text:
# Virtual Drums
# Author: Navendu Pottekkat https://bit.ly/navendupottekkat
# MIT License

# This project use Open CV to create a virtual drum set. It uses color detection in a region of interest
# and plays the corresponding drum beat. The drums are shown in the video output and hitting the drums
# with a green object would make you able to virtually play it.

# Source code: https://www.github.com/navendu-pottekkat/virtual-drums

# Importing the necessary libraries
import numpy as np
import time
import cv2
from pygame import mixer

# This function plays the corresponding drum beat if a green color object is detected in the region
def play_beat(detected,sound):

	# Checks if the detected green color is greater that a preset value
	play = (detected) > hat_thickness[0]*hat_thickness[1]*0.8

	# If it is detected play the corresponding drum beat
	if play and sound==1:
		drum_snare.play()

	elif play and sound==2:
		drum_hat.play()
		time.sleep(0.001)

# This function is used to check if green color is present in the small region
def detect_in_region(frame,sound):

	# Converting to HSV
	hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

	# Creating mask
	mask = cv2.inRange(hsv, greenLower, greenUpper)

	# Calculating the number of green pixels
	detected = np.sum(mask)

	# Call the function to play the drum beat
	play_beat(detected,sound)

	return mask

# A flag variable to choose whether to show the region that is being detected
verbose = False

# Importing drum beats
mixer.init()
drum_hat = mixer.Sound('./sounds/high_hat_1.ogg')
drum_snare = mixer.Sound('./sounds/snare_1.wav')

# Set HSV range for detecting green color
greenLower = (25,52,72)
greenUpper = (102,255,255)

# Obtain input from the webcam
camera = cv2.VideoCapture(0)
ret,frame = camera.read()
H,W = frame.shape[:2]

kernel = np.ones((7,7),np.uint8)

# Read the image of High Hat and the Snare drum
hat = cv2.resize(cv2.imread('./images/high_hat.png'),(200,100),interpolation=cv2.INTER_CUBIC)
snare = cv2.resize(cv2.imread('./images/snare_drum.png'),(200,100),interpolation=cv2.INTER_CUBIC)

# Set the region area for detecting green color
hat_center = [np.shape(frame)[1]*2//8,np.shape(frame)[0]*6//8]
snare_center = [np.shape(frame)[1]*6//8,np.shape(frame)[0]*6//8]

hat_thickness = [200,100]
hat_top = [hat_center[0]-hat_thickness[0]//2,hat_center[1]-hat_thickness[1]//2]
hat_btm = [hat_center[0]+hat_thickness[0]//2,hat_center[1]+hat_thickness[1]//2]

snare_thickness = [200,100]
snare_top = [snare_center[0]-snare_thickness[0]//2,snare_center[1]-snare_thickness[1]//2]
snare_btm = [snare_center[0]+snare_thickness[0]//2,snare_center[1]+snare_thickness[1]//2]

time.sleep(1)

while True:

	# Select the current frame
	ret, frame = camera.read()
	frame = cv2.flip(frame,1)

	if not(ret):
	    break

	# Select region corresponding to the Snare drum
	snare_region = np.copy(frame[snare_top[1]:snare_btm[1],snare_top[0]:snare_btm[0]])
	mask = detect_in_region(snare_region,1)

	# Select region corresponding to the High Hat
	hat_region = np.copy(frame[hat_top[1]:hat_btm[1],hat_top[0]:hat_btm[0]])
	mask = detect_in_region(hat_region,2)

	# Output project title
	cv2.putText(frame,'Virtual Drums',(10,30),2,1,(20,20,20),2)

	# If flag is selected, display the region under detection
	if verbose:
		frame[snare_top[1]:snare_btm[1],snare_top[0]:snare_btm[0]] = cv2.bitwise_and(frame[snare_top[1]:snare_btm[1],snare_top[0]:snare_btm[0]],frame[snare_top[1]:snare_btm[1],snare_top[0]:snare_btm[0]], mask=mask[snare_top[1]:snare_btm[1],snare_top[0]:snare_btm[0]])
		frame[hat_top[1]:hat_btm[1],hat_top[0]:hat_btm[0]] = cv2.bitwise_and(frame[hat_top[1]:hat_btm[1],hat_top[0]:hat_btm[0]],frame[hat_top[1]:hat_btm[1],hat_top[0]:hat_btm[0]],mask=mask[hat_top[1]:hat_btm[1],hat_top[0]:hat_btm[0]])

	# If flag is not selected, display the drums
	else:
		frame[snare_top[1]:snare_btm[1],snare_top[0]:snare_btm[0]] = cv2.addWeighted(snare, 1, frame[snare_top[1]:snare_btm[1],snare_top[0]:snare_btm[0]], 1, 0)
		frame[hat_top[1]:hat_btm[1],hat_top[0]:hat_btm[0]] = cv2.addWeighted(hat, 1, frame[hat_top[1]:hat_btm[1],hat_top[0]:hat_btm[0]], 1, 0)

	cv2.imshow('Output',frame)
	key = cv2.waitKey(1) & 0xFF
	# 'Q' to exit
	if key == ord("q"):
		break

# Clean up the open windows
camera.release()
cv2.destroyAllWindows()

ext: py

reason: import numpy


text:
"""Prediction of user based on tweet embeddings"""

import numpy as np
from sklearn.linear_model import LogisticRegression
from .models import User
from .twitter import vectorize_tweet

def predict_user(user0_name, user1_name, hypo_tweet_text):
    """
    Determine and return which user is more likely to say given 'tweet'.

    Args:
        user0_name (str): 1st user to compare
        user1_name (str): 2nd user to compare
        hypo_tweet_text (str): hypothetical tweet

    Returns:
        0 or 1: a.k.a. user0 or user1
    """

    # Get users from database
    user0 = User.query.filter(User.name == user0_name).one()
    user1 = User.query.filter(User.name == user1_name).one()

    # Get vectors from each users' tweets
    user0_vects = np.array([tweet.vect for tweet in user0.tweets])
    user1_vects = np.array([tweet.vect for tweet in user1.tweets])

    # Vertically stack vectors to train model
    vects = np.vstack([user0_vects, user1_vects])

    # Generate labels for vects array
    labels = np.concatenate(
        [np.zeros(len(user0.tweets)), np.ones(len(user1.tweets))])

    # Train model
    log_reg = LogisticRegression().fit(vects, labels)

    # Using nlp to generate embeddings - hypo_tweet_text
    hypo_tweet_vect = vectorize_tweet(hypo_tweet_text).reshape(1, -1)

    return log_reg.predict(hypo_tweet_vect)

max_stars_repo_name: dakotagporter/twitoff

max_issues_repo_name: dakotagporter/twitoff

max_forks_repo_name: dakotagporter/twitoff


text:
import xarray as xr
import numpy as np
import importlib
import sys
import warnings

from CASutils import lensread_utils as lens
from CASutils import readdata_utils as read
from CASutils import calendar_utils as cal

importlib.reload(lens)
importlib.reload(read)
importlib.reload(cal)
warnings.filterwarnings('ignore')

filepath="/project/cas02/islas/CESM1LE/"
pathout="/project/cas/islas/python_savs/snowpaper/DATA_SORT/LENS1/BULKSNOW/"

nmems=40
memnames = lens.lens1memnamegen(nmems)

for imem in np.arange(0,len(memnames),1):
    print(imem)
    snowice = read.read_sfc_cesm(filepath+"SNOWICE/SNOWICE_"+memnames[imem]+"*.nc","1850-01","2100-12")
    snowliq = read.read_sfc_cesm(filepath+"SNOWLIQ/SNOWLIQ_"+memnames[imem]+"*.nc","1850-01","2100-12")
    snowdp = read.read_sfc_cesm(filepath+"SNOWDP/SNOWDP_"+memnames[imem]+"*.nc","1850-01","2100-12")
    rhosnowt = (snowice.SNOWICE + snowliq.SNOWLIQ) / snowdp.SNOWDP
    rhosnow_djf = cal.season_ts(rhosnowt,"DJF")
    if (imem == 0):
        rhosnow = xr.DataArray(np.zeros([len(memnames),rhosnow_djf.time.size, rhosnow_djf.lat.size, rhosnow_djf.lon.size]),
                              dims=['member','time','lat','lon'], coords = [memnames,rhosnow_djf.time, rhosnow_djf.lat, rhosnow_djf.lon], name='rhosnow')

    rhosnow[imem,:,:,:] = rhosnow_djf[:,:,:]

rhosnow.to_netcdf(pathout+"rhosnow_lens1_djf.nc")


text:
import argparse
import os
import numpy as np
import itertools
import sys
from tqdm import tqdm

from torch.utils.data import DataLoader
from torch.autograd import Variable

from models import *
from data_proc import DataProc

import torch.nn as nn
import torch.nn.functional as F
import torch

from utils import plot_batch_train
import random

parser = argparse.ArgumentParser()
parser.add_argument("--epoch", type=int, default=0, help="epoch to start training from")
parser.add_argument("--n_epochs", type=int, default=100, help="number of epochs of training")
parser.add_argument("--model_name", type=str, help="name of the model")
parser.add_argument("--dataset", type=str, help="path to dataset for training")
parser.add_argument("--n_spkrs", type=int, default=2, help="number of speakers for conversion, must match that of preprocessing")
parser.add_argument("--batch_size", type=int, default=4, help="size of the batches")
parser.add_argument("--lr", type=float, default=0.0001, help="adam: learning rate")
parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient")
parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient")
parser.add_argument("--decay_epoch", type=int, default=50, help="epoch from which to start lr decay")
parser.add_argument("--n_cpu", type=int, default=8, help="number of cpu threads to use during batch generation")
parser.add_argument("--img_height", type=int, default=128, help="size of image height")
parser.add_argument("--img_width", type=int, default=128, help="size of image width")
parser.add_argument("--channels", type=int, default=1, help="number of image channels")
parser.add_argument("--plot_interval", type=int, default=-1, help="epoch interval between saving plots (disable with -1)")
parser.add_argument("--checkpoint_interval", type=int, default=1, help="interval between saving model checkpoints")
parser.add_argument("--n_downsample", type=int, default=2, help="number downsampling layers in encoder")
parser.add_argument("--dim", type=int, default=32, help="number of filters in first encoder layer")

opt = parser.parse_args()
print(opt)

cuda = True if torch.cuda.is_available() else False

# Create sample and checkpoint directories
os.makedirs("saved_models/%s" % opt.model_name, exist_ok=True)

# Losses
criterion_GAN = torch.nn.MSELoss()
criterion_pixel = torch.nn.L1Loss()

input_shape = (opt.channels, opt.img_height, opt.img_width)

# Dimensionality (channel-wise) of image embedding
shared_dim = opt.dim * 2 ** opt.n_downsample

# Initialize generators and discriminators
encoder = Encoder(dim=opt.dim, in_channels=opt.channels, n_downsample=opt.n_downsample)
shared_G = ResidualBlock(features=shared_dim)
G = [Generator(dim=opt.dim, out_channels=opt.channels, n_upsample=opt.n_downsample, shared_block=shared_G) for _ in range(opt.n_spkrs)]
D = [Discriminator(input_shape) for _ in range(opt.n_spkrs)]

if cuda:
    encoder = encoder.cuda()
    G = [G_n.cuda() for G_n in G]
    D = [D_n.cuda() for D_n in D]
    criterion_GAN.cuda()
    criterion_pixel.cuda()

if opt.epoch != 0:
    # Load pretrained models
    encoder.load_state_dict(torch.load("saved_models/%s/encoder_%02d.pth" % (opt.model_name, opt.epoch-1)))
    for n in range(opt.n_spkrs):
        G[n].load_state_dict(torch.load("saved_models/%s/G%d_%02d.pth" % (opt.model_name, n+1, opt.epoch-1)))
        D[n].load_state_dict(torch.load("saved_models/%s/D%d_%02d.pth" % (opt.model_name, n+1, opt.epoch-1)))
else:
    # Initialize weights
    encoder.apply(weights_init_normal)
    for n in range(opt.n_spkrs):
        G[n].apply(weights_init_normal)
        D[n].apply(weights_init_normal)

# Loss weights
lambda_0 = 10   # GAN
lambda_1 = 0.1  # KL (encoded spect)
lambda_2 = 100  # ID pixel-wise
lambda_3 = 0.1  # KL (encoded translated spect)
lambda_4 = 100  # Cycle pixel-wise
lambda_5 = 10   # latent space L1

# Optimizers
optimizer_G = torch.optim.Adam(
    itertools.chain(encoder.parameters(), *[G_n.parameters() for G_n in G]),
    lr=opt.lr,
    betas=(opt.b1, opt.b2),
)
optimizer_D = [torch.optim.Adam(D_n.parameters(), lr=opt.lr, betas=(opt.b1, opt.b2)) for D_n in D]

# Learning rate update schedulers
lr_scheduler_G = torch.optim.lr_scheduler.LambdaLR(
    optimizer_G, lr_lambda=LambdaLR(opt.n_epochs, opt.epoch, opt.decay_epoch).step
)
lr_scheduler_D = [torch.optim.lr_scheduler.LambdaLR(
                    optimizer_D_n, lr_lambda=LambdaLR(opt.n_epochs, opt.epoch, opt.decay_epoch).step
                ) for optimizer_D_n in optimizer_D]

Tensor = torch.cuda.FloatTensor if cuda else torch.Tensor

# Prepare dataloader
dataloader = torch.utils.data.DataLoader(
	DataProc(opt, split='train'),
	batch_size=opt.batch_size,
	shuffle=True,
	num_workers=opt.n_cpu,
    pin_memory=True
)

def compute_kl(mu):
    mu_2 = torch.pow(mu, 2)
    loss = torch.mean(mu_2)
    return loss

# ---------------------------------------------------------
#  Training (local)
# ---------------------------------------------------------

def train_local(i, epoch, batch, id_1, id_2, losses):

        # Create plot output directories if doesn't exist already
        if opt.plot_interval != -1:
            os.makedirs("out_train/%s/plot_%dt%d/" % (opt.model_name, id_1, id_2), exist_ok=True)
            os.makedirs("out_train/%s/plot_%dt%d/" % (opt.model_name, id_2, id_1), exist_ok=True)

        # Set model input
        X1 = Variable(batch[id_1].type(Tensor))
        X2 = Variable(batch[id_2].type(Tensor))

        # Adversarial ground truths
        valid = Variable(Tensor(np.ones((X1.size(0), *D[id_1].output_shape))), requires_grad=False)
        fake = Variable(Tensor(np.zeros((X1.size(0), *D[id_1].output_shape))), requires_grad=False)

        # -------------------------------
        #  Train Encoder and Generators
        # -------------------------------

        optimizer_G.zero_grad()

        # Get shared latent representation
        mu1, Z1 = encoder(X1)
        mu2, Z2 = encoder(X2)

        # Latent space feat
        feat_1 = mu1.view(mu1.size()[0], -1).mean(dim=0)
        feat_2 = mu2.view(mu2.size()[0], -1).mean(dim=0)

        # Reconstruct speech
        recon_X1 = G[id_1](Z1)
        recon_X2 = G[id_2](Z2)

        # Translate speech
        fake_X1 = G[id_1](Z2)
        fake_X2 = G[id_2](Z1)

        # Cycle translation
        mu1_, Z1_ = encoder(fake_X1)
        mu2_, Z2_ = encoder(fake_X2)
        cycle_X1 = G[id_1](Z2_)
        cycle_X2 = G[id_2](Z1_)

        # Losses
        loss_GAN_1 = lambda_0 * criterion_GAN(D[id_1](fake_X1), valid)
        loss_GAN_2 = lambda_0 * criterion_GAN(D[id_2](fake_X2), valid)
        loss_KL_1 = lambda_1 * compute_kl(mu1)
        loss_KL_2 = lambda_1 * compute_kl(mu2)
        loss_ID_1 = lambda_2 * criterion_pixel(recon_X1, X1)
        loss_ID_2 = lambda_2 * criterion_pixel(recon_X2, X2)
        loss_KL_1_ = lambda_3 * compute_kl(mu1_)
        loss_KL_2_ = lambda_3 * compute_kl(mu2_)
        loss_cyc_1 = lambda_4 * criterion_pixel(cycle_X1, X1)
        loss_cyc_2 = lambda_4 * criterion_pixel(cycle_X2, X2)
        loss_feat = lambda_5 * criterion_pixel(feat_1, feat_2)

        # Total loss
        loss_G = (
            loss_KL_1
            + loss_KL_2
            + loss_ID_1
            + loss_ID_2
            + loss_GAN_1
            + loss_GAN_2
            + loss_KL_1_
            + loss_KL_2_
            + loss_cyc_1
            + loss_cyc_2
            + loss_feat
        )

        loss_G.backward()
        optimizer_G.step()

        # -----------------------
        #  Train Discriminator 1
        # -----------------------

        optimizer_D[id_1].zero_grad()

        loss_D1 = criterion_GAN(D[id_1](X1), valid) + criterion_GAN(D[id_1](fake_X1.detach()), fake)

        loss_D1.backward()
        optimizer_D[id_1].step()

        # -----------------------
        #  Train Discriminator 2
        # -----------------------

        optimizer_D[id_2].zero_grad()

        loss_D2 = criterion_GAN(D[id_2](X2), valid) + criterion_GAN(D[id_2](fake_X2.detach()), fake)

        loss_D2.backward()
        optimizer_D[id_2].step()

        # --------------
        #  Log Progress
        # --------------

        # Plot first batch every epoch or few epochs
        if opt.plot_interval != -1 and (epoch+1) % opt.plot_interval == 0 and i == 0:
            plot_batch_train(opt.model_name, 'plot_%dt%d'%(id_1, id_2), epoch, X1, cycle_X1, fake_X2, X2)
            plot_batch_train(opt.model_name, 'plot_%dt%d'%(id_2, id_1), epoch, X2, cycle_X2, fake_X1, X1)

        losses['G'].append(loss_G.item())
        losses['D'].append((loss_D1 + loss_D2).item())

        return losses

# ---------------------------------------------------------
#  Training (global)
# ---------------------------------------------------------

def train_global():
    for epoch in range(opt.epoch, opt.n_epochs):

        losses = {'G': [],'D': []}
        progress = tqdm(enumerate(dataloader),desc='',total=len(dataloader))

	for i, batch in progress:

		# For each target, randomly choose a source for training
		for trg_id in range(opt.n_spkrs):

		    src_id = random.randint(0, opt.n_spkrs-1)
		    losses = train_local(i, epoch, batch, pair[src_id], pair[trg_id], losses)

		    # Update progress bar
		    progress.set_description("[Epoch %d/%d] [D loss: %f] [G loss: %f] "
			% (epoch, opt.n_epochs, np.mean(losses['D']), np.mean(losses['G'])))

        # Update learning rates
        lr_scheduler_G.step()
        for n in range(opt.n_spkrs):
            lr_scheduler_D[n].step()

        if opt.checkpoint_interval != -1 and (epoch+1) % opt.checkpoint_interval == 0:
            # Save model checkpoints
            torch.save(encoder.state_dict(), "saved_models/%s/encoder_%02d.pth" % (opt.model_name, epoch))
            for n in range(opt.n_spkrs):
                torch.save(G[n].state_dict(), "saved_models/%s/G%d_%02d.pth" % (opt.model_name, n+1, epoch))
                torch.save(D[n].state_dict(), "saved_models/%s/D%d_%02d.pth" % (opt.model_name, n+1, epoch))

if __name__ == '__main__':
    train_global()

max_stars_repo_name: Daanddj/test

max_issues_repo_name: Daanddj/test

max_forks_repo_name: Daanddj/test


text:
# To import required modules:
import numpy as np
import time
import os
import sys
import matplotlib
import matplotlib.cm as cm #for color maps
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec #for specifying plot attributes
from matplotlib import ticker #for setting contour plots to log scale
import scipy.integrate #for numerical integration
import scipy.misc #for factorial function
from scipy.special import erf #error function, used in computing CDF of normal distribution
import scipy.interpolate #for interpolation functions
import corner #corner.py package for corner plots
#matplotlib.rc('text', usetex=True)

sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))

from src.functions_general import *
from src.functions_compare_kepler import *
from src.functions_load_sims import *
from src.functions_plot_catalogs import *
from src.functions_plot_params import *

savefigures = False

savefigures_directory = '/Users/hematthi/Documents/GradSchool/Misc_Presentations/PhD_Thesis_Defense/Figures/'
save_name = 'Models_Compare_Kepler'

compute_ratios = compute_ratios_adjacent
AD_mod = True
weights_all = load_split_stars_weights_only()
dists_include = ['delta_f',
                 'mult_CRPD_r',
                 'periods_KS',
                 'period_ratios_KS',
                 #'durations_KS',
                 #'durations_norm_circ_KS',
                 'durations_norm_circ_singles_KS',
                 'durations_norm_circ_multis_KS',
                 'duration_ratios_nonmmr_KS',
                 'duration_ratios_mmr_KS',
                 'depths_KS',
                 'radius_ratios_KS',
                 'radii_partitioning_KS',
                 'radii_monotonicity_KS',
                 'gap_complexity_KS',
                 ]

##### To load the files with the systems with observed planets:

# Model 1:
loadfiles_directory1 = '/Users/hematthi/Documents/GradSchool/Research/ACI/Simulated_Data/Split_stars/Clustered_P_R_fswp_bprp/Params13_KS/durations_KS/GP_med/incl0_ecc0p02/' #incl0_ecc0p02/ #ecc0_incl1/
run_number1 = ''

N_sim, cos_factor, P_min, P_max, radii_min, radii_max = read_targets_period_radius_bounds(loadfiles_directory1 + 'periods%s.out' % run_number1)

param_vals_all1 = read_sim_params(loadfiles_directory1 + 'periods%s.out' % run_number1)
sss_per_sys1, sss1 = compute_summary_stats_from_cat_obs(file_name_path=loadfiles_directory1, run_number=run_number1, compute_ratios=compute_ratios)

# Model 2:
loadfiles_directory2 = '/Users/hematthi/Documents/GradSchool/Research/ACI/Simulated_Data/Split_stars/Clustered_P_R_fswp_bprp/Params13_KS/durations_KS/GP_med/incl2_ecc0p02/' #incl2_ecc0p02/ #ecc0p1_incl1/
run_number2 = ''

param_vals_all2 = read_sim_params(loadfiles_directory2 + 'periods%s.out' % run_number2)
sss_per_sys2, sss2 = compute_summary_stats_from_cat_obs(file_name_path=loadfiles_directory2, run_number=run_number2, compute_ratios=compute_ratios)

# Model 3:
#loadfiles_directory3 = '/Users/hematthi/Documents/GradSchool/Research/ExoplanetsSysSim_Clusters/SysSimExClusters/examples/New_algorithm_AMD/Same_params/Stable/New_sampler_P_Pc/No_final_mHill_check/' #'ACI/Simulated_Data/Julia_v0.7/Kepler_catalog_optimization/q1q17_dr25_gaia_fgk_stars79935/Non_Clustered/f_high_incl_low_incl_mmr/Fit_rate_mult_P_Pratios_D_Dratios_dur_durratios_mmr/Some8_params_CRPDr_KS/Fixed_Rbreak3_Ncrit8/lc_1_8_alphaP_-2_2_alphaR1_-4_2_alphaR2_-6_0_ecc_0_0p1_incl_inclmmr_0_90/targs79935_maxincl0_maxiters5000/sigma_i_greater_sigma_i_mmr/GP_med/'
#run_number3 = ''

#param_vals_all3 = read_sim_params(loadfiles_directory3 + 'periods%s.out' % run_number3)
#sss_per_sys3, sss3 = compute_summary_stats_from_cat_obs(file_name_path=loadfiles_directory3, run_number=run_number3, compute_ratios=compute_ratios)

model_sss = [sss1, sss2]
model_sss_per_sys = [sss_per_sys1, sss_per_sys2]
model_names = [r'$\sigma_i = 0^\circ$', r'$\sigma_i = 2^\circ$'] #[r'$\sigma_e = 0$', r'$\sigma_e = 0.1$'] #[r'$\sigma_i = 0^\circ$', r'$\sigma_i = 2^\circ$'] #[r'$\omega = {\rm atan}(x,y)$', r'$\omega \sim {\rm Unif}(-\pi,\pi)$'] # Make sure this matches the models loaded!
model_linestyles = ['-', '-']
model_colors = ['b', 'r'] #['b', 'r']

# To load and process the observed Kepler catalog and compare with our simulated catalog:
ssk_per_sys, ssk = compute_summary_stats_from_Kepler_catalog(P_min, P_max, radii_min, radii_max)

logxi_Kep_2 = np.log10(ssk_per_sys['xi_obs'][ssk_per_sys['Mtot_obs'] == 2, 0])
logxi_Kep_3 = np.log10(ssk_per_sys['xi_obs'][ssk_per_sys['Mtot_obs'] == 3, :2].flatten())
logxi_Kep_4 = np.log10(ssk_per_sys['xi_obs'][ssk_per_sys['Mtot_obs'] == 4, :3].flatten())
xi_Kep_4p = ssk_per_sys['xi_obs'][ssk_per_sys['Mtot_obs'] >= 4]
logxi_Kep_4p = np.log10(xi_Kep_4p[xi_Kep_4p != -1])
xi_Kep_5p = ssk_per_sys['xi_obs'][ssk_per_sys['Mtot_obs'] >= 5]
logxi_Kep_5p = np.log10(xi_Kep_5p[xi_Kep_5p != -1])

##### To plot the simulated and Kepler catalogs as marginal distributions:

subdirectory = '' #'Paper_Figures/'; 'Talk_Figures/'

fig_size = (8,3) #size of each panel (figure)
fig_lbrt = [0.15, 0.3, 0.95, 0.925]

n_bins = 100
lw = 1 #linewidth
#alpha = 0.2 #transparency of histograms

afs = 20 #axes labels font size
tfs = 20 #text labels font size
lfs = 12 #legend labels font size

#'''
# Multiplicities:
plot_fig_counts_hist_simple(fig_size, [sss_per_sys['Mtot_obs'] for sss_per_sys in model_sss_per_sys], [ssk_per_sys['Mtot_obs']], x_min=0, y_min=1e-2, y_max=1e4, x_llim=0.5, N_sim_Kep_factor=float(N_sim)/N_Kep, log_y=True, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text='Observed planets per system', afs=afs, tfs=tfs, lfs=lfs, legend=True, show_counts_sim=True, show_counts_Kep=True, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_multiplicities_compare.pdf', save_fig=savefigures)

# Periods:
plot_fig_pdf_simple(fig_size, [sss['P_obs'] for sss in model_sss], [ssk['P_obs']], x_min=3., x_max=300., y_min=1e-3, y_max=0.1, n_bins=n_bins, log_x=True, c_sim=model_colors, log_y=True, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xticks_custom=[3,10,30,100,300], xlabel_text=r'$P$ (days)', afs=afs, tfs=tfs, lfs=lfs, legend=True, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_periods_compare.pdf', save_fig=savefigures)

# Period ratios (all, with some upper cut-off):
R_max_cut = 30. #upper cut-off for plotting period ratios; np.max(sss['Rm_obs'])
plot_fig_pdf_simple(fig_size, [sss['Rm_obs'][sss['Rm_obs'] < R_max_cut] for sss in model_sss], [ssk['Rm_obs'][ssk['Rm_obs'] < R_max_cut]], x_min=1., x_max=R_max_cut, n_bins=n_bins, log_x=True, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xticks_custom=[1,2,3,4,5,10,20], xlabel_text=r'$P_{i+1}/P_i$', afs=afs, tfs=tfs, lfs=lfs, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_periodratios_compare.pdf', save_fig=savefigures)

# Period ratios (< 5):
plot_fig_pdf_simple(fig_size, [sss['Rm_obs'][sss['Rm_obs'] < 5.] for sss in model_sss], [ssk['Rm_obs'][ssk['Rm_obs'] < 5.]], x_min=1., x_max=5., n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$P_{i+1}/P_i$', afs=afs, tfs=tfs, lfs=lfs, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_periodratios_less5_compare.pdf', save_fig=savefigures)

# Transit durations:
plot_fig_pdf_simple(fig_size, [sss['tdur_obs'] for sss in model_sss], [ssk['tdur_obs']], x_min=0., x_max=15., n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$t_{\rm dur}$ (hrs)', afs=afs, tfs=tfs, lfs=lfs, legend=True, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_durations_compare.pdf', save_fig=savefigures)

# Circular normalized transit durations:
plot_fig_pdf_simple(fig_size, [sss['tdur_tcirc_obs'] for sss in model_sss], [ssk['tdur_tcirc_obs']], x_min=0., x_max=1.5, n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$t_{\rm dur}/t_{\rm circ}$', afs=afs, tfs=tfs, lfs=lfs, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_tdur_tcirc_compare.pdf', save_fig=savefigures)

# Transit depths:
plot_fig_pdf_simple(fig_size, [sss['D_obs'] for sss in model_sss], [ssk['D_obs']], x_min=10.**(-5.), x_max=10.**(-1.5), n_bins=n_bins, log_x=True, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$\delta$', afs=afs, tfs=tfs, lfs=lfs, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_depths_compare.pdf', save_fig=savefigures)

# Planet radii:
plot_fig_pdf_simple(fig_size, [sss['radii_obs'] for sss in model_sss], [ssk['radii_obs']], x_min=radii_min, x_max=radii_max, n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$R_p (R_\oplus)$', afs=afs, tfs=tfs, lfs=lfs, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_radii_compare.pdf', save_fig=savefigures)

# Stellar radii:
plot_fig_pdf_simple(fig_size, [sss['Rstar_obs'] for sss in model_sss], [ssk['Rstar_obs']], x_min=0.5, x_max=2.5, n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$R_\star (R_\odot)$', afs=afs, tfs=tfs, lfs=lfs, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_stellar_radii_compare.pdf', save_fig=savefigures)

# Transit depth ratios:
plot_fig_pdf_simple(fig_size, [sss['D_ratio_obs'] for sss in model_sss], [ssk['D_ratio_obs']], x_min=10.**(-1.5), x_max=10.**(1.5), n_bins=n_bins, log_x=True, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$\delta_{i+1}/\delta_i$', afs=afs, tfs=tfs, lfs=lfs, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_depthratios_compare.pdf', save_fig=savefigures)

# Log(xi):
plot_fig_pdf_simple(fig_size, [np.log10(sss['xi_obs']) for sss in model_sss], [np.log10(ssk['xi_obs'])], x_min=-0.5, x_max=0.5, n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$\log{\xi}$', afs=afs, tfs=tfs, lfs=lfs, fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_logxi_all_compare.pdf', save_fig=savefigures)

plt.show()
plt.close()
#'''

##### To plot the inner vs. outer period ratios of triplets (in 3+ systems) (similar to Fig 6 in Zhu et al. 2019 and Fig 7 in Weiss et al. 2018a):
'''
compute_pratio_in_out_and_plot_fig_pdf([sss_per_sys['P_obs'] for sss_per_sys in model_sss_per_sys], last_is_Kep=True, fig_size=(8,6), n_bins=50, x_min=0.1, x_max=10., colors=['b','r'], ls=['-',':'], lw=2, labels=['Clustered P+R', 'Non-clustered'], afs=afs, tfs=tfs, lfs=lfs, save_name=savefigures_directory + subdirectory + save_name + '_observed_pratio_out_in_ratio.pdf', save_fig=savefigures)
plt.show()
plt.close()
'''

##### To plot the circular normalized transit durations again (observed singles vs. multis):
plot_fig_pdf_simple(fig_size, [sss['tdur_tcirc_1_obs'] for sss in model_sss], [ssk['tdur_tcirc_1_obs']], x_min=0., x_max=1.5, n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$t_{\rm dur}/t_{\rm circ}$', afs=afs, tfs=tfs, lfs=lfs, extra_text='Singles', fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_tdur_tcirc_singles_compare.pdf', save_fig=savefigures)
plot_fig_pdf_simple(fig_size, [sss['tdur_tcirc_2p_obs'] for sss in model_sss], [ssk['tdur_tcirc_2p_obs']], x_min=0., x_max=1.5, n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=lw, labels_sim=model_names, xlabel_text=r'$t_{\rm dur}/t_{\rm circ}$', afs=afs, tfs=tfs, lfs=lfs, extra_text='Multis', fig_lbrt=fig_lbrt, save_name=savefigures_directory + subdirectory + save_name + '_tdur_tcirc_multis_compare.pdf', save_fig=savefigures)

##### To plot the xi distribution separated by observed multiplicities (m=2,3,4+):

logxi_2_model1 = np.log10(sss_per_sys1['xi_obs'][sss_per_sys1['Mtot_obs'] == 2, 0])
logxi_3_model1 = np.log10(sss_per_sys1['xi_obs'][sss_per_sys1['Mtot_obs'] == 3, :2].flatten())
logxi_4_model1 = np.log10(sss_per_sys1['xi_obs'][sss_per_sys1['Mtot_obs'] == 4, :3].flatten())
xi_4p_model1 = sss_per_sys1['xi_obs'][sss_per_sys1['Mtot_obs'] >= 4]
logxi_4p_model1 = np.log10(xi_4p_model1[xi_4p_model1 != -1])
xi_5p_model1 = sss_per_sys1['xi_obs'][sss_per_sys1['Mtot_obs'] >= 5]
logxi_5p_model1 = np.log10(xi_5p_model1[xi_5p_model1 != -1])

logxi_2_model2 = np.log10(sss_per_sys2['xi_obs'][sss_per_sys2['Mtot_obs'] == 2, 0])
logxi_3_model2 = np.log10(sss_per_sys2['xi_obs'][sss_per_sys2['Mtot_obs'] == 3, :2].flatten())
logxi_4_model2 = np.log10(sss_per_sys2['xi_obs'][sss_per_sys2['Mtot_obs'] == 4, :3].flatten())
xi_4p_model2 = sss_per_sys2['xi_obs'][sss_per_sys2['Mtot_obs'] >= 4]
logxi_4p_model2 = np.log10(xi_4p_model2[xi_4p_model2 != -1])
xi_5p_model2 = sss_per_sys2['xi_obs'][sss_per_sys2['Mtot_obs'] >= 5]
logxi_5p_model2 = np.log10(xi_5p_model2[xi_5p_model2 != -1])

c2, c3, c4p = 'r', 'b', 'g'
ymax = 0.14

xi_bins = np.linspace(-0.5, 0.5, n_bins+1)

fig = plt.figure(figsize=(8,14))
plot = GridSpec(7,1,left=0.2,bottom=0.07,right=0.95,top=0.98,wspace=0,hspace=0)

ax = plt.subplot(plot[0,0])
plot_panel_cdf_simple(ax, [logxi_2_model1, logxi_3_model1, logxi_4p_model1], [logxi_Kep_2, logxi_Kep_3, logxi_Kep_4p], x_min=np.min(xi_bins), x_max=np.max(xi_bins), c_sim=[c2,c3,c4p], c_Kep=[c2,c3,c4p], ls_sim=['-','-','-'], ls_Kep=[':',':',':'], lw=2, labels_sim=[r'$m=2$', r'$m=3$', r'$m=4+$'], labels_Kep=['Kepler data',None,None], xlabel_text='', legend=True, afs=afs, tfs=tfs, lfs=lfs, label_dist=False)

ax = plt.subplot(plot[1:3,0])
plot_panel_pdf_simple(ax, [logxi_2_model1, logxi_3_model1, logxi_4p_model1], [], x_min=np.min(xi_bins), x_max=np.max(xi_bins), y_max=ymax, n_bins=n_bins, c_sim=[c2,c3,c4p], ls_sim=['-','-','-'], lw=2, labels_sim=[r'$m=2$', r'$m=3$', r'$m=4+$'], xlabel_text='', legend=False, afs=afs, tfs=tfs, lfs=lfs)
plt.text(x=0.98, y=0.9, s=model_names[0], ha='right', fontsize=lfs, transform=ax.transAxes)

ax = plt.subplot(plot[3:5,0])
plot_panel_pdf_simple(ax, [logxi_2_model2, logxi_3_model2, logxi_4p_model2], [], x_min=np.min(xi_bins), x_max=np.max(xi_bins), y_max=ymax, n_bins=n_bins, c_sim=[c2,c3,c4p], ls_sim=['-','-','-'], lw=2, labels_sim=[r'$m=2$', r'$m=3$', r'$m=4+$'], xlabel_text='', legend=False, afs=afs, tfs=tfs, lfs=lfs)
plt.text(x=0.98, y=0.9, s=model_names[1], ha='right', fontsize=lfs, transform=ax.transAxes)

ax = plt.subplot(plot[5:,0])
plot_panel_pdf_simple(ax, [logxi_Kep_2, logxi_Kep_3, logxi_Kep_4p], [], x_min=np.min(xi_bins), x_max=np.max(xi_bins), y_max=ymax, n_bins=n_bins, c_sim=[c2,c3,c4p], ls_sim=['-','-','-'], lw=2, labels_sim=[r'$m=2$', r'$m=3$', r'$m=4+$'], labels_Kep=[None], xlabel_text=r'$\log{\xi}$', legend=False, afs=afs, tfs=tfs, lfs=lfs)
plt.text(x=0.98, y=0.9, s='Kepler data', ha='right', fontsize=lfs, transform=ax.transAxes)

if savefigures:
    plt.savefig(savefigures_directory + subdirectory + save_name + '_logxi_per_mult.pdf')
    plt.close()

##### To remake the log(xi) plot for defense talk:

plot_fig_pdf_simple((8,4), [np.log10(sss['xi_obs']) for sss in model_sss], [], x_min=-0.5, x_max=0.5, n_bins=n_bins, c_sim=model_colors, ls_sim=model_linestyles, lw=3, labels_sim=model_names, xlabel_text=r'$\log{\xi}$', afs=afs, tfs=tfs, lfs=20, legend=True, fig_lbrt=[0.15, 0.2, 0.95, 0.925], save_name=savefigures_directory + subdirectory + save_name + '_logxi_incl.pdf', save_fig=savefigures)
plt.show()

reason: import numpy,import scipy,from scipy


text:
import matplotlib.pyplot as plt
import numpy as np

def showCategoryChart(nameList, countListList, countLabelList, countColorList, ylabel, title, cleanNameDict = None, figsize = None, width=0.35, barLabelSize=None):
    N = len(countListList[0])
    print(N)

    ind = np.arange(N)  # the x locations for the groups

    if figsize:
      fig, ax = plt.subplots(figsize=figsize)
    else:
      fig, ax = plt.subplots()

    rectList = []
    for i, countList in enumerate(countListList):
      # print "Added bar for %s" % countList
      currInd = np.arange(len(countList))
      if countColorList != None:
        currColor = countColorList[i]
      else:
        currColor = np.random.rand(3,1)
      rectList.append(ax.bar(currInd + i * width, countList, width, color = currColor))
      # rectList.append(ax.bar(ind, countList, width, color = 'r'))

    cleanedNameList = []
    if cleanNameDict:
      for name in nameList:
          if name in cleanNameDict:
              cleanedNameList.append(cleanNameDict[name])
          else:
              cleanedNameList.append(name)
    else:
      cleanedNameList = nameList

    # add some
    ax.set_ylabel(ylabel)
    ax.set_title(title)
    ax.set_xticks(ind+width/2)
    # ax.set_xticklabels(cleanedNameList, rotation = 45, rotation_mode = "anchor")
    ax.set_xticklabels(cleanedNameList)

    print("len(rectList) = %d, len(countLabelList) = %d" % (len(rectList), len(countLabelList)))
    if len(countLabelList) > 1:
      if len(countLabelList) > 3:
        nCols = len(countLabelList) // 3
      else:
        nCols = len(countLabelList)
      plt.legend(rectList, countLabelList, loc="best", framealpha=0.3, ncol = nCols + 1)
      # ax.legend(rectList, countLabelList)

    def autolabel(rects):
        # attach some text labels
        for rect in rects:
            height = rect.get_height()
            ax.text(rect.get_x()+rect.get_width()/2., 1.05*height, '%d'%int(height),
                    ha='center', va='bottom', size=barLabelSize)

    if not barLabelSize == None:
      for rect in rectList:
        autolabel(rect)
    return (fig, ax)

def showHorizCategoryChart(nameList, countListList, countLabelList, countColorList, ylabel, title, cleanNameDict = None, figsize = None, width=0.35, barLabelSize=None):
    N = len(countListList[0])
    print(N)

    ind = np.arange(N)  # the x locations for the groups

    if figsize:
      fig, ax = plt.subplots(figsize=figsize)
    else:
      fig, ax = plt.subplots()

    rectList = []
    for i, countList in enumerate(countListList):
      currInd = np.arange(len(countList))
      rectList.append(ax.barh(currInd + i * width, countList, width, color = countColorList[i]))
      # rectList.append(ax.bar(ind, countList, width, color = 'r'))

    cleanedNameList = []
    if cleanNameDict:
      for name in nameList:
          if name in cleanNameDict:
              cleanedNameList.append(cleanNameDict[name])
          else:
              cleanedNameList.append(name)
    else:
      cleanedNameList = nameList

    # add some
    ax.set_xlabel(ylabel)
    ax.set_title(title)
    ax.set_yticks(ind+width/2)
    # ax.set_xticklabels(cleanedNameList, rotation = 45, rotation_mode = "anchor")
    ax.set_yticklabels(cleanedNameList)

    if len(countLabelList) > 1:
      # ax.legend(rectList, countLabelList)
      plt.legend(rectList, countLabelList, loc="best", framealpha=0.5)

    def autolabel(rects):
        # attach some text labels
        for rect in rects:
            height = rect.get_height()
            bw = rect.get_width()
            print(rect.get_y(), rect.get_height(), bw)
            ax.text(bw + 0.02 * (ax.get_xlim()[1] - ax.get_xlim()[0]),
                    rect.get_y()+rect.get_height()/2.,'%d'%int(bw),
                    ha='left', va='center', size=barLabelSize)

    for rect in rectList:
      autolabel(rect)
    return (fig, ax)

max_forks_repo_name: corinne-hcr/e-mission-eval-private-data


text:
"""
S2 band dropout
"""
# built-in
import os, pickle, logging, sys

# packages
from tensorflow.python import keras
from keras.utils import Sequence
import tensorflow as tf
import keras.backend as K
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tqdm import tqdm

# lib
from solarpv.training.train_S2_unet import DataGenerator

# conf
K.set_image_data_format('channels_last')
from tensorflow.compat.v1 import ConfigProto
from tensorflow.compat.v1 import InteractiveSession
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)

logging.basicConfig(stream=sys.stdout, level=logging.INFO)

print ('gpu',tf.test.is_gpu_available())

class BandDropoutS2:

    def __init__(self,records_path, model_path, batch_size):
        self.records = pickle.load(open(records_path,'rb'))
        self.model = keras.models.load_model(model_path)
        self.band_list = ['red', 'green', 'blue', 'nir', 'red-edge','red-edge-2', 'red-edge-3', 'red-edge-4', 'swir1','swir2','water-vapor','cirrus','coastal-aerosol','alpha']
        self.BATCH_SIZE=batch_size

    def loop_inference(self):

        mi = pd.MultiIndex.from_product([range(len(self.records)),self.band_list+['none']],names=['records','bands'])
        df = pd.DataFrame(index=mi, columns=None)

        logging.info(f'Running band dropout...')
        df['band_dropout'] = self._run_inference(perturbation=None)['data']
        df.to_csv(os.path.join(os.getcwd(),'solarpv','analysis','chkpt.csv'))

        for perturbation_magnitude in [0.1,0.2,0.3]:
            df['additive_'+str(perturbation_magnitude)] = self._run_inference(perturbation='additive', perturbation_magnitude=perturbation_magnitude)['data']
            df.to_csv(os.path.join(os.getcwd(),'solarpv','analysis','chkpt.csv'))

        for perturbation_magnitude in [0.1, 0.2, 0.3]:
            df['multiplicative_'+str(perturbation_magnitude)] = self._run_inference(perturbation='multiplicative', perturbation_magnitude=perturbation_magnitude)['data']
            df.to_csv(os.path.join(os.getcwd(),'solarpv','analysis','chkpt.csv'))

        df.to_csv(os.path.join(os.getcwd(),'solarpv','analysis','band_perturbation.csv'))

    def _run_inference(self, perturbation=None, perturbation_magnitude=0.05):

        """
        run inferences to check sensitivity. Check against:
        Band dropout: perturbation=None
        Additive noise: perturbation='additive'
        Multiplicative noise: perturbation='multiplicative'
        perturbation_magnitude: magnitude of perturbation
        """

        mi = pd.MultiIndex.from_product([range(len(self.records)),self.band_list+['none']],names=['records','bands'])
        df = pd.DataFrame(index=mi, columns=['data'])

        if not perturbation:
            perturbation_str = 'dropout'
        else:
            perturbation_str = perturbation + '_' + str(perturbation_magnitude)

        generator = DataGenerator(self.records,
                                    batch_size=self.BATCH_SIZE,
                                    dim=(200,200,14),
                                    n_channels=1,
                                    n_classes=2,
                                    shuffle=False,
                                    augment=False)

        logging.info(f'len generator: {len(generator)}')

        ious = np.zeros((len(self.records), len(self.band_list+['none'])))

        for ii_b in range(len(self.band_list+['none'])):

            for ii_g in tqdm(range(len(generator)), desc=f'band: {(self.band_list+["none"])[ii_b]}'):   # ii_batch

                sample, ann = generator.__getitem__(ii_g)

                ann = np.argmax(ann,axis=-1)

                do_sample = sample.copy()

                if ii_b<(len(self.band_list+['none'])-1):
                    if perturbation==None:
                        do_sample[:,:,:,ii_b]=0#do_sample[:,:,ii]+noise

                    elif perturbation=='additive':
                        noise = perturbation_magnitude*np.random.rand(self.BATCH_SIZE*200*200).reshape(self.BATCH_SIZE,200,200) - perturbation_magnitude/2
                        do_sample[:,:,:,ii_b]=do_sample[:,:,:,ii_b]+noise

                    elif perturbation=='multiplicative':
                        noise = np.ones((self.BATCH_SIZE,200,200))+ perturbation_magnitude*np.random.rand(self.BATCH_SIZE*200*200).reshape(self.BATCH_SIZE,200,200) - perturbation_magnitude/2
                        do_sample[:,:,:,ii_b]=do_sample[:,:,:,ii_b]*noise

                do_sample = do_sample.clip(0,1)

                pred = np.squeeze(self.model.predict(do_sample))
                pred = (pred[...,1] - pred[...,0]).clip(0,1)

                ious_batch = np.sum((pred>0.1).astype(int) * ann, axis=(1,2)) / np.sum(((pred>0.1).astype(int) + ann).clip(0,1), axis=(1,2))  #batch size

                ious[ii_g*self.BATCH_SIZE:(ii_g+1)*self.BATCH_SIZE,ii_b]=ious_batch

        df.loc[:,'data'] = ious.ravel()
        return df

    def loop_figs(self,n_figs, savepath):

        indices = np.random.choice(len(self.records), n_figs, replace=False)
        logging.info(f'generating figures')
        self._run_inference_fig(indices,perturbation=None, savepath=savepath)

        for perturbation_magnitude in [0.1,0.2,0.3]:
            self._run_inference_fig(indices,perturbation='additive', perturbation_magnitude=perturbation_magnitude, savepath=savepath)

        for perturbation_magnitude in [0.1, 0.2, 0.3]:
            self._run_inference_fig(indices,perturbation='multiplicative', perturbation_magnitude=perturbation_magnitude,savepath=savepath)

    def _run_inference_fig(self, indices, perturbation=None, perturbation_magnitude=0.05, savepath=None):

        if not perturbation:
            perturbation_str = 'dropout'
        else:
            perturbation_str = perturbation + '_' + str(perturbation_magnitude)

        for ii_f in indices:
            logging.info(f'Generating figs for record {self.records[ii_f]}')
            sample = np.load(open(self.records[ii_f]['data'],'rb'))['data']
            ann = np.load(open(self.records[ii_f]['data'],'rb'))['annotation'].astype(int)/255

            fig, axs = plt.subplots(4,4,figsize=(20,20))

            for ii in range(sample.shape[-1]):
                do_sample = sample.copy()
                #do_sample[:,:,ii]=0
                #print (do_sample.shape)
                #print (do_sample.mean(axis=0).mean(axis=0))

                if perturbation==None:
                    do_sample[:,:,ii]=0#do_sample[:,:,ii]+noise

                elif perturbation=='additive':
                    noise = perturbation_magnitude*np.random.rand(200*200).reshape(200,200) - perturbation_magnitude/2
                    do_sample[:,:,ii]=do_sample[:,:,ii]+noise

                elif perturbation=='multiplicative':
                    noise = np.ones((200,200))+ perturbation_magnitude*np.random.rand(200*200).reshape(200,200) - perturbation_magnitude/2
                    do_sample[:,:,ii]=do_sample[:,:,ii]*noise

                do_sample = do_sample.clip(0,1)

                pred = np.squeeze(self.model.predict(do_sample[np.newaxis,...]))
                pred = (pred[...,1] - pred[...,0]).clip(0,1)

                intersection = np.sum((pred>0.1).astype(int) * ann)
                union = np.sum(((pred>0.1).astype(int) + ann).clip(0,1))
                #logging.info(f'intersection:{intersection}, union: {union}')
                iou =  intersection/union

                axs[ii//4, ii%4].imshow(pred)
                axs[ii//4, ii%4].set_title(f'{self.band_list[ii]} IoU: {iou}',fontsize=12)
                axs[ii//4, ii%4].set_axis_off()

            pred = np.squeeze(self.model.predict(sample[np.newaxis,...]))
            pred = (pred[...,1] - pred[...,0]).clip(0,1)

            iou = np.sum((pred>0.1).astype(int) * ann) / np.sum(((pred>0.1).astype(int) + ann).clip(0,1))

            axs[3,2].imshow(pred)
            axs[3,2].set_title(f'All bands IoU: {iou}',fontsize=12)
            axs[3,3].imshow((sample[:,:,0:3]*2.5).clip(0,1))
            axs[3,2].set_axis_off()
            axs[3,3].set_axis_off()
                #logging.info(f'saving fig {str(ii_r)+perturbation_str+".png"}')
            fig.savefig(os.path.join(savepath,str(ii_f)+'_'+perturbation_str+'.png'))
            plt.close(fig)

if __name__=="__main__":
    bdo = BandDropoutS2(
        records_path=os.path.join(os.getcwd(),'data','crossvalidation','S2_unet','records.pickle'),
        model_path=os.path.join(os.getcwd(),'data','S2_unet.h5'),
        batch_size=1)
    bdo.loop_figs(n_figs=10, savepath=os.path.join(os.getcwd(),'solarpv','analysis','band_perturbation'))


text:
"""
Created on 2013-10-03

@author: guillaume
"""

# Imports
from scipy import eye, kron, diag

# Define the basis for the liouvillian
# States: B, C or all states
___B, ___C, _ABC = (
    diag([0.0, 1.0, 0.0]),
    diag([0.0, 0.0, 1.0]),
    diag([1.0, 1.0, 1.0]),
)

# Auto-relaxation rates
R_IXY, DR_IXY_AB, DR_IXY_AC = (
    kron(_ABC, -eye(2)),
    kron(___B, -eye(2)),
    kron(___C, -eye(2)),
)
# Chemical shifts
_CS = [[+0.0, -1.0],
       [+1.0, +0.0]]

DW_AB = kron(___B, _CS)
DW_AC = kron(___C, _CS)

# Exchange rates
KAB = kron([[-1.0, +0.0, +0.0],
            [+1.0, +0.0, +0.0],
            [+0.0, +0.0, +0.0]], eye(2))

KBA = kron([[+0.0, +1.0, +0.0],
            [+0.0, -1.0, +0.0],
            [+0.0, +0.0, +0.0]], eye(2))

KBC = kron([[+0.0, +0.0, +0.0],
            [+0.0, -1.0, +0.0],
            [+0.0, +1.0, +0.0]], eye(2))

KCB = kron([[+0.0, +0.0, +0.0],
            [+0.0, +0.0, +1.0],
            [+0.0, +0.0, -1.0]], eye(2))

KAC = kron([[-1.0, +0.0, +0.0],
            [+0.0, +0.0, +0.0],
            [+1.0, +0.0, +0.0]], eye(2))

KCA = kron([[+0.0, +0.0, +1.0],
            [+0.0, +0.0, +0.0],
            [+0.0, +0.0, -1.0]], eye(2))

# 180 degree y pulse
P_180Y = diag([1.0, -1.0, 1.0, -1.0, 1.0, -1.0])

reason: from scipy


text:
import geopandas as gpd

import pdal
import json
from shapely.geometry import shape, GeometryCollection, Polygon,Point,MultiPolygon
from typing import TypeVar, Union

import os
import sys
from pathlib import Path
sys.path.append(os.path.abspath(os.path.join('..')))

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d

from scipy.interpolate import griddata
from matplotlib.tri import Triangulation, LinearTriInterpolator,CubicTriInterpolator
from math import floor
import math
import pyproj

from shapely.ops import transform
import numpy as np

from logger_util import CreateLogger

logger = CreateLogger('FetchData')
logger = logger.get_default_logger()

class FetchData:
    def __init__(self,region:str) -> None:
        """Initialize Fetch With Region

        Args:
            region (str): region in list of regions found at https://s3-us-west-2.amazonaws.com/usgs-lidar-public/
        """
        self.region=region
        logger.info('Initialized fetch object')
    def fetch_elevation(self,boundary:gpd.GeoDataFrame, crs:str)->list[gpd.GeoDataFrame]:
        """Fetch Elevation Dataframe

        Args:
            boundary (GeoDataFrame): boundary defined in a geopandas dataframe
            crs (string): Coordinate reference system in string

        Returns:
            list[GeoDataFrame]: Returns a list of geopandas dataframes
        """

        logger.info('Fetch point cloud data stareted')
        try:
            pat=Path(__file__).parent.joinpath("fetch_template.json")
            with open(pat, 'r') as json_file:
                pipeline = json.load(json_file)
        except FileNotFoundError as e:
            logger.exception('Fetch JSON template File not found')

        boundary_repojected=boundary.to_crs(epsg=3857)

        Xmin,Ymin,Xmax,Ymax=boundary_repojected.total_bounds

        pipeline[0]["filename"]=f"https://s3-us-west-2.amazonaws.com/usgs-lidar-public/{self.region}/ept.json"
        pipeline[0]["bounds"]=f"([{Xmin},{Xmax}],[{Ymin},{Ymax}])"
        pipeline[1]["polygon"]=boundary_repojected.geometry.unary_union.wkt
        pipeline[4]["out_srs"]=crs
        # print( pipeline[0]["polygon"])
        # print(pipeline)
        try:
            pipe = pdal.Pipeline(json.dumps(pipeline))
            count = pipe.execute()
            arrays = pipe.arrays
            metadata = pipe.metadata
        except:
            logger.info('Error occured: Could not fetch point cloud data')

        log = pipe.log
        years=[]
        for i in arrays:
            geometry_points = [Point(x, y) for x, y in zip(i["X"], i["Y"])]
            elevetions = i["Z"]
            frame=gpd.GeoDataFrame(columns=["elevation", "geometry"])
            frame['elevation'] = elevetions
            frame['geometry'] = geometry_points
            frame.set_geometry("geometry",inplace=True)
            frame.set_crs(crs , inplace=True)
            years.append(frame)

        logger.info('Fetch point cloud data completed')

        return years
    def repoject(self,polygon:Union[Polygon,Point,MultiPolygon],crs:str)->Union[Polygon,Point,MultiPolygon]:
        """Reproject Shape from crs to EPSG:3857

        Args:
            polygon (Union[Polygon,Point,MultiPolygon]): Polygon to be reporojected
            crs (str): Coordinate reference system

        Returns:
            Union[Polygon,Point,MultiPolygon]: Reprojected Polygon
        """

        wgs84 = pyproj.CRS(crs)
        utm = pyproj.CRS('EPSG:3857')

        project = pyproj.Transformer.from_crs(wgs84, utm, always_xy=True).transform
        utm_point = transform(project, polygon)
        return utm_point
    def visualize3D(self,data:gpd.GeoDataFrame)->None:
        """Create a 3D pointcloud Visualization

        Args:
            data (GeoDataFrame): Dataframe to be visualized
        """

        logger.info('Visualization 3D Started')
        fig = plt.figure()
        ax = fig.add_subplot(111,projection='3d')
        ax.scatter( data.geometry.x,data.geometry.y,data.elevation, cmap='Spectral_r',s=0.001, c=data.elevation)
        plt.show()

        logger.info('Visualization 3D complete')
    def visualize2D(self,data:gpd.GeoDataFrame)->None:
        """Create a 2D visualization

        Args:
            data (GeoDataFrame): Dataframe to be visualized
        """

        logger.info('Visualization 2D complete')
        fig, ax = plt.subplots(1, 1)
        try:
            data.plot(column='elevation',  cmap = 'Spectral_r', ax=ax, legend=True)
        except KeyError:
            logger.exception('Elevation column does not exist')
        except:
            logger.exception("Exception Occured: Could not complete visualization")

        plt.show()

        logger.info('Visualization 2D complete')
    def standardize(self,data,resolution):
        """Standardize and interpolate dataframe

        Args:
            data (GeoDataFrame): Dataframe to be standardized
            resolution (int): Resolution of the standardization

        Returns:
            GeoDataFrame: Returns the interpolated and standardized dataframe
        """

        logger.info('Standardization Started')

        data_meters= data.to_crs({'init': 'epsg:3395'})

        rasterRes = resolution

        totalPointsArray = np.zeros([data_meters.shape[0],3])
        for index, point in data_meters.iterrows():
            pointArray = np.array([point.geometry.coords.xy[0][0],point.geometry.coords.xy[1][0],point['elevation']])
            totalPointsArray[index] = pointArray
        xCoords = np.arange(totalPointsArray[:,0].min(), totalPointsArray[:,0].max()+rasterRes, rasterRes)
        yCoords = np.arange(totalPointsArray[:,1].min(), totalPointsArray[:,1].max()+rasterRes, rasterRes)
        zCoords = np.zeros([yCoords.shape[0],xCoords.shape[0]])
        polygons=[]
        elevations=[]

        logger.info('Grid Created Proceeding to interpolation')
        triFn = Triangulation(totalPointsArray[:,0],totalPointsArray[:,1])
        #linear triangule interpolator funtion
        linTriFn = LinearTriInterpolator(triFn,totalPointsArray[:,2])
        #loop among each cell in the raster extension
        for indexX, x in np.ndenumerate(xCoords):
            for indexY, y in np.ndenumerate(yCoords):
                tempZ = linTriFn(x,y)
                #filtering masked values
                if tempZ == tempZ:
                    zCoords[indexY,indexX]=tempZ

                    polygons.append(Point(x,y))
                    elevations.append(float(tempZ.data[()]))
                else:
                    zCoords[indexY,indexX]=np.nan
        logger.info("Interploation Completed")
        frame=gpd.GeoDataFrame(columns=["elevation", "geometry"])
        frame['elevation'] = elevations
        frame['geometry'] = polygons
        frame.set_geometry("geometry",inplace=True)
        frame.set_crs(crs=data_meters.crs, inplace=True)

        logger.info('Standardization Completed')
        return frame.to_crs(data.crs)
    def standardize2(self,data,resolution):

        logger.info('Standardization Started')
        data_meters= data.to_crs({'init': 'epsg:3395'})

        rasterRes = resolution
        try:
            points = [[i.x,i.y] for i in data_meters.geometry.values]
        except KeyError:
            logger.exception('Geometry column does not exist')
        except:
            logger.exception("Exception Occured: Could not complete Standardization")

        try:
            values = data_meters['elevation'].values
        except KeyError:
            logger.exception('Elevation column does not exist')
        except:
            logger.exception("Exception Occured: Could not complete visualization")

        xDim = floor(data_meters.total_bounds[2])-floor(data_meters.total_bounds[0])
        yDim = floor(data_meters.total_bounds[3])-floor(data_meters.total_bounds[1])
        # rasterRes = 0.5
        nCols = xDim / rasterRes
        nRows = yDim / rasterRes

        grid_y, grid_x = np.mgrid[floor(data_meters.total_bounds[1])+rasterRes/2:floor(data_meters.total_bounds[3])-rasterRes/2:nRows*1j,
                                floor(data_meters.total_bounds[0])+rasterRes/2:floor(data_meters.total_bounds[2])+rasterRes/2:nCols*1j]

        logger.info('Grid Created Proceeding to interpolation')
        mtop = griddata(points, values, (grid_x, grid_y), method='linear')

        logger.info("Interploation Completed")
        x=grid_x.flatten()
        y=grid_y.flatten()
        geometry_points = [Point(x, y) for x, y in zip(x, y)]
        elevetions = mtop.flatten()
        frame=gpd.GeoDataFrame(columns=["elevation", "geometry"])
        frame['elevation'] = elevetions
        frame['geometry'] = geometry_points
        frame.set_geometry("geometry",inplace=True)
        frame.set_crs("epsg:3395" , inplace=True)

        logger.info("Standardization Completed")
        return frame.to_crs(data.crs)
    def topographicWetnessIndex(self,data, resolution):
        """Calculate Topographic Wetness Index for elevations

        Args:
            data (GeoDataFRame): GeoDataFrame containing points and thier elevation
            resolution (int): Resolution of the generated TWI

        Returns:
            GeoDataFrame: GeoDataFrame With a new Column For the index
        """

        logger.info("Topographic Wetness Index Generation Started")
        data_meters= data.to_crs({'init': 'epsg:3395'})

        rasterRes = resolution
        try:
            points = [[i.x,i.y] for i in data_meters.geometry.values]
        except KeyError:
            logger.exception('Geometry column does not exist')
        except:
            logger.exception("Exception Occured: Could not complete Standardization")

        try:
            values = data_meters['elevation'].values
        except KeyError:
            logger.exception('Elevation column does not exist')
        except:
            logger.exception("Exception Occured: Could not complete visualization")

        xDim = round(data_meters.total_bounds[2])-round(data_meters.total_bounds[0])
        yDim = round(data_meters.total_bounds[3])-round(data_meters.total_bounds[1])
        # rasterRes = 0.5
        nCols = xDim // rasterRes
        nRows = yDim // rasterRes

        grid_y, grid_x = np.mgrid[round(data_meters.total_bounds[1])+rasterRes/2:round(data_meters.total_bounds[3])-rasterRes/2:nRows*1j,
                                round(data_meters.total_bounds[0])+rasterRes/2:round(data_meters.total_bounds[2])+rasterRes/2:nCols*1j]
        mtop = griddata(points, values, (grid_x, grid_y), method='linear')

        slopes=[]

        slopeMatrix = np.zeros([nRows,nCols])
        for indexX in range(0,nCols):
            for indexY in range(0,nRows):
                # print(indexX,indexY)
                # if not np.isnan(zCoords[indexY,indexX]):
                # slopeMatrix

                mat=neighbors(mtop,indexY+1,indexX+1)
                ind=twi(mat,rasterRes)
                # if indexX[0]>508:
                #     print(pointSlope,indexY)
                slopeMatrix[indexY,indexX]=ind
                slopes.append(ind)
        logger.info("Topographic Wetness Index Generation Completed")

        x=grid_x.flatten()
        y=grid_y.flatten()
        geometry_points = [Point(x, y) for x, y in zip(x, y)]
        elevetions = mtop.flatten()
        frame=gpd.GeoDataFrame(columns=["elevation", "geometry"])
        frame['elevation'] = elevetions
        frame['geometry'] = geometry_points
        frame['twi']=slopes
        frame.set_geometry("geometry",inplace=True)
        frame.set_crs("epsg:3395" , inplace=True)

        return frame.to_crs(data.crs)

def slope(matrix, res):
    """Generate Slope From Matrix

    Args:
        matrix (np.array): Numpy array representing our matrix
        res (int): Resolution of the given matrix in

    Returns:
        float: Slope of the point in the middle of our matrix
    """
    logger.info("Slope Generation Started")
    if not np.isnan(np.sum(matrix)):
        mat=np.nan_to_num(matrix)
        fx=(mat[2,0] - mat[2,2] +mat[1,0]-mat[1,2]+mat[0,0]-mat[0,2])/(6 *res)
        fy=(mat[0,2]-mat[2,2]+mat[0,1]-mat[2,1]+mat[0,0]-mat[2,0])/(6*res)
        slope=math.atan(((fx**2) + (fy**2))**0.5)

        logger.info("Slope Generation Completed")
        return slope
    else:

        logger.exception("Matrix contains nan values: Returning a nan slope")
        return np.nan

def slope2(matrix,res):
    """Generate Slope From Matrix

    Args:
        matrix (np.array): Numpy array representing our matrix
        res (int): Resolution of the given matrix in

    Returns:
        float: Slope of the point in the middle of our matrix
    """
    logger.info("Slope Generation Started")
    if not np.isnan(np.sum(matrix)):
        mat=np.nan_to_num(matrix)
        fx=(mat[1,0]-mat[1,2])/(2 *res)
        fy=(mat[0,1]-mat[2,1])/(2*res)
        slope=math.atan(((fx**2) + (fy**2))**0.5)

        logger.info("Slope Generation Completed")
        return slope
    else:

        logger.exception("Matrix contains nan values: Returning a nan slope")
        return np.nan

def twi(matrix,res):
    """Generate TWI for Matrix

    Args:
        matrix (array): Array Containing our point and its neighbors
        res (int): Resolution of our matrix

    Returns:
        float: Return TWI for the center point of our matrix
    """
    logger.info("TWI Calculation Started")
    if not np.isnan(np.sum(matrix)):
        mat=np.array(matrix)
        fx=(mat[2,0] - mat[2,2] +mat[1,0]-mat[1,2]+mat[0,0]-mat[0,2])/(6 *res)
        fy=(mat[0,2]-mat[2,2]+mat[0,1]-mat[2,1]+mat[0,0]-mat[2,0])/(6*res)
        slope=((fx**2) + (fy**2))**0.5
        arr=np.array(matrix)
        contributingarea=len(arr[arr>arr[1,1]]) * (res**2)
        if contributingarea==0:
            return 0
        if abs(slope)==0.0:
            return 0
        index=np.log(contributingarea/slope)

        logger.info("TWI Calculation Completed")
        return index
    else:

        logger.exception("Matrix contains nan values: Returning a nan TWI")
        return 0

def neighbors(mat, row, col, radius=1):
    """Generate neigbors matrix for point

    Args:
        mat (array): Matrix from which the neighbors will be extracted
        row (int): Row of the value for which we are fining the neighbors
        col (int): Col of of the value for which we are fining the neighbors
        radius (int, optional): Radius of neighbors. Defaults to 1.

    Returns:
        [array]: Returns a matrix of the neighbors with the value of the requred index in the middle
    """

    logger.info("Neighbors Calculation Started")

    rows, cols = len(mat), len(mat[0])
    out = []

    for i in range(row - radius - 1, row + radius):
        row = []
        for j in range(col - radius - 1, col + radius):

            if 0 <= i < rows and 0 <= j < cols:
                try:
                    row.append(mat[i][j])
                except:
                    logger.exception("Error fininding neighbor")
            else:
                row.append(np.nan)

        out.append(row)

    logger.info("Neighbors Calculation Completed")
    return out

reason: import numpy,from scipy


text:
import numpy as np
import os
import glob
import argparse

def parse_args(script):
    parser = argparse.ArgumentParser(description= 'few-shot script %s' %(script))
    parser.add_argument('--dataset'     , default='miniImagenet',  help='miniImagenet/cub')
    parser.add_argument('--model'       , default='ResNet18',      help='model: Conv{4|6} / ResNet{10|18|34|50|101}') # we use ResNet18 in the paper
    parser.add_argument('--method'      , default='maml_approx',   help='maml{_approx}') #maml_approx use first-order approximation in the gradient for efficiency
    parser.add_argument('--train_n_way' , default=5, type=int,  help='class num to classify for training')
    parser.add_argument('--test_n_way'  , default=5, type=int,  help='class num to classify for testing (validation) ')
    parser.add_argument('--n_shot'      , default=5, type=int,  help='number of labeled data in each class, same as n_support')
    parser.add_argument('--train_aug'   , action='store_true',  help='perform data augmentation or not during training ')
    parser.add_argument('--name'        , default='tmp', type=str, help='')
    parser.add_argument('--save_dir'    , default='./output', type=str, help='')
    parser.add_argument('--data_dir'    , default='/home/project/nasa_2020/data', type=str, help='')
    parser.add_argument('--night'       , action='store_true', help='Use gray scale transformation to simulate night view')

    if script == 'train':
        parser.add_argument('--num_classes' , default=200, type=int, help='total number of classes in softmax, only used in baseline')
        parser.add_argument('--save_freq'   , default=50, type=int, help='Save frequency')
        parser.add_argument('--start_epoch' , default=0, type=int,help ='Starting epoch')
        parser.add_argument('--stop_epoch'  , default=400, type=int, help ='Stopping epoch') # for meta-learning methods, each epoch contains 100 episodes
        parser.add_argument('--resume'      , default='', type=str, help='continue from previous trained model with largest epoch')
        parser.add_argument('--warmup'      , default='gg3b0', type=str, help='continue from baseline, neglected if resume is true') #never used in the paper
        parser.add_argument('--dropout_method', default='none', help='none/binary/gaussian')
        parser.add_argument('--dropout_rate', default=0.1, type=float, help='dropout rate, recommend [0.1, 0.2]')
        parser.add_argument('--dropout_schedule', default='constant', help='constant/linear')
        parser.add_argument('--seed'        , default=41608, type=int, help='random seed')
    elif script == 'test':
        parser.add_argument('--split'       , default='novel', help='base/val/novel') #default novel, but you can also test base/val class accuracy if you want
        parser.add_argument('--save_iter', default=-1, type=int,help ='save feature from the model trained in x epoch, use the best model if x is -1')
    else:
       raise ValueError('Unknown script')

    return parser.parse_args()

def get_assigned_file(checkpoint_dir,num):
    assign_file = os.path.join(checkpoint_dir, '{:d}.tar'.format(num))
    return assign_file

def get_resume_file(checkpoint_dir):
    filelist = glob.glob(os.path.join(checkpoint_dir, '*.tar'))
    if len(filelist) == 0:
        return None

    filelist =  [ x  for x in filelist if os.path.basename(x) != 'best_model.tar' ]
    epochs = np.array([int(os.path.splitext(os.path.basename(x))[0]) for x in filelist])
    max_epoch = np.max(epochs)
    resume_file = os.path.join(checkpoint_dir, '{:d}.tar'.format(max_epoch))
    return resume_file

def get_best_file(checkpoint_dir):
    best_file = os.path.join(checkpoint_dir, 'best_model.tar')
    if os.path.isfile(best_file):
        return best_file
    else:
        return get_resume_file(checkpoint_dir)


text:
# importing libraries
import cv2
import numpy as np

# initialize camera settings
frameWidth = 640
frameHeight = 480
cap = cv2.VideoCapture(0)
cap.set(3, frameWidth)
cap.set(4, frameHeight)

def empty(a):
    pass

# name of the window
cv2.namedWindow("Parameters")

# resize the window
cv2.resizeWindow("Parameters",640,240)

#creating trackbars
cv2.createTrackbar("Threshold1","Parameters",23,255,empty)
cv2.createTrackbar("Threshold2","Parameters",20,255,empty)
cv2.createTrackbar("Area","Parameters",5000,30000,empty)

# function for stacking images together
def stackImages(scale,imgArray):
    rows = len(imgArray)
    cols = len(imgArray[0])
    rowsAvailable = isinstance(imgArray[0], list)
    width = imgArray[0][0].shape[1]
    height = imgArray[0][0].shape[0]
    if rowsAvailable:
        for x in range ( 0, rows):
            for y in range(0, cols):
                if imgArray[x][y].shape[:2] == imgArray[0][0].shape [:2]:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (0, 0), None, scale, scale)
                else:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (imgArray[0][0].shape[1], imgArray[0][0].shape[0]), None, scale, scale)
                if len(imgArray[x][y].shape) == 2: imgArray[x][y]= cv2.cvtColor( imgArray[x][y], cv2.COLOR_GRAY2BGR)
        imageBlank = np.zeros((height, width, 3), np.uint8)
        hor = [imageBlank]*rows
        hor_con = [imageBlank]*rows
        for x in range(0, rows):
            hor[x] = np.hstack(imgArray[x])
        ver = np.vstack(hor)
    else:
        for x in range(0, rows):
            if imgArray[x].shape[:2] == imgArray[0].shape[:2]:
                imgArray[x] = cv2.resize(imgArray[x], (0, 0), None, scale, scale)
            else:
                imgArray[x] = cv2.resize(imgArray[x], (imgArray[0].shape[1], imgArray[0].shape[0]), None,scale, scale)
            if len(imgArray[x].shape) == 2: imgArray[x] = cv2.cvtColor(imgArray[x], cv2.COLOR_GRAY2BGR)
        hor= np.hstack(imgArray)
        ver = hor
    return ver

def getContours(img,imgContour):
    contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)

    # detect the area of each contour and based on the area we can remove all the contours that we are not interested in
    # so in order to do that we will need a for loop
    for cnt in contours:
        area = cv2.contourArea(cnt)
        areaMin = cv2.getTrackbarPos("Area", "Parameters")
        if area > areaMin:
            cv2.drawContours(imgContour, cnt, -1, (255, 0, 255), 7)

            # find the corner points, so in order to do that we need to first find the length of contours
            # true basically means that the contour is closed
            peri = cv2.arcLength(cnt, True)

            # to approximate what type of shape this is, we will use the approximation of poly method
            # we will input the contour, will give it a resolution and then we will define again that this is a closed contour
            approx = cv2.approxPolyDP(cnt, 0.02 * peri, True)
            print(len(approx))

            # create a bounding box because we need to highlight the area of where the object is
            x , y , w, h = cv2.boundingRect(approx)
            cv2.rectangle(imgContour, (x , y ), (x + w , y + h ), (0, 255, 0), 5)

            # to display values so that we can easily see the number of points and the area detected
            cv2.putText(imgContour, "Points: " + str(len(approx)), (x + w + 20, y + 20), cv2.FONT_HERSHEY_COMPLEX, .7, (0, 255, 0), 2)
            cv2.putText(imgContour, "Area: " + str(int(area)), (x + w + 20, y + 45), cv2.FONT_HERSHEY_COMPLEX, 0.7, (0, 255, 0), 2)

# reading each frame
while True:
    success, img = cap.read()
    imgContour = img.copy()

    # converting img into blur version
    imgBlur = cv2.GaussianBlur(img, (7, 7), 1)

    # converting it into gray scale
    imgGray = cv2.cvtColor(imgBlur, cv2.COLOR_BGR2GRAY)

    # define trackbar positions
    threshold1 = cv2.getTrackbarPos("Threshold1", "Parameters")
    threshold2 = cv2.getTrackbarPos("Threshold2", "Parameters")

    # canny edge detector
    imgCanny = cv2.Canny(imgGray,threshold1,threshold2)

    # define kernel for dilation
    kernel = np.ones((5, 5))

    # to overcome the overlaps and noise, we use dilation function
    imgDil = cv2.dilate(imgCanny, kernel, iterations=1)
    getContours(imgDil,imgContour)

    # stacking images together as we want images side by side instead of different window
    imgStack = stackImages(0.8,([img,imgCanny],
                                [imgDil,imgContour]))

    # displaying it
    cv2.imshow("Result", imgStack)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

max_stars_repo_name: A-kriti/Amazing-Python-Scripts

max_issues_repo_name: A-kriti/Amazing-Python-Scripts

max_forks_repo_name: A-kriti/Amazing-Python-Scripts


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import random
import seaborn as sns
import imgkit
import re
import nltk
import spacy
import string
from textblob import TextBlob
from collections import Counter
from sklearn import tree
from sklearn import preprocessing
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, OrdinalEncoder
from sklearn.feature_selection import SelectFromModel
from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth, association_rules
from mlxtend.classifier import EnsembleVoteClassifier
from mlxtend.data import iris_data
from mlxtend.plotting import plot_decision_regions
from mlxtend.preprocessing import TransactionEncoder
from pandas.plotting import table
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from emot.emo_unicode import UNICODE_EMO, EMOTICONS# Converting emojis to words

#df['class'].value_counts()

#diffrenciating between diffrent attributes with similar values
#data = df[['Tweet Id','Text','name','Screen Name','UTC','Created At','Favorites','Retweets','Language','Client','Tweet Type','URLs','Hasgtags','Mentions','Media Type','Media URLs']].values

def is_number(s):
	try:
		float(s)
		return True
	except ValueError:
		return False

#remove extra new line characters
f = open('to_only.csv', 'w')
with open('opinion_2.csv', 'r') as fp:
	s = fp.readline()
	while s:
		ids = str(s[1:19])
		#print(ids)
		if is_number(ids):
			f.write('\n')
			s = s.strip('\n')
			f.write(s)
		else:
			s = s.strip('\n')
			f.write(s)
		s=fp.readline()
f.close()

# read the dataset
#dp = pd.read_csv('to_only.csv', header=None, sep='\n')
#dp.head()
#print (dp)

#data = dp[['Tweet Id','Text','name','Screen Name','UTC','Created At','Favorites','Retweets','Language','Client','Tweet Type','URLs','Hasgtags','Mentions','Media Type','Media URLs']].values

#remove all entries that are not in english
f = open('eng_only.csv', 'w')
with open('to_only.csv', 'r') as fp:
	s = fp.readline()
	f.write(s)
	s = fp.readline()
	while s:
		sts = re.split(r'\t+', s)
		#l = ""
		if sts[8] == 'en':
			#for i in sts:
			#	if i.startswith('\"') and i.endswith('\"'):
			#		i = i[1:-1]
			#		l = l + i + "\t"
			#	else:
			#		l = l+
			f.write(s)
		s=fp.readline()
f.close()

# read the dataset
df = pd.read_csv('eng_only.csv', sep='\t')
df.head()

#data = df[['Tweet Id','Text','name','Screen Name','UTC','Created At','Favorites','Retweets','Language','Client','Tweet Type','URLs','Hasgtags','Mentions','Media Type','Media URLs']].values

def prep(text):
	#all to lower case
	df['text_lower'] = text.str.lower()
	df['text_lower'].head()

	#remove links
	df['text_lower'] = df['text_lower'].str.replace('http\S+','')

	#remove punctuation
	df['text_punct'] = df['text_lower'].str.replace('[^\w\s]',' ')
	df['text_punct'].head()

	#remove all single char
	df['text_punct'] = df['text_punct'].str.replace('\s+[a-zA-Z]\s+',' ')
	df['text_punct'] = df['text_punct'].str.replace('\^[a-zA-Z]\s+',' ')

	#multiple spaces to single space
	df['text_punct'] = df['text_punct'].str.replace('\s+',' ')

	#remove prefixed 'b' if bite data
	df['text_punct'] = df['text_punct'].str.replace('^b\s+','')

	#stop-word removal
	'''from nltk.corpus import stopwords
	STOPWORDS = set(stopwords.words('english'))

	def stopwords(text):
		return " ".join([word for word in str(text).split() if word not in STOPWORDS])

	df["text_stop"] = df["text_punct"].apply(stopwords)
	df["text_stop"].head()
	'''
	#common word removal
	cnt = Counter()
	for text in df["text_punct"].values:
		for word in text.split():
			cnt[word] += 1

	cnt.most_common(10)
	freq = set([w for (w, wc) in cnt.most_common(10)])
	def freqwords(text):
		return " ".join([word for word in str(text).split() if word not in freq])
	df["text_common"] = df["text_punct"].apply(freqwords)
	df["text_common"].head()

	#rare word removal
	freq = pd.Series(' '.join(df['text_common']).split()).value_counts()[-10:] # 10 rare words
	freq = list(freq.index)
	df['text_rare'] = df['text_common'].apply(lambda x: " ".join(x for x in x.split() if x not in freq))
	df['text_rare'].head()

	#spell correction
	df['text_rare'][:5].apply(lambda x: str(TextBlob(x).correct()))

	#emoji removal
	def emoji(string):
		emoji_pattern = re.compile("["
		                   #u"\U0001F600-\U0001F64F"  # emoticons
		                   u"\U0001F300-\U0001F5FF"  # symbols & pictographs
		                   u"\U0001F680-\U0001F6FF"  # transport & map symbols
		                   u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
		                   u"\U00002702-\U000027B0"
		                   u"\U000024C2-\U0001F251"
		                   "]+", flags=re.UNICODE)
		return emoji_pattern.sub(r'', string)
	df['text_emo'] = df['text_rare'].apply(emoji)

	#verbalize emoji & emoticons
	def convert_emojis(text):
		for emot in UNICODE_EMO:
			text = text.replace(emot, "_".join(UNICODE_EMO[emot].replace(",","").replace(":","").split()))
			return text
	# Converting emoticons to words
	def convert_emoticons(text):
		for emot in EMOTICONS:
			text = re.sub(u'('+emot+')', "_".join(EMOTICONS[emot].replace(",","").split()), text)
			return text
	# Passing both functions to 'text_rare'
	df['text_v'] = df['text_emo'].apply(convert_emoticons)
	df['text_v'] = df['text_emo'].apply(convert_emojis)

	pd.set_option("display.max_rows", None, "display.max_columns", None)
	df.to_csv("formated.csv")

def get_tweet_sentiment(tweet):
	'''
	Utility function to classify sentiment of passed tweet
	using textblob's sentiment method
	'''
	# create TextBlob object of passed tweet text
	analysis = TextBlob(tweet)
	# set sentiment
	if analysis.sentiment.polarity > 0:
		return 'positive'
	elif analysis.sentiment.polarity == 0:
		return 'neutral'
	else:
		return 'negative'

prep(df['Text'])

df['sentiment_blob'] = df['text_v'].apply(get_tweet_sentiment)
df['sentiment_blob'].head()

#create labels
labels = df["sentiment_blob"].values
labels.shape

indices = df.index.values
indices.shape

#define text to be processed
data = df['text_v'].values
data.shape

#stop words bag of words
vectorizer = TfidfVectorizer(max_features=2500, min_df=7, max_df=0.8, stop_words=stopwords.words('english'))
data = vectorizer.fit_transform(data).toarray()

#divide into train and test
X_train, X_test, indices_train, indices_test = train_test_split(data, indices, test_size=0.3, random_state=0)

y_train, y_test = labels[indices_train],  labels[indices_test]

#train the model - classifier
text_classifier = RandomForestClassifier(n_estimators=200, random_state=0)
text_classifier.fit(X_train, y_train)

#predictions
predictions = text_classifier.predict(X_test)
pred_pr = text_classifier.predict_proba(X_test)
pred_pro = text_classifier.predict_proba(X_train)

#put into table
df1 = pd.DataFrame(pred_pr, index = indices_test)
df2 = pd.DataFrame(pred_pro, index = indices_train)
frames = [df1,df2]
res = pd.concat(frames)
res = res.sort_index()

df['sentiment_confidence'] = res.max(axis=1)

#model evaluation
print('Confiusion matrix: ')
conf_matrix = pd.crosstab(y_test, predictions, rownames=['Actual'], colnames=['Predicted'])
print(conf_matrix)
print('\nClassification report: \n')
class_r = classification_report(y_test,predictions)
print(class_r)
print("Accuracy score: " + str(accuracy_score(y_test, predictions)))

#pretty charts and graphs
plot_size = plt.rcParams["figure.figsize"]
print(plot_size[0])
print(plot_size[1])

plot_size[0] = 8
plot_size[1] = 6
plt.rcParams["figure.figsize"] = plot_size

#piechart of pos/neg/neu
df.sentiment_blob.value_counts().plot(kind='pie', autopct='%1.0f%%')
plt.savefig('sentiment.png')
plt.clf()

#average conf
sns.barplot(x='sentiment_blob', y='sentiment_confidence' , data=df)
plt.savefig('sentiment_confidence.png')
plt.clf()

#conf matrix
sns.heatmap(conf_matrix, annot=True)
plt.savefig('confiusion_matrix_sent.png')
plt.clf()

#word cloud
from wordcloud import WordCloud, STOPWORDS

def wc(data,bgcolor,title):
    plt.figure()
    wc = WordCloud(background_color = bgcolor, max_words = 1000,  max_font_size = 50)
    wc.generate(' '.join(data))
    plt.imshow(wc)
    plt.axis('off')

wc(df['text_v'], 'black', 'Common words')
plt.savefig('word_cloud_sentiment.png')
plt.clf()


text:
import matplotlib as mpl
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import matplotlib.pyplot as plt
import math as maths

points, approximate, exact = np.loadtxt("../data/solution.dat", unpack=True)

plt.figure(1)

plt.plot(points, approximate, 'b-', label="Approximation")
plt.plot(points, exact,       'g-', label="Exact")
plt.grid(True)
plt.xlabel("x")
plt.ylabel("u")
plt.title("Solution Plot")
plt.legend(loc="lower right")

plt.show()


text:
import pyaudio
import numpy as np
import math
import time
from numpy import inf

RATE = 44100
BUFFER = 882

BANDS = 24
BAND_DATA = np.zeros(BANDS)

p = pyaudio.PyAudio()

stream = p.open(
    format = pyaudio.paFloat32,
    channels = 1,
    rate = RATE,
    input = True,
    output = False,
    frames_per_buffer = BUFFER
)

while True:
  try:
    data = np.fft.rfft(np.fromstring(
      stream.read(BUFFER, exception_on_overflow=False), dtype=np.float32)
    )
  except IOError:
    pass

  data = np.log10(np.sqrt(
    np.real(data)**2+np.imag(data)**2) / BUFFER) * 10

  size = len(data)
  band_size = int(math.floor(size / BANDS))
  new_bands = np.zeros(BANDS)
  count = np.zeros(BANDS)
  for i in range(size):
    band = int(math.floor(float(i) / (band_size + 1)))
    # print i, band
    new_bands[band] += abs(data[i])
    count[band] += 1

  # smooth out the last point
  if count[BANDS - 1] < band_size:
    new_bands[BANDS - 1] *= (band_size / count[BANDS - 1] * 1.06)

  # print new_bands, count, size

  for i in range(BANDS):
    new_bands[i] = (50 - ((new_bands[i] / band_size ))) * 7
    if (new_bands[i] == inf):
        new_bands[i] = 256
    if (new_bands[i] == -inf):
        new_bands[i] = 0

    BAND_DATA[i] = BAND_DATA[i] * 0.9 + new_bands[i] * 0.1
    BAND_DATA[i] = min(max(0, BAND_DATA[i]), 256)

  print BAND_DATA

max_stars_repo_name: amcolash/hyperion-audio-effects

max_issues_repo_name: amcolash/hyperion-audio-effects

max_forks_repo_name: amcolash/hyperion-audio-effects

reason: import numpy,from numpy


text:
# Copyright (c) 2016-2019 Florian Wagner
#
# This file is part of XL-mHG.

"""Contains the `mHGResult` class."""

import sys
import hashlib
import logging

import numpy as np

try:
    # This is a duct-tape fix for the Google App Engine, on which importing
    # the C extension fails.
    from . import mhg_cython
except ImportError:
    print('Warning (xlmhg): Failed to import "mhg_cython" C extension.',
          file=sys.stderr)
    from . import mhg as mhg_cython

logger = logging.getLogger(__name__)

class mHGResult(object):
    """The result of an XL-mHG test.

    This class is used by the `get_xlmhg_test_result` function to represent the
    result of an XL-mHG test.

    Parameters
    ----------
    N: int
        See :attr:`N` attribute.
    indices
        See :attr:`indices` attribute.
    X: int
        See :attr:`X` attribute.
    L: int
        See :attr:'L' attribute.
    stat: float
        See :attr:`stat` attribute.
    cutoff: int
        See :attr:`cutoff` attribute.
    pval: float
        See :attr:`pval` attribute.
    pval_thresh: float, optional
        See :attr:`pval_thresh` attribute.
    escore_pval_thresh: float, optional
        See :attr:`escore_pval_thresh` attribute.
    escore_tol: float, optional
        See :attr:`escore_tol` attribute.

    Attributes
    ----------
    N: int
        The length of the ranked list (i.e., the number of elements in it).
    indices: `numpy.ndarray` with ``ndim=1`` and ``dtype=np.uint16``.
        A sorted (!) list of indices of all the 1's in the ranked list.
    X: int
        The XL-mHG X parameter.
    L: int
        The XL-mHG L parameter.
    stat: float
        The XL-mHG test statistic.
    cutoff: int
        The XL-mHG cutoff.
    pval: float
        The XL-mHG p-value.
    pval_thresh: float or None
        The user-specified significance (p-value) threshold for this test.
    escore_pval_thresh: float or None
        The user-specified p-value threshold used in the E-score calculation.
    escore_tol: float or None
        The floating point tolerance used in the E-score calculation.
    """
    def __init__(self, N, indices, X, L, stat, cutoff, pval,
                 pval_thresh=None, escore_pval_thresh=None, escore_tol=None):

        assert isinstance(N, int)
        assert isinstance(indices, np.ndarray) and indices.ndim == 1 and \
               np.issubdtype(indices.dtype, np.uint16) and \
               indices.flags.c_contiguous
        assert isinstance(X, int)
        assert isinstance(L, int)
        assert isinstance(stat, float)
        assert isinstance(cutoff, int)
        assert isinstance(pval, float)
        if pval_thresh is not None:
            assert isinstance(pval_thresh, float)
        if escore_pval_thresh is not None:
            assert isinstance(escore_pval_thresh, float)
        if escore_tol is not None:
            assert isinstance(escore_tol, float)

        self.indices = indices
        self.N = N
        self.X = X
        self.L = L
        self.stat = stat
        self.cutoff = cutoff
        self.pval = pval
        self.pval_thresh = pval_thresh
        self.escore_pval_thresh = escore_pval_thresh
        self.escore_tol = escore_tol

    def __repr__(self):
        return '<%s object (N=%d, K=%d, pval=%.1e, hash="%s")>' \
               % (self.__class__.__name__,
                  self.N, self.K, self.pval, self.hash)

    def __str__(self):
        return '<%s object (N=%d, K=%d, X=%d, L=%d, pval=%.1e)>' \
               % (self.__class__.__name__,
                  self.N, self.K, self.X, self.L, self.pval)

    def __eq__(self, other):
        if self is other:
            return True
        elif type(self) == type(other):
            return self.hash == other.hash
        else:
            return NotImplemented

    def __ne__(self, other):
        return not self.__eq__(other)

    @property
    def v(self):
        """(property) Returns the list as a `numpy.ndarray`
                      (with dtype ``np.uint8``).
        """
        v = np.zeros(self.N, dtype=np.uint8)
        v[self.indices] = 1
        return v

    @property
    def K(self):
        """(property) Returns the number of 1's in the list."""
        return self.indices.size

    @property
    def k(self):
        """(property) Returns the number of 1's above the XL-mHG cutoff."""
        return int(np.sum(self.indices < self.cutoff))

    @property
    def hash(self):
        """(property) Returns a unique hash value for the result."""
        data_str = ';'.join(
            [str(repr(var)) for var in
             [self.N, self.K, self.X, self.L,
              self.stat, self.cutoff, self.pval,
              self.pval_thresh, self.escore_pval_thresh]])
        data_str += ';'
        data = data_str.encode('UTF-8') + self.indices.tobytes()
        return str(hashlib.md5(data).hexdigest())

    @property
    def fold_enrichment(self):
        """(property) Returns the fold enrichment at the XL-mHG cutoff."""
        return self.k / (self.K*(self.cutoff/float(self.N)))

    @property
    def escore(self):
        """(property) Returns the E-score associated with the result."""
        hg_pval_thresh = self.escore_pval_thresh or self.pval
        escore_tol = self.escore_tol or mhg_cython.get_default_tol()
        es = mhg_cython.get_xlmhg_escore(
            self.indices, self.N, self.K, self.X, self.L,
            hg_pval_thresh, escore_tol)
        return es

max_stars_repo_name: flo-compbio/xlmhg

max_issues_repo_name: flo-compbio/xlmhg

max_forks_repo_name: flo-compbio/xlmhg


text:
# Copyright 2017-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import os

import boto3
import pytest
from sagemaker.tensorflow import TensorFlow
from sagemaker.tuner import HyperparameterTuner, IntegerParameter
from six.moves.urllib.parse import urlparse

from test.test_utils import is_pr_context, SKIP_PR_REASON
from ...integration.utils import processor, py_version, unique_name_from_base  # noqa: F401
from .timeout import timeout

@pytest.mark.skipif(is_pr_context(), reason=SKIP_PR_REASON)
@pytest.mark.model("mnist")
@pytest.mark.deploy_test
def test_mnist(sagemaker_session, ecr_image, instance_type, framework_version):
    resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
    script = os.path.join(resource_path, 'mnist', 'mnist.py')
    estimator = TensorFlow(entry_point=script,
                           role='SageMakerRole',
                           train_instance_type=instance_type,
                           train_instance_count=1,
                           sagemaker_session=sagemaker_session,
                           image_name=ecr_image,
                           framework_version=framework_version,
                           script_mode=True)
    inputs = estimator.sagemaker_session.upload_data(
        path=os.path.join(resource_path, 'mnist', 'data'),
        key_prefix='scriptmode/mnist')
    estimator.fit(inputs, job_name=unique_name_from_base('test-sagemaker-mnist'))
    _assert_s3_file_exists(sagemaker_session.boto_region_name, estimator.model_data)

@pytest.mark.skipif(is_pr_context(), reason=SKIP_PR_REASON)
@pytest.mark.model("mnist")
@pytest.mark.multinode(2)
@pytest.mark.integration("no parameter server")
def test_distributed_mnist_no_ps(sagemaker_session, ecr_image, instance_type, framework_version):
    resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
    script = os.path.join(resource_path, 'mnist', 'mnist.py')
    estimator = TensorFlow(entry_point=script,
                           role='SageMakerRole',
                           train_instance_count=2,
                           train_instance_type=instance_type,
                           sagemaker_session=sagemaker_session,
                           image_name=ecr_image,
                           framework_version=framework_version,
                           script_mode=True)
    inputs = estimator.sagemaker_session.upload_data(
        path=os.path.join(resource_path, 'mnist', 'data'),
        key_prefix='scriptmode/mnist')
    estimator.fit(inputs, job_name=unique_name_from_base('test-tf-sm-distributed-mnist'))
    _assert_s3_file_exists(sagemaker_session.boto_region_name, estimator.model_data)

@pytest.mark.model("mnist")
@pytest.mark.multinode(2)
@pytest.mark.integration("parameter server")
def test_distributed_mnist_ps(sagemaker_session, ecr_image, instance_type, framework_version):
    resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
    script = os.path.join(resource_path, 'mnist', 'mnist_estimator.py')
    estimator = TensorFlow(entry_point=script,
                           role='SageMakerRole',
                           hyperparameters={'sagemaker_parameter_server_enabled': True},
                           train_instance_count=2,
                           train_instance_type=instance_type,
                           sagemaker_session=sagemaker_session,
                           image_name=ecr_image,
                           framework_version=framework_version,
                           script_mode=True)
    inputs = estimator.sagemaker_session.upload_data(
        path=os.path.join(resource_path, 'mnist', 'data-distributed'),
        key_prefix='scriptmode/mnist-distributed')
    estimator.fit(inputs, job_name=unique_name_from_base('test-tf-sm-distributed-mnist'))
    _assert_checkpoint_exists(sagemaker_session.boto_region_name, estimator.model_dir, 0)
    _assert_s3_file_exists(sagemaker_session.boto_region_name, estimator.model_data)

@pytest.mark.skipif(is_pr_context(), reason=SKIP_PR_REASON)
@pytest.mark.model("mnist")
@pytest.mark.integration("s3 plugin")
def test_s3_plugin(sagemaker_session, ecr_image, instance_type, region, framework_version):
    resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
    script = os.path.join(resource_path, 'mnist', 'mnist_estimator.py')
    estimator = TensorFlow(entry_point=script,
                           role='SageMakerRole',
                           hyperparameters={
                               # Saving a checkpoint after every 5 steps to hammer the S3 plugin
                               'save-checkpoint-steps': 10,
                               # Reducing throttling for checkpoint and model saving
                               'throttle-secs': 1,
                               # Without the patch training jobs would fail around 100th to
                               # 150th step
                               'max-steps': 200,
                               # Large batch size would result in a larger checkpoint file
                               'batch-size': 1024,
                               # This makes the training job exporting model during training.
                               # Stale model garbage collection will also be performed.
                               'export-model-during-training': True
                           },
                           train_instance_count=1,
                           train_instance_type=instance_type,
                           sagemaker_session=sagemaker_session,
                           image_name=ecr_image,
                           framework_version=framework_version,
                           script_mode=True)
    estimator.fit('s3://sagemaker-sample-data-{}/tensorflow/mnist'.format(region),
                  job_name=unique_name_from_base('test-tf-sm-s3-mnist'))
    _assert_s3_file_exists(region, estimator.model_data)
    _assert_checkpoint_exists(region, estimator.model_dir, 200)

@pytest.mark.skipif(is_pr_context(), reason=SKIP_PR_REASON)
@pytest.mark.model("mnist")
@pytest.mark.integration("hpo")
def test_tuning(sagemaker_session, ecr_image, instance_type, framework_version):
    resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
    script = os.path.join(resource_path, 'mnist', 'mnist.py')

    estimator = TensorFlow(entry_point=script,
                           role='SageMakerRole',
                           train_instance_type=instance_type,
                           train_instance_count=1,
                           sagemaker_session=sagemaker_session,
                           image_name=ecr_image,
                           framework_version=framework_version,
                           script_mode=True)

    hyperparameter_ranges = {'epochs': IntegerParameter(1, 2)}
    objective_metric_name = 'accuracy'
    metric_definitions = [{'Name': objective_metric_name, 'Regex': 'accuracy = ([0-9\\.]+)'}]

    tuner = HyperparameterTuner(estimator,
                                objective_metric_name,
                                hyperparameter_ranges,
                                metric_definitions,
                                max_jobs=2,
                                max_parallel_jobs=2)

    with timeout(minutes=20):
        inputs = estimator.sagemaker_session.upload_data(
            path=os.path.join(resource_path, 'mnist', 'data'),
            key_prefix='scriptmode/mnist')

        tuning_job_name = unique_name_from_base('test-tf-sm-tuning', max_length=32)
        tuner.fit(inputs, job_name=tuning_job_name)
        tuner.wait()

@pytest.mark.skip(reason="skip the test temporarily due to timeout issue")
@pytest.mark.model("mnist")
@pytest.mark.integration("smdebug")
@pytest.mark.skip_py2_containers
def test_smdebug(sagemaker_session, ecr_image, instance_type, framework_version):
    resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
    script = os.path.join(resource_path, 'mnist', 'mnist_smdebug.py')
    hyperparameters = {'smdebug_path': '/tmp/ml/output/tensors'}
    estimator = TensorFlow(entry_point=script,
                           role='SageMakerRole',
                           train_instance_type=instance_type,
                           train_instance_count=1,
                           sagemaker_session=sagemaker_session,
                           image_name=ecr_image,
                           framework_version=framework_version,
                           script_mode=True,
                           hyperparameters=hyperparameters)
    inputs = estimator.sagemaker_session.upload_data(
        path=os.path.join(resource_path, 'mnist', 'data'),
        key_prefix='scriptmode/mnist_smdebug')
    estimator.fit(inputs, job_name=unique_name_from_base('test-sagemaker-mnist-smdebug'))
    _assert_s3_file_exists(sagemaker_session.boto_region_name, estimator.model_data)

def _assert_checkpoint_exists(region, model_dir, checkpoint_number):
    _assert_s3_file_exists(region, os.path.join(model_dir, 'graph.pbtxt'))
    _assert_s3_file_exists(region,
                           os.path.join(model_dir, 'model.ckpt-{}.index'.format(checkpoint_number)))
    _assert_s3_file_exists(region,
                           os.path.join(model_dir, 'model.ckpt-{}.meta'.format(checkpoint_number)))

def _assert_s3_file_exists(region, s3_url):
    parsed_url = urlparse(s3_url)
    s3 = boto3.resource('s3', region_name=region)
    s3.Object(parsed_url.netloc, parsed_url.path.lstrip('/')).load()

max_stars_repo_name: Elizaaaaa/deep-learning-containers

max_issues_repo_name: Elizaaaaa/deep-learning-containers

max_forks_repo_name: Elizaaaaa/deep-learning-containers

reason: from sage


text:
# <Copyright 2022, Argo AI, LLC. Released under the MIT license.>

"""Primitive types for vector maps (point, polyline)."""

from __future__ import annotations

from dataclasses import dataclass
from typing import Dict, List

import numpy as np

from av2.utils.typing import NDArrayFloat

@dataclass
class Point:
    """Represents a single 3-d point."""

    x: float
    y: float
    z: float

    @property
    def xyz(self) -> NDArrayFloat:
        """Return (3,) vector."""
        return np.array([self.x, self.y, self.z])

    def __eq__(self, other: object) -> bool:
        """Check for equality with another Point object."""
        if not isinstance(other, Point):
            return False

        return all([self.x == other.x, self.y == other.y, self.z == other.z])

@dataclass
class Polyline:
    """Represents an ordered point set with consecutive adjacency."""

    waypoints: List[Point]

    @property
    def xyz(self) -> NDArrayFloat:
        """Return (N,3) array representing ordered waypoint coordinates."""
        return np.vstack([wpt.xyz for wpt in self.waypoints])

    @classmethod
    def from_json_data(cls, json_data: List[Dict[str, float]]) -> Polyline:
        """Generate object instance from list of dictionaries, read from JSON data.

        Args:
            json_data: JSON formatted data.

        Returns:
            `Polyline` object.
        """
        return cls(waypoints=[Point(x=v["x"], y=v["y"], z=v["z"]) for v in json_data])

    @classmethod
    def from_array(cls, array: NDArrayFloat) -> Polyline:
        """Generate object instance from a (N,3) Numpy array.

        Args:
            array: array of shape (N,3) representing N ordered three-dimensional points.

        Returns:
            Polyline object.
        """
        return cls(waypoints=[Point(*pt.tolist()) for pt in array])

    def __eq__(self, other: object) -> bool:
        """Check for equality with another Polyline object."""
        if not isinstance(other, Polyline):
            return False

        if len(self.waypoints) != len(other.waypoints):
            return False

        return all([wpt == wpt_ for wpt, wpt_ in zip(self.waypoints, other.waypoints)])

    def __len__(self) -> int:
        """Return the number of waypoints in the polyline."""
        return len(self.waypoints)