text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
import numpy as np
from six.moves import zip
import utool as ut

def test_simple_parallel():
    r"""
    CommandLine:
        python -m tests.test_pyhesaff_simple_parallel --test-test_simple_parallel --show

    Example:
        >>> # ENABLE_DOCTEST
        >>> from tests.test_pyhesaff_simple_parallel import *  # NOQA
        >>> import matplotlib as mpl
        >>> from matplotlib import pyplot as plt
        >>> img_fpaths, kpts_array, desc_array = test_simple_parallel()
        >>> ut.quit_if_noshow()
        >>> # Do not plot by default
        >>> fig = plt.figure()
        >>> for count, (img_fpath, kpts, desc) in enumerate(zip(img_fpaths, kpts_array,
        >>>                                                     desc_array)):
        >>>     if count > 3:
        >>>         break
        >>>     ax = fig.add_subplot(2, 2, count + 1)
        >>>     img = mpl.image.imread(img_fpath)
        >>>     plt.imshow(img)
        >>>     _xs, _ys = kpts.T[0:2]
        >>>     ax.plot(_xs, _ys, 'ro', alpha=.5)
        >>> ut.show_if_requested()
    """
    import pyhesaff

    test_fnames = ['carl.jpg', 'lena.png', 'zebra.png', 'ada.jpg', 'star.png']
    img_fpaths = list(map(ut.grab_test_imgpath, test_fnames)) * 2

    # Time parallel computation
    with ut.Timer('Timing Parallel'):
        kpts_array, desc_array = pyhesaff.detect_feats_list(img_fpaths)

    # Time serial computation
    kpts_list2 = []
    desc_list2 = []
    with ut.Timer('Timing Iterative'):
        for img_fpath in img_fpaths:
            kpts_, desc_ = pyhesaff.detect_feats(img_fpath)
            kpts_list2.append(kpts_)
            desc_list2.append(desc_)

    print('Checking for errors')
    for (kpts_, desc_, kpts, desc) in zip(kpts_list2, desc_list2, kpts_array, desc_array):
        print(
            'shape(kpts, kpts_, desc, desc_) = %9r, %9r, %11r, %11r'
            % (kpts.shape, kpts_.shape, desc.shape, desc_.shape)
        )
        try:
            assert np.all(kpts_ == kpts), 'parallel computation inconsistent'
            assert np.all(desc_ == desc), 'parallel computation inconsistent'
            assert len(kpts_) > 0, 'no kpts detected'
            # assert False, 'deliberate triggering to see printouts'
        except Exception as ex:
            ut.printex(ex)
            raise
    print('Keypoints seem consistent')
    return img_fpaths, kpts_array, desc_array

if __name__ == '__main__':
    """
    CommandLine:
        python -m tests.test_pyhesaff_simple_parallel
        python -m tests.test_pyhesaff_simple_parallel --allexamples
        python -m tests.test_pyhesaff_simple_parallel --allexamples --noface --nosrc
    """
    import multiprocessing

    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA

    ut.doctest_funcs()

max_stars_repo_name: WildbookOrg/hesaff

max_issues_repo_name: WildbookOrg/hesaff

max_forks_repo_name: WildbookOrg/hesaff


text:
# -*- coding: utf-8 -*-
"""
Created on Sun Nov 22 14:14:34 2020

@author: mclea
"""
import numpy as np
import matplotlib.pyplot as plt

dx = 0.01
X = np.arange(-4, 4+dx, dx)
Y = np.arange(-4, 4+dx, dx)
XY = np.meshgrid(X, Y)
a = 1
epsilon = 0.02
N =  np.array([[1,0],
               [0,1]])

def cartesian_to_polar(x, y):
    """

    Parameters
    ----------
    X : Numpy array
        A numpy array of X coordinates
    Y : Numpy array
        A numpy array of Y coordinats

    Returns
    -------
    r : The radial distance
    theta : The angle, in radians

    """
    r = np.sqrt(x**2 + y**2)
    theta = np.arctan(y/x)
    return (r, theta)

def calc_nearest_point(r, theta, a):
    """

    Parameters
    ----------
    r : Numpy array
        The radial distance
    theta : Numpy array
        The angle, in radians
    a : float
        The diameter of the circle

    Returns
    -------
    None.

    """
    new_r = np.ones(r.shape)*a
    return (new_r, theta)

def calc_distance(original, closest):
    """

    Parameters
    ----------
    original : Numpy array
        The coordinates where you want to know the distance to the closest surface
    closest : TYPE
        The closest point to the points of interest

    Returns
    -------
    distance : Numpy array
        The distance to the nearest surface, negative within the body.

    """
    distance = original[0]-closest[0]
    return distance

def calc_delta_solid_body(d, epsilon):
    """
    Calculates the interpolation function delta for a immersed solid body

    Parameters
    ----------
    d : numpy array
        The signed distance to the nearest point on a fluid/solid interface,
        chosen negative within the solid.
    epsilon : float
        The kernal diameter.

    Returns
    -------
    delta : numpy array
        Returns the kernal zeroth moment over a body, delta epsilon B. The
        integrated value is 1 within the body, 0 on the exterior and a smooth
        transition over 2*epsilon.
    """
    in_kernal = abs(d)/epsilon < 1
    outside_kernal = d/epsilon < -1
    delta = np.zeros(d.shape)
    delta[in_kernal] = 0.5*(1-np.sin((np.pi/2)*(d[in_kernal]/epsilon)))
    delta[outside_kernal] = 1
    return delta

def divergence(f):
    """
    Calculates the divergence of a field f(x, y, z,...)

    Parameters
    ----------
    f : Numpy array
        A numpy array of a vector field [U, V, W]

    Returns
    -------
    divergence:
       The divergence of the vector field [U, V, W]

    """
    num_dims = len(f)
    return np.ufunc.reduce(np.add, [np.gradient(f[i], axis=i) for i in range(num_dims)])

U = -1*np.ones(XY[0].shape) # Uniform flow of magnitude 1
V = 0*np.ones(XY[0].shape) # Zero flow up
velocity = np.array([U, V])

polar_coords = cartesian_to_polar(XY[0], XY[1])
nearest_coords = calc_nearest_point(polar_coords[0], polar_coords[1], a)
distance = calc_distance(polar_coords, nearest_coords)
delta = calc_delta_solid_body(distance, epsilon)

velocity = delta*velocity
g = divergence(velocity)

# uu = delta * U
# B = np.diff(delta*U, axis=1)
# plt.figure()
plt.contourf(XY[0], XY[1], g, cmap="RdGy")
plt.colorbar();

ext: py

max_stars_repo_path: Fluid Dynamics/BDIM/Potential flow.py

max_stars_repo_name: mcleantom/CFDPython

max_issues_repo_path: Fluid Dynamics/BDIM/Potential flow.py

max_issues_repo_name: mcleantom/CFDPython

max_forks_repo_path: Fluid Dynamics/BDIM/Potential flow.py

max_forks_repo_name: mcleantom/CFDPython

reason: import numpy


text:
import random
import numpy as np
import tensorflow as tf

from data import *
from model import SegCaps
from config import get_config
from utils import ImageMaskGenerator
#import tensorflow.contrib.eager as tfe

#TODO: update code for Eager Execution

#tf.enable_eager_execution()
#print("TensorFlow version: {}".format(tf.VERSION))
#print("Eager execution: {}".format(tf.executing_eagerly()))

#adpapted for Siamese NN for One-Shot Image Recognition:
def contrastive_loss(model1, model2, y, margin):
	with tf.name_scope("contrastive-loss"):
		d = tf.sqrt(tf.reduce_sum(tf.pow(model1-model2, 2), 1, keep_dims=True))
		tmp= y * tf.square(d)
		tmp2 = (1 - y) * tf.square(tf.maximum((margin - d),0))
		return tf.reduce_mean(tmp + tmp2) /2

def triplet_loss(model1):
  return None

#Hyperparameters:
#train_loss_results=[]
#train_accuracy_results=[]

#keep results for plotting
#train_loss_results=[]
#train_accuracy_results=[]
num_epochs=201

def main():
  # Get configuration
  config = get_config()
  tf_config = tf.compat.v1.ConfigProto(allow_soft_placement=True)
  tf_config.gpu_options.visible_device_list = config.device
  #with tf.Session(init) as sess:
  with tf.compat.v1.Session(config=tf_config) as sess:
    #for d in ['/device:GPU:0','/device:GPU:1','/device:GPU:2']:
    with tf.device('/device:GPU:0'):
      # model = SegCaps(sess, config, is_train=True)
      # with tf.variable_scope("model") as scope:
      model = SegCaps(sess, config, is_train=True)
      sess.run(tf.compat.v1.global_variables_initializer())
      # with tf.variable_scope(scope, reuse=True):
      # model2 = SegCaps(sess, config, is_train=True)
      images, labels = ImageMaskGenerator('dataset/annotations/val2017/', 'dataset/images/val2017/')
      print(images.shape)
      print(labels.shape)
      for epoch in range(num_epochs):
        loss_value = model.fit(images, labels)
        print(loss_value)
        model.save()

if __name__=="__main__":
  # To reproduce the result
  tf.random.set_seed(2018)
  np.random.seed(2018)
  random.seed(2018)
  main()


text:
def demo():
	import numpy as np
	import matplotlib.pyplot as plt
	import xhorizon as xh
	from xhorizon.evap.massplot import massplot
	from xhorizon.evap import helpers

	"""
	"""

	## setup figure #################
	## rc
	plt.rcParams['font.size'] = 7
	plt.rcParams['axes.linewidth'] = .4
	plt.rcParams['xtick.major.size'] = 2
	plt.rcParams['ytick.major.size'] = 2
	plt.rcParams['lines.solid_capstyle'] = 'butt'
	plt.rcParams['lines.dash_capstyle'] = 'butt'
	## figure size
	fig_width  = 2.5  ## inches
	fig_aspect = 1.68   ## height/width
	fig_height = fig_width * fig_aspect
	## axis size
	width  = 0.86   ## x fraction
	aspect = 1.84   ## absolute height/width
	height = aspect * width / fig_aspect    ## y fraction
	## axis loc
	left, bottom = .12, .05     ## x fraction, y fraction
	## define figure and axes
	plt.figure(1,figsize=(fig_width,fig_height))
	ax1 = plt.axes([left,bottom,width,height], aspect=1.)
	ax = [ax1]
	## format axes
	for axx in ax:
		## set axis
		plt.sca(axx)
		## labels
		plt.xlabel('$V-U$', labelpad=-7)
		plt.ylabel('$V+U$', labelpad=-4)
		## ticks
		plt.xticks([-1,1])
		plt.yticks([-1,1])
		## lims
		sq = 2.4
		x0 = -1.2
		y0 = 0.
		plt.xlim(x0,x0+sq)
		plt.ylim(y0-0.5*aspect*sq,y0+0.5*aspect*sq)
	################################

	## save
	path = "temp-figs/demo"
	sfp = dict(dpi=800)
	temp_only = False

	## draw
	draw = True

	## label
	label = '$(b)$'

	## input
	l  = .01
	le = .01

	## input
	R  = .2

	## input
	L = 5.

	## accrete
	Tacc = 1.25

	## vv
	voff = -.4*Tacc - 0.

	## params
	seed = 0
	mpgo=True
	params = dict()
	chainparams = dict()

##################################

	## func
	func0 = xh.mf.dS(L=1.*L)
	func1 = xh.mf.Hay_dS(l=1.*l, R=1.*R, L=1.*L)
	func2 = xh.mf.Hay_dS_cutoff(l=1.*l, R=1.*R, L=1.*L)
	func3 = xh.mf.dS(L=1.*L)
	funclist = [func0, func1, func2, func3]

	## need to match 1 and 2 at infinty
	c12 = func2.F(10.*L) - func1.F(10.*L)

	## regions
	deflines = False
	reg0 = xh.reg.MAXreg(func0, boundary=deflines, rlines=deflines)
	reg1 = xh.reg.EFreg( func1, boundary=deflines, rlines=deflines)
	reg2 = xh.reg.MAXreg(func2, rparams=dict(c=1.*c12), boundary=deflines, rlines=deflines)
	reg3 = xh.reg.MAXreg(func3, boundary=deflines, rlines=deflines)
	reglist = [reg0, reg1, reg2, reg3]

	## modify Hay-AdS region to add extra blocks
	## top
	b  = reg1.blocks[3]
	cdlu = 1.*b.bparams['cdlu'] + 1.
	cdlv = 1.*b.bparams['cdlv'] + 1.
	epsu = -1.*b.bparams['epsu']
	epsv = -1.*b.bparams['epsv']
	bparams = dict(cdlu=1.*cdlu, cdlv=1.*cdlv, epsu=1.*epsu, epsv=1.*epsv)
	reg1.blocks += [xh.block(b.master, b.j, dict(cdlu=1.*cdlu, cdlv=1.*cdlv, epsu=1.*epsu, epsv=1.*epsv))]
	## right
	b  = reg1.blocks[2]
	cdlu = 1.*b.bparams['cdlu'] - 1.
	cdlv = 1.*b.bparams['cdlv'] + 1.
	epsu = -1.*b.bparams['epsu']
	epsv = -1.*b.bparams['epsv']
	bparams = dict(cdlu=1.*cdlu, cdlv=1.*cdlv, epsu=1.*epsu, epsv=1.*epsv)
	reg1.blocks += [xh.block(b.master, b.j, dict(cdlu=1.*cdlu, cdlv=1.*cdlv, epsu=1.*epsu, epsv=1.*epsv))]

	## chain params
	m  = np.array([0., 0.5*R, 0., 0.])
	Rh = np.array([0., func1.rj[2], 0., 0.])
	va = 1.*voff
	vb = 1.*va + 1.*Tacc

	#### initial region
	## passive slice
	sliceloc0 = dict(v0=1.*va, r0=3.*L)
	pslice0 = xh.junc.pslice(reg0, ublocks=[1,2], vblocks=[0,1], **sliceloc0)

	#### Hay-AdS region
	## active slice
	aslice1 = xh.junc.aslice(reg1, ublocks=[3,5], vblocks=[0,1,2,3], U0=pslice0.U_of_r_at_v0, V0=pslice0.V_of_r_at_u0, r_refs=[pslice0.reg.metfunc.r_ref], **sliceloc0)
	## modify transformations
	reg1.U_of_udl = aslice1.U_of_udl_at_v0
	reg1.V_of_vdl = aslice1.V_of_vdl_at_u0
	## passive slice
	sliceloc1 = dict(v0=1.*vb, r0=Rh[1]+le)
	pslice1 = xh.junc.pslice(reg1, ublocks=[2,4], vblocks=[0,1,2,3], **sliceloc1)

	#### Cutoff region
	## modify transformations
	reg2.U_of_udl = reg1.U_of_udl
	reg2.V_of_vdl = reg1.V_of_vdl

	#### Final region
	## active slice
	aslice3 = xh.junc.aslice(reg3, ublocks=[0,3], vblocks=[0,1], U0=pslice1.U_of_r_at_v0, V0=pslice1.V_of_r_at_u0, r_refs=[pslice1.reg.metfunc.r_ref], **sliceloc1)
	ub = 1.*aslice3.u0
	## modify transformations
	reg3.U_of_udl = aslice3.U_of_udl_at_v0
	reg3.V_of_vdl = aslice3.V_of_vdl_at_u0

	#### mask
	## initial region
	reg0.blocks = reg0.blocks[0:2]
	for b in reg0.blocks:
		b.uvbounds.update(dict(vmax=1.*va))
	# Cutoff region
	reg2.blocks = reg2.blocks[2:3]
	reg2.blocks[0].bparams.update(reg1.blocks[-1].bparams)
	reg2.blocks[0].update()
	# Hay-AdS region past slice
	reg1.blocks = reg1.blocks[0:5]
	for b in reg1.blocks[0:4]:
		b.uvbounds.update(dict(vmin=1.*va))
	# Hay-AdS future slice
	newblocks = []
	for i in [0,1]:
		newblocks += [xh.block(reg1.blocks[i].master, reg1.blocks[i].j, reg1.blocks[i].bparams)]
		newblocks[-1].uvbounds.update(reg1.blocks[i].uvbounds)
		newblocks[-1].uvbounds.update(dict(vmax=1.*vb))
	for i in [2]:
		newblocks += [xh.block(reg1.blocks[i].master, reg1.blocks[i].j, reg1.blocks[i].bparams)]
		newblocks[-1].uvbounds.update(reg1.blocks[i].uvbounds)
		newblocks[-1].uvbounds.update(dict(umax=1.*ub, vmax=1.*vb))
	for i in [2]:
		newblocks += [xh.block(reg1.blocks[i].master, reg1.blocks[i].j, reg1.blocks[i].bparams)]
		newblocks[-1].uvbounds.update(reg1.blocks[i].uvbounds)
		newblocks[-1].uvbounds.update(dict(umax=1.*ub, vmin=1.*vb))
	for i in [4]:
		newblocks += [xh.block(reg1.blocks[i].master, reg1.blocks[i].j, reg1.blocks[i].bparams)]
		newblocks[-1].uvbounds.update(reg1.blocks[i].uvbounds)
		newblocks[-1].uvbounds.update(dict(umax=1.*ub))
	for i in [3]:
		newblocks += [xh.block(reg1.blocks[i].master, reg1.blocks[i].j, reg1.blocks[i].bparams)]
		newblocks[-1].uvbounds.update(reg1.blocks[i].uvbounds)
		newblocks[-1].uvbounds.update(dict())
	## reset Hay-AdS blocks to masked ones
	reg1.blocks = newblocks
	## final region
	reg3.blocks = [reg3.blocks[0],reg3.blocks[3]]
	reg3.blocks[0].uvbounds.update(dict(vmin=1.*vb, umin=1.*ub))
	reg3.blocks[1].uvbounds.update(dict(umin=1.*ub))

	## chainparams
	m = 1.*m
	Rh = 1.*Rh
	fs_v0 = np.array([1.*va, 1.*vb, 0., 0.])
	fs_u0 = np.array([0., 1.*ub, 0., 0.])
	fs_r0 = 1.*Rh+le
	chainparams.update(dict(m=1.*m, Rh=1.*Rh, fs_r0=1.*fs_r0, fs_u0=1.*fs_u0, fs_v0=1.*fs_v0))

	print(chainparams)

######################################

	## draw
	if draw:
		print("plot")
		pp = dict(l=1.*l, R=1.*R)
		xh.evap.draw_dS.drawreg(reglist, chainparams, fparams=pp)

	## label
	if True:
		plt.annotate(s=label, xy=(.95,.97), xycoords='axes fraction', ha='right', va='top', size=8)

	## auto param label
	if True:
		#plabel = [r"", r"$\tau_{acc}=%3s$"%(Tacc), r"$\tau_{ev}=%3s$"%(Tevap)]
		plabel = [r"$l_{ev}=%3s$"%(le), r"", r"$l=%3s$"%(l), r"$2M=%3s$"%(R), r"$L=%3s$"%(L)]
		plabel = "\n".join(plabel)
		plt.annotate(s=plabel, xy=(.95,.03), xycoords='axes fraction', ha='right', va='bottom', size=8)

	## manual param label
	if False:
		plabel = [r"$\tau_{ev}=%s$"%(Tevap), r"$\tau_{acc}=%s$"%(Tacc)]
		plabel += [r"", r"$M=10^{-3}$", r"$l_{ev}=l=10^{-4}$"]
		plabel = "\n".join(plabel)
		plt.annotate(s=plabel, xy=(.95,.03), xycoords='axes fraction', ha='right', va='bottom', size=8)

	## r=infty label
	if True:
		plt.annotate(s=r"$r=\infty$", xy=(0., 1.), ha='center', va='center', size=7)
		plt.annotate(s=r"$r=\infty$", xy=(0.,-1.05), ha='center', va='center', size=7)
		plt.annotate(s=r"$r=0$", xy=(-1.1, 0.), ha='center', va='center', size=7, rotation=90)
		plt.annotate(s=r"$r=0$", xy=(1.02, 0.), ha='center', va='center', size=7, rotation=-90)

	## save
	if True:
		xh.evap.evapsave(path=path, params=params, chainparams=chainparams, seed=seed, sfp=sfp, temp_only=temp_only, massplot=mpgo)

	## show
	if False:
		plt.show()

	## return
	return reglist, chainparams

if __name__=="__main__":
	demo()

max_stars_repo_name: xh-diagrams/xhorizon

max_issues_repo_name: xh-diagrams/xhorizon

max_forks_repo_name: xh-diagrams/xhorizon


text:
import numpy as np
import model_selection
from definitions import PROJECT_ROOT

def get_data(dataset, n_subsamples=10):
    src_fpath = PROJECT_ROOT + '/data/' + dataset + '/sources/' + dataset + '-src.csv'
    tgt_fpath = PROJECT_ROOT + '/data/' + dataset + '/targets/' + dataset + '-tgt.csv'
    src_mat = np.loadtxt(open(src_fpath, "rb"), delimiter=",")
    tgt_mat = np.loadtxt(open(tgt_fpath, "rb"), delimiter=",")
    subsamples = []
    for _ in range(n_subsamples):
        X_tgt_known, X_tgt_unknown = model_selection.active_da_train_test_split(tgt_mat)
        X_src = src_mat[:,:-1]
        y_src = src_mat[:,-1]
        X_tgt_known = X_tgt_known[:,:-1]
        y_tgt_known = X_tgt_known[:,-1]
        X_tgt_unknown= X_tgt_unknown[:, :-1]
        y_tgt_unknown = X_tgt_unknown[:, -1]
        subsamples.append((X_src, y_src, X_tgt_known, y_tgt_known, X_tgt_unknown, y_tgt_unknown))

    return subsamples

max_stars_repo_name: dainis-boumber/domain-adaptation-kit

max_issues_repo_name: dainis-boumber/domain-adaptation-kit

max_forks_repo_name: dainis-boumber/domain-adaptation-kit


text:
import getdist.plots as gplot
import matplotlib.pyplot as plt
import numpy as np
import plotsetup
plotsetup.halfpaperfig()

g = gplot.getPlotter(chain_dir = '/scratch/midway/rkessler/dscolnic/cosmomc')
g.settings.fig_width_inch = 5.0
g.settings.lw_contour = 2.0
g.settings.solid_contour_palefactor = 0.2
g.settings.legend_frame = False
g.settings.axes_fontsize=12
g.settings.legend_fontsize=12
g.settings.lab_fontsize=14
#roots = ['chains/noSN_omw', 'chains2/DS17_ALL_omw_alone','chains2/DS17_ALL_nosys_omw_alone','chains/noSN_bao_omw','chains2/DS17_ALL_omw','chains2/DS17_ALL_nosys_omw']
#names=['CMB','SN','SN(stat)','CMB+BAO','SN+CMB','SN(stat)+CMB']
#colors=['teal','k','gray','blue','purple','gray']

roots = ['chains2/noSN_omw','chains2/DS17_ALL_omw_alone','chains2/DS17_ALL_nosys_omw_alone','chains2/noSN_bao_omw','chains2/DS17_ALL_omw']
names=['CMB','SN','SN(stat)','CMB+BAO','SN+CMB']
colors=['teal','k','seashell','blue','purple']
colors=['#235789','#C1292E','#778899','#F1D302','#020100']

#roots=['chains2/DS17_ALL_omw','chains/noSN_bao_omw','chains2/DS17_ALL_nosys_omw_alone','chains2/DS17_ALL_omw_alone','chains/noSN_omw']
#colors=['#020100','#F1D302','#FDFFFC','#C1292E','#235789']

g.add_y_bands(-1,0.01,alpha1=0.15)

g.plot_2d(roots,'omegam','w',
                                filled=[True,True,False,True,True,True],colors=colors)
#g.add_y_marker(-1)
#g.plot_2d(roots, 'omegam', 'w', filled=True)
#g.legend(['CMB', 'CMB+BAO','SN+CMB','SN'], legend_loc='upper left',frameon=False,fontsize=8,ncol=2);
#line, = plt.plot(np.arange(-5,5,0.1), np.arange(-5,5,0.1)*0-1, lw=1,alpha=0.1)

plt.xlabel(r'$\Omega_m$',labelpad=10)
plt.ylabel(r'$w$',labelpad=5)
plt.xlim(0.15,0.5)
plt.ylim(-1.5,-.5)
plt.title(r'$w \rm{CDM}$'+' Constraints For Combined Samples')

plt.text(0.42,-.75,"CMB", color='blue',rotation=40)
plt.text(0.38,-1.1,"Pantheon SN", color='red',rotation=306,alpha=0.5)
plt.text(0.29,-1.23,"Pantheon SN (Stat)", color='gray',rotation=305,alpha=0.5)
#plt.text(0.28,-.75,"CMB+BAO", color='yellow')
props = dict(boxstyle='round', facecolor='yellow', alpha=0.5)
plt.text(0.26,-.8,"CMB+BAO", color='blue', bbox=props)

props = dict(boxstyle='round', facecolor='black', alpha=0.8)
plt.text(0.34,-1.0,"Pantheon SN + CMB",color='white', bbox=props)

#plt.xticks([])
#plt.yticks([])
str2=colors
#colors=['black','green','blue','red','purple']
proxy = [plt.Rectangle((0,0),.05,.05,fc = str2[pc]) for pc in range(0,5)]
#names=['CMB','SN','SN(stat)','CMB+BAO','SN+CMB','SN(stat)+CMB']
#plt.legend(proxy,names,loc='upper left',prop={'size':12},frameon=False,ncol=3)

plt.figure(figsize=(20,10))
str2=colors
#colors=['black','green','blue','red','purple']
#'CMB', 'CMB+BAO','SN+CMB','SN'

g.export('plot_omw.png')

max_stars_repo_name: LBJ-Wade/Pantheon

max_issues_repo_name: LBJ-Wade/Pantheon

max_forks_repo_name: LBJ-Wade/Pantheon


text:
import json
import glob
import pandas as pd
import numpy as np

from IPython import embed

if hasattr(json.decoder, 'JSONDecodeError'):
    jsonerror = json.decoder.JSONDecodeError
else:
    jsonerror = ValueError

# get all raw result files
#files = glob.glob('/Users/dvm/Documents/projects/covert_sequence/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/d_prime/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/dist_onset/RT/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/dist_onset/RT_exp2/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/dist_onset/RT_exp3/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/dist_onset/RT_exp4/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/present_absent/target_SL/exp_1/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/present_absent/target_SL/exp_2/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/present_absent/target_SL/exp_3/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/present_absent/dist_SL_dprime/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/spatial_transfer/object/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/spatial_transfer/control/raw/*.txt')
#files = glob.glob('/Users/dvm/Documents/projects/spatial_transfer/object_barble_cross/raw/*.txt')
files = glob.glob('/Users/dvm/Documents/projects/probe/exp_5/raw/*.txt')

# set general parameters
control_sj = True
column_idx = 'nr_trials'
sjs = []
comb = []

for f, file in enumerate(files):
    print('start reading file {} out of {} files'.format(f+1, len(files)))
    dfs = []
    partial = []

    with open(file, "r") as fd:
        # loop over each logged datafile
        for line in fd:
            # strip the lines so that they are valid json
            line = line.strip()
            # strip unwanted characters
            if line.startswith('[{'):
                line = line[1:]
            if not line.endswith('}') and '}' in line:
                idx = len(line) - line[::-1].index('}')
                line = line[:idx]
            try:
                d = json.loads(line)
            except:
                continue

            # data from succesfully finished experiments
            if 'data' in d:
                if d['data'] != []:
                    df = pd.DataFrame(d['data'])
                    while control_sj and df.loc[0,'subject_nr'] in sjs:
                        df.loc[:,'subject_nr'] = '{}#'.format(df.loc[0,'subject_nr'])
                    dfs.append(df)
                    sjs.append(df.loc[0,'subject_nr'])
            # data from incomplete experimemts
            elif d != []:
                # store all incomplete files as dictionaries
                partial.append(d)

    # print info
    print('There are {} complete data files, with the following IDs: \n{}'.format(len(sjs), sjs))

    # check whether dataset contains partial log files
    if partial != []:
        print('Data set contains incomplete log files')
        partial = pd.DataFrame(partial, dtype=object).fillna('')
        if control_sj:
            prev = 0
            for idx, sj in partial['subject_nr'].iteritems():
                while control_sj and partial.loc[idx,'subject_nr'] in sjs:
                    partial.loc[idx,'subject_nr'] = '{}#'.format(partial.loc[idx,'subject_nr'])
                if partial.loc[idx, column_idx] < prev: # start new subject
                    sjs.append(partial.loc[idx-1, 'subject_nr'])
                prev = partial.loc[idx, column_idx]

        sj_info = np.unique(partial.subject_nr)
        print('There are {} incomplete data files, with the following IDs: \n{}'.format(sj_info.size, list(sj_info)))

    # create final dataframe
    if isinstance(partial, pd.DataFrame) and dfs != []:
        df = pd.concat([pd.concat(dfs, sort = True), partial], sort = True, ignore_index = True)
    elif isinstance(partial, pd.DataFrame):
        df = partial
    elif dfs != []:
        df = pd.concat(dfs, sort = True, ignore_index = True)
    else:
        continue

    comb.append(df)

if len(comb)>1:
    df =  pd.concat(comb, sort = True, ignore_index = True)
elif len(comb) == 1:
    df = comb[0]
else:
    print('no data file found')

# save datafile
#df.to_csv('/Users/dvm/Documents/projects/covert_sequence/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/d_prime/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/dist_onset/RT/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/dist_onset/RT_exp2/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/dist_onset/RT_exp3/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/dist_onset/RT_exp4/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/present_absent/target_SL/exp_1/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/present_absent/target_SL/exp_2/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/present_absent/target_SL/exp_3/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/present_absent/dist_SL_dprime/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/present_absent/dist_SL_dprime/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/spatial_transfer/object/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/spatial_transfer/control/raw/raw_data.csv')
#df.to_csv('/Users/dvm/Documents/projects/spatial_transfer/object_barble_cross/raw/raw_data.csv')
df.to_csv('/Users/dvm/Documents/projects/probe/exp_5/raw/raw_data.csv')

max_stars_repo_name: dvanmoorselaar/DvM

max_issues_repo_name: dvanmoorselaar/DvM

max_forks_repo_name: dvanmoorselaar/DvM


text:
"""
Various tools which may be needed in various processes.
"""

from subprocess import Popen, PIPE
from math import sin, asin, cos, acos, atan, atan2, degrees, radians, sqrt, pi
from warnings import warn
from typing import Union

import numpy as np

from qcore.binary_version import get_unversioned_bin

R_EARTH = 6378.139

class InputError(Exception):
    pass

def get_distances(locations: np.ndarray, lon: Union[float, np.ndarray], lat: Union[float, np.ndarray]):
    """
    Calculates the distance between the array of locations and
    the specified reference location / locations

    Parameters
    ----------
    locations : np.ndarray
        List of locations
        Shape [n_locations, 2], column format (lon, lat)
    lon : Union[float, np.ndarray]
        Array or singular float of Longitude reference locations to compare
    lat : Union[float, np.ndarray]
        Array or singular float of Latitude reference locations to compare

    Returns
    -------
    np.ndarray
        The distances, shape [n_locations] or shape [n_references, n_locations]
        based on the input lon, lat values being a single float or array
    """
    d = (
        np.sin(np.radians(np.expand_dims(locations[:, 1], axis=1) - lat) / 2.0) ** 2
        + np.cos(np.radians(lat))
        * np.cos(np.radians(np.expand_dims(locations[:, 1], axis=1)))
        * np.sin(np.radians(np.expand_dims(locations[:, 0], axis=1) - lon) / 2.0) ** 2
    )
    d = (R_EARTH * 2.0 * np.arctan2(np.sqrt(d), np.sqrt(1 - d))).T
    return d[0] if d.shape[0] == 1 else d

def closest_location(locations, lon, lat):
    """
    Find position and distance of closest location in 2D np.array of (lon, lat).
    """
    d = get_distances(locations, lon, lat)
    i = np.argmin(d)

    return i, d[i]

def ll2gp_multi(
    coords,
    mlon,
    mlat,
    rot,
    nx,
    ny,
    hh,
    dx=1,
    dy=1,
    decimated=False,
    verbose=False,
    keep_outside=False,
):
    """
    Converts longitude/latitude positions to gridpoints.
    Three main modes of operation:
    1: No dx, dy (= 1): gridpoint
    2: dx or dy != 1: closest gridpoint considering dx/dy
    3: decimated: gridpoint number if only decimated points existed
    coords: 2d list in format [[lon0, lat0], [lon1, lat1], ...]
    keep_outside: False will remove values outside the sim domain,
            True will replace those entries with None
    """
    # derived parameters
    xlen = nx * hh
    ylen = ny * hh
    # where the x plane points
    xazim = (rot + 90) % 360

    # run binary, get output
    # output is displacement (x, y) from center, in kilometres
    cmd = [
        get_unversioned_bin("ll2xy"),
        "mlat=%s" % (mlat),
        "mlon=%s" % (mlon),
        "geoproj=1",
        "center_origin=1",
        "h=%s" % (hh),
        "xazim=%s" % (xazim),
        "xlen=%s" % (xlen),
        "ylen=%s" % (ylen),
    ]
    if verbose:
        print(" ".join(cmd))

    # Has to be a byte string
    p_conv = Popen(cmd, stdin=PIPE, stdout=PIPE)
    stdout = p_conv.communicate(
        "\n".join(["%s %s" % tuple(c) for c in coords]).encode()
    )[0].decode()
    xy = [list(map(float, line.split())) for line in stdout.rstrip().split("\n")]

    # convert displacement to grid points
    # has to be 'nx - 1', because the first gridpoint is offset 0km
    # nx = 1400 means the greatest offset is 1399 * hh km
    mid_x = (nx - 1) * hh * 0.5
    mid_y = (ny - 1) * hh * 0.5
    for i, c in enumerate(xy[::-1]):
        # make the distance relative to top corner
        # convert back to grid spacing
        # gridpoints are discrete
        c[0] = int(round((c[0] + mid_x) / hh))
        c[1] = int(round((c[1] + mid_y) / hh))

        # x values range from 0 -> nx - 1
        if not (0 <= c[0] < nx and 0 <= c[1] < ny):
            if keep_outside:
                xy[-(i + 1)] = None
            else:
                # this is why xy is looped in reverse
                xy.remove(c)
            continue

        if decimated:
            c[0] //= dx
            c[1] //= dy
        else:
            # closest gridpoint considering decimation
            c[0] -= c[0] % dx
            c[1] -= c[1] % dy

    return xy

def ll2gp(
    lat, lon, mlat, mlon, rot, nx, ny, hh, dx=1, dy=1, decimated=False, verbose=False
):
    """
    Converts latitude/longitude to a gridpoint position.
    """
    try:
        return ll2gp_multi(
            [[lon, lat]],
            mlon,
            mlat,
            rot,
            nx,
            ny,
            hh,
            dx=dx,
            dy=dy,
            decimated=decimated,
            verbose=verbose,
            keep_outside=False,
        )[0]
    except IndexError:
        raise InputError("Input outside simulation domain.")

def gp2ll_multi(coords, mlat, mlon, rot, nx, ny, hh):
    """
    Converts gridpoint positions to longitude, latitude.
    coords: 2d list in format [[x0, y0], [x1, y1], ...]
    """
    # derived parameters
    xlen = nx * hh
    ylen = ny * hh
    # where the x plane points
    xazim = (rot + 90) % 360

    # convert gridpoint to offset km
    max_x = (nx - 1) * hh
    max_y = (ny - 1) * hh
    for c in coords:
        # length from corner
        c[0] *= hh
        c[1] *= hh
        # length from centre origin
        c[0] -= max_x * 0.5
        c[1] -= max_y * 0.5

    # run binary, get output
    p_conv = Popen(
        [
            get_unversioned_bin("xy2ll"),
            "mlat=%s" % (mlat),
            "mlon=%s" % (mlon),
            "geoproj=1",
            "center_origin=1",
            "h=%s" % (hh),
            "xazim=%s" % (xazim),
            "xlen=%s" % (xlen),
            "ylen=%s" % (ylen),
        ],
        stdin=PIPE,
        stdout=PIPE,
    )
    stdout = p_conv.communicate(
        "\n".join(["%s %s" % tuple(c) for c in coords]).encode()
    )[0].decode()

    # lon, lat
    return [list(map(float, line.split())) for line in stdout.rstrip().split("\n")]

def gp2ll(x, y, mlat, mlon, rot, nx, ny, hh):
    """
    Converts a gridpoint position to latitude/longitude.
    """
    return gp2ll_multi([[x, y]], mlat, mlon, rot, nx, ny, hh)[0]

def gen_mat(mrot, mlon, mlat):
    """
    Precursor for xy2ll and ll2xy functions.
    mrot: model rotation
    mlon: model centre longitude
    mlat: model centre latitude
    """
    arg = radians(mrot)
    cosA = cos(arg)
    sinA = sin(arg)

    arg = radians(90.0 - mlat)
    cosT = cos(arg)
    sinT = sin(arg)

    arg = radians(mlon)
    cosP = cos(arg)
    sinP = sin(arg)

    amat = np.array(
        [
            [
                cosA * cosT * cosP + sinA * sinP,
                sinA * cosT * cosP - cosA * sinP,
                sinT * cosP,
            ],
            [
                cosA * cosT * sinP - sinA * cosP,
                sinA * cosT * sinP + cosA * cosP,
                sinT * sinP,
            ],
            [-cosA * sinT, -sinA * sinT, cosT],
        ],
        dtype="f",
    )
    ainv = amat.T * 1.0 / np.linalg.det(amat)

    return amat.flatten(), ainv.flatten()

def xy2ll(xy_km, amat):
    """
    Converts km offsets to longitude and latitude.
    xy_km: 2D np array of [X, Y] offsets from origin (km)
    amat: from gen_mat function
    """
    x = xy_km[:, 0] / R_EARTH
    sinB = np.sin(x)
    y = xy_km[:, 1] / R_EARTH
    sinG = np.sin(y)
    z = np.sqrt(1.0 + sinB * sinB * sinG * sinG)
    xp = sinG * np.cos(x) * z
    yp = sinB * np.cos(y) * z
    zp = np.sqrt(1.0 - xp * xp - yp * yp)

    xg = xp * amat[0] + yp * amat[1] + zp * amat[2]
    yg = xp * amat[3] + yp * amat[4] + zp * amat[5]
    zg = xp * amat[6] + yp * amat[7] + zp * amat[8]

    lat = np.where(
        zg == 0.0, 0.0, 90.0 - np.degrees(np.arctan(np.sqrt(xg * xg + yg * yg) / zg))
    )
    lat[np.where(zg < 0.0)] -= 180.0

    lon = np.where(xg == 0.0, 0.0, np.degrees(np.arctan(yg / xg)))
    lon[np.where(xg < 0.0)] -= 180.0
    lon[np.where(lon < -180.0)] += 360.0

    return np.column_stack((lon, lat))

def ll2xy(ll, ainv):
    """
    Converts longitude and latitude to km offsets.
    ll: 2D np array of [lon, lat]
    ainv: from gen_mat function
    """
    lon = np.radians(ll[:, 0])
    lat = np.radians(90.0 - ll[:, 1])

    xg = np.sin(lat) * np.cos(lon)
    yg = np.sin(lat) * np.sin(lon)
    zg = np.cos(lat)

    xp = xg * ainv[0] + yg * ainv[1] + zg * ainv[2]
    yp = xg * ainv[3] + yg * ainv[4] + zg * ainv[5]
    zp = xg * ainv[6] + yg * ainv[7] + zg * ainv[8]

    # X km offsets from centre origin, Y km offsets from centre origin
    return np.column_stack(
        (
            R_EARTH * np.arcsin(yp / np.sqrt(1.0 - xp * xp)),
            R_EARTH * np.arcsin(xp / np.sqrt(1.0 - yp * yp)),
        )
    )

def xy2gp(xy, nx, ny, hh):
    """
    Converts km offsets to grid points.
    xy: 2D np array of [X, Y] offsets from origin (km)
    nx: number of X grid positions
    ny: number of Y grid positions
    hh: grid spacing
    """
    gp = np.copy(xy)

    # distance from corner
    gp[:, 0] += (nx - 1) * hh * 0.5
    gp[:, 1] += (ny - 1) * hh * 0.5

    # gridpoint from top corner
    gp /= hh

    return np.round(gp).astype(np.int32, copy=False)

def gp2xy(gp, nx, ny, hh):
    """
    Converts grid points to km offsets.
    xy: 2D np array of [X, Y] gridpoints
    nx: number of X grid positions
    ny: number of Y grid positions
    hh: grid spacing
    """
    xy = gp.astype(np.float32) * hh

    # shift for centre origin
    xy[:, 0] -= (nx - 1) * hh * 0.5
    xy[:, 1] -= (ny - 1) * hh * 0.5

    return xy

def ll_shift(lat, lon, distance, bearing):
    """
    Shift lat/long by distance at bearing.
    """
    # formula is for radian values
    lat, lon, bearing = list(map(radians, [lat, lon, bearing]))

    shift = distance / R_EARTH
    lat2 = asin(sin(lat) * cos(shift) + cos(lat) * sin(shift) * cos(bearing))
    lon2 = lon + atan2(
        sin(bearing) * sin(shift) * cos(lat), cos(shift) - sin(lat) * sin(lat2)
    )

    return degrees(lat2), degrees(lon2)

def ll_mid(lon1, lat1, lon2, lat2):
    """
    Return midpoint between a pair of lat, long points.
    """
    # functions based on radians
    lon1, lat1, lat2, dlon = list(map(radians, [lon1, lat1, lat2, (lon2 - lon1)]))

    Bx = cos(lat2) * cos(dlon)
    By = cos(lat2) * sin(dlon)

    lat3 = atan2(sin(lat1) + sin(lat2), sqrt((cos(lat1) + Bx) ** 2 + By ** 2))
    lon3 = lon1 + atan2(By, cos(lat1) + Bx)

    return degrees(lon3), degrees(lat3)

def ll_dist(lon1, lat1, lon2, lat2):
    """
    Return distance between a pair of lat, long points.
    """
    # functions based on radians
    lat1, lat2, dlon, dlat = list(
        map(radians, [lat1, lat2, (lon2 - lon1), (lat2 - lat1)])
    )

    a = sin(dlat / 2.0) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2.0) ** 2
    return R_EARTH * 2.0 * atan2(sqrt(a), sqrt(1 - a))

def ll_bearing(lon1, lat1, lon2, lat2, midpoint=False):
    """
    Initial bearing when traveling from 1 -> 2.
    Direction facing from point 1 when looking at point 2.
    """
    if midpoint:
        lon1, lat1 = ll_mid(lon1, lat1, lon2, lat2)
    lat1, lat2, lon_diff = map(radians, [lat1, lat2, (lon2 - lon1)])
    return (
        degrees(
            atan2(
                cos(lat2) * sin(lon_diff),
                cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(lon_diff),
            )
        )
        % 360
    )

def ll_cross_along_track_dist(
    lon1, lat1, lon2, lat2, lon3, lat3, a12=None, a13=None, d13=None
):
    """
    Returns both the distance of point 3 to the nearest point on the great circle line that passes through point 1 and
    point 2 and how far away that nearest point is from point 1, along the great line circle
    If any of a12, a13, d13 are given the calculations for them are skipped
    If all of a12, a13, d13 are given, none of the lon, lat values are used and may be junk data
    Taken from https://www.movable-type.co.uk/scripts/latlong.html
    :param lon1, lat1: The lon, lat coordinates for point 1
    :param lon2, lat2: The lon, lat coordinates for point 2
    :param lon3, lat3: The lon, lat coordinates for point 3
    :param a12: The angle between point 1 (lon1, lat1) and point 2 (lon2, lat2) in radians
    :param a13: The angle between point 1 (lon1, lat1) and point 3 (lon3, lat3) in radians
    :param d13: The distance between point 1 (lon1, lat1) and point 3 (lon3, lat3) in km
    """
    if a12 is None:
        a12 = radians(ll_bearing(lon1, lat1, lon2, lat2))
    if a13 is None:
        a13 = radians(ll_bearing(lon1, lat1, lon3, lat3))
    if d13 is None:
        d13 = ll_dist(lon1, lat1, lon3, lat3)
    d13 /= R_EARTH
    # Only keeping the cross track angle saves a division on the following line
    xta = asin(sin(d13) * sin(a13 - a12))
    # The sign factor of this line is a modification of the original formula to distinguish between up/down strike
    # locations
    ata = acos(cos(d13) / cos(xta)) * np.sign(np.cos(a13 - a12))

    # Multiply the angles by radius to get the distance across the earth surface
    return xta * R_EARTH, ata * R_EARTH

def ll_cross_track_dist(
    lon1, lat1, lon2, lat2, lon3, lat3, a12=None, a13=None, d13=None
):
    """
    Returns the distance of point 3 to the nearest point on the great circle line that passes through point 1 and point 2
    If any of a12, a13, d13 are given the calculations for them are skipped
    :param a12: The angle between point 1 (lon1, lat1) and point 2 (lon2, lat2) in radians
    :param a13: The angle between point 1 (lon1, lat1) and point 3 (lon3, lat3) in radians
    :param d13: The distance between point 1 (lon1, lat1) and point 3 (lon3, lat3)
    """
    warn(
        "This function is deprecated in favour of ll_cross_along_track_dist",
        DeprecationWarning,
    )
    return ll_cross_along_track_dist(lon1, lat1, lon2, lat2, lon3, lat3, a12, a13, d13)[
        0
    ]

def angle_diff(b1, b2):
    """
    Return smallest difference (clockwise, -180 -> 180) from b1 to b2.
    """
    r = (b2 - b1) % 360
    if r > 180:
        return r - 360
    return r

def avg_wbearing(angles):
    """
    Return average angle given angles and weightings.
    NB: angles are clockwise from North, not anti-clockwise from East.
    angles: 2d list of (angle, weight)
    """
    x = 0
    y = 0
    for a in angles:
        x += a[1] * sin(radians(a[0]))
        y += a[1] * cos(radians(a[0]))
    q_diff = 0
    if y < 0:
        q_diff = pi
    elif x < 0:
        q_diff = 2 * pi
    return degrees(atan(x / y) + q_diff)

def build_corners(origin, rot, xlen, ylen):
    """
    Return 4 coordinates at corners centered at origin in [longitude, latitude] format.
    Parameters
    ----------
    origin :  a tuple of (lon,lat)
    rot : bearing (in degrees 0~360)
    xlen : width between corner points
    ylen : height between corner points

    Returns
    -------
    (c1, c2, c3, c4) where each c1, c2, c3 and c4 are in tuple (lon,lat)  format
    """
    # amount to shift from middle
    x_shift = xlen / 2.0
    y_shift = ylen / 2.0

    y_len_mid_shift = R_EARTH * asin(sin(y_shift / R_EARTH) / cos(x_shift / R_EARTH))

    top_mid = ll_shift(
        lat=origin[1], lon=origin[0], distance=y_len_mid_shift, bearing=rot
    )[::-1]
    bottom_mid = ll_shift(
        lat=origin[1],
        lon=origin[0],
        distance=y_len_mid_shift,
        bearing=(rot + 180) % 360,
    )[::-1]

    top_mid_bearing = ll_bearing(*top_mid, *origin) + 180 % 360
    bottom_mid_bearing = ll_bearing(*bottom_mid, *origin)

    c2 = ll_shift(
        lat=top_mid[1],
        lon=top_mid[0],
        distance=x_shift,
        bearing=(top_mid_bearing - 90) % 360,
    )[::-1]
    c1 = ll_shift(
        lat=top_mid[1],
        lon=top_mid[0],
        distance=x_shift,
        bearing=(top_mid_bearing + 90) % 360,
    )[::-1]

    c3 = ll_shift(
        lat=bottom_mid[1],
        lon=bottom_mid[0],
        distance=x_shift,
        bearing=(bottom_mid_bearing - 90) % 360,
    )[::-1]
    c4 = ll_shift(
        lat=bottom_mid[1],
        lon=bottom_mid[0],
        distance=x_shift,
        bearing=(bottom_mid_bearing + 90) % 360,
    )[::-1]

    # at this point we have a perfect square (by corner distance)
    # c1 -> c4 == c2 -> c3 (right == left), c1 -> c2 == c3 -> c4 (top == bottom)
    return c1, c2, c3, c4

def path_from_corners(
    corners=None, output="sim.modelpath_hr", min_edge_points=100, close=True
):
    """
    corners: python list (4 by 2) containing (lon, lat) in order
        otherwise take from velocity model
    output: where to store path of (lon, lat) values
    min_edge_points: at least this many points wanted along edges
    """

    # close the box by going back to origin
    if close:
        corners.append(corners[0])

    # until each side has at least wanted number of points
    while len(corners) < 4 * min_edge_points:
        # work backwards, insertions don't change later indexes
        for i in range(len(corners) - 1, 0, -1):
            val = ll_mid(
                corners[i][0], corners[i][1], corners[i - 1][0], corners[i - 1][1]
            )
            corners.insert(i, val)

    # write points the make the path
    if output != None:
        with open(output, "w") as mp:
            for point in corners:
                mp.write("%s %s\n" % (point[0], point[1]))
    else:
        return corners

def wgs_nztm2000x(points):
    """
    Coordinates Transform: between WGS84 lon, lat and NZ Tranverse Mercator 2000
    author: X. Bellagamba, Matlab -> Python by Viktor Polak

    points: series of decimal longitude, latitude or nztm2k x, y
    """
    if type(points).__name__ == "list":
        points = np.array(points)
    points = np.atleast_2d(points)

    if np.max(np.abs(points)) > 180:
        wgs_out = True
    else:
        wgs_out = False
        points = np.radians(points)

    # NZTM2000 definitions (PROJ4 naming)
    # origin
    lon_0 = radians(173.0)
    lat_0 = 0.0
    # false northing, easting (metres)
    y_0 = 10000000
    x_0 = 1600000
    # central meridian scaling factor
    k_0 = 0.9996
    # flattening of the ellipsoid (%)
    f = 1 / 298.257222101
    # semimajor, semiminor ellipsoid radius
    a = 6378137.0
    b = a * (1 - f)
    # eccentricity of the ellipsoid squared
    es = (f + f) - (f * f)

    # conversion specific
    A_0 = 1 - es / 4.0 - 3 * es ** 2 / 64.0 - 5 * es ** 3 / 256.0
    A_2 = 3 / 8.0 * (es + es ** 2 / 4.0 + 15 * es ** 3 / 128.0)
    A_4 = 15 / 256.0 * (es ** 2 + 3 * es ** 3 / 4.0)
    A_6 = 35 * es ** 3 / 3072.0
    m_0 = a * (
        A_0 * lat_0 - A_2 * sin(2 * lat_0) + A_4 * sin(4 * lat_0) - A_6 * sin(6 * lat_0)
    )
    n = (a - b) / (a + b)
    G = a * (1 - n) * (1 - n ** 2) * radians(1 + 9 * n ** 2 / 4.0 + 225 * n ** 4 / 64.0)

    ###
    ### process all points
    ###
    if wgs_out:
        N_prime = points[:, 1] - y_0
        E_prime = points[:, 0] - x_0
        m_prime = m_0 + N_prime / k_0
        sigma = np.radians(m_prime) * 1 / G
        phi_prime = (
            sigma
            + (3 * n / 2.0 - 27 * n ** 3 / 32.0) * np.sin(2 * sigma)
            + (21 * n ** 2 / 16.0 - 55 * n ** 4 / 32.0) * np.sin(4 * sigma)
            + (151 * n ** 3 / 96.0) * np.sin(6 * sigma)
            + (1097 * n ** 4 / 512.0) * np.sin(8 * sigma)
        )
        y_factors = phi_prime
    else:
        y_factors = points[:, 1]
        m = a * (
            A_0 * y_factors
            - A_2 * np.sin(2 * y_factors)
            + A_4 * np.sin(4 * y_factors)
            - A_6 * np.sin(6 * y_factors)
        )
        omega = points[:, 0] - lon_0

    # ellipsoid radius in the prime vertical
    nu = a / np.sqrt(1 - es * np.sin(y_factors) ** 2)
    # projection parameters
    rho = (a * (1 - es)) / ((1 - es * np.sin(y_factors) ** 2) ** 1.5)
    psi = nu / rho
    t = np.tan(y_factors)
    if wgs_out:
        rs = rho * nu * k_0 ** 2
        x = E_prime / (k_0 * nu)

        # terms for the north coordinates
        T1_N = t / (k_0 * rho) * E_prime * x / 2.0
        T2_N = (
            t
            / (k_0 * rho)
            * E_prime
            * x ** 3
            / 24.0
            * (-4 * psi ** 2 + 9 * psi * (1 - t ** 2) + 12 * t ** 2)
        )
        T3_N = (
            t
            / (k_0 * rho)
            * E_prime
            * x ** 5
            / 720.0
            * (
                8 * psi ** 4 * (11 - 24 * t ** 2)
                - 12 * psi ** 3 * (21 - 7 * t ** 2)
                + 15 * psi ** 2 * (15 - 98 * t ** 2 + 15 * t ** 4)
                + 180 * psi * (5 * t ** 2 - 3 * t ** 4)
                + 360 * t ** 4
            )
        )
        T4_N = (
            t
            / (k_0 * rho)
            * E_prime
            * x ** 7
            / 40320.0
            * (1385 + 3633 * t ** 2 + 4095 * t ** 4 + 1575 * t ** 6)
        )
        # north coordinates
        y = y_factors - T1_N + T2_N - T3_N + T4_N

        # terms for the east coordinates
        T1_E = x * 1 / np.cos(y_factors)
        T2_E = x ** 3 * 1 / np.cos(y_factors) / 6 * (psi + 2 * t ** 2)
        T3_E = (
            x ** 5
            * 1
            / np.cos(y_factors)
            / 120
            * (
                -4 * psi ** 3 * (1 - 6 * t ** 2)
                + psi ** 2 * (9 - 68 * t ** 2)
                + 72 * psi * t ** 2
                + 24 * t ** 4
            )
        )
        T4_E = (
            x ** 7
            * 1
            / np.cos(y_factors)
            / 5040
            * (61 + 662 * t ** 2 + 1320 * t ** 4 + 720 * t ** 6)
        )
        # east coordinates
        x = lon_0 + T1_E - T2_E + T3_E - T4_E

        return np.dstack((np.degrees(x), np.degrees(y)))[0]

    # terms for the north coordinates
    T1_N = (omega ** 2 / 2.0) * nu * np.sin(y_factors) * np.cos(y_factors)
    T2_N = (
        (omega ** 4 / 24.0)
        * nu
        * np.sin(y_factors)
        * np.cos(y_factors) ** 3
        * (4 * psi ** 2 + psi - t ** 2)
    )
    T3_N = (
        (omega ** 6 / 720.0)
        * nu
        * np.sin(y_factors)
        * np.cos(y_factors) ** 5
        * (
            8 * psi ** 4 * (11 - 24 * t ** 2)
            - 28 * psi ** 3 * (1 - 6 * t ** 2)
            + psi ** 2 * (1 - 32 * t ** 2)
            - psi * (2 * t ** 2)
            + t ** 4
        )
    )
    T4_N = (
        (omega ** 8 / 40320.0)
        * nu
        * np.sin(y_factors)
        * np.cos(y_factors) ** 7
        * (1385 - 3111 * t ** 2 + 543 * t ** 4 - t ** 6)
    )
    # north coordinates
    lat = y_0 + k_0 * (m - m_0 + T1_N + T2_N + T3_N + T4_N)

    # terms for the east coordinates
    T1_E = (omega ** 2 / 6.0) * np.cos(y_factors) ** 2 * (psi - t ** 2)
    T2_E = (
        (omega ** 4 / 120.0)
        * np.cos(y_factors) ** 4
        * (
            4 * psi ** 3 * (1 - 6 * t ** 2)
            + psi ** 2 * (1 + 8 * t ** 2)
            - psi * 2 * t ** 2
            + t ** 4
        )
    )
    T3_E = (
        (omega ** 6 / 5040.0)
        * np.cos(y_factors) ** 6
        * (61 - 479 * t ** 2 + 179 * t ** 4 - t ** 6)
    )
    # east coordinates
    lon = x_0 + k_0 * nu * omega * np.cos(y_factors) * (1 + T1_E + T2_E + T3_E)

    return np.dstack((lon, lat))[0]

def compute_intermediate_latitudes(lon_lat1, lon_lat2, lon_in):
    """
    Calculates the latitudes of points along the shortest path between the points lon_lat1 and lon_lat2, taking the
    shortest path on the sphere, using great circle calculations.
    Note that this is different from the path obtained by interpolating the latitude and longitude between the two points.
    Equation taken from http://www.edwilliams.org/avform.htm#Int
    :param lon_lat1: A tuple containing the longitude and latitude of the start point
    :param lon_lat2: A tuple containing the longitude and latitude of the end point
    :param lon_in: A float or iterable of floats compliant with numpy of the longitude(s) to have latitudes calculated for
    :return: A float or iterable of floats which represent the latitude(s) of the value(s) given in lon_in
    """
    conversion_factor = np.pi / 180
    lon1, lat1 = lon_lat1
    lon2, lat2 = lon_lat2
    lat1 *= conversion_factor
    lon1 *= conversion_factor
    lat2 *= conversion_factor
    lon2 *= conversion_factor
    lon = lon_in * conversion_factor
    if lon1 == lon2:
        return np.linspace(lat1, lat2, len(lon_in)) / conversion_factor
    return (
        np.arctan(
            (
                np.sin(lat1) * np.cos(lat2) * np.sin(lon - lon2)
                - np.sin(lat2) * np.cos(lat1) * np.sin(lon - lon1)
            )
            / (np.cos(lat1) * np.cos(lat2) * np.sin(lon1 - lon2))
        )
    ) / conversion_factor

max_stars_repo_name: QuakeKorea/qcore

max_issues_repo_name: QuakeKorea/qcore

max_forks_repo_name: QuakeKorea/qcore


text:
#######################################################################
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
# Permission given to modify the code as long as you keep this        #
# declaration at the top                                              #
#######################################################################

from deep_rl import *
import numpy as np

from arguments import get_args

# DQN
def dqn_feature(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    if config.control == False:
        config.fixed_policy = np.array([np.ones(13)*-1,
        [-1,0,2,2,2,2,-1,1,1,1,1,1,-1],
        [-1,0,0,0,0,0,-1,1,1,1,1,1,-1],
        [-1,0,0,0,0,0, 2,2,2,2,2,2,-1],
        [-1,0,0,0,0,0,-1,0,0,0,0,0,-1],
        [-1,0,0,0,0,0,-1,0,0,0,0,0,-1],
        [-1,-1,0,-1,-1,-1,-1,0,0,0,0,0,-1],
        [-1,3,0,2,2,2,-1,-1,-1,0,-1,-1,-1],
        [-1,0,0,0,0,0,-1, 1, 3,0, 2, 2,-1],
        [-1,0,0,0,0,0,-1, 1, 3,0, 2, 2,-1],
        [-1,0,0,0,0,0, 2, 2, 2,2, 2, 2,-1],
        [-1,0,0,0,0,0,-1, 0, 0,0, 0, 0,-1],
        np.ones(13)*-1])

        config.obs_map = np.array([np.ones(13)*-1,
        [-1,0,1,2,3,4,-1,5,6,7,8,9,-1],
        [-1,10,11,12,13,14,-1,15,16,17,18,19,-1],
        [-1,20,21,22,23,24,25,26,27,28,29,30,-1],
        [-1,31,32,33,34,35,-1,36,37,38,39,40,-1],
        [-1,41,42,43,44,45,-1,46,47,48,49,50,-1],
        [-1,-1,51,-1,-1,-1,-1,52,53,54,55,56,-1],
        [-1,57,58,59,60,61,-1,-1,-1,62,-1,-1,-1],
        [-1,63,64,65,66,67,-1,68,69,70,71,72,-1],
        [-1,73,74,75,76,77,-1,78,79,80,81,82,-1],
        [-1,83,84,85,86,87,88,89,90,91,92,93,-1],
        [-1,94,95,96,97,98,-1,99,100,101,102,103,-1],
        np.ones(13)*-1])

    config.task_fn = lambda: Task(config.game, seed = config.seed, goal_location = config.goal_location)
    config.eval_env = Task(config.game, seed = config.seed, goal_location = config.goal_location)

    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.01)
    config.network_fn = lambda: VanillaNet(config.action_dim, FCBody(config.state_dim))
    # config.network_fn = lambda: DuelingNet(config.action_dim, FCBody(config.state_dim))
    config.replay_fn = lambda: Replay(memory_size=int(1e4), batch_size=10)
    #config.replay_fn = lambda: AsyncReplay(memory_size=int(1e5), batch_size=30)

    if config.control == True:
        config.random_action_prob = LinearSchedule(1.0, 0.1, 1e6)
        config.max_steps = config.max_steps
    elif config.control == False:
        config.random_action_prob = LinearSchedule(0.1, 0.1, 1e6)
        config.max_steps = 200000
    config.discount = 0.99
    config.target_network_update_freq = 200
    config.exploration_steps = 10000
    # config.double_q = True
    config.double_q = False
    config.sgd_update_frequency = 4
    config.gradient_clip = 5
    config.eval_interval = int(5e3)
    config.async_actor = False
    run_steps(DQNAgent(config))

def dqn_pixel(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()

    config.optimizer_fn = lambda params: torch.optim.RMSprop(
        params, lr=0.00025, alpha=0.95, eps=0.01, centered=True)
    config.network_fn = lambda: VanillaNet(config.action_dim, NatureConvBody(in_channels=config.history_length))
    # config.network_fn = lambda: DuelingNet(config.action_dim, NatureConvBody(in_channels=config.history_length))
    config.random_action_prob = LinearSchedule(1.0, 0.01, 1e6)

    # config.replay_fn = lambda: Replay(memory_size=int(1e6), batch_size=32)
    config.replay_fn = lambda: AsyncReplay(memory_size=int(1e6), batch_size=32)

    config.batch_size = 32
    config.state_normalizer = ImageNormalizer()
    config.reward_normalizer = SignNormalizer()
    config.discount = 0.99
    config.target_network_update_freq = 10000
    config.exploration_steps = 50000
    config.sgd_update_frequency = 4
    config.gradient_clip = 5
    config.history_length = 4
    # config.double_q = True
    config.double_q = False
    config.max_steps = int(2e7)
    run_steps(DQNAgent(config))

# QR DQN
def quantile_regression_dqn_feature(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
    config.network_fn = lambda: QuantileNet(config.action_dim, config.num_quantiles, FCBody(config.state_dim))

    # config.replay_fn = lambda: Replay(memory_size=int(1e4), batch_size=10)
    config.replay_fn = lambda: AsyncReplay(memory_size=int(1e4), batch_size=10)

    config.random_action_prob = LinearSchedule(1.0, 0.1, 1e4)
    config.discount = 0.99
    config.target_network_update_freq = 200
    config.exploration_steps = 100
    config.num_quantiles = 20
    config.gradient_clip = 5
    config.sgd_update_frequency = 4
    config.eval_interval = int(5e3)
    config.max_steps = 1e5
    run_steps(QuantileRegressionDQNAgent(config))

def quantile_regression_dqn_pixel(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()

    config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.00005, eps=0.01 / 32)
    config.network_fn = lambda: QuantileNet(config.action_dim, config.num_quantiles, NatureConvBody())
    config.random_action_prob = LinearSchedule(1.0, 0.01, 1e6)

    # config.replay_fn = lambda: Replay(memory_size=int(1e6), batch_size=32)
    config.replay_fn = lambda: AsyncReplay(memory_size=int(1e6), batch_size=32)

    config.state_normalizer = ImageNormalizer()
    config.reward_normalizer = SignNormalizer()
    config.discount = 0.99
    config.target_network_update_freq = 10000
    config.exploration_steps = 50000
    config.sgd_update_frequency = 4
    config.gradient_clip = 5
    config.num_quantiles = 200
    config.max_steps = int(2e7)
    run_steps(QuantileRegressionDQNAgent(config))

# C51
def categorical_dqn_feature(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
    config.network_fn = lambda: CategoricalNet(config.action_dim, config.categorical_n_atoms, FCBody(config.state_dim))
    config.random_action_prob = LinearSchedule(1.0, 0.1, 1e4)

    # config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10)
    config.replay_fn = lambda: AsyncReplay(memory_size=10000, batch_size=10)

    config.discount = 0.99
    config.target_network_update_freq = 200
    config.exploration_steps = 100
    config.categorical_v_max = 100
    config.categorical_v_min = -100
    config.categorical_n_atoms = 50
    config.gradient_clip = 5
    config.sgd_update_frequency = 4

    config.eval_interval = int(5e3)
    config.max_steps = 1e5
    run_steps(CategoricalDQNAgent(config))

def categorical_dqn_pixel(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()
    config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.00025, eps=0.01 / 32)
    config.network_fn = lambda: CategoricalNet(config.action_dim, config.categorical_n_atoms, NatureConvBody())
    config.random_action_prob = LinearSchedule(1.0, 0.01, 1e6)

    # config.replay_fn = lambda: Replay(memory_size=int(1e6), batch_size=32)
    config.replay_fn = lambda: AsyncReplay(memory_size=int(1e6), batch_size=32)

    config.discount = 0.99
    config.state_normalizer = ImageNormalizer()
    config.reward_normalizer = SignNormalizer()
    config.target_network_update_freq = 10000
    config.exploration_steps = 50000
    config.categorical_v_max = 10
    config.categorical_v_min = -10
    config.categorical_n_atoms = 51
    config.sgd_update_frequency = 4
    config.gradient_clip = 0.5
    config.max_steps = int(2e7)
    run_steps(CategoricalDQNAgent(config))

# A2C
def a2c_feature(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.num_workers = 5
    config.task_fn = lambda: Task(config.game, num_envs=config.num_workers)
    config.eval_env = Task(config.game)
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
    config.network_fn = lambda: CategoricalActorCriticNet(
        config.state_dim, config.action_dim, FCBody(config.state_dim, gate=F.tanh))
    config.discount = 0.99
    config.use_gae = True
    config.gae_tau = 0.95
    config.entropy_weight = 0.01
    config.rollout_length = 5
    config.gradient_clip = 0.5
    run_steps(A2CAgent(config))

def a2c_pixel(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.num_workers = 16
    config.task_fn = lambda: Task(config.game, num_envs=config.num_workers)
    config.eval_env = Task(config.game)
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=1e-4, alpha=0.99, eps=1e-5)
    config.network_fn = lambda: CategoricalActorCriticNet(config.state_dim, config.action_dim, NatureConvBody())
    config.state_normalizer = ImageNormalizer()
    config.reward_normalizer = SignNormalizer()
    config.discount = 0.99
    config.use_gae = True
    config.gae_tau = 1.0
    config.entropy_weight = 0.01
    config.rollout_length = 5
    config.gradient_clip = 5
    config.max_steps = int(2e7)
    run_steps(A2CAgent(config))

def a2c_continuous(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.num_workers = 16
    config.task_fn = lambda: Task(config.game, num_envs=config.num_workers)
    config.eval_env = Task(config.game)
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.0007)
    config.network_fn = lambda: GaussianActorCriticNet(
        config.state_dim, config.action_dim,
        actor_body=FCBody(config.state_dim), critic_body=FCBody(config.state_dim))
    config.discount = 0.99
    config.use_gae = True
    config.gae_tau = 1.0
    config.entropy_weight = 0.01
    config.rollout_length = 5
    config.gradient_clip = 5
    config.max_steps = int(2e7)
    run_steps(A2CAgent(config))

# N-Step DQN
def n_step_dqn_feature(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game, num_envs=config.num_workers)
    config.eval_env = Task(config.game)
    config.num_workers = 5
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
    config.network_fn = lambda: VanillaNet(config.action_dim, FCBody(config.state_dim))
    config.random_action_prob = LinearSchedule(1.0, 0.1, 1e4)
    config.discount = 0.99
    config.target_network_update_freq = 200
    config.rollout_length = 5
    config.gradient_clip = 5
    run_steps(NStepDQNAgent(config))

def n_step_dqn_pixel(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game, num_envs=config.num_workers)
    config.eval_env = Task(config.game)
    config.num_workers = 16
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=1e-4, alpha=0.99, eps=1e-5)
    config.network_fn = lambda: VanillaNet(config.action_dim, NatureConvBody())
    config.random_action_prob = LinearSchedule(1.0, 0.05, 1e6)
    config.state_normalizer = ImageNormalizer()
    config.reward_normalizer = SignNormalizer()
    config.discount = 0.99
    config.target_network_update_freq = 10000
    config.rollout_length = 5
    config.gradient_clip = 5
    config.max_steps = int(2e7)
    run_steps(NStepDQNAgent(config))

# Option-Critic
def option_critic_feature(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.num_workers = 5
    config.task_fn = lambda: Task(config.game, num_envs=config.num_workers)
    config.eval_env = Task(config.game)
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
    config.network_fn = lambda: OptionCriticNet(FCBody(config.state_dim), config.action_dim, num_options=2)
    config.random_option_prob = LinearSchedule(1.0, 0.1, 1e4)
    config.discount = 0.99
    config.target_network_update_freq = 200
    config.rollout_length = 5
    config.termination_regularizer = 0.01
    config.entropy_weight = 0.01
    config.gradient_clip = 5
    run_steps(OptionCriticAgent(config))

def option_critic_pixel(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game, num_envs=config.num_workers)
    config.eval_env = Task(config.game)
    config.num_workers = 16
    config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=1e-4, alpha=0.99, eps=1e-5)
    config.network_fn = lambda: OptionCriticNet(NatureConvBody(), config.action_dim, num_options=4)
    config.random_option_prob = LinearSchedule(0.1)
    config.state_normalizer = ImageNormalizer()
    config.reward_normalizer = SignNormalizer()
    config.discount = 0.99
    config.target_network_update_freq = 10000
    config.rollout_length = 5
    config.gradient_clip = 5
    config.max_steps = int(2e7)
    config.entropy_weight = 0.01
    config.termination_regularizer = 0.01
    run_steps(OptionCriticAgent(config))

# PPO
def ppo_continuous(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()

    config.network_fn = lambda: GaussianActorCriticNet(
        config.state_dim, config.action_dim, actor_body=FCBody(config.state_dim, gate=torch.tanh),
        critic_body=FCBody(config.state_dim, gate=torch.tanh))
    config.actor_opt_fn = lambda params: torch.optim.Adam(params, 3e-4)
    config.critic_opt_fn = lambda params: torch.optim.Adam(params, 1e-3)
    config.discount = 0.99
    config.use_gae = True
    config.gae_tau = 0.95
    config.gradient_clip = 0.5
    config.rollout_length = 2048
    config.optimization_epochs = 10
    config.mini_batch_size = 64
    config.ppo_ratio_clip = 0.2
    config.log_interval = 2048
    config.max_steps = 3e6
    config.target_kl = 0.01
    config.state_normalizer = MeanStdNormalizer()
    run_steps(PPOAgent(config))

# DDPG
def ddpg_continuous(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()
    config.max_steps = int(1e6)
    config.eval_interval = int(1e4)
    config.eval_episodes = 20

    config.network_fn = lambda: DeterministicActorCriticNet(
        config.state_dim, config.action_dim,
        actor_body=FCBody(config.state_dim, (400, 300), gate=F.relu),
        critic_body=TwoLayerFCBodyWithAction(
            config.state_dim, config.action_dim, (400, 300), gate=F.relu),
        actor_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-4),
        critic_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-3))

    config.replay_fn = lambda: Replay(memory_size=int(1e6), batch_size=64)
    config.discount = 0.99
    config.random_process_fn = lambda: OrnsteinUhlenbeckProcess(
        size=(config.action_dim,), std=LinearSchedule(0.2))
    config.warm_up = int(1e4)
    config.target_network_mix = 1e-3
    run_steps(DDPGAgent(config))

# TD3
def td3_continuous(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()
    config.max_steps = int(1e6)
    config.eval_interval = int(1e4)
    config.eval_episodes = 20

    config.network_fn = lambda: TD3Net(
        config.action_dim,
        actor_body_fn=lambda: FCBody(config.state_dim, (400, 300), gate=F.relu),
        critic_body_fn=lambda: FCBody(
            config.state_dim+config.action_dim, (400, 300), gate=F.relu),
        actor_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-3),
        critic_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-3))

    config.replay_fn = lambda: Replay(memory_size=int(1e6), batch_size=100)
    config.discount = 0.99
    config.random_process_fn = lambda: GaussianProcess(
        size=(config.action_dim,), std=LinearSchedule(0.1))
    config.td3_noise = 0.2
    config.td3_noise_clip = 0.5
    config.td3_delay = 2
    config.warm_up = int(1e4)
    config.target_network_mix = 5e-3
    run_steps(TD3Agent(config))

if __name__ == '__main__':
    args = get_args()
    print(args.transfer, args.seed)
    mkdir('log')
    mkdir('tf_log')
    set_one_thread()
    seed = args.seed
    random_seed(seed=seed)
    select_device(-1)
    # select_device(0)

    num_runs = 15   # DON'T CHANGE THIS - NEED TO KEEP THIS SAME TO ENSURE REPRODUCIBILITY. IF THIS IS CHANGED, DIFFERENT GOALS WILL BE CHOSEN
    goal_locations = np.random.randint(104,size=(2,num_runs))
    blocked_hallways = np.random.randint(4,size=(num_runs))

    iteration = args.run_index-1
    original_goal_location = goal_locations[0][iteration]
    transfer_goal_location = goal_locations[1][iteration]

    algo = args.algo
    game = "FourRooms-v1"
    goal_location = original_goal_location
    new_goal_location = transfer_goal_location
    num_eigvals = args.num_eigvals
    transfer = args.transfer
    stochastic = args.stochastic
    if algo == 'mrpvf':
        decomp_type = args.decomp_type
    elif algo == 'pvf':
        decomp_type = ""

    # for policy evaluation
    if args.control == False:
        goal_location = 0
        tag = "_"+game+"_"+algo+"_"+decomp_type+"_seed="+str(seed)+"_run_iteration="+str(iteration)+"_num_eigvals="+str(num_eigvals)+"_"
        max_steps = 1500000
        dqn_feature(game=game, decomp_type = decomp_type, seed = seed, goal_location = goal_location, tag = tag, algo = algo, num_eigvals = num_eigvals, transfer = transfer, max_steps = max_steps, control = args.control)

    # for control
    elif args.control == True:
        if stochastic == False:
            tag = "_"+game+"_"+algo+"_"+decomp_type+"_seed="+str(seed)+"_run_iteration="+str(iteration)+"_num_eigvals="+str(num_eigvals)+"_"

            if transfer == False:
                max_steps = 1500000
                dqn_feature(game=game, decomp_type = decomp_type, seed = seed, goal_location = goal_location, tag = tag, algo = algo, num_eigvals = num_eigvals, transfer = transfer, max_steps = max_steps, control = args.control)

            elif transfer == True:
                max_steps = 800000
                #load_weights_location = "log/"+algo+"_"+decomp_type+"_n="+str(num_eigvals)+"/"+"_"+game+"_"+algo+"_"+decomp_type+"_seed="+str(seed)+"_run_iteration="+str(iteration)+"_num_eigvals="+str(num_eigvals)+"__ntimesteps=1200000.pt"
                load_weights_location = "log/"+algo+"_"+decomp_type+"_n="+str(num_eigvals)+"_weight=2/"+"_"+game+"_"+algo+"_"+decomp_type+"_seed="+str(seed)+"_run_iteration="+str(iteration)+"_num_eigvals="+str(num_eigvals)+"__ntimesteps=1200000.pt"
                print(load_weights_location)
                tag = "_transfer_"+game+"_"+algo+"_"+decomp_type+"_seed="+str(seed)+"_run_iteration="+str(iteration)+"_num_eigvals="+str(num_eigvals)+"_"
                dqn_feature(game=game, decomp_type = decomp_type, seed = seed, goal_location = new_goal_location, tag = tag, algo = algo, num_eigvals = num_eigvals, transfer = transfer, max_steps = max_steps, load_weights_location = load_weights_location, control = args.control)

        elif stochastic == True:
            tag = "_stochastic_"+game+"_"+algo+"_"+decomp_type+"_seed="+str(seed)+"_run_iteration="+str(iteration)+"_num_eigvals="+str(num_eigvals)+"_"

            if transfer == False:
                max_steps = 2000000
                dqn_feature(game=game, decomp_type = decomp_type, seed = seed, goal_location = goal_location, tag = tag, algo = algo, num_eigvals = num_eigvals, transfer = transfer, max_steps = max_steps, control = args.control)

            elif transfer == True:
                max_steps = 800000
                load_weights_location = "log/stochastic_"+algo+"_"+decomp_type+"_n="+str(num_eigvals)+"/"+"_stochastic_"+game+"_"+algo+"_"+decomp_type+"_seed="+str(seed)+"_run_iteration="+str(iteration)+"_num_eigvals="+str(num_eigvals)+"__ntimesteps=1200000.pt"
                print(load_weights_location)
                tag = "_stochastic_transfer_"+game+"_"+algo+"_"+decomp_type+"_seed="+str(seed)+"_run_iteration="+str(iteration)+"_num_eigvals="+str(num_eigvals)+"_"
                dqn_feature(game=game, decomp_type = decomp_type, seed = seed, goal_location = new_goal_location, tag = tag, algo = algo, num_eigvals = num_eigvals, transfer = transfer, max_steps = max_steps, load_weights_location = load_weights_location, control = args.control)

    # quantile_regression_dqn_feature(game=game)
    # categorical_dqn_feature(game=game)
    # a2c_feature(game=game)
    # n_step_dqn_feature(game=game)
    # option_critic_feature(game=game)

    # game = 'Hopper-v2'
    # a2c_continuous(game=game)
    # ppo_continuous(game=game)
    # ddpg_continuous(game=game)
    # td3_continuous(game=game)

    # game = 'BreakoutNoFrameskip-v4'
    # dqn_pixel(game=game)
    # quantile_regression_dqn_pixel(game=game)
    # categorical_dqn_pixel(game=game)
    # a2c_pixel(game=game)
    # n_step_dqn_pixel(game=game)
    # option_critic_pixel(game=game)


text:
import pandas as p
import numpy as np
from file_setup_helper import FileSetupHelper as fsh
from preprocess_data import PreprocessData as pd
from model_export import ModelExport as me
import sys
from sklearn.linear_model import Ridge

def main():
    #call for file download with given date
    file_name = fsh(sys.argv[1], 1460, 0).download_csv()

    #aquire features and target data filenames
    features_file_name, target_file_name = pd(file_name).preprocess(True, None)

    #init model, and open the features and target file
    #(drop column added by pandas and bypass 29.02 potential error)
    model = Ridge(alpha=1, fit_intercept=True)
    X_train = p.read_csv(features_file_name).drop(['Unnamed: 0', 'Date_29.02'], axis=1)
    y_train = p.read_csv(target_file_name).drop(['Unnamed: 0'], axis=1)

    #export model with the given datasets and model
    me(model, X_train, y_train).fit_and_export(sys.argv[2])

if __name__ == '__main__':
    main()

max_stars_repo_name: Slavkata/Forecast-Report

max_issues_repo_name: Slavkata/Forecast-Report

max_forks_repo_name: Slavkata/Forecast-Report


text:
# script used to create the data set gliderflight_data.npy
import numpy as np

import dbdreader
try:
    # use my fast private gsw implementation if available
    from fast_gsw import rho as gsw_rho
except ImportError:
    # or fall back on the community one, installable via pip, for example.
    import gsw

    def gsw_rho(C, T, P, lon, lat):
        SP = gsw.SP_from_C(C, T, P)
        # This particular data set was collected in the Baltic.
        SA = gsw.SA_from_SP_Baltic(SP, lon, lat)
        # in-situ density
        rho = gsw.density.rho_t_exact(SA, T, P)
        return rho

dbd = dbdreader.MultiDBD(pattern="/home/lucas/gliderdata/subex2016/hd/comet-2016-174-05-00?.[de]bd")

_tmp = dbd.get_sync("sci_ctd41cp_timestamp", "sci_water_pressure m_pitch m_ballast_pumped sci_water_cond sci_water_temp".split())
_, tctd, P, pitch, buoyancy_drive, C, T = _tmp.compress(_tmp[1]>0, axis=1)

tilt_correction_factor=0.86
pitch_offset=-0.004,
pitch = pitch*tilt_correction_factor + pitch_offset
density = gsw_rho(C*10, T, P*10, 15, 54)
U_adcp = np.interp(tctd, *np.load("../data/U_kalman_datasetIII.npy"))
U_adcp[np.where(np.logical_or(P*10<12, pitch>0))[0]] = np.nan
np.save("../data/gliderflight_data.npy", (tctd, P, pitch, buoyancy_drive, density, U_adcp))

max_stars_repo_name: smerckel/gliderflight

max_issues_repo_name: smerckel/gliderflight

max_forks_repo_name: smerckel/gliderflight


text:
# Data here:  https://www.kaggle.com/c/instant-gratification/data
# Original source here: https://www.kaggle.com/prashantkikani/ig-pca-nusvc-knn-lr-stack
import argparse
import pickle
import time

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

from instant_utils import *
from willump.evaluation.willump_executor import willump_execute

base_path = "tests/test_resources/instant_gratification/"

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--cascades", action="store_true", help="Cascades?")
parser.add_argument("-k", "--top_k", type=int, help="Top-K to return", required=True)
parser.add_argument("-d", "--disable", help="Disable Willump", action="store_true")
args = parser.parse_args()

if args.cascades:
    cascades = pickle.load(open(base_path + "cascades.pk", "rb"))
else:
    cascades = None
top_K = args.top_k

@willump_execute(disable=args.disable, eval_cascades=cascades, top_k=top_K)
def predict_stacked_model(X, model, clf_svnu, clf_knn, clf_lr, clf_mlp, clf_svc):
    pred_svnu = model_prediction(X, clf_svnu)
    # pred_knn = model_prediction(X, clf_knn)
    pred_lr = model_prediction(X, clf_lr)
    pred_mlp = model_prediction(X, clf_mlp)
    pred_svc = model_prediction(X, clf_svc)
    combined_pred = np.hstack([pred_svnu, pred_lr, pred_mlp, pred_svc])
    preds = willump_predict_proba_function(model, combined_pred)
    return preds

if __name__ == "__main__":
    data = pd.read_csv(base_path + "train.csv")
    _, valid_data = train_test_split(data, test_size=0.1, random_state=42)
    valid_data = valid_data[valid_data['wheezy-copper-turtle-magic'] < NUM_PARTITIONS].reset_index(drop=True)
    print("Validation Data Length: %d" % len(valid_data))
    valid_y = valid_data.pop('target')
    partition_column = valid_data.pop('wheezy-copper-turtle-magic').values.reshape(-1, 1)
    del data

    cols = [c for c in valid_data.columns if c not in ['id', 'wheezy-copper-turtle-magic']]

    pca_scaler, standard_scaler = pickle.load(open(base_path + "scaler.pk", "rb"))

    valid_data = standard_scaler.transform(pca_scaler.transform(valid_data[cols]))
    valid_data = np.append(valid_data, partition_column, 1)

    clf_svnu, clf_knn, clf_lr, clf_mlp, clf_svc = pickle.load(open(base_path + "clf.pk", "rb"))

    model = pickle.load(open(base_path + "model.pk", "rb"))

    mini_data = valid_data[:3]
    orig_preds = predict_stacked_model(valid_data, model, clf_svnu, clf_knn, clf_lr, clf_mlp, clf_svc)
    predict_stacked_model(mini_data, model, clf_svnu, clf_knn, clf_lr, clf_mlp, clf_svc)

    t0 = time.time()
    preds = predict_stacked_model(valid_data, model, clf_svnu, clf_knn, clf_lr, clf_mlp, clf_svc)
    time_elapsed = time.time() - t0

    orig_model_top_k_idx = np.argsort(orig_preds)[-1 * top_K:]
    actual_model_top_k_idx = np.argsort(preds)[-1 * top_K:]
    precision = len(np.intersect1d(orig_model_top_k_idx, actual_model_top_k_idx)) / top_K

    orig_model_sum = sum(orig_preds[orig_model_top_k_idx])
    actual_model_sum = sum(preds[actual_model_top_k_idx])
    set_size = len(valid_data)
    print("Title Processing Time %fs Num Rows %d Throughput %f rows/sec" %
          (time_elapsed, set_size, set_size / time_elapsed))
    print("Precision: %f Orig Model Sum: %f Actual Model Sum: %f" % (precision, orig_model_sum, actual_model_sum))

max_stars_repo_name: stanford-futuredata/Willump

max_issues_repo_name: stanford-futuredata/Willump

max_forks_repo_name: stanford-futuredata/Willump


text:
# -*- coding:utf-8 -*-
# Author:Richard Fang

import time
import numpy as np
import torch

def soft_nms_pytorch(dets, box_scores, sigma=0.5, thresh=0.001, cuda=0):
    """
    Build a pytorch implement of Soft NMS algorithm.
    # Augments
        dets:        boxes coordinate tensor (format:[y1, x1, y2, x2])
        box_scores:  box score tensors
        sigma:       variance of Gaussian function
        thresh:      score thresh
        cuda:        CUDA flag
    # Return
        the index of the selected boxes
    """

    # Indexes concatenate boxes with the last column
    N = dets.shape[0]
    if cuda:
        indexes = torch.arange(0, N, dtype=torch.float).cuda().view(N, 1)
    else:
        indexes = torch.arange(0, N, dtype=torch.float).view(N, 1)
    dets = torch.cat((dets, indexes), dim=1)

    # The order of boxes coordinate is [y1,x1,y2,x2]
    y1 = dets[:, 0]
    x1 = dets[:, 1]
    y2 = dets[:, 2]
    x2 = dets[:, 3]
    scores = box_scores
    areas = (x2 - x1 + 1) * (y2 - y1 + 1)

    for i in range(N):
        # intermediate parameters for later parameters exchange
        tscore = scores[i].clone()
        pos = i + 1

        if i != N - 1:
            maxscore, maxpos = torch.max(scores[pos:], dim=0)
            if tscore < maxscore:
                dets[i], dets[maxpos.item() + i + 1] = dets[maxpos.item() + i + 1].clone(), dets[i].clone()
                scores[i], scores[maxpos.item() + i + 1] = scores[maxpos.item() + i + 1].clone(), scores[i].clone()
                areas[i], areas[maxpos + i + 1] = areas[maxpos + i + 1].clone(), areas[i].clone()

        # IoU calculate
        yy1 = np.maximum(dets[i, 0].to("cpu").numpy(), dets[pos:, 0].to("cpu").numpy())
        xx1 = np.maximum(dets[i, 1].to("cpu").numpy(), dets[pos:, 1].to("cpu").numpy())
        yy2 = np.minimum(dets[i, 2].to("cpu").numpy(), dets[pos:, 2].to("cpu").numpy())
        xx2 = np.minimum(dets[i, 3].to("cpu").numpy(), dets[pos:, 3].to("cpu").numpy())

        w = np.maximum(0.0, xx2 - xx1 + 1)
        h = np.maximum(0.0, yy2 - yy1 + 1)
        inter = torch.tensor(w * h).cuda() if cuda else torch.tensor(w * h)
        ovr = torch.div(inter, (areas[i] + areas[pos:] - inter))

        # Gaussian decay
        weight = torch.exp(-(ovr * ovr) / sigma)
        scores[pos:] = weight * scores[pos:]

    # select the boxes and keep the corresponding indexes
    keep = dets[:, 4][scores > thresh].int()

    return keep

def speed():
    boxes = 1000 * torch.rand((1000, 100, 4), dtype=torch.float)
    boxscores = torch.rand((1000, 100), dtype=torch.float)

    # cuda flag
    cuda = 1 if torch.cuda.is_available() else 0
    if cuda:
        boxes = boxes.cuda()
        boxscores = boxscores.cuda()

    start = time.time()
    for i in range(1000):
        soft_nms_pytorch(boxes[i], boxscores[i], cuda=cuda)
    end = time.time()
    print("Average run time: %f ms" % (end - start))

def test():
    # boxes and boxscores
    boxes = torch.tensor([[200, 200, 400, 400],
                          [220, 220, 420, 420],
                          [200, 240, 400, 440],
                          [240, 200, 440, 400],
                          [1, 1, 2, 2]], dtype=torch.float)
    boxscores = torch.tensor([0.8, 0.7, 0.6, 0.5, 0.9], dtype=torch.float)

    # cuda flag
    cuda = 1 if torch.cuda.is_available() else 0
    if cuda:
        boxes = boxes.cuda()
        boxscores = boxscores.cuda()

    print(soft_nms_pytorch(boxes, boxscores, cuda=cuda))

if __name__ == '__main__':
    test()
    # speed()

max_issues_repo_name: LY-Road/Mask-Face-Recognition

max_forks_repo_name: LY-Road/Mask-Face-Recognition


text:
from cycle_gan import UnPairedDataset
from skimage.io import imsave
import numpy as np
import os

def test_dataset():
    data_paths = []

    path_a = os.path.join(os.path.split(__file__)[0], "test_dset", "A")
    path_b = os.path.join(os.path.split(__file__)[0], "test_dset", "B")
    os.makedirs(path_a)
    os.makedirs(path_b)

    for i in range(25):
        imsave(os.path.join(path_a, "image_%03d.png" %
                            i), np.random.rand(224, 224, 3))

        data_paths.append(os.path.join(path_a, "image_%03d.png" %
                                       i))

    for i in range(30):
        imsave(os.path.join(path_b, "image_%03d.png" %
                            i), np.random.rand(224, 224, 3))
        data_paths.append(os.path.join(path_b, "image_%03d.png" %
                                       i))

    dset = UnPairedDataset(path_a, path_b, 50, 0.5, img_size=64, n_channels=1)

    assert len(dset) == 30

    assert isinstance(dset[(5, 3)], dict)
    assert dset[(5, 3)]["domain_a"].shape == (1, 64, 64)

    try:
        dset[5]
        assert False, "should raise TypeError since index must be iterable"

    except TypeError:
        assert True

    for file in data_paths:
        os.remove(file)

    os.rmdir(path_a)
    os.rmdir(path_b)
    os.rmdir(os.path.join(os.path.split(__file__)[0], "test_dset"))


text:
# -*- coding: utf-8 -*-
"""MTG jamendo autotagging moodtheme Dataset Loader

.. admonition:: Dataset Info
    :class: dropdown

    The MTG Jamendo autotagging mood/theme Dataset is a new open dataset for music auto-tagging. It
    is built using music available at Jamendo under Creative Commons licenses and tags provided by content uploaders. The
    dataset contains 18,486 full audio tracks with 195 tags from mood/theme. It is provided
    five fixed data splits for a better and fair replication. For more information please visit: https://github.com/MTG/mtg-jamendo-dataset .

    The moodtheme tags are:

    action, adventure, advertising, ambiental, background, ballad, calm, children, christmas, commercial, cool,
    corporate, dark, deep, documentary, drama, dramatic, dream, emotional, energetic, epic, fast, film, fun, funny,
    game, groovy, happy, heavy, holiday, hopeful, horror, inspiring, love, meditative, melancholic, mellow, melodic,
    motivational, movie, nature, party, positive, powerful, relaxing, retro, romantic, sad, sexy, slow, soft,
    soundscape, space, sport, summer, trailer, travel, upbeat, uplifting.

    Emotion and theme recognition is a popular task in music information retrieval that is relevant for music search and
    recommendation systems.

    This task involves the prediction of moods and themes conveyed by a music track, given the raw audio. The examples
    of moods and themes are: happy, dark, epic, melodic, love, film, space etc. The full list is available at:
    https://github.com/mir-dataset-loaders/mirdata/pull/505 Each track is tagged with at least one
    tag that serves as a ground-truth.

    Acknowledgments

    This work was funded by the predoctoral grant MDM-2015-0502-17-2 from the Spanish Ministry of Economy and
    Competitiveness linked to the Maria de Maeztu Units of Excellence Programme (MDM-2015-0502).

    This work has received funding from the European Union's Horizon 2020 research and innovation programme under the
    Marie Skłodowska-Curie grant agreement No. 765068 "MIP-Frontiers".

    This work has received funding from the European Union's Horizon 2020 research and innovation programme under
    grant agreement No 688382 "AudioCommons".
"""
import csv
import os
from typing import Optional, Tuple, BinaryIO

from deprecated.sphinx import deprecated
import librosa
import numpy as np
from mirdata import download_utils, jams_utils, core
from smart_open import open

BIBTEX = """@conference {bogdanov2019mtg,
    author = "Bogdanov, Dmitry and Won, Minz and Tovstogan, Philip and Porter, Alastair and Serra, Xavier",
    title = "The MTG-Jamendo Dataset for Automatic Music Tagging",
    booktitle = "Machine Learning for Music Discovery Workshop, International Conference on Machine Learning (ICML 2019)",
    year = "2019",
    address = "Long Beach, CA, United States",
    url = "http://hdl.handle.net/10230/42015"
}"""
INDEXES = {
    "default": "1.0",
    "test": "1.0",
    "1.0": core.Index(filename="mtg_jamendo_autotagging_moodtheme_index_1.0.json"),
}
DOWNLOAD_INFO = """
    The audio files can be downloaded following the path described in https://github.com/MTG/mtg-jamendo-dataset#downloading-the-data

    To download audio, unpack and validate all tar archives:

    .. code-block:: console

          mkdir /path/to/download
          python3 scripts/download/download.py --dataset autotagging_moodtheme --type audio /path/to/download --unpack --remove

    Later add the files to a folder into mir_datasets called audio/ with the following structure:
        > mtg_jamendo_autotagging_moodtheme/
            > audios/
                > 00/
                ...
                > 99/
"""
REMOTES = {
    "metadata": download_utils.RemoteFileMetadata(
        filename="metadata.zip",
        url="https://zenodo.org/record/3826813/files/data.zip?download=1",
        checksum="039ce10f267f6e4e9f72837c76d72b2f",
    )
}

class Track(core.Track):
    """MTG jamendo autotagging moodtheme Track class

    Args:
        track_id (str): track id of the track (JAMENDO track id)

    Attributes:
        audio_path (str): Path to the audio file

    Cached Properties:
        artist_id (str): JAMENDO artist id
        album_id (str): JAMENDO album id
        duration (float): track duration
        tags (str): autotagging moodtheme annotations

    """

    def __init__(
        self,
        track_id,
        data_home,
        dataset_name,
        index,
        metadata,
    ):
        super().__init__(
            track_id,
            data_home,
            dataset_name,
            index,
            metadata,
        )

        self.track_id = track_id
        self.audio_path = self.get_path("audio")

    @property
    def audio(self) -> Optional[Tuple[np.ndarray, float]]:
        """The track's audio

        Returns:
            * np.ndarray - audio signal
            * float - sample rate

        """
        return load_audio(self.audio_path)

    @core.cached_property
    def artist_id(self) -> str:
        return self._track_metadata.get("ARTIST_ID")

    @core.cached_property
    def album_id(self) -> str:
        return self._track_metadata.get("ALBUM_ID")

    @core.cached_property
    def duration(self) -> float:
        return float(self._track_metadata.get("DURATION"))

    @core.cached_property
    def tags(self) -> str:
        return self._track_metadata.get("TAGS")

    def to_jams(self):
        # Initialize top-level JAMS container
        return jams_utils.jams_converter(
            audio_path=self.audio_path,
            metadata={
                "track_id": self.track_id,
                "artist_id": self.artist_id,
                "album_id": self.album_id,
                "duration": self.duration,
                "tags": self.tags,
            },
        )

def load_audio(fhandle: BinaryIO) -> Tuple[np.ndarray, float]:
    """Load a MTG jamendo autotagging moodtheme audio file.

    Args:
        fhandle (str or file-like): path or file-like object pointing to an audio file

    Returns:
        * np.ndarray - the mono audio signal
        * float - The sample rate of the audio file

    """
    return librosa.load(fhandle, sr=None, mono=False)

@core.docstring_inherit(core.Dataset)
class Dataset(core.Dataset):
    """
    The MTG jamendo autotagging moodtheme dataset
    """

    def __init__(self, data_home=None, version="default"):
        super().__init__(
            data_home,
            version,
            name="mtg_jamendo_autotagging_moodtheme",
            track_class=Track,
            bibtex=BIBTEX,
            download_info=DOWNLOAD_INFO,
            indexes=INDEXES,
            remotes=REMOTES,
            license_info=(
                "Creative Commons Attribution NonCommercial Share Alike 4.0 International."
            ),
        )

    @core.cached_property
    def _metadata(self):
        meta_path = os.path.join(self.data_home, "data/autotagging_moodtheme.tsv")

        try:
            with open(meta_path, "r") as fhandle:
                reader = csv.DictReader(fhandle, delimiter="\t")
                meta = {
                    row["TRACK_ID"]: {k: row[k] for k in row if k != "TRACK_ID"}
                    for row in reader
                }
        except FileNotFoundError:
            raise FileNotFoundError("Metadata not found. Did you run .download()?")

        splits = {}
        for split_number in range(5):
            split = {}
            path_train = os.path.join(
                self.data_home,
                "data",
                "splits",
                f"split-{split_number}",
                "autotagging_moodtheme-train.tsv",
            )
            with open(path_train, "r") as fhandle:
                reader = csv.reader(fhandle, delimiter="\t")
                track_uris = [line[0] for line in reader]
                split["train"] = track_uris[1:]
            path_validation = os.path.join(
                self.data_home,
                "data",
                "splits",
                f"split-{split_number}",
                "autotagging_moodtheme-validation.tsv",
            )
            with open(path_validation, "r") as fhandle:
                reader = csv.reader(fhandle, delimiter="\t")
                track_uris = [line[0] for line in reader]
                split["validation"] = track_uris[1:]
            path_test = os.path.join(
                self.data_home,
                "data",
                "splits",
                f"split-{split_number}",
                "autotagging_moodtheme-test.tsv",
            )
            with open(path_test, "r") as fhandle:
                reader = csv.reader(fhandle, delimiter="\t")
                track_uris = [line[0] for line in reader]
                split["test"] = track_uris[1:]

            splits[split_number] = split
        meta["splits"] = splits
        return meta

    @deprecated(
        reason="Use mirdata.datasets.mtg_jamendo_autotagging_moodtheme.load_audio",
        version="0.3.4",
    )
    def load_audio(self, *args, **kwargs):
        return load_audio(*args, **kwargs)

    def get_track_ids_for_split(self, split_number):
        """Load a MTG jamendo autotagging moodtheme pre-defined split. There are five different train/validation/tests splits.
        Args:
             split_number (int): split to be retrieved from 0 to 4
        Returns:
            * dict: {"train": [...], "validation": [...], "test": [...]} - the train split

        """
        if not (0 <= split_number <= 4):
            raise Exception("Splits avaiables from num 0 to 4")

        return self._metadata["splits"][split_number]

max_stars_repo_name: migperfer/mirdata

max_issues_repo_name: migperfer/mirdata

max_forks_repo_name: migperfer/mirdata