text:
# Gavin Mischler
# 12/12/2019

import pytest
import numpy as np
from mvlearn.semi_supervised.ctclassifier import CTClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import RidgeClassifier

@pytest.fixture(scope='module')
def data():
    random_seed = 10
    N = 100
    D1 = 10
    D2 = 6
    N_test = 5
    random_data = []
    np.random.seed(random_seed)
    random_data.append(np.random.rand(N,D1))
    random_data.append(np.random.rand(N,D2))
    random_labels = np.floor(2*np.random.rand(N,)+2)
    random_labels[:-10] = np.nan
    random_test = []
    random_test.append(np.random.rand(N_test, D1))
    random_test.append(np.random.rand(N_test, D2))
    gnb1 = GaussianNB()
    gnb2 = GaussianNB()
    clf_test = CTClassifier(gnb1, gnb2, random_state=random_seed)

    return {'N_test' : N_test, 'clf_test' : clf_test,
    'random_data' : random_data, 'random_labels' : random_labels,
    'random_test' : random_test, 'random_seed' : random_seed}

def test_fit_one_class(data):
    random_labels_bad = np.floor(np.random.rand(100,))
    random_labels_bad[:-10] = np.nan
    data['clf_test'].fit(data['random_data'], random_labels_bad)

'''
EXCEPTION TESTING
'''

def test_no_predict_proba_attribute():
    with pytest.raises(AttributeError):
        clf = CTClassifier(RidgeClassifier(), RidgeClassifier())

def test_no_wrong_view_number(data):
    with pytest.raises(ValueError):
        Xs = []
        for i in range(5):
            Xs.append(np.zeros(10))

        data['clf_test'].fit(Xs, data['random_labels'])

def test_fit_incompatible_views(data):
    X1 = np.ones((100,10))
    X2 = np.zeros((99,10))
    with pytest.raises(ValueError):
        data['clf_test'].fit([X1, X2], data['random_labels'])

def test_fit_over_two_classes(data):
    with pytest.raises(ValueError):
        random_labels_bad = np.floor(2*np.random.rand(100,)+2)
        random_labels_bad[:-10] = np.nan
        random_labels_bad[0] = 10
        data['clf_test'].fit(data['random_data'], random_labels_bad)

def test_fit_zero_classes(data):
    with pytest.raises(ValueError):
        random_labels_bad = np.zeros((100,))
        random_labels_bad[:] = np.nan
        data['clf_test'].fit(data['random_data'], random_labels_bad)

def test_predict_incompatible_views(data):
    X1 = np.ones((100,10))
    X2 = np.zeros((99,10))
    with pytest.raises(ValueError):
        data['clf_test'].predict([X1, X2])

def test_predict_over_two_classes(data):
    data['clf_test'].fit(data['random_data'], data['random_labels'])
    bad_data = []
    for i in range(3):
        bad_data.append(np.ones((100,10)))
    with pytest.raises(ValueError):
        random_labels_bad = np.ones((100,))
        data['clf_test'].predict(bad_data)

def test_predict_one_class(data):
    data['clf_test'].fit(data['random_data'], data['random_labels'])
    bad_data = []
    bad_data.append(np.zeros((100,10)))
    with pytest.raises(ValueError):
        random_labels_bad = np.zeros(100)
        data['clf_test'].predict(bad_data)

def test_predict_zero_classes(data):
    data['clf_test'].fit(data['random_data'], data['random_labels'])
    bad_data = []
    with pytest.raises(ValueError):
        random_labels_bad = np.zeros(100)
        data['clf_test'].predict(bad_data)

def test_zero_p():
    with pytest.raises(ValueError):
        clf = CTClassifier(GaussianNB(), GaussianNB(), p=0)

def test_negative_p():
    with pytest.raises(ValueError):
        clf = CTClassifier(GaussianNB(), GaussianNB(), p=-1)

def test_zero_n():
    with pytest.raises(ValueError):
        clf = CTClassifier(GaussianNB(), GaussianNB(), n=0)

def test_negative_n():
    with pytest.raises(ValueError):
        clf = CTClassifier(GaussianNB(), GaussianNB(), n=-1)

def test_zero_pool_size():
    with pytest.raises(ValueError):
        clf = CTClassifier(GaussianNB(), GaussianNB(), unlabeled_pool_size=0)

def test_negative_pool_size():
    with pytest.raises(ValueError):
        clf = CTClassifier(GaussianNB(), GaussianNB(), unlabeled_pool_size=-1)

def test_zero_num_iter():
    with pytest.raises(ValueError):
        clf = CTClassifier(GaussianNB(), GaussianNB(), num_iter=0)

def test_negative_num_iter():
    with pytest.raises(ValueError):
        clf = CTClassifier(GaussianNB(), GaussianNB(), num_iter=-1)

'''
FUNCTION TESTING
'''

def test_fit_defaultclassifier(data):
    clf = CTClassifier(random_state=10)
    clf.fit(data['random_data'], data['random_labels'])
    y_pred_test = clf.predict(data['random_test'])
    y_pred_prob = clf.predict_proba(data['random_test'])

    truth = [2., 3., 2., 2., 2.]
    truth_proba = [[0.90643161, 0.09356839],
    [0.35361377, 0.64638623],
    [0.50968072, 0.49031928],
    [0.58446856, 0.41553144],
    [0.87548943, 0.12451057]]

    for i in range(data['N_test']):
        assert y_pred_test[i] == truth[i]

    for i in range(data['N_test']):
        for j in range(2):
            assert abs(y_pred_prob[i,j] - truth_proba[i][j]) < 0.00000001

def test_predict_default(data):
    data['clf_test'].fit(data['random_data'], data['random_labels'])
    y_pred_test = data['clf_test'].predict(data['random_test'])
    y_pred_prob = data['clf_test'].predict_proba(data['random_test'])

    truth = [2., 3., 2., 2., 2.]
    truth_proba = [[0.90643161, 0.09356839],
    [0.35361377, 0.64638623],
    [0.50968072, 0.49031928],
    [0.58446856, 0.41553144],
    [0.87548943, 0.12451057]]

    for i in range(data['N_test']):
        assert y_pred_test[i] == truth[i]

    print('test_predict_default')
    print(y_pred_prob)

    for i in range(data['N_test']):
        for j in range(2):
            assert abs(y_pred_prob[i,j] - truth_proba[i][j]) < 0.00000001

def test_predict_check_p_n(data):
    labels1 = np.zeros(100,)
    labels1[:5] = 4 # 5 "negative"
    labels1[5:15] = 6 # 10 "positive"
    labels1[15:] = np.nan
    clf = CTClassifier(random_state=0)
    clf.fit(data['random_data'], labels1)
    assert clf.p_ == 2
    assert clf.n_ == 1

    labels2 = np.zeros(100,)
    labels2[:5] = 6 # 5 "positive"
    labels2[5:15] = 4 # 10 "negative"
    labels2[15:] = np.nan
    clf = CTClassifier(random_state=0)
    clf.fit(data['random_data'], labels2)
    assert clf.p_ == 1
    assert clf.n_ == 2

    labels1 = np.zeros(100,)
    labels1[:5] = 4 # 5 "negative"
    labels1[5:15] = 6 # 10 "positive"
    labels1[15:] = np.nan
    clf = CTClassifier(p=4, n=3, random_state=0)
    clf.fit(data['random_data'], labels1)
    assert clf.p_ == 4
    assert clf.n_ == 3

def test_predict_set_p(data):
    random_seed = 10
    gnb1 = GaussianNB()
    gnb2 = GaussianNB()
    clf = CTClassifier(gnb1, gnb2, p=12, random_state=random_seed)
    clf.fit(data['random_data'], data['random_labels'])
    y_pred_test = clf.predict(data['random_test'])
    y_pred_prob = clf.predict_proba(data['random_test'])

    truth = [3., 3., 3., 3., 3.]
    truth_proba = [[0.31422418, 0.68577582],
    [0.40938282, 0.59061718],
    [0.48448605, 0.51551395],
    [0.38853225, 0.61146775],
    [0.22972488, 0.77027512]]

    for i in range(data['N_test']):
        assert y_pred_test[i] == truth[i]

    for i in range(data['N_test']):
        for j in range(2):
            assert abs(y_pred_prob[i,j] - truth_proba[i][j]) < 0.00000001

def test_predict_set_n(data):
    random_seed = 10
    gnb1 = GaussianNB()
    gnb2 = GaussianNB()
    clf = CTClassifier(gnb1, gnb2, n=9, random_state=random_seed)
    clf.fit(data['random_data'], data['random_labels'])
    y_pred_test = clf.predict(data['random_test'])
    y_pred_prob = clf.predict_proba(data['random_test'])

    truth = [3., 3., 2., 3., 3.]
    truth_proba = [[0.29020704, 0.70979296],
    [0.44024614, 0.55975386],
    [0.5710383 , 0.4289617 ],
    [0.37366059, 0.62633941],
    [0.22157484, 0.77842516]]

    for i in range(data['N_test']):
        assert y_pred_test[i] == truth[i]

    for i in range(data['N_test']):
        for j in range(2):
            assert abs(y_pred_prob[i,j] - truth_proba[i][j]) < 0.00000001

def test_predict_unlabeled_pool_size(data):
    random_seed = 10
    gnb1 = GaussianNB()
    gnb2 = GaussianNB()
    clf = CTClassifier(gnb1, gnb2, unlabeled_pool_size=20, random_state=random_seed)
    clf.fit(data['random_data'], data['random_labels'])
    y_pred_test = clf.predict(data['random_test'])
    y_pred_prob = clf.predict_proba(data['random_test'])

    truth = [2., 3., 3., 2., 2.]
    truth_proba = [[0.75269031, 0.24730969],
    [0.37316567, 0.62683433],
    [0.45283102, 0.54716898],
    [0.59666021, 0.40333979],
    [0.81196019, 0.18803981]]

    for i in range(data['N_test']):
        assert y_pred_test[i] == truth[i]

    for i in range(data['N_test']):
        for j in range(2):
            assert abs(y_pred_prob[i,j] - truth_proba[i][j]) < 0.00000001

def test_predict_num_iter(data):
    random_seed = 10
    gnb1 = GaussianNB()
    gnb2 = GaussianNB()
    clf = CTClassifier(gnb1, gnb2, num_iter=9, random_state=random_seed)
    clf.fit(data['random_data'], data['random_labels'])
    y_pred_test = clf.predict(data['random_test'])
    y_pred_prob = clf.predict_proba(data['random_test'])

    truth = [2., 3., 2., 2., 2.]
    truth_proba = [[0.90643161, 0.09356839],
    [0.35361377, 0.64638623],
    [0.50968072, 0.49031928],
    [0.58446856, 0.41553144],
    [0.87548943, 0.12451057]]

    for i in range(data['N_test']):
        assert y_pred_test[i] == truth[i]

    for i in range(data['N_test']):
        for j in range(2):
            assert abs(y_pred_prob[i,j] - truth_proba[i][j]) < 0.00000001

max_stars_repo_name: NeuroDataDesign/mvlearn

max_issues_repo_name: neurodata/mvlearn

max_forks_repo_name: NeuroDataDesign/mvlearn


text:
import numpy as np
import pandas as pd
import os
from benchmark_util import *
from sklearn.metrics.cluster import adjusted_rand_score
import argparse

current_path = os.path.abspath('.')

parser = argparse.ArgumentParser(description='arg')
parser.add_argument('--dir', type=str, default='/outputdir/',
                    help='outputdir')
parser.add_argument('--inputfile', type=str, default='151673_all_results',
                    help='151673_all_results.txt')
parser.add_argument('--datasetname', type=str, default='151673',
                    help='dataset name')
args = parser.parse_args()

fileFolder = current_path+'/'+args.datasetname+'/'

labelname = fileFolder+'label.csv'
# labelname = current_path+'/outputdir_ccG/151673_noregu_0.9_1.0_0.0_results_6.txt'
# resultname = current_path+'/outputdir/151673_dummy_add_0.0_results_5.txt'
# resultname = current_path+'/outputdir/151673_all_results.txt'
# resultname = current_path+'/outputdir/151673_geom_lowf_add_2.0_results_7.txt'
# resultname = current_path+'/outputdirscgnn/151673_noregu_0.9_1.0_0.0_results_9.txt'

resultname = current_path+args.dir+args.inputfile+'.txt'

df = pd.read_csv(labelname)
listBench = df['layer'].to_numpy().tolist()
# listBench = df['Celltype'].to_numpy().tolist()

df = pd.read_csv(resultname)
listResult = df['Celltype'].to_numpy().tolist()

ari, ami, nmi, cs, fms, vms, hs = measureClusteringTrueLabel(listBench, listResult)
resultstr = args.inputfile+' '+str(ari)+' ' + \
            str(ami)+' '+str(nmi)+' '+str(cs)+' ' + \
            str(fms)+' '+str(vms)+' '+str(hs)
print(resultstr)

ext: py

reason: import numpy


text:
import os
import sys

sys.path.append(os.path.join('..', '..'))

from rascil.data_models.parameters import rascil_path

results_dir = rascil_path('test_results')

from matplotlib import pylab

pylab.rcParams['figure.figsize'] = (8.0, 8.0)
pylab.rcParams['image.cmap'] = 'rainbow'

import numpy

from astropy.coordinates import SkyCoord
from astropy import units as u
from astropy import constants

from matplotlib import pyplot as plt

from rascil.processing_components.visibility.base import create_visibility
from rascil.processing_components.simulation.configurations import create_named_configuration

from rascil.data_models.polarisation import PolarisationFrame

import logging

log = logging.getLogger()
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler(sys.stdout))
mpl_logger = logging.getLogger("matplotlib")
mpl_logger.setLevel(logging.WARNING)

pylab.rcParams['figure.figsize'] = (12.0, 12.0)
pylab.rcParams['image.cmap'] = 'rainbow'

from rascil.processing_components.visibility.base import create_blockvisibility_from_ms

vis = create_blockvisibility_from_ms('PE_0.0_arcsec.ms')[0]

print(vis.uvw.shape)
plt.clf()
uvw=vis.uvw.reshape([65*197*197,3])
vu=uvw[...,0]
vv=uvw[...,1]
plt.plot(vu, vv, '.', color='b')
plt.plot(-vu, -vv, '.', color='b')
plt.show()

print(vis.configuration)

max_stars_repo_name: PhilippaHartley/sim-mid-pointing

max_issues_repo_name: PhilippaHartley/sim-mid-pointing

max_forks_repo_name: PhilippaHartley/sim-mid-pointing

reason: import numpy,from astropy


text:
"""This example file demonstrates the construction of a simple rocket.
The corresponding .ork file is the OpenRocket implementation of the same rocket for comparison.
"""
import numpy as np
import matplotlib.pyplot as plt

import rocketPy as rp
from rocketPy import ureg

## First, create a rocket.
r = rp.Rocket(name='Simple Rocket')

## create a nose cone
nc = rp.NoseCone(name='Nose Cone', diameter=6*ureg.inch, fineness=3, material=rp.materials.PLA())
# assign to rocket
r.set_nose_cone(nc)

## create a BodyTube
bt = rp.BodyTube(name = 'Body Tube', diameter=6*ureg.inch, length=48*ureg.inch, wall_thickness=2*ureg.mm, material=rp.materials.Phenolic())
# define its location
bt.set_position(after=nc)
# assign to rocket
r.set_body_tube(bt)

# create a boat tail
boat_tail = rp.Transition(name='Boat Tail', fore_dia=6*ureg.inch, aft_dia=4*ureg.inch, length=4*ureg.inch, material=rp.materials.Phenolic())
# define its location
boat_tail.set_position(after=bt)
# assign to rocket
r.set_boat_tail(boat_tail)

## create the fins
fins = rp.FinSet(name='Fins', n=3, span=6*ureg.inch, root_chord=12*ureg.inch, tip_chord=6*ureg.inch, mid_sweep=10*ureg.degree, tube_dia=6*ureg.inch,  thickness=2*ureg.mm, material=rp.materials.Aluminium())
# define its location
fins.set_position(end_of=bt, offset=-fins.root_chord)
#  assign to rocket
r.set_fins(fins)

# plot the entire rocket
fig = plt.figure()
ax = plt.gca()
r.plot(ax, unit=ureg.inch)
plt.draw()

# describe the rocket
r.describe(describe_components=True)

plt.show()


text:
'''
This is a sample class for a model. You may choose to use it as-is or make any changes to it.
This has been provided just to give you an idea of how to structure your model class.
'''
import numpy as np
import time
from openvino.inference_engine import IENetwork, IECore
import os
import cv2
import argparse
import logging as log
import sys
from model_reuse import Model
class FacialLandmarks(Model):
    '''
    Class for the facial_landmarks_detetion_Model.
    '''
    def __init__(self,model_name,device="CPU",extensions=None,threshold=0.6):
        Model.__init__(self, model_name, device='CPU', extensions=None)
        self.input_name=next(iter(self.model.inputs))
        self.input_shape=self.model.inputs[self.input_name].shape
        self.output_name=next(iter(self.model.outputs))
        self.output_shape=self.model.outputs[self.output_name].shape

    def predict(self, image):
        '''
        TODO: You will need to complete this method.
        This method is meant for running predictions on the input image.
        '''
        img_processed = self.preprocess_input(image.copy())
        outputs = self.network.infer({self.input_name:img_processed})
        coords = self.preprocess_output(outputs,image)
        left_eye_x_min = coords['leftx'] - 10
        left_eye_x_max = coords['leftx'] + 10
        left_eye_y_min = coords['lefty'] - 10
        left_eye_y_max = coords['lefty'] + 10

        right_eye_x_min = coords['rightx'] - 10
        right_eye_x_max = coords['rightx'] + 10
        right_eye_y_min = coords['righty'] - 10
        right_eye_y_max = coords['righty'] + 10

        eye_coord = [[left_eye_x_min, left_eye_y_min, left_eye_x_max, left_eye_y_max],
                          [right_eye_x_min, right_eye_y_min, right_eye_x_max, right_eye_y_max]]
        left_eye_image = image[left_eye_y_min:left_eye_y_max, left_eye_x_min:left_eye_x_max]
        right_eye_image = image[right_eye_y_min:right_eye_y_max, right_eye_x_min:right_eye_x_max]

        return left_eye_image, right_eye_image, eye_coord

    def check_model(self):
        ''

    def preprocess_output(self, outputs,image):
        '''
        Before feeding the output of this model to the next model,
        you might have to preprocess the output. This function is where you can do that.
        '''
        h=image.shape[0]
        w=image.shape[1]
        #preprocessing_output  Adapted from https://github.com/Rahul24-06/Computer-pointer-Controller-using-Gaze-Estimation/blob/master/src/facial_landmarks_detection.py
        try:
            outputs = outputs[self.output_name][0]
            left_eye_x = int(outputs[0] * w)
            left_eye_y = int(outputs[1] * h)
            right_eye_x = int(outputs[2] * w)
            right_eye_y = int(outputs[3] * h)

        except Exception as e:
            log.error("Error While preprocessing output for facial landmark detection model" + str(e))
        return {'leftx': left_eye_x, 'lefty': left_eye_y,
                'rightx': right_eye_x, 'righty': right_eye_y}


text:
from collections import OrderedDict

import numpy as np
import pyFAI

from ..Operation import Operation

inputs=OrderedDict(
    wxd_file=None,
    pixel_size_um=None,
    fpolz=None)
outputs=OrderedDict(poni_dict=None)

class WXDToPONI(Operation):
    """
    Convert WXDIFF .calib output
    to a dict of PyFAI PONI parameters,
    by first converting from WXDIFF to Fit2D,
    then using a pyFAI.AzimuthalIntegrator
    to convert from Fit2D to PONI format.

    The conversion from WXDiff parameters to Fit2D parameters
    was originally contributed to paws by Fang Ren.

    Input .calib file from WXDIFF automated calibration,
    input pixel size and polarization factor,
    output dict of pyFAI PONI calibration parameters.
    """

    def __init__(self):
        super(WXDToPONI,self).__init__(inputs,outputs)
        self.input_doc['wxd_file'] = '.calib file produced by WXDIFF calibration'
        self.input_doc['pixel_size_um'] = 'pixel size in microns'
        self.input_doc['fpolz'] = 'polarization factor'
        self.inputs['fpolz'] = float(1.)
        self.output_doc['poni_dict'] = 'Dict of pyFAI calibration parameters, as found in a .poni file'

    def run(self):
        fpath = self.inputs['wxd_file']
        pxsz_um = self.inputs['pixel_size_um']
        pxsz_m = pxsz_um*1E-6
        fpolz = self.inputs['fpolz']
        for line in open(fpath,'r'):
            kv = line.strip().split('=')
            if kv[0] == 'detect_dist':
                d_px = float(kv[1])         # WXDIFF direct detector distance,
                                            # from sample to where beam axis intersects detector plane, in pixels.
                                            # Fit2D uses this input in mm to set its spatial scale,
                                            # so it has to be converted to distance units (mm for Fit2D).
                                            # Because this scales the spatial representation for Fit2D,
                                            # the other inputs can still be given in pixel units,
                                            # without converting between pixel sizes.
                d_m = d_px*pxsz_m
                d_mm = d_m*1E3
            if kv[0] == 'bcenter_x':
                bcx_px = float(kv[1])       # WXDIFF x coord relative to 'bottom left' corner of detector
                                            # where beam axis intersects detector plane, in pixels
                                            # Fit2D centerX is also in pixels
            if kv[0] == 'bcenter_y':
                bcy_px = float(kv[1])       # WXDIFF y coord relative to 'bottom left' corner of detector
                                            # where beam axis intersects detector plane, in pixels
                                            # Fit2D centerY is also in pixels
            if kv[0] == 'detect_tilt_alpha':
                rot_rad = float(kv[1])      # WXDIFF alpha is tilt plane rotation in rad
                                            # Fit2D tiltPlanRotation  = 360-rot_rad*180/pi
            if kv[0] == 'detect_tilt_delta':
                tilt_rad = float(kv[1])     # WXDIFF delta is the tilt in rad
            if kv[0] == 'wavelenght':
                wl_A = float(kv[1])         # WXDIFF 'wavelenght' is in Angstroms
        # get wavelength in m
        wl_m = wl_A*1E-10
        # Mapping between Fit2D and WXDIFF originally contributed by Fang Ren.
        # TODO: Verify this mapping
        rot_deg = rot_rad*float(180)/np.pi
        rot_fit2d = float(360)-rot_deg
        tilt_deg = tilt_rad*float(180)/np.pi
        # use a pyFAI.AzimuthalIntegrator() to do the conversion
        p = pyFAI.AzimuthalIntegrator(wavelength = wl_m)
        p.setFit2D(d_mm,bcx_px,bcy_px,tilt_deg,rot_fit2d,pxsz_um,pxsz_um)
        poni_dict = p.getPyFAI()
        poni_dict['fpolz'] = fpolz
        self.outputs['poni_dict'] = poni_dict

max_stars_repo_name: slaclab/paws

max_issues_repo_name: slaclab/paws

max_forks_repo_name: slaclab/paws


text:
import matplotlib.pyplot as plt
import numpy as np

from simulate import popgen_simulate

# parameters to replicate figure 5a of 2014 valley crossing paper
# for each param combo, they average over 500 independent runs
delta = -2e-4
s = 0.1
mu_0 = 1e-5
mu_1 = 1e-4
mutant_traits = [(0.0, mu_0),    # base pop
                 (delta, mu_1),  # 1-mutant
                 (s, 0.0)]       # 2-mutant
N_list = [1e2, 5*1e2, 1e3, 5*1e3, 1e4, 5*1e4, 1e5, 5*1e5, 1e6, 5*1e6, 1e7]

REPEATS=10
def get_average_run(N, mutant_traits, repeats=REPEATS):
    trials = [0 for i in xrange(repeats)]
    for i in xrange(repeats):
        population, t = popgen_simulate(N, mutant_traits)
        trials[i] = t
    return sum(trials)/float(repeats)

t_list = [0 for N in N_list]
for i, N in enumerate(N_list):
    t = get_average_run(N, mutant_traits)
    t_list[i] = t

# plot output
plt.plot(N_list, t_list, '--x', label='default')
ax = plt.gca()
ax.set_xscale('log')
ax.set_yscale('log')
plt.xlabel('N')
plt.ylabel('time')
plt.legend(loc='upper right')
plt.title('Mean fixation time (avg %d runs per data point)' % REPEATS)
plt.show()

max_stars_repo_name: mattsmart/biomodels

max_issues_repo_name: mattsmart/biomodels

max_forks_repo_name: mattsmart/biomodels


text:
import subprocess
from subprocess import call
import pandas as pd
import os
import shutil
import gzip
import json
import sys
import mygene
import pickle

def uncompress_gzip(file_name, new_name=None, delete=True):
    # Read the stream and write that stream to a new file:
    in_file = gzip.open(file_name, 'rb')
    if new_name is None:
        out_file = open(file_name.strip('.gz'), 'wb')
    else:
        out_file = open(new_name, 'wb')
    out_file.write(in_file.read())
    in_file.close()
    out_file.close()
    if delete:
        os.remove(file_name)

def execute(comando, doitlive=False, input_to_use=None, verbose=True):
    # result = subprocess.run(['ls', '-l'], stdout=subprocess.PIPE)
    comando = comando.split(' ')

    if doitlive:
        popen = subprocess.Popen(comando, stdout=subprocess.PIPE, universal_newlines=True)

        to_return = popen.stdout.read()
        for line in to_return:
            if verbose:  # I see no reason to doitlive and have it be not verbose, but to each their own.
                print(line, end='')
        popen.stdout.close()
        return_code = popen.wait()
        if return_code:
            raise subprocess.CalledProcessError(return_code, comando)
    else:
        if input_to_use is not None:
            input_to_use = input_to_use.ecode('utf-8')
        result = subprocess.run(comando, stdout=subprocess.PIPE, stderr=subprocess.PIPE, input=input_to_use)

        to_return = result.stdout.decode('utf-8')
        if verbose:
            print(to_return)
    return to_return.strip('\n')

def make_sample_information_file(name, manifest_df, name_id_dict):

    # pwd = execute('pwd', doitlive=False, verbose=False)
    # destination = os.path.join(pwd, 'unused_files')
    # if os.path.isdir(destination):
    #     shutil.rmtree(destination)

    # name = 'TCGA_' + dataset_name + '.txt'
    file = open(name, 'w')
    file.write('File\tClass\tSample_Name\n')
    ignored_files = []
    ignored_twice = []
    ignored_flag = False
    for f in manifest_df['filename']:

        # TODO: add a filter here for the types of samples we want. I am using all "0x"and "1x" samples...
        # but presumably we only want "01" and "11" but then we should remove those from the directory (moved them to "unused")

            if name_id_dict[f][13:15] == '01':
                # file.write('\t'.join([name_id_dict[f]+'.htseq',class_dict[name_id_dict[f][17:19]] ,name_id_dict[f]]))
                file.write('\t'.join([name_id_dict[f]+'.htseq', 'Tumor', name_id_dict[f]]))
                file.write('\n')
                # print(name_id_dict[f])
            elif name_id_dict[f][13:15] == '03':
                print('\tNot ignoring file named "{}" because sample is tagged as'
                      ' "Primary Blood Derived Cancer - Peripheral Blood"'
                      '(i.e., sample id = {}), this is usually expected for LAML.'
                      .format(name_id_dict[f], name_id_dict[f][13:15]))
                # file.write('\t'.join([name_id_dict[f]+'.htseq',class_dict[name_id_dict[f][17:19]] ,name_id_dict[f]]))
                file.write('\t'.join([name_id_dict[f]+'.htseq', 'Tumor', name_id_dict[f]]))
                file.write('\n')

                # print(name_id_dict[f])
            elif name_id_dict[f][13:15] == '11':
                file.write('\t'.join([name_id_dict[f]+'.htseq', 'Normal', name_id_dict[f]]))
                file.write('\n')
                # print(name_id_dict[f])
            else:
                print('\tIgnoring file named "{}" because sample is neither Primary Tumor nor Matched Normal tissue '
                      '(i.e., sample id = {}).'.format(name_id_dict[f], name_id_dict[f][13:15]))
                # Move from raw_count_files to unused_files
                if name_id_dict[f] not in ignored_files:
                    ignored_flag = True
                    ignored_files.append(name_id_dict[f])  #keeping a list of the files which have been deleted prevent double deletion.
                    pwd = execute('pwd', doitlive=False, verbose=False)
                    destination = os.path.join(pwd, 'unused_files')
                    # if os.path.isdir(destination):
                    #     shutil.rmtree(destination)
                    if not os.path.isdir(destination):
                        os.mkdir(destination)
                    # Move the downloaded files to a folder
                    source = os.path.join(pwd, 'raw_count_files', name_id_dict[f]+'.htseq.counts')
                    print("source", source)
                    print("destination", destination)
                    # try:
                    shutil.move(source, os.path.join(destination, name_id_dict[f]+'.htseq.counts'))
                    # except shutil.Error:
                    #     shutil.move(source, destination)

                    # shutil.rmtree(os.path.join(pwd, 'raw_count_files'))  # Remove those files/folders from current directory
                    # print(f)
                    # print(name_id_dict[f]+'.htseq.counts')
                else:
                    print("This sample has been removed already only one sample with the same ID is allowed. "
                          "Consider setting 'long IDs' to 'True'"
                          "[as of 2018-07-06 this feature is yet to be implemented]")
                    ignored_twice.append(name_id_dict[f])
    file.close()

    if ignored_flag:
        print("The following files were ignored due to having the same ID as other sample:")
        print(ignored_twice)
    return

def remove_duplicate_genes(df):
    """
    TCGA has two duplicated genes RGS5 and POLR2J4.
    Rather than getting them manually, we'll check for all duplicated genes.
    """

    try:
        new_ix = df.index.droplevel(1).values
        df.index = new_ix
    except AttributeError:
        print("Dataframe only has one index, that's alright.")
    except IndexError:
        print("Using Pandas>1.0; Dataframe only has one index, that's alright.")
    except ValueError:
        print("Dataframe only has one index but it thinks it's multi-indexed, that's weird but alright.")

    # print(df)
    # print("")
    # print(df.columns)
    # print('getting first row')
    # print(df.ix[1,:])
    # print('getting first column')
    # print(df.ix[:, 0].ix[:, 0])
    # print('index')
    # print(df.index)
    # print(df.index.values)

    s = pd.Series(df['Name'])
    # print(s)
    import numpy as np
    repeated = s[s.duplicated()]
    repeated = np.unique(repeated.values)

    # print("---")
    # print(s)
    # print("@@@@@@@@@@")
    # print("Repeated are")
    # print(repeated)
    print(f"There were {len(repeated)} repeated genes.")
    print("Note that mygene.info's ENSEMBL ID to HUGO ID are not a 1 to 1 mapping, hence the replication.")
    df.set_index('Name', inplace=True)
    # print(df.head())
    # print(df.loc['TSPAN6',:])
    print(repeated)

    for gene in repeated:
        # if gene == 'RF00019':
        #     # print(gene)
        #     temp = df.loc[gene, :]
        #     df.drop(gene, axis=0, inplace=True)
        #     # print('before:')
        #     # print(temp)
        #     # print('after max:')
        #     temp_max = temp.max(axis=0, numeric_only=True)
        #     # print(temp_max)
        #     # temp_max.insert(0, 'ENSEMBL ID', 'Multiple ENSEMBL IDs')
        #     temp_max = pd.concat([pd.Series(['Multiple ENSEMBL IDs']), temp_max])
        #     # print(temp_max)
        #     columns = temp_max.values
        #     # columns = columns[-1:] + columns[:-1]  # Moving the ENSEMBL IDs first
        #     # print(columns)
        #
        #     df.loc[gene] = columns
        #     # print('after collapse')
        #     # print(df.loc[gene])
        #     # exit(192)

        # temp = df.loc[gene,:]
        # df.drop(gene, axis=0, inplace=True)
        # df.loc[gene] = temp.max(axis=0)
        # if gene == 'RF00019':
        temp = df.loc[gene,:]
        df.drop(gene, axis=0, inplace=True)
        temp_max = temp.max(axis=0, numeric_only=True)
        temp_max = pd.concat([pd.Series(['Multiple ENSEMBL IDs']), temp_max])
        df.loc[gene] = temp_max.values

    return df

def make_gct(file_list, translate_bool, file_name, cls_bool):
    """
    This function makes a GCT file by concatenating all the files present in file_list
    """
    df_gct = None

    if translate_bool:
        print('translation done with mygene.info')

    # get sample names
    sample_list = []
    # sample_list.append("GID")  # 2018-02-07 Changing this to Name
    sample_list.append("Name")
    # sample_list.append("NAME")  # 2018-02-07 Changing this to Description to conform to other GenePattern GCT files
    sample_list.append("Description")

    #CLS file generation
    #01 - 09 tumor, 10 - 19 normal, 20 - 29 control
    cls_list = []

    removed_for_repetition = 0

    # add data from every file in list to dataframe if exists
    for file in file_list:
        if os.path.exists(file):

            # get sample name
            splited = file.split('/')
            # # splited = splited[len(splited) - 1].split('.')[0][:19]  # After 15 IDs become redundant
            splited = splited[len(splited) - 1].split('.')[0][:15]

            if splited not in sample_list:

                sample_list.append(splited)
                cls_list.append(splited[-2:])

                # read in file
                df_curr = pd.read_table(file, header=None)
                df_curr.columns = ['ENSEMBL ID', 'Counts']

                # if first file, get gene translations and ensembl ids
                if df_gct is None:
                    df_gct = df_curr.copy()
                    df_curr.drop(df_curr.columns[1, ], axis=1, inplace=True)
                    df_curr[df_curr.columns[0]] = df_curr[df_curr.columns[0]].apply(lambda x: x.split(".")[0])

                    if translate_bool:
                        print("Translating genes now")
                        df_curr[df_curr.columns[0]] = df_curr[df_curr.columns[0]].apply(lambda x: translate(x))
                    df_curr.columns = ['Name']
                    df_gct = pd.concat([df_curr, df_gct], axis=1)

                # otherwise just concatenate
                else:
                    # get counts column and concatenate
                    df_curr.drop(df_curr.columns[0,], axis=1, inplace=True)
                    df_gct = pd.concat([df_gct, df_curr], axis=1)
            else:
                removed_for_repetition += 1

    print("{} samples were not included due to repetition, only one sample per unique ID is being used.".format(
        removed_for_repetition))

    # remove last 5 rows, which are not genes
    df_gct = df_gct[:-5]

    # remove repeated genes
    df_gct = remove_duplicate_genes(df_gct)

    # start writing gct file
    f = open(str(file_name+".gct"), "w")
    # headers
    f.write("#1.2")
    #  # The next two lines add enough tabs, removed for now on 2018-02-07
    # for i in range(len(sample_list)):
    #     f.write('\t')
    f.write('\n')

    f.write(str(len(df_gct)) + "\t" + str((len(sample_list) - 2)))
    #  # The next two lines add enough tabs, removed for now on 2018-02-07
    # for i in range(len(sample_list) - 2):
    #     f.write('\t')
    f.write('\n')

    # sample names
    f.write('\t'.join(sample_list))
    # # The following lines do the same but add one extra tab
    # for i in range(len(sample_list)):
    #     f.write(sample_list[i])
    #     print(sample_list[i])
    #     f.write('\t')
    f.write('\n')

    # dataframe
    df_gct.to_csv(f, sep='\t', index=True, header=False)
    f.close()

    if cls_bool:
        print("cls_list= ", cls_list)
        # start writing cls file
        f = open(str(file_name+".cls"), "w")
        types = set(cls_list)
        # headers
        f.write(str(len(cls_list)))
        f.write(' ')
        f.write(str(len(types)))
        f.write(' 1\n#')

        type_dict = {}
        type_count = 0
        for t in types:
            type_dict[t] = type_count
            type_count += 1
            f.write(' ')
            f.write(t)
            print(t)

        f.write('\n')

        for val in cls_list[:-1]:
            f.write(str(type_dict[val]))
            f.write(' ')
        f.write(str(type_dict[cls_list[-1]]))

        f.close()

mg = mygene.MyGeneInfo()
try:
    with open('TCGA_ENSEMBL2HUGO_dictionary.p', 'rb') as handle:
        ENSEMBL2HUGO = pickle.load(handle)
except FileNotFoundError:
    try:
        print("Local version of dictionary not found, trying to explicitly add the PWD")
        pwd = os.path.dirname(os.path.realpath(__file__))
        print(pwd)
        with open(pwd+'/TCGA_ENSEMBL2HUGO_dictionary.p', 'rb') as handle:
            ENSEMBL2HUGO = pickle.load(handle)
    except FileNotFoundError:
        print("Local version of dictionary not found again, trying the docker container version")
        with open('/usr/local/bin/TCGAImporter/TCGA_ENSEMBL2HUGO_dictionary.p', 'rb') as handle:
            ENSEMBL2HUGO = pickle.load(handle)

def translate(ESNG):
    hugo_id = ESNG
    try:
        hugo_id = ENSEMBL2HUGO[ESNG]
    except KeyError:
        try:
            hugo_id = mg.getgene(ESNG)['symbol']
        except TypeError:
            hugo_id = ESNG
    return hugo_id

class_dict = {
    '01': 'Tumor',
    '02': 'Tumor',
    '03': 'Tumor',
    '04': 'Tumor',
    '05': 'Tumor',
    '06': 'Tumor',
    '07': 'Tumor',
    '08': 'Tumor',
    '09': 'Tumor',
    '10': 'Normal',
    '11': 'Normal',
    '12': 'Normal',
    '13': 'Normal',
    '14': 'Normal',
    '15': 'Normal',
    '16': 'Normal',
    '17': 'Normal',
    '18': 'Normal',
    '19': 'Normal',
}

max_stars_repo_name: genepattern/TCGAImporter

max_issues_repo_name: genepattern/TCGAImporter

max_forks_repo_name: genepattern/TCGAImporter


text:
import numpy as np
from carla.transform import Transform

from core.datadescriptor import KittiDescriptor
from core.constants import WINDOW_HEIGHT, WINDOW_WIDTH, MAX_RENDER_DEPTH_IN_METERS, MIN_VISIBLE_VERTICES_FOR_RENDER, \
    VISIBLE_VERTEX_COLOR, OCCLUDED_VERTEX_COLOR, MIN_BBOX_AREA_IN_PX
from utils import degrees_to_radians
import logging

from utils.camera_utils import point_in_canvas, point_is_occluded, draw_rect, midpoint_from_agent_location, \
    calc_projected_2d_bbox, draw_3d_bounding_box, proj_to_camera, proj_to_2d

def bbox_2d_from_agent(agent, intrinsic_mat, extrinsic_mat, ext, bbox_transform, agent_transform, rot_rp):
    """ Creates bounding boxes for a given agent and camera/world calibration matrices.
        Returns the modified image that contains the screen rendering with drawn on vertices from the agent """
    bbox = vertices_from_extension(ext)
    # transform the vertices respect to the bounding box transform
    bbox = bbox_transform.transform_points(bbox)
    # the bounding box transform is respect to the agents transform
    # so let's transform the points relative to it's transform
    bbox = agent_transform.transform_points(bbox)
    # agents's transform is relative to the world, so now,
    # bbox contains the 3D bounding box vertices relative to the world
    # Additionally, you can logging.info these vertices to check that is working
    # Store each vertex 2d points for drawing bounding boxes later
    vertices_pos2d = vertices_to_2d_coords(bbox, intrinsic_mat, extrinsic_mat)
    return vertices_pos2d

def calculate_occlusion_stats(image, vertices_pos2d, depth_map, draw_vertices=True):
    """ Draws each vertex in vertices_pos2d if it is in front of the camera
        The color is based on whether the object is occluded or not.
        Returns the number of visible vertices and the number of vertices outside the camera.
    """
    num_visible_vertices = 0
    num_vertices_outside_camera = 0

    for y_2d, x_2d, vertex_depth in vertices_pos2d:
        # if the point is in front of the camera but not too far away
        if MAX_RENDER_DEPTH_IN_METERS > vertex_depth > 0 and point_in_canvas((y_2d, x_2d)):
            is_occluded = point_is_occluded(
                (y_2d, x_2d), vertex_depth, depth_map)
            if is_occluded:
                vertex_color = OCCLUDED_VERTEX_COLOR
            else:
                num_visible_vertices += 1
                vertex_color = VISIBLE_VERTEX_COLOR
            if draw_vertices:
                draw_rect(image, (y_2d, x_2d), 4, vertex_color)
        else:
            num_vertices_outside_camera += 1
    return num_visible_vertices, num_vertices_outside_camera

def create_kitti_data_point(agent, intrinsic_mat, extrinsic_mat, image, depth_image, player_measurements,
                            rot_rp, draw_3d_bbox=True):
    """ Calculates the bounding box of the given agent, and returns a KittiDescriptor which describes the object
    to be labeled """
    obj_type, agent_transform, bbox_transform, ext, location = transforms_from_agent(
        agent)

    if obj_type is None:
        logging.warning(
            "Could not get bounding box for agent. Object type is None")
        return image, None
    vertices_pos2d = bbox_2d_from_agent(
        agent, intrinsic_mat, extrinsic_mat, ext, bbox_transform, agent_transform, rot_rp)

    depth_map = depth_image.data * 1000
    num_visible_vertices, num_vertices_outside_camera = calculate_occlusion_stats(
        image, vertices_pos2d, depth_map, draw_vertices=draw_3d_bbox)

    midpoint = midpoint_from_agent_location(
        image, location, extrinsic_mat, intrinsic_mat)

    # At least N vertices has to be visible in order to draw bbox
    if num_visible_vertices >= MIN_VISIBLE_VERTICES_FOR_RENDER > num_vertices_outside_camera:
        midpoint[:3] = rot_rp * midpoint[:3]

        bbox_2d = calc_projected_2d_bbox(vertices_pos2d)
        area = calc_bbox2d_area(bbox_2d)
        if area < MIN_BBOX_AREA_IN_PX:
            logging.info("Filtered out bbox with too low area {}".format(area))
            return image, None
        if draw_3d_bbox:
            draw_3d_bounding_box(image, vertices_pos2d)
        from math import pi
        rotation_y = get_relative_rotation_y(agent, player_measurements) % pi

        datapoint = KittiDescriptor()
        datapoint.set_bbox(bbox_2d)
        datapoint.set_3d_object_dimensions(ext)
        datapoint.set_type(obj_type)
        datapoint.set_3d_object_location(midpoint)
        datapoint.set_rotation_y(rotation_y)
        return image, datapoint
    else:
        return image, None

def get_relative_rotation_y(agent, player_measurements):
    """ Returns the relative rotation of the agent to the camera in yaw
    The relative rotation is the difference between the camera rotation (on car) and the agent rotation"""
    # We only car about the rotation for the classes we do detection on
    if agent.vehicle.transform:
        rot_agent = agent.vehicle.transform.rotation.yaw
        rot_car = player_measurements.transform.rotation.yaw
        return degrees_to_radians(rot_agent - rot_car)

def vertices_to_2d_coords(bbox, intrinsic_mat, extrinsic_mat):
    """ Accepts a bbox which is a list of 3d world coordinates and returns a list
        of the 2d pixel coordinates of each vertex.
        This is represented as a tuple (y, x, d) where y and x are the 2d pixel coordinates
        while d is the depth. The depth can be used for filtering visible vertices.
    """
    vertices_pos2d = []
    for vertex in bbox:
        pos_vector = vertex_to_world_vector(vertex)
        # Camera coordinates
        transformed_3d_pos = proj_to_camera(pos_vector, extrinsic_mat)
        # 2d pixel coordinates
        pos2d = proj_to_2d(transformed_3d_pos, intrinsic_mat)

        # The actual rendered depth (may be wall or other object instead of vertex)
        vertex_depth = pos2d[2]
        x_2d, y_2d = WINDOW_WIDTH - pos2d[0],  WINDOW_HEIGHT - pos2d[1]
        vertices_pos2d.append((y_2d, x_2d, vertex_depth))
    return vertices_pos2d

def vertex_to_world_vector(vertex):
    """ Returns the coordinates of the vector in correct carla world format (X,Y,Z,1) """
    return np.array([
        [vertex[0, 0]],  # [[X,
        [vertex[0, 1]],  # Y,
        [vertex[0, 2]],  # Z,
        [1.0]  # 1.0]]
    ])

def vertices_from_extension(ext):
    """ Extraxts the 8 bounding box vertices relative to (0,0,0)
    https://github.com/carla-simulator/carla/commits/master/Docs/img/vehicle_bounding_box.png
    8 bounding box vertices relative to (0,0,0)
    """
    return np.array([
        [ext.x,   ext.y,   ext.z],  # Top left front
        [- ext.x,   ext.y,   ext.z],  # Top left back
        [ext.x, - ext.y,   ext.z],  # Top right front
        [- ext.x, - ext.y,   ext.z],  # Top right back
        [ext.x,   ext.y, - ext.z],  # Bottom left front
        [- ext.x,   ext.y, - ext.z],  # Bottom left back
        [ext.x, - ext.y, - ext.z],  # Bottom right front
        [- ext.x, - ext.y, - ext.z]  # Bottom right back
    ])

def transforms_from_agent(agent):
    """ Returns the KITTI object type and transforms, locations and extension of the given agent """
    if agent.HasField('pedestrian'):
        obj_type = 'Pedestrian'
        agent_transform = Transform(agent.pedestrian.transform)
        bbox_transform = Transform(agent.pedestrian.bounding_box.transform)
        ext = agent.pedestrian.bounding_box.extent
        location = agent.pedestrian.transform.location
    elif agent.HasField('vehicle'):
        obj_type = 'Car'
        agent_transform = Transform(agent.vehicle.transform)
        bbox_transform = Transform(agent.vehicle.bounding_box.transform)
        ext = agent.vehicle.bounding_box.extent
        location = agent.vehicle.transform.location
    else:
        return None, None, None, None, None
    return obj_type, agent_transform, bbox_transform, ext, location

def calc_bbox2d_area(bbox_2d):
    """ Calculate the area of the given 2d bbox
    Input is assumed to be xmin, ymin, xmax, ymax tuple
    """
    x_min, y_min, x_max, y_max = bbox_2d
    return (y_max - y_min) * (x_max - x_min)

max_stars_repo_name: dd-iuonac/object-detector-in-carla

max_issues_repo_name: dd-iuonac/object-detector-in-carla

max_forks_repo_name: dd-iuonac/object-detector-in-carla


text:
'''
Many of the variables  and equations are in reference to the paper,
Inferring Morphology and Strength of Magnetic Fields From Proton Radiographs,
This script will reconstruct magnetic fields given the data
from a Proton Radiography experiment.
'''
import sys
import math

import rad_ut as ru
from constants import M_PROTON_G, ESU, C, V_PER_E

from re import match
import numpy as np

def b_field(s2r_cm, s2d_cm, Ep_MeV):
    '''
    Calculates the Uniform Magnetic field.

    Parameters
    ----------
    s2r_cm (float): Distance from the proton source to the detector, in cm
    s2d_cm (float): Distance from the proton source to the interaction region, in cm
    Ep_MeV (float): Kinetic energy

    Returns
    -------
    Bconst (float): Calculates the B, uniform magnetic field strength
    '''
    v = math.sqrt(2 * (Ep_MeV * V_PER_E) /
                  M_PROTON_G)  # Velocity of Proton

    Bconst = M_PROTON_G * C * v / \
        (ESU * (s2r_cm - s2d_cm))  # Uniform B Field Strength

    return Bconst

def steady_state(flux, flux_ref):
    '''
    The goal is the obtain the steady-state diffusion Equation

    Parameters
    ----------
    flux (2D array): Number of protons per bin
    flux_ref (2D array): Number protons per bin without an interaction region
    type (int): if 1 Lam is calcuated using taylor expansion if not Lam is
                calcuated using the equation specified in the paper
    Returns
    -------
    Lam (2D array): fluence contrast
    Src (2D array): Source term from multiplying the fluence contrast and exp(fluence contrast)
    '''
    num_bins = flux_ref.shape[0]  # num_bins x num_bins

    # Obtaining the fluence contrast from Equation 6
    #Lam = np.multiply(2.0, np.subtract(1.0, np.sqrt(np.divide(flux_ref, flux))))
    Lam = np.zeros((num_bins, num_bins))

    for i in range(num_bins):
        for j in range(num_bins):
            if ((flux_ref[i,j] == 0) or (flux[i,j] == 0)):
                continue
            else:
                # Original Equationfrom the paperEquation in Paper
                #Lam[i, j] = (flux[i, j] - flux_ref[i, j]) / (flux_ref[i, j])
                # Taylor expansion
                Lam[i,j] = 2.0 * ( 1.0 - math.sqrt(flux_ref[i,j]/flux[i,j]))

    # Obtaining the exponential fluence contrast
    ExpLam = np.exp(Lam)
    # Source Term
    # RHS of the Steady-State Diffusion Equation
    Src = np.multiply(Lam, ExpLam)

    return (Src, Lam)

def D(i, j, y):
    '''
    Supplemental function used during Gauss-Seidel Iteration
    '''
    d = -2.0 * y[i, j] - 0.5 * (ru.bc_enforce_N(y, i + 1, j) +
                                ru.bc_enforce_N(y, i - 1, j) +
                                ru.bc_enforce_N(y, i, j + 1) +
                                ru.bc_enforce_N(y, i, j - 1))

    return d

def O(i, j, x, y):
    '''
    Supplemental function used during Gauss-Seidel Iteration
    '''
    a = 0.5 * (ru.bc_enforce_D(x, i + 1, j) * (ru.bc_enforce_N(y, i + 1, j) + y[i, j]) +
               ru.bc_enforce_D(x, i - 1, j) * (ru.bc_enforce_N(y, i - 1, j) + y[i, j]) +
               ru.bc_enforce_D(x, i, j + 1) * (ru.bc_enforce_N(y, i, j + 1) + y[i, j]) +
               ru.bc_enforce_D(x, i, j - 1) * (ru.bc_enforce_N(y, i, j - 1) + y[i, j]))

    return a

def B_recon(flux, flux_ref, Bperp, s2r_cm, s2d_cm, bin_um, Ep_MeV, tol_iter, max_iter):
    '''
    Produces a reconstructed magnetic field

    Parameters
    ----------
    flux (2D array): Number of protons per bin
    flux_ref (2D array): Number protons per bin without an interaction region
    s2r_cm (float): Distance from the proton source to the detector, in cm
    s2d_cm (float): Distance from the proton source to the interaction region, in cm
    bin_um (float): Length of the side of a bin, in cm
    Ep_MeV (float): Kinetic Energy

    Returns
    -------
    BperpR (2D array of (x,y)): Reconstructed Magnetic Field
    BperpS (2D array of (x,y)): True Magnetic Field
    '''
    ru.delta = bin_um / 10000.0

    num_bins = flux_ref.shape[0]  # num_bins x num_bins
    # RHS of the Steady-State Diffusion Equation and Fluence Contrast
    Src, Lam = steady_state(flux, flux_ref)
    # The real component after Lam is transformed then convolved and then inversely transformed
    phi = ru.solve_poisson(Lam)
    # Uniform B Field Strength
    Bconst = b_field(s2r_cm, s2d_cm, Ep_MeV)
    # Iterate to solution
    print "Gauss-Seidel Iteration..."
    GS = ru.Gauss_Seidel(phi, np.exp(Lam), D, O, Src,
                         talk=20, tol=tol_iter, maxiter=max_iter)
    # Multiplying by the area of the bin
    phi *= (ru.delta**2)
    # Reconstructed perpendicular B Fields
    BperpR = np.zeros((num_bins, num_bins, 2))
    # True perpendicular B Fields
    BperpS = np.zeros((num_bins, num_bins, 2))
    # Reconstructed Lateral motion of proton
    deltaXR = np.zeros((num_bins, num_bins, 2))

    for i in range(num_bins):
        for j in range(num_bins):
            # Reconstructed Data
            deltaXR[i, j] = -ru.gradient(phi, (i, j))
            BperpR[i, j, 0] = Bconst * deltaXR[i, j, 1]
            BperpR[i, j, 1] = -Bconst * deltaXR[i, j, 0]
            # True Data
            x = ru.idx2vec((i, j))
            x = x + deltaXR[i, j]
            idx = ru.vec2idx(x)
            BperpS[i, j, :] = Bperp[idx[0] % num_bins, idx[1] % num_bins, :]

    print "#L2 norm of residual = %12.5E ;  Number of Gauss-Seidel iterations = %d\n" % GS
    return BperpR, BperpS


text:
import cv2
import os

def get_file_name(path):
    file_names = [x for x in os.walk(path)][0][2]

    file_names = ['.'.join(x.split('.')[:-1]) for x in file_names]
    return file_names

def load_data(data_path:str,label_path:str):
    cap = cv2.VideoCapture(data_path)
    data_list = []
    while cap.isOpened():
        ret,frame = cap.read()
        if not ret:
            break
        data_list.append(cv2.resize(frame,(300,300)))
    save_shape=[300,300]
    with open(label_path, "r", encoding="utf-8") as f:
        readlines = f.readlines()
        boxs = []
        for readline in readlines:
            readline_split = readline.strip().split(",")
            box = [
                [int(float(readline_split[0]) * save_shape[1]), int(float(readline_split[1]) * save_shape[0])],
                [int(float(readline_split[2]) * save_shape[1]), int(float(readline_split[3]) * save_shape[0])],
                readline_split[4]]
            boxs.append(box)
    return data_list,boxs

if __name__ == '__main__':
    import cv2
    import numpy as np

    org_data_dirs=[
        # "D:\data\smoke_car\\rebuild_data_slim\\base_dataset",
        # "D:\data\smoke_car\\rebuild_data_slim\\DeAn_dataset",
        # "D:\data\smoke_car\\rebuild_data_slim\\GuRun_dataset",
        # "D:\data\smoke_car\\rebuild_data_slim\\HeNeng_dataset",
        # "D:\data\smoke_car\\rebuild_data_slim\\TongHua_dataset",
        # "D:\data\smoke_car\\rebuild_data_slim\\WanZai_dataset",
        # "D:\data\smoke_car\\rebuild_data_slim\\XinXiang_dataset",
        # "D:\data\smoke_car\\rebuild_data_slim\\YunJing_dataset",
        # "D:\data\smoke_car\\rebuild_data_slim\\ZhangYe_dataset",
        "D:\data\smoke_car\\rebuild_data_slim\\test_dataset",

                   ]
    save_data_pardir="D:\data\smoke_car/smoke_classification_data"

    for org_data_dir in org_data_dirs:
        save_data_dir = save_data_pardir+"/%s"%os.path.basename(org_data_dir)
        if not os.path.exists(save_data_dir):
            os.makedirs(save_data_dir)
        file_names = get_file_name(org_data_dir+"/data")
        for file_name in file_names:
            data, label = load_data(
                "%s/data/%s.mp4"%(org_data_dir,file_name),
                "%s/label/%s.txt"%(org_data_dir,file_name))
            have_smoke=False
            for x in label:
                if x[-1] == "smoke":
                    have_smoke=True
                    label = x
                    break
                elif "smoke" in x[-1]:
                    print("")
                    continue
            if not have_smoke:
                continue
                center_point = np.random.randint(0, 300, 2)
                max_wh = 50
            else:
                # continue
                center_point = (np.array(label[0])+np.array(label[1]))//2
                max_wh = np.max(np.array(label[1])-np.array(label[0]))//2*3
                max_wh = min(80,max_wh)
            center_point=np.clip(center_point,max_wh,300-max_wh)
            video_data=[]
            for d in data:
                show_data = d[center_point[1]-max_wh:center_point[1]+max_wh,center_point[0]-max_wh:center_point[0]+max_wh,:]
                show_data = cv2.resize(show_data,(100,100))
                video_data.append(show_data)
            video_data = np.concatenate(video_data,0)
            save_path = "%s/data/%s.jpg"%(save_data_dir,file_name)
            if not os.path.exists(os.path.dirname(save_path)):
                os.makedirs(os.path.dirname(save_path))
            cv2.imencode(".jpg",video_data)[1].tofile(save_path)
            save_path = "%s/label/%s.txt"%(save_data_dir,file_name)
            if not os.path.exists(os.path.dirname(save_path)):
                os.makedirs(os.path.dirname(save_path))
            with open(save_path,"w",encoding="utf-8") as f:
                if have_smoke:
                    f.write("T")
                else:
                    f.write("F")


text:
import json
from test_labels_loader.config_name_creator import create_cnn_model_name
import numpy as np
import matplotlib.pyplot as plt
from pandas import DataFrame

def plot_learning_curves(file_name1, file_name2):
    with open(file_name1, 'rb') as f:
        d1 = np.loadtxt(f)

    with open(file_name2, 'rb') as f:
        d2 = np.loadtxt(f)

    ax = plt.gca()
    for label in (ax.get_xticklabels() + ax.get_yticklabels()):
        label.set_fontsize(20)
    ax.set_xticks([2500,5000,7500, 10000, 12500, 15000, 17500])
    ax.set_xticklabels([25,50,75,100,125,150,175])
    plt.xlim([0, 17500])
    plt.ylim([0, 0.55])
    plt.xlabel('Iterations, $10^3$', fontsize=20)
    plt.ylabel('Negative log-likelihood', fontsize=20)
    plt.plot(d1, color='grey', label='ADADELTA 1.0')
    plt.plot(d2, linewidth=1.3, color='black', label='ADADELTA 0.01')
    plt.legend(loc='upper right')
    plt.show()

def plot_learning_curve(file_name):
    with open(file_name, 'rb') as f:
        d1 = np.loadtxt(f)
    print min(d1), max(d1)

    ax = plt.gca()
    for label in (ax.get_xticklabels() + ax.get_yticklabels()):
        label.set_fontsize(20)
    ax.set_xticks([2500,5000,7500, 10000, 12500, 15000, 17500])
    ax.set_xticklabels([25,50,75,100,125,150,175])
    plt.xlim([0, 17500])
    plt.xlabel('Iterations, $10^3$', fontsize=20)
    plt.ylabel('Negative log-likelihood', fontsize=20)
    plt.plot(d1, 'r')
    plt.show()

s = '/mnt/sda4/CODING/python/kaggle_data/models/nfreq6featumeanlog_stdhighc180lowcu0.1win_l60strid60globa1recep[1, 2]use_t0activ[urelu, urelu, utanh]dropo[0.3, 0.6]overl9strid[1, 2]train10weigh0.01scale1nkern[32, 64, 512]pool_[1, 1]l2_re0.0001valid10max_i150000rando1'
s1 = s + '/output_1.0.txt'
s2 = s + '/output_0.01.txt'

plot_learning_curves(s1,s2)
plot_learning_curve(s1)
plot_learning_curve(s2)

# nrow = 10
# ncol = len(d1)/nrow
# d1 = d1[:ncol*nrow]
# d1 = np.reshape(d1, (nrow, ncol))
# df = DataFrame(data=d1)
# labels = xrange(0, ncol, 1000)
# plt.boxplot(df.values)
# plt.xticks(range(len(labels)), labels)
# plt.show()
#plt.plot(np.abs(d1[1:len(d1)]-d1[0:len(d1)-1]), 'r')
#plt.show()

max_stars_repo_name: jizongFox/kaggle-seizure-prediction

max_issues_repo_name: jizongFox/kaggle-seizure-prediction

max_forks_repo_name: jizongFox/kaggle-seizure-prediction


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Aug 11 15:42:06 2018

@author: admin
"""

import numpy as np

img1 = cv2.imread("the_book_thief.jpg", cv2.IMREAD_GRAYSCALE)
img2 = cv2.imread("me_holding_book.jpg", cv2.IMREAD_GRAYSCALE)

# ORB Detector
orb = cv2.ORB_create()
kp1, des1 = orb.detectAndCompute(img1, None)
kp2, des2 = orb.detectAndCompute(img2, None)

# Brute Force Matching
bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
matches = bf.match(des1, des2)
matches = sorted(matches, key = lambda x:x.distance)

matching_result = cv2.drawMatches(img1, kp1, img2, kp2, matches[:50], None, flags=2)

cv2.imshow("Img1", img1)
cv2.imshow("Img2", img2)
cv2.imshow("Matching result", matching_result)
cv2.waitKey(0)
cv2.destroyAllWindows()


text:
from scipy import sparse
#import matplotlib.pyplot as plt
import os
# import tensorflow as tf
import numpy as np
import scipy

from plyfile import PlyData, PlyElement
import time

def totuple(a):
    return [ tuple(i) for i in a]

def TicTocGenerator():
    # Generator that returns time differences
    ti = 0           # initial time
    tf = time.time() # final time
    while True:
        ti = tf
        tf = time.time()
        yield tf-ti # returns the time difference

TicToc = TicTocGenerator() # create an instance of the TicTocGen generator

# This will be the main function through which we define both tic() and toc()
def toc(tempBool=True):
    # Prints the time difference yielded by generator instance TicToc
    tempTimeInterval = next(TicToc)
    if tempBool:
        print( "Elapsed time: %f seconds.\n" %tempTimeInterval )

def tic():
    # Records a time in TicToc, marks the beginning of a time interval
    toc(False)

def tfeval(X):
    gpu_options = tf.GPUOptions(allow_growth = True)
    config = tf.ConfigProto(device_count={'CPU': 1, 'GPU': 0},
    allow_soft_placement = False, gpu_options=gpu_options)
    sess = tf.Session(config=config)
    tf.global_variables_initializer().run(session=sess)
    #[tfEvals, tfEvecs] = tf.self_adjoint_eig(X)
    #[evals, evecs]  = sess.run( [tfEvals, tfEvecs] );
    x = sess.run(tf.identity(X) );
    sess.close();
    return x

def tfeig(X):
    gpu_options = tf.GPUOptions(allow_growth = True)
    config = tf.ConfigProto(device_count={'CPU': 1, 'GPU': 0},
    allow_soft_placement = False, gpu_options=gpu_options)
    sess = tf.Session(config=config)
    tf.global_variables_initializer().run(session=sess)
    #[tfEvals, tfEvecs] = tf.self_adjoint_eig(X)
    #[evals, evecs]  = sess.run( [tfEvals, tfEvecs] );
    LAP = sess.run(tf.identity(X) );
    sess.close();

    [evals, evecs] = scipy.linalg.eigh(LAP);
    evals = np.diag(evals)
    return evals, evecs, LAP

def load_mesh(path):
    VERT = np.loadtxt(path+'/mesh.vert')
    TRIV = np.loadtxt(path+'/mesh.triv',dtype='int32')-1

    return VERT, TRIV

def totuple(a):
    return [ tuple(i) for i in a]

def save_ply(V,T,filename):
    if(V.shape[1]==2):
        Vt = np.zeros((V.shape[0],3))
        Vt[:,0:2] = V
        V = Vt

    vertex = np.array(totuple(V),dtype=[('x', 'f4'), ('y', 'f4'),('z', 'f4')])
    face = np.array([ tuple([i]) for i in T],dtype=[('vertex_indices', 'i4', (3,))])
    el1 = PlyElement.describe(vertex, 'vertex')
    el2 = PlyElement.describe(face, 'face')
    PlyData([el1,el2]).write(filename)

def ismember(T, pts):
    out = np.zeros(np.shape(T)[0])
    for r in range(np.shape(T)[0]):
        s=0
        for c in range(np.shape(T)[1]):
            if np.sum(T[r,c]==pts)>0: s=s+1;
        out[r] = s>0;
    return out

def prepare_mesh(VERT,TRIV,dtype='float32'):
    edges = np.ones(shape=(np.shape(VERT)[0],np.shape(VERT)[0],2),dtype='int32')*-1
    edges_count = np.zeros(shape=(np.shape(VERT)[0],np.shape(VERT)[0]),dtype='int32')

    def setedg(i,j,k):
        _setedg(i,j,k)
        _setedg(j,i,k)

    def _setedg(i,j,k):
        edges_count[i,j] +=1
        if edges[i,j,0]==k:
            return
        if edges[i,j,1]==k:
            return
        if edges[i,j,0]==-1:
    #         print(edges[i,j,0])
            edges[i,j,0]=k
        else:
            edges[i,j,1]=k

    for ti in range(np.shape(TRIV)[0]):
        setedg(TRIV[ti,0],TRIV[ti,1],TRIV[ti,2])
        setedg(TRIV[ti,2],TRIV[ti,0],TRIV[ti,1])
        setedg(TRIV[ti,1],TRIV[ti,2],TRIV[ti,0])

    n = np.shape(VERT)[0]
    m = int(np.sum( ((edges[:,:,0]>=0) + (edges[:,:,1]>=0)) >0)/2);

    map_ = np.ones(shape=(n,n),dtype='int32')*-1;
    invmap = np.ones(shape=(m,2),dtype='int32')*-1;
    iM = np.zeros(shape=(m,n),dtype=dtype);
    bound_edges = np.zeros(shape=(m,1),dtype='bool');

    idx=0
    for i in range(n):
        for j in range(i+1,n):
            if(edges[i,j,0]==-1 and edges[i,j,1]==-1): continue;
            map_[i,j]  = idx;
            map_[j,i]  = idx;
            invmap[idx,:] = [i,j]
            iM[idx,i] = 1;
            iM[idx,j] = -1;
            bound_edges[idx,0] = edges_count[i,j]<2
            idx=idx+1;
    #print(idx)

    Ael = np.zeros(shape=(n,m),dtype=dtype);
    for i in range(n):
        Ael[i,map_[i,np.nonzero(map_[i,:]+1)]]=1

    Ik  = np.zeros(shape=(m,m),dtype=dtype);
    Ih  = np.zeros(shape=(m,m),dtype=dtype);
    Ik_k  = np.zeros(shape=(m,m,3),dtype=dtype);
    Ih_k  = np.zeros(shape=(m,m,3),dtype=dtype);
    for i in range(n):
        for j in range(i+1,n):
            if(edges[i,j,0]==-1): continue

            k = edges[i,j,0]
            Ik[map_[i,j],map_[i,j]]=-1;
            Ik[map_[i,j],map_[j,k]]=1;
            Ik[map_[i,j],map_[k,i]]=1

            Ik_k[map_[i,j],map_[i,j],0] = 1;
            Ik_k[map_[i,j],map_[j,k],1] = 1;
            Ik_k[map_[i,j],map_[k,i],2] = 1;

            if(edges[i,j,1]==-1): continue

            k = edges[i,j,1]
            Ih[map_[i,j],map_[i,j]]=-1;
            Ih[map_[i,j],map_[j,k]]=1;
            Ih[map_[i,j],map_[k,i]]=1;

            Ih_k[map_[i,j],map_[i,j],0] = 1;
            Ih_k[map_[i,j],map_[j,k],1] = 1;
            Ih_k[map_[i,j],map_[k,i],2] = 1;

    Tni =  np.zeros(shape=(np.shape(TRIV)[0],n),dtype=dtype);
    Tpi =  np.zeros(shape=(np.shape(TRIV)[0],n),dtype=dtype);
    Txi =  np.zeros(shape=(np.shape(TRIV)[0],n),dtype=dtype);
    for i in range(np.shape(TRIV)[0]):
        Tni[i,TRIV[i,0]] = -1;
        Tni[i,TRIV[i,1]] =  1;
        Tpi[i,TRIV[i,0]] =  1;
        Txi[i,TRIV[i,2]] =  1;

    #Windices = np.zeros(shape=(n*n,m),dtype=dtype)
    #for i in range(m):
    #    Windices[invmap[i,0]*n+invmap[i,1],i] = -1;

    Windices = np.zeros(shape=(m,2),dtype=dtype)
    for i in range(m):
        #Windices[i,:] = [invmap[i,0],invmap[i,1]];
        Windices[i,:] = [invmap[i,0]*n+invmap[i,1], i];

    def calc_adj_matrix(VERT,TRIV):
        n = np.shape(VERT)[0]
        A = np.zeros((n,n))
        A[TRIV[:,0],TRIV[:,1]] = 1
        A[TRIV[:,1],TRIV[:,2]] = 1
        A[TRIV[:,2],TRIV[:,0]] = 1
        return A

    A = calc_adj_matrix(VERT, TRIV)
    A = np.matmul(np.diag(1/np.sum(A, axis=1)),A);
    Bary = A - np.eye(np.shape(VERT)[0]);

    return np.asarray(VERT,dtype),TRIV, n, m, Ik, Ih, Ik_k, Ih_k, Tpi, Txi, Tni, iM, Windices, Ael, Bary

max_stars_repo_name: yuyingyeh/isospectralization-pytorch

max_issues_repo_name: yuyingyeh/isospectralization-pytorch

max_forks_repo_name: yuyingyeh/isospectralization-pytorch

reason: import numpy,import scipy,from scipy


text:
import json
import numpy as np

### ---- SCORING FUNCTIONS ---- ###

def bert_scores(dataset):
    raise NotImplementedError

def rarity_scores(dataset, vocab):
    """ Less likely (more rare) sentences are more difficult.
    @returns rarity_scores: scores to be ranked by rank_scores
    """
    train_data, dev_data = dataset
    src_vocab, tgt_vocab = vocab.src, vocab.tgt

    train_scores = []
    for (src_sent, tgt_sent) in train_data:
        src_word_scores, src_sent_score = src_vocab.words2rarity(src_sent)
        tgt_word_scores, tgt_sent_score = tgt_vocab.words2rarity(tgt_sent)
        train_scores.append((src_sent_score, tgt_sent_score))
    dev_scores = []
    for (src_sent, tgt_sent) in dev_data:
        src_word_scores, src_sent_score = src_vocab.words2rarity(src_sent)
        tgt_word_scores, tgt_sent_score = tgt_vocab.words2rarity(tgt_sent)
        dev_scores.append((src_sent_score, tgt_sent_score))

    return (train_scores, dev_scores)

def length_scores(dataset):
    raise NotImplementedError

### -------------------------- ###

def get_difficulty_scores(order_name, dataset, vocab):
    """ Returns the difficulty scores given a specificied ordering function.
    @returns difficulty_scores
    """
    if order_name == "bert":
        raise NotImplementedError
    elif order_name == "rarity":
        scores = rarity_scores(dataset, vocab)
    elif order_name == "length":
        raise NotImplementedError
    else:
        raise ValueError("Order name {} is not supported!".format(order_name))
    return scores

def rank_scores(difficulty_scores):
    """ Ranks the sentences of a dataset according to the difficulty scores.
    @param difficulty_scores
    @returns order_ids (list) a list of ints specifying the ordering
    """
    # TODO: for now, the difficulty is for the *src* sentence only
    train_scores, dev_scores = difficulty_scores
    train_scores = [src_tgt_pair[0] for src_tgt_pair in train_scores]
    dev_scores = [src_tgt_pair[0] for src_tgt_pair in dev_scores]
    train_order_ids = sorted(range(len(train_scores)), key=train_scores.__getitem__)
    dev_order_ids = sorted(range(len(dev_scores)), key=dev_scores.__getitem__)
    return (train_order_ids, dev_order_ids)

def load_order(order_name, dataset, vocab):
    """ Takes in a dataset and returns an ordering (ranking)
    based on a scoring (difficulty) function.
    @param order_name (str): the scoring function to use
    @param dataset (tuple): a tuple of lists (train_data, dev_data)
    @param vocab (Vocab): a Vocab object containing (vocab.src, vocab.tgt)
    @returns order (Dataset): an ordered dataset of (train_data, dev_data)
        in increasing order (easy difficulty to hard difficulty)
    """
    orderings = ["none", "bert", "rarity", "length"]
    print("order name is:", order_name)
    if order_name not in orderings:
        raise ValueError("Order name {} is not supported!".format(order_name))

    if order_name == "none":
        return dataset

    difficulty_scores = get_difficulty_scores(order_name, dataset, vocab)
    order_ids = rank_scores(difficulty_scores)

    # Sort the dataset according to the ordering
    train_order_ids, dev_order_ids = order_ids
    train_data, dev_data = dataset
    ordered_train = [train_data[i] for i in train_order_ids]
    ordered_dev = [dev_data[i] for i in dev_order_ids]
    return (ordered_train, ordered_dev)

def balance_order(order, dataset):
    """ Ensures that the ordered dataset according to difficulty
    is balanced, i.e. each class has equal number of samples.
    """
    pass

def visualize_scoring(ordered_dataset, vocab):
    """ Visualizes difficulty of sentences.
    @param ordered_dataset (Dataset): dataset of (train_data, dev_data)
    @param vocab (Vocab): Vocabulary with vocab.src and vocab.tgt
    """
    train_data, dev_data = ordered_dataset
    src_vocab, tgt_vocab = vocab.src, vocab.tgt
    print("Creating scoring visualizations.")
    num_easy = 1
    skip = 30
    num_medium = 1
    num_hard = 1

    easy_examples = train_data[skip:skip+num_easy]
    medium_examples = train_data[skip + int(len(train_data) / 2):skip + int(len(train_data) / 2) + num_medium]
    hard_examples = train_data[-num_hard:]

    print("\n***** [ Easy examples ] *****")
    print("Source:\n", " ".join(easy_examples[0][0]))
    print("Target:\n", " ".join(easy_examples[0][1]))
    src_word_scores, src_sent_score = src_vocab.words2rarity(easy_examples[0][0])
    tgt_word_scores, tgt_sent_score = tgt_vocab.words2rarity(easy_examples[0][1])
    print("src_word_scores:", src_word_scores)
    print("src_sent_score:", src_sent_score)
    print("tgt_word_scores:", tgt_word_scores)
    print("tgt_sent_score:", tgt_sent_score)

    print("\n***** [ Medium examples ] *****")
    print("Source:\n", " ".join(medium_examples[0][0]))
    print("Target:\n", " ".join(medium_examples[0][1]))
    src_word_scores, src_sent_score = src_vocab.words2rarity(medium_examples[0][0])
    tgt_word_scores, tgt_sent_score = tgt_vocab.words2rarity(medium_examples[0][1])
    print("src_word_scores:", src_word_scores)
    print("src_sent_score:", src_sent_score)
    print("tgt_word_scores:", tgt_word_scores)
    print("tgt_sent_score:", tgt_sent_score)

    print("\n***** [ Hard examples ] *****")
    print("Source:\n", " ".join(hard_examples[0][0]))
    print("Target:\n", " ".join(hard_examples[0][1]))
    src_word_scores, src_sent_score = src_vocab.words2rarity(hard_examples[0][0])
    tgt_word_scores, tgt_sent_score = tgt_vocab.words2rarity(hard_examples[0][1])
    print("src_word_scores:", src_word_scores)
    print("src_sent_score:", src_sent_score)
    print("tgt_word_scores:", tgt_word_scores)
    print("tgt_sent_score:", tgt_sent_score)