text:
# -*- coding: utf-8 -*-
"""
Created on Tue Jul 17 18:38:40 2018

@author: badat
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import pandas as pd
import os.path
import os
import numpy as np
import time
from average_precision import apk
import global_setting_OpenImage
import D_utility
#%% logging level
tf.logging.set_verbosity(tf.logging.INFO)
#%% override
#global_setting_OpenImage.learning_rate_base = 0.001
global_setting_OpenImage.batch_size=32
global_setting_OpenImage.n_cycles*=1#60*global_setting_OpenImage.report_interval
global_setting_OpenImage.report_interval = 100
global_setting_OpenImage.n_cycles = 3657355//global_setting_OpenImage.batch_size
#%% data flag
idx_GPU=7
beta = 0.02
os.environ["CUDA_VISIBLE_DEVICES"]="{}".format(idx_GPU)
df_image = pd.read_csv('./data/2017_11/train/images.csv')

list_alphas = [0,0.001,0.01,0.1,1]

global_step = tf.Variable(0, trainable=False,dtype=tf.float32)
learning_rate = 1.0/(tf.sqrt(global_step)+1.0)*global_setting_OpenImage.learning_rate_base#tf.Variable(global_setting_OpenImage.learning_rate_base,trainable = False,dtype=tf.float32)

n_iters = 1

schedule_wrt_report_interval = 80

name = 'baseline_logistic_OpenImages'
save_name = name+'.csv'
df_result = pd.DataFrame()
is_save = False
global_setting_OpenImage.report_interval = 100
#n_process_map = 4
#%%
print('number of cycles {}'.format(global_setting_OpenImage.n_cycles))
#%% label mapping function
def LoadLabelMap(labelmap_path, dict_path):
  """Load index->mid and mid->display name maps.

  Args:
    labelmap_path: path to the file with the list of mids, describing
        predictions.
    dict_path: path to the dict.csv that translates from mids to display names.
  Returns:
    labelmap: an index to mid list
    label_dict: mid to display name dictionary
  """
  labelmap = [line.rstrip() for line in tf.gfile.GFile(labelmap_path)]

  label_dict = {}
  for line in tf.gfile.GFile(dict_path):
    words = [word.strip(' "\n') for word in line.split(',', 1)]
    label_dict[words[0]] = words[1]

  return labelmap, label_dict

predictions_eval = 0
predictions_eval_resize = 0
#%%
labelmap, label_dict = LoadLabelMap(global_setting_OpenImage.labelmap_path, global_setting_OpenImage.dict_path)
list_label = []
for id_name in labelmap:
    list_label.append(label_dict[id_name])

#%% Dataset

def parser(record):
    feature = {'img_id': tf.FixedLenFeature([], tf.string),
               'feature': tf.FixedLenFeature([], tf.string),
               'label': tf.FixedLenFeature([], tf.string)}

    parsed = tf.parse_single_example(record, feature)
    img_id = parsed['img_id']
    feature = tf.decode_raw( parsed['feature'],tf.float32)
    label = tf.decode_raw( parsed['label'],tf.int32)
    return img_id,feature,label

#%% load in memory
sess = tf.InteractiveSession()#tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))
g = tf.get_default_graph()
#with g.as_default():
saver = tf.train.import_meta_graph(global_setting_OpenImage.model_path+ '.meta')
saver.restore(sess, global_setting_OpenImage.model_path)
weight = tf.squeeze( g.get_tensor_by_name('resnet_v1_101/logits/weights:0'))
bias =  g.get_tensor_by_name('resnet_v1_101/logits/biases:0')[tf.newaxis,:]
init_Theta = tf.concat([weight,bias],name='Theta',axis = 0).eval()
#%%
Theta = tf.get_variable('Theta',shape=[2049,5000])
#%%
dataset = tf.data.TFRecordDataset(global_setting_OpenImage.record_path)
dataset = dataset.map(parser)
dataset = dataset.shuffle(20000)
dataset = dataset.batch(global_setting_OpenImage.batch_size)
dataset = dataset.repeat()
iterator = dataset.make_initializable_iterator()
(img_ids,img_features,labels) = iterator.get_next()

dataset_in = tf.data.TFRecordDataset(global_setting_OpenImage.validation_path)
dataset_in = dataset_in.map(parser).batch(50000)
(img_val_ids,F_val,val_labels) = dataset_in.make_one_shot_iterator().get_next()
F_val = tf.concat([F_val,tf.ones([tf.shape(F_val)[0],1])],axis = 1)
(img_val_ids,F_val,val_labels)=sess.run([img_val_ids,F_val,val_labels])

#%%
def AP(prediction,label,names):
    mask = np.abs(label)==1
    if np.sum(label==1)==0:
        return 0.0
    groundtruth = names[label == 1]
    prediction = prediction[mask]
    retrieval = names[mask]
    sort_idx = np.argsort(prediction)[::-1]
    retrieval = retrieval[sort_idx]
    return apk(groundtruth,retrieval,len(prediction))

def compute_AP(Prediction,Label,names):
    num_class = Prediction.shape[1]
    ap=np.zeros(num_class)
    for idx_cls in range(num_class):
        prediction = Prediction[:,idx_cls]
        label = Label[:,idx_cls]
        ap[idx_cls]=AP(prediction,label,names)
    return ap
#%%
F = tf.squeeze(img_features)
F = tf.concat([F,tf.ones([tf.shape(F)[0],1])],axis = 1)

#%%
alpha_regularizer_var = tf.get_variable('alpha_regularizer',dtype=tf.float32,trainable=False, shape=())
alpha_regularizer_var_fh = tf.placeholder(dtype=tf.float32, shape=())
#%%
op_alpha_regularizer = alpha_regularizer_var.assign(alpha_regularizer_var_fh)
#%%
G = np.load('./label_graph/graph_label_naive.npy').astype(np.float32)
G=D_utility.preprocessing_graph(G)
G_empty_diag = G - np.diag(np.diag(G))
G_init=G_empty_diag[G_empty_diag!=0]
G_var = tf.get_variable("G_var", G_init.shape)
op_G_var=G_var.assign(G_init)
indices = []
#updates = []
shape = tf.constant(G.shape)
counter = 0

diag_G = tf.diag(np.diag(G))

for idx_row in range(G_empty_diag.shape[1]):
    idx_cols = np.where(G_empty_diag[idx_row,:]!=0)[0]
    for idx_col in idx_cols:
        if G[idx_row,idx_col]-G_init[counter] != 0:
            raise Exception('error relation construction')
        if idx_row != idx_col:
            indices.append([idx_row,idx_col])
        counter += 1
part_G_var = tf.scatter_nd(indices, G_var, shape)+diag_G
#%% disperse measurement
dispersion=tf.reduce_sum(tf.abs(part_G_var)) - tf.reduce_sum(tf.diag_part(tf.abs(part_G_var)))
#%%

with tf.variable_scope("logistic"):
    logits = tf.matmul(F,Theta)
    labels_binary = tf.div(labels+1,2)
    labels_weight = tf.abs(labels)
    loss_logistic = tf.losses.sigmoid_cross_entropy(multi_class_labels=labels_binary, logits=logits,weights=labels_weight)

with tf.variable_scope("regularizer"):
    loss_regularizer = tf.square(tf.norm(Theta))
#%% shared operation
grad_logistic = tf.gradients(loss_logistic, Theta)
grad_regularizer = tf.gradients(loss_regularizer,Theta)

norm_grad_logistic = tf.norm(grad_logistic)
norm_grad_regularizer = tf.norm(grad_regularizer)
norm_Theta = tf.norm(Theta)
raitio_regularizer_grad = norm_grad_logistic/norm_grad_regularizer

validate_Prediction = tf.matmul(F_val,Theta)
#%%
tf.global_variables_initializer().run()
sess.run(iterator.initializer)
#%%
def append_info(m_AP,loss_value,lr_v):

    res_mAP[index]=m_AP
    res_loss[index] = loss_value
    res_lr[index]=lr_v

    df_result['mAP: regularizer {}'.format(alpha_regularizer)]=res_mAP
    df_result['loss: regularizer {}'.format(alpha_regularizer)]=res_loss
    df_result['lr: regularizer {}'.format(alpha_regularizer)]=res_lr

#%%
Theta_fh = tf.placeholder(dtype=tf.float32, shape=[2049,5000])
op_assign_Theta = Theta.assign(Theta_fh)
global_step_fh=tf.placeholder(dtype=tf.float32,shape=())
op_assign_global_step = global_step.assign(global_step_fh)

#%%
tf.global_variables_initializer().run()
sess.run(op_G_var)
sess.run(op_assign_Theta,{Theta_fh:init_Theta})
sess.run(iterator.initializer)

#%%
#optimizer = tf.train.AdamOptimizer(learning_rate=global_setting_OpenImage.learning_rate_base)#tf.train.RMSPropOptimizer(learning_rate=learning_rate)#,momentum=0.9
optimizer = tf.train.RMSPropOptimizer(
      learning_rate,
      0.9,  # decay
      0.9,  # momentum
      1.0   #rmsprop_epsilon
  )
loss = loss_logistic
#%% hypergradient
grad_loss = tf.gradients(loss, Theta)
#%%
train = optimizer.minimize(loss,var_list=[Theta],global_step = global_step)
print('done placeholder assignment')
def experiment_cond_success():
    return alpha_colaborative_o == 0.1 and alpha_feature_o ==0

n_experiment= 0

for idx_alpha_colaborative,alpha_colaborative_o in enumerate(list_alphas):
    for idx_alpha_feature,alpha_feature_o in enumerate(list_alphas):
        for idx_alpha_regularizer,alpha_regularizer_o in enumerate([0]):
            if not experiment_cond_success():#index_column <= 4:#(idx_alpha_colaborative == 0 and idx_alpha_feature != 1) or idx_alpha_regularizer != 0 or
                print('skip')
                continue
            n_experiment += 1
print('Total number of experiment: {}'.format(n_experiment))

Thetas = np.zeros((2049,5000,n_experiment))
Gs = np.zeros((5000,5000,n_experiment))
idx_experiment = 0
expon_moving_avg_old = np.inf
expon_moving_avg_new = 0
for idx_alpha_colaborative,alpha_colaborative_o in enumerate(list_alphas):
    for idx_alpha_feature,alpha_feature_o in enumerate(list_alphas):
        for idx_alpha_regularizer,alpha_regularizer_o in enumerate([0]):
            index_column = idx_alpha_colaborative*len(list_alphas)*len(list_alphas)+idx_alpha_feature*len(list_alphas)+idx_alpha_regularizer

            if not experiment_cond_success():#index_column <= 4:#(idx_alpha_colaborative == 0 and idx_alpha_feature != 1) or idx_alpha_regularizer != 0 or
                print('skip')
                continue

            report_length = global_setting_OpenImage.n_cycles*n_iters//global_setting_OpenImage.report_interval +1 #in case that my lousy computation is wrong
            print('report length {}'.format(report_length))
            res_mAP = np.zeros(report_length)
            res_loss = np.zeros(report_length)
            res_loss_logistic=np.zeros(report_length)
            res_loss_R=np.zeros(report_length)
            res_loss_feature=np.zeros(report_length)
            res_grad_logistic=np.zeros(report_length)
            res_grad_R=np.zeros(report_length)
            res_lr=np.zeros(report_length)
            #loss_R#
            sess.run(iterator.initializer)
            alpha_regularizer = alpha_regularizer_o

            tf.global_variables_initializer().run()
            print('reset Theta')
            sess.run(op_G_var)
            sess.run(op_assign_Theta,{Theta_fh:init_Theta})
            sess.run(op_alpha_regularizer,{alpha_regularizer_var_fh:alpha_regularizer})

            df_ap = pd.DataFrame()
            df_ap['label']=list_label
            #%%
            tic = time.clock()
            for idx_cycle in range(global_setting_OpenImage.n_cycles):
                _,loss_value,lr_v  = sess.run([train,loss,learning_rate])

                index = (idx_cycle*n_iters)//global_setting_OpenImage.report_interval

                if (idx_cycle*n_iters) % global_setting_OpenImage.report_interval == 0 :#or idx_iter == n_iters-1:
                    print('Elapsed time udapte: {}'.format(time.clock()-tic))
                    tic = time.clock()
                    print('index {} -- compute mAP'.format(index))
                    print('Loss {} lr {}'.format(loss_value,lr_v))
                    validate_Prediction_val = validate_Prediction.eval()
                    ap = compute_AP(validate_Prediction_val,val_labels,img_val_ids)
                    df_ap['index {}'.format(index)]=ap
                    m_AP=np.mean(ap)
#
                    append_info(m_AP,loss_value,lr_v)
                    print('mAP {}'.format(m_AP))
                    if is_save:
                        Thetas[:,:,idx_experiment]=Theta.eval()
                        Gs[:,:,idx_experiment]=part_G_var.eval()
                        df_result.to_csv('./result/'+save_name)
                        ap_save_name = './result/baseline_ap_{}.csv'
                        df_ap.to_csv(ap_save_name.format(alpha_regularizer))
                        if index%(int(report_length/4)) == 0:
                            np.savez('./result/'+name, Thetas=Thetas, Gs=Gs)
            idx_experiment+=1
if is_save:
    np.savez('./result/'+name, Thetas=Thetas, Gs=Gs)
#%%
sess.close()
tf.reset_default_graph()


text:
"""
.. role:: bash(code)
    :language: bash

This is the test module for the SNR evaluation of the reconstructed signal.
|br|

It tests the SNR evaluator with the number of test cases,
and analyzes the results.

In every case random multitone signals are generated with a specified
level of noise. The Random Multitone Signal Generator is used to generate
the signals. The non noisy signal from the generator is treated as the orignal
signal, the noisy signal is treated as the reconstructed signal. |br|

The following tests are performed on the results of evaluation:

- if the measured level of noise for every signal is correct?

- if the measured average level of noise is correct?

- if the measured success ratio of the reconsteruction is correct?

To start the test run this module directly as a script:

    :bash:`$ python SNR_test.py`

when in *rxcs/test/analysis* directory.
The results are then printed to the console.
|br|

*Author*:
    Jacek Pierzchlewski, Aalborg University, Denmark. <jap@es.aau.dk>

*Version*:
    0.1  | 20-MAY-2014 : * Initial version. |br|
    0.2  | 21-MAY-2014 : * Success Ratio check added. |br|
    0.3  | 21-MAY-2014 : * Docstrings added. |br|
    0.4  | 21-MAY-2014 : * UUT now configured with a dicionary. |br|
    1.0  | 21-MAY-2014 : * Version 1.0 released. |br|
    2.0  | 21-AUG-2015 : * Adjusted to SNR analysis v2.0 (objectified) |br|

*License*:
    BSD 2-Clause
"""
from __future__ import division
import numpy as np
import rxcs

# =====================================================================
# Main function of the test
# =====================================================================
def _SNR_test():
    """
    This is main function of the test.

    It runs the test case functions.
    An info that a test was passed is printed to the console if a case
    function returns.

    Args:
        None

    Returns:
        Nothing
    """

    # Print out the header of the signal generator test
    print('')
    rxcs.console.progress('Function under test',
                          'SNR evaluation of the reconstructed signal')

    # -----------------------------------------------------------------
    # Tests start here:

    # Tolerance of expected vs measured is 0.1%
    iTolerance = 0.1 * 1e-2

    _TestCase1(iTolerance)
    rxcs.console.info('case 1 OK!')

    _TestCase2(iTolerance)
    rxcs.console.info('case 2 OK!')

    _TestCase3(iTolerance)
    rxcs.console.info('case 3 OK!')

    _TestCase4(iTolerance)
    rxcs.console.info('case 4 OK!')

    _TestCase5(iTolerance)
    rxcs.console.info('case 5 OK!')

# =====================================================================
# Test case 1
# =====================================================================
def _TestCase1(iTolerance):
    """
    This is test case function #1. |br|

    The function sets up the configuration dictionary for the Random Multitone
    Signal Generator and runs the engine of the test.

    Args:
        iTolerance: maximum tolerance of a difference between an expected value
        and a real value

    Returns:
        Nothing
    """
    # Put the generator on board
    gen = rxcs.sig.randMult()

    gen.tS = 1e-3   # Time of the signal is 1 ms
    gen.fR = 2e4    # The signal representation sampling frequency is 20 kHz
    gen.fMax = 8e3  # The highest possible frequency in the signal is 8 kHz
    gen.fRes = 1e3  # The signal spectrum resolution is 1 kHz

    gen.iSNR = 30   # Signal noise

    gen.iP = 2      # Signal power adjustments

    gen.vFrqs = np.array([2e3, np.nan, 4e3])   # Vector with given frequencies
    gen.vAmps = np.array([np.nan, 1, np.nan])  # Vector with given amplitudes
    gen.vPhs = np.array([np.nan, np.nan, 90])  # Vector with given phases

    # The number of random tones
    gen.nTones = 2

    # Amplitude and phase parameters of ranimd tones:
    gen.iMinAmp = 0.1  # Minimum amplitude
    gen.iGraAmp = 0.1  # Gradation of amplitude
    gen.iMaxAmp = 1.0  # Maximum amplitude

    gen.iMinPhs = 0   # Minimum phase of additional tones
    gen.iGraPhs = 2   # Gradation of phase of additional tones
    gen.iMaxPhs = 90  # Maximum phase of additional tones

    gen.nSigs = int(1e5)  # The number of signals to be generated

    gen.bMute = 1   # Mute the output from the gneerator

    # -----------------------------------------------------------------
    # ENGINE OF THE TEST:
    # Generate signals with a given SNR, measure the SNR and check if
    # the measured value is correct
    _checkSNR(gen, iTolerance)

    return

# =====================================================================
# Test case 2
# =====================================================================
def _TestCase2(iTolerance):
    """
    This is test case function #2. |br|

    The function sets up the configuration dictionary for the Random Multitone
    Signal Generator and runs the engine of the test.

    Args:
        iTolerance: maximum tolerance of a difference between an expected value
        and a real value

    Returns:
        Nothing
    """

    # Put the generator on board
    gen = rxcs.sig.randMult()

    gen.tS = 1e-1     # Time of the signal is 100 ms
    gen.fR = 1e6      # The signal representation sampling frequency is 1 MHz
    gen.fMax = 100e3  # The highest possible frequency in the signal is 100 kHz
    gen.fRes = 1e3    # The signal spectrum resolution is 1 kHz

    gen.iSNR = 100    # Signal noise

    gen.iP = 10      # Signal power adjustments

    gen.vFrqs = np.array([1e3])   # Vector with given frequencies
    gen.vAmps = np.array([0.5])   # Vector with given amplitudes
    gen.vPhs = np.array([180])    # Vector with given phases

    # The number of random tones
    gen.nTones = 10

    # Amplitude and phase parameters of ranimd tones:
    gen.iMinAmp = 1.0   # Minimum amplitude
    gen.iGraAmp = 0.1   # Gradation of amplitude
    gen.iMaxAmp = 10.0  # Maximum amplitude

    gen.nSigs = int(1e3)  # The number of signals to be generated

    gen.bMute = 1   # Mute the output from the gneerator

    # -----------------------------------------------------------------
    # ENGINE OF THE TEST:
    # Generate signals with a given SNR, measure the SNR and check if
    # the measured value is correct
    _checkSNR(gen, iTolerance)

    return

# =====================================================================
# Test case 3
# =====================================================================
def _TestCase3(iTolerance):
    """
    This is test case function #3. |br|

    The function sets up the configuration dictionary for the Random Multitone
    Signal Generator and runs the engine of the test.

    Args:
        iTolerance: maximum tolerance of a difference between an expected value
        and a real value

    Returns:
        Nothing
    """

    # Put the generator on board
    gen = rxcs.sig.randMult()

    gen.tS = 1       # Time of the signal is 1 s
    gen.fR = 1e6     # The signal representation sampling frequency is 1 MHz
    gen.fMax = 20e3  # The highest possible frequency in the signal is 20 kHz
    gen.fRes = 1     # The signal spectrum resolution is 1 Hz

    gen.iSNR = 5   # Signal noise

    gen.vFrqs = np.array([1e3, np.nan, np.nan])   # Vector with given frequencies
    gen.vAmps = np.array([0.5, 1, 10])            # Vector with given amplitudes
    gen.vPhs = np.array([78, np.nan, np.nan])     # Vector with given phases

    # The number of random tones
    gen.nTones = 20

    # Amplitude and phase parameters of ranimd tones:
    gen.iMinAmp = 5.0   # Minimum amplitude
    gen.iGraAmp = 0.1   # Gradation of amplitude
    gen.iMaxAmp = 10.0  # Maximum amplitude

    gen.iMinPhs = 1   # Minimum phase of additional tones
    gen.iGraPhs = 1   # Gradation of phase of additional tones
    gen.iMaxPhs = 20  # Maximum phase of additional tones

    gen.nSigs = int(1e2)  # The number of signals to be generated

    gen.bMute = 1   # Mute the output from the gneerator

    # -----------------------------------------------------------------
    # ENGINE OF THE TEST:
    # Generate signals with a given SNR, measure the SNR and check if
    # the measured value is correct
    _checkSNR(gen, iTolerance)

    return

# =====================================================================
# Test case 4
# =====================================================================
def _TestCase4(iTolerance):
    """
    This is test case function #4. |br|

    The function sets up the configuration dictionary for the Random Multitone
    Signal Generator and runs the engine of the test.

    Args:
        iTolerance: maximum tolerance of a difference between an expected value
        and a real value

    Returns:
        Nothing
    """

    # Put the generator on board
    gen = rxcs.sig.randMult()

    gen.tS = 1e-6    # Time of the signal is 1 us
    gen.fR = 1e9     # The signal representation sampling frequency is 1 GHz
    gen.fMax = 50e6  # The highest possible frequency in the signal is 50 MHz
    gen.fRes = 1e6   # The signal spectrum resolution is 1 MHz

    gen.iSNR = 45   # Signal noise

    gen.iP = 0.1      # Signal power adjustments

    gen.vFrqs = np.array([10e6, 20e6, 30e6, 40e6])   # Vector with given frequencies
    gen.vAmps = np.array([1.0, 1, 10, 20])   # Vector with given amplitudes
    gen.vPhs = np.array([78, 10, -50, -40])  # Vector with given phases

    # The number of random tones
    gen.nTones = 10

    # Amplitude and phase parameters of ranimd tones:
    gen.iMinAmp = 0.1  # Minimum amplitude
    gen.iGraAmp = 0.01  # Gradation of amplitude
    gen.iMaxAmp = 10.0  # Maximum amplitude

    gen.iMinPhs = -45   # Minimum phase of additional tones
    gen.iGraPhs = 1     # Gradation of phase of additional tones
    gen.iMaxPhs = 45    # Maximum phase of additional tones

    gen.nSigs = int(1e4)  # The number of signals to be generated

    gen.bMute = 1   # Mute the output from the gneerator

    # -----------------------------------------------------------------
    # ENGINE OF THE TEST:
    # Generate signals with a given SNR, measure the SNR and check if
    # the measured value is correct
    _checkSNR(gen, iTolerance)

    return

# =====================================================================
# Test case 5
# =====================================================================
def _TestCase5(iTolerance):
    """
    This is test case function #5. |br|

    The function sets up the configuration dictionary for the Random Multitone
    Signal Generator and runs the engine of the test.

    Args:
        iTolerance: maximum tolerance of a difference between an expected value
        and a real value

    Returns:
        Nothing
    """

    # Put the generator on board
    gen = rxcs.sig.randMult()

    gen.tS = 1e-6    # Time of the signal is 1 us
    gen.fR = 1e9     # The signal representation sampling frequency is 1 GHz
    gen.fMax = 50e6  # The highest possible frequency in the signal is 50 MHz
    gen.fRes = 1e6   # The signal spectrum resolution is 1 MHz

    gen.iSNR = 12   # Signal noise

    # The number of random tones
    gen.nTones = 10

    # Amplitude and phase parameters of ranimd tones:
    gen.iMinAmp = 0.1   # Minimum amplitude
    gen.iGraAmp = 0.01  # Gradation of amplitude
    gen.iMaxAmp = 10.0  # Maximum amplitude

    gen.iMinPhs = -45   # Minimum phase of additional tones
    gen.iGraPhs = 1     # Gradation of phase of additional tones
    gen.iMaxPhs = 45    # Maximum phase of additional tones

    gen.nSigs = int(1e4)  # The number of signals to be generated

    gen.bMute = 1   # Mute the output from the gneerator

    # -----------------------------------------------------------------
    # ENGINE OF THE TEST:
    # Generate signals with a given SNR, measure the SNR and check if
    # the measured value is correct
    _checkSNR(gen, iTolerance)

    return

# =====================================================================
# ENGINE OF THE TEST: Generate signals with a given SNR, measure the SNR and
#                     check it
# =====================================================================
def _checkSNR(gen, iTolerance):
    """
    This is the engine of the test.

    The function runs the signal generator, measure the SNR using the tested
    SNR evaluation and tests the measured SNR values for every signal and
    the average level of SNR. Is also checks the reported Success Ratio.

    Args:
        gen:  The Random Multitone Signal Generator
        iTolerance (float): maximum tolerance of a difference between an
                            expected value and a real value
    Returns:
        nothing

    """
    # Succcess ratio if SNR > 20 [db]
    iSNRSuccess = 20

    # -----------------------------------------------------------------
    # Run the multtone signal generator
    tStart = \
        rxcs.console.module_progress('signals generation')
    gen.run()
    rxcs.console.progress_done(tStart)

    # -----------------------------------------------------------------
    # Run the SNR evaluation
    analysisSNR = rxcs.ana.SNR()

    analysisSNR.mSigRef = gen.mSigNN        # Nonnoisy signal from the generator is a a reference signal
    analysisSNR.mSig = gen.mSig             # Generated signal is a signal under test
    analysisSNR.iSNRSuccess = iSNRSuccess
    analysisSNR.bMute = 1
    tStart = rxcs.console.module_progress('SNR computation')
    analysisSNR.run()
    rxcs.console.progress_doneNL(tStart)

    # -----------------------------------------------------------------
    # Check the measured SNR vs the reported orignal SNR

    # Get the number of signals
    nSigPack = gen.nSigs

    # Get the vector with measured SNR values
    vSNR = analysisSNR.vSNR

    # Check the vector with measured SNR values
    for inxSig in np.arange(nSigPack):
        iSNR_meas = vSNR[inxSig]
        if not _isequal(gen.iSNR, iSNR_meas, iTolerance):
            raise Exception('measured SNR: error!!!')

    # Check the average SNR
    iSNR_avg = analysisSNR.iSNR
    if not _isequal(gen.iSNR, iSNR_avg, iTolerance):
        raise Exception('measured average SNR: error!!!')

    # -----------------------------------------------------------------
    # Check the success ratio

    # The expected success ratio
    if gen.iSNR >= iSNRSuccess:
        iSR_expected = 1
    else:
        iSR_expected = 0

    # Check the expected success ratio
    iSR = analysisSNR.iSR  # Measured success ratio
    if not _isequal(iSR_expected, iSR, iTolerance):
        raise Exception('measured SNR success ratio: error!!!')

    # -----------------------------------------------------------------
    return

# =====================================================================
# This function compares two values.
# The function allows for a very small error margin
# =====================================================================
def _isequal(iX, iY, iMargin):
    """
    This function checks if a difference between two values is in the
    given allowed margin.

    Args:
        iX: the first value |br|
        iY: the second value |br|
        iMargin: the allowed margin |br|

    Returns:
        1: if the difference between the given values does not exceed the
           given margin |br|
        0: if the difference between the given values does exceed the
           given margin |br|
    """

    if (abs(iX - iY) <= np.abs(iMargin)):
        return 1
    else:
        return 0

# =====================================================================
# Trigger when start as a script
# =====================================================================
if __name__ == '__main__':
    _SNR_test()

ext: py

max_stars_repo_name: JacekPierzchlewski/RxCS

max_issues_repo_name: JacekPierzchlewski/RxCS

max_forks_repo_name: JacekPierzchlewski/RxCS

reason: import numpy


text:
""" Handles DS Prediction Model """
from .twitter import vectorize_tweet
import numpy as np
from .model import User
from sklearn.linear_model import LogisticRegression
# user0 = 'elonmusk'
# user1 = 'JackBlack'

log_reg = LogisticRegression()

def predict_user(user0_name, user1_name, hypo_text):
    """ Which user is more likely to have tweeted a tweet?

    Returns 0 for user0_name and 1 for user1_name

     """

    user0 = User.query.filter(User.username == user0_name).one()
    user1 = User.query.filter(User.username == user1_name).one()

    user0_vectors = np.array([tweet.vect for tweet in user0.tweets])
    user1_vectors = np.array([tweet.vect for tweet in user1.tweets])

    vectors = np.vstack([user0_vectors, user1_vectors])

    labels = np.concatenate(
        [np.zeros(len(user0.tweets)), np.ones(len(user1.tweets))])

    log_reg = LogisticRegression().fit(vectors, labels)

    hypo_tweet_vect = vectorize_tweet(hypo_text).reshape(1,-1)

    return log_reg.predict(hypo_tweet_vect)

    user_test = User.query.filter(User.username == "elonmusk").one()
    print(user_test)

max_stars_repo_name: yaobviously/twitterapp

max_issues_repo_name: yaobviously/twitterapp

max_forks_repo_name: yaobviously/twitterapp


text:
import scipy as sc
from scipy.misc import imread
import numpy as np
import pickle
import os
import cv2
import operator
# filename = 'get-face/faces/face_laurynas_0.jpeg'
#
# im = imread(filename, mode='RGB')
# print(im.shape)
# print(im)
#
missed = 0
cascPath = "get_face/haarcascade_frontalface_default.xml"
faceCascade = cv2.CascadeClassifier(cascPath)

images = []
labels = []

targets = []
idv = 0
for root, d, files in os.walk('lfw'):
    for file in files:
        if '.jpg' in file:
            print(os.path.join(root, file))
            targets.append(os.path.join(root, file))

for file in os.listdir('get_face/faces'):
    if '.jpg' in file:
        print(os.path.join('get_face/faces', file))
        targets.append(os.path.join('get_face/faces', file))

for target in targets:
    im = imread(target, mode='RGB')
    label = int('laurynas' in target)
    print(target, label)
    if im.shape != (224,224,3):
        gray = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)
        faces = faceCascade.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30)
            # flags = cv2.CV_HAAR_SCALE_IMAGE
        )
        if len(faces)>0:
            (x, y, w, h) = max(faces, key=(lambda f: operator.itemgetter(2)(f)))
            im = cv2.resize(im[y:y + h, x:x + w], (224, 224), 0, 0, cv2.INTER_LANCZOS4)
            # im = cv2.cvtColor(subface, cv2.COLOR_BGR2RGB)
        else:
            missed += 1
            continue
            # raise Exception("Faceless image")
    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
    # cv2.imshow("Face", im)
    # cv2.waitKey(1)
    images.append(im)
    labels.append(label)
    if label:
        cv2.imwrite('faces/one/{}.jpg'.format(idv), im)
    else:
        cv2.imwrite('faces/zero/{}.jpg'.format(idv), im)
    idv +=1

print(missed, len(targets))

# X = np.array(images)
# Y = np.array(labels)
# obj= X,Y
# print("Writing")
# np.savez('dataset_properX.npz', X)
# np.savez('dataset_properY.npz', Y)

max_stars_repo_name: PetarV-/facejack

max_issues_repo_name: PetarV-/facejack

max_forks_repo_name: PetarV-/facejack

reason: import numpy,import scipy,from scipy


text:
import numpy as np

y_true = [int(x) for x in input().split()]
y_pred =  [int(x) for x in input().split()]
tp, tn, fp, fn = 0, 0, 0, 0
for i in range(len(y_true)):
    if y_true[i] == 0 and y_true[i]==y_pred[i]:
        tn += 1
    elif y_true[i] == 0 and y_true[i]!=y_pred[i]:
        fp += 1
    elif y_true[i] == 1 and y_true[i]==y_pred[i]:
        tp += 1
    else:
        fn += 1
print(np.vstack([[float(tp), float(fp)],[float(fn), float(tn)]]))

max_stars_repo_path: Regular/Binary Disorder.py

max_issues_repo_path: Regular/Binary Disorder.py

max_forks_repo_path: Regular/Binary Disorder.py


text:
import sys,os

from numpy.core.fromnumeric import shape
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

import os
import trimesh
import numpy as np
import json
import open3d as o3d
import torch
from tqdm import tqdm
from timeit import default_timer as timer
import utils.deepsdf_utils as deepsdf_utils
import matplotlib.pyplot as plt

from models.pose_decoder import PoseDecoder, PoseDecoderSE3
from models.shape_decoder import ShapeDecoder

from utils.pcd_utils import (BBox,
                            trimesh_to_open3d,
                            transform_pointcloud_to_opengl_coords,
                            rotate_around_axis,
                            origin, normalize_transformation)

import utils.nnutils as nnutils
import config as cfg
import data_scripts.config_data as cfg_data

class ViewerShapeTransfer:
    def __init__(
        self,
        labels,
        labels_tpose,
        shape_codes,
        pose_codes,
        out_root_dir,
        dataset_class=None,
        source_identity=None, # we'll use the shape code from this identity
        target_identity=None, # we'll use the pose code from this identity
        target_identity_animation=None,
        target_identity_sample=None,
        use_gt_tpose_mesh=False,
        render_video_options="/rhome/ppalafox/workspace/render_video_options",
    ):
        mult = 2
        self.reconstruction_res = 256 * mult
        self.max_batch = (mult * 32)**3

        self.labels = labels
        self.labels_tpose = labels_tpose
        self.num_to_eval = num_to_eval
        self.src_identity = source_identity
        self.tgt_identity = target_identity
        self.target_identity_animation = target_identity_animation
        self.target_identity_sample = target_identity_sample

        self.shape_codes = shape_codes
        self.pose_codes = pose_codes

        self.dataset_class = dataset_class

        self.src_ref = None
        self.tgt_ref = None
        self.src_refw = None
        self.tgt_refw = None

        self.show_src_ref = False
        self.show_tgt_ref = False
        self.show_src_refw = False
        self.show_tgt_refw = False

        # Find the source and target identity's id
        self.src_identity_id, self.tgt_identity_id = None, None
        for identity_id, label_tpose in enumerate(self.labels_tpose):
            if label_tpose['identity_name'] == self.src_identity:
                self.src_identity_id = identity_id
            if label_tpose['identity_name'] == self.tgt_identity:
                self.tgt_identity_id = identity_id
        assert self.src_identity_id is not None and self.tgt_identity_id is not None

        # Find the index of the tgt sample_id
        self.frame_idx = None
        for pose_idx, label in enumerate(self.labels):
            if label['identity_name' ] == self.tgt_identity \
            and label['animation_name'] == self.target_identity_animation \
            and label['sample_id'] == self.target_identity_sample:
                self.frame_idx = pose_idx
        assert self.frame_idx is not None

        # print(self.src_identity_id, self.tgt_identity_id, self.frame_idx)

        self.SRC_REFW_COLOR = SRC_REFW_COLOR
        self.TGT_REFW_COLOR = TGT_REFW_COLOR

        self.use_gt_tpose_mesh = use_gt_tpose_mesh

        # Recording options
        self.view = "frontal"
        self.render_json            = os.path.join(render_video_options, "render_options.json")
        self.viewpoint_json         = os.path.join(render_video_options, "viewpoint.json")
        self.viewpoint_lateral_json = os.path.join(render_video_options, "viewpoint_lateral.json")
        os.makedirs(render_video_options, exist_ok=True)
        self.out_dir = os.path.join(out_root_dir, f"{target_identity}__ON__{target_identity_animation}__{target_identity_sample}__TO__{source_identity}")
        os.makedirs(self.out_dir, exist_ok=True)

        self.initialize()

    def initialize(self):

        self.loaded_frames = 0

        src_ref_path = os.path.join(data_dir, self.dataset_class, self.src_identity, "a_t_pose", "000000")
        tgt_ref_path = os.path.join(data_dir, self.dataset_class, self.tgt_identity, "a_t_pose", "000000")

        ################################################################################################################
        # src ref
        ################################################################################################################
        if self.use_gt_tpose_mesh:
            src_ref_sample_path = os.path.join(src_ref_path, 'mesh_normalized.ply')
            assert os.path.isfile(src_ref_sample_path), src_ref_sample_path
            src_ref_mesh = trimesh.load(src_ref_sample_path)
        else:
            print("Generating src tpose mesh from code")
            src_ref_mesh = deepsdf_utils.create_mesh(
                shape_decoder, self.shape_codes, identity_ids=[self.src_identity_id], shape_codes_dim=shape_codes_dim,
                N=self.reconstruction_res, max_batch=self.max_batch
            )
            print("Generated")
        p_src_ref = src_ref_mesh.vertices.astype(np.float32)
        p_src_ref_cuda = torch.from_numpy(p_src_ref)[None, :].cuda()

        # src ref (mesh)
        self.src_ref = trimesh_to_open3d(src_ref_mesh, self.SRC_REFW_COLOR)

        ################################################################################################################
        # tgt ref
        ################################################################################################################
        if self.use_gt_tpose_mesh:
            tgt_ref_sample_path = os.path.join(tgt_ref_path, 'mesh_normalized.ply')
            assert os.path.isfile(tgt_ref_sample_path), tgt_ref_sample_path
            tgt_ref_mesh = trimesh.load(tgt_ref_sample_path)
        else:
            print("Generating tgt tpose mesh from code")
            tgt_ref_mesh = deepsdf_utils.create_mesh(
                shape_decoder, self.shape_codes, identity_ids=[self.tgt_identity_id], shape_codes_dim=shape_codes_dim,
                N=self.reconstruction_res, max_batch=self.max_batch
            )
            print("Generated")
        p_tgt_ref = tgt_ref_mesh.vertices.astype(np.float32)
        p_tgt_ref_cuda = torch.from_numpy(p_tgt_ref)[None, :].cuda()

        # tgt ref (mesh)
        self.tgt_ref = trimesh_to_open3d(tgt_ref_mesh, self.TGT_REFW_COLOR)

        # o3d.visualization.draw_geometries([self.src_ref])
        # o3d.visualization.draw_geometries([self.tgt_ref])
        # exit()

        ################################################################################################################
        ################################################################################################################
        # Pose both src and tgt identities with the pose code of tgt
        ################################################################################################################
        ################################################################################################################

        ##########################################################################################
        # SRC
        ##########################################################################################
        points = p_src_ref_cuda    # [1, 100000, 3]
        points_flat = points.reshape(-1, 3) # [100000, 3]

        with torch.no_grad():
            ##########################################################################################
            ### Prepare shape codes
            shape_codes_batch = self.shape_codes[[self.src_identity_id], :] # [bs, 1, C]
            assert shape_codes_batch.shape == (1, 1, shape_codes_dim), f"{shape_codes_batch} vs {(1, 1, shape_codes_dim)}"

            # Extent latent code to all sampled points
            shape_codes_repeat = shape_codes_batch.expand(-1, points_flat.shape[0], -1) # [bs, N, C]
            shape_codes_inputs = shape_codes_repeat.reshape(-1, shape_codes_dim) # [bs*N, C]

            ### Prepare pose codes
            pose_codes_batch = self.pose_codes[[self.frame_idx], ...] # [bs, 1, C]
            assert pose_codes_batch.shape == (1, 1, pose_codes_dim), f"{pose_codes_batch.shape} vs {(1, 1, pose_codes_dim)}"

            # Extent latent code to all sampled points
            pose_codes_repeat = pose_codes_batch.expand(-1, points_flat.shape[0], -1) # [bs, N, C]
            pose_codes_inputs = pose_codes_repeat.reshape(-1, pose_codes_dim) # [bs*N, C]
            ##########################################################################################

            # Concatenate pose and shape codes
            shape_pose_codes_inputs = torch.cat([shape_codes_inputs, pose_codes_inputs], 1)

            # Concatenate (for each sample point), the corresponding code and the p_cur coords
            pose_inputs = torch.cat([shape_pose_codes_inputs, points_flat], 1)

            # Predict delta flow
            p_src_ref_warped, _ = pose_decoder(pose_inputs) # [bs*N, 3]

            # REFW
            p_src_ref_warped = p_src_ref_warped.detach().cpu().numpy()
            src_ref_warped_mesh_o3d = o3d.geometry.TriangleMesh(
                o3d.utility.Vector3dVector(p_src_ref_warped),
                o3d.utility.Vector3iVector(src_ref_mesh.faces),
            )
            src_ref_warped_mesh_o3d.compute_vertex_normals()
            src_ref_warped_mesh_o3d.paint_uniform_color(self.SRC_REFW_COLOR)
            self.src_refw = src_ref_warped_mesh_o3d

        ##########################################################################################
        # TGT
        ##########################################################################################
        points = p_tgt_ref_cuda    # [1, 100000, 3]
        points_flat = points.reshape(-1, 3) # [100000, 3]

        with torch.no_grad():
            ##########################################################################################
            ### Prepare shape codes
            shape_codes_batch = self.shape_codes[[self.tgt_identity_id], :] # [bs, 1, C]
            assert shape_codes_batch.shape == (1, 1, shape_codes_dim), f"{shape_codes_batch} vs {(1, 1, shape_codes_dim)}"

            # Extent latent code to all sampled points
            shape_codes_repeat = shape_codes_batch.expand(-1, points_flat.shape[0], -1) # [bs, N, C]
            shape_codes_inputs = shape_codes_repeat.reshape(-1, shape_codes_dim) # [bs*N, C]

            ### Prepare pose codes
            pose_codes_batch = self.pose_codes[[self.frame_idx], ...] # [bs, 1, C]
            assert pose_codes_batch.shape == (1, 1, pose_codes_dim), f"{pose_codes_batch.shape} vs {(1, 1, pose_codes_dim)}"

            # Extent latent code to all sampled points
            pose_codes_repeat = pose_codes_batch.expand(-1, points_flat.shape[0], -1) # [bs, N, C]
            pose_codes_inputs = pose_codes_repeat.reshape(-1, pose_codes_dim) # [bs*N, C]
            ##########################################################################################

            # Concatenate pose and shape codes
            shape_pose_codes_inputs = torch.cat([shape_codes_inputs, pose_codes_inputs], 1)

            # Concatenate (for each sample point), the corresponding code and the p_cur coords
            pose_inputs = torch.cat([shape_pose_codes_inputs, points_flat], 1)

            # Predict delta flow
            p_tgt_ref_warped, _ = pose_decoder(pose_inputs) # [bs*N, 3]

            # REFW
            # REFW
            p_tgt_ref_warped = p_tgt_ref_warped.detach().cpu().numpy()
            tgt_ref_warped_mesh_o3d = o3d.geometry.TriangleMesh(
                o3d.utility.Vector3dVector(p_tgt_ref_warped),
                o3d.utility.Vector3iVector(tgt_ref_mesh.faces),
            )
            tgt_ref_warped_mesh_o3d.compute_vertex_normals()
            tgt_ref_warped_mesh_o3d.paint_uniform_color(self.TGT_REFW_COLOR)
            self.tgt_refw = tgt_ref_warped_mesh_o3d

            ##########################################################################################
            ##########################################################################################

        # ###############################################################################################
        # # Generate additional meshes.
        # ###############################################################################################
        # unit bbox
        p_min = -0.5
        p_max =  0.5
        self.unit_bbox = BBox.compute_bbox_from_min_point_and_max_point(
            np.array([p_min]*3), np.array([p_max]*3), color=[0.7, 0.7, 0.7]
        )

        # world frame
        self.world_frame = o3d.geometry.TriangleMesh.create_coordinate_frame(
            size=0.00001, origin=[0, 0, 0]
        )

    def update_src_refw(self, vis):
        param = vis.get_view_control().convert_to_pinhole_camera_parameters()

        if self.src_refw is not None:
            vis.remove_geometry(self.src_refw)

        if self.show_src_refw:
            vis.add_geometry(self.src_refw)

        ctr = vis.get_view_control()
        ctr.convert_from_pinhole_camera_parameters(param)

    def update_tgt_refw(self, vis):
        param = vis.get_view_control().convert_to_pinhole_camera_parameters()

        if self.tgt_refw is not None:
            vis.remove_geometry(self.tgt_refw)

        if self.show_tgt_refw:
            vis.add_geometry(self.tgt_refw)

        ctr = vis.get_view_control()
        ctr.convert_from_pinhole_camera_parameters(param)

    def update_src_ref(self, vis):
        param = vis.get_view_control().convert_to_pinhole_camera_parameters()

        if self.src_ref is not None:
            vis.remove_geometry(self.src_ref)

        if self.show_src_ref:
            vis.add_geometry(self.src_ref)

        ctr = vis.get_view_control()
        ctr.convert_from_pinhole_camera_parameters(param)

    def _load_render_and_viewpoint_option(self, vis, view):
        vis.get_render_option().load_from_json(self.render_json)

        # change viewpoint
        ctr = vis.get_view_control()
        if view == "frontal":
            param = o3d.io.read_pinhole_camera_parameters(self.viewpoint_json)
        elif view == "lateral":
            param = o3d.io.read_pinhole_camera_parameters(self.viewpoint_lateral_json)
        else:
            exit()
        ctr.convert_from_pinhole_camera_parameters(param)

    def render_image(self, vis, out_filename):
        image_np = np.asarray(vis.capture_screen_float_buffer(False))
        h, w, _ = image_np.shape
        new_h, new_w = h, h
        image_np = image_np[(h-new_h)//2:(h+new_h)//2, (w-new_w)//2:(w+new_w)//2,:]
        plt.imsave(f"{self.out_dir}/{out_filename}.jpg", image_np)

    def run(self):

        def update_all(vis):
            self.update_src_refw(vis)
            self.update_tgt_refw(vis)
            self.update_src_ref(vis)
            return False

        def toggle_src_refw(vis):
            self.show_src_refw = not self.show_src_refw
            self.update_src_refw(vis)
            return False

        def toggle_tgt_refw(vis):
            self.show_tgt_refw = not self.show_tgt_refw
            self.update_tgt_refw(vis)
            return False

        def toggle_src_ref(vis):
            self.show_src_ref = not self.show_src_ref
            self.update_src_ref(vis)
            return False

        def render(vis):
            print("::render")

            self._load_render_and_viewpoint_option(vis, self.view)

            self.show_src_ref = False
            self.show_tgt_ref = False
            self.show_src_refw = False
            self.show_tgt_refw = False

            ##################################################
            # Render the posed target
            ##################################################
            self.show_tgt_refw = True
            update_all(vis)
            vis.poll_events()
            vis.update_renderer()
            self.render_image(vis, "tgt_posed")
            self.show_tgt_refw = False

            ##################################################
            # Render the tpose of the src identity
            ##################################################
            self.show_src_ref = True
            update_all(vis)
            vis.poll_events()
            vis.update_renderer()
            self.render_image(vis, "src_Tpose")
            self.show_src_ref = False

            ##################################################
            # Render the posed src
            ##################################################
            self.show_src_refw = True
            update_all(vis)
            vis.poll_events()
            vis.update_renderer()
            self.render_image(vis, "src_posed")
            self.show_src_refw = False

            return False

        key_to_callback = {}
        key_to_callback[ord("T")] = toggle_src_ref
        key_to_callback[ord("W")] = toggle_src_refw
        key_to_callback[ord("V")] = toggle_tgt_refw
        key_to_callback[ord("R")] = render

        # Start showing the tgt tpose
        self.show_src_refw = True

        # o3d.visualization.draw_geometries_with_key_callbacks([self.unit_bbox, self.src_refw, self.src_cur], key_to_callback)
        o3d.visualization.draw_geometries_with_key_callbacks([self.world_frame, self.src_refw], key_to_callback)

################################################################################################################################
################################################################################################################################
################################################################################################################################
################################################################################################################################
################################################################################################################################
################################################################################################################################
################################################################################################################################
################################################################################################################################
################################################################################################################################
################################################################################################################################
################################################################################################################################

if __name__ == "__main__":

    torch.backends.cudnn.benchmark = True

    ########################################################################################################################
    # Options
    ########################################################################################################################
    # import argparse
    # parser = argparse.ArgumentParser(
    #     description='Run Model'
    # )
    # parser.add_argument('-o', '--optimize_codes', action='store_true')
    # parser.add_argument('-e', '--extra_name', default="")
    # parser.add_argument('-v', '--viz', action='store_true')
    # parser.add_argument('-n', '--optim_name', default=None)

    # try:
    #     args = parser.parse_args()
    # except:
    #     args = parser.parse_known_args()[0]

    demo_name = "qualitative_results__transfer_shape"

    out_root_dir = f"/cluster_HDD/lothlann/ppalafox/{demo_name}"

    viz = False

    # ------------------------------------------- #
    DATASET_TYPE = "HUMAN"
    print("#"*30)
    print(f"DATASET_TYPE: {DATASET_TYPE}")
    print("#"*30)
    # input("Continue?")
    # ------------------------------------------- #

    if DATASET_TYPE == "HUMAN":
        from configs_eval.config_eval_HUMAN import *
    elif DATASET_TYPE == "MANO":
        from configs_eval.config_eval_MANO import *

    ########################################################################################################################
    ########################################################################################################################

    dataset_name = "MIX-POSE__AMASS-50id-10349__MIXAMO-165id-40000__CAPE-35id-20533"

    from utils.parsing_utils import get_dataset_type_from_dataset_name
    dataset_type = get_dataset_type_from_dataset_name(dataset_name)
    splits_dir = f"{cfg.splits_dir}_{dataset_type}"

    labels_json       = os.path.join(data_dir, splits_dir, dataset_name, "labels.json")
    labels_tpose_json = os.path.join(data_dir, splits_dir, dataset_name, "labels_tpose.json")

    print("Reading from:")
    print(labels_json)
    print("Dataset name:")
    print(dataset_name)
    print()

    train_to_augmented_json = os.path.join(data_dir, splits_dir, dataset_name, "train_to_augmented.json")

    #######################################################################################################
    # Data
    #######################################################################################################
    with open(labels_json, "r") as f:
        labels = json.loads(f.read())

    with open(labels_tpose_json, "r") as f:
        labels_tpose = json.loads(f.read())

    train_to_augmented = None
    if os.path.isfile(train_to_augmented_json):
        with open(train_to_augmented_json, "r") as f:
            train_to_augmented = json.loads(f.read())

    batch_size = min(batch_size, len(labels))
    print("batch_size", batch_size)

    print("clamping distance", clamping_distance)

    num_identities = len(labels_tpose)
    num_frames = len(labels)

    print()
    print("#"*60)
    print("Num identities", num_identities)
    print("Num frames    ", num_frames)
    print()
    print('Interval', interval)
    print('Factor', factor)
    print("#"*60)
    print()

    ########################################################################################################################
    ########################################################################################################################

    # Pose MLP
    exp_dir = os.path.join(exps_dir, exp_name)
    checkpoint = nnutils.load_checkpoint(exp_dir, checkpoint_epoch)

    ########################
    # Shape decoder
    ########################
    if checkpoint_tpose is None:
        checkpoint_tpose = checkpoint

    shape_decoder = ShapeDecoder(shape_codes_dim, **shape_network_specs).cuda()
    shape_decoder.load_state_dict(checkpoint_tpose['model_state_dict_shape_decoder'])
    for param in shape_decoder.parameters():
        param.requires_grad = False
    shape_decoder.eval()
    nnutils.print_num_parameters(shape_decoder)

    ########################
    # Pose decoder
    ########################
    if use_se3:
        print()
        print("Using SE(3) formulation for the PoseDecoder")
        pose_decoder = PoseDecoderSE3(
            pose_codes_dim + shape_codes_dim, **pose_network_specs
        ).cuda()
    else:
        print()
        print("Using normal (translation) formulation for the PoseDecoder")
        pose_decoder = PoseDecoder(
            pose_codes_dim + shape_codes_dim, **pose_network_specs
        ).cuda()
    pose_decoder.load_state_dict(checkpoint['model_state_dict_pose_decoder'])
    for param in pose_decoder.parameters():
        param.requires_grad = False
    pose_decoder.eval()
    nnutils.print_num_parameters(pose_decoder)

    ########################
    # SHAPE Codes
    ########################
    shape_codes = torch.ones(num_identities, 1, shape_codes_dim).normal_(0, 0.1).cuda()

    pretrained_shape_codes = checkpoint['shape_codes'].cuda().detach().clone()

    if shape_codes.shape[0] != pretrained_shape_codes.shape[0] and train_to_augmented is not None:
        print("Loading shape codes - A")
        shape_codes = pretrained_shape_codes[list(train_to_augmented.values())].detach().clone()
        if len(shape_codes.shape) == 2:
            shape_codes = shape_codes.unsqueeze(0)
    else:
        print("Loading shape codes - B")
        shape_codes = checkpoint['shape_codes'].cuda().detach().clone()

    ##################################################################
    # Use codes from training
    ##################################################################

    print()
    print("Using pretrained pose codes")
    print()

    pretrained_pose_codes = checkpoint['pose_codes'].cuda().detach().clone()

    if pretrained_pose_codes.shape[0] != len(labels):
        raise Exception("Number of pose codes != lenght of dataset")

    pose_codes = pretrained_pose_codes
    pose_codes.requires_grad = False

    assert pose_codes.shape[1] == 1 and pose_codes.shape[2] == pose_codes_dim

    ##################################################################################################################
    ##################################################################################################################

    TGT_REFW_COLOR = [0.6, 0.4, 0.4]

    print()
    print()
    print("#######################################################################")
    print("Final visualization")
    print("#######################################################################")

    # source_identity = "adam"
    # target_identity = "kate"
    # target_identity_animation = "breakdance_uprock"
    # target_identity_sample = 13
    # SRC_REFW_COLOR = [0.4, 0.6, 0.4]

    source_identity = "adam"
    target_identity = "megan"
    target_identity_animation = "afoxe_samba_reggae_dance"
    target_identity_sample = 2
    SRC_REFW_COLOR = [0.4, 0.6, 0.6]

    # source_identity = "racer"
    # target_identity = "megan"
    # target_identity_animation = "afoxe_samba_reggae_dance"
    # target_identity_sample = 2
    # SRC_REFW_COLOR = [0.4, 0.6, 0.4]

    # source_identity = "douglas"
    # target_identity = "megan"
    # target_identity_animation = "afoxe_samba_reggae_dance"
    # target_identity_sample = 2
    # SRC_REFW_COLOR = [0.6, 0.6, 0.4]

    viewer = ViewerShapeTransfer(
        labels,
        labels_tpose,
        shape_codes,
        pose_codes,
        out_root_dir,
        dataset_class="mixamo_trans_all",
        source_identity=source_identity, # we'll use the shape code from this identity
        target_identity=target_identity, # we'll use the pose code from this identity
        target_identity_animation=target_identity_animation,
        target_identity_sample= str(target_identity_sample).zfill(6),
    )
    viewer.run()

max_stars_repo_name: Tiamat-Tech/npms

max_issues_repo_name: Tiamat-Tech/npms

max_forks_repo_name: Tiamat-Tech/npms

reason: import numpy,from numpy


text:
"""
================================================================
Use the RidgeCV and LassoCV to set the regularization parameter
================================================================

"""

############################################################
# Load the diabetes dataset
from sklearn.datasets import load_diabetes
data = load_diabetes()
X, y = data.data, data.target
print(X.shape)

############################################################
# Compute the cross-validation score with the default hyper-parameters
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import Ridge, Lasso

for Model in [Ridge, Lasso]:
    model = Model()
    print('%s: %s' % (Model.__name__,
                      cross_val_score(model, X, y).mean()))

############################################################
# We compute the cross-validation score as a function of alpha, the
# strength of the regularization for Lasso and Ridge
import numpy as np
from matplotlib import pyplot as plt

alphas = np.logspace(-3, -1, 30)

plt.figure(figsize=(5, 3))

for Model in [Lasso, Ridge]:
    scores = [cross_val_score(Model(alpha), X, y, cv=3).mean()
            for alpha in alphas]
    plt.plot(alphas, scores, label=Model.__name__)

plt.legend(loc='lower left')
plt.xlabel('alpha')
plt.ylabel('cross validation score')
plt.tight_layout()
plt.show()

max_stars_repo_name: zmoon/scipy-lecture-notes

max_issues_repo_name: zmoon/scipy-lecture-notes

max_forks_repo_name: zmoon/scipy-lecture-notes


text:
import argparse
import os
import pstats
from cProfile import Profile

import numpy as np
import torch
import torch.nn as nn
import torch.optim as opt
from torch.optim.lr_scheduler import LambdaLR
from torch.utils.data import DataLoader
from tqdm import tqdm

from dataset import MyDataset
from torchvision import models

'''
    A script for testing computation time.

'''

def count_params(model):
    total = 0
    for i in model.parameters():
        total += np.prod(i.shape)
    return total

def main(args):
    SEED = 12345

    torch.manual_seed(SEED)
    torch.cuda.manual_seed(SEED)
    np.random.seed(SEED)

    model = models.vgg16(pretrained=True)
    model.classifier[0] = nn.Linear(8*8*512, 4096)
    model.classifier[6] = nn.Linear(4096, 27)
    model.load_state_dict(torch.load("model.pth"))
    print("total params:{}".format(count_params(model)))

    dataset_test = MyDataset(args.pickle_path, args.test_list_path, args.test_dir)
    test_loader = DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False)

    device = torch.device("cpu" if args.no_cuda else "cuda:0")
    model = model.to(device)

    model.eval()

    if not args.no_prof:
        pr = Profile()
        pr.runcall(bench, model, test_loader, device)
        pr.dump_stats("profile.txt")

        ps = pstats.Stats("profile.txt")
        ps.sort_stats('time').print_stats(15)

def bench(model, test_loader, device):
    print("start benchmark...")
    with torch.no_grad():
        for data in test_loader:
            x, y = data
            x = x.to(device)
            y = y.to(device)
            out = model(x)
            _, pred = torch.max(out.data, 1)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--test_list_path", required=True, help="location of test files list")
    parser.add_argument("--test_dir", required=True, help="location of test files directory")
    parser.add_argument("--pickle_path", required=True, help="location of pickle file")
    parser.add_argument("--batch_size", type=int, default=1, help="batch size")
    parser.add_argument("--model_path", required=True, help="location of saved weight")
    parser.add_argument("--no-cuda", action="store_true", help="disable GPU")
    parser.add_argument("--no-prof", action="store_true", help="disable Profiling")

    args = parser.parse_args()

    main(args)

max_stars_repo_name: hideki-kaneko/TGN

max_issues_repo_name: hideki-kaneko/TGN

max_forks_repo_name: hideki-kaneko/TGN


text:
import sqlite3
import requests

import numpy as np
import tensorflow as tf
from tensorflow import keras
import chesslib

from chessai.dataset import ChessGmGamesDataset
from chessai.pretrain import ChessDrawGenerator

class DrawGenTrainingSession(object):

    def __init__(self, params: dict):

        super(DrawGenTrainingSession, self).__init__()
        self.params = params

        # create training and evaluation datasets
        dataset = ChessGmGamesDataset(params['batch_size'])
        self.train_dataset, self.eval_dataset = dataset.load_datasets()

        # create model to be trained
        self.model = ChessDrawGenerator(params)
        # self.model.build((None, 8, 8, 7))
        # print(self.model.summary())

        # create learning rate decay func
        self.lr_decay_func = tf.keras.optimizers.schedules.ExponentialDecay(
            initial_learning_rate=params['learn_rate'],
            decay_steps = params['total_train_batches'] * params['lr_decay_epochs'],
            decay_rate = params['lr_decay_rate'],
            staircase=False
        )

        # create optimizer and loss func
        self.optimizer = tf.optimizers.SGD(learning_rate=self.lr_decay_func)
        self.loss_func = tf.losses.CategoricalCrossentropy()

        # create model checkpoints
        self.checkpoint = tf.train.Checkpoint(optimizer=self.optimizer, net=self.model)
        self.manager = tf.train.CheckpointManager(self.checkpoint, '/app/models/pretrain-fx', max_to_keep=5)

        # create logging metrics
        self.train_loss = tf.keras.metrics.Mean(name="train_loss")
        self.train_acc = tf.keras.metrics.Mean(name="train_acc")
        self.eval_loss = tf.keras.metrics.Mean(name="eval_loss")
        self.eval_acc = tf.keras.metrics.Mean(name="eval_acc")

        # create TensorBoard summary writer
        self.train_summary_writer = tf.summary.create_file_writer('/app/logs/train')
        self.eval_summary_writer = tf.summary.create_file_writer('/app/logs/eval')

    def run_training(self):

        step = 0

        # loop through all epochs
        for epoch in range(self.params['epochs']):

            print('starting training epoch', epoch)

            # loop through all batches on the training dataset
            for batch_data in self.train_dataset:
                self.train_step(batch_data)
                step += 1

                # log training progress and metrics
                if step % self.params['log_interval'] == 0:
                    print('training progress:', int((step % self.params['total_train_batches']) / self.params['total_train_batches'] * 100), '%')
                    with self.train_summary_writer.as_default():
                        print('train results:', 'acc={}, loss={}'.format(self.train_acc.result(), self.train_loss.result()))
                        # tf.summary.scalar('train_acc', self.train_acc.result(), step=step)
                        tf.summary.scalar('train_loss', self.train_loss.result(), step=step)
                        # self.train_acc.reset_states()
                        self.train_loss.reset_states()

            # loop through all batches on the evaluation dataset
            for batch_data in self.eval_dataset:
                self.eval_step(batch_data)

            with self.eval_summary_writer.as_default():
                print('eval results:', 'acc={}, loss={}'.format(self.eval_acc.result(), self.eval_loss.result()))
                # tf.summary.scalar('eval_acc', self.eval_acc.result(), step=step)
                tf.summary.scalar('eval_loss', self.eval_loss.result(), step=step)
                # self.eval_acc.reset_states()
                self.eval_loss.reset_states()

            # store the model weights using the checkpoint manager
            self.manager.save(epoch)

    @tf.function
    def train_step(self, batch_data):

        # unwrap batch data as SARS data
        states, actions, rewards, next_states = batch_data
        input_positions = tf.bitwise.bitwise_and(actions, 0x3F)
        target_positions = tf.bitwise.bitwise_and(actions, 0xFC0)
        target_positions = tf.one_hot(target_positions, depth=64, dtype=tf.float32)

        with tf.GradientTape() as tape:

            # initialize the LSTM states by showing the chess boards to it
            h = self.model.show_chessboard(states)

            # now, predict the most likely target position for the acting chess pieces
            pred_positions = tf.nn.softmax(self.model(input_positions, h))

            # compute the loss (check if the correct target positions were predicted)
            loss = self.loss_func(y_true=target_positions, y_pred=pred_positions)

            # scale the loss according to the rewards (-> favour better actions)
            # loss = loss * tf.math.log(1 - tf.cast(rewards, dtype=tf.float32))

        # compute gradients and optimize the model
        gradients = tape.gradient(loss, self.model.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, self.model.trainable_variables))

        # write loss and accuracy to the metrics cache
        self.train_loss(loss)
        # self.train_acc(1 - loss)

    @ tf.function
    def eval_step(self, batch_data):

        # unwrap batch data as SARS data
        states, actions, rewards, next_states = batch_data
        input_positions = tf.bitwise.bitwise_and(actions, 0x3F)
        target_positions = tf.bitwise.bitwise_and(actions, 0xFC0)
        target_positions = tf.one_hot(target_positions, depth=64, dtype=tf.float32)

        # initialize the LSTM states by showing the chess boards to it
        h = self.model.show_chessboard(states)

        # now, predict the most likely target position for the acting chess pieces
        pred_positions = tf.nn.softmax(self.model(input_positions, h))

        # compute the loss (check if the correct target positions were predicted)
        # scale the loss according to the rewards (-> favour better actions)
        loss = self.loss_func(y_true=target_positions, y_pred=pred_positions)
        # loss = loss * tf.math.log(1 - rewards)

        # write loss and accuracy to the metrics cache
        self.eval_loss(loss)
        # self.eval_acc(1 - loss)


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

__author__ = "Christian Heider Nielsen"

from collections import namedtuple

import numpy

class InitStateDistribution(object):
    StateDist = namedtuple("StateDist", ("state", "prob"))

    def __init__(self):
        self.state_tuples = []

    def add(self, state, prob):
        self.state_tuples.append(self.StateDist(state, prob))

    def sample(self):
        sds = self.StateDist(*zip(*self.state_tuples))
        return numpy.random.choice(sds.state, p=sds.prob)

max_stars_repo_name: gitter-badger/agent

max_issues_repo_name: gitter-badger/agent

max_forks_repo_name: gitter-badger/agent


text:
import sys
sys.path.append('..')

from config import config
data_dir = config['data_dir']
from config import logger
from utils import get_natural_constraints, get_balanced_constraints, get_beta_distributed_links, links_accuracy
from sklearn.model_selection import KFold

import numpy as np
from CEC import CEC
import os
import pandas as pd

def run(data_fname, metrics, params):

    X = np.loadtxt(os.path.join(data_dir, data_fname + '_data.csv'), delimiter=',')
    link_fname = data_fname.split('_')[0]
    links = np.loadtxt(os.path.join(data_dir, link_fname + '_links.csv'), delimiter=',').astype(int)

    ### Process experiment params
    # n_folds
    n_folds = params.get('n_folds', 5)
    # seed
    seed = params.get('seed', np.random.randint(np.iinfo(np.int32).max))

    # probability of bad links
    bad_links = params.get('bad_links', 0.)
    assert bad_links >= 0 and bad_links <= 1

    ### Process  CEC params

    # k
    k = params.get('k', 10)
    # n_jobs
    n_jobs = params.get('n_jobs', 10)
    # n_init
    n_init = params.get('n_init', 10)
    # link_weight
    link_weight = params.get('link_weight', 1)
    # cov_type
    cov_type = params.get('cov_type', 'fixed_spherical')
    # verbosity
    verbose = params.get('verbose', False)

    # Prepare results
    results = {m: [] for m in metrics}

    logger.info("Starting dataset {} with metrics: {} and params: {}".format(data_fname.upper(), ", ".join(metrics), params))

    # prepare folds
    folds = KFold(n_splits=n_folds, shuffle=False)

    # run CEC
    cec = CEC(k=k,
              n_init=n_init,
              n_jobs=n_jobs,
              link_weight=link_weight,
              seed=seed,
              verbose=verbose,
              cov_type=cov_type)

    for i, (train, test) in enumerate(folds.split(links)):

        p = cec.fit_predict(X, constraints=links[train])

        for metric in results.keys():
            results[metric].append(globals().get(metric)(links[test], p))

        logger.info("\t fold {}/{}".format(i+1, n_folds))

    # Average results
    for metric, scores in results.iteritems():
        results[metric] = np.mean(scores)

    return results


text:
import matplotlib.pyplot as plt
import numpy as np
from latency_lists import *

flask_get = np.array(flask_get) * 1000
flask_post = np.array(flask_post) * 1000
flask_delete = np.array(flask_delete) * 1000
cherry_get = np.array(cherry_get) * 1000
cherry_post = np.array(cherry_post) * 1000
cherry_delete = np.array(cherry_delete) * 1000

flask_get_avg = np.average(flask_get)
flask_post_avg = np.average(flask_post)
flask_delete_avg = np.average(flask_delete)
cherry_get_avg = np.average(cherry_get)
cherry_post_avg = np.average(cherry_post)
cherry_delete_avg = np.average(cherry_delete)

print(flask_get_avg)
print(flask_post_avg)
print(flask_delete_avg)
print(cherry_get_avg)
print(cherry_post_avg)
print(cherry_delete_avg)

labels = ['GET', 'POST', 'DELETE']

flask_means = [round(flask_get_avg, 3), round(flask_post_avg, 3), round(flask_delete_avg, 3)]
cherry_means = [round(cherry_get_avg, 3), round(cherry_post_avg, 3), round(cherry_delete_avg, 3)]

x = np.arange(len(labels))  # the label locations
width = 0.35  # the width of the bars

fig, ax = plt.subplots()
rects1 = ax.bar(x - width/2, flask_means, width, label='Flask')
rects2 = ax.bar(x + width/2, cherry_means, width, label='Cherry')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('Latency [ms]')
ax.set_title('Latency by method and server')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.legend()

def autolabel(rects):
    """Attach a text label above each bar in *rects*, displaying its height."""
    for rect in rects:
        height = rect.get_height()
        ax.annotate('{}'.format(height),
                    xy=(rect.get_x() + rect.get_width() / 2, height),
                    xytext=(0, 3),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom')

autolabel(rects1)
autolabel(rects2)

fig.tight_layout()

plt.show()
plt.savefig('multi_tutti.png')

max_stars_repo_name: kwitnacy/wti

max_issues_repo_name: kwitnacy/wti

max_forks_repo_name: kwitnacy/wti


text:
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from tqdm import tqdm

import utils
from model import DuelingNetwork
from utils import ExperienceReplayMemory, AgentConfig, BranchingTensorEnv, init_seeds

class BranchingDQN(nn.Module):

    def __init__(self, obs, ac, config):
        super().__init__()

        self.q = DuelingNetwork(obs, ac)
        self.target = DuelingNetwork(obs, ac)

        self.target.load_state_dict(self.q.state_dict())

        self.target_net_update_freq = config.target_net_update_freq
        self.update_counter = 0
        self.config = config

    def get_action(self, x):
        with torch.no_grad():
            # a = self.q(x).max(1)[1]
            out = self.q(x)
            return torch.argmax(out).item()

    def update_policy(self, adam, memory, params):
        b_states, b_actions, b_rewards, b_next_states, b_masks = memory.sample(params.batch_size)

        states = torch.tensor(b_states).float()
        actions = torch.tensor(b_actions).long().reshape(states.shape[0], -1)
        rewards = torch.tensor(b_rewards).float().reshape(-1, 1)
        next_states = torch.tensor(b_next_states).float()
        masks = torch.tensor(b_masks).float().reshape(-1, 1)

        qvals = self.q(states)
        current_q_values = qvals.gather(1, actions).squeeze(-1)

        with torch.no_grad():
            argmax = torch.argmax(self.q(next_states), dim=1)

            max_next_q_vals = self.target(next_states).gather(1, argmax.unsqueeze(1))
            # max_next_q_vals = max_next_q_vals.mean(1, keepdim=True).expand(-1, max_next_q_vals.shape[1])   # Branching DQN

        expected_q_vals = rewards + max_next_q_vals * self.config.gamma * masks
        # print(expected_q_vals[:5])
        loss = F.mse_loss(expected_q_vals, current_q_values)

        # input(loss)

        # print('\n'*5)

        adam.zero_grad()
        loss.backward()

        for p in self.q.parameters():
            p.grad.data.clamp_(-1., 1.)
        adam.step()

        self.update_counter += 1
        if self.update_counter % self.target_net_update_freq == 0:
            self.update_counter = 0
            # self.target.load_state_dict(self.q.state_dict())
            self.target.soft_update(self.q, tau=self.config.target_net_update_tau)

if __name__ == "__main__":
    args = utils.arguments()

    init_seeds(500)

    env = BranchingTensorEnv(args.env)
    env.seed(1234)

    config = AgentConfig()
    memory = ExperienceReplayMemory(config.memory_size)
    agent = BranchingDQN(env.observation_space.shape[0], env.action_space.n, config)
    adam = optim.Adam(agent.q.parameters(), lr=config.lr)

    s = env.reset()
    ep_reward = 0.
    recap = []  # stores cumulative reward per episode (episode rewards)

    p_bar = tqdm(total=config.max_frames)
    for frame in range(config.max_frames):

        epsilon = config.epsilon_by_frame(frame)

        if np.random.random() > 0.5:
            action = agent.get_action(s)
        else:
            action = np.random.randint(0, env.action_space.n)

        ns, r, done, infos = env.step(action)
        ep_reward += r

        if done:
            ns = env.reset()
            recap.append(ep_reward)
            p_bar.set_description('Rew: {:.3f}'.format(ep_reward))
            ep_reward = 0.

        memory.push((s.reshape(-1).numpy().tolist(), action, r, ns.reshape(-1).numpy().tolist(), 0. if done else 1.))
        s = ns

        p_bar.update(1)

        # Update online and target model only from specified number of frames
        if frame > config.learning_starts:
            agent.update_policy(adam, memory, config)

        if frame % 1000 == 0:
            utils.save(agent, recap, args)

    p_bar.close()

max_stars_repo_name: ornean/cas-artificial-intelligence

max_issues_repo_name: ornean/cas-artificial-intelligence

max_forks_repo_name: ornean/cas-artificial-intelligence


text:
import os
import sys
import shutil
import tarfile
import pickle
import numpy as np
import tables

from .. import config
from ..image.utils import ImageArrayUInt8ToFloat32
from . import dataset

# Pickle encoding parameters depend on Python version
_PICKLE_ENC = {} if sys.version_info[0] == 2 else {'encoding': 'latin1'}

_CIFAR100_BASE_URL = 'http://www.cs.toronto.edu/~kriz/'
_SHA256_CIFAR100_TARBALL = \
    '85cd44d02ba6437773c5bbd22e183051d648de2e7d6b014e1ef29b855ba677a7'
_H5_FILENAME = 'cifar100.h5'

_CIFAR100_SRC = dataset.DownloadSourceFile(
    'cifar-100-python.tar.gz', base_url=_CIFAR100_BASE_URL,
    sha256=_SHA256_CIFAR100_TARBALL)

@dataset.fetch_and_convert_dataset([_CIFAR100_SRC], _H5_FILENAME)
def fetch_cifar100(source_paths, target_path):
    tarball_path = source_paths[0]

    # Get the paths to the member files
    download_dir = os.path.dirname(tarball_path)
    train_path = os.path.join(download_dir,
                              'cifar-100-python', 'train')
    test_path = os.path.join(download_dir,
                             'cifar-100-python', 'test')
    meta_path = os.path.join(download_dir,
                             'cifar-100-python', 'meta')

    # Unpack
    print('Unpacking CIFAR-100 tarball {}'.format(tarball_path))
    tarfile.open(name=tarball_path, mode='r:gz').extractall(
        path=download_dir)

    # Create HDF5 output file
    f_out = tables.open_file(target_path, mode='w')
    g_out = f_out.create_group(f_out.root, 'cifar100', 'CIFAR-100 data')

    print('Converting CIFAR-100 training set to HDF5')
    train_batch = pickle.load(open(train_path, 'rb'), **_PICKLE_ENC)
    train_X_u8 = train_batch['data'].reshape((-1, 3, 32, 32))
    train_y = np.array(train_batch['fine_labels'], dtype=np.int32)
    train_y_coarse = np.array(train_batch['coarse_labels'], dtype=np.int32)
    f_out.create_array(g_out, 'train_X_u8', train_X_u8)
    f_out.create_array(g_out, 'train_y', train_y)
    f_out.create_array(g_out, 'train_y_coarse', train_y_coarse)

    print('Converting CIFAR-100 test set to HDF5')
    tst_batch = pickle.load(open(test_path, 'rb'), **_PICKLE_ENC)
    test_X_u8 = tst_batch['data'].reshape((-1, 3, 32, 32))
    test_y = np.array(tst_batch['fine_labels'], dtype=np.int32)
    test_y_coarse = np.array(tst_batch['coarse_labels'], dtype=np.int32)
    f_out.create_array(g_out, 'test_X_u8', test_X_u8)
    f_out.create_array(g_out, 'test_y', test_y)
    f_out.create_array(g_out, 'test_y_coarse', test_y_coarse)

    print('Converting CIFAR-100 metadata to HDF5')
    meta = pickle.load(open(meta_path, 'rb'), **_PICKLE_ENC)
    class_names = meta['fine_label_names']
    class_names_coarse = meta['coarse_label_names']
    f_out.create_array(g_out, 'class_names', class_names)
    f_out.create_array(g_out, 'class_names_coarse', class_names_coarse)

    f_out.close()

    # Remove the contents unpacked from the tarball
    shutil.rmtree(os.path.join(download_dir, 'cifar-100-python'))

    return target_path

def delete_cache():  # pragma: no cover
    dataset.delete_dataset_cache(_H5_FILENAME)

class CIFAR100 (object):
    def __init__(self, n_val=10000, val_lower=0.0, val_upper=1.0):
        h5_path = fetch_cifar100()
        if h5_path is not None:
            f = tables.open_file(h5_path, mode='r')

            train_X_u8 = f.root.cifar100.train_X_u8
            train_y = f.root.cifar100.train_y
            train_y_coarse = f.root.cifar100.train_y_coarse
            self.test_X_u8 = f.root.cifar100.test_X_u8
            self.test_y = f.root.cifar100.test_y
            self.test_y_coarse = f.root.cifar100.test_y_coarse
            self.class_names = dataset.classnames_from_h5(
                f.root.cifar100.class_names)
            self.class_names_coarse = dataset.classnames_from_h5(
                f.root.cifar100.class_names_coarse)

            if n_val == 0 or n_val is None:
                self.train_X_u8 = train_X_u8
                self.train_y = train_y
                self.train_y_coarse = train_y_coarse
                self.val_X_u8 = np.zeros((0, 3, 32, 32), dtype=np.uint8)
                self.val_y = np.zeros((0,), dtype=np.int32)
                self.val_y_coarse = np.zeros((0,), dtype=np.int32)
            else:
                self.train_X_u8 = train_X_u8[:-n_val]
                self.val_X_u8 = train_X_u8[-n_val:]
                self.train_y, self.val_y = train_y[:-n_val], train_y[-n_val:]
                self.train_y_coarse, self.val_y_coarse = \
                    (train_y_coarse[:-n_val], train_y_coarse[-n_val:])
        else:
            raise RuntimeError('Could not load CIFAR-10 dataset')

        self.train_X = ImageArrayUInt8ToFloat32(self.train_X_u8, val_lower,
                                                val_upper)
        self.val_X = ImageArrayUInt8ToFloat32(self.val_X_u8, val_lower,
                                              val_upper)
        self.test_X = ImageArrayUInt8ToFloat32(self.test_X_u8, val_lower,
                                               val_upper)

max_stars_repo_name: Britefury/batchup

max_issues_repo_name: Britefury/batchup

max_forks_repo_name: Britefury/batchup


text:
# Copyright 2021 Kotaro Terada
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time

import numpy

import sawatabi

def test_current_time_type():
    assert isinstance(sawatabi.utils.current_time(), float)
    assert isinstance(sawatabi.utils.current_time_ms(), float)
    assert isinstance(sawatabi.utils.current_time_us(), float)
    assert isinstance(sawatabi.utils.current_time_ns(), float)

    assert isinstance(sawatabi.utils.current_time_as_int(), int)
    assert isinstance(sawatabi.utils.current_time_ms_as_int(), int)
    assert isinstance(sawatabi.utils.current_time_us_as_int(), int)
    assert isinstance(sawatabi.utils.current_time_ns_as_int(), int)

def test_current_time_value():
    current_time = time.time()

    # Check the time approximately

    # sec
    numpy.testing.assert_approx_equal(actual=sawatabi.utils.current_time(), desired=current_time, significant=9)
    numpy.testing.assert_approx_equal(actual=sawatabi.utils.current_time_as_int(), desired=current_time, significant=9)

    # milli sec
    numpy.testing.assert_approx_equal(
        actual=sawatabi.utils.current_time_ms(),
        desired=current_time * 1_000,
        significant=9,
    )
    numpy.testing.assert_approx_equal(
        actual=sawatabi.utils.current_time_ms_as_int(),
        desired=current_time * 1_000,
        significant=9,
    )

    # micro sec
    numpy.testing.assert_approx_equal(
        actual=sawatabi.utils.current_time_us(),
        desired=current_time * 1_000_000,
        significant=9,
    )
    numpy.testing.assert_approx_equal(
        actual=sawatabi.utils.current_time_us_as_int(),
        desired=current_time * 1_000_000,
        significant=9,
    )

    # nano sec
    numpy.testing.assert_approx_equal(
        actual=sawatabi.utils.current_time_ns(),
        desired=current_time * 1_000_000_000,
        significant=9,
    )
    numpy.testing.assert_approx_equal(
        actual=sawatabi.utils.current_time_ns_as_int(),
        desired=current_time * 1_000_000_000,
        significant=9,
    )

max_stars_repo_name: marblejenka/sawatabi

max_issues_repo_name: marblejenka/sawatabi

max_forks_repo_name: marblejenka/sawatabi


text:
#  MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
#  Copyright (C) 2020-2021, UChicago Argonne, LLC. All rights reserved.
#  Released under the modified BSD license. See COPYING.md for more details.
from typing import Dict
from unittest.mock import Mock

import numpy as np
import pytest
from numpy.testing import assert_array_equal

from miplearn.classifiers import Regressor
from miplearn.components.objective import ObjectiveValueComponent
from miplearn.features.sample import Sample, MemorySample
from miplearn.solvers.learning import LearningSolver
from miplearn.solvers.pyomo.gurobi import GurobiPyomoSolver
from miplearn.solvers.tests import assert_equals

@pytest.fixture
def sample() -> Sample:
    sample = MemorySample(
        {
            "mip_lower_bound": 1.0,
            "mip_upper_bound": 2.0,
            "lp_instance_features": np.array([1.0, 2.0, 3.0]),
        },
    )
    return sample

def test_sample_xy(sample: Sample) -> None:
    x_expected = {
        "Lower bound": np.array([[1.0, 2.0, 3.0]]),
        "Upper bound": np.array([[1.0, 2.0, 3.0]]),
    }
    y_expected = {
        "Lower bound": np.array([[1.0]]),
        "Upper bound": np.array([[2.0]]),
    }
    xy = ObjectiveValueComponent().sample_xy(None, sample)
    assert xy is not None
    x_actual, y_actual = xy
    assert_equals(x_actual, x_expected)
    assert_equals(y_actual, y_expected)

def test_fit_xy() -> None:
    x: Dict[str, np.ndarray] = {
        "Lower bound": np.array([[0.0, 0.0], [1.0, 2.0]]),
        "Upper bound": np.array([[0.0, 0.0], [1.0, 2.0]]),
    }
    y: Dict[str, np.ndarray] = {
        "Lower bound": np.array([[100.0]]),
        "Upper bound": np.array([[200.0]]),
    }
    reg = Mock(spec=Regressor)
    reg.clone = Mock(side_effect=lambda: Mock(spec=Regressor))
    comp = ObjectiveValueComponent(regressor=reg)
    assert "Upper bound" not in comp.regressors
    assert "Lower bound" not in comp.regressors
    comp.fit_xy(x, y)
    assert reg.clone.call_count == 2
    assert "Upper bound" in comp.regressors
    assert "Lower bound" in comp.regressors
    assert comp.regressors["Upper bound"].fit.call_count == 1  # type: ignore
    assert comp.regressors["Lower bound"].fit.call_count == 1  # type: ignore
    assert_array_equal(
        comp.regressors["Upper bound"].fit.call_args[0][0],  # type: ignore
        x["Upper bound"],
    )
    assert_array_equal(
        comp.regressors["Lower bound"].fit.call_args[0][0],  # type: ignore
        x["Lower bound"],
    )
    assert_array_equal(
        comp.regressors["Upper bound"].fit.call_args[0][1],  # type: ignore
        y["Upper bound"],
    )
    assert_array_equal(
        comp.regressors["Lower bound"].fit.call_args[0][1],  # type: ignore
        y["Lower bound"],
    )

def test_sample_predict(sample: Sample) -> None:
    x, y = ObjectiveValueComponent().sample_xy(None, sample)
    comp = ObjectiveValueComponent()
    comp.regressors["Lower bound"] = Mock(spec=Regressor)
    comp.regressors["Upper bound"] = Mock(spec=Regressor)
    comp.regressors["Lower bound"].predict = Mock(  # type: ignore
        side_effect=lambda _: np.array([[50.0]])
    )
    comp.regressors["Upper bound"].predict = Mock(  # type: ignore
        side_effect=lambda _: np.array([[60.0]])
    )
    pred = comp.sample_predict(sample)
    assert pred == {
        "Lower bound": 50.0,
        "Upper bound": 60.0,
    }
    assert_array_equal(
        comp.regressors["Upper bound"].predict.call_args[0][0],  # type: ignore
        x["Upper bound"],
    )
    assert_array_equal(
        comp.regressors["Lower bound"].predict.call_args[0][0],  # type: ignore
        x["Lower bound"],
    )

def test_sample_evaluate(sample: Sample) -> None:
    comp = ObjectiveValueComponent()
    comp.regressors["Lower bound"] = Mock(spec=Regressor)
    comp.regressors["Lower bound"].predict = lambda _: np.array([[1.05]])  # type: ignore
    comp.regressors["Upper bound"] = Mock(spec=Regressor)
    comp.regressors["Upper bound"].predict = lambda _: np.array([[2.50]])  # type: ignore
    ev = comp.sample_evaluate(None, sample)
    assert ev == {
        "Lower bound": {
            "Actual value": 1.0,
            "Predicted value": 1.05,
            "Absolute error": 0.05,
            "Relative error": 0.05,
        },
        "Upper bound": {
            "Actual value": 2.0,
            "Predicted value": 2.50,
            "Absolute error": 0.5,
            "Relative error": 0.25,
        },
    }

def test_usage() -> None:
    solver = LearningSolver(components=[ObjectiveValueComponent()])
    instance = GurobiPyomoSolver().build_test_instance_knapsack()
    solver.solve(instance)
    solver.fit([instance])
    stats = solver.solve(instance)
    assert stats["mip_lower_bound"] == stats["Objective: Predicted lower bound"]
    assert stats["mip_upper_bound"] == stats["Objective: Predicted upper bound"]

max_stars_repo_name: neuyhwu/MIPLearn

max_issues_repo_name: neuyhwu/MIPLearn

max_forks_repo_name: zclab/MIPLearn


text:
import torch
import numpy as np

from torchvision import datasets
import torchvision.transforms as transforms
from torch.utils.data.sampler import SubsetRandomSampler

import torch.nn as nn
import torch.nn.functional as F

num_workers = 0
batch_size = 20
valid_size = 0.2

# convert data to torch.FloatTensor
transform = transforms.ToTensor()

# choose the training and test datasets
train_data = datasets.MNIST(root='data', train=True, download=True, transform=transform)
test_data = datasets.MNIST(root='data', train=False, download=True, transform=transform)

# prepare data loaders
train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, num_workers=num_workers)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, num_workers=num_workers)

# obtain training indices that will be used for validation
num_train = len(train_data)
indices = list(range(num_train))
np.random.shuffle(indices)
split = int(np.floor(valid_size * num_train))
train_idx, valid_idx = indices[split:], indices[:split]

# define samplers for obtaining training and validation batches
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)

# prepare data loaders
train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,
 sampler=train_sampler, num_workers=num_workers)
valid_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,
 sampler=valid_sampler, num_workers=num_workers)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size,
 num_workers=num_workers)

## TODO: Define the NN architecture
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

    # linear layer (784 -> 1 hidden node)
        self.fc1 = nn.Linear(28 * 28, 64)
        self.fc2 = nn.Linear(64, 24)
        self.fc3 = nn.Linear(24, 10)

    def forward(self, x):
        # flatten image input
        x = x.view(-1, 28 * 28)
        # add hidden layer, with relu activation function
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))

        return x

# initialize the NN
model = Net()
print(model)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
n_epochs = 11

# initialize tracker for minimum validation loss
valid_loss_min = np.Inf # set initial "min" to infinity

for epoch in range(n_epochs):
    # monitor training loss
    train_loss = 0.0
    valid_loss = 0.0

    ###################
    # train the model #
    ###################
    model.train() # prep model for training
    for data, target in train_loader:
        # clear the gradients of all optimized variables
        optimizer.zero_grad()
        # forward pass: compute predicted outputs by passing inputs to the model
        output = model(data)
        # calculate the loss
        loss = criterion(output, target)
        # backward pass: compute gradient of the loss with respect to model parameters
        loss.backward()
        # perform a single optimization step (parameter update)
        optimizer.step()
        # update running training loss
        train_loss += loss.item() * data.size(0)

    ######################
    # validate the model #
    ######################
    model.eval() # prep model for evaluation
    for data, target in valid_loader:
        # forward pass: compute predicted outputs by passing inputs to the model
        output = model(data)
        # calculate the loss
        loss = criterion(output, target)
        # update running validation loss
        valid_loss += loss.item() * data.size(0)

    # print training/validation statistics
    # calculate average loss over an epoch
    train_loss = train_loss / len(train_loader.sampler)
    valid_loss = valid_loss / len(valid_loader.sampler)

    print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}'.format(
    epoch + 1,
    train_loss,
    valid_loss
    ))

    # save model if validation loss has decreased
    if valid_loss <= valid_loss_min:
        print('Validation loss decreased ({:.6f} --> {:.6f}). Saving model ...'.format(
        valid_loss_min,
        valid_loss))
        torch.save(model.state_dict(), 'model.pt')
        valid_loss_min = valid_loss


text:
import glob
import numpy as np
from pathlib import Path
from ..dataset import DatasetTemplate

class CoilsDataset(DatasetTemplate):
    def __init__(self, dataset_cfg, class_names, training=True, root_path=None, logger=None):
        '''
        Args:
        root_path:
        dataset_cfg:
        class_names:
        training:
        logger:

        '''
        super().__init__(
        dataset_cfg=dataset_cfg, class_names=class_names, training=training, root_path=root_path, logger=logger
        )
        points_file_list = glob.glob(str(self.root_path / 'train/points' / '*.txt'))
        labels_file_list = glob.glob(str(self.root_path / 'train/labels' / '*.txt'))
        points_file_list.sort()
        labels_file_list.sort()
        self.sample_file_list = points_file_list
        self.samplelabel_file_list = labels_file_list

    def __len__(self):
        return len(self.sample_file_list)

    def __getitem__(self, index):
        sample_idx = Path(self.sample_file_list[index]).stem  # 0000.txt -> 0000 样本id(文件编号) n：点的个数 m：标注的个数
        points = np.loadtxt(self.sample_file_list[index], dtype=np.float32).reshape(-1, 3)  # 每个点云文件里的所有点 n*3
        points_label = np.loadtxt(self.samplelabel_file_list[index], dtype=np.float32).reshape(-1, 7) # 每个点云标注文件里的所有点 m*7
        gt_names = np.array(['Coil']*points_label.shape[0])

        input_dict = {
            'points': points,
            'frame_id': sample_idx,
            'gt_names': gt_names,
            'gt_boxes': points_label
        }

        data_dict = self.prepare_data(data_dict=input_dict)
        return data_dict

max_stars_repo_name: jeacwen/OpenPCDet

max_issues_repo_name: jeacwen/OpenPCDet

max_forks_repo_name: jeacwen/OpenPCDet