text:
# ===========================================
#
# mian Analysis Data Mining/ML Library
# @author: tbj128
#
# ===========================================

#
# Imports
#
import pandas as pd
from sklearn.impute import SimpleImputer
from sklearn.linear_model import ElasticNet, SGDRegressor
from sklearn.metrics import mean_absolute_error, mean_squared_error
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold

from mian.model.otu_table import OTUTable
import numpy as np
import random

class LinearRegression(object):

    def run(self, user_request):
        table = OTUTable(user_request.user_id, user_request.pid)
        otu_table, headers, sample_labels = table.get_table_after_filtering_and_aggregation_and_low_count_exclusion(user_request)

        expvar = user_request.get_custom_attr("expvar")
        metadata_vals = table.get_sample_metadata().get_metadata_column_table_order(sample_labels, expvar)

        return self.analyse(user_request, otu_table, headers, metadata_vals)

    def analyse(self, user_request, otu_table, headers, metadata_vals):
        cross_validate_set = user_request.get_custom_attr("crossValidate")
        cross_validate_folds = int(user_request.get_custom_attr("crossValidateFolds"))
        fix_training = user_request.get_custom_attr("fixTraining")
        training_proportion = user_request.get_custom_attr("trainingProportion")
        seed = int(user_request.get_custom_attr("seed")) if user_request.get_custom_attr("seed") is not "" else random.randint(0, 100000)
        mixing_ratio = float(user_request.get_custom_attr("mixingRatio"))
        max_iterations = int(user_request.get_custom_attr("maxIterations"))

        if int(user_request.level) == -1:
            # OTU tables are returned as a CSR matrix
            X = pd.DataFrame.sparse.from_spmatrix(otu_table, columns=headers, index=range(otu_table.shape[0]))
        else:
            X = pd.DataFrame(otu_table, columns=headers, index=range(otu_table.shape[0]))

        Y = np.array(metadata_vals)

        def performCrossValidationForAUC(X_cv, metadata_vals_cv, Y_cv):
            cv = KFold(n_splits=cross_validate_folds, shuffle=True, random_state=seed)

            classifier = ElasticNet(l1_ratio=mixing_ratio, max_iter=max_iterations)

            test_maes = []
            test_mses = []

            for i, (train, test) in enumerate(cv.split(X_cv, metadata_vals_cv)):
                classifier.fit(X_cv[X_cv.index.isin(train)], Y_cv[train])
                preds = classifier.predict(X_cv[X_cv.index.isin(test)])
                test_mae = mean_absolute_error(Y_cv[test].astype(float), preds)
                test_mse = mean_squared_error(Y_cv[test].astype(float), preds)
                test_maes.append(test_mae)
                test_mses.append(test_mse)

            cv_obj = {
                "cv_mae": np.array(test_maes).mean(),
                "cv_mae_std": np.array(test_maes).std(),
                "cv_mse": np.array(test_mses).mean(),
                "cv_mse_std": np.array(test_mses).std()
            }
            return cv_obj

        if cross_validate_set == "full":
            cv_obj = performCrossValidationForAUC(X, metadata_vals, Y)
            return {
                "cv_mae": round(cv_obj["cv_mae"], 2),
                "cv_mae_std": round(cv_obj["cv_mae_std"], 2),
                "cv_mse": round(cv_obj["cv_mse"], 2),
                "cv_mse_std": round(cv_obj["cv_mse_std"], 2)
            }
        else:
            if fix_training == "yes":
                X_train, X_test, y_train, y_test = train_test_split(X, Y, train_size=training_proportion, random_state=seed)
                X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, train_size=0.5, random_state=seed)
            else:
                # Use a random seed each time (not recommended)
                X_train, X_test, y_train, y_test = train_test_split(X, Y, train_size=training_proportion)
                X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, train_size=0.5)

            imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
            X_train = imp_mean.fit_transform(X_train)
            X_val = imp_mean.transform(X_val)
            X_test = imp_mean.transform(X_test)

            classifier = ElasticNet(l1_ratio=mixing_ratio, fit_intercept=True, max_iter=max_iterations)
            classifier.fit(X_train, y_train)
            preds_train = classifier.predict(X_train)
            train_mae = mean_absolute_error(y_train.astype(float), preds_train)
            train_mse = mean_squared_error(y_train.astype(float), preds_train)

            preds_val = classifier.predict(X_val)
            val_mae = mean_absolute_error(y_val.astype(float), preds_val)
            val_mse = mean_squared_error(y_val.astype(float), preds_val)

            preds_test = classifier.predict(X_test)
            test_mae = mean_absolute_error(y_test.astype(float), preds_test)
            test_mse = mean_squared_error(y_test.astype(float), preds_test)

            abundances_obj = {
                "train_mae": round(train_mae, 2),
                "train_mse": round(train_mse, 2),
                "val_mae": round(val_mae, 2),
                "val_mse": round(val_mse, 2),
                "test_mae": round(test_mae, 2),
                "test_mse": round(test_mse, 2),
                "train_size": X_train.shape,
                "val_size": X_val.shape,
                "test_size": X_test.shape,
                "seed": seed
            }

            return abundances_obj

ext: py

reason: import numpy


text:
# Copyright 2019 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: get_data_statistics
   :platform: Unix
   :synopsis: A plugin to calculate global statistics (max, min, sum, mean) of the input data

.. moduleauthor:: Daniil Kazantsev <scientificsoftware@diamond.ac.uk>
"""

from savu.plugins.plugin import Plugin
from savu.plugins.driver.cpu_plugin import CpuPlugin
from savu.plugins.utils import register_plugin

import numpy as np

@register_plugin
class GetDataStatistics(Plugin, CpuPlugin):
    def __init__(self):
        super(GetDataStatistics, self).__init__("GetDataStatistics")

    def setup(self):

        in_dataset, out_dataset = self.get_datasets()
        in_pData, out_pData = self.get_plugin_datasets()
        pattern_type=self.parameters['pattern']
        in_pData[0].plugin_data_setup(pattern_type, 'single')
        out_dataset[0].create_dataset(in_dataset[0])
        out_pData[0].plugin_data_setup(pattern_type, 'single')

        fullData = in_dataset[0]
        slice_dirs = list(in_dataset[0].get_slice_dimensions())
        self.new_shape = (np.prod(np.array(fullData.get_shape())[slice_dirs]), 4)
        out_dataset[1].create_dataset(shape=self.new_shape,
                                      axis_labels=['stattype', 'value'],
                                      remove=True,
                                      transport='hdf5')
        out_dataset[1].add_pattern("METADATA", core_dims=(1,), slice_dims=(0,))
        out_pData[1].plugin_data_setup('METADATA', self.get_max_frames())

    def process_frames(self, data):
        data_temp = data[0]
        indices = np.where(np.isnan(data_temp))
        data_temp[indices] = 0.0
        # collecting values for each slice
        slice_values = [np.max(data_temp), np.min(data_temp), np.sum(data_temp), np.mean(data_temp)]
        return [data_temp, np.array([slice_values])]

    def post_process(self):
        data, slice_values = self.get_out_datasets()
        all_slice_values = slice_values.data[...]
        max_stat = np.max(all_slice_values[:,0])
        min_stat = np.max(all_slice_values[:,1])
        sum_stat = np.sum(all_slice_values[:,2])
        mean_stat = np.sum(all_slice_values[:,3])
        data.meta_data.set(['stats', 'max', 'pattern'], max_stat)
        data.meta_data.set(['stats', 'min', 'pattern'], min_stat)
        data.meta_data.set(['stats', 'sum', 'pattern'], sum_stat)
        data.meta_data.set(['stats', 'mean', 'pattern'], mean_stat)

    def nInput_datasets(self):
        return 1
    def nOutput_datasets(self):
        return 2
    def get_max_frames(self):
        return 'single'

max_stars_repo_name: elainehoml/Savu

max_issues_repo_name: elainehoml/Savu

max_forks_repo_name: elainehoml/Savu


text:
'''
    Copyright (c) 2019-2020, Takashi Shirakawa. All rights reserved.
    e-mail: tkshirakawa@gmail.com

    Released under the MIT license.
    https://opensource.org/licenses/mit-license.php

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''

# from __future__ import absolute_import
# from __future__ import division
# from __future__ import print_function

import sys
if sys.argv[1] == '-h':
    print('### Help for argv ###')
    print('  argv[1] : Path to a neural network model file (.py).')
    print('  argv[2] : Path to a CSV/.h5 file for training image paths')
    print('  argv[3] : Path to a CSV/.h5 file for validation image paths')
    print('  argv[4] : Path to a directory to save results in it')
    print('  argv[5] : Training mode: 0=Normal, 1=Resume by load_model(), 2=Retrain by load_weights()')
    print('  argv[6] : Path to a model for mode 1 or 2')       # print('  argv[6] : Path to a model for Mode-1 or Mode-2')
    print('  argv[7] : Initial epoch to resume training for Mode-1')
    print('  NOTE : Input images must be gray-scale without alpha values')
    sys.exit()

import os
import platform
import numpy as np
import importlib.machinery as imm

if platform.system() == 'Darwin':
    os.environ['KMP_DUPLICATE_LIB_OK']='True'

# The absolute path to this file and directory
trainFilePath = os.path.abspath(__file__)
trainDirPath = os.path.dirname(trainFilePath)

#############################################################################################
# Define loss and metrics.

lossAndMetricsPath = os.path.join(trainDirPath, 'utils', 'Loss_and_metrics.py')

# The following get_loss() and get_metrics() will be used in coremltools when converting a trained model.
# NOTE: The metrics defined in Loss_and_metrics.py may be estimated / assumed values, NOT final results.

_LM = imm.SourceFileLoader('Loss_and_metrics', lossAndMetricsPath).load_module()

# Select a loss function for training
# loss = _LM.MSE_loss
# loss = _LM.MSE_loss_w_iou_score
# loss = _LM.MSE_loss_w_dice_coef               # Good
# loss = _LM.LogCosh_loss
# loss = _LM.LogCosh_loss_w_iou_score
# loss = _LM.LogCosh_loss_w_dice_coef           # Good
# loss = _LM.KLD_loss
# loss = _LM.KLD_loss_w_iou_score
# loss = _LM.KLD_loss_w_dice_coef

# loss = _LM.focal_CE_loss
# loss = _LM.focal_CE_loss_w_iou_score
# loss = _LM.focal_CE_loss_w_dice_coef
# loss = _LM.constrained_focal_CE_loss
# loss = _LM.constrained_focal_CE_loss_w_iou_score
# loss = _LM.constrained_focal_CE_loss_w_dice_coef    # Good
# loss = _LM.hausdorff_distance_loss
# loss = _LM.hausdorff_distance_loss_w_iou_score
loss = _LM.hausdorff_distance_loss_w_dice_coef      # Good

# loss = _LM.series_combination_loss_w_iou_score
# loss = _LM.series_combination_loss_w_dice_coef

def get_loss(): return {loss.__name__: loss}

# Metrics
# NOTE: The metrics defined here MUST include the keys 'iou_score' and 'dice_coef' for monitoring the best performance model.
# NOTE: The metrics defined in Loss_and_metrics.py are estimated / assumed values for each batch, NOT final results.
def get_metrics(): return {'iou_score': _LM.iou_score, 'dice_coef': _LM.dice_coef, 'rou': _LM.rou, 'fpoet': _LM.fpoet}

# Define loss and metrics.
#############################################################################################

#############################################################################################
# Define custom callbacks

# from tensorflow.keras.callbacks import Callback     ##### For TensorFlow v2 #####
from keras.callbacks import Callback

# NOTE: Call this BestMetricsMonitor prior to the other callbacks to update mIoU metrics with the key 'val_iou_score' and 'val_dice_coef'.
class BestMetricsMonitor(Callback):

    def __init__(self, validation, model_base_path, nn_name, patience, **kwargs):
        super(BestMetricsMonitor, self).__init__(**kwargs)
        # self.val_gen = validation
        self.val_img, gt = validation.getdata()     # Numpy array of float between 0.0 and 1.0
        self.truth = np.clip(gt, 0.0, 1.0)          # Numpy array of float between 0.0 and 1.0, Simple clip keeps the gradient within the range
        self.model_base_path = model_base_path
        self.nn_name = nn_name
        self.patience = patience
        print('Callback: BestMetricsMonitor - Start monitoring mean IoU and Dice coef to save the best model on each epoch end...')

    def on_train_begin(self, logs=None):
        self.best_miou = -np.Inf
        self.best_dice = -np.Inf
        self.wait = 0
        self.metrics_updated = False
        self.stopped_epoch = 0

    def on_epoch_end(self, epoch, logs=None):
        print('\nCalling prediction and calculating metrics for validation data...')
        # predc = np.where(self.model.predict_generator(self.val_gen) >= 0.5, 1, 0)
        # predc = np.where(self.model.predict(self.val_img) >= 0.5, 1, 0)
        t = self.truth
        p = np.clip(self.model.predict(self.val_img), 0.0, 1.0)     # Prediction for validation data
        u = np.clip(t + p, 0.0, 1.0)
        truth = t.sum(axis=(-3,-2,-1))
        predc = p.sum(axis=(-3,-2,-1))
        union = u.sum(axis=(-3,-2,-1))
        plane = np.where(t.max(axis=(-3,-2,-1)) >= 0.5, 1.0, 0.0)
        count = max(plane.sum(), 1.0)
        t_or_p = truth + predc
        intsec = t_or_p - union
        miou = np.sum(intsec / (union + 1e-4), axis=None) / count
        dice = np.sum(2.0 * intsec / (t_or_p + 1e-4), axis=None) / count

        # Update metrics dictionary with the key 'val_iou_score' / 'val_dice_coef'
        try:    logs['val_iou_score'] = miou
        except: print('ALERT: Failed to update logs[val_iou_score].')
        try:    logs['val_dice_coef'] = dice
        except: print('ALERT: Failed to update logs[val_dice_coef].')

        self.metrics_updated = False
        def create_model_path(metrics_name, metrics_val):
            return '{0}, {1}={2:.4f}, {3}.h5'.format(self.model_base_path, metrics_name, metrics_val, self.nn_name)

        # Save model with the best mean IoU
        if miou > self.best_miou:
            print('Val. mean IoU  = {0:.5f} (updated from the value = {1:.5f})'.format(miou, self.best_miou))
            path = create_model_path('mIoU', self.best_miou)
            if os.path.exists(path): os.remove(path)
            self.model.save(filepath=create_model_path('mIoU', miou), overwrite=True, include_optimizer=True)
            self.best_miou = miou
            self.metrics_updated = True
        else:
            print('Val. mean IoU  = {0:.5f} (not updated, the current best = {1:.5f})'.format(miou, self.best_miou))

        # Save model with the best Dice coef
        if dice > self.best_dice:
            print('Val. Dice coef = {0:.5f} (updated from the value = {1:.5f})'.format(dice, self.best_dice))
            path = create_model_path('Dice', self.best_dice)
            if os.path.exists(path): os.remove(path)
            self.model.save(filepath=create_model_path('Dice', dice), overwrite=True, include_optimizer=True)
            self.best_dice = dice
            self.metrics_updated = True
        else:
            print('Val. Dice coef = {0:.5f} (not updated, the current best = {1:.5f})'.format(dice, self.best_dice))

        # Early stopping of myself
        if self.metrics_updated:
            self.wait = 0
        else:
            self.wait += 1
            print('Patience count = {0} (early stop at {1} patience)'.format(self.wait, self.patience))
            if self.wait >= self.patience:
                self.stopped_epoch = epoch
                self.model.stop_training = True
                print('Epoch {0}: early stopping...'.format(self.stopped_epoch + 1))
        print(' ')

    def metrics_updated_in_Monitor(self):
        return self.metrics_updated

class AutoLRManager(Callback):

    def __init__(self, param, bm_monitor, **kwargs):
        super(AutoLRManager, self).__init__(**kwargs)
        self.p = param
        self.bm_monitor = bm_monitor
        print('Callback: AutoLRManager - Start monitoring the best metrics for learning rate management...')

    def on_train_begin(self, logs=None):
        self.multiplier = 1.0
        self.n_good = 0
        self.n_bad = 0

    def on_epoch_end(self, epoch, logs=None):
        metrics_updated = self.bm_monitor.metrics_updated_in_Monitor()
        print('Monitoring metrics for learning rate management... [metrics: {0}]'.format('updated' if metrics_updated else 'Not updated'))

        if metrics_updated:
            self.n_good += 1
            self.n_bad = 0
            step = self.p['step'][1]
            print('Count(s) of update = {0} (LR increased at {1} updates by x{2})'.format(self.n_good, self.p['patience'][1], step))
            if self.n_good >= self.p['patience'][1] and step != 1.0:
                if   step < 1.0: self.multiplier = max(self.p['limit'][1], self.multiplier * step)
                elif step > 1.0: self.multiplier = min(self.p['limit'][1], self.multiplier * step)
                print('LR multiplier is set to {0} for the next epoch (step {1}, max {2})'.format(self.multiplier, step, self.p['limit'][1]))
                self.n_good = 0
        else:
            self.n_good = 0
            self.n_bad += 1
            step = self.p['step'][0]
            print('Count(s) of not-update = {0} (LR decreased at {1} patiences by x{2})'.format(self.n_bad, self.p['patience'][0], step))
            if self.n_bad >= self.p['patience'][0] and step != 1.0:
                if   step < 1.0: self.multiplier = max(self.p['limit'][0], self.multiplier * step)
                elif step > 1.0: self.multiplier = min(self.p['limit'][0], self.multiplier * step)
                print('LR multiplier is set to {0} for the next epoch (step {1}, min {2})'.format(self.multiplier, step, self.p['limit'][0]))
                self.n_bad = 0

        print(' ')

    def get_LR_multiplier(self):
        return self.multiplier

# Define custom callbacks
#############################################################################################

#############################################################################################
# Main training sequence

def Train():

    import shutil
    import time
    import platform
    import matplotlib.pyplot as plt
    from datetime import datetime, timedelta, timezone

    import tensorflow as tf

    import keras as _keras
    import keras.backend as K
    # from keras.callbacks import ModelCheckpoint, EarlyStopping, CSVLogger, TensorBoard, LearningRateScheduler
    from keras.callbacks import CSVLogger, LearningRateScheduler
    from keras.utils import plot_model
    from keras.models import load_model

    ##### For TensorFlow v2 #####
#    from tensorflow import keras
#    from tensorflow.keras import backend as K
#    # from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, CSVLogger, TensorBoard, LearningRateScheduler
#    from tensorflow.keras.callbacks import CSVLogger, LearningRateScheduler
#    from tensorflow.keras.utils import plot_model
#    from tensorflow.keras.models import load_model

    K.clear_session()

    # NOTE: Data must be (samples, height, width, channels)
    if K.image_data_format() is not 'channels_last': K.set_image_data_format('channels_last')

    JST = timezone(timedelta(hours=+9), 'JST')      # Japan Standard Time, Change it for your time zone
    startdate = datetime.now(JST)
    starttime = time.time()

    ############################################
    # Training parameters

    # Load loss and metrics
    custom_loss = get_loss()
    custom_metrics = get_metrics()

    # Neural network model
    NN_model_path = sys.argv[1]
    NN = imm.SourceFileLoader(os.path.splitext(os.path.basename(NN_model_path))[0], NN_model_path).load_module()
    NN_info = []

    try:    NN_model_name    = NN.Model_Name()
    except: NN_model_name, _ = NN.__name__, NN_info.append('ALERT: Define a model name in the neural network model file.')

    try:    NN_model_descript    = NN.Model_Description()
    except: NN_model_descript, _ = 'Empty description.', NN_info.append('ALERT: Define description for the model in the neural network model file.')

    try:    NN_num_classes    = NN.Number_of_Classes()
    except: NN_num_classes, _ = 1, NN_info.append('NOTE: The number of classes was not defined in the neural network model file, automatically set to 1.')

    try:    NN_batch_size    = NN.Batch_Size()
    except: NN_batch_size, _ = 32, NN_info.append('NOTE: The batch size was not defined in the neural network model file, automatically set to 32.')

    # Initial parameters
    LR_params = {'formula'          : [None, 0.0, 0],               # Learning rate formula calculates LR at points of epochs - ['poly', base_lr, number_of_epochs] is available
                 'graph'            : [[0,4e-3], [100,2e-3]],       # Learning rate graph defines LR at points of epochs - [[epoch_1, LR_1], [epoch_2, LR_2], ... [epoch_last, LR_last]]
                 'step'             : [0.1, 2.5],                   # Multiplying values to LR - will be applied when mIoU is [NOT improved, improved]
                 'limit'            : [0.001, 1.0] ,                # Limitation of LR multiplier - when [NOT improved, improved]
                 'patience'         : [1000, 1000],                 # Patience counts before applying step for LR - when [NOT improved, improved]
                 'stop_count'       : 50 }                          # Define a count number before early stopping

    LR_params_set = True
    try:    LR_params = NN.Learning_Rate_Parameters()
    except: LR_params_set, _ = False, NN_info.append('ALERT: The list of learning rate parameters was not defined in the neural network model file, trying to set separately.')

    if not LR_params_set:
        try:    LR_params['formula']    = NN.Learning_Rate_Formula()
        except: NN_info.append('ALERT: The LR formula was not defined in the neural network model file, automatically deactivated.')

        try:    LR_params['graph']    = NN.Learning_Rate_Lsit()
        except: NN_info.append('ALERT: The LR graph was not defined in the neural network model file, automatically set.')
        # LR_params['graph'] = [[0,3e-3], [3,3.2e-3], [12,4.5e-3], [30,4.5e-3], [50,3e-3], [80,1e-3], [100,5e-4], [150,2e-4]]
        # LR_params['graph'] = [[0, 7.81e-4], [5, 7.81e-4], [15, 6e-4], [30, 2e-4], [35, 1e-4], [50, 2e-5]]     # For aorta
        # LR_params['graph'] = [[0, 7.81e-4], [30, 7.81e-4], [50, 6e-4], [100, 2e-4], [140, 1e-4], [200, 1e-5]]     # For heart
        # LR_params['graph'] = [[0,2e-3], [50,2e-3], [80,1.5e-3], [100,1.1e-3], [150,3e-4], [200,1e-4]]     # For heart 20191116
        # LR_params['graph'] = [[0,3e-3], [3,3.2e-3], [12,4.5e-3], [30,4.5e-3], [50,3e-3], [80,1e-3], [100,5e-4], [150,2e-4]]                     # For heart 20200106
        # LR_params['graph'] = [[0,3e-3], [3,3.2e-3], [12,4.8e-3], [30,4.8e-3], [50,3.8e-3], [80,2e-3], [100,1.1e-3], [150,3e-4], [200,1e-4]]     # For heart 20190903
        # LR_params['graph'] = [[0,3e-3], [3,3.2e-3], [12,4.8e-3], [30,4.8e-3], [50,3.8e-3], [80,2e-3], [100,1.1e-3], [180,3e-4], [250,6e-5]]     # For heart 20190903
        # LR_params['graph'] = [[0,3e-3], [3,3.2e-3], [12,4.8e-3], [30,4.8e-3], [50,3.8e-3], [80,2e-3], [120,1.1e-3], [180,3e-4], [250,1e-4]]       # For heart 20191107
        # LR_params['graph'] = [[0,1e-3], [100,1e-3]]       # For heart 20191118 Paper
        # LR_params['graph'] = [[0, 1.5e-4], [30, 1.0e-4], [80, 0.2e-4], [130, 0.05e-4], [180, 0.01e-4]]     # For heart
        # LR_params['graph'] = [[0, 7.81e-4], [2, 7.81e-4], [10, 2e-5]]     # For bone

        try:    LR_params['patience']    = NN.Count_before_LR_Step()
        except: NN_info.append('ALERT: The patience count for LR step was not defined in the neural network model file, automatically set.')

        try:    LR_params['stop_count']    = NN.Count_before_Stop()
        except: NN_info.append('ALERT: The count for early stopping was not defined in the neural network model file, automatically set.')

    if LR_params['formula'][0] is None: number_of_epochs = LR_params['graph'][-1][0]
    else:                               number_of_epochs = LR_params['formula'][2]

    # Training parameters
    ############################################

    # Resume training
    if sys.argv[5] == '1':
        init_epoch = int(sys.argv[7]) - 1      # Starting from zero
        if init_epoch < 0 or init_epoch >= number_of_epochs:
            init_epoch = min(number_of_epochs-1, max(0, init_epoch))
            print('ALART : Initial epoch [{0}] is clipped between 0 and {1}'.format(sys.argv[7], number_of_epochs-1))
        trained_model_path = sys.argv[6]
        training_mode = 'Resume training'
        NN_model_descript = 'The following model will be resumed - ' + trained_model_path + '\n\t\t\t' + NN_model_descript

    # Normal training or Retraining
    elif sys.argv[5] == '0' or sys.argv[5] == '2':
        init_epoch = 0       # Starting from zero
        if sys.argv[5] == '0':
            trained_model_path = None
            training_mode = 'Normal training'
        else:
            trained_model_path = sys.argv[6]
            training_mode = 'Retraining with trained wieghts'
            NN_model_descript = 'Trained weights will be loaded from - ' + trained_model_path + '\n\t\t\t' + NN_model_descript

    else:
        print('ERROR : Invalid training mode!!!')
        sys.exit()

    # Loaded neural network code may not have Custom_Layers()
    try:    custom_layers    = NN.Custom_Layers()
    except: custom_layers, _ = {}, \
                NN_info.append('ALERT: The dictionary of custom layers was not defined in the neural network model file, automatically set to empty.')

    # Training mode: 0=Normal, 1=Resume the model, 2=Boost the model weights
    if sys.argv[5] == '0':
        model = NN.Build_Model()
    elif sys.argv[5] == '1':
        model = load_model(trained_model_path, custom_objects=dict(**custom_loss, **custom_metrics, **custom_layers), compile=False)
    elif sys.argv[5] == '2':
        model = NN.Build_Model()
        model.load_weights(trained_model_path, by_name=True)
    else:
        print('Invalid mode.')
        sys.exit()

    # Optimizers
    from keras.optimizers import SGD, Adam, Nadam
    # from tensorflow.keras.optimizers import SGD, Adam, Nadam      ##### For TensorFlow v2 #####
    # from optimizers.AdaBound1.adabound1 import AdaBound1
    # from optimizers.AdaBound2.adabound2 import AdaBound2
    # from optimizers.Santa.Santa import Santa
    if LR_params['formula'][0] is not None: base_lr = LR_params['formula'][1]
    elif LR_params['graph'] is not None:    base_lr = LR_params['graph'][0][1]
    else:
        print('Invalid learning rate.')
        sys.exit()

    try:    optimizer    = NN.Optimizer(base_lr=base_lr)
    except: optimizer, _ = Nadam(lr=base_lr, beta_1=0.9, beta_2=0.999), \
                NN_info.append('ALERT: The optimizer was not defined in the neural network model file, automatically set to Nadam.')
    # except: optimizer, _ = SGD(lr=base_lr, momentum=0.9, nesterov=True), \
    # except: optimizer, _ = Adam(lr=base_lr, beta_1=0.9, beta_2=0.999, amsgrad=False), \
    # except: optimizer, _ = Nadam(lr=base_lr, beta_1=0.9, beta_2=0.999), \
    # except: optimizer, _ = AdaBound1(lr=base_lr, final_lr=0.5, beta_1=0.9, beta_2=0.999, gamma=1e-3, amsbound=False, weight_decay=0.0), \
    # except: optimizer, _ = AdaBound2(lr=base_lr, beta_1=0.9, beta_2=0.999, terminal_bound=0.5, lower_bound=0.0, upper_bound=None), \
    # except: optimizer, _ = Santa(lr=base_lr, exploration=max(number_of_epochs/2, number_of_epochs-20), rho=0.95, anne_rate=0.5), \

    # Compile
    model.compile(optimizer          = optimizer,
                  loss               = loss,
                  metrics            = list(custom_metrics.values()),
                  loss_weights       = None,
                  sample_weight_mode = None,
                  weighted_metrics   = None,
                  target_tensors     = None )

    # Paths and directories
    datestr = startdate.strftime("%Y%m%d%H%M%S")
    work_dir_path   = os.path.join(sys.argv[4], 'run'+datestr+' ('+training_mode+')')
    code_dir_path   = os.path.join(work_dir_path, 'code')
    NN_dir_path     = os.path.join(code_dir_path, 'neural_networks')
    utils_dir_path  = os.path.join(code_dir_path, 'utils')
    model_base_path = os.path.join(work_dir_path, 'model'+datestr)

    os.makedirs(NN_dir_path)
    os.makedirs(utils_dir_path)
    shutil.copy2(trainFilePath,   os.path.join(code_dir_path,  os.path.basename(trainFilePath)))            # Copy this file
    shutil.copy2(NN_model_path, os.path.join(NN_dir_path,    os.path.basename(NN_model_path)))              # Copy model file
    shutil.copy2(lossAndMetricsPath, os.path.join(utils_dir_path, os.path.basename(lossAndMetricsPath)))    # Copy loss and metrics file
    if custom_layers:
        shutil.copy2(os.path.join(trainDirPath, 'neural_networks', 'Custom_layers.py'), os.path.join(NN_dir_path, 'Custom_layers.py'))   # Copy layer file

    # Descriptions
    model.summary()
    print('Date                    : {0}'.format(startdate))
    print('TensorFlow version      : {0}'.format(tf.version.VERSION))
    print('TF-Keras version        : {0}'.format(tf.keras.__version__))
    print('Keras version           : {0}'.format(_keras.__version__))
    print('OS-version              : {0}'.format(platform.platform()))
    print('Processor               : {0}'.format(platform.processor()))
    print('__________________________________________________________________________________________________')
    print('Training mode           : {0}'.format(training_mode))
    print('Model name              : {0}'.format(NN_model_name))
    print('Model description       : {0}'.format(NN_model_descript))
    print('Number of classes       : {0}'.format(NN_num_classes))
    print('Loaded model path       : {0}'.format(NN_model_path))
    print('Working directory       : {0}'.format(work_dir_path))
    print('__________________________________________________________________________________________________')
    print('Keras data format       : {0}'.format(K.image_data_format()))
    print('Optimizer               : {0}'.format(optimizer.__class__.__name__))
    print('Loss                    : {0}'.format(loss.__name__))
    print('Metrics                 : {0}'.format(model.metrics_names[1:]))      # model.metrics_names[0] = 'loss'
    print('Custom layers           : {0}'.format(list(custom_layers.keys()) ))
    print('Batch size              : {0}'.format(NN_batch_size))
    print('Epochs                  : {0} - {1}'.format(init_epoch+1, number_of_epochs))
    print('Learning rate formula   : {0}'.format(LR_params['formula']))
    print('Learning rate graph     : {0}'.format(LR_params['graph']))
    print('LR step                 : {0}'.format(LR_params['step']))
    print('LR limit                : {0}'.format(LR_params['limit']))
    print('Patience for LR step    : {0}'.format(LR_params['patience']))
    print('Patience for early stop : {0}'.format(LR_params['stop_count']))
    print('__________________________________________________________________________________________________')
    if len(NN_info) > 0:
        for info in NN_info: print(info)
        print('__________________________________________________________________________________________________')

    # Image data generator
    '''
        For TensorFlow 2.0
        Model.fit_generator IS DEPRECATED.
        To use Model.fit, generator classes, ImageDataGenerator_XXX(), were updated as subclasses of keras.utils.Sequence.

        See:
        https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit
        https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit_generator
    '''
    from utils.Image_data_generator import ImageDataGenerator_CSV_with_Header, ImageDataGenerator_h5_Dataset
    print('\n- Loading images for training...')
    ext = os.path.splitext(sys.argv[2])[1]
    if   ext == '.csv' :  training_images = ImageDataGenerator_CSV_with_Header('Train data from CSV', sys.argv[2], batch_size=NN_batch_size, rescale=1.0/255.0, shuffle=True)
    elif ext == '.h5'  :  training_images = ImageDataGenerator_h5_Dataset('image_training', sys.argv[2], batch_size=NN_batch_size, rescale=1.0/255.0)
    else               :  sys.exit()
    print('\n- Loading images for validation...')
    ext = os.path.splitext(sys.argv[3])[1]
    if   ext == '.csv' :  validation_images = ImageDataGenerator_CSV_with_Header('Validation data from CSV', sys.argv[3], batch_size=NN_batch_size, rescale=1.0/255.0, shuffle=True)
    elif ext == '.h5'  :  validation_images = ImageDataGenerator_h5_Dataset('image_validation', sys.argv[3], batch_size=NN_batch_size, rescale=1.0/255.0)
    else               :  sys.exit()

    # Allocate callbacks
    print('\n- Defining callbacks...')
    bm_monitor = BestMetricsMonitor(validation=validation_images, model_base_path=model_base_path, nn_name=NN_model_name, patience=LR_params['stop_count'])
    lr_manager = AutoLRManager(param=LR_params, bm_monitor=bm_monitor)

    def ScheduleLR(epoch, lr):
        import math
        raw_lr = lr
        if LR_params['formula'][0] == 'poly':
            # See https://arxiv.org/pdf/1506.04579.pdf
            print('Learning rate by poly: base_lr = {0}, power = 0.9'.format(LR_params['formula'][1]))
            raw_lr = LR_params['formula'][1] * math.pow(1 - epoch / number_of_epochs, 0.9)
        # elif LR_params['formula'][0] == 'XXX':
        #     print('Learning rate by XXX: ...
        #     raw_lr = LR_params['formula'][1] ...
        elif LR_params['graph'] is not None:
            def LR_at_epoch(epoch, pt1, pt2): return (pt2[1] - pt1[1]) / (pt2[0] - pt1[0]) * (epoch - pt1[0]) + pt1[1]
            print('Learning rate by graph [epoch, LR] : {0}'.format(LR_params['graph']))
            for i in range(len(LR_params['graph'])-1):
                if LR_params['graph'][i][0] <= epoch and epoch < LR_params['graph'][i+1][0]:
                    raw_lr = LR_at_epoch(epoch, LR_params['graph'][i], LR_params['graph'][i+1])
                    break
        m = lr_manager.get_LR_multiplier()
        new_LR = m * raw_lr
        print('LR = {0} (raw LR = {1}, multiplier = {2})'.format(new_LR, raw_lr, m))
        return new_LR

    # check_pointer = ModelCheckpoint(model_base_path, monitor=LR_params['monitor_for_best'][0], verbose=1, save_best_only=True, mode=LR_params['monitor_for_best'][1])
    # early_stopper = EarlyStopping(monitor=LR_params['monitor_for_best'][0], min_delta=0, patience=LR_params['stop_count'], verbose=1, mode=LR_params['monitor_for_best'][1])
    lr_scheduler = LearningRateScheduler(ScheduleLR, verbose=0)
    csv_logger = CSVLogger(os.path.join(work_dir_path,'training_log.csv'), separator=',', append=False)
    # tensorboard = TensorBoard(log_dir=work_dir_path, histogram_freq=0, write_graph=True, write_images=True)

    # Save descriptions, network figure and parameters
    plot_model(model, to_file=os.path.join(work_dir_path,'model_figure.png'), show_shapes=True, show_layer_names=False)
    with open(os.path.join(work_dir_path,'training_parameters.txt'), mode='w') as path_file:
        path_file.write('Date                    : {0}\n'.format(startdate))
        path_file.write('TensorFlow version      : {0}\n'.format(tf.version.VERSION))
        path_file.write('TF-Keras version        : {0}\n'.format(tf.keras.__version__))
        path_file.write('Keras version           : {0}\n'.format(_keras.__version__))
        path_file.write('OS-version              : {0}\n'.format(platform.platform()))
        path_file.write('Processor               : {0}\n\n'.format(platform.processor()))
        path_file.write('Training mode           : {0}\n'.format(training_mode))
        path_file.write('Model name              : {0}\n'.format(NN_model_name))
        path_file.write('Model description       : {0}\n'.format(NN_model_descript))
        path_file.write('Number of classes       : {0}\n'.format(NN_num_classes))
        path_file.write('Loaded model path       : {0}\n'.format(NN_model_path))
        path_file.write('Working directory       : {0}\n\n'.format(work_dir_path))
        path_file.write('Training images         : {0} sets in {1}\n'.format(training_images.datalength(), sys.argv[2]))
        path_file.write('Validation images       : {0} sets in {1}\n\n'.format(validation_images.datalength(), sys.argv[3]))
        path_file.write('Keras data format       : {0}\n'.format(K.image_data_format()))
        path_file.write('Optimizer               : {0}\n'.format(optimizer.__class__.__name__))
        path_file.write('Loss                    : {0}\n'.format(loss.__name__))
        path_file.write('Metrics                 : {0}\n'.format(model.metrics_names[1:]))
        path_file.write('Custom layers           : {0}\n'.format(list(custom_layers.keys()) ))
        path_file.write('Batch size              : {0}\n'.format(NN_batch_size))
        path_file.write('Epochs                  : {0} - {1}\n'.format(init_epoch+1, number_of_epochs))
        path_file.write('Learning rate formula   : {0}\n'.format(LR_params['formula']))
        path_file.write('Learning rate graph     : {0}\n'.format(LR_params['graph']))
        path_file.write('LR step                 : {0}\n'.format(LR_params['step']))
        path_file.write('LR limit                : {0}\n'.format(LR_params['limit']))
        path_file.write('Patience for LR step    : {0}\n\n'.format(LR_params['patience']))
        path_file.write('Patience for early stop : {0}\n'.format(LR_params['stop_count']))
        if len(NN_info) > 0:
            for info in NN_info: path_file.write('{}\n'.format(info))
        path_file.write('\n')
        model.summary(print_fn=lambda x: path_file.write(x + '\n'))

    # Train the model
    '''
        For TensorFlow 2.0
        fit_generator -> fit

        Warning: Model.fit_generator IS DEPRECATED. It will be removed in a future version.
        Instructions for updating: Please use Model.fit, which supports generators.

        See:
        https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit
        https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit_generator
    '''
    print('\n- Starting model training...')
    results = model.fit_generator(
        generator               = training_images,
        epochs                  = number_of_epochs,
        verbose                 = 1,
        callbacks               = [bm_monitor, lr_manager, lr_scheduler, csv_logger],
        validation_data         = validation_images,
        max_queue_size          = 10,
        workers                 = 1,
        use_multiprocessing     = False,
        shuffle                 = False,
        initial_epoch           = init_epoch )

#    results = model.fit_generator(
#        generator               = training_images.flow(),
#        steps_per_epoch         = training_images.datalength() // NN_batch_size,
#        epochs                  = number_of_epochs,
#        verbose                 = 1,
#        callbacks               = [bm_monitor, lr_manager, lr_scheduler, csv_logger],
#        validation_data         = validation_images.flow(),
#        validation_steps        = validation_images.datalength() // NN_batch_size,
#        max_queue_size          = 10,
#        workers                 = 1,
#        use_multiprocessing     = False,
#        shuffle                 = False,
#        initial_epoch           = init_epoch )

    ##### For TensorFlow v2 #####
#    results = model.fit(
#        x                       = training_images,                  # keras.utils.Sequence
#        epochs                  = number_of_epochs,
#        verbose                 = 1,
#        callbacks               = [bm_monitor, lr_manager, lr_scheduler, csv_logger],
#        validation_data         = validation_images.getdata(),      # tuple of Numpy arrays
#        shuffle                 = False,
#        initial_epoch           = init_epoch,
#        validation_freq         = 1,
#        max_queue_size          = 10,
#        workers                 = 1,
#        use_multiprocessing     = False )

    # Show results
    print('\n- Saving training graph...')
    try:
        his_loss = results.history['loss']
        his_miou = results.history['iou_score']
        his_dice = results.history['dice_coef']
        his_valloss = results.history['val_loss']
        his_valmiou = results.history['val_iou_score']
        his_valdice = results.history['val_dice_coef']
        xlen = range(len(his_loss))

        fig = plt.figure()
        ax1 = fig.add_subplot(111)      # Loss
        ax2 = ax1.twinx()

        ax1.plot(xlen, his_loss, marker='.', color='salmon', label='Loss - training')
        ax1.plot(xlen, his_valloss, marker='.', color='red', label='Loss - validation')
        ax2.plot(xlen, his_miou, marker='.', color='deepskyblue', label='mIoU - training')
        ax2.plot(xlen, his_valmiou, marker='.', color='blue', label='mIoU - validation')
        ax2.plot(xlen, his_dice, marker='.', color='limegreen', label='Dice - training')
        ax2.plot(xlen, his_valdice, marker='.', color='green', label='Dice - validation')

        ax1.set_xlabel('Epoch')
        ax1.set_ylabel('Loss:'+loss.__name__)
        ax1.set_yscale("log")
        ax1.set_ylim([0.001, 10.0])
        ax2.set_ylabel('Metrics')
        ax2.set_yscale("log")
        ax2.set_ylim([0.6, 1.0])

        h1, l1 = ax1.get_legend_handles_labels()
        h2, l2 = ax2.get_legend_handles_labels()
        ax1.legend(h1+h2, l1+l2, loc='lower center')

        plt.savefig(os.path.join(work_dir_path,'training_graph.png'))
        # plt.show()

    except:
        print('ALERT: Failed to save the training graph figure.')

    print('\n==================================================================================================')
    print('Computation time        : {0}'.format(timedelta(seconds=time.time()-starttime)))
    print('From the date           : {0}\n'.format(startdate))
    print('==================================================================================================')

# Main
if __name__ == '__main__': Train()


text:
#!/usr/bin/env python3

"""
File: section3/3_1_embeddings.py
Author: Brandon McKinzie
Description: Simple autoencoder model to illustrate embeddings with TensorFlow.
    1. Build {word => counts} dictionary from training dataset.
    2. Store as vocabulary file (always useful to have on-hand).
"""

import os
import numpy as np
import argparse
import tensorflow as tf
from tensorflow.contrib.training import HParams

import dataset
import components
from util import newsgroups, tfrecords

tf.logging.set_verbosity(tf.logging.INFO)

parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, default=64)
parser.add_argument('--data_dir', default='data')
parser.add_argument('--processed_data_dir', default='processed_data')
parser.add_argument('--model_dir', default='model_dir')

parser.add_argument('--predict_only', action='store_true', default=False)

parser.add_argument('-V', '--vocab_size', default=5000)
parser.add_argument('-T', '--max_seq_len', default=20)
parser.add_argument('-E', '--embed_size', default=100)

parser.add_argument('--hidden_size', default=256)
args = parser.parse_args()

def input_fn(data_dir, params):
    text_batch, _ = dataset.Dataset(data_dir, params).make_batch(params.batch_size)
    return text_batch, text_batch

def model_fn(features, labels, mode, params):
    """
    Args:
        features: Tensor with shape (batch_size, max_seq_len) with dtype int.
        labels: same as features, since we are training an autoencoder.
        params: tf.contrib.HParams object containing...
            embed_size, vocab_size
    """
    if mode == tf.estimator.ModeKeys.PREDICT:
        features = features['x']

    # Load and build the embedding layer, initialized with pre-trained
    # GloVe embeddings.
    embedded_features = components.glove_embed(
        features,
        embed_shape=(params.vocab_size, params.embed_size),
        vocabulary=params.vocab)

    hidden_layer = tf.layers.Dense(
        params.hidden_size, activation=tf.tanh)(embedded_features)
    logits = tf.layers.Dense(params.vocab_size)(hidden_layer)

    if mode == tf.estimator.ModeKeys.PREDICT:
        output_probs = tf.nn.softmax(logits)
        preds = tf.argmax(output_probs, -1)
        table = tf.contrib.lookup.index_to_string_table_from_tensor(params.vocab)
        preds_words = table.lookup(preds)
        return tf.estimator.EstimatorSpec(
            mode, predictions=preds_words)

    loss = tf.losses.sparse_softmax_cross_entropy(
        labels=labels, logits=logits)
    train_op = tf.contrib.layers.optimize_loss(
        loss=loss, global_step=tf.train.get_or_create_global_step(),
        learning_rate=0.01,
        optimizer='Adam')

    with tf.variable_scope('accuracy', values=[labels, logits]):
        flattened_logits = tf.reshape(
            logits, [params.batch_size * params.max_seq_len, -1])
        flattened_labels = tf.reshape(
            labels, [params.batch_size * params.max_seq_len,])
        output_probs = tf.nn.softmax(flattened_logits)
        num_correct = tf.to_float(tf.nn.in_top_k(
            output_probs, tf.to_int64(flattened_labels), 1))
        accuracy = tf.reduce_mean(num_correct)

    tf.summary.scalar('accuracy', accuracy)
    if mode == tf.estimator.ModeKeys.TRAIN:
        logging_hook = tf.train.LoggingTensorHook({
            'loss': loss,
            'acc': accuracy
        }, every_n_iter=100)
        return tf.estimator.EstimatorSpec(
            mode,
            loss=loss,
            train_op=train_op,
            training_hooks=[logging_hook])

def main(_):

    # tf.estimator will load/reuse anything found in its model_dir, so
    # we make sure to clear its contents before every training run.
    # For predictions, however, we of course want to load the previously
    # trained model from disk.
    if tf.gfile.Exists(args.model_dir) and not args.predict_only:
        tf.gfile.DeleteRecursively(args.model_dir)
    tf.gfile.MakeDirs(args.model_dir)

    hparams = HParams(**vars(args))

    # We will use the 20 newsgroups dataset to train our model.
    # Note that we won't be using the labels, since our model is simply
    # learning to reconstruct its inputs as its output.
    train_file_path = os.path.join(hparams.data_dir, '20ng-train-all-terms.txt')

    # Define the path to the file that we'll store our vocabulary in.
    # This file will have the same number of lines as our vocab_size.
    # Each line will contain a single word in our vocabulary, listed in
    # order of decreasing frequency seen in our training data.
    vocab_path = os.path.join(hparams.processed_data_dir, 'vocab.txt')

    # Data preparation: getting vocabulary and saving tfrecords format.
    if not tf.gfile.Exists(vocab_path):
        print('Extracting vocab, labels, and tokenized texts from data.')
        vocab, labels, texts = newsgroups.fit_and_extract(
            train_file_path, hparams.vocab_size)
        print('Saving vocabulary to {}.'.format(vocab_path))
        with open(vocab_path, 'w+') as f:
            f.write('\n'.join(vocab))

        tfrecords_path = os.path.join(hparams.processed_data_dir, 'embed.tfrecords')
        print('Saving tfrecords to {}.'.format(tfrecords_path))
        tfrecords.save_tfrecords(
            out_path=tfrecords_path,
            labels=labels,
            texts=texts,
            vocab=vocab)
    else:
        print('Reading existing vocabulary from {}.'.format(vocab_path))
        with open(vocab_path) as f:
            vocab = [l.strip() for l in f.readlines()]

    hparams.vocab = vocab
    print('Creating autoencoder.')
    autoencoder = tf.estimator.Estimator(
        model_fn=model_fn,
        model_dir=hparams.model_dir,
        config=tf.estimator.RunConfig(log_step_count_steps=10000),
        params=hparams)

    if not args.predict_only:
        print('Training autoencoder.')
        autoencoder.train(
            input_fn=lambda: input_fn(hparams.processed_data_dir, hparams),
            steps=1000)

    sample_sentences = [
        'i like dogs',
        'i am a test sentence',
        'TensorFlow is a fun library to use']
    pred_inputs = []
    for sent in sample_sentences:
        token_ids = [vocab.index(w)
                     for w in sent.split()[:args.max_seq_len]
                     if w in vocab]
        # Pad if necessary.
        if len(token_ids) < args.max_seq_len:
            token_ids.extend([0] * (args.max_seq_len - len(token_ids)))
        pred_inputs.append(token_ids)

    pred_inp_fn = tf.estimator.inputs.numpy_input_fn(
        x={'x': np.asarray(pred_inputs)}, shuffle=False)
    predictions = autoencoder.predict(input_fn=pred_inp_fn)

    print('Sample predictions:')
    for i, prediction in enumerate(predictions):
        clean_prediction = ' '.join([tok.decode() for tok in prediction if tok != b'_UNK'])
        print('\nExpected:', sample_sentences[i], sep='\t')
        print('Actual:  ', clean_prediction, sep='\t')

if __name__ == '__main__':
    tf.app.run()


text:
#!/usr/bin/python3
import cv2
import numpy as np
import sys, os
import json
import traceback
from pynput import mouse
import argparse
import matplotlib.pyplot as plt
import matplotlib
import threading
import queue
import time

minFrames = 30      # minimum amount of frames to capture per coordinate
aparser = argparse.ArgumentParser()
aparser.add_argument('-tSet', help='Choose training set')
aparser.add_argument('-outputDir', help='Choose directory to save training data')
aparser.add_argument('-view', help='Visualise training data', action='store_true')
aparser.add_argument('-fs', help='Full screen training: supply display config (e.g. xx/xx for 4 displays in 2 rows)')
aparser.add_argument('-save', help='Save config into cfg.json', action='store_true')
args = aparser.parse_args()

# read / write cfg file
if os.path.isfile('cfg.json'):
    with open('cfg.json', 'r') as cfgfile:
        cfg = json.load(cfgfile)
else:
    cfg = {}
if args.tSet or 'tSet' not in cfg:
    cfg['tSet'] = args.tSet
if args.outputDir or 'capturepath' not in cfg:
    cfg['capturepath'] = args.outputDir
if args.save:
    with open('cfg.json', 'w') as cfgfile:
        cfgfile.write(json.dumps(cfg, sort_keys=True, indent=2))

outputDir = os.path.realpath(cfg['capturepath'])
trainingFile = os.path.join(outputDir, '{}.json'.format(cfg['tSet']))
outputDataDir = os.path.join(outputDir, cfg['tSet'])

# read database file:
if os.path.isfile(trainingFile):
    with open(trainingFile) as jsonfile:
        trainingData = json.load(jsonfile)

    if args.view:
        X = np.asarray([k['x'] for k in trainingData])
        Y = np.asarray([k['y'] for k in trainingData])
        fig, axes = plt.subplots(2, 1, constrained_layout=True)
        axes[0].hist2d(X, Y, (50, 50), cmap=plt.cm.jet, norm = matplotlib.colors.LogNorm())
        axes[0].set_title('log hist')
        axes[1].scatter(X, Y, marker = 'x')
        axes[1].set_title('{} samples'.format(len(trainingData)))
        for ax in axes:
            ax.set_xlabel('pixel vertical')
            ax.set_ylabel('pixel horizontal')
        fig = plt.gcf()
        fig.canvas.set_window_title(cfg['tSet'])
        plt.show()
        sys.exit()
else:
    trainingData = []

def savefile():
    # save json file
    with open(trainingFile, 'w') as jsonfile:
        jsonfile.write(json.dumps(trainingData,
                                    sort_keys = True,
                                    indent = 2))
        print('saved as ' + trainingFile)

if args.fs:
    # show fs training app
    import tkinter as tk
    print('Use arrowkeys to navigate to a node.')
    print('Use <ENTER> to start recording at this node.')
    print('Use <SPACE> to switch to next screen.')
    print('Once a node is green enough frames have been recorded!')
    print('Recording can be resumed any time.')

    capQ = queue.Queue(20)
    class CaptureThread(threading.Thread):
        def run(self):
            # configure capture
            cap = cv2.VideoCapture(0)
            cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
            cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
            cap.set(cv2.CAP_PROP_FPS, 60)
            self.capture = True
            if not cap.isOpened():
                print('Error opening device!')
                sys.exit()

            while self.capture:
                ret, frame = cap.read()
                if ret:
                    frame = cv2.flip(frame, 1)
                    capQ.put(frame)
            cap.release()

        def stop(self):
            self.capture = False

    class CaptureHandler(threading.Thread):
        def record(self, x, y):
            self.x = x
            self.y = y
            self.recording = True
            self.frames = 0

        def stopRecording(self):
            self.recording = False

        def run(self):
            self.capture = True
            self.recording = False
            if not os.path.exists(outputDataDir):
                os.makedirs(outputDataDir)
            while self.capture:
                start = time.time()
                frame = capQ.get()
                if self.recording:
                    self.frames += 1
                    fname = 'cap-{:06d}.jpg'.format(len(trainingData))
                    cv2.imwrite(os.path.join(outputDataDir, fname), frame)
                    trainingData.append({'x': self.x,
                                         'y': self.y,
                                         'f': fname})
                    print(' -> ' + os.path.join(outputDataDir, fname) + ' ({:5.2f} fps)'.format(1.0 / (time.time() - start)))

        def stop(self):
            self.capture = False

    class gPoint(object):
        def __init__(self, x, y):
            self.x = x
            self.y = y
            self.frames = 0

    class xyz(object):
        def __init__(self):
            self.cThread = CaptureThread()
            self.cThread.start()
            self.cHandler = CaptureHandler()
            self.cHandler.start()

            self.root = tk.Tk()
            self.canv = tk.Canvas(self.root)
            self.canv.pack()
            self.width, self.height = (self.root.winfo_screenwidth(), self.root.winfo_screenheight())
            self.currPosX = 0
            self.currPosY = 0
            self.offsetX = 0
            self.offsetY = 0
            self.cwidth = 0
            self.cheight = 0
            self.incCol = False
            self.incRow = False
            self.killed = False
            self.recording = False
            self.root.attributes('-fullscreen', True)
            self.root.after(100, self.root.quit)
            self.root.mainloop()
            # now we have our display dimensions
            self.updGeom()
            self.drawCurr()
            self.bindKeys()

        def bindKeys(self):
            self.root.bind('<Escape>', self.kill)
            self.root.bind('<Left>', self.moveL)
            self.root.bind('<Right>', self.moveR)
            self.root.bind('<Up>', self.moveU)
            self.root.bind('<Down>', self.moveD)
            self.root.bind('<Return>', self.record)
            self.root.bind('<space>', self.nextw)

        def unbindKeys(self):
            self.root.unbind('<Escape>')
            self.root.unbind('<Left>')
            self.root.unbind('<Right>')
            self.root.unbind('<Up>')
            self.root.unbind('<Down>')
            self.root.unbind('<space>')

        def kill(self, ev):
            self.cThread.stop()
            self.cHandler.stop()
            self.killed = True
            self.root.destroy()

        def moveR(self, ev):
            if self.currPosX < len(self.grid) - 1:
                self.currPosX += 1
                self.drawCurr()

        def moveL(self, ev):
            if self.currPosX > 0:
                self.currPosX -= 1
                self.drawCurr()

        def moveU(self, ev):
            if self.currPosY > 0:
                self.currPosY -= 1
                self.drawCurr()

        def moveD(self, ev):
            if self.currPosY < len(self.grid[0]) - 1:
                self.currPosY += 1
                self.drawCurr()

        def drawCurr(self):
            self.canv.delete('all')
            self.drawXs()
            currpt = self.grid[self.currPosX][self.currPosY]
            self.drawX(currpt.x, currpt.y, 'blue', 3.)

        def updGeom(self):
            self.offsetX += self.cwidth if self.incCol else 0
            self.offsetY += self.cheight if self.incRow else 0
            self.cwidth, self.cheight = [int(k) for k in self.root.geometry().split('+')[0].split('x')]
            self.canv.config(width=self.cwidth, height=self.cheight)
            rangeY = range(20, self.cheight + 10, 100)
            rangeX = range(20, self.cwidth + 10, 100)
            self.grid = [[gPoint(x, y) for y in rangeY] for x in rangeX]
            if os.path.exists(trainingFile):
                with open(trainingFile) as tf:
                    content = json.load(tf)
                for data in content:
                    if data['x'] - self.offsetX in rangeX and data['y'] - self.offsetY in rangeY:
                        self.grid[rangeX.index(data['x'] - self.offsetX)][rangeY.index(data['y'] - self.offsetY)].frames += 1
            self.drawXs()

        def drawX(self, x, y, col='black', width = 1.):
            self.canv.create_oval(x - 1, y - 1, x + 1, y + 1, fill=col)
            self.canv.create_line(x - 20, y - 20, x + 20, y + 20, fill=col, width=width)
            self.canv.create_line(x - 20, y + 20, x + 20, y - 20, fill=col, width=width)

        def drawXs(self):
            for rows in self.grid:
                for pt in rows:
                    if pt.frames >= minFrames:
                        self.drawX(pt.x, pt.y, col='green', width = 2.)
                    else:
                         self.drawX(pt.x, pt.y, col='grey')

        def record(self, ev):
            if not self.recording:
                self.recording = True
                self.unbindKeys()
                currpt = self.grid[self.currPosX][self.currPosY]
                self.drawX(currpt.x, currpt.y, 'red', 3.)
                self.cHandler.record(currpt.x + self.offsetX, currpt.y + self.offsetY)
            else:
                self.recording = False
                self.cHandler.stopRecording()
                self.grid[self.currPosX][self.currPosY].frames += self.cHandler.frames
                self.drawCurr()
                self.bindKeys()

        def nextw(self, ev):
            self.currPosX = self.currPosY = 0
            self.root.geometry('{}x{}+{}+{}'.format(self.width, self.height,
                int((self.ridx - .5)*self.width/self.row), int((self.cidx - .5)*self.height/len(self.disprows))))
            self.updGeom()
            self.drawCurr()
            self.root.quit()

        def run(self):
            self.disprows = [len(k) for k in args.fs.split('/')]
            for ridx, row in enumerate(self.disprows, 1):
                self.ridx = ridx
                self.row = row
                if self.killed:
                    break
                for cidx in range(1, row + 1):
                    self.cidx = cidx
                    self.incCol = True
                    # root.attributes('-zoomed', True)
                    self.root.attributes('-fullscreen', True)
                    self.root.mainloop()
                    self.incCol = False
                    self.incRow = False
                    if self.killed:
                        break
                self.incRow = True

    xyz().run()
    savefile()

else:
    # configure capture
    cap = cv2.VideoCapture(0)
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
    cap.set(cv2.CAP_PROP_FPS, 60)

    if not cap.isOpened():
        print('Error opening device!')
        sys.exit()

    def handleClick(x, y, button, pressed):
        # will not trigger on any wayland window
        if pressed:
            sys.stdout.write('mouseDown @ {}, button {}'.format((x, y), button))
            if button == mouse.Button.left:
                # capture cam
                ret, frame = cap.read()
                if ret:
                    frame = cv2.flip(frame, 1)
                    if not os.path.exists(outputDataDir):
                        os.makedirs(outputDataDir)

                    fname = 'cap-{:06d}.jpg'.format(len(trainingData))
                    cv2.imwrite(os.path.join(outputDataDir, fname), frame)
                    trainingData.append({'x': x,
                                         'y': y,
                                         'f': fname})
                    print(' -> ' + fname)
                else:
                    print(' -> discarded')

            else:
                print()

    with mouse.Listener(on_click = handleClick) as listener:
        try:
            print('Listening for left-clicks...')
            listener.join()
        except:
            print()
            traceback.print_exc()
            savefile()

    cap.release()

max_stars_repo_name: r-or/cnn-eyetrack

max_issues_repo_name: r-or/cnn-eyetrack

max_forks_repo_name: r-or/cnn-eyetrack


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Greeks for option pricing
=========================

"""
from __future__ import print_function, division

import numpy as np
import scipy.stats as scs

__all__ = ['delta', 'vega']

def delta(moneyness, maturity, sigma, call):
    """Delta of an option with unit current underlying price.

    .. math::

        Delta_{t}\left(\tau,x\right)=\Phi\left(d_{1}\right)

    Log-forward moneyness:

    .. math::

        x=\log\left(K/F\right)=\log\left(K/S\right)-r\tau

        d_{1}=-\frac{x}{\sigma\sqrt{\tau}}+\frac{1}{2}\sigma\sqrt{\tau}

    Parameters
    ----------
    moneyness : array_like
        Log-forward moneyness
    maturity : array_like
        Fraction of the year, i.e. = 30/365
    sigma : array_like
        Volatility
    call : bool array_like
        Call/put flag. True for call, False for put

    Returns
    -------
    array_like
        Delta of the option

    """
    sq_maturity = maturity**.5
    int_vol = sigma * sq_maturity
    out = scs.norm.cdf(int_vol/2 - moneyness/int_vol)
    return out * call + (out - 1) * np.logical_not(call)

def vega(moneyness, maturity, sigma):
    """Vega of an option with unit current underlying price.

    .. math::

        Vega_{t}\left(\tau,x\right)=S_{t}\phi\left(d_{1}\right)\sqrt{\tau}

    Log-forward moneyness:

    .. math::

        x=\log\left(K/F\right)=\log\left(K/S\right)-r\tau

        d_{1}=-\frac{x}{\sigma\sqrt{\tau}}+\frac{1}{2}\sigma\sqrt{\tau}

    Parameters
    ----------
    moneyness : array_like
        Log-forward moneyness
    maturity : array_like
        Fraction of the year, i.e. = 30/365
    sigma : array_like
        Volatility

    Returns
    -------
    array_like
        Vega of the option

    """
    sq_maturity = maturity**.5
    int_vol = sigma * sq_maturity
    return sq_maturity * scs.norm.pdf(int_vol/2 - moneyness/int_vol)

max_stars_repo_name: khrapovs/impvol

max_issues_repo_name: khrapovs/impvol

max_forks_repo_name: khrapovs/impvol

reason: import numpy,import scipy


text:
__connections__ = {
    0: [1, 17, 36],
	1: [2, 36, 41],
	2: [3, 31, 41],
	3: [4, 31, 48],
	4: [5, 48],
	5: [6, 48, 59],
	6: [7, 58, 59],
	7: [8, 57, 58],
	8: [9, 57],
	9: [10, 56, 57],
	10: [11, 55, 56],
	11: [12, 55],
	12: [13, 54, 55],
	13: [14, 35, 54],
	14: [15, 35, 46],
	15: [16, 45, 46],
	16: [26, 45],
	17: [18, 36],
	18: [19, 36, 37],
	19: [20, 37],
	20: [21, 37, 38],
	21: [22, 27, 38, 39],
	22: [23, 27, 42, 43],
	23: [24, 43, 44],
	24: [25, 44],
	25: [26, 44, 45],
	26: [45],
	27: [28, 39, 42],
	28: [29, 39, 42],
	29: [30, 31, 35, 39, 40, 42, 47],
	30: [31, 32, 33, 34, 35],
	31: [32, 40, 41, 48, 49, 50],
	32: [33, 50, 51],
	33: [34, 51],
	34: [35, 51, 52],
	35: [46, 47, 52, 53, 54],
	36: [37, 41],
	37: [38],
	38: [39],
	39: [40],
	40: [41],
	41: [],
	42: [43, 47],
	43: [44],
	44: [45],
	45: [46],
	46: [47],
	47: [],
	48: [49, 59, 60],
	49: [50, 60, 61],
	50: [51, 61, 62],
	51: [52, 62],
	52: [53, 62, 63],
	53: [54, 63, 64],
	54: [55, 64],
	55: [56, 64, 65],
	56: [57, 65, 66],
	57: [58, 66],
	58: [59, 66, 67],
	59: [60, 67],
	60: [61, 67],
	61: [62],
	62: [63],
	63: [64],
	64: [65],
	65: [66],
	66: [67],
}

connections = []
for i in __connections__.keys():
    for j in __connections__[i]:
        connections.append((i, j))

del __connections__
import numpy as np
connections = np.array(connections)

max_stars_repo_name: pandov/myitacademy

max_issues_repo_name: pandov/myitacademy

max_forks_repo_name: pandov/myitacademy


text:
import tensorflow as tf
import math
import numpy as np
import sys

sys.path.append('../../../../')
import tensornet

NUM_CLASSES = 10
IMAGE_SIZE = 32
IMAGE_DEPTH = 3
IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE * IMAGE_DEPTH

opts = {}

opts['hidden_units'] = [4096, 4096, 4096, 4096]
opts['use_dropout'] = True
opts['learning_rate_init'] = 1.0
opts['learning_rate_decay_steps'] = 2000
opts['learning_rate_decay_weight'] = 0.64

def placeholder_inputs():
    """Generate placeholder variables to represent the input tensors.

    Returns:
        images_ph: Images placeholder.
        labels_ph: Labels placeholder.
        train_phase_ph: Train phase indicator placeholder.
    """
    # Note that the shapes of the placeholders match the shapes of the full
    # image and label tensors, except the first dimension is now batch_size
    # rather than the full size of the train or test data sets.
    images_ph = tf.placeholder(tf.float32, shape=(None, IMAGE_PIXELS), name='placeholder/images')
    labels_ph = tf.placeholder(tf.int32, shape=(None), name='placeholder/labels')
    train_phase_ph = tf.placeholder(tf.bool, name='placeholder/train_phase')
    return images_ph, labels_ph, train_phase_ph

def inference(images, train_phase):
    """Build the model up to where it may be used for inference.
    Args:
        images: Images placeholder.
        train_phase: Train phase placeholder
    Returns:
        logits: Output tensor with the computed logits.
    """
    tn_init = lambda dev: lambda shape: tf.truncated_normal(shape, stddev=dev)
    tu_init = lambda bound: lambda shape: tf.random_uniform(shape, minval = -bound, maxval = bound)
    dropout_rate = lambda p: (opts['use_dropout'] * (p - 1.0)) * tf.to_float(train_phase) + 1.0

    layers = []
    layers.append(images)

    cnt = len(opts['hidden_units'])
    for i in range(cnt):
        n_out = opts['hidden_units'][i]

        layers.append(tensornet.layers.linear(layers[-1],
                                              n_out,
                                              scope='linear_' + str(len(layers)),
                                              biases_initializer=None))

        layers.append(tensornet.layers.batch_normalization(layers[-1],
                                                           train_phase,
                                                           scope='BN_' + str(len(layers)),
                                                           ema_decay=0.8))
        layers.append(tf.nn.relu(layers[-1],
                                 name='relu_' + str(len(layers))))

        layers.append(tf.nn.dropout(layers[-1],
                                        dropout_rate(0.77),
                                        name='dropout_' + str(len(layers))))

    layers.append(tensornet.layers.linear(layers[-1],
                                          NUM_CLASSES,
                                          scope='linear_' + str(len(layers))))

    return layers[-1]

def loss(logits, labels):
    """Calculates the loss from the logits and the labels.
    Args:
        logits: input tensor, float - [batch_size, NUM_CLASSES].
        labels: Labels tensor, int32 - [batch_size].
    Returns:
        loss: Loss tensor of type float.
    """
    # Convert from sparse integer labels in the range [0, NUM_CLASSES)
    # to 1-hot dense float vectors (that is we will have batch_size vectors,
    # each with NUM_CLASSES values, all of which are 0.0 except there will
    # be a 1.0 in the entry corresponding to the label).
    batch_size = tf.size(labels)
    labels = tf.expand_dims(labels, 1)
    indices = tf.expand_dims(tf.range(0, batch_size), 1)
    concated = tf.concat([indices, labels], 1)
    onehot_labels = tf.sparse_to_dense(concated,
                                       tf.shape(logits), 1.0, 0.0)

    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
                                                            labels=onehot_labels,
                                                            name='xentropy')
    loss = tf.reduce_mean(cross_entropy, name='loss')
    tf.summary.scalar('summary/loss', loss)
    return loss

def training(loss):
    """Sets up the training Ops.
    Creates an optimizer and applies the gradients to all trainable variables.
    The Op returned by this function is what must be passed to the
    `sess.run()` call to cause the model to train.
    Args:
        loss: Loss tensor, from loss().
    Returns:
        train_op: The Op for training.
    """
    # Create a variable to track the global step.
    global_step = tf.Variable(0, name='global_step', trainable=False)
    learning_rate = tf.train.exponential_decay(opts['learning_rate_init'],
                                               global_step,
                                               opts['learning_rate_decay_steps'],
                                               opts['learning_rate_decay_weight'],
                                               staircase=True,
                                               name='learning_rate')
    tf.summary.scalar('summary/learning_rate', learning_rate)
    # Create the gradient descent optimizer with the given learning rate.
    optimizer = tf.train.GradientDescentOptimizer(learning_rate, name='optimizer')

    grads_and_vars = optimizer.compute_gradients(loss)
    train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step, name='train_op')
    return train_op

def evaluation(logits, labels):
    """Evaluate the quality of the logits at predicting the label.
    Args:
        logits: Logits tensor, float - [batch_size, NUM_CLASSES].
        labels: Labels tensor, int32 - [batch_size], with values in the
        range [0, NUM_CLASSES).
    Returns:
        A scalar int32 tensor with the number of examples (out of batch_size)
        that were predicted correctly.
    """
    # For a classifier model, we can use the in_top_k Op.
    # It returns a bool tensor with shape [batch_size] that is true for
    # the examples where the label's is was in the top k (here k=1)
    # of all logits for that example.
    correct_flags = tf.nn.in_top_k(logits, labels, 1)
    # Return the number of true entries.
    correct_count = tf.reduce_sum(tf.cast(correct_flags, tf.int32), name='correct_count')
    return correct_count

def build(new_opts={}):
    """ Build graph
        Args:
            new_opts: dict with additional opts, which will be added to opts dict/
    """
    opts.update(new_opts)
    images_ph, labels_ph, train_phase_ph = placeholder_inputs()
    logits = inference(images_ph, train_phase_ph)
    loss_out = loss(logits, labels_ph)
    train = training(loss_out)
    eval_out = evaluation(logits, labels_ph)

max_stars_repo_name: akatoshking/matrix-project

max_issues_repo_name: akatoshking/matrix-project

max_forks_repo_name: akatoshking/matrix-project


text:
import os
import re
import copy
import torch
import numpy as np
# import matplotlib
# matplotlib.use("Agg")
# import matplotlib.pyplot as plt
from shutil import copyfile

def get_proper_cuda_device(device, verbose=True):
    if not isinstance(device, list):
        device = [device]
    count = torch.cuda.device_count()
    if verbose:
        print("[Builder]: Found {} gpu".format(count))
    for i in range(len(device)):
        d = device[i]
        did = None
        if isinstance(d, str):
            if re.search("cuda:[\d]+", d):
                did = int(d[5:])
        elif isinstance(d, int):
            did = d
        if did is None:
            raise ValueError("[Builder]: Wrong cuda id {}".format(d))
        if did < 0 or did >= count:
            if verbose:
                print("[Builder]: {} is not found, ignore.".format(d))
            device[i] = None
        else:
            device[i] = did
    device = [d for d in device if d is not None]
    return device

def get_proper_device(devices, verbose=True):
    origin = copy.copy(devices)
    devices = copy.copy(devices)
    if not isinstance(devices, list):
        devices = [devices]
    use_cpu = any([d.find("cpu")>=0 for d in devices])
    use_gpu = any([(d.find("cuda")>=0 or isinstance(d, int)) for d in devices])
    assert not (use_cpu and use_gpu), "{} contains cpu and cuda device.".format(devices)
    if use_gpu:
        devices = get_proper_cuda_device(devices, verbose)
        if len(devices) == 0:
            if verbose:
                print("[Builder]: Failed to find any valid gpu in {}, use `cpu`.".format(origin))
            devices = ["cpu"]
    return devices

def _file_at_step(step):
    return "save_{}k{}.pkg".format(int(step // 1000), int(step % 1000))

def _file_best():
    return "trained.pkg"

def save(global_step, graph, optim, criterion_dict=None, pkg_dir="", is_best=False, max_checkpoints=None):
    if optim is None:
        raise ValueError("cannot save without optimzier")
    state = {
        "global_step": global_step,
        # DataParallel wrap model in attr `module`.
        "graph": graph.module.state_dict() if hasattr(graph, "module") else graph.state_dict(),
        "optim": optim.state_dict(),
        "criterion": {}
    }
    if criterion_dict is not None:
        for k in criterion_dict:
            state["criterion"][k] = criterion_dict[k].state_dict()
    save_path = os.path.join(pkg_dir, _file_at_step(global_step))
    best_path = os.path.join(pkg_dir, _file_best())
    torch.save(state, save_path)
    if is_best:
        copyfile(save_path, best_path)
    if max_checkpoints is not None:
        history = []
        for file_name in os.listdir(pkg_dir):
            if re.search("save_\d*k\d*\.pkg", file_name):
                digits = file_name.replace("save_", "").replace(".pkg", "").split("k")
                number = int(digits[0]) * 1000 + int(digits[1])
                history.append(number)
        history.sort()
        while len(history) > max_checkpoints:
            path = os.path.join(pkg_dir, _file_at_step(history[0]))
            print("[Checkpoint]: remove {} to keep {} checkpoints".format(path, max_checkpoints))
            if os.path.exists(path):
                os.remove(path)
            history.pop(0)

def load(step_or_path, graph, optim=None, criterion_dict=None, pkg_dir="", device=None):
    step = step_or_path
    save_path = None
    if isinstance(step, int):
        save_path = os.path.join(pkg_dir, _file_at_step(step))
    if isinstance(step, str):
        if pkg_dir is not None:
            if step == "best":
                save_path = os.path.join(pkg_dir, _file_best())
            else:
                save_path = os.path.join(pkg_dir, step)
        else:
            save_path = step
    if save_path is not None and not os.path.exists(save_path):
        print("[Checkpoint]: Failed to find {}".format(save_path))
        return
    if save_path is None:
        print("[Checkpoint]: Cannot load the checkpoint with given step or filename or `best`")
        return

    # begin to load
    state = torch.load(save_path, map_location=device)
    global_step = state["global_step"]
    graph.load_state_dict(state["graph"])
    if optim is not None:
        optim.load_state_dict(state["optim"])
    if criterion_dict is not None:
        for k in criterion_dict:
            criterion_dict[k].load_state_dict(state["criterion"][k])

    graph.set_actnorm_init(inited=True)

    print("[Checkpoint]: Load {} successfully".format(save_path))
    return global_step

def __save_figure_to_numpy(fig):
    # save it to a numpy array.
    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
    return data

def __to_ndarray_list(tensors, titles):
    if not isinstance(tensors, list):
        tensors = [tensors]
        titles = [titles]
    assert len(titles) == len(tensors),\
        "[visualizer]: {} titles are not enough for {} tensors".format(
            len(titles), len(tensors))
    for i in range(len(tensors)):
        if torch.is_tensor(tensors[i]):
            tensors[i] = tensors[i].cpu().detach().numpy()
    return tensors, titles

def __get_figures(num_tensors, figsize):
    fig, axes = plt.subplots(num_tensors, 1, figsize=figsize)
    if not isinstance(axes, np.ndarray):
        axes = np.asarray([axes])
    return fig, axes

def __make_dir(file_name, plot_dir):
    if file_name is not None and not os.path.exists(plot_dir):
        os.makedirs(plot_dir)

def __draw(fig, file_name, plot_dir):
    if file_name is not None:
        plt.savefig('{}/{}.png'.format(plot_dir, file_name), format='png')
        plt.close(fig)
        return None
    else:
        fig.tight_layout()
        fig.canvas.draw()
        data = __save_figure_to_numpy(fig)
        plt.close(fig)
        return data

def __get_size_for_spec(tensors):
    spectrogram = tensors[0]
    fig_w = np.min([int(np.ceil(spectrogram.shape[1] / 10.0)), 10])
    fig_w = np.max([fig_w, 3])
    fig_h = np.max([3 * len(tensors), 3])
    return (fig_w, fig_h)

def __get_aspect(spectrogram):
    fig_w = np.min([int(np.ceil(spectrogram.shape[1] / 10.0)), 10])
    fig_w = np.max([fig_w, 3])
    aspect = 3.0 / fig_w
    if spectrogram.shape[1] > 50:
        aspect = aspect * spectrogram.shape[1] / spectrogram.shape[0]
    else:
        aspect = aspect * spectrogram.shape[1] / (spectrogram.shape[0])
    return aspect

def plot_prob(done, title="", file_name=None, plot_dir=None):
    __make_dir(file_name, plot_dir)

    done, title = __to_ndarray_list(done, title)
    for i in range(len(done)):
        done[i] = np.reshape(done[i], (-1, done[i].shape[-1]))
    figsize = (5, 5 * len(done))
    fig, axes = __get_figures(len(done), figsize)
    for ax, d, t in zip(axes, done, title):
        im = ax.imshow(d, vmin=0, vmax=1, cmap="Blues", aspect=d.shape[1]/d.shape[0])
        ax.set_title(t)
        ax.set_yticks(np.arange(d.shape[0]))
        lables = ["Frame{}".format(i+1) for i in range(d.shape[0])]
        ax.set_yticklabels(lables)
        ax.set_yticks(np.arange(d.shape[0])-.5, minor=True)
        ax.grid(which="minor", color="g", linestyle='-.', linewidth=1)
        ax.invert_yaxis()
    return __draw(fig, file_name, plot_dir)

max_stars_repo_name: swyoon/pytorch-glow-that-works

max_issues_repo_name: swyoon/pytorch-glow-that-works

max_forks_repo_name: swyoon/pytorch-glow-that-works


text:
import numpy as np
import apricot
import math
from .nonadaptivedataloader import NonAdaptiveDSSDataLoader

class SubmodDataLoader(NonAdaptiveDSSDataLoader):
    # Currently split dataset with size of |max_chunk| then proportionably select samples in every chunk
    # Otherwise distance matrix will be too large
    def __init__(self, train_loader, val_loader, dss_args, logger, *args,
                 **kwargs):

        super(SubmodDataLoader, self).__init__(train_loader, val_loader, dss_args,
                                               logger, *args, **kwargs)

        assert "size_chunk" in dss_args.keys(), "'size_chunk' is a compulsory agument for submodular dataloader"
        if dss_args.size_chunk:
            self.logger.info("You are using max_chunk: %s" % dss_args.size_chunk)
        self.size_chunk = dss_args.size_chunk

    def _init_subset_indices(self):
        X = np.array([x for (x, _y) in self.dataset])
        m = X.shape[0]
        # Chunking dataset to calculate pairwise distance with limited memory
        sample_indices = []
        size_chunk, budget = self.size_chunk, self.budget
        n_chunks = math.ceil(m / self.size_chunk)
        budget_chunk = math.ceil(budget / n_chunks)
        for i_chunk in range(n_chunks):
            l_idx = i_chunk * size_chunk
            r_idx = min(m, (i_chunk + 1) * size_chunk)
            n_samples = min(budget_chunk, budget - len(sample_indices))
            chunk = X[l_idx: r_idx, :]
            _sample_indices = self._chunk_select(chunk, n_samples)
            _sample_indices = [_sample_indice + l_idx for _sample_indice in _sample_indices]
            sample_indices += _sample_indices
        return np.array(sample_indices)

# Submodular optimization based
class FacLocDataLoader(SubmodDataLoader):
    def _chunk_select(self, chunk, n_samples):
        f = apricot.functions.facilityLocation.FacilityLocationSelection(n_samples=n_samples)
        m = f.fit(chunk)
        return list(m.ranking)

class GraphCutDataLoader(SubmodDataLoader):
    def _chunk_select(self, chunk, n_samples):
        f = apricot.functions.graphCut.GraphCutSelection(n_samples=n_samples)
        m = f.fit(chunk)
        return list(m.ranking)

class SumRedundancyDataLoader(SubmodDataLoader):
    def _chunk_select(self, chunk, n_samples):
        f = apricot.functions.sumRedundancy.SumRedundancySelection(n_samples=n_samples)
        m = f.fit(chunk)
        return list(m.ranking)

class SaturatedCoverageDataLoader(SubmodDataLoader):
    def _chunk_select(self, chunk, n_samples):
        f = apricot.functions.facilityLocation.FacilityLocationSelection(n_samples=n_samples)
        m = f.fit(chunk)
        return list(m.ranking)


text:
import pybullet as p
import numpy as np
from gym import spaces
from diy_gym.model import Model
from diy_gym.addons.addon import Addon

class Camera(Addon):
    """Captures RGB, depth and segmentation images from the perspective of a model or fixed in the world frame.

    Configs:
        clipping_boundaries (list of float, optional, [0.1, 100]): the boundaries in the z-direction of the volume
            captured by the depth images (i.e objects at a distance that is outside these boundaries won't
            appear in the depth image)
        field_of_view (float, optional, 70): defines the perspective matrix of the camera
        frame (str, optional): the frame on the parent model to which the camera will be attached
            (defaults to the base frame)
        resolution (list of float, optional, [640, 480]): defines the width and height (respectively) of the
            images captured by the camera
        xyz (list of float, optional, [0,0,0]): the offset relative to the `frame` at which to spawn the camera
        rpy (list of float, optional, [0,0,0]): the orientation relative to the `frame` at which to spawn the camera
            expressed in euler angles
        use_depth (bool, optional, False): whether to include depth images in observations
        use_segmentation_mask (bool, optional, False): whether to include segmentation images in observations
    """
    def __init__(self, parent, config):
        super(Camera, self).__init__(parent, config)

        self.near, self.far = config.get('clipping_boundaries', [0.01, 100])
        self.fov = config.get('field_of_view', 70.0)
        self.resolution = config.get('resolution', [640, 480])
        self.aspect = self.resolution[0] / self.resolution[1]

        self.uid = parent.uid if isinstance(parent, Model) else -1
        self.frame_id = parent.get_frame_id(config.get('frame')) if 'frame' in config else -1

        xyz = config.get('xyz', [0., 0., 0.])
        rpy = config.get('rpy', [0., 0., 0.])

        self.use_depth = config.get('use_depth', True)
        self.use_seg_mask = config.get('use_segmentation_mask', False)

        self.T_parent_cam = self.trans_from_xyz_quat(xyz, p.getQuaternionFromEuler(rpy))
        self.projection_matrix = p.computeProjectionMatrixFOV(self.fov, self.aspect, self.near, self.far)
        self.K = np.array(self.projection_matrix).reshape([4, 4]).T

        self.observation_space = spaces.Dict({
            'rgb': spaces.Box(0., 1., shape=self.resolution + [3], dtype='float32'),
        })

        if self.use_depth:
            self.observation_space.spaces.update({'depth': spaces.Box(0., 10., shape=self.resolution, dtype='float32')})

        if self.use_seg_mask:
            self.observation_space.spaces.update(
                {'segmentation_mask': spaces.Box(0., 10., shape=self.resolution, dtype='float32')})

    def observe(self):
        if self.frame_id >= 0:
            link_state = p.getLinkState(self.uid, self.frame_id)
            T_world_parent = self.trans_from_xyz_quat(link_state[4], link_state[5])
        elif self.uid >= 0:
            model_state = p.getBasePositionAndOrientation(self.uid)
            T_world_parent = self.trans_from_xyz_quat(model_state[0], model_state[1])
        else:
            T_world_parent = np.eye(4)

        T_world_cam = np.linalg.inv(T_world_parent.dot(self.T_parent_cam))

        image = p.getCameraImage(self.resolution[0],
                                 self.resolution[1],
                                 T_world_cam.T.flatten(),
                                 self.projection_matrix,
                                 flags=p.ER_NO_SEGMENTATION_MASK if not self.use_seg_mask else 0)

        obs = {
            'rgb': np.array(image[2], copy=False).reshape(self.resolution + [4])[:, :, :3] / 255.
        }  # discard the alpha channel and normalise to [0 1]

        if self.use_depth:
            # the depth buffer is normalised to [0 1] whereas NDC coords require [-1 1] ref: https://bit.ly/2rcXidZ
            depth_ndc = np.array(image[3], copy=False).reshape(self.resolution) * 2 - 1

            # recover eye coordinate depth using the projection matrix ref: https://bit.ly/2vZJCsx
            depth = self.K[2, 3] / (self.K[3, 2] * depth_ndc - self.K[2, 2])

            obs['depth'] = depth

        if self.use_seg_mask:
            obs['segmentation_mask'] = np.array(image[4], copy=False).reshape(self.resolution)

        return obs

    def trans_from_xyz_quat(self, xyz, quat):
        T = np.eye(4)
        T[:3, 3] = xyz
        T[:3, :3] = np.array(p.getMatrixFromQuaternion(quat)).reshape(3, 3)
        return T

max_stars_repo_name: ktlichkid/diy-gym

max_issues_repo_name: ktlichkid/diy-gym

max_forks_repo_name: ktlichkid/diy-gym


text:
from django.views.generic import TemplateView
from django.db.models import Count
from django.db.models.functions import TruncMonth
from va_explorer.utils.mixins import CustomAuthMixin
from datetime import date, timedelta
from dateutil.relativedelta import relativedelta
import numpy as np
import pandas as pd
from django.db.models import F
# TODO: We're using plotly here since it's already included in the project, but there may be slimmer options
import plotly.offline as opy
import plotly.graph_objs as go
from va_explorer.va_data_management.utils.loading import get_va_summary_stats
from va_explorer.va_data_management.utils.date_parsing import parse_date, to_dt, get_submissiondates

# Simple helper function for creating the plotly graphs used on the home page
def graph(x, y):
    figure = go.Figure(data=go.Scatter(x=x, y=y))
    figure.update_layout(autosize=False, width=300, height=100, margin=dict(l=0, r=0, t=0, b=0, pad=0))
    config = {'displayModeBar': False}
    return opy.plot(figure, auto_open=False, output_type='div', config=config)

class Index(CustomAuthMixin, TemplateView):

    template_name = "home/index.html"

    def get_context_data(self, **kwargs):
        # TODO: interviewers should only see their own data
        context = super().get_context_data(**kwargs)

        user = self.request.user
        today = date.today()
        start_month = pd.to_datetime(date(today.year - 1, today.month, 1))
        location_restrictions = user.location_restrictions
        if location_restrictions.count() > 0:
            context['locations'] = ', '.join([location.name for location in location_restrictions.all()])
        else:
            context['locations'] = 'All Regions'
        # NOTE: using SUBMISSIONDATE to drive stats/views. To change this, change all references to submissiondate
        user_vas = user.verbal_autopsies()
        if user_vas.count() > 0:
            va_df = pd.DataFrame(user_vas\
                .only("id","Id10023","location","Id10011", "submissiondate","created","Id10010", "Id10011") \
                .select_related("location") \
                .select_related("causes") \
                .values("id","Id10023", "created", "Id10011", "submissiondate",\
                    name=F("Id10010"),facility=F("location__name"),cause=F("causes__cause"))
                )
            va_df["date"] = get_submissiondates(va_df)

            # clean date fields - strip timezones from submissiondate and created dates
            va_df["date"] = to_dt(va_df["date"])
            va_df["created"] = to_dt(va_df["created"])
            va_df["Id10023"] = to_dt(va_df["Id10023"])
            va_df["yearmonth"] = va_df["date"].dt.strftime("%Y-%m")

            context.update(get_va_summary_stats(user_vas))
            # Load the VAs that are collected over various periods of time
            today = pd.to_datetime(date.today())

            vas_24_hours = va_df[va_df["date"] == today].index
            vas_1_week = va_df[va_df['date'] >= (today - timedelta(days=7))].index
            vas_1_month = va_df[va_df['date'] >= (today - relativedelta(months=1))].index
            vas_overall = va_df.sort_values(by="id").index

            # VAs collected in the past 24 hours, 1 week, and 1 month
            context['vas_collected_24_hours'] = len(vas_24_hours)
            context['vas_collected_1_week'] = len(vas_1_week)
            context['vas_collected_1_month'] = len(vas_1_month)
            context['vas_collected_overall'] = len(vas_overall)
            # VAs successfully coded in the past 24 hours, 1 week, and 1 month
            context['vas_coded_24_hours'] = va_df.loc[vas_24_hours,:].query("cause == cause").shape[0]
            context['vas_coded_1_week'] = va_df.loc[vas_1_week,:].query("cause == cause").shape[0]
            context['vas_coded_1_month'] = va_df.loc[vas_1_month,:].query("cause == cause").shape[0]
            context['vas_coded_overall'] = va_df.loc[vas_overall,:].query("cause == cause").shape[0]
            # VAs not able to be coded in the past 24 hours, 1 week, and 1 month
            context['vas_uncoded_24_hours'] = context['vas_collected_24_hours'] - context['vas_coded_24_hours']
            context['vas_uncoded_1_week'] = context['vas_collected_1_week'] - context['vas_coded_1_week']
            context['vas_uncoded_1_month'] = context['vas_collected_1_month'] - context['vas_coded_1_month']
            context['vas_uncoded_overall'] = context['vas_collected_overall'] - context['vas_coded_overall']

            # Graphs of the past 12 months, not including this month (current month will almost
            # always show the month with artificially low numbers)
            plot_df = (pd.DataFrame({'month':[start_month + relativedelta(months=i) for i in range(12)]})
                        .assign(month_num = lambda df: df['month'].dt.month)
                )

            months = [start_month + relativedelta(months=i) for i in range(12)]
            x = [month.strftime('%b') for month in months]
            plot_df = pd.DataFrame({'yearmonth': [month.strftime('%Y-%m') for month in months], 'x': x})

            # Collected; total VAs by month
            plot_df = plot_df.merge(va_df.query("date >= @start_month").groupby("yearmonth")["id"].count().rename("y_collected").reset_index(), how="left").fillna(0)
            context['graph_collected'] = graph(plot_df['x'], plot_df['y_collected'])

            # Coded; same query as above, just filtered by whether the va has been coded
            plot_df = plot_df.merge(va_df.query("date >= @start_month & cause==cause").groupby("yearmonth")["id"].count().rename("y_coded").reset_index(), how="left").fillna(0)
            # y_coded = va_df.query().groupby("yearmonth")["id"].count().values
            #y_coded = [coded_by_month.get(month, 0) for month in months]
            context['graph_coded'] = graph(plot_df['x'], plot_df['y_coded'])

            # Uncoded; just take the difference between the two previous queries
            context['graph_uncoded'] = graph(plot_df['x'], (plot_df['y_collected'] - plot_df['y_coded']))

            # List the VAs that need attention; requesting certain fields and prefetching makes this more efficient
            vas_to_address = user_vas.only('id', 'location_id', 'Id10007', 'Id10010', 'Id10017',
                'Id10018', 'submissiondate', 'Id10011', 'Id10023').filter(causes__isnull=True)[:10].prefetch_related("causes", "coding_issues", "location")

            context['issue_list'] = [{
                "id": va.id,
                "deceased": f"{va.Id10017} {va.Id10018}",
                "interviewer": va.Id10010,
                "submitted":  parse_date(va.submissiondate) if (va.submissiondate != 'dk') else "Unknown", #django stores the date in yyyy-mm-dd
                "dod":  parse_date(va.Id10023) if (va.Id10023 != 'dk') else "Unknown",
                "facility": va.location.name if va.location else "",
                "cause": va.causes.all()[0].cause if len(va.causes.all()) > 0 else "",
                "warnings": len([issue for issue in va.coding_issues.all() if issue.severity == 'warning']),
                "errors": len([issue for issue in va.coding_issues.all() if issue.severity == 'error'])
            } for va in vas_to_address]

            # If there are more than 10 show a link to where the rest can be seen
            context['additional_issues'] = max(context['vas_uncoded_overall'] - 10, 0)
        # NO VAS FOUND - RETURN EMPTY STATS
        else:
            # empty stats
            for va_type in ['collected', 'coded', 'uncoded']:
                for va_time_period in ['24_hours', '1_week', '1_mont',  'overall']:
                    context[f"vas_{va_type}_{va_time_period}"] = 0
            # empty graphs
            months = [start_month + relativedelta(months=i) for i in range(12)]
            x = [month.strftime('%b') for month in months]
            for va_type in ['collected', 'coded', 'uncoded']:
                context[f"graph_{va_type}"] = graph(x, np.repeat(0, len(x)))
            context['issue_list'] = []

        return context

class About(CustomAuthMixin, TemplateView):
    template_name = "home/about.html"


text:
"""
unsupervised learning algorithm

implement k-means method
"""
import numpy as np

def thresh_kmeans(img_input, itr=5, mu1_init=50, mu2_init=150):
    """
    binarize input image using k-means algorithm

        :param img_input: ndarray uint8, 1ch image
        :param itr: int, iterator of k-means algorithm
        :param mu1_init: int, initial centroid of cluster1
        :param mu2_init: int, initial centroid of cluster2
        :return img_bin: ndarray uint8, binarized image
    """

    # histogram
    hist, bins = np.histogram(img_input.ravel(), 256, [0, 256])

    # -- k-means algoritym -- #
    # initialize mean (centroid)
    mu1_current = mu1_init  # initial centroid of cluster1
    mu2_current = mu2_init  # initial centroid of cluster2

    # iterative calculation
    itr_cnt = 0  # iterate conter
    while(itr_cnt < itr):
        # initialize
        mu1_updated = 0
        mu2_updated = 0
        sum1 = 0
        sum2 = 0
        cnt1 = 0
        cnt2 = 0
        for i in range(len(hist)):
            # in the case of belonging to cluster1
            if (np.abs(i - mu1_current) < np.abs(i - mu2_current)):
                sum1 += hist[i]*i
                cnt1 += hist[i]
            # in the case of belonging to cluster2
            else:
                sum2 += hist[i]*i
                cnt2 += hist[i]
        # update each centroid
        mu1_updated = sum1 / cnt1
        mu2_updated = sum2 / cnt2
        mu1_current = mu1_updated
        mu2_current = mu2_updated
        itr_cnt += 1

    # calculate binary threshold
    thresh = (mu1_current + mu2_current) / 2
    # convert gray image to binary image
    # Note: THRESH_OTSU is also good
    max_pixel = 255
    ret, img_bin = cv2.threshold(img_input,
                                 thresh,
                                 max_pixel,
                                 cv2.THRESH_BINARY)

    return img_bin

max_stars_repo_name: hsmtknj/ml

max_issues_repo_name: hsmtknj/ml

max_forks_repo_name: hsmtknj/ml