text:
# Copyright (C) [2021] by Cambricon, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# pylint: disable=missing-docstring, invalid-name, too-many-locals
"""A multi-platform code link example test for BANGPy TCP."""

import numpy as np
import torch
import pytest

import bangpy as bp
from bangpy.common import load_op_by_type
from logsumexp import DTYPES, KERNEL_NAME, TARGET_LIST

def create_random_shape(length, size):
    return np.random.randint(low=1, high=size, size=length)

ranshp = create_random_shape(2, 10)
ranshp1 = create_random_shape(5, 3)
ranshp2 = create_random_shape(10, 2)

@pytest.mark.parametrize(
    "shape",
    [
        (1, 2, 101 * 100 ),
        (1, 1, 1),
        (2, 1, 1, 1, 3, 2, 2, 3, 1, 2, 5, 4 ),
        (30000, 1),
        (1, 3),
        (3, 1, 2),
        (4, 5, 2),
        (3, 1, 2),
        (1, 482 * 1024),
        ranshp,
        ranshp1,
        ranshp2
    ],
)
@pytest.mark.parametrize(
    "dtype", DTYPES,
)

@pytest.mark.parametrize(
    "dim", [0, 1, 2, 3, 9, 10, 100, -3, -1000],
)

@pytest.mark.parametrize(
    "keepdim", [True, False],
)

def test_logsumexp(target, shape, dim, dtype, keepdim):
    if target not in TARGET_LIST:
        return

    try:
        total_input_len = 1
        for s in shape:
            total_input_len *= s

        input_tensor = np.random.uniform(low=-5, high=5, size=shape).astype(dtype.as_numpy_dtype)

        def get_total_size(shp):
            size = 1
            for s in shp:
                size *= s
            return size

        def check_dim_range(dim, shape):
            if dim < 0 or dim >= len(shape):
                return False
            return True

        _dev = bp.device(0)
        shp_len = len(shape)

        if dim < 0:
            dim += len(shape)

        if not check_dim_range(dim, shape):
            raise Exception('dim err')

        # mlu 输入参数
        _pd_len = shape[dim]
        _pd_height = 1
        _pd_width = 1

        for i in range(0, dim + 1):
            _pd_height *= shape[i]

        if dim == shp_len - 1:
            pass
        else:
            for i in range(dim + 1, shp_len):
                _pd_width *= shape[i]

        # mlu 输入
        _mlu_input1 = bp.Array(input_tensor.flatten(), _dev)

        # mlu 输出
        _output_len = get_total_size(shape) // shape[dim]
        output_buffer = -np.ones(_output_len, dtype=dtype.as_numpy_dtype)
        _mlu_output = bp.Array(output_buffer, _dev)

        output_count = 256
        output_buffer2 = np.zeros(output_count, dtype=dtype.as_numpy_dtype)
        _mlu_border_output = bp.Array(output_buffer2, _dev)

        output_buffer3 = -np.ones(output_count, dtype=np.int32)
        _mlu_border_idx_output = bp.Array(output_buffer3, _dev)

        # 调用mlu
        func = load_op_by_type(KERNEL_NAME, dtype.name)
        func(_mlu_input1,
             _pd_len, _pd_height, _pd_width, _output_len
             , _mlu_border_output, _mlu_border_idx_output, _mlu_output)

        result = _mlu_output.numpy()
        def create_reshape(keepdim, dim, shape):
            outputshape = []
            if keepdim:
                for item in shape:
                    outputshape.append(item)
                outputshape[dim] = 1
            else:
                i = 0
                for s in shape:
                    if i == dim:
                        i += 1
                        continue
                    outputshape.append(s)
                    i += 1
            return outputshape

        outputshape = create_reshape(keepdim, dim, shape)

        mlu_ret = result.reshape(outputshape)

        x = torch.Tensor(input_tensor)
        cpu_ret = torch.logsumexp(x, dim, keepdim)

        bp.assert_allclose( cpu_ret.numpy(), mlu_ret,rtol = 0.01, atol = 0.01)

    except Exception as err:
        strerr = str(err)
        print(strerr)

        if strerr == "dim err":
            return

        raise Exception(strerr) from err

max_stars_repo_name: testouya/mlu-ops

max_issues_repo_name: testouya/mlu-ops

max_forks_repo_name: testouya/mlu-ops


text:
#!/usr/bin/env python3

import sys, os
from os.path import basename, dirname, join as p_join
from glob import glob

import numpy as np
from kaldiio import WriteHelper
import scipy.io.wavfile as wav_file
from python_speech_features import mfcc as psf_mfcc

if __name__ == '__main__':
    input_mfcc_dir= sys.argv[1]
    output_prefix = sys.argv[2]

    mfccs = glob(p_join(input_mfcc_dir, '*.npy'))
    os.makedirs(dirname(output_prefix), exist_ok=True)
    new_scp = output_prefix + '.scp'
    new_ark = output_prefix + '.ark'
    with WriteHelper('ark,scp:{},{}'.format(new_ark, new_scp)) as writer:
        for mfcc in mfccs:
            utt_id = basename(mfcc)[:-4]
            writer[utt_id] = np.load(mfcc)

ext: py

max_stars_repo_name: zeek-han/kaldi

max_issues_repo_name: zeek-han/kaldi

max_forks_repo_name: zeek-han/kaldi

reason: import numpy,import scipy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 28 10:50:57 2020

@author: Aaron Watkins

Citation: Graham, A. (2019), PASA 36, 035
"""

import numpy as np
from scipy.special import gammaincinv
from scipy.special import gamma
from astropy.io import ascii as asc
from astropy.table import Table

class Dwarfs():
    '''
    When an instance is created, attaches Sersic parameters to each stellar
    mass with some scatter.

    Args:
        mstar: array of log(Mstar/Msun), or single log(Mstar/Msun)
        sigma1: dispersion around log(Mstar)-log(n) relationship
        sigma2: dispersion around log(Mstar)-log(mu0) relationship
        fit1: tuple, slope and intercept of linear fits between log(Mstar)-log(n)
        fit2: tuple, slope and intercept of linear fits between log(Mstar)-mu0

    Attributes:
        self.mstar: log(Mstar/Msun)
        self.sigma1: dispersion in log(Mstar) vs. log(n) relationship
        self.sigma2: dispersion in log(Mstar) vs. mu0 relationship
        self.fit1: linear fit params to log(Mstar) vs. log(n) relationship
        self.fit2: linear fit params to log(Mstar) vs. mu0 relationship
        self.re: effective radius in pc
        self.n: Sersic index
        self.I0: central mass surface density in Msun/pc^2
        self.mabs: dictionary of absolute magnitudes in ugrizy (LSST, AB)
        self.q: axial ratio (randomly sampled from U(0.05,1))
        self.pa: position angle in degrees (randomly sampled from U(-90, 90))
    '''
    def __init__(self, mstar, sigma1=0.03, sigma2=1.2, fit1=(0.1, -0.73), fit2=(-1.48, 34.77)):
        self.mstar = mstar
        self.sigma1 = sigma1
        self.sigma2 = sigma2
        self.fit1 = fit1
        self.fit2 = fit2
        self.re, self.n, self.I0, self.mabs = self.derive_params(mstar,
                                                      sigma1,
                                                      sigma2,
                                                      fit1,
                                                      fit2)
        self.q = np.random.uniform(0.05, 1, size=len(mstar)) # Minimum is GalSim min
        self.pa = np.random.uniform(-90, 90, size=len(mstar))
        self.n[self.n < 0.3] = 0.3 # GalSim doesn't take n < 0.3

    def sersic(self, r, I0, re, n):
        '''
        For testing/verification purposes

        Args:
            r: radius array
            I0: central surface brightness
            re: effective radius
            n: Sersic index

        Returns:
            I(r) in Mstar/pc^2 for chosen Sersic parameters
        '''
        b = bnn(n, 0.5)

        return I0 * np.exp(-b*(r/re)**(1/n))

    def mstar_v_n(self, mstar, sigma, fit):
        '''
        Predicts Sersic n given log-linear relation with log(Mstar)

        Args:
            m: slope
            b: intercept

        Returns:
            n: Sersic index given value of log(Mstar)
        '''

        noise = np.random.normal(0, sigma, size=len(mstar))
        logn = fit[0]*mstar + fit[1]
        logn += noise

        return 10**logn

    def mstar_v_mu0(self, mstar, sigma, fit):
        '''
        Predicts central surface brightness given log-linear relation with
        log(Mstar)

        Args:
            m: slope
            b: intercept

        Returns:
            Central surface brightness in mstar/pc^2
        '''

        noise = np.random.normal(0, sigma, size=len(mstar))
        mu0 = fit[0]*mstar + fit[1]
        mu0 += noise
        # Derived from SPITZER 3.6um magnitudes
        I0 = 10**(-0.4*(mu0 - 27.59)) * 0.5

        return I0

    def get_mags(self, mstar):
        '''
        Derive absolute magnitudes for galaxies given stellar mass, assuming
        the Kirby + (2013) mass-metallicity relationship for dwarfs holds

        Model parameters, using online tool:
        E-MILES basti kb 1.30 baseFe parametric
        3000 - 12000 A, linear, AB mags

        Args:
            mstar: log(solar mass) value

        Returns:
            Dictionary of absolute AB magnitudes in ugrizy LSST bands
        '''
        magtab = asc.read('lsst.MAG', comment='#')
        masstab = asc.read('lsst.MASS', comment='#')
        m_on_h = np.arange(-2.4, 0.7, 0.1)

        # Applying errors on the params from Kirby + 2013
        ran1 = np.random.normal(0.0, 0.04, len(mstar))
        ran2 = np.random.normal(0.0, 0.02, len(mstar))
        z_data = -1.69+ran1 + (0.3+ran2)*(mstar - 6)

        filters = [key for key in magtab.keys() if key!='model']
        mabs = {}
        if len(mstar)==0:
            for filt in filters:
                idx = find_nearest(m_on_h, z_data)
                mag = magtab[filt][idx]
                mfrac = masstab['Mstar_ali'][idx]

                mag1msun = mag + 2.5*np.log10(mfrac)
                mabs[filt] = mag1msun - 2.5*mstar \
                             + np.random.uniform(0, 0.05)
        else:
            for filt in filters:
                mabs[filt] = np.zeros(len(mstar))
                for i in range(len(mstar)):
                    idx = find_nearest(m_on_h, z_data[i])
                    mag = magtab[filt][idx]
                    mfrac = masstab['Mstar_ali'][idx]

                    mag1msun = mag + 2.5*np.log10(mfrac)
                    mabs[filt][i] = mag1msun - 2.5*mstar[i] \
                                    + np.random.uniform(0, 0.05)

        return mabs

    def derive_params(self, mstar, sigma1, sigma2, fit1, fit2):
        '''
        Derive effective radius using total stellar mass
        Returns the two parameters needed by GalSim (Re and n) to build model
        galaxies.

        Args:
            mstar: log10(mstar/msun)
            fit1: tuple of slope and intercept for Mstar vs. n relationship
            fit2: tuple of slope and intercept for Mstar vs. mu0 relationship

        Returns:
            Effective radius (in pc)
            Sersic index
            Central surface brightness (Mstar/pc^2)
        '''

        n = self.mstar_v_n(mstar, sigma1, fit1)
        I0 = self.mstar_v_mu0(mstar, sigma2, fit2)
        bn = bnn(n, 0.5)

        re = np.sqrt((10**(mstar)*bn**(2*n)) / (I0 * 2*n * np.pi * gamma(2*n)))

        mabs = self.get_mags(mstar)

        return re, n, I0, mabs

# =============================================================================
# Utility Functions
# =============================================================================

def find_nearest(array, value):
    '''
    Utility function to find array element closest to input value

    Args:
        array: array in which you want to find the value
        value: value you want to find in the array

    Returns:
        Index of the array containing the element closest to input value
    '''
    array = array[np.isfinite(array)]
    idx = (np.abs(array-value)).argmin()
    return idx

def bnn(n, frac):
    '''
    Calculates value of b for Sersic index/light fraction nn

    Args:
        n: Sersic index
        frac: fraction of total light (e.g. 0.5 = 50%)

    Returns:
        b: Sersic constant b_n
    '''

    return gammaincinv(2*n, frac)

def schechter(logmstar, phi_star=1, logm_star=10.2, alpha=-1.35):
    '''
    Output a Schechter function for stellar mass
    Default values approximately derived from Venhola et al. (2019)

    Args:
        phi_star: normalization at m_star
        m_star: the knee of the function
        alpha: the slope at the low mass end

    Returns:
        A Schechter function with the given parameters
    '''
    mstar = 10**logmstar
    m_star = 10**logm_star
    return phi_star * np.exp(-mstar/m_star) * (mstar/m_star)**(alpha+1)

def mass_from_schechter(logmstar, N, phi_star=1, logm_star=10.2, alpha=-1.35):
    '''
    Creates a random distribution of stellar masses that follow the low-mass
    end of the Schechter function.

    Args:
        logmstar: array of stellar masses in log(M/Msun)
        N: output sample size

    Returns:
        Array of logmstar sampled randomly from a Schechter function
    '''
    phi = np.log10(schechter(logmstar, phi_star, logm_star, alpha))
    if np.min(phi) < 0:
        phi -= np.min(phi)
    phi /= np.sum(phi) # Using this as the weights for random sampling

    return np.random.choice(logmstar, size=N, p=phi)

def distribute_coords_as_gaussian(cenra, cendec, stdra, stddec, size):
    '''
    Outputs random RA, DEC positions centered somewhere on the sky as a 2D
    Gaussian

    Args:
        cenra: right ascension of central coordinate
        cendec: declination of central coordinate
        stdra: standard deviation along the RA direction
        stddec: standard deviation along the DEC direction
        size: number of random (RA, DEC) pairs to output

    Returns:
        Tuple containing right ascensions and declinations distributed as
        N(mu, Sigma) with mu = (cenra, cendec) and Sigma=[[stdra, 0], [0, stddec]]
    '''
    mean = [cenra, cendec]
    cov = [[stdra, 0], [0, stddec]]
    rand_ra, rand_dec = np.random.default_rng().multivariate_normal(mean,
                                                                    cov,
                                                                    size).T
    coords = (rand_ra, rand_dec)
    return coords

# =============================================================================
# Outputting values as FITS table
# =============================================================================

def write_to_table(dwarf, coords, fname):
    '''
    Outputs dwarf galaxy catalog as a FITS table in HSC injection format

    Args:
        dwarf: instance of Dwarfs() class
        coords: tuple of RA and DEC coordinates (see distribute_as_gaussian)
        fname: output table filename (throws error if not a FITS table)

    Returns:
        Nothing, but write table to working directory
    '''
    if '.fits' not in fname:
        print('Table filename must be ***.fits')
    else:
        model = {
                'raJ2000': coords[0],
                'decJ2000': coords[1],
                'a_b': np.zeros(len(dwarf.n))+1, # Normalized semi-major axis to 1
                'b_b': dwarf.q, # q = b/a = b given above
                'pa_bulge': dwarf.pa,
                'umagVar': dwarf.mabs['umagVar'],
                'gmagVar': dwarf.mabs['gmagVar'],
                'rmagVar': dwarf.mabs['rmagVar'],
                'imagVar': dwarf.mabs['imagVar'],
                'zmagVar': dwarf.mabs['zmagVar'],
                'ymagVar': dwarf.mabs['ymagVar'],
                'BulgeHalfLightRadius': dwarf.re,
                'bulge_n': dwarf.n,
                'DiskHalfLightRadius': dwarf.re, # same params as bulge
                'disk_n': dwarf.n #same params as bulge
                }

        t = Table(model)
        t.write(fname, format='fits', overwrite=True)
        print('File written as ./'+fname)

reason: import numpy,from scipy,from astropy


text:
import random
import numpy as np
from collections import defaultdict

class L2LSH(object):
    """L2LSH class, each hash value is computed by np.ceil((a @ X + b) / r)

    Args:
        prob_dim (int): size of probability vector
        r (float, optional): r is a real positive number
        num_l (int, optional): number of band,
            default=40
        num_k (int, optional): number of concatenated hash function,
            default=4
        seed (int, optional): random seed that is used to generate hash
            function coefficients, default=1

    """

    def __init__(self, prob_dim, r=8, num_l=40, num_k=4, seed=None):
        """Initialize the L2LSH.

        Args:
            prob_dim (int): size of probability vector
            r (float, optional): a real positive number, default=1
            num_l (int, optional): number of band,
            default=40
        num_k (int, optional): number of concatenated hash function,
            default=4
        seed (int, optional): random seed that is used to generate hash
            function coefficients, default=1

        """

        if seed != None:
            self.seed = seed
            np.random.seed(seed)

        self.num_l = num_l
        self.num_k = num_k
        self.r = r

        # Initiate the coefficients, num_l*num_k hash functions in total
        self.a = np.random.normal(size=(num_l*num_k, prob_dim))
        # b is uniformly at random on [0, r]
        self.b = np.random.uniform(low=0, high=self.r, size=(num_l*num_k, 1))

        # generate the hash ranges for each band
        self.hash_ranges = [(i*self.num_k, (i+1)*self.num_k)
                            for i in range(self.num_l)]
        self.hash_table = defaultdict(set)

    def _compute_concatenated_lsh(self, hash_matrix):
        """Compute concatenated hash value for each band

        Args:
            hash_matrix (array): a hash value vector

        Return:
            int: concatenated hash value
        """

        str_value = [str(x) for x in hash_matrix]
        concatenated_hash_value = ''.join(str_value)

        return concatenated_hash_value

    def compute_lsh(self, prob):
        """Compute hash value for the given probability vector

        Args:
            prob (array): a square root probability vector
        """
        return np.ceil((self.a @ prob + self.b) / self.r)

    def insert(self, prob, key):
        """Insert a probability vector with key name and store them in hash_table

        Args:
            prob (array): a square root probability vector
            key (string): key name of the inserted probability vector
        """
        # Reshape from (dim, ) to (dim, 1) to avoid Python broadcasting error
        prob = np.reshape(prob, (-1,1))

        lsh_values = self.compute_lsh(prob).flatten()

        # Insert each band hash value into hash table
        index = 0

        # Concatenation AND
        for start, end in self.hash_ranges:

            concatenated_hash_value = self._compute_concatenated_lsh(lsh_values[start: end])

            dict_key = ''.join(['L', str(index), '_', concatenated_hash_value])

            self._insert(dict_key, key)

            index += 1

    def compute_signatures_for_bands(self, prob):
        # Reshape from (dim, ) to (dim, 1) to avoid Python broadcasting error
        prob = np.reshape(prob, (-1,1))
        index = 0
        lsh_values = self.compute_lsh(prob).flatten()
        lsh_bands_signatures = []
        for start, end in self.hash_ranges:

            concatenated_hash_value = self._compute_concatenated_lsh(lsh_values[start: end])

            dict_key = ''.join(['L', str(index), '_', concatenated_hash_value])
            index += 1
            lsh_bands_signatures.append(dict_key)
        return lsh_bands_signatures

    def query(self, prob, threshold=0, allow_duplicate=False):
        """Retrieve the keys of probs that are similiar to the
        given probability vector

        Args:
            prob (array): a square root probability vector
            threshold (int, optional): number of collided hash value

        """
        # Reshape from (dim, ) to (dim, 1) to avoid Python broadcasting error
        prob = np.reshape(prob, (-1,1))

        # Compute LSH for the given probability vector
        candidates = list()
        lsh_values = self.compute_lsh(prob).flatten()

        # Construct a hit dictonary
        candidate_hit = defaultdict(int)

        # Retrieve the probs
        index = 0
        for start, end in self.hash_ranges:

            concatenated_hash_value = self._compute_concatenated_lsh(lsh_values[start: end])

            dict_key = ''.join(['L', str(index), '_', concatenated_hash_value])

            for val in self.hash_table.get(dict_key, []):
                candidate_hit[val] += 1

            index += 1

        for k, v in candidate_hit.items():
            if v >= threshold:
                # add to result multiple times
                for _ in range(v):
                    candidates.append(k)

        # TODO Can be optimized to in the future
        if allow_duplicate:
            return candidates
        else:
            return set(candidates)

    def _insert(self, key, value):
        """Insert a band hash value to the hash table

        Args:
            key (int): band hash value
            value (str): key name of the given probability vector

        """
        self.hash_table[key].add(value)

reason: import numpy


text:
import numpy as np
import timeWarpOB as tw
import unittest

class test_full(unittest.TestCase):
	'''
	Unit testing suite for TimeWarpOB.  To run, use python full.py
	at a Command Prompt.
	'''

	def testSinCos(self):
		'''Runs multiple sin/cos and check the warp is equal to pi/2'''
		def testSinCos(n,l):
			ts = list(np.linspace(0, n*np.pi, l))
			x = list(np.sin(ts))
			y = list(np.cos(ts))

			wo = tw.timeWarp(x,y,method='DTW')

			avgWarp = wo["warpStats"]["avgBehind"]

			avgWarpAng = ( (n*np.pi) / l) * avgWarp

			return (np.pi / 2) - avgWarpAng

		for i in range(2,500):
			d = testSinCos(i,(2*i))
			self.assertTrue( (round(d * 1000000)/1000000) == 0)

	def testWindow(self):
		'''Checks the warp window works by setting it equal to 1
		(0 warp deviation should be reported)'''

		ts = list(np.linspace(0, 4*np.pi, 1000))
		x = list(np.sin(ts))
		y = list(np.cos(ts))

		wo = tw.timeWarp(x,y,method='DTW', window=1)
		self.assertTrue(wo["warpStats"]["avgWarp"] == 0)

	def testL1Flat(self):
		'''Check that the L1 distance is calcuated properly,
		using two "flat" series'''

		for i in range(1,500):
			for j in range(0,100,20):
				x = list(np.ones(i) * j)
				y = list(np.ones(i))
				c = tw.L1distances(x,y)

				expected = abs(((i*i*1)-(i*i*j)))

				self.assertTrue(abs(c.sum()) == expected)

	def testL1Slope(self):
		'''Check that the L1 distance is calcuated properly,
		using two linearly sloping series'''

		x = []
		y = []
		l = 500

		for i in range(1,l+1):
			x.append(i)
			y.append(i)

		c = tw.L1distances(x,y)

		self.assertTrue(c[0].sum() == l*(0+(l-1))/2)

		for i in range(1,l):
			self.assertTrue(c[i][i] == 0)

	def testCostMat(self):
		''' Test cost matrix calcualtion for DTW and ERP with
		linearly sloping time series'''

		x = []
		y = []
		l = 500

		for i in range(1,l+1):
			x.append(i)
			y.append(i)

		c = tw.L1distances(x,y)

		self.assertTrue(c[0].sum() == l*(0+(l-1))/2)
		d = tw.timeWarpOB.DTWwarp(c,x,y,w=0)
		e = tw.timeWarpOB.ERPwarp(c,x,y,w=500,g=0)

		for i in range(0,l):
			r1 = 1/(c[i].sum() / d[i].sum())
			r2 = 1/(c[l-i-1].sum() / d[l-i-1].sum())
			self.assertTrue(r1 == r2)

			self.assertTrue(d[i][i] == 0)
			self.assertTrue(e[i][i] == 0)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: oliverbinns/timeWarpOB

max_issues_repo_name: oliverbinns/timeWarpOB

max_forks_repo_name: oliverbinns/timeWarpOB


text:
import os
import pickle
import pprint

import ego4d.utils.logging as logging
import numpy as np
import torch
from ego4d.tasks.short_term_anticipation import ShortTermAnticipationTask
from ego4d.utils.c2_model_loading import get_name_convert_func
from ego4d.utils.parser import load_config, parse_args
from pytorch_lightning import Trainer, seed_everything
from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
from scripts.slurm import copy_and_run_with_config
from pytorch_lightning.plugins import DDPPlugin

logger = logging.get_logger(__name__)

def main(cfg):
    seed_everything(cfg.RNG_SEED)

    logging.setup_logging(cfg.OUTPUT_DIR)
    logger.info("Run with config:")
    logger.info(pprint.pformat(cfg))

    # Choose task type based on config.
    if cfg.DATA.TASK == "short_term_anticipation":
        TaskType = ShortTermAnticipationTask

    task = TaskType(cfg)

    # Load model from checkpoint if checkpoint file path is given.
    ckp_path = cfg.CHECKPOINT_FILE_PATH
    if len(ckp_path) > 0:
        if cfg.CHECKPOINT_VERSION == "caffe2":
            with open(ckp_path, "rb") as f:
                data = pickle.load(f, encoding="latin1")
            state_dict = data["blobs"]
            fun = get_name_convert_func()
            state_dict = {
                fun(k): torch.from_numpy(np.array(v))
                for k, v in state_dict.items()
                if "momentum" not in k and "lr" not in k and "model_iter" not in k
            }

            if not cfg.CHECKPOINT_LOAD_MODEL_HEAD:
                state_dict = {k: v for k, v in state_dict.items() if "head" not in k}
            print(task.model.load_state_dict(state_dict, strict=False))
            print(f"Checkpoint {ckp_path} loaded")
        elif cfg.DATA.CHECKPOINT_MODULE_FILE_PATH != "":

            # Load slowfast weights into backbone submodule
            ckpt = torch.load(
                cfg.DATA.CHECKPOINT_MODULE_FILE_PATH,
                map_location=lambda storage, loc: storage,
            )

            def remove_first_module(key):
                return ".".join(key.split(".")[1:])

            state_dict = {
                remove_first_module(k): v
                for k, v in ckpt["state_dict"].items()
                if "head" not in k
            }
            missing_keys, unexpected_keys = task.model.backbone.load_state_dict(
                state_dict, strict=False
            )

            # Ensure only head key is missing.
            assert len(unexpected_keys) == 0
            assert all(["head" in x for x in missing_keys])
        else:
            # Load all child modules except for "head" if CHECKPOINT_LOAD_MODEL_HEAD is
            # False.
            pretrained = TaskType.load_from_checkpoint(ckp_path)
            state_dict_for_child_module = {
                child_name: child_state_dict.state_dict()
                for child_name, child_state_dict in pretrained.model.named_children()
            }
            for child_name, child_module in task.model.named_children():
                if not cfg.CHECKPOINT_LOAD_MODEL_HEAD and "head" in child_name:
                    continue

                state_dict = state_dict_for_child_module[child_name]
                child_module.load_state_dict(state_dict)

    checkpoint_callback = ModelCheckpoint(
        monitor=task.checkpoint_metric, mode="min", save_last=True, save_top_k=1
    )
    if cfg.ENABLE_LOGGING:
        args = {"callbacks": [LearningRateMonitor(), checkpoint_callback]}
    else:
        args = {"logger": False, "callbacks": checkpoint_callback}

    trainer = Trainer(
        gpus=cfg.NUM_GPUS,
        num_nodes=cfg.NUM_SHARDS,
        accelerator=cfg.SOLVER.ACCELERATOR,
        max_epochs=cfg.SOLVER.MAX_EPOCH,
        num_sanity_val_steps=3,
        benchmark=True,
        log_gpu_memory="min_max",
        replace_sampler_ddp=False,
        fast_dev_run=cfg.FAST_DEV_RUN,
        default_root_dir=cfg.OUTPUT_DIR,
        plugins=DDPPlugin(find_unused_parameters=False),
        **args,
    )

    if cfg.TRAIN.ENABLE and cfg.TEST.ENABLE:
        trainer.fit(task)

        # Calling test without the lightning module arg automatically selects the best
        # model during training.
        return trainer.test()

    elif cfg.TRAIN.ENABLE:
        return trainer.fit(task)

    elif cfg.TEST.ENABLE:
        return trainer.test(task)

if __name__ == "__main__":
    args = parse_args()
    cfg = load_config(args)
    if args.on_cluster:
        copy_and_run_with_config(
            main,
            cfg,
            args.working_directory,
            job_name=args.job_name,
            time="72:00:00",
            partition="learnfair",
            gpus_per_node=cfg.NUM_GPUS,
            ntasks_per_node=cfg.NUM_GPUS,
            cpus_per_task=10,
            mem="470GB",
            nodes=cfg.NUM_SHARDS,
            constraint="volta32gb",
        )
    else:  # local
        main(cfg)

max_stars_repo_name: VJWQ/forecasting

max_issues_repo_name: VJWQ/forecasting

max_forks_repo_name: VJWQ/forecasting


text:
#  Copyright (c) 2022 Manuel Olguín Muñoz <molguin@kth.se>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import contextlib
import socket
import time
from collections import deque
from typing import Callable, Literal

import click
import pandas as pd
from loguru import logger

from ..common import response_stream_unpack, pack_frame
from ... import data as e_data
from ...models import (
    EdgeDroidModel,
    EmpiricalExecutionTimeModel,
    ExecutionTimeModel,
    FrameModel,
    ModelFrame,
    TheoreticalExecutionTimeModel,
)

import numpy.typing as npt

from ...models.timings import NaiveExecutionTimeModel

class StreamSocketEmulation:
    def __init__(
        self,
        neuroticism: float,
        trace: str,
        fade_distance: int,
        model: Literal["theoretical", "empirical", "naive"] = "theoretical",
    ):
        logger.info(
            f"Initializing EdgeDroid model with neuroticism {neuroticism:0.2f} and "
            f"fade distance {fade_distance:d} steps"
        )
        logger.info(f"Model type: {model}")
        logger.info(f"Trace: {trace}")

        # should be able to use a single thread for everything

        # first thing first, prepare data
        frameset = e_data.load_default_trace(trace)

        match model:
            case "theoretical":
                timing_model: ExecutionTimeModel = (
                    TheoreticalExecutionTimeModel.from_default_data(
                        neuroticism=neuroticism,
                        transition_fade_distance=fade_distance,
                    )
                )
            case "empirical":
                timing_model: ExecutionTimeModel = (
                    EmpiricalExecutionTimeModel.from_default_data(
                        neuroticism=neuroticism,
                        transition_fade_distance=fade_distance,
                    )
                )
            case "naive":
                timing_model: ExecutionTimeModel = (
                    NaiveExecutionTimeModel.from_default_data()
                )
            case _:
                raise NotImplementedError(f"Unrecognized execution time model: {model}")

        frame_model = FrameModel(e_data.load_default_frame_probabilities())

        self._model = EdgeDroidModel(
            frame_trace=frameset, frame_model=frame_model, timing_model=timing_model
        )

        self._frame_records = deque()

    def get_step_metrics(self) -> pd.DataFrame:
        return self._model.model_step_metrics()

    def get_frame_metrics(self) -> pd.DataFrame:
        return pd.DataFrame(self._frame_records).set_index("seq")

    def emulate(
        self,
        sock: socket.SocketType,
        emit_cb: Callable[[ModelFrame], None] = lambda _: None,
        resp_cb: Callable[[bool, npt.NDArray, str], None] = lambda t, i, s: None,
    ) -> None:
        """
        # TODO: document

        Parameters
        ----------
        sock
        emit_cb
        resp_cb

        Returns
        -------

        """

        logger.warning("Starting emulation")
        start_time = time.time()
        start_time_mono = time.monotonic()
        with contextlib.closing(response_stream_unpack(sock)) as resp_stream:
            for step_num, model_step in enumerate(self._model.play_steps()):
                logger.info(f"Current step: {step_num}")
                for model_frame in model_step:
                    # package and send the frame
                    logger.debug(
                        f"Sending frame:\n"
                        f"\tSeq: {model_frame.seq}\n"
                        f"\tTag: {model_frame.frame_tag}\n"
                        f"\tStep index: {model_frame.step_index}\n"
                        f"\tFrame step seq: {model_frame.step_seq}"
                    )
                    payload = pack_frame(model_frame.seq, model_frame.frame_data)
                    send_time = time.monotonic()
                    sock.sendall(payload)
                    emit_cb(model_frame)

                    # wait for response
                    logger.debug("Waiting for response from server")
                    transition, guidance_img, guidance_text = next(resp_stream)
                    recv_time = time.monotonic()
                    rtt = recv_time - send_time
                    logger.debug("Received response from server")
                    logger.debug(f"Frame round-trip-time: {rtt:0.3f} seconds")
                    logger.info(f"Guidance: {guidance_text}")
                    resp_cb(transition, guidance_img, guidance_text)

                    # log the frame
                    self._frame_records.append(
                        {
                            "seq": model_frame.seq,
                            "step_index": model_frame.step_index,
                            "step_seq": model_frame.step_seq,
                            "expected_tag": model_frame.frame_tag,
                            "transition": transition,
                            "send_time": start_time + (send_time - start_time_mono),
                            "rtt": rtt,
                        }
                    )

                    if model_frame.frame_tag in ("success", "initial"):
                        if transition:
                            # if we receive a response for a success frame, advance the
                            # model
                            logger.success("Advancing to next step")
                            break
                        else:
                            logger.error(
                                "Received unexpected unsuccessful response from "
                                "server, aborting"
                            )
                            raise click.Abort()

        logger.warning("Emulation finished")


text:
import numpy as np

arr = np.array([1, 0, 3])

newarr = arr.astype(bool)

print(newarr)
print(newarr.dtype)


text:
from sympy.concrete.tests.test_sums_products import NS

from sympy import sqrt, S
from sympy.physics.units import (
    convert_to,
    coulomb_constant,
    elementary_charge,
    gravitational_constant,
    planck,
)
from sympy.physics.units.definitions.unit_definitions import (
    statcoulomb,
    coulomb,
    second,
    gram,
    centimeter,
    erg,
    newton,
    joule,
    dyne,
    speed_of_light,
    meter,
)
from sympy.physics.units.systems import SI
from sympy.physics.units.systems.cgs import cgs_gauss

def test_conversion_to_from_si():

    assert convert_to(statcoulomb, coulomb, cgs_gauss) == 5 * coulomb / 149896229
    assert convert_to(coulomb, statcoulomb, cgs_gauss) == 149896229 * statcoulomb / 5
    assert (
        convert_to(statcoulomb, sqrt(gram * centimeter ** 3) / second, cgs_gauss)
        == centimeter ** (S(3) / 2) * sqrt(gram) / second
    )
    assert convert_to(
        coulomb, sqrt(gram * centimeter ** 3) / second, cgs_gauss
    ) == 149896229 * centimeter ** (S(3) / 2) * sqrt(gram) / (5 * second)

    # SI units have an additional base unit, no conversion in case of electromagnetism:
    assert convert_to(coulomb, statcoulomb, SI) == coulomb
    assert convert_to(statcoulomb, coulomb, SI) == statcoulomb

    # SI without electromagnetism:
    assert convert_to(erg, joule, SI) == joule / 10 ** 7
    assert convert_to(erg, joule, cgs_gauss) == joule / 10 ** 7
    assert convert_to(joule, erg, SI) == 10 ** 7 * erg
    assert convert_to(joule, erg, cgs_gauss) == 10 ** 7 * erg

    assert convert_to(dyne, newton, SI) == newton / 10 ** 5
    assert convert_to(dyne, newton, cgs_gauss) == newton / 10 ** 5
    assert convert_to(newton, dyne, SI) == 10 ** 5 * dyne
    assert convert_to(newton, dyne, cgs_gauss) == 10 ** 5 * dyne

def test_cgs_gauss_convert_constants():

    assert (
        convert_to(speed_of_light, centimeter / second, cgs_gauss)
        == 29979245800 * centimeter / second
    )

    assert convert_to(coulomb_constant, 1, cgs_gauss) == 1
    assert convert_to(
        coulomb_constant, newton * meter ** 2 / coulomb ** 2, cgs_gauss
    ) == 22468879468420441 * meter ** 2 * newton / (25000000000 * coulomb ** 2)
    assert convert_to(
        coulomb_constant, newton * meter ** 2 / coulomb ** 2, SI
    ) == 22468879468420441 * meter ** 2 * newton / (2500000 * coulomb ** 2)
    assert (
        convert_to(
            coulomb_constant, dyne * centimeter ** 2 / statcoulomb ** 2, cgs_gauss
        )
        == centimeter ** 2 * dyne / statcoulomb ** 2
    )
    assert convert_to(coulomb_constant, 1, SI) == coulomb_constant
    assert (
        NS(convert_to(coulomb_constant, newton * meter ** 2 / coulomb ** 2, SI))
        == "8987551787.36818*meter**2*newton/coulomb**2"
    )

    assert convert_to(elementary_charge, statcoulomb, cgs_gauss)
    assert convert_to(
        gravitational_constant, dyne * centimeter ** 2 / gram ** 2, cgs_gauss
    )
    assert (
        NS(convert_to(planck, erg * second, cgs_gauss)) == "6.62607015e-27*erg*second"
    )

max_stars_repo_name: msgoff/sympy

max_issues_repo_name: msgoff/sympy

max_forks_repo_name: msgoff/sympy

reason: from sympy


text:
# coding: utf-8

# # Brain Tumor Detection Using a Convolutional Neural Network

# **About the Brain MRI Images dataset:**

# The dataset contains 2 folders: yes and no which contains 253 Brain MRI Images. The folder yes contains 155 Brain MRI Images that are tumorous and the folder no contains 98 Brain MRI Images that are non-tumorous. You can find it [here](https://www.kaggle.com/navoneel/brain-mri-images-for-brain-tumor-detection).

# This code uses [mlflow](https://mlflow.org/) to track the models and experiments of the project.

# You can read more about details about the project in this Medium [blog](https://medium.com/@mohamedalihabib7/brain-tumor-detection-using-convolutional-neural-networks-30ccef6612b0).

# ## Import Necessary Modules

# In[1]:

import tensorflow as tf
from tensorflow.keras.layers import Conv2D, Input, ZeroPadding2D, BatchNormalization, Activation, MaxPooling2D, Flatten, Dense
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint, Callback
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score, accuracy_score, log_loss
from sklearn.utils import shuffle
import cv2
import imutils
import numpy as np
import matplotlib.pyplot as plt
import time
import math
from os import listdir

import mlflow
import mlflow.keras

#get_ipython().run_line_magic('matplotlib', 'inline')

# ## Data Preparation & Preprocessing

# In order to crop the part that contains only the brain of the image, I used a cropping technique to find the extreme top, bottom, left and right points of the brain. You can read more about it here [Finding extreme points in contours with OpenCV](https://www.pyimagesearch.com/2016/04/11/finding-extreme-points-in-contours-with-opencv/).

# In[2]:

def crop_brain_contour(image, plot=False):

    #import imutils
    #import cv2
    #from matplotlib import pyplot as plt

    # Convert the image to grayscale, and blur it slightly
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    gray = cv2.GaussianBlur(gray, (5, 5), 0)

    # Threshold the image, then perform a series of erosions +
    # dilations to remove any small regions of noise
    thresh = cv2.threshold(gray, 45, 255, cv2.THRESH_BINARY)[1]
    thresh = cv2.erode(thresh, None, iterations=2)
    thresh = cv2.dilate(thresh, None, iterations=2)

    # Find contours in thresholded image, then grab the largest one
    cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    cnts = imutils.grab_contours(cnts)
    c = max(cnts, key=cv2.contourArea)

    # Find the extreme points
    extLeft = tuple(c[c[:, :, 0].argmin()][0])
    extRight = tuple(c[c[:, :, 0].argmax()][0])
    extTop = tuple(c[c[:, :, 1].argmin()][0])
    extBot = tuple(c[c[:, :, 1].argmax()][0])

    # crop new image out of the original image using the four extreme points (left, right, top, bottom)
    new_image = image[extTop[1]:extBot[1], extLeft[0]:extRight[0]]

    if plot:
        plt.figure()

        plt.subplot(1, 2, 1)
        plt.imshow(image)

        plt.tick_params(axis='both', which='both',
                        top=False, bottom=False, left=False, right=False,
                        labelbottom=False, labeltop=False, labelleft=False, labelright=False)

        plt.title('Original Image')

        plt.subplot(1, 2, 2)
        plt.imshow(new_image)

        plt.tick_params(axis='both', which='both',
                        top=False, bottom=False, left=False, right=False,
                        labelbottom=False, labeltop=False, labelleft=False, labelright=False)

        plt.title('Cropped Image')

        plt.show()

    return new_image

# In order to better understand what it's doing, let's grab an image from the dataset and apply this cropping function to see the result:

# In[3]:

#ex_img = cv2.imread('augmented data/yes/aug_Y1_0_2900.jpg')
#ex_new_img = crop_brain_contour(ex_img, True)

# ### Load up the data:

# The following function takes two arguments, the first one is a list of directory paths for the folders 'yes' and 'no' that contain the image data and the second argument is the image size, and for every image in both directories and does the following:
# 1. Read the image.
# 2. Crop the part of the image representing only the brain.
# 3. Resize the image (because the images in the dataset come in different sizes (meaning width, height and # of channels). So, we want all of our images to be (240, 240, 3) to feed it as an input to the neural network.
# 4. Apply normalization because we want pixel values to be scaled to the range 0-1.
# 5. Append the image to X and its label to y.

#
# After that, Shuffle X and y, because the data is ordered (meaning the arrays contains the first part belonging to one class and the second part belonging to the other class, and we don't want that).

# Finally, Return X and y.

# In[4]:

def load_data(dir_list, image_size):
    """
    Read images, resize and normalize them.
    Arguments:
        dir_list: list of strings representing file directories.
    Returns:
        X: A numpy array with shape = (#_examples, image_width, image_height, #_channels)
        y: A numpy array with shape = (#_examples, 1)
    """

    # load all images in a directory
    X = []
    y = []
    image_width, image_height = image_size

    for directory in dir_list:
        for filename in listdir(directory):
            # load the image
            image = cv2.imread(directory + '\\' + filename)
            # crop the brain and ignore the unnecessary rest part of the image
            image = crop_brain_contour(image, plot=False)
            # resize image
            image = cv2.resize(image, dsize=(image_width, image_height), interpolation=cv2.INTER_CUBIC)
            # normalize values
            image = image / 255.
            # convert image to numpy array and append it to X
            X.append(image)
            # append a value of 1 to the target array if the image
            # is in the folder named 'yes', otherwise append 0.
            if directory[-3:] == 'yes':
                y.append([1])
            else:
                y.append([0])

    X = np.array(X)
    y = np.array(y)

    # Shuffle the data
    X, y = shuffle(X, y)

    print(f'Number of examples is: {len(X)}')
    print(f'X shape is: {X.shape}')
    print(f'y shape is: {y.shape}')

    return X, y

# Load up the data that we augmented earlier in the Data Augmentation notebook.

# **Note:** the augmented data directory contains not only the new generated images but also the original images.

# In[5]:

augmented_path = 'augmented data/'

# augmented data (yes and no) contains both the original and the new generated examples
augmented_yes = augmented_path + 'yes'
augmented_no = augmented_path + 'no'

IMG_WIDTH, IMG_HEIGHT = (240, 240)

X, y = load_data([augmented_yes, augmented_no], (IMG_WIDTH, IMG_HEIGHT))

# As we see, we have 2065 images. Each images has a shape of **(240, 240, 3)=(image_width, image_height, number_of_channels)**

# ### Plot sample images:

# In[5]:

def plot_sample_images(X, y, n=50):
    """
    Plots n sample images for both values of y (labels).
    Arguments:
        X: A numpy array with shape = (#_examples, image_width, image_height, #_channels)
        y: A numpy array with shape = (#_examples, 1)
    """

    for label in [0,1]:
        # grab the first n images with the corresponding y values equal to label
        images = X[np.argwhere(y == label)]
        n_images = images[:n]

        columns_n = 10
        rows_n = int(n/ columns_n)

        plt.figure(figsize=(20, 10))

        i = 1 # current plot
        for image in n_images:
            plt.subplot(rows_n, columns_n, i)
            plt.imshow(image[0])

            # remove ticks
            plt.tick_params(axis='both', which='both',
                            top=False, bottom=False, left=False, right=False,
                           labelbottom=False, labeltop=False, labelleft=False, labelright=False)

            i += 1

        label_to_str = lambda label: "Yes" if label == 1 else "No"
        plt.suptitle(f"Brain Tumor: {label_to_str(label)}")
        plt.show()

# In[7]:

#plot_sample_images(X, y)

# ### Split the data:
# Split X and y into training, validation (development) and validation sets.

# In[6]:

def split_data(X, y, test_size=0.2):

    """
    Splits data into training, development and test sets.
    Arguments:
        X: A numpy array with shape = (#_examples, image_width, image_height, #_channels)
        y: A numpy array with shape = (#_examples, 1)
    Returns:
        X_train: A numpy array with shape = (#_train_examples, image_width, image_height, #_channels)
        y_train: A numpy array with shape = (#_train_examples, 1)
        X_val: A numpy array with shape = (#_val_examples, image_width, image_height, #_channels)
        y_val: A numpy array with shape = (#_val_examples, 1)
        X_test: A numpy array with shape = (#_test_examples, image_width, image_height, #_channels)
        y_test: A numpy array with shape = (#_test_examples, 1)
    """

    X_train, X_test_val, y_train, y_test_val = train_test_split(X, y, test_size=test_size)
    X_test, X_val, y_test, y_val = train_test_split(X_test_val, y_test_val, test_size=0.5)

    return X_train, y_train, X_val, y_val, X_test, y_test

# Let's use the following way to split:

# 1. 70% of the data for training.
# 2. 15% of the data for validation.
# 3. 15% of the data for testing.

# In[9]:

X_train, y_train, X_val, y_val, X_test, y_test = split_data(X, y, test_size=0.3)

# In[10]:

print ("number of training examples = " + str(X_train.shape[0]))
print ("number of development examples = " + str(X_val.shape[0]))
print ("number of test examples = " + str(X_test.shape[0]))
print ("X_train shape: " + str(X_train.shape))
print ("Y_train shape: " + str(y_train.shape))
print ("X_val (dev) shape: " + str(X_val.shape))
print ("Y_val (dev) shape: " + str(y_val.shape))
print ("X_test shape: " + str(X_test.shape))
print ("Y_test shape: " + str(y_test.shape))

# Some helper functions:

# In[13]:

# Nicely formatted time string
def hms_string(sec_elapsed):
    h = int(sec_elapsed / (60 * 60))
    m = int((sec_elapsed % (60 * 60)) / 60)
    s = sec_elapsed % 60
    return f"{h}:{m}:{round(s,1)}"

# In[14]:

def compute_f1_score(prefix, y_true, prob, epoch):

    f1score = f1_score(y_true, np.round(prob))
    mlflow.log_metric("{}_f1_score".format(prefix), f1score, step=epoch)

    return f1score

# In[15]:

def compute_loss(prefix, y_true, prob, epoch):

    #print('predictions: ', pred)
    #print('rounded pred: ', np.round(pred))
    #print('actual: ', actual)

    binary_cross_entropy = log_loss(y_true, prob)
    mlflow.log_metric("{}_loss".format(prefix), binary_cross_entropy, step=epoch)

    #print("{}_loss_epoch_{} = {}".format(prefix, epoch, binary_cross_entropy))

    return binary_cross_entropy

# In[16]:

def compute_accuracy(prefix, y_true, prob, epoch):

    accuracy = accuracy_score(y_true, np.round(prob))
    mlflow.log_metric("{}_acc".format(prefix), accuracy, step=epoch)

    return accuracy

# In[17]:

class MLflowCheckpoint(Callback):
    """
    A Keras MLflow logger that does the following:
        1. Logs training metrics and final model with MLflow.
        2. Keeps track of the best model (best loss on validation dataset). Every improvement of the best model is also
        evaluated on the test set and the following metrics are logged: loss, accuracy and f1_score
        3. At the end of the training, log the best model with MLflow.
    """

    def __init__(self, test_x, test_y, loss="loss", metric='acc'):
        self._test_x = test_x
        self._test_y = test_y

        self.train_loss = "train_{}".format(loss)
        self.val_loss = "val_{}".format(loss)
        self.test_loss = "test_{}".format(loss)

        self.train_acc = 'train_{}'.format(metric)
        self.val_acc = 'val_{}'.format(metric)
        self.test_acc = 'test_{}'.format(metric)

        self._best_train_loss = math.inf
        self._best_val_loss = math.inf
        self._best_train_acc = math.inf
        self._best_val_acc = math.inf
        self._best_model = None

        self._next_step = 0

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        """
        Log the best model at the end of the training run.
        """
        if not self._best_model:
            raise Exception("Failed to build any model")

        mlflow.log_metric(self.train_loss, self._best_train_loss, step=self._next_step)
        mlflow.log_metric(self.train_acc, self._best_train_acc, step=self._next_step)
        mlflow.log_metric(self.val_loss, self._best_val_loss, step=self._next_step)
        mlflow.log_metric(self.val_acc, self._best_val_acc, step=self._next_step)

        mlflow.keras.log_model(self._best_model, "model")

    def on_epoch_end(self, epoch, logs=None):
        """
        Log Keras metrics with MLflow. If model improved on the validation data, evaluate it on
        a test set and store it as the best model.
        """
        if not logs:
            return

        self._next_step = epoch + 1

        train_loss = logs["loss"]
        val_loss = logs["val_loss"]
        train_acc = logs['acc']
        val_acc = logs['val_acc']

        mlflow.log_metrics({
            self.train_loss: train_loss,
            self.val_loss: val_loss,
            self.train_acc: train_acc,
            self.val_acc: val_acc
        }, step=epoch)

        if val_acc < self._best_val_acc:
            # The result improved in the validation set.
            # Log the model with mlflow and also evaluate and log on test set.
            self._best_train_loss = train_loss
            self._best_val_loss = val_loss

            self._best_train_acc = train_acc
            self._best_val_acc = val_acc

            self._best_model = tf.keras.models.clone_model(self.model)
            self._best_model.set_weights([x.copy() for x in self.model.get_weights()])

            #print("New_best_model_on_epoch:{}".format(epoch))
            #print('val_acc:{}'.format(val_acc))
            #print("best_val_acc:{}".format(self._best_val_acc))

            preds = self._best_model.predict(self._test_x)

            # evaluate the best model on the testing set
            compute_loss("test", self._test_y, preds, epoch) # loss
            compute_accuracy("test", self._test_y, preds, epoch) # accuracy
            compute_f1_score("test", self._test_y, preds, epoch) # f1_score

# # Build the model

# Let's build a convolutional neural network model:

#

# In[18]:

def build_model(input_shape):
    """
    Arugments:
        input_shape: A tuple representing the shape of the input of the model. shape=(image_width, image_height, #_channels)
    Returns:
        model: A Model object.
    """
    # Define the input placeholder as a tensor with shape input_shape.
    X_input = Input(input_shape) # shape=(?, 240, 240, 3)

    # Zero-Padding: pads the border of X_input with zeroes
    X = ZeroPadding2D((2, 2))(X_input) # shape=(?, 244, 244, 3)

    # CONV -> BN -> RELU Block applied to X
    X = Conv2D(32, (7, 7), strides = (1, 1), name = 'conv0')(X)
    X = BatchNormalization(axis = 3, name = 'bn0')(X)
    X = Activation('relu')(X) # shape=(?, 238, 238, 32)

    # MAXPOOL
    X = MaxPooling2D((4, 4), name='max_pool0')(X) # shape=(?, 59, 59, 32)

    # MAXPOOL
    X = MaxPooling2D((4, 4), name='max_pool1')(X) # shape=(?, 14, 14, 32)

    # FLATTEN X
    X = Flatten()(X) # shape=(?, 6272)
    # FULLYCONNECTED
    X = Dense(1, activation='sigmoid', name='fc')(X) # shape=(?, 1)

    # Create model. This creates your Keras model instance, you'll use this instance to train/test the model.
    model = Model(inputs = X_input, outputs = X, name='BrainDetectionModel')

    return model

# In[21]:

IMG_SHAPE = (IMG_WIDTH, IMG_HEIGHT, 3)

# In[22]:

model = build_model(IMG_SHAPE)

# In[23]:

model.summary()

# Compile the model:

# In[24]:

model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

# In[25]:

# tensorboard
log_file_name = f'brain_tumor_detection_cnn_{int(time.time())}'
tensorboard = TensorBoard(log_dir=f'logs/{log_file_name}')

# In[26]:

# checkpoint
# unique file name that will include the epoch and the validation (development) accuracy
filepath="cnn-parameters-improvement-{epoch:02d}-{val_acc:.2f}"
# save the model with the best validation (development) accuracy till now
checkpoint = ModelCheckpoint("models/{}.model".format(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max'))

# ## Train the model

# In[27]:

start_time = time.time()

mlflow.warnings.filterwarnings("ignore")

with mlflow.start_run():
    with MLflowCheckpoint(X_test, y_test) as mlflow_logger:
        model.fit(x=X_train, y=y_train, batch_size=32, epochs=25, validation_data=(X_val, y_val),
                          callbacks=[mlflow_logger, tensorboard, checkpoint])

end_time = time.time()
execution_time = (end_time - start_time)
print(f"Elapsed time: {hms_string(execution_time)}")

# In[28]:

history = model.history.history

# In[29]:

#for key in history.keys():
#    print(key)

# ## Plot Loss & Accuracy

# In[30]:

def plot_metrics(history):

    train_loss = history['loss']
    val_loss = history['val_loss']
    train_acc = history['acc']
    val_acc = history['val_acc']

    # Loss
    plt.figure()
    plt.plot(train_loss, label='Training Loss')
    plt.plot(val_loss, label='Validation Loss')
    plt.title('Loss')
    plt.legend()
    plt.show()

    # Accuracy
    plt.figure()
    plt.plot(train_acc, label='Training Accuracy')
    plt.plot(val_acc, label='Validation Accuracy')
    plt.title('Accuracy')
    plt.legend()
    plt.show()

# In[31]:

plot_metrics(history)

# # Results

# Let's experiment with the best model (the one with the best validation accuracy):

# In[56]:

# find the iteration in which we had the best validation accuracy
best_validation_accuracy = max(history['val_acc'])
best_iteration = history['val_acc'].index(best_validation_accuracy) + 1

# Concretely, the model at the 24th iteration with validation accuracy of 90%

# In[41]:

def best_model_file_name(path, best_validation_accuracy):
    """
    This method returns the file path for the model with best validation accuracy.
    Arguemnts:
        path: directory path for the file where the files .models exist.
    Returns:
        file_name: a string representing the file name for the model with the best validation accuracy
    """

    best_models = [] # a list of file names for models that has the best validation accuracy
    for filename in listdir(path):

        # grab the part of the name that represents the accuracy and compare it with the best validation accuracy
        acc_part = filename[-10:-6] # the part of the file name that represents the accuracy
        best_val_acc = str(np.round(best_validation_accuracy, 2))
        # if best_val_acc is equal to something like '0.4', append a zero to look like this '0.40'
        # that's to make easier to compare with accuracy part of the file names.
        if len(best_val_acc) < 4:
            best_val_acc += '0'
        #print(best_val_acc)
        if acc_part == best_val_acc:
            best_models.append(filename)

    #print(best_models)
    # grab the last model of the models that has the best validation accuracy
    file_name = best_models[-1]
    return file_name

# grab the name of the file for the model with the best validation accuracy
filename = best_model_file_name('models/', best_validation_accuracy)

best_model = load_model(filepath=f'models/{filename}')
#best_model = load_model(filepath='models/cnn-parameters-improvement-24-0.90.model')

# In[42]:

#best_model.metrics_names

# Evaluate the best model on the testing data:

# In[43]:

print("Evaluate the best model on the testing data:")
loss, acc = best_model.evaluate(x=X_test, y=y_test)

# ### Accuracy of the best model on the testing data:

# In[44]:

print("Accuracy of the best model on the testing data")
print (f"Test Loss = {loss}")
print (f"Test Accuracy = {acc}")

# ### F1 score for the best model on the testing data:

# In[45]:

print("F1 score for the best model on the testing data:")
y_test_prob = best_model.predict(X_test)

# In[47]:

f1score = f1_score(y_test, np.round(y_test_prob))
print(f"F1 score: {f1score}")

# Let's also find the f1 score on the validation data:

# In[48]:

y_val_prob = best_model.predict(X_val)

# In[49]:

f1score_val = f1_score(y_val, np.round(y_val_prob))
print(f"F1 score: {f1score_val}")

# ### Results Interpretation

# In[50]:

def data_percentage(y):

    m=len(y)
    n_positive = np.sum(y)
    n_negative = m - n_positive

    pos_prec = (n_positive* 100.0)/ m
    neg_prec = (n_negative* 100.0)/ m

    print(f"Number of examples: {m}")
    print(f"Percentage of positive examples: {pos_prec}%, number of pos examples: {n_positive}")
    print(f"Percentage of negative examples: {neg_prec}%, number of neg examples: {n_negative}")

# In[54]:

# the whole data
data_percentage(y)

# In[55]:

print("Training Data:")
data_percentage(y_train)
print("Validation Data:")
data_percentage(y_val)
print("Testing Data:")
data_percentage(y_test)

# As expectred, the percentage of positive examples are around 50%.

# # Conclusion:

# #### Now, the model detects brain tumor with:

# **88.3%** accuracy on the **test set**.

# **0.88** f1 score on the **test set**.

# These resutls are very good considering that the data is balanced.

# **Performance Table:**

# |   | Validation set | Test set |
# | --------- | -------------- | -------- |
# | Accuracy  | 90%            | 88%      |
# | F1 score  | 0.91           | 0.88     |

# Hooray!


text:
import tvm
import sys
from tvm.contrib import graph_runtime
import numpy as np
from PIL import Image
from utils.timerecorder import TimeRecoder

class Mobilenet():
    def __init__(self, mode, lib_path, graph_path, param_path, class_path):
        # Load compiled lib and params from file
        graph = open(graph_path).read()
        lib = tvm.module.load(lib_path)
        params = bytearray(open(param_path, "rb").read())

        # load classes
        with open(class_path) as f:
            self.classes = eval(f.read())

        # Init target mode
        self.ctx = tvm.cpu(0)
        if mode == 'gpu':
            self.ctx = tvm.gpu(0)

        self.runtime = graph_runtime.create(graph, lib, self.ctx)
        self.runtime.load_params(params)

        # init Timer
        self.timer = TimeRecoder()

    def run(self, im_fname):
        # Load Image
        img = Image.open(im_fname).resize((224, 224))
        img_bgr = np.array(img)[:,:,::-1]
        x = np.transpose(img_bgr, (2, 0, 1))[np.newaxis, :]
        dtype = 'float32'
        tvm_input = tvm.nd.array(x.astype(dtype))
        print(tvm_input.shape)

        # do inference
        self.timer.store_timestamp('inference')
        self.runtime.set_input('image', tvm_input)
        self.runtime.run()
        self.timer.store_timestamp('inference')

        tvm_output = self.runtime.get_output(0)
        top1 = np.argmax(tvm_output.asnumpy()[0])
        results = {'id': str(top1), 'class': self.classes[top1]}
        return results

if __name__ == '__main__':
    mode = sys.argv[1]
    model_dir = sys.argv[2]
    image_path = sys.argv[3]

    lib_path = model_dir + "deploy_lib.tar"
    graph_path = model_dir + "deploy_graph.json"
    param_path = model_dir + "deploy_param.params"
    class_path = model_dir + "classes.txt"

    mobilenet = Mobilenet(mode, lib_path, graph_path, param_path, class_path)
    results = mobilenet.run(image_path)
    print(results)

max_stars_repo_name: mengyaliu/DLRU

max_issues_repo_name: mengyaliu/DLRU

max_forks_repo_name: mengyaliu/DLRU


text:
"""
Copyright: Wenyi Tang 2017-2018
Author: Wenyi Tang
Email: wenyi.tang@intel.com
Created Date: May 11th 2018
Updated Date: May 17th 2018

SR model running environment
"""
import tensorflow as tf
import numpy as np
import time
import tqdm
from pathlib import Path

from VLLV.Framework.SuperResolution import SuperResolution
from VLLV.DataLoader.Loader import MpLoader, QuickLoader
from VLLV.DataLoader.Dataset import Dataset
from VLLV.Util.Utility import to_list

class Environment:
    """A model wrapper like tf.Estimator

    Args:
        model: the compiled model object. Have to call model.compile explicitly
        save_dir: the dir to save training checkpoints
        log_dir: the dir to save tensorboard log events
        feature_callbacks: a list of callable called in turn to process features.
                           the signature of this callable is `fn(x)->x`
        label_callbacks: a list of callable called in turn to process labels
                         the signature of this callable is `fn(x)->x`
        output_callbacks: a list of callable called in turn to post-process outputs
                          the signature of this callable is `fn(input, output, save_dir, step)->output`
        feature_index: the index to access in the return list of BatchLoader, default the 2nd item in list
        label_index: the index to access in the return list of BatchLoader, default the 1st item in list
        verbose: tf logger level
    """

    def __init__(self,
                 model,
                 save_dir,
                 log_dir,
                 feature_callbacks=None,
                 label_callbacks=None,
                 output_callbacks=None,
                 feature_index=None,
                 label_index=None,
                 verbose=tf.logging.INFO,
                 **kwargs):
        assert isinstance(model, SuperResolution)

        self.model = model
        self.savedir = Path(save_dir)
        self.logdir = Path(log_dir)
        self.savedir.mkdir(parents=True, exist_ok=True)
        self.logdir.mkdir(parents=True, exist_ok=True)
        self.feature_callbacks = feature_callbacks or []
        self.label_callbacks = label_callbacks or []
        self.output_callbacks = output_callbacks or []
        self.fi = feature_index if feature_index is not None else 1
        self.li = label_index if label_index is not None else 0
        tf.logging.set_verbosity(verbose)

    def __enter__(self):
        """Create session of tensorflow and build model graph"""

        sess = tf.Session()
        sess.__enter__()
        if not self.model.compiled:
            self.model.compile()
        self.savers = self.model.savers
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        """Close session and
        #TODO ? clear graph ?
        """

        sess = tf.get_default_session()
        sess.__exit__(exc_type, exc_val, exc_tb)
        # tf.reset_default_graph() ?

    def _make_ckpt_name(self, name, step):
        return f'{name}-sc{self.model.scale[0]}-ep{step:04d}.ckpt'

    def _parse_ckpt_name(self, name):
        # sample name: {model}-sc{scale}-ep{epoch}.ckpt(.index)
        if not name:
            return 0
        model_name, scale, epochs = Path(name).stem.split('.')[0].split('-')
        return int(epochs[2:])

    def _find_last_ckpt(self):
        # restore the latest checkpoint in savedir
        ckpt = tf.train.get_checkpoint_state(self.savedir)
        if ckpt and ckpt.model_checkpoint_path:
            return tf.train.latest_checkpoint(self.savedir)
        # try another way
        ckpt = to_list(self.savedir.glob('*.ckpt.index'))
        # sort as modification time
        ckpt = sorted(ckpt, key=lambda x: x.stat().st_mtime_ns)
        return self.savedir / ckpt[-1].stem if ckpt else None

    def _restore_model(self, sess):
        last_checkpoint_step = 0
        for name in self.savers:
            saver = self.savers.get(name)
            ckpt = to_list(self.savedir.glob(f'{name}*.index'))
            if ckpt:
                ckpt = sorted(ckpt, key=lambda x: x.stat().st_mtime_ns)
                ckpt = self.savedir / ckpt[-1].stem
                try:
                    saver.restore(sess, str(ckpt))
                except:
                    tf.logging.warning(f'{name} of model {self.model.name} counld not be restored')
                last_checkpoint_step = self._parse_ckpt_name(ckpt)
        return last_checkpoint_step

    def _save_model(self, sess, step):
        for name in self.savers:
            saver = self.savers.get(name)
            file = self.savedir / self._make_ckpt_name(name, step)
            saver.save(sess, str(file))

    def _early_exit(self):
        # Todo not implemented
        return False

    def fit(self,
            batch=32,
            epochs=1,
            steps_per_epoch=200,
            dataset=None,
            learning_rate=1e-4,
            learning_rate_schedule=None,
            restart=False,
            validate_numbers=1,
            validate_every_n_epoch=1,
            augmentation=False,
            parallel=1,
            memory_usage=None,
            **kwargs):
        """Train the model.

        Args:
            batch: the size of mini-batch during training
            epochs: the total training epochs
            steps_per_epoch: training steps of each epoch
            dataset: the Dataset object, used to get training and validation frames
            learning_rate: the initial learning rate
            learning_rate_schedule: a callable to adjust learning rate. The signature is
                                    `fn(learning_rate, epochs, steps, loss)`
            restart: if True, start training from scratch, regardless of saved checkpoints
            validate_numbers: the number of patches in validation
            validate_every_n_epoch: run validation every n epochs
            augmentation: a boolean representing conduct image augmentation (random flip and rotate)
            parallel: a scalar representing threads number of loading dataset
            memory_usage: a string or integer, limiting maximum usage of CPU memory. (i.e. 1024MB, 8GB...)
        """

        sess = tf.get_default_session()
        if sess is None:
            raise RuntimeError('No session initialized')
        if not self.model.compiled:
            tf.logging.error('[Warning] model not compiled, compiling now...')
            self.model.compile()
        sess.run(tf.global_variables_initializer())
        init_epoch = 1 if restart else self._restore_model(sess) + 1
        if init_epoch > epochs:
            return
        print('===================================')
        print(f'Training model: {self.model.name.upper()}')
        print('===================================')
        self.model.display()
        summary_writer = tf.summary.FileWriter(str(self.logdir), graph=tf.get_default_graph())
        lr = learning_rate
        global_step = self.model.global_steps.eval()
        if learning_rate_schedule and callable(learning_rate_schedule):
            lr = learning_rate_schedule(lr, epochs=init_epoch, steps=global_step)
        if parallel == 1:
            train_loader = QuickLoader(batch, dataset, 'train', self.model.scale, steps_per_epoch,
                                       crop='random', augmentation=augmentation, **kwargs)
        else:
            train_loader = MpLoader(batch, dataset, 'train', self.model.scale, steps_per_epoch,
                                    crop='random', augmentation=augmentation, **kwargs)
        val_loader = QuickLoader(batch, dataset, 'val', self.model.scale, validate_numbers, crop='center', **kwargs)
        for epoch in range(init_epoch, epochs + 1):
            train_iter = train_loader.make_one_shot_iterator(memory_usage, shard=parallel, shuffle=True)
            date = time.strftime('%Y-%m-%d %T', time.localtime())
            print(f'| {date} | Epoch: {epoch}/{epochs} | LR: {lr} |')
            avg_meas = {}
            with tqdm.tqdm(train_iter, unit='batch', ascii=True) as r:
                for img in r:
                    feature, label, name = img[self.fi], img[self.li], img[-1]
                    for fn in self.feature_callbacks:
                        feature = fn(feature, name=name)
                    for fn in self.label_callbacks:
                        label = fn(label, name=name)
                    loss = self.model.train_batch(
                        feature=feature, label=label, learning_rate=lr, epochs=epoch)
                    global_step = self.model.global_steps.eval()
                    if learning_rate_schedule and callable(learning_rate_schedule):
                        lr = learning_rate_schedule(lr, epochs=epoch, steps=global_step)
                    for k, v in loss.items():
                        avg_meas[k] = avg_meas[k] + [v] if avg_meas.get(k) else [v]
                        loss[k] = '{:08.5f}'.format(v)
                    r.set_postfix(loss)
            for k, v in avg_meas.items():
                print(f'| Epoch average {k} = {np.mean(v):.6f} |')

            if epoch % validate_every_n_epoch: continue
            val_metrics = {}
            val_iter = val_loader.make_one_shot_iterator(memory_usage, shard=parallel, shuffle=False)
            for img in val_iter:
                feature, label, name = img[self.fi], img[self.li], img[-1]
                for fn in self.feature_callbacks:
                    feature = fn(feature, name=name)
                for fn in self.label_callbacks:
                    label = fn(label, name=name)
                metrics, val_summary_op, _ = self.model.validate_batch(
                    feature=feature, label=label, epochs=epoch)
                for k, v in metrics.items():
                    if k not in val_metrics:
                        val_metrics[k] = []
                    val_metrics[k] += [v]
                summary_writer.add_summary(val_summary_op, global_step)
            for k, v in val_metrics.items():
                print(f'{k}: {np.mean(v):.6f}', end=', ')
            print('')
            self._save_model(sess, epoch)
        # flush all pending summaries to disk
        summary_writer.close()

    def test(self, dataset, **kwargs):
        r"""Test model with test sets in dataset

        Args:
            dataset: instance of dataset, with dataset.test valid
        """

        sess = tf.get_default_session()
        ckpt_last = self._restore_model(sess)
        loader = QuickLoader(1, dataset, 'test', self.model.scale, -1, crop=None, **kwargs)
        it = loader.make_one_shot_iterator()
        if len(it):
            print('===================================')
            print(f'Testing model: {self.model.name} by {ckpt_last}')
            print('===================================')
        else:
            return
        for img in tqdm.tqdm(it, 'Test', ascii=True):
            feature, label, name = img[self.fi], img[self.li], img[-1]
            tf.logging.debug('output: ' + str(name))
            for fn in self.feature_callbacks:
                feature = fn(feature, name=name)
            for fn in self.label_callbacks:
                label = fn(label, name=name)
            outputs = self.model.test_batch(feature, None)
            for fn in self.output_callbacks:
                outputs = fn(outputs, input=img[self.fi], label=img[self.li], mode=loader.color_format,
                             name=name, subdir=dataset.name)

    def predict(self, files, mode='pil-image1', depth=1, **kwargs):
        r"""Predict output for frames

        Args:
            files: a list of frames as inputs
            mode: specify file format. `pil-image1` for PIL supported images, or `NV12/YV12/RGB` for raw data
            depth: specify length of sequence of images. 1 for images, >1 for videos
        """

        sess = tf.get_default_session()
        ckpt_last = self._restore_model(sess)
        files = [Path(file) for file in to_list(files)]
        data = Dataset(test=files, mode=mode, depth=depth, modcrop=False, **kwargs)
        loader = QuickLoader(1, data, 'test', self.model.scale, -1, crop=None, **kwargs)
        it = loader.make_one_shot_iterator()
        if len(it):
            print('===================================')
            print(f'Predicting model: {self.model.name} by {ckpt_last}')
            print('===================================')
        else:
            return
        for img in tqdm.tqdm(it, 'Infer', ascii=True):
            feature, label, name = img[self.fi], img[self.li], img[-1]
            tf.logging.debug('output: ' + str(name))
            for fn in self.feature_callbacks:
                feature = fn(feature, name=name)
            outputs = self.model.test_batch(feature, None)
            for fn in self.output_callbacks:
                outputs = fn(outputs, input=img[self.fi], label=img[self.li], mode=loader.color_format, name=name)

    def export(self, export_dir='.'):
        """Export model as protobuf

        Args:
            export_dir: directory to save the exported model
        """

        sess = tf.get_default_session()
        self._restore_model(sess)
        self.model.export_saved_model(export_dir)

max_stars_repo_name: rzumer/VideoLowLevelVision

max_issues_repo_name: rzumer/VideoLowLevelVision

max_forks_repo_name: rzumer/VideoLowLevelVision


text:
"""Serialize a pyvista plotter to a pythreejs scene."""

import warnings

import numpy as np

try:
    import pythreejs as tjs
except ImportError:  # pragma: no cover
    raise ImportError('Please install pythreejs to use this feature')

from ipywidgets import GridspecLayout

import pyvista as pv

# TODO: Consider adding sprites for point labels
# def add_sprite():
#     text = tjs.TextTexture('hi', size=60, color='black')
#     sprite = tjs.Sprite(tjs.SpriteMaterial(map=text,
#                                            sizeAttenuation=False,
#                                            transparent=False,
#                                            # depthWrite=False,
#                                            # depthTest=False
#                                            useScreenCoordinates=True,
#                                            ),
#                         scale=(.1,.1,.1),
#                         center=(0.0, 0.0)
#                         )
#     children.append(sprite)

def segment_poly_cells(mesh):
    """Segment lines from a mesh into line segments."""
    if not pv.is_pyvista_dataset(mesh):  # pragma: no cover
        mesh = pv.wrap(mesh)
    polylines = []
    i, offset = 0, 0
    cc = mesh.lines  # fetch up front
    while i < mesh.n_cells:
        nn = cc[offset]
        polylines.append(cc[offset + 1 : offset + 1 + nn])
        offset += nn + 1
        i += 1

    lines = []
    for poly in polylines:
        lines.append(np.column_stack((poly[:-1], poly[1:])))
    return np.vstack(lines)

def buffer_normals(trimesh):
    """Extract surface normals and return a buffer attribute."""
    if 'Normals' in trimesh.point_data:
        normals = trimesh.point_data['Normals']
    else:
        normals = trimesh.point_normals
    normals = normals.astype(np.float32, copy=False)
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", message="Given trait value dtype")
        return tjs.BufferAttribute(array=normals)

def get_coloring(mapper, dataset):
    """Return the three.js coloring type for a given actor."""
    coloring = 'NoColors'
    if mapper.GetScalarModeAsString() == 'UsePointData':
        scalars = dataset.point_data.active_scalars
        if scalars is not None:
            coloring = 'VertexColors'
    elif mapper.GetScalarModeAsString() == 'UseCellData':
        scalars = dataset.cell_data.active_scalars
        if scalars is not None:
            coloring = 'FaceColors'
    return coloring

def extract_surface_mesh(obj):
    """Extract a surface mesh from a pyvista or vtk dataset.

    Parameters
    ----------
    obj : pyvista compatible object
        Any object compatible with pyvista.  Includes most ``vtk``
        objects.

    Returns
    -------
    pyvista.PolyData
        Surface mesh

    """
    # attempt to wrap non-pyvista objects
    if not pv.is_pyvista_dataset(obj):  # pragma: no cover
        mesh = pv.wrap(obj)
        if not pv.is_pyvista_dataset(mesh):
            raise TypeError(f'Object type ({type(mesh)}) cannot be converted to a pyvista dataset')
    else:
        mesh = obj

    if not isinstance(obj, pv.PolyData):
        # unlikely case that mesh does not have extract_surface
        if not hasattr(mesh, 'extract_surface'):  # pragma: no cover
            mesh = mesh.cast_to_unstructured_grid()
        return mesh.extract_surface()

    return mesh

def map_scalars(mapper, scalars):
    """Map scalars to a RGB array.

    Parameters
    ----------
    mapper : vtk.vtkMapper
        Mapper containing lookup table.
    scalars : vtk array, numpy.ndarray, or pyvista.pyvista_ndarray
        Scalars to map

    Returns
    -------
    pyvista.pyvista_ndarray
        Array of mapped scalars.

    """
    if isinstance(scalars, np.ndarray):
        if hasattr(scalars, 'VTKObject') and scalars.VTKObject is not None:
            scalars = scalars.VTKObject
        else:
            scalars = pv._vtk.numpy_to_vtk(scalars)

    table = mapper.GetLookupTable()
    return pv.wrap(table.MapScalars(scalars, 0, 0))[:, :3] / 255

def array_to_float_buffer(points):
    """Convert a numpy array to a pythreejs compatible point buffer."""
    # create buffered points
    points = points.astype(np.float32, copy=False)

    # ignore invalid warning.  BufferAttribute type is None and is
    # improperly recognized as float64.  Same for the rest
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", message="Given trait value dtype")
        position = tjs.BufferAttribute(array=points, normalized=False)
    return position

def cast_to_min_size(ind, max_index):
    """Return a buffered attribute of the minimum index size."""
    ind = ind.ravel()
    if max_index < np.iinfo(np.uint16).max:
        ind = ind.astype(np.uint16, copy=False)
    elif max_index < np.iinfo(np.uint32).max:
        ind = ind.astype(np.uint32, copy=False)
    else:
        raise ValueError(
            f'pythreejs does not support a maximum index more than {np.iinfo(np.uint32).max}'
        )
    return tjs.BufferAttribute(array=ind, normalized=False)

def to_surf_mesh(actor, surf, mapper, prop, add_attr=None):
    """Convert a pyvista surface to a buffer geometry.

    General Notes
    -------------

    * THREE.BufferGeometry expects position and index attributes
      representing a triangulated mesh points and face indices or just
      a position array representing individual faces of a mesh.
    * The normals attribute is needed for physically based rendering,
      but not for the other mesh types.
    * Colors must be a RGB array with one value per point.

    Shading Notes
    -------------
    To match VTK, the following materials are used to match VTK's shading:

    * MeshPhysicalMaterial when physically based rendering is enabled
    * MeshPhongMaterial when physically based rendering is disabled,
      but lighting is enabled.
    * MeshBasicMaterial when lighting is disabled.

    """
    if add_attr is None:
        add_attr = {}
    # convert to an all-triangular surface
    if surf.is_all_triangles():
        trimesh = surf
    else:
        trimesh = surf.triangulate()

    position = array_to_float_buffer(trimesh.points)

    # convert to minimum index type
    face_ind = trimesh.faces.reshape(-1, 4)[:, 1:]
    index = cast_to_min_size(face_ind, trimesh.n_points)
    attr = {
        'position': position,
        'index': index,
    }

    if prop.GetInterpolation():  # something other than flat shading
        attr['normal'] = buffer_normals(trimesh)

    # extract point/cell scalars for coloring
    colors = None
    scalar_mode = mapper.GetScalarModeAsString()
    if scalar_mode == 'UsePointData':
        colors = map_scalars(mapper, trimesh.point_data.active_scalars)
    elif scalar_mode == 'UseCellData':
        # special handling for RGBA
        if mapper.GetColorMode() == 2:
            scalars = trimesh.cell_data.active_scalars.repeat(3, axis=0)
            scalars = scalars.astype(np.float32, copy=False)
            colors = scalars[:, :3] / 255  # ignore alpha
        else:
            # must repeat for each triangle
            scalars = trimesh.cell_data.active_scalars.repeat(3)
            colors = map_scalars(mapper, scalars)

        position = array_to_float_buffer(trimesh.points[face_ind])
        attr = {'position': position}

    # add colors to the buffer geometry attributes
    if colors is not None:
        attr['color'] = array_to_float_buffer(colors)

    # texture coordinates
    t_coords = trimesh.active_t_coords
    if t_coords is not None:
        attr['uv'] = array_to_float_buffer(t_coords)

    # TODO: Convert PBR textures
    # base_color_texture = prop.GetTexture("albedoTex")
    # orm_texture = prop.GetTexture("materialTex")
    # anisotropy_texture = prop.GetTexture("anisotropyTex")
    # normal_texture = prop.GetTexture("normalTex")
    # emissive_texture = prop.GetTexture("emissiveTex")
    # coatnormal_texture = prop.GetTexture("coatNormalTex")
    if prop.GetNumberOfTextures():  # pragma: no cover
        warnings.warn('pythreejs converter does not support PBR textures (yet).')

    # create base buffer geometry
    surf_geo = tjs.BufferGeometry(attributes=attr)

    # add texture to the surface buffer if available
    texture = actor.GetTexture()
    tjs_texture = None
    if texture is not None:
        wrapped_tex = pv.wrap(texture.GetInput())
        data = wrapped_tex.active_scalars
        dim = (wrapped_tex.dimensions[0], wrapped_tex.dimensions[1], data.shape[1])
        data = data.reshape(dim)
        # fmt = "RGBFormat" if data.shape[1] == 3 else "RGBAFormat"

        # Create data texture and catch invalid warning
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", message="Given trait value dtype")
            tjs_texture = tjs.DataTexture(data=data, format="RGBFormat", type="UnsignedByteType")

    # these attributes are always used regardless of the material
    shared_attr = {
        'vertexColors': get_coloring(mapper, trimesh),
        'wireframe': prop.GetRepresentation() == 1,
        'opacity': prop.GetOpacity(),
        'wireframeLinewidth': prop.GetLineWidth(),
        # 'side': 'DoubleSide'  # enabling seems to mess with textures
    }

    if colors is None:
        shared_attr['color'] = color_to_hex(prop.GetColor())

    if tjs_texture is not None:
        shared_attr['map'] = tjs_texture
    else:
        shared_attr['side'] = 'DoubleSide'

    if prop.GetOpacity() < 1.0:
        shared_attr['transparent'] = True

    if prop.GetInterpolation() == 3:  # using physically based rendering
        material = tjs.MeshPhysicalMaterial(
            flatShading=False,
            roughness=prop.GetRoughness(),
            metalness=prop.GetMetallic(),
            reflectivity=0,
            **shared_attr,
            **add_attr,
        )
    elif prop.GetLighting():
        # specular disabled to fix lighting issues
        material = tjs.MeshPhongMaterial(
            shininess=0,
            flatShading=prop.GetInterpolation() == 0,
            specular=color_to_hex((0, 0, 0)),
            reflectivity=0,
            **shared_attr,
            **add_attr,
        )
    else:  # no lighting
        material = tjs.MeshBasicMaterial(**shared_attr, **add_attr)

    return tjs.Mesh(geometry=surf_geo, material=material)

def to_edge_mesh(surf, mapper, prop, use_edge_coloring=True, use_lines=False):
    """Convert a pyvista surface to a three.js edge mesh."""
    # extract all edges from the surface.  Should not use triangular
    # mesh here as mesh may contain more than triangular faces
    if use_lines:
        edges_mesh = surf
        edges = segment_poly_cells(surf)
    else:
        edges_mesh = surf.extract_all_edges()
        edges = edges_mesh.lines.reshape(-1, 3)[:, 1:]

    attr = {
        'position': array_to_float_buffer(edges_mesh.points),
        'index': cast_to_min_size(edges, surf.n_points),
    }

    # add in colors
    coloring = get_coloring(mapper, surf)
    if coloring != 'NoColors' and not use_edge_coloring:
        if mapper.GetScalarModeAsString() == 'UsePointData':
            edge_scalars = edges_mesh.point_data.active_scalars
            edge_colors = map_scalars(mapper, edge_scalars)
            attr['color'] = array_to_float_buffer(edge_colors)

    edge_geo = tjs.BufferGeometry(attributes=attr)

    mesh_attr = {}
    if coloring != 'NoColors':
        mesh_attr['vertexColors'] = coloring

    if use_edge_coloring:
        edge_color = prop.GetEdgeColor()
    else:
        edge_color = prop.GetColor()

    edge_mat = tjs.LineBasicMaterial(
        color=color_to_hex(edge_color),
        linewidth=prop.GetLineWidth(),
        opacity=prop.GetOpacity(),
        side='FrontSide',
        **mesh_attr,
    )
    return tjs.LineSegments(edge_geo, edge_mat)

def to_tjs_points(dataset, mapper, prop):
    """Extract the points from a dataset and return a buffered geometry."""
    attr = {
        'position': array_to_float_buffer(dataset.points),
    }

    coloring = get_coloring(mapper, dataset)
    if coloring == 'VertexColors':
        colors = map_scalars(mapper, dataset.point_data.active_scalars)
        attr['color'] = array_to_float_buffer(colors)

    geo = tjs.BufferGeometry(attributes=attr)

    m_attr = {
        'color': color_to_hex(prop.GetColor()),
        'size': prop.GetPointSize() / 100,
        'vertexColors': coloring,
    }

    point_mat = tjs.PointsMaterial(**m_attr)
    return tjs.Points(geo, point_mat)

def pvcamera_to_threejs_camera(pv_camera, lights, aspect):
    """Return an ipygany camera dict from a ``pyvista.Plotter`` object."""
    # scene will be centered at focal_point, so adjust the position
    position = np.array(pv_camera.position) - np.array(pv_camera.focal_point)
    far = np.linalg.norm(position) * 100000

    return tjs.PerspectiveCamera(
        up=pv_camera.up,
        children=lights,
        position=position.tolist(),
        fov=pv_camera.view_angle,
        aspect=aspect,
        far=far,
        near=0.01,
    )

def color_to_hex(color, linear_to_srgb=False):
    """Convert a 0 - 1 RGB color tuple to a HTML hex color.

    Optionally convert linear colors to sRGB.

    """
    color = np.array(color)
    if linear_to_srgb:
        mask = color < 0.0031308
        color[mask] *= 12.92
        color[~mask] = 1.055 * color[~mask] ** (1 / 2.4) - 0.055

    color = tuple((color * 255).astype(np.uint8))
    return '#%02x%02x%02x' % tuple(color)

def pvlight_to_threejs_light(pvlight):
    """Convert a pyvista headlight into a three.js directional light."""
    if pvlight.is_camera_light or pvlight.is_headlight:
        # extend the position of the light to make "near infinite"
        position = np.array(pvlight.position) * 100000
        return tjs.DirectionalLight(
            color=color_to_hex(pvlight.diffuse_color, True),
            position=position.tolist(),
            intensity=pvlight.intensity,
        )

def extract_lights_from_renderer(renderer):
    """Extract and convert all pyvista lights to pythreejs compatible lights."""
    return [pvlight_to_threejs_light(pvlight) for pvlight in renderer.lights]

def actor_to_mesh(actor, focal_point):
    """Convert a VTK actor to a threejs mesh or meshes."""
    mapper = actor.GetMapper()
    if mapper is None:
        return

    dataset = mapper.GetInputAsDataSet()
    has_faces = True
    if hasattr(dataset, 'faces'):
        has_faces = np.any(dataset.faces)

    prop = actor.GetProperty()
    rep_type = prop.GetRepresentationAsString()

    meshes = []
    if rep_type == 'Surface' and has_faces:
        surf = extract_surface_mesh(dataset)
        add_attr = {}
        if prop.GetEdgeVisibility():
            # must offset polygons to have mesh render property with lines
            add_attr = {'polygonOffset': True, 'polygonOffsetFactor': 1, 'polygonOffsetUnits': 1}

            meshes.append(to_edge_mesh(surf, mapper, prop, use_edge_coloring=True))

        meshes.append(to_surf_mesh(actor, surf, mapper, prop, add_attr))

    elif rep_type == 'Points':
        meshes.append(to_tjs_points(dataset, mapper, prop))
    else:  # wireframe
        if has_faces:
            surf = extract_surface_mesh(dataset)
            mesh = to_edge_mesh(surf, mapper, prop, use_edge_coloring=False)
            meshes.append(mesh)
        elif np.any(dataset.lines):
            mesh = to_edge_mesh(dataset, mapper, prop, use_edge_coloring=False, use_lines=True)
            meshes.append(mesh)
        else:  # pragma: no cover
            warnings.warn('Empty or unsupported dataset attached to actor')

    # the camera in three.js has no concept of a "focal point".  In
    # three.js, the scene is always centered at the origin, which
    # serves as the focal point of the camera.  Therefore, we need to
    # shift the entire scene by the focal point of the pyvista camera
    for mesh in meshes:
        mesh.position = -focal_point[0], -focal_point[1], -focal_point[2]

    return meshes

def meshes_from_actors(actors, focal_point):
    """Convert a pyvista plotter to a scene."""
    meshes = []
    for actor in actors:
        mesh = actor_to_mesh(actor, focal_point)
        if mesh is not None:
            meshes.extend(mesh)

    return meshes

def convert_renderer(pv_renderer):
    """Convert a pyvista renderer to a pythreejs renderer."""
    # verify plotter hasn't been closed

    width, height = pv_renderer.width, pv_renderer.height
    pv_camera = pv_renderer.camera
    children = meshes_from_actors(pv_renderer.actors.values(), pv_camera.focal_point)

    lights = extract_lights_from_renderer(pv_renderer)
    aspect = width / height
    camera = pvcamera_to_threejs_camera(pv_camera, lights, aspect)

    children.append(camera)

    if pv_renderer.axes_enabled:
        children.append(tjs.AxesHelper(0.1))

    scene = tjs.Scene(children=children, background=color_to_hex(pv_renderer.background_color))

    # replace inf with a real value here due to changes in
    # ipywidges==6.4.0 see
    # https://github.com/ipython/ipykernel/issues/771
    inf = 1e20
    orbit_controls = tjs.OrbitControls(
        controlling=camera,
        maxAzimuthAngle=inf,
        maxDistance=inf,
        maxZoom=inf,
        minAzimuthAngle=-inf,
    )

    renderer = tjs.Renderer(
        camera=camera,
        scene=scene,
        alpha=True,
        clearOpacity=0,
        controls=[orbit_controls],
        width=width,
        height=height,
        antialias=pv_renderer.GetUseFXAA(),
    )

    if pv_renderer.has_border:
        bdr_color = color_to_hex(pv_renderer.border_color)
        renderer.layout.border = f'solid {pv_renderer.border_width}px {bdr_color}'

    # for now, we can't dynamically size the render windows.  If
    # unset, the renderer widget will attempt to resize and the
    # threejs renderer will not resize.
    # renderer.layout.width = f'{width}px'
    # renderer.layout.height = f'{height}px'
    return renderer

def convert_plotter(pl):
    """Convert a pyvista plotter to a pythreejs widget."""
    if not hasattr(pl, 'ren_win'):
        raise AttributeError(
            'This plotter is closed and unable to export to html.\n'
            'Please run this before showing or closing the plotter.'
        )

    if len(pl.renderers) == 1:
        # return HBox(children=(convert_renderer(pl.renderers[0]),))
        return convert_renderer(pl.renderers[0])

    # otherwise, determine if we can use a grid layout
    if len(pl.shape) == 2:
        n_row = pl.shape[0]
        n_col = pl.shape[1]
        grid = GridspecLayout(int(n_row), int(n_col))
        width, height = 0, 0
        for i in range(n_row):
            for j in range(n_col):
                pv_ren = pl.renderers[j + n_row * i]
                if j == 0:
                    height += pv_ren.height + pv_ren.border_width * 2
                if i == 0:
                    width += pv_ren.width + pv_ren.border_width * 2
                grid[i, j] = convert_renderer(pv_ren)

        # this is important to ignore when building the gallery
        if not pv.BUILDING_GALLERY:
            grid.layout.width = f'{width}px'
            grid.layout.height = f'{height+4}px'

        return grid

    raise RuntimeError(
        'Unsupported plotter shape.  The ``pythreejs`` backend only '
        'supports single or regular grids of plots'
    )


text:
import unittest

import numpy as np

from nfoursid.nfoursid import NFourSID
from nfoursid.state_space import StateSpace

class TestNFourSid(unittest.TestCase):
    def test_which_is_actually_regression_test(self):
        n_datapoints = 100
        model = StateSpace(
            np.array([[.5]]),
            np.array([[.6]]),
            np.array([[.7]]),
            np.array([[.8]]),
        )
        np.random.seed(0)
        for _ in range(n_datapoints):
            model.step(np.random.standard_normal((1, 1)))

        nfoursid = NFourSID(
            model.to_dataframe(),
            model.y_column_names,
            input_columns=model.u_column_names,
            num_block_rows=2
        )
        nfoursid.subspace_identification()
        identified_model, covariance_matrix = nfoursid.system_identification(rank=1)

        # matrices `a` and `d` don't have freedom of choice: they should be fitted well
        self.assertTrue(is_slightly_close(.5, identified_model.a))
        self.assertTrue(is_slightly_close(.8, identified_model.d))
        self.assertTrue(np.all(is_slightly_close(0, covariance_matrix)))

def is_slightly_close(matrix, number):
    return np.isclose(matrix, number, rtol=0, atol=1e-3)

max_stars_repo_name: spmvg/nfoursid

max_issues_repo_name: spmvg/nfoursid

max_forks_repo_name: spmvg/nfoursid


text:
import sys
import os
import glob
import csv
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

def write_test_to_file(file_output, data):
    rewards_labels = ["episode", "rewards"]
    with open(file_output, 'a+') as csvfile:
        writer = csv.writer(csvfile)
        if sum(1 for line in open(file_output)) == 0:
            writer.writerow(rewards_labels)
        for row in data:
            writer.writerow(row)

def test(episode_checkpoint, task, agent, prefix):
    state = agent.reset_episode() # start a new episode
    while True:
        action = agent.act(state, noise=False)
        next_state, splitted_rewards, done = task.step(action)
        net_reward = splitted_rewards[-1]
        agent.step(action, net_reward, next_state, done, episode_checkpoint, learn=False)
        state = next_state
        if done:
            write_test_to_file(f'data/{prefix}_test.txt', [[int(episode_checkpoint), round(task.score, 2)]])
            break

def plot_tests(prefix):
    results = pd.read_csv(f'data/{prefix}_test.txt')
    plt.title('Test scores (each 25 episodes without noise)')
    plt.plot(results['episode'], results['rewards'])
    plt.legend()
    plt.show()

max_stars_repo_name: MichelML/udacity-quadopter

max_issues_repo_name: MichelML/udacity-quadopter

max_forks_repo_name: MichelML/udacity-quadopter