text:
'''
File name: emulator.py
    AI environment emulator class implementation (Implements an AI emulator based
    on Open AI Gym [1])

Author: Vasileios Saveris
enail: vsaveris@gmail.com

License: MIT

Date last modified: 01.12.2019

References:
    [1] arXiv:1606.01540 [cs.LG]

Python Version: 3.6
'''

from collections import deque
import numpy as np
import pandas as pd
import gym

'''
Constants
'''
C_VERBOSE_NONE   = 0  # Printing is disabled
C_VERBOSE_INFO   = 1  # Only information printouts (constructor)
C_VERBOSE_DEBUG  = 2  # Debugging printing level (all printouts)

class Emulator():
    '''
    Summary:
        Emulator class implementation using OpenAI gym API.

    Private Attributes:
        __verbose: int
            Verbose level (0: None, 1: INFO, 2: DEBUG

        __rendering: boolean
            If True, rendering of the OpenAI gym environment is enabled.

        __statistics: boolean
            If True, printing and saving of statistics for each episode are both enabled.

        __current_state: object
            The current state of the OpenAI gym environment.

        __average_reward_episodes: int
            The number of last concecutive episodes in which the average reward should be
            calculated.

        __environment: OpenAI gym environment object
            The environment returned by the make() OpenAI gym function.

        __last_x_rewards: deque
            The last X concecutive rewards, where X = __average_reward_episodes.

    Public Attributes:
        emulator_started: boolean
            Indicates the state of the Emulator (True means an episode is running)

        average_reward: float
            The average of last X concecutive episode rewards.

        episode_number: int
            The sequential number of the episode.

        episode_total_reward: float
            The total reward of the current episode.

        episode_total_steps: int
            The total steps of the current episode.

        state_size: int
            The dimensions of the state object for the given environment.

        actions_number: int
            The number of different actions supported by the given environment.

        execution_statistics: pandas.DataFrame
            Emulator statistics ('episode', 'steps', 'total_reward', 'last_X_average_reward')

    Private Methods:
        __init__(): returns None
            Class constructor.

    Public Methods:
        start(): returns environment's initial state.
            Resets the environment and initializes all the attributes for the current episode.
            Should be used as the first call when we want to start a new episode.

        applyAction(action: int or None): returns an experience list [s, a, r, s']
            Apply an action to the environment. If action is none, a random action is selected.

    '''

    def __init__(self, scenario, average_reward_episodes = 1, statistics = False, rendering = False,
                 seed = None, verbose = C_VERBOSE_NONE):
        '''
        summary:
            Class constructor. Creates the OpenAI gym environment and initializes
            all the attributes.

        Args:
            scenario: string
                The OpenAI gym scenario to be emulated.

            average_reward_episodes: int
                On how many concecutive episodes the averaged reward should be calculated.

            statistics: boolean
                If True, execution statistics are printed and stored in a class attribute. Verbose can
                be C_VERBOSE_NONE.

            rendering: boolean
                If True, then rendering is enabled and a preview of the environment
                is shown for each state change.

            seed: int
                Optional Seed to be used with the OpenAI gym environment, for results reproducability.

            verbose: int
                Verbose level (0: None, 1: INFO, 2: DEBUG, see CONSTANTS section).

        Raises:
            -

        Returns:
            -

        notes:
            -

        '''

        self.__verbose = verbose

        if self.__verbose > C_VERBOSE_NONE:
            print('\nRL Emulator initialization (scenario = ', scenario, ', average_reward_episodes = ', average_reward_episodes,
                  ', statistics = ', statistics, ', rendering = ', rendering, ', seed = ', seed, ')', sep = '')

        self.__rendering = rendering
        self.__statistics = statistics
        self.__current_state = None
        self.__average_reward_episodes = average_reward_episodes

        #Create Open AI gym environment
        try:
            self.__environment = gym.make(scenario)
        except:
            print('ERROR: class Emulator, \'', scenario, '\' is not a valid Open AI Gym scenario, script exits.', sep = '')
            exit(-1)

        #Seed the environment
        if seed is not None:
            self.__environment.seed(seed)

        #Keep track of the last x consecutive total rewards
        self.__last_x_rewards = deque(maxlen = self.__average_reward_episodes)

        #Public Attributes
        self.emulator_started = False  #If False the Emulator not started, or episode finished
        self.average_reward = 0
        self.episode_number = 0
        self.episode_total_reward = 0
        self.episode_total_steps  = 0

        #Keeps statistics for all the episodes
        if self.__statistics:
            self.execution_statistics = pd.DataFrame(data = None, index = None, columns = ['episode', 'steps', 'total_reward',
                'last_X_average_reward'], dtype = None, copy = False)

        #Get the observation space size based on its type
        if isinstance(self.__environment.observation_space, gym.spaces.box.Box):
            self.state_size = 1
            for i in range(len(self.__environment.observation_space.shape)):
                self.state_size *= self.__environment.observation_space.shape[i]

        elif isinstance(self.__environment.observation_space, gym.spaces.discrete.Discrete):
            self.state_size = self.__environment.observation_space.n

        else:
            print('ERROR: class RLEmulator, \'', type(self.__environment.observation_space), '\' Observation Space type is not supported, script exits.')
            exit(-1)

        #Get the action space size based on its type
        if isinstance(self.__environment.action_space, gym.spaces.box.Box):
            self.actions_number = 1
            for i in range(len(self.__environment.action_space.shape)):
                self.actions_number *= self.__environment.action_space.shape[i]

        elif isinstance(self.__environment.action_space, gym.spaces.discrete.Discrete):
            self.actions_number = self.__environment.action_space.n

        else:
            print('ERROR: class RLEmulator, \'', type(self.__environment.observation_space), '\' Actions Space type is not supported, script exits.')
            exit(-1)

        if self.__verbose > C_VERBOSE_NONE:
            print('- RL Emulator created (observations_size = ', self.state_size, ', actions_size = ', self.actions_number, ')', sep = '')

    def start(self):
        '''
        summary:
            Starts or restarts the Emulator (starts an episode).

        Args:
            -

        Raises:
            -

        Returns:
            current_state: Environment's state object
                The current (initial) state of the Open AI gym environment. Returned
                after the OpenAI gym reset() function called.

        notes:
            -

        '''

        #Initialze attributes at the begining of the episode.
        self.emulator_started = True
        self.episode_number += 1
        self.episode_total_reward = 0
        self.episode_total_steps  = 0

        #Reset the environment
        self.__current_state = self.__environment.reset()

        if self.__rendering:
            self.__environment.render()

        if self.__verbose > C_VERBOSE_INFO:
            print('RL Emulator Started')

        return self.__current_state

    def applyAction(self, action = None):
        '''
        summary:
            Applies a specific or random action to the environment.

        Args:
            action: integer or None
                Action to be applied in the envoronment. If None, then a random action
                should be used.

        Raises:
            -

        Returns:
            experience_tuple: list
                An experience tuple for this step in the form of [s, a, r, s']

        notes:
            -

        '''

        if not self.emulator_started:
            print('ERROR: Emulator is not started yet, script exits.')
            exit(-1)

        #If no action given, select a random one
        if action is None:
            action = self.__environment.action_space.sample()

        new_state, reward, done, info = self.__environment.step(action)

        self.episode_total_reward += reward
        self.episode_total_steps  += 1

        if self.__rendering:
            self.__environment.render()

        if self.__verbose > C_VERBOSE_INFO:
            print('RL Emulator Apply Action (action = ', action, ', reward = ', reward,
                  ', episode_is_done = ', done, ')', sep = '')

        current_state = self.__current_state #Keep the current state for the return statememt

        #Episode finished
        if done:
            new_state = None

            #Update Emulator status, the caller has to restart the Emulator for another episode (start())
            self.emulator_started = False

            #Calculate average reward
            self.__last_x_rewards.append(self.episode_total_reward)
            self.average_reward = np.mean(self.__last_x_rewards)

            if self.__statistics:
                print('Statistics: episode = %05d' % self.episode_number, ', steps = %4d' % self.episode_total_steps,
                      ', total_reward = %9.3f' % round(self.episode_total_reward, 3), ', ', self.__average_reward_episodes,
                      '_episodes_average_reward = %9.3f' % round(self.average_reward, 3), sep = '')

                #Store the statistics for this episode
                self.execution_statistics.loc[len(self.execution_statistics.index)] = ([self.episode_number, self.episode_total_steps,
                    round(self.episode_total_reward, 3), round(self.average_reward, 3)])

        #Episode is not done yet
        else:
            self.__current_state = new_state     #Update the current state for the next step

        #Return an experience tuple (list)
        return [current_state, action, reward, new_state]

max_stars_repo_name: fy-meng/lunarlander-saliency

max_issues_repo_name: fy-meng/lunarlander-saliency

max_forks_repo_name: fy-meng/lunarlander-saliency


text:
from cupy import _core

j0 = _core.create_ufunc(
    'cupyx_scipy_special_j0', ('f->f', 'd->d'),
    'out0 = j0(in0)',
    doc='''Bessel function of the first kind of order 0.

    .. seealso:: :meth:`scipy.special.j0`

    ''')

j1 = _core.create_ufunc(
    'cupyx_scipy_special_j1', ('f->f', 'd->d'),
    'out0 = j1(in0)',
    doc='''Bessel function of the first kind of order 1.

    .. seealso:: :meth:`scipy.special.j1`

    ''')

y0 = _core.create_ufunc(
    'cupyx_scipy_special_y0', ('f->f', 'd->d'),
    'out0 = y0(in0)',
    doc='''Bessel function of the second kind of order 0.

    .. seealso:: :meth:`scipy.special.y0`

    ''')

y1 = _core.create_ufunc(
    'cupyx_scipy_special_y1', ('f->f', 'd->d'),
    'out0 = y1(in0)',
    doc='''Bessel function of the second kind of order 1.

    .. seealso:: :meth:`scipy.special.y1`

    ''')

# Note: oddly, unlike for y0 or y1, SciPy always returns double for yn
# dd->d because SciPy will accept 2.0
yn = _core.create_ufunc(
    'cupyx_scipy_special_yn', ('id->d', 'dd->d'),
    'out0 = yn((int)in0, in1)',
    doc='''Bessel function of the second kind of order n.

    Args:
        n (cupy.ndarray): order (integer)
        x (cupy.ndarray): argument (float)

    Returns:
        cupy.ndarray: The result.

    Notes
    -----
    Unlike SciPy, no warning will be raised on unsafe casting of `order` to
    32-bit integer.

    .. seealso:: :meth:`scipy.special.yn`
    ''')

i0 = _core.create_ufunc(
    'cupyx_scipy_special_i0', ('f->f', 'd->d'),
    'out0 = cyl_bessel_i0(in0)',
    doc='''Modified Bessel function of order 0.

    .. seealso:: :meth:`scipy.special.i0`

    ''')

i0e = _core.create_ufunc(
    'cupyx_scipy_special_i0e', ('f->f', 'd->d'),
    'out0 = exp(-abs(in0)) * cyl_bessel_i0(in0)',
    doc='''Exponentially scaled modified Bessel function of order 0.

    .. seealso:: :meth:`scipy.special.i0e`

    ''')

i1 = _core.create_ufunc(
    'cupyx_scipy_special_i1', ('f->f', 'd->d'),
    'out0 = cyl_bessel_i1(in0)',
    doc='''Modified Bessel function of order 1.

    .. seealso:: :meth:`scipy.special.i1`

    ''')

i1e = _core.create_ufunc(
    'cupyx_scipy_special_i1e', ('f->f', 'd->d'),
    'out0 = exp(-abs(in0)) * cyl_bessel_i1(in0)',
    doc='''Exponentially scaled modified Bessel function of order 1.

    .. seealso:: :meth:`scipy.special.i1e`

    ''')

ext: py

reason: from cupy


text:
from numpy import full, nan

from .check_nd_array_for_bad import check_nd_array_for_bad

def clip_nd_array_by_standard_deviation(
    nd_array, standard_deviation, raise_for_bad=True
):

    is_good = ~check_nd_array_for_bad(nd_array, raise_for_bad=raise_for_bad)

    nd_array_clipped = full(nd_array.shape, nan)

    if is_good.any():

        nd_array_good = nd_array[is_good]

        nd_array_good_mean = nd_array_good.mean()

        nd_array_good_interval = nd_array_good.std() * standard_deviation

        nd_array_clipped[is_good] = nd_array[is_good].clip(
            min=nd_array_good_mean - nd_array_good_interval,
            max=nd_array_good_mean + nd_array_good_interval,
        )

    return nd_array_clipped

max_stars_repo_name: alex-wenzel/ccal

max_issues_repo_name: alex-wenzel/ccal

max_forks_repo_name: alex-wenzel/ccal

reason: from numpy


text:
import requests
import cv2
import numpy as np

url = "http://192.168.43.1:8080/shot.jpg"

while True:

	img_resp = requests.get(url)
	img_arr = np.array(bytearray(img_resp.content), dtype=np.uint8)
	img = cv2.imdecode(img_arr, -1)

	cv2.imshow("AndroidCam", img)

	if cv2.waitKey(1) == 27:
		break

max_stars_repo_name: jjhenkel/dockerizeme

max_issues_repo_name: jjhenkel/dockerizeme

max_forks_repo_name: jjhenkel/dockerizeme

reason: import numpy


text:
import pandas as pd
import numpy as np
import scipy as sp

from scipy.optimize import linprog, minimize, NonlinearConstraint
from .pdf_quantile_functions import pdf_quantile_builder
from .support import diffMatMetalog, pdfMetalog, quantileMetalog, newtons_method_metalog

import time
import warnings

def a_vector_OLS_and_LP(m_dict,
           bounds,
           boundedness,
           term_limit,
           term_lower_bound,
           fit_method,
           alpha,
           diff_error = .001,
           diff_step = 0.001):

    """ Main workhorse function of pymetalog package.
        Called during metalog.__init__ method call.

    Args:
        m_dict (:obj:`dict` with keys ['params', 'dataValues', 'Y']): Initialized output_dict variable from metalog class.
            - m_dict['params']: (:obj:`dict` with keys ['bounds', 'boundedness', 'term_limit', 'term_lower_bound', 'step_len', 'fit_method']):
                * 'bounds': metalog.bounds
                * 'boundedness': metalog.boundedness
                * 'term_limit': metalog.term_limit
                * 'term_lower_bound': metalog.term_lower_bound
                * 'step_len': metalog.step_len
                * 'fit_method': metalog.fit_method

            - m_dict['dataValues']: (:obj:`pandas.DataFrame` with columns ['x','probs','z']  of type numeric):
                * 'x': metalog.x
                * 'probs': metalog.probs
                * 'z': column calculated in metalog.append_zvector method
                    - depends on metalog.boundedness attribute
                    - metalog.boundedness = 'u':
                        * 'z' = metalog.x
                    - metalog.boundedness = 'sl':
                        * 'z' = log( (metalog.x-lower_bound) )
                    - metalog.boundedness = 'su':
                        * 'z' = = log( (upper_bound-metalog.x) )
                    - metalog.boundedness = 'b':
                        * 'z' = log( (metalog.x-lower_bound) / (upper_bound-metalog.x) )

            - m_dict['Y']: (:obj:`pandas.DataFrame` with columns ['y1','y2','y3','y4', ... ,'yn']  of type numeric):
                * 'y1': numpy.array of ones with length equal to len(x)
                * 'y2': numpy.array of numeric values equal to the term attached to s in the logistic quantile function np.log(m_dict['dataValues']['probs'] / (1 - m_dict['dataValues']['probs']))
                * 'y3': numpy.array of numeric values (m_dict['dataValues']['probs'] - 0.5) * m_dict['Y']['y2']
                * 'y4': numpy.array of numeric values m_dict['Y']['y4'] = m_dict['dataValues']['probs'] - 0.5
                * 'yn': numpy.array of numeric values:
                    - if n in 'yn' is odd,
                        m_dict['Y']['yn'] = m_dict['Y']['y4']**(int(i//2))
                    - if n in 'yn' is even,
                        zn = 'y' + str(n-1)
                        m_dict['Y'][yn] = m_dict['Y']['y2'] * m_dict['Y'][zn]

        bounds (:obj:`list`): Upper and lower limits to filter the data with before calculating metalog quantiles/pdfs.
            - should be set in conjunction with the `boundedness` parameter

        boundedness (:obj:`str`): String that is used to specify the type of metalog to fit.
            - must be in set ('u','sl','su','b')
            - Default: 'u'
                * Fits an unbounded metalog
            - 'sl' fits a strictly lower bounded metalog
                * len(bounds) must == 1
            - 'su' fits a strictly upper bounded metalog
                * len(bounds) must == 1
            - 'b' fits a upper/lower bounded metalog
                * len(bounds) must == 2
                * bounds[1] must be > bounds[0]

        term_limit (:obj:`int`): The upper limit of the range of metalog terms to use to fit the data.
            - strictly > term_lower_bound
            - in range [3,30]

        term_lower_bound (:obj:`int`): The lower limit of the range of metalog terms to use to fit the data.
            - strictly < term_limit
            - in range [2,29]

        fit_method (:obj:`str`): Fit method to use to fit metalog distribution.
            - must be in set ('any','OLS','LP','MLE')
            - Default: 'any'
                * first tries 'OLS' method than 'LP'
            - 'OLS' only tries to fit by solving directly for a coefficients using ordinary least squares method
            - 'LP' only tries to estimate fit using simplex linear program optimization routine
            - 'MLE' first tries 'OLS' method than falls back to a maximum likelihood estimation routine

        alpha (:obj:`float`, optional): Regularization term to add to OLS fit
            - strictly >= 0.
            - should be set in conjunction with `penalty` parameter
            - Default: 0. (no regularization, OLS)

        diff_error (:obj:`float`, optional): Value used to in scipy.optimize.linprog method call
                                             to init the array of values representing the
                                             upper-bound of each inequality constraint (row) in A_ub.
            - #TODO: Insert maths

        diff_step (:obj:`float`, optional): Value passed to `step_len` parameter in support.py diffMatMetalog method call
                                             defines the bin width for the Reimann sum of the differences differentiation method
            - diffMatMetalog differentiates the metalog pdf
                * Differentiation reference: https://math.stackexchange.com/a/313135
    Returns:
        m_dict: (:obj:`dict` with keys ['params', 'dataValues', 'Y', 'A', 'M', 'Validation'])
            - m_dict['A']: (:obj:`pandas.DataFrame` with columns ['a2','a3', ... ,'an'] of type numeric):
                * a2, a3, ... , an are our a coefficients returned by the method specified in `fit_method`

            - m_dict['M']: (:obj:`pandas.DataFrame` with columns 0:'pdf_1',1:'cdf_1',2:'pdf_2',3:'cdf_2',
                            ...,((2*(term_limit-term_lower_bound))+1)-1:'pdf_n',
                                ((2*(term_limit-term_lower_bound))+1):'cdf_n'
                            where n is the total number of metalog fits determined by (term_limit-term_lower_bound)+1
                            )
                * pdf_1, pdf_2, ... , pdf_n are the metalog pdfs returned by pdf_quantile_builder.pdfMetalog method
                * cdf_1, cdf_2, ... , cdf_n are the metalog quantiles returned by pdf_quantile_builder.quantileMetalog method

            - m_dict['y']: (:obj: `numpy.ndarray` of type float):
                * Array of bin widths for both the pdf_n and cdf_n

            - m_dict['Validation']: (:obj:`pandas.DataFrame` with columns ['term', 'valid', 'method'] of type str):
                * 'term': each metalog estimation given a number of terms
                * 'valid': boolean flag indicating if the metalog estimation was valid or not
                * 'method': a string indicating which method was used for the metalog estimation

    """

    A = pd.DataFrame()
    c_a_names = []
    c_m_names = []
    Mh = pd.DataFrame()
    Validation = pd.DataFrame()
    df_MH_temp_list = list()
    df_A_temp_list = list()
    df_Validation_temp_list = list()

    # TODO: Large for-loop can probably be factored into smaller functions
    for i in range(term_lower_bound,term_limit+1):
        Y = m_dict['Y'].iloc[:,0:i]
        eye = np.eye(Y.shape[1])
        z = m_dict['dataValues']['z']
        y = m_dict['dataValues']['probs']
        step_len = m_dict['params']['step_len']
        methodFit = 'OLS'
        a_name = 'a'+str(i)
        m_name = 'm'+str(i)
        M_name = 'M'+str(i)
        c_m_names = np.append(c_m_names, [m_name, M_name])
        c_a_names = np.append(c_a_names, a_name)

        if fit_method == 'any' or fit_method == 'MLE':
            try:
                temp = np.dot(np.dot(np.linalg.inv(np.dot(Y.T, Y) + alpha*eye), Y.T), z)
            except:
                # use LP solver if OLS breaks
                temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
                methodFit = 'Linear Program'
        if fit_method == 'OLS':
            try:
                temp = np.dot(np.dot(np.linalg.inv(np.dot(Y.T, Y) + alpha*eye), Y.T), z)
            except:
                raise RuntimeError("OLS was unable to solve infeasible or poorly formulated problem")
        if fit_method == "LP":
            temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
            methodFit = 'Linear Program'

        if fit_method == 'MLE':
            temp = a_vector_MLE(temp, y, i, m_dict, bounds, boundedness)

        temp = np.append(temp, np.zeros(term_limit-i))

        # build a y vector for smaller data sets
        if len(z) < 100:
            y2 = np.linspace(step_len, 1 - step_len, int((1 - step_len) / step_len))
            tailstep = step_len / 10
            y1 = np.linspace(tailstep, (min(y2) - tailstep), int((min(y2) - tailstep) / tailstep))
            y3 = np.linspace((max(y2) + tailstep), (max(y2) + tailstep * 9), int((tailstep * 9) / tailstep))
            y = np.hstack((y1, y2, y3))

        # Get the dict and quantile values back for validation
        temp_dict = pdf_quantile_builder(temp, y=y, term_limit=i, bounds=bounds, boundedness=boundedness)

        # If it not a valid pdf run and the OLS version was used the LP version
        if (temp_dict['valid'] == 'no') and (fit_method != 'OLS'):
            temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
            temp = np.append(temp, np.zeros(term_limit-i))
            methodFit = 'Linear Program'

            # Get the dict and quantile values back for validation
            temp_dict = pdf_quantile_builder(temp, y=y, term_limit=i, bounds=bounds, boundedness=boundedness)

        df_MH_temp_list.append(pd.DataFrame(temp_dict['m']))
        df_MH_temp_list.append(pd.DataFrame(temp_dict['M']))
        df_A_temp_list.append(pd.DataFrame(temp))

        tempValidation = pd.DataFrame(data={'term': [i], 'valid': [temp_dict['valid']], 'method': [methodFit]})
        df_Validation_temp_list.append(tempValidation)

    Validation = pd.concat(df_Validation_temp_list, axis=0)
    Mh = pd.concat(df_MH_temp_list, axis=1)
    A = pd.concat(df_A_temp_list, axis=1)

    A.columns = c_a_names
    Mh.columns = c_m_names

    m_dict['A'] = A
    m_dict['M'] = Mh
    m_dict['M']['y'] = temp_dict['y']
    m_dict['Validation'] = Validation

    A = np.column_stack((np.repeat(1.,len(A)), A))
    Est = np.dot(m_dict['Y'], A)
    ncols = A.shape[1]
    Z = np.column_stack((np.array(m_dict['dataValues']['z']),np.repeat(m_dict['dataValues']['z'].values,ncols-1).reshape(len(m_dict['dataValues']['z']),ncols-1)))

    m_dict['square_residual_error'] = ((Z-Est)**2).sum(axis=1)

    return m_dict

def a_vector_LP(m_dict, term_limit, term_lower_bound, diff_error = .001, diff_step = 0.001):
    """TODO: write docstring

    """
    cnames = np.array([])

    for i in range(term_lower_bound, term_limit + 1):
        Y = m_dict['Y'].iloc[:, 0:i]
        z = m_dict['dataValues']['z']

        # Bulding the objective function using abs value LP formulation
        Y_neg = -Y

        new_Y = pd.DataFrame({'y1': Y.iloc[:, 0], 'y1_neg': Y_neg.iloc[:, 0]})

        for c in range(1,len(Y.iloc[0,:])):
            new_Y['y'+str(c+1)] = Y.iloc[:,c]
            new_Y['y' + str(c+1)+'_neg'] = Y_neg.iloc[:, c]

        a = np.array([''.join(['a', str(i)])])
        cnames = np.append(cnames, a, axis=0)

        # Building the constraint matrix
        error_mat = np.array([])

        for j in range(1,len(Y.iloc[:,0])+1):
            front_zeros = np.zeros(2 * (j - 1))
            ones = [1, -1]
            trail_zeroes = np.zeros(2 * (len(Y.iloc[:, 1]) - j))
            if j == 1:
                error_vars = np.append(ones, trail_zeroes)

            elif j != 1:
                error_vars = np.append(front_zeros, ones)
                error_vars = np.append(error_vars, trail_zeroes)

            if error_mat.size == 0:
                error_mat = np.append(error_mat, error_vars, axis=0)
            else:
                error_mat = np.vstack((error_mat, error_vars))

        new = pd.concat((pd.DataFrame(data=error_mat), new_Y), axis=1)
        diff_mat = diffMatMetalog(i, diff_step)
        diff_zeros = []

        for t in range(0,len(diff_mat.iloc[:, 0])):
            zeros_temp = np.zeros(2 * len(Y.iloc[:, 0]))

            if np.size(diff_zeros) == 0:
                diff_zeros = zeros_temp
            else:
                diff_zeros = np.vstack((zeros_temp, diff_zeros))

        diff_mat = np.concatenate((diff_zeros, diff_mat), axis=1)

        # Combine the total constraint matrix
        lp_mat = np.concatenate((new, diff_mat), axis=0)

        # Objective function coeficients
        c = np.append(np.ones(2 * len(Y.iloc[:, 1])), np.zeros(2*i))

        # Constraint matrices
        A_eq = lp_mat[:len(Y.iloc[:, 1]),:]
        A_ub = -1*lp_mat[len(Y.iloc[:, 1]):,:]
        b_eq = z
        b_ub = -1*np.repeat(diff_error, len(diff_mat[:,0]))

        # Solving the linear program w/ scipy (for now)
        lp_sol = linprog(c, A_ub=A_ub, b_ub=b_ub, A_eq=A_eq, b_eq=b_eq, method='simplex', options={"maxiter":5000, "tol":1.0e-5,"disp": False})

        # Consolidating solution back into the a vector
        tempLP = lp_sol.x[(2 * len(Y.iloc[:, 1])):(len(lp_sol.x)+1)]
        temp = []

        for r in range(0,((len(tempLP) // 2))):
            temp.append(tempLP[(r * 2)] - tempLP[(2 * r)+1])

    return temp

def a_vector_MLE(a, y, term, m_dict, bounds, boundedness):
    """TODO: write docstring

    """
    ym = [newtons_method_metalog(a, xi, term, bounds, boundedness) for xi in m_dict['dataValues']['x']]

    def MLE_quantile_constraints(x):
        M = [quantileMetalog(x[:term], yi, term, bounds=bounds, boundedness=boundedness) for yi in x[term:]]
        return m_dict['dataValues']['x'] - M

    def MLE_objective_function(x, y, term, m_dict):
        return -np.sum([np.log10(pdfMetalog(x[:term], yi, term, bounds, boundedness)) for yi in np.absolute(x[term:])])

    m_dict[str('MLE' + str(term))] = {}

    x0 = np.hstack((a[:term],ym))
    m_dict[str('MLE' + str(term))]['oldobj'] = -MLE_objective_function(x0, y, term, m_dict)
    bnd = ((None, None),)*len(a)+((0, 1),)*(len(x0)-len(a))
    con = NonlinearConstraint(MLE_quantile_constraints, 0, 0)

    mle = minimize(MLE_objective_function, x0, args=(y, term, m_dict), bounds=bnd, constraints=con)

    m_dict[str('MLE' + str(term))]['newobj'] = -MLE_objective_function(mle.x, y, term, m_dict)
    m_dict[str('MLE'+str(term))]['A'] = mle.x[:term]
    m_dict[str('MLE'+str(term))]['Y'] = mle.x[term:]

    m_dict[str('MLE' + str(term))]['oldA'] = a
    m_dict[str('MLE' + str(term))]['oldY'] = y

    out_temp = np.zeros_like(a)
    for i in range(term):
        out_temp[i] = mle.x[i]

    return out_temp

reason: import numpy,import scipy,from scipy


text:
"""
Operation representation classes for the `stabilizer_slow` evolution type.
"""
#***************************************************************************************************
# Copyright 2015, 2019 National Technology & Engineering Solutions of Sandia, LLC (NTESS).
# Under the terms of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights
# in this software.
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License.  You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0 or in the LICENSE file in the root pyGSTi directory.
#***************************************************************************************************

import numpy as _np

from .. import basereps as _basereps
from pygsti.baseobjs.statespace import StateSpace as _StateSpace
from ...tools import internalgates as _itgs
from ...tools import matrixtools as _mt
from ...tools import symplectic as _symp

class OpRep(_basereps.OpRep):
    def __init__(self, state_space):
        assert(state_space.num_qubits is not None), "State space does not contain a definite number of qubits!"
        self.state_space = state_space

    def acton(self, state):
        raise NotImplementedError()

    def adjoint_acton(self, state):
        raise NotImplementedError()

    @property
    def nqubits(self):
        return self.state_space.num_qubits

    #@property
    #def dim(self):
    #    return 2**(self.nqubits)  # assume "unitary evolution"-type mode

class OpRepClifford(OpRep):
    def __init__(self, unitarymx, symplecticrep, basis, state_space):

        if symplecticrep is not None:
            self.smatrix, self.svector = symplecticrep
        else:
            # compute symplectic rep from unitary
            self.smatrix, self.svector = _symp.unitary_to_symplectic(unitarymx, flagnonclifford=True)

        self.inv_smatrix, self.inv_svector = _symp.inverse_clifford(
            self.smatrix, self.svector)  # cache inverse since it's expensive

        #nQubits = len(self.svector) // 2
        #dim = 2**nQubits  # "stabilizer" is a "unitary evolution"-type mode
        self.unitary = unitarymx
        self.basis = basis

        state_space = _StateSpace.cast(state_space)
        assert(state_space.num_qubits == self.smatrix.shape[0] // 2)
        super(OpRepClifford, self).__init__(state_space)

    @property
    def unitary_dagger(self):
        return _np.conjugate(self.unitary.T)

    def acton(self, state):
        """ Act this gate map on an input state """
        state = state.copy()  # (copies any qubit filters in .sframe too)
        state.sframe.clifford_update(self.smatrix, self.svector, self.unitary)
        return state

    def adjoint_acton(self, state):
        """ Act the adjoint of this operation matrix on an input state """
        # Note: cliffords are unitary, so adjoint == inverse
        state = state.copy()  # (copies any qubit filters in .sframe too)
        state.sframe.clifford_update(self.smatrix_inv, self.svector_inv,
                                     _np.conjugate(self.unitary.T))
        return state

    def __str__(self):
        """ Return string representation """
        s = "Clifford operation with matrix:\n"
        s += _mt.mx_to_string(self.smatrix, width=2, prec=0)
        s += " and vector " + _mt.mx_to_string(self.svector, width=2, prec=0)
        return s

class OpRepStandard(OpRepClifford):
    def __init__(self, name, basis, state_space):
        std_unitaries = _itgs.standard_gatename_unitaries()
        self.name = name
        if self.name not in std_unitaries:
            raise ValueError("Name '%s' not in standard unitaries" % self.name)

        U = std_unitaries[self.name]
        super(OpRepStandard, self).__init__(U, None, basis, state_space)

class OpRepComposed(OpRep):

    def __init__(self, factor_op_reps, state_space):
        state_space = _StateSpace.cast(state_space)
        self.factor_reps = factor_op_reps
        super(OpRepComposed, self).__init__(state_space)

    def reinit_factor_op_reps(self, factor_op_reps):
        self.factors_reps = factor_op_reps

    def acton(self, state):
        """ Act this gate map on an input state """
        for gate in self.factor_reps:
            state = gate.acton(state)
        return state

    def adjoint_acton(self, state):
        """ Act the adjoint of this operation matrix on an input state """
        for gate in reversed(self.factor_reps):
            state = gate.adjoint_acton(state)
        return state

class OpRepSum(OpRep):
    def __init__(self, factor_reps, state_space):
        state_space = _StateSpace.cast(state_space)
        self.factor_reps = factor_reps
        super(OpRepSum, self).__init__(state_space)

    def reinit_factor_reps(self, factor_reps):
        self.factors_reps = factor_reps

    def acton(self, state):
        """ Act this gate map on an input state """
        # need further stabilizer frame support to represent the sum of stabilizer states
        raise NotImplementedError()

    def adjoint_acton(self, state):
        """ Act the adjoint of this operation matrix on an input state """
        # need further stabilizer frame support to represent the sum of stabilizer states
        raise NotImplementedError()

class OpRepEmbedded(OpRep):

    def __init__(self, state_space, target_labels, embedded_rep):
        # assert that all state space labels == qubits, since we only know
        # how to embed cliffords on qubits...
        state_space = _StateSpace.cast(state_space)
        assert(state_space.num_tensor_product_blocks == 1
               and all([state_space.label_udimension(l) == 2 for l in state_space.tensor_product_block_labels(0)])), \
            "All state space labels must correspond to *qubits*"

        #Cache info to speedup representation's acton(...) methods:
        # Note: ...labels[0] is the *only* tensor-prod-block, asserted above
        qubitLabels = state_space.tensor_product_block_labels(0)
        qubit_indices = _np.array([qubitLabels.index(targetLbl)
                                   for targetLbl in target_labels], _np.int64)

        self.embedded_rep = embedded_rep
        self.qubits = qubit_indices  # qubit *indices*
        super(OpRepEmbedded, self).__init__(state_space)

    def acton(self, state):
        state = state.copy()  # needed?
        state.sframe.push_view(self.qubits)
        outstate = self.embedded_rep.acton(state)  # works b/c sfame has "view filters"
        state.sframe.pop_view()  # return input state to original view
        outstate.sframe.pop_view()
        return outstate

    def adjoint_acton(self, state):
        state = state.copy()  # needed?
        state.sframe.push_view(self.qubits)
        outstate = self.embedded_rep.adjoint_acton(state)  # works b/c sfame has "view filters"
        state.sframe.pop_view()  # return input state to original view
        outstate.sframe.pop_view()
        return outstate

class OpRepExpErrorgen(OpRep):

    def __init__(self, errorgen_rep):
        state_space = errorgen_rep.state_space
        self.errorgen_rep = errorgen_rep
        super(OpRepExpErrorgen, self).__init__(state_space)

    def errgenrep_has_changed(self, onenorm_upperbound):
        pass

    def acton(self, state):
        raise AttributeError("Cannot currently act with statevec.OpRepExpErrorgen - for terms only!")

    def adjoint_acton(self, state):
        raise AttributeError("Cannot currently act with statevec.OpRepExpErrorgen - for terms only!")

class OpRepRepeated(OpRep):
    def __init__(self, rep_to_repeat, num_repetitions, state_space):
        state_space = _StateSpace.cast(state_space)
        self.repeated_rep = rep_to_repeat
        self.num_repetitions = num_repetitions
        super(OpRepRepeated, self).__init__(state_space)

    def acton(self, state):
        """ Act this gate map on an input state """
        for i in range(self.num_repetitions):
            state = self.repeated_rep.acton(state)
        return state

    def adjoint_acton(self, state):
        """ Act the adjoint of this operation matrix on an input state """
        for i in range(self.num_repetitions):
            state = self.repeated_rep.adjoint_acton(state)
        return state

class OpRepLindbladErrorgen(OpRep):
    def __init__(self, lindblad_coefficient_blocks, state_space):
        super(OpRepLindbladErrorgen, self).__init__(state_space)
        self.Lterms = None
        self.Lterm_coeffs = None
        self.lindblad_coefficient_blocks = lindblad_coefficient_blocks

max_stars_repo_name: pyGSTi-Developers/pyGSTi

max_issues_repo_name: pyGSTi-Developers/pyGSTi

max_forks_repo_name: pyGSTi-Developers/pyGSTi


text:
import numpy as np

from mytorch.simple_core import Variable, add

def ex1():
    x = Variable(np.array(3.0))
    y = add(x, x)
    y.backward()
    print(x.grad)

    y = add(add(x, x), x)
    y.backward()
    print(x.grad)  # wrong

def ex2():
    x = Variable(np.array(3.0))
    y = add(x, x)
    y.backward()
    print(x.grad)

    x.zerograd()
    y = add(add(x, x), x)
    y.backward()
    print(x.grad)

ex1()
ex2()


text:
import numpy as np
import rosbag

def to_rads_sec(deg_sec):
    return float(np.deg2rad(deg_sec))

def fix_bag_data():
    data_dir='/home/akshitjain/catkin_ws/data'
    bag_file_name = 'cam_imu'
    in_bag_file = "{}/{}.bag".format(data_dir, bag_file_name)
    out_bag_file = "{}/{}_fixed.bag".format(data_dir, bag_file_name)

    with rosbag.Bag(out_bag_file, 'w') as outbag:
        for topic, msg, t in rosbag.Bag(in_bag_file).read_messages():
            if topic == "/vio/imu":
                msg.angular_velocity.x = to_rads_sec(msg.angular_velocity.x)
                msg.angular_velocity.y = to_rads_sec(msg.angular_velocity.y)
                msg.angular_velocity.z = to_rads_sec(msg.angular_velocity.z)

            outbag.write(topic, msg, msg.header.stamp)

if __name__ == '__main__':
    fix_bag_data()


text:
import nltk
from nltk.stem.porter import PorterStemmer
import numpy as np
from functools import lru_cache
import spacy

stemmer = PorterStemmer()

@lru_cache()
def loader():
    nltk.download('punkt')

def tokenize(sentence):
    return nltk.word_tokenize(sentence)

def stem(word):
    return stemmer.stem(word)

def bag_of_words(tokenized_sentence, all_words):
    tokenized_sentence = [stem(w) for w in tokenized_sentence]
    bag = np.zeros(len(all_words), dtype=np.float32)
    for idx, w in enumerate(all_words):
        if w in tokenized_sentence:
            bag[idx] = 1.0
    return bag

@lru_cache()
def remove_stop(sentence):
    nlp = spacy.load('en_core_web_trf')
    stopwords = [',', '.', '?', '!']
    stopwords.extend(spacy.lang.en.stop_words.STOP_WORDS)
    sent_without_stop = [w for w in tokenize(sentence) if w not in stopwords]
    return sent_without_stop

max_stars_repo_name: MichalPodlaszuk/AI-Dispatcher

max_issues_repo_name: MichalPodlaszuk/AI-Dispatcher

max_forks_repo_name: MichalPodlaszuk/AI-Dispatcher


text:
# -*- coding: utf-8 -*-
"""
SEGM.BOT: bag-of-things related functions.
@author: vlad
"""
from __future__ import (absolute_import, division, print_function, unicode_literals)
from builtins import *

__version__ = 0.01
__author__ = 'Vlad Popovici'
__all__ = ['grow_bag_from_new_image', 'grow_bag_with_new_features',
           'grow_bag_from_images', 'read_bag', 'build_codebook_kmeans_online']

import numpy as np

from sklearn.cluster import MiniBatchKMeans
import skimage.io
from skimage.util import img_as_float

from util.explore import random_window_on_regions, random_window, \
    sliding_window, sliding_window_on_regions
from util.misc import intg_image
from descriptors.txtgrey import HaarLikeDescriptor
from ml.gap import gap

import joblib

def worker1(img_, r_, woff_, desc_, discard_empty_):
    if discard_empty_ and img_[r_[0]:r_[1], r_[2]:r_[3]].sum() < 1e-16:
        return (None, None)
    # adjust if needed:
    r2 = (r_[0], r_[1] - woff_[1], r_[2], r_[3] - woff_[0])
    return (desc_.compute(img_[r_[0]:r_[1], r_[2]:r_[3]]), r2)

def grow_bag_from_new_image(image, desc, w_size, n_obj, **kwargs):
    """
    Extracts local descriptors from a new image.

    :param image: numpy.array
        Image data (single channel).
    :param desc: LocalDescriptor
        Local descriptor for feature extraction.
    :param w_size: tuple
        (width, height) of the sub-windows from the image.
    :param n_obj: int
        Maximum number of objects to be added to the bag.
    :param kwargs: dict
        Other parameters:
        'roi': region of interest (default: None)
        'sampling_strategy': how the image should be sampled:
            'random' for random sampling
            'sliding' for systematic, sliding window scanning
             of the image
        'it_start': where the scanning of the image starts (for
            sliding window sampling strategy) (default (0,0))
        'it_step': step from one window to the next (for
            sliding window sampling strategy) (default (1,1))
        'discard_empty': (boolean) whether an empy patch should still
            be processed or simply discarded. Default: False
    :return: dict
        A dictionary with two elements:
            <name of the descriptor>: list
            'regions': list
        The first list contains the feature descriptors.
        The second list contains the corresponding window positions.

    See also: grow_bag_with_new_features
    """

    if 'roi' not in kwargs:
        roi = None
    else:
        roi = kwargs['roi']

    if 'it_start' not in kwargs:
        it_start = (0,0)
    else:
        it_start = kwargs['it_start']

    if 'it_step' not in kwargs:
        it_step = (1,1)
    else:
        it_step = kwargs['it_step']

    if 'sampling_strategy' not in kwargs:
        sampling_strategy = 'random'
    else:
        sampling_strategy = kwargs['sampling_strategy']

    if 'discard_empty' in kwargs:
        discard_empty = kwargs['discard_empty']
    else:
        discard_empty = False

    if discard_empty:
        if image.dtype == np.uint8:
            discard_empty_threshold = w_size[0]*w_size[1]/20
        elif image.dtype == np.float32 or image.dtype == np.float64:
            discard_empty_threshold = 1e-5
        else:
            discard_empty_threshold = 0

    w_offset = (0, 0)
    if isinstance(desc, HaarLikeDescriptor):
        # this one works on integral images
        image = intg_image(image)
        # the sliding window should also be increased by 1:
        w_offset = (1, 1)
        w_size = (w_size[0] + w_offset[0], w_size[1] + w_offset[1])

    # create iterator:
    sampling_strategy = sampling_strategy.lower()
    if sampling_strategy == 'random':
        if roi is None:
            itw = random_window(image.shape, w_size, n_obj)
        else:
            itw = random_window_on_regions(image.shape, roi, w_size, n_obj)
    elif sampling_strategy == 'sliding':
        if roi is None:
            itw = sliding_window(image.shape, w_size, start=it_start, step=it_step)
        else:
            itw = sliding_window_on_regions(image.shape, roi, w_size, step=it_step)
    else:
        raise ValueError('Unknown strategy.')

    bag = []
    wnd = []
    n = 0

    for r in itw:
        if discard_empty and image[r[0]:r[1], r[2]:r[3]].sum() <= discard_empty_threshold:
            continue

        # adjust if needed:
        r2 = (r[0], r[1] - w_offset[1], r[2], r[3] - w_offset[0])
        wnd.append(r2)
        bag.append(desc.compute(image[r[0]:r[1], r[2]:r[3]]))

        n += 1
        if n > n_obj:
            break

#    wnd = [w for w in itw]             # pre-generate windows
#    wnd = wnd[:min(len(wnd), n_obj)]   # keep only at most n_obj windows
#
#    # parallelized:
#    res = joblib.Parallel(n_jobs=joblib.cpu_count())( joblib.delayed(worker1)(image, r, w_offset, desc, discard_empty) for r in wnd )
#
#    wnd = []
#    bag = []
#    for b, w in res:
#        if not b is None:
#            bag.append(b)
#            wnd.append(w)

    return {desc.name: bag, 'regs': wnd}

def grow_bag_with_new_features(image, regions, desc):
    """
    Returns the features corresponding to a list of regions. This
    is usually used for adding new features to an existing bag,
    where the list of regions has been obtained from an iterator.

    :param image: numpy.array
        Image data (single channel).
    :param regions: list
        A list of regions [(row_min, row_max, col_min, col_max),...]
    :param desc: LocalDescriptor
        Descriptor used for extracting the features.
    :return: dict
        A dictionary with two elements:
            <name of the descriptor>: list
            'regions': list
        The first list contains the feature descriptors.
        The second list contains the corresponding window positions
        (identical with the input regions list)

    See also: grow_bag_from_new_image
    """

    w_offset = (0, 0)
    if isinstance(desc, HaarLikeDescriptor):
        # this one works on integral images
        image = intg_image(image)
        # the sliding window should also be increased by 1:
        w_offset = (1, 1)

    bag = []

    for w in regions:
        # adjust if needed:
        r = (w[0], w[1] + w_offset[1], w[2], w[3] + w_offset[0])
        bag.append(desc.compute(image[r[0]:r[1], r[2]:r[3]]))

    return {desc.name: bag, 'regs': regions}

def grow_bag_from_images(img_list, desc_list, w_size, img_n):
    big_bag = None
    inames = []
    for img in img_list:
        im = skimage.io.imread(img)
        im = img_as_float(im)
        bag = None
        for d in desc_list:
            if bag is None:
                bag = grow_bag_from_new_image(im, d, w_size, img_n)
            else:
                bag[d.name] = grow_bag_with_new_features(im, bag['regs'], d)[d.name]

        if big_bag is None:
            big_bag = bag
        else:
            for k in big_bag.keys():
                big_bag[k].extend(bag[k])

        inames.extend(len(bag['regs'])*[img])  # add the name of the image

    big_bag['image'] = inames

    return big_bag

def read_bag(infile, desc_name):
    """
    Read a bag of features from an external file. It is assumed that the
    file contains values corresponding to a single type of descriptors,
    since no information distinguishing the descriptors is stored in the
    file.

    :param infile: string
        file name
    :param desc_name: string
        the name of the descriptor whose values are read
    :return: dict
        a bag of features with 'regs' and <desc_name> keys
    """

    with open(infile, 'r') as f:
        x = np.loadtxt(f)

    # x contains, as a matrix, all the local deascriptors, supposedly of the
    # same type. The following data is in x:
    # -first 4 columns in x correspond to the rectangular region on which
    #  the feature was computed
    # -the remaining columns correspond to descriptor values
    # -each row corresponds to a patch in the image (at coordinates given
    #  by the first 4 values)
    ndesc, nfeat = x.shape
    nfeat -= 4  # first 4 values are coords

    # put everything in a regular B-o-T dictionary (as above):
    regs = []
    desc = []
    for i in range(ndesc):
        regs.append(np.int32(x[i, 0:4]))
        desc.append(x[i, 4:])

    return {desc_name: desc, 'regs': regs}

def build_codebook_kmeans_online(bag, codebook_size, desc_names, standardize=False):
    desc = [np.array(bag[dn_]) for dn_ in desc_names]                  # ensures a strict ordering

    X = np.hstack(desc)                                                # put all feature vectors in an array
    if standardize:
        # make sure each variable (column) is mean-centered and has unit standard deviation
        Xm = np.mean(X, axis=0)
        Xs = np.std(X, axis=0)
        Xs[np.isclose(Xs, 1e-16)] = 1.0
        X = (X - Xm) / Xs

    if codebook_size is None:
        # try to estimate a suitable codebook size based on gap statistic:
        codebook_size,_ = gap(X, Ks=np.linspace(start=10, stop=100, num=10, dtype=np.int32),
                              Wstar=None, B=20)
        print("Best codebook size:", codebook_size)

    rng = np.random.RandomState(0)
    vq = MiniBatchKMeans(n_clusters=codebook_size, random_state=rng,
                         batch_size=500, compute_labels=True, verbose=True)   # vector quantizer

    vq.fit(X)

    return vq

max_stars_repo_name: vladpopovici/WSItk

max_issues_repo_name: vladpopovici/WSItk

max_forks_repo_name: vladpopovici/WSItk


text:
#!/usr/bin/env python

#=========================================================================
# Copyright (c)  2014 Geoscience Australia
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither Geoscience Australia nor the names of its contributors may be
#       used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#=========================================================================
'''
THREDDSCatalog Class
Created on 22/03/2016

@author: Alex Ip
'''
import os
import re
from collections import OrderedDict
import logging
import subprocess
#from osgeo import gdal, osr
import numpy as np
import netCDF4
from owslib.csw import CatalogueServiceWeb
from owslib.fes import PropertyIsEqualTo  # , PropertyIsLike, BBox
from datetime import datetime
import tempfile
import dateutil.parser
from dateutil import tz
import pytz
import yaml

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)  # Logging level for this module

# Ignore failed import of URL modules
try:
    import urllib
except:
    logger.warning(
        'WARNING: Unable to import urllib. Any HTML function calls will fail.')

try:
    import lxml.html
except:
    logger.warning(
        'WARNING: Unable to import lxml.html. Any HTML function calls will fail.')

class THREDDSCatalog(object):
    '''
    Class definition for THREDDSCatalog
    '''
    # DEFAULT_THREDDS_CATALOGUE_URL = 'http://dapds00.nci.org.au/thredds/catalog.html'
    DEFAULT_THREDDS_CATALOGUE_URLS = 'http://dapds00.nci.org.au/thredds/catalog/rr2/National_Coverages/catalog.html,\
http://dapds00.nci.org.au/thredds/catalog/rr2/National_Coverages/http/catalog.html'

    def __init__(self, thredds_catalog_urls=None,
                 yaml_path=None, verbose=False):
        '''
        Constructor for class THREDDSCatalog
        Launches a crawler to examine every THREDDS catalog page underneath the nominated thredds_catalog_urls
        '''
        assert (yaml_path and not thredds_catalog_urls) or (
            thredds_catalog_urls and not yaml_path), 'yaml_path or thredds_catalog_urls should be specified, but not both.'
        self.verbose = verbose
        if yaml_path:
            self.load(yaml_path)
        else:
            thredds_catalog_urls = thredds_catalog_urls or self.DEFAULT_THREDDS_CATALOGUE_URLS
            self.thredds_catalog_url_list = [thredds_catalog_url.strip() for thredds_catalog_url in thredds_catalog_urls.split(',')]

            # Top level dict is an OrderedDict to implement prioritised search through specified catalogues
            self.thredds_catalog_dict = OrderedDict((thredds_catalog_url, self.get_thredds_dict(thredds_catalog_url))
                                                     for thredds_catalog_url in self.thredds_catalog_url_list
                                                    )

    def get_thredds_dict(self, thredds_catalog_urls):
        '''
        get_thredds_dict - recursive function to parse specified THREDDS catalogue URL and return a nested dict
        Parameter: thredds_catalog_urls - string specifying URL of THREDDS catalog
        '''
        def get_absolute_url(href):
            # Create absolute URL
            if href.startswith(
                    '/'):  # Absolute href - should start with "/thredds/"
                return re.sub('/thredds/.*', href, thredds_catalog_urls)
            else:  # Relative href
                return re.sub('catalog.html$', href, thredds_catalog_urls)

        thredds_catalog_dict = {}

        if self.verbose:
            logger.info('Opening %s', thredds_catalog_urls)
        data = urllib.urlopen(thredds_catalog_urls).read()
        logger.debug('data = %s', data)

        tree = lxml.html.fromstring(data)

        title = tree.find('.//title')
        title_text = [e.strip() for e in title.xpath(
            './/text()') if len(e.strip()) > 0][0]
        logger.debug('title_text = %s', title_text)

        if title_text == 'Catalog Services':  # This is a landing page for a file
            # Iterate through all service endpoints for file
            for ol in tree.iterfind('.//ol'):
                for li in ol.iterfind('.//li'):
                    text = [e.strip() for e in li.xpath(
                        './/text()') if len(e.strip()) > 0]

                    if len(text) == 0:  # No li text found
                        continue

                    endpoint_type = text[0].replace(':', '')
                    url = get_absolute_url(text[1])

        #======================================================================
        #             # Not sure why a isn't found
        #             a = li.find('.//a')
        #             if a is None:
        #                 logger.debug('a not found')
        #                 continue
        #
        #             href = a.get('href')
        #             if href is None:
        #                 logger.debug('href not found')
        #                 continue
        #
        #             url = get_absolute_url(href)
        #======================================================================

                    logger.debug(
                        'Service endpoint: endpoint_type = %s, href = %s', endpoint_type, url)
                    thredds_catalog_dict[endpoint_type] = url
                break  # Only process first "<ol>"

        else:  # Catalog page for virtual subdirectory

            for table in tree.iterfind('.//table'):
                #                first_row = True
                for row in table.iterfind('.//tr'):
                    a = row.find('.//a')
                    if a is None:
                        continue

                    href = a.get('href')
                    logger.debug('href = %s', href)
                    if href is None:
                        continue

                    url = get_absolute_url(href)

                    if href.endswith('catalog.html'):  # Virtual subdirectory
                        logger.debug('Virtual subdirectory: url = %s', url)
                        thredds_catalog_dict[url] = self.get_thredds_dict(url)

                    elif href.startswith('catalog.html?dataset='):  # File
                        filename = os.path.basename(href)

                        logger.debug(
                            'File: filename = %s, url = %s', filename, url)
                        try:
                            thredds_catalog_dict[
                                url] = self.get_thredds_dict(url)
                        except Exception as e:
                            logger.error('ERROR: %s', e.message)

                    # Get rid of empty dicts
                    if thredds_catalog_dict.get(url) == {}:
                        del thredds_catalog_dict[url]

        logger.debug('thredds_catalog_dict = %s', thredds_catalog_dict)
        return thredds_catalog_dict

    def dump(self, yaml_path=None):
        yaml_path = os.path.abspath(yaml_path or (re.sub('\W', '_', re.sub(
            'http://', '', self.thredds_catalog_dict.keys()[0])) + '.yaml'))
        yaml_file = open(yaml_path, 'w')
        yaml.dump(self.thredds_catalog_dict, yaml_file)
        yaml_file.close()
        logger.info('THREDDS catalogue dumped to file %s', yaml_path)

    def load(self, yaml_path):
        yaml_file = open(yaml_path, 'r')
        self.thredds_catalog_dict = yaml.load(yaml_file)
        yaml_file.close()
        logger.info('THREDDS catalogue loaded from file %s', yaml_path)

    def endpoint_tuple_list(self, type_filter='.*',
                            url_filter='.*', catalog_dict=None):
        '''
        Function to return a list of (protocol, endpoint) tuples contained in the leaf nodes of self.thredds_catalog_dict
        Arguments:
            type_filter: regular expression string matching one or more of ['HTTPServer', 'NetcdfSubset', OPENDAP', 'WCS, 'WMS']
            url_filter: regular expression string to restrict URLs returned: e.g. '.*\.nc$' to return all NetCDF endpoints
        '''
        result_list = []
        catalog_dict = catalog_dict or self.thredds_catalog_dict

        for key in sorted(catalog_dict.keys()):
            value = catalog_dict[key]

            if isinstance(value, dict):
                result_list += self.endpoint_tuple_list(
                    type_filter, url_filter, catalog_dict[key])
            else:  # Leaf node
                if (re.search(type_filter, key)
                        and re.search(url_filter, value)):
                    result_list.append((key, value))

        return result_list

    def endpoint_list(self, type_filter='.*',
                      url_filter='.*', catalog_dict=None):
        '''
        Function to return a list of endpoints contained in the leaf nodes of self.thredds_catalog_dict
        Arguments:
            type_filter: regular expression string matching one or more of ['HTTPServer', 'NetcdfSubset', OPENDAP', 'WCS, 'WMS']
            url_filter: regular expression string to restrict URLs returned: e.g. '.*\.nc$' to return all NetCDF endpoints
        '''
        return [endpoint for _protocol, endpoint in self.endpoint_tuple_list(
            type_filter=type_filter, url_filter=url_filter, catalog_dict=catalog_dict)]

    def find_url_list(self, file_path):
        '''
        Function to return list of (<protocol>, <url>) tuples for a given filename
        Returns empty list for failed match.
        '''
        # Narrow down search to tuples matching basename
        basename = os.path.basename(file_path)
        base_list = self.endpoint_tuple_list(
            type_filter='.*', url_filter=basename)
        if base_list:
            logger.debug('%d possible URLs initially found for basename %s', len(
                base_list), basename)
        else:  # Nothing found
            logger.debug('No possible URLs found for basename %s', basename)
            return []

        # Find URL matches for longest possible sub-path
        find_path = os.path.abspath(file_path).replace(os.path.sep, '/') # Convert path separators to '/'
        find_list = []
        while find_path and not find_list:
            logger.debug('Searching for %s in URL list' % find_path)
            find_list = [(protocol, url)
                         for protocol, url in base_list if url.find(find_path)]
            if find_list:  # Matches found for maximum-length sub-path
                logger.debug('%d URLs found for %s', len(find_list), find_path)
                return find_list  # Search complete
            elif '/' in find_path:  # More leading directories to strip
                # Strip leading directory from find_path
                find_path = re.sub('^[^/]*/', '', find_path)
            else:  # Nothing found for basename - should never happen
                logger.debug('No URLs found for %s', find_path)
                return []

    def find_url_dict(self, file_path):
        '''
        Function to return dict of {<protocol>: <url>, <protocol>: <url>,...} for a given filename
        Returns empty dict for failed match, returns first of multiple URLs when duplicate protocols found
        '''
        # Convert list of tuples to dict - remove duplicate protocols
        result_dict = {}
        for protocol, url in self.find_url_list(file_path):
            existing_url = result_dict.get(protocol)
            # Keep the shorter of the two URLs when duplicate protocols found
            if existing_url is None:
                result_dict[protocol] = url

        return result_dict

    def find_catalogs(self, file_path, distribution_types=[
                      'NetcdfSubset'], catalog_dict=None):
        '''
        Recursive function to return list of catalog URLS containing specified distribution type(s) for specified file_path
        Returns empty dict for failed match, keeps the shorter of two URLs when duplicates found
        '''
        basename = os.path.basename(file_path)
        base_list = []
        catalog_dict = catalog_dict or self.thredds_catalog_dict

        for key in sorted(catalog_dict.keys()):
            value = catalog_dict[key]

            if isinstance(value, dict):
                if re.search(
                        basename + '$', key) and (set(distribution_types) <= set(value.keys())):
                    base_list.append(key)
                else:
                    base_list += self.find_catalogs(file_path,
                                                    distribution_types, value)

        if not base_list:
            return base_list

        # Find URL matches for longest possible sub-path
        find_path = os.path.abspath(file_path).replace(os.path.sep, '/')
        find_list = []
        while find_path and not find_list:
            logger.debug('Searching for %s in URL list' % find_path)
            find_list = [catalog_url for catalog_url in base_list if re.search(
                find_path + '$', catalog_url)]
            if find_list:  # Matches found for maximum-length sub-path
                logger.debug('%d URLs found for %s', len(find_list), find_path)
                return find_list  # Search complete
            elif '/' in find_path:  # More leading directories to strip
                # Strip leading directory from find_path
                find_path = re.sub('^[^/]*/', '', find_path)
            else:  # Nothing found for basename - should never happen
                logger.debug('No URLs found for %s', find_path)
                return []

    def indented_text(self, subtree_dict=None, level=0):
        '''
        Recursive function to return indented text representing catalog tree
        '''
        subtree_dict = subtree_dict or self.thredds_catalog_dict
        result = ''

        key_list = subtree_dict.keys()
        if type(subtree_dict) != OrderedDict:
            key_list = sorted(key_list)

        for key in key_list:
            result += ('  ' * level) + key
            value = subtree_dict[key]

            if type(value) == str:
                result += ': ' + value + '\n'
            elif type(value) == dict:
                result += '\n' + self.indented_text(value, level+1)
            else:
                raise Exception('Unrecognised value type')

        return result


text:
import math
import numpy as np
import pandas as pd
import scipy
import scipy.io
import scipy.sparse
from rdkit import Chem
from rdkit.Chem import MolFromSmiles
from rdkit.Chem import AllChem
from rdkit import DataStructs
import pickle
import os
import argparse
import rdkit
from rdkit.Chem import MACCSkeys

file_path_now=os.getcwd()
catalog=file_path_now.split('python_code')[0]
print(catalog)
parser = argparse.ArgumentParser()
parser.add_argument("-supplfile", help="sdf file path", type=str,default=catalog+"/test_data/test.sdf")
parser.add_argument("-lstmfeature", help="lstm sparse feature", default=False)
parser.add_argument("-featureoutfile", help="pckl file path", type=str, default=catalog+"/test_data/")
parser.add_argument("-featureoutname", help="pckl file name", type=str, default="test")
args = parser.parse_args()

sdf_supply = args.supplfile
lstmfeature = args.lstmfeature
featureoutfile = args.featureoutfile
featureoutname = args.featureoutname
feature_format='dense'

suppl = Chem.SDMolSupplier(sdf_supply)
ms = [x for x in suppl if x is not None]
n = len(ms)
ecfpMat = np.zeros((n, 2048), dtype=int)
for i in range(n):
    fp = AllChem.GetMorganFingerprintAsBitVect(ms[i], 3, 2048)
    ecfpMat[i] = np.array(list(fp.ToBitString()))
if lstmfeature:
    ecfpMat = scipy.sparse.csr_matrix(ecfpMat)
    feature_format='sparse'
sampleECFPInd = pd.Series(data=np.arange(len(ms)), index=np.arange(len(ms)))
f = open(featureoutfile+featureoutname+'ecfp6'+feature_format+'.pckl', "wb")
pickle.dump(ecfpMat, f, -1)
pickle.dump(sampleECFPInd, f, -1)
f.close()

fcfpMat = np.zeros((n, 2048), dtype=int)
for i in range(n):
    fp = AllChem.GetMorganFingerprintAsBitVect(ms[i], 3, 2048, useFeatures=True)
    fcfpMat[i] = np.array(list(fp.ToBitString()))
if lstmfeature:
    fcfpMat = scipy.sparse.csr_matrix(fcfpMat)
sampleFCFPInd = pd.Series(data=np.arange(len(ms)), index=np.arange(len(ms)))
f = open(featureoutfile+featureoutname+'fcfp6'+feature_format+'.pckl', "wb")
pickle.dump(fcfpMat, f, -1)
pickle.dump(sampleFCFPInd, f, -1)
f.close()

if not lstmfeature:
    MACCSArray = []
    for x in ms:
        MACCSArray.append(MACCSkeys.GenMACCSKeys(x))
    MACCSMat=np.array(MACCSArray)
    sampleMACCSInd = pd.Series(data=np.arange(len(ms)), index=np.arange(len(ms)))
    f=open(featureoutfile+featureoutname+'MACCSFNN'+feature_format+'.pckl', "wb")
    pickle.dump(MACCSMat, f, -1)
    pickle.dump(sampleMACCSInd, f, -1)
    f.close()

max_stars_repo_name: zhonghua-wang/NPTP

max_issues_repo_name: zhonghua-wang/NPTP

max_forks_repo_name: zhonghua-wang/NPTP

reason: import numpy,import scipy


text:
from utils.preprocessing import to_ndarray
import pandas as pd
import numpy as np
from utils.math_functions import euclidian_distance
from typing import Union

class KNNClassifier:
    k: int
    x: np.ndarray
    y: np.ndarray
    labels: np.ndarray

    def __init__(self, k=3):
        self.k = k

    def fit(self, x: Union[pd.DataFrame, np.ndarray], y: Union[pd.DataFrame, np.ndarray]):
        self.x = to_ndarray(x)
        self.y = to_ndarray(y)
        self.labels = np.unique(self.y)

    def count_labels(self, labels: np.ndarray):
        labels_dict = dict()

        for label in self.labels:
            labels_dict[label] = 0

        for label in labels:
            labels_dict[label] += 1

        labels_dict = list(labels_dict.items())
        labels_dict = sorted(
            labels_dict, key=lambda item: item[1], reverse=True)

        labels_dict = [(item[0], item[1] / self.k) for item in labels_dict]

        return labels_dict

    def predict_one(self, sample:  np.ndarray):
        distances = euclidian_distance(sample, self.x)
        index = np.argsort(distances)
        k_best_index = index[:self.k]
        k_best = self.y[k_best_index]

        return self.count_labels(k_best)

    def predict(self, values: Union[pd.DataFrame, np.ndarray]):
        if type(values) == pd.DataFrame:
            values = values.values.tolist()

        return np.array([self.predict_one(val)[0][0] for val in values])

    def predict_proba(self, values: Union[pd.DataFrame, np.ndarray]):
        if type(values) == pd.DataFrame:
            values = values.values.tolist()

        return np.array([self.predict_one(val)[0] for val in values])


text:
"""Filter for detecting posterized images.

The posterized image detection first loads the given image as a grayscale
image and calculates a histogram of its pixel intensities. From this
histogram the value for each local max point is calculated which measures
how sharp the peak is. Calculating the value takes into account how wide
the peak is, meaning how large the distance between the local minimum
before and the local maximum after it is. The second feature measured is
how large the peak is, meaning how big is the average difference between
the value at the local maximum and two local minimums next to it.
Posterized images have naturally more sharp peaks since colors in the
image are lacking many different shades.
"""

import cv2
import numpy as np

from ..utils.image_utils import read_image
from ..utils.histogram_analyzation import normalize, largest, calculate_peak_value
from ..utils.statistic_common import linear_normalize

from .filter import Filter

def get_input_vector(img):
    """Returns a numpy array which contains the average of 20 percent of
    the largest peak values of the histogram of a given image. This value
    can be used to predict whether an image is posterized or not. The
    result can also be given to an SVM.

    :param img: image to be processed
    :type img: numpy.ndarray
    :returns: a numpy array which contains the prediction
    """
    hist = cv2.calcHist([img], [0], None, [256], [0, 255]).T[0]
    hist = normalize(hist)
    peak_values = calculate_peak_value(hist)

    if len(peak_values) == 0:
        return np.array([0.0]).astype(np.float32)

    largest_peak_values = largest(peak_values, 0.2)
    return np.array([np.average(largest_peak_values)]).astype(np.float32)

class Posterized(Filter):

    """Filter for detecting posterized images"""

    name = 'posterized'
    speed = 1

    def __init__(self, threshold=0.5, invert_threshold=False):
        """Initializes a posterized image filter

        :param threshold: threshold at which the given prediction is changed
                          from negative to positive
        :type threshold: float
        :param invert_threshold: whether the result should be greater than
                                 the given threshold (default) or lower
                                 for an image to be considered positive
        :type invert_threshold: bool
        """
        super(Posterized, self).__init__(threshold, invert_threshold)

    def predict(self, image_path, return_boolean=True, ROI=None):
        """Predict if a given image is posterized

        :param image_path: path to the image
        :type image_path: str
        :param return_boolean: whether to return the result as a
                               float between 0 and 1 or as a boolean
                               (threshold is given to the class)
        :type return_boolean: bool
        :param ROI: possible region of interest as a 4-tuple
                    (x0, y0, width, height), None if not needed
        :returns: the prediction as a bool or float depending on the
                  return_boolean parameter
        """
        image = read_image(image_path, ROI)
        prediction = get_input_vector(image)[0]

        if prediction > 0.004:
            prediction = 1.0
        elif prediction < 0.0:
            prediction = 0.0
        else:
            prediction = linear_normalize(prediction, 0.0, 0.004).item(0)

        if return_boolean:
            return self.boolean_result(prediction)
        return prediction

max_stars_repo_name: kwrobert/qualipy

max_issues_repo_name: kwrobert/qualipy

max_forks_repo_name: kwrobert/qualipy


text:
import numpy as np

N_THRESHOLD = 15

def reset_record():
    record = {
        'n_threshold': N_THRESHOLD,
        'eq_correct_count': np.zeros(N_THRESHOLD),
        'not_eq_correct_count': np.zeros(N_THRESHOLD),
        'eq_count': 0,
        'not_eq_count': 0,
        'threshold': 0.1*np.array(range(N_THRESHOLD)),
        'WKDR': np.zeros((N_THRESHOLD, 4))
    }
    return record

def _classify(zA, zB, gt, threshold):
    if zA - zB > threshold:
        res = 1
    elif zA - zB < -threshold:
        res = -1
    else:
        res = 0
    return (res == gt)

def _count_correct(output, target, record):
    for idx in range(len(target)):
        yA = target[idx][0]
        xA = target[idx][1]
        yB = target[idx][2]
        xB = target[idx][3]

        zA = output[yA][xA]
        zB = output[yB][xB]

        gt = target[idx][4]

        for tau_idx in range(record['n_threshold']):
            if _classify(zA, zB, gt, record['threshold'][tau_idx]):
                if gt == 0:
                    record['eq_correct_count'][tau_idx] += 1
                else:
                    record['not_eq_correct_count'][tau_idx] += 1

        if gt == 0:
            record['eq_count'] += 1
        else:
            record['not_eq_count'] += 1

def evaluate(output, target):
    """
    Args:
        Output: np array, N x height x width x 1
        target: np array, N x 800 x 5
    """
    record = reset_record()

    n_iters = output.shape[0]
    for i in range(n_iters):
        _count_correct(output[i], target[i], record)

    max_min = 0
    max_min_k = 1
    for tau_idx in range(record['n_threshold']):
        record['WKDR'][tau_idx][0] = record['threshold'][tau_idx]
        record['WKDR'][tau_idx][1] = float(record['eq_correct_count'][tau_idx]+record['not_eq_correct_count'][tau_idx])/float(record['eq_count']+record['not_eq_count'])
        record['WKDR'][tau_idx][2] = float(record['eq_correct_count'][tau_idx])/float(record['eq_count'])
        record['WKDR'][tau_idx][3] = float(record['not_eq_correct_count'][tau_idx])/float(record['not_eq_count'])

        if min(record['WKDR'][tau_idx][2], record['WKDR'][tau_idx][3]) > max_min:
            max_min = min(record['WKDR'][tau_idx][2], record['WKDR'][tau_idx][3])
            max_min_k = tau_idx

    return 1 - max_min


text:
#!/usr/bin/env python
import csv
import math
import string
import numpy as np
from std_msgs.msg import String
from math import sqrt, atan, sin, cos, pi, radians
from datetime import datetime, timedelta

def lla2ned(lat_deg, long_deg, altitude, lat0_deg, long0_deg, altitude0):
    deg2rad = np.pi/180
    rad2deg = 180/np.pi

    lat_rad   = lat_deg * deg2rad
    long_rad  = long_deg * deg2rad
    lat0_rad  = lat0_deg * deg2rad
    long0_rad = long0_deg * deg2rad

    #From vik 2012 - page 4 - table 1.1
    re        = 6378137.0  # [m]
    rp        = 6356752.0  # [m]
    rad_curve = pow(re,2) / (math.sqrt(pow(re,2) * pow(math.cos(lat_rad),2) + pow(rp,2) * pow(math.sin(lat_rad),2)))  #N in vik

    #LLA to ECEF
    xe = (rad_curve + altitude) * math.cos(lat_rad) * math.cos(long_rad)
    ye = (rad_curve + altitude) * math.cos(lat_rad) * math.sin(long_rad)
    ze = ((pow(rp, 2) / pow(re, 2)) * rad_curve + altitude) * math.sin(lat_rad)

    xe0 = (rad_curve + altitude0) * math.cos(lat0_rad) * math.cos(long0_rad)
    ye0 = (rad_curve + altitude0) * math.cos(lat0_rad) * math.sin(long0_rad)
    ze0 = ((pow(rp, 2) / pow(re, 2)) * rad_curve + altitude0) * math.sin(lat0_rad)

    # ECEF to NED
    cosPhi    = math.cos(lat0_rad)
    sinPhi    = math.sin(lat0_rad)
    cosLambda = math.cos(long0_rad)
    sinLambda = math.sin(long0_rad)

    dist   = cosLambda * xe + sinLambda * ye
    uNorth = -sinPhi * dist + cosPhi * ze
    vEast  = -sinLambda * xe + cosLambda * ye
    wDown  = -(cosPhi * dist + sinPhi * ze)

    # ECEF0 to NED0
    cosPhi0    = cosPhi
    sinPhi0    = sinPhi
    cosLambda0 = cosLambda
    sinLambda0 = sinLambda

    dist0   = cosLambda0 * xe0 + sinLambda0 * ye0
    vEast0  = -sinLambda0 * xe0 + cosLambda0 * ye0
    wDown0  = -(cosPhi0 * dist0 + sinPhi0 * ze0)
    uNorth0 = -sinPhi0 * dist0 + cosPhi0 * ze0

    # Compute difference
    N = uNorth - uNorth0  # [m]
    E = vEast - vEast0    # [m]
    D = wDown - wDown0    # [m]

    NED = [N, E, D]
    return NED

'''
def lla2ned(latitude, longitude, latitude_ref, longitude_ref):
	R = 6378137.0#Earth radius, WGS84
	f = 0.003352810664747#Flattening factor, WGS84

	d_lat = radians(latitude - latitude_ref)
	d_long = radians(longitude - longitude_ref)

	R_N = R/(sqrt(1.0-(2.0*f-f*f)*(sin(radians(latitude_ref))**2)))
	R_M = R_N*(1-(2*f-f*f))/(1-(2*f-f*f)*(sin(radians(latitude_ref))**2))

	dN = d_lat/(atan(1.0/R_M))
	dE = d_long/(atan(1.0/(R_N*cos(radians(latitude_ref)))))

	return [dN, dE, 0]
'''
def main():
    ### Piren
    lat0_deg  = 63.4389029083
    long0_deg = 10.39908278
    altitude0 = 39.923

    lla_points = np.loadtxt('track_points_fix_elev.csv', delimiter=',', skiprows=1, dtype=String)

    #lla_points = np.delete(lla_points, np.s_[2:6],1)
    #lla_points = np.delete(lla_points, np.s_[5:29],1)

    a = np.asarray([ ["N","E","D","time"]])
    with open('GPS_fix_elev.csv','ab') as fd:
        np.savetxt(fd, a, delimiter=",", fmt="%s")

    for row in lla_points:
        NED = lla2ned(float(row[1]), float(row[0]), float(row[5]) ,lat0_deg, long0_deg, altitude0)

        #string = row[6]
        #new_time = string[0:11] + str(int(string[11:12]) + 2) + string[13:len(string)]
        date_str = row[6]
        date = datetime.strptime(date_str, '%Y/%m/%d %H:%M:%S+%f')
        new_time = date + timedelta(hours=2) #+ timedelta(seconds=4)

        b = np.asarray([ [NED, new_time]])

        with open('GPS_fix_elev.csv','ab') as fd:
            np.savetxt(fd, b, delimiter=",", fmt="%s")

if __name__ == '__main__':
    main()

max_stars_repo_name: trineoo/stereo

max_issues_repo_name: trineoo/stereo

max_forks_repo_name: trineoo/stereo


text:
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

import tensorflow as tf
from tensorflow.keras.layers.experimental import preprocessing

import numpy as np
import time

def get_sentance(chars,seed,model,states,next_char,result):
    for n in range(chars):
      next_char, states = model.generate_one_step(next_char, states=states)
      result.append(next_char)
    return tf.strings.join(result)[0].numpy().decode("utf-8")

max_stars_repo_name: DanielFlockhart/RNN-language-Model

max_issues_repo_name: DanielFlockhart/RNN-language-Model

max_forks_repo_name: DanielFlockhart/RNN-language-Model


text:
import unittest

import numpy as np
import tensorflow as tf
import torch

import fastestimator as fe
from fastestimator.test.unittest_util import is_equal

class TestArgmax(unittest.TestCase):
    def test_argmax_np_input_axis_0(self):
        n = np.array([[2, 7, 5], [9, 1, 3], [4, 8, 2]])
        obj1 = fe.backend.argmax(n, axis=0)
        obj2 = np.array([1, 2, 0])
        self.assertTrue(is_equal(obj1, obj2))

    def test_argmax_np_input_axis_1(self):
        n = np.array([[2, 7, 5], [9, 1, 3], [4, 8, 2]])
        obj1 = fe.backend.argmax(n, axis=1)
        obj2 = np.array([1, 0, 1])
        self.assertTrue(is_equal(obj1, obj2))

    def test_argmax_tf_input_axis_0(self):
        t = tf.constant([[2, 7, 5], [9, 1, 3], [4, 8, 2]])
        obj1 = fe.backend.argmax(t, axis=0)
        obj2 = tf.constant([1, 2, 0])
        self.assertTrue(is_equal(obj1, obj2))

    def test_argmax_tf_input_axis_1(self):
        t = tf.constant([[2, 7, 5], [9, 1, 3], [4, 8, 2]])
        obj1 = fe.backend.argmax(t, axis=1)
        obj2 = tf.constant([1, 0, 1])
        self.assertTrue(is_equal(obj1, obj2))

    def test_argmax_torch_input_axis_0(self):
        p = torch.tensor([[2, 7, 5], [9, 1, 3], [4, 8, 2]])
        obj1 = fe.backend.argmax(p, axis=0)
        obj2 = torch.tensor([1, 2, 0])
        self.assertTrue(is_equal(obj1, obj2))

    def test_argmax_torch_input_axis_1(self):
        p = torch.tensor([[2, 7, 5], [9, 1, 3], [4, 8, 2]])
        obj1 = fe.backend.argmax(p, axis=1)
        obj2 = torch.tensor([1, 0, 1])
        self.assertTrue(is_equal(obj1, obj2))

max_stars_repo_name: hanskrupakar/fastestimator

max_issues_repo_name: hanskrupakar/fastestimator

max_forks_repo_name: hanskrupakar/fastestimator


text:
import math
import os
import argparse
import numpy as np
import tensorflow as tf
from cleverhans.loss import CrossEntropy
from cleverhans.attacks import FastGradientMethod

from data import make_dataset, read_image
from data import decode_png, resize
from util import init_logger
from model import InceptionV3

def eval_acc(sess, logits, labels, num_batches, is_training, dataset_init_op):
  sess.run(dataset_init_op)

  pred = tf.equal(tf.to_int32(tf.argmax(logits, 1)),
                  tf.to_int32(tf.argmax(labels, 1)))

  num_correct, num_samples = 0, 0
  for batch in range(num_batches):
    pred_val = sess.run(pred, {is_training: False})
    num_correct += pred_val.sum()
    num_samples += pred_val.shape[0]

  acc = num_correct / num_samples
  return acc

def main(args):
  logger = init_logger(args.run_name)

  # Datasets
  img_height, img_width, _ = InceptionV3.SHAPE

  def prep_func(f, x, y):
    x = read_image(x)
    x = decode_png(x)
    x = resize(x, img_height, img_width)
    return f, x, y

  trn_ds = make_dataset(args.train_dir, args.batch_size, prep_func,
                        shuffle=True, repeat=True, add_filenames=True)
  val_ds = make_dataset(args.train_dir, args.batch_size, prep_func,
                        shuffle=False, repeat=False, add_filenames=True)
  tst_ds = make_dataset(args.train_dir, args.batch_size, prep_func,
                        shuffle=False, repeat=False, add_filenames=True)

  num_classes = len(trn_ds.labels_map)

  it = tf.data.Iterator.from_structure(
    trn_ds.dataset.output_types, trn_ds.dataset.output_shapes)

  num_trn_batches = int(math.ceil(float(trn_ds.size) / args.batch_size))
  num_val_batches = int(math.ceil(float(val_ds.size) / args.batch_size))
  num_tst_batches = int(math.ceil(float(tst_ds.size) / args.batch_size))

  trn_init_op = it.make_initializer(trn_ds.dataset)
  val_init_op = it.make_initializer(val_ds.dataset)
  tst_init_op = it.make_initializer(tst_ds.dataset)

  # Filename, input image and corrsponding one hot encoded label
  f, x, y = it.get_next()

  sess = tf.Session()

  # Model and logits
  is_training = tf.placeholder(dtype=tf.bool)
  model = InceptionV3(nb_classes=num_classes, is_training=is_training)
  logits = model.get_logits(x)

  attacks_ord = {
    'inf': np.inf,
    '1': 1,
    '2': 2
  }

  # FGM attack
  attack_params = {
    'eps': args.eps,
    'clip_min': 0.0,
    'clip_max': 1.0,
    'ord': attacks_ord[args.ord],
  }
  attack = FastGradientMethod(model, sess)

  # Learning rate with exponential decay
  global_step = tf.Variable(0, trainable=False)
  global_step_update_op = tf.assign(global_step, tf.add(global_step, 1))
  lr = tf.train.exponential_decay(
    args.initial_lr, global_step, args.lr_decay_steps,
    args.lr_decay_factor, staircase=True)

  cross_entropy = CrossEntropy(model, attack=attack,
                               smoothing=args.label_smth,
                               attack_params=attack_params,
                               adv_coeff=args.adv_coeff)
  loss = cross_entropy.fprop(x, y)

  # Gradients clipping
  opt = tf.train.RMSPropOptimizer(learning_rate=lr, decay=args.opt_decay,
                                  epsilon=1.0)
  gvs = opt.compute_gradients(loss)
  clip_min, clip_max = -args.grad_clip, args.grad_clip

  capped_gvs = []
  for g, v in gvs:
    capped_g = tf.clip_by_value(g, clip_min, clip_max) \
      if g is not None else tf.zeros_like(v)
    capped_gvs.append((capped_g, v))

  train_op = opt.apply_gradients(capped_gvs)

  saver = tf.train.Saver()
  global_init_op = tf.global_variables_initializer()

  if args.load_model and args.restore_path:
    saver.restore(sess, args.restore_path)
    logger.info("Model restored from: ".format(args.restore_path))

  with sess.as_default():
    sess.run(global_init_op)

    best_val_acc = -1
    for epoch in range(args.num_epochs):
      logger.info("Epoch: {:04d}/{:04d}".format(epoch + 1, args.num_epochs))
      sess.run(trn_init_op)

      for batch in range(num_trn_batches):
        loss_np, lr_np, _ = sess.run([loss, lr, train_op],
                                     feed_dict={is_training: True})
        logger.info("Batch: {:04d}/{:04d}, loss: {:.05f}, lr: {:.05f}"
          .format(batch + 1, num_trn_batches, loss_np, lr_np))

      logger.info("Epoch completed...")

      sess.run(global_step_update_op)
      val_acc = eval_acc(sess, logits, y, num_val_batches,
                         is_training, val_init_op)
      logger.info("Validation set accuracy: {:.05f}".format(val_acc))

      if best_val_acc < val_acc:
        output_path = saver.save(sess, args.model_path)
        logger.info("Model was successfully saved: {}".format(output_path))
        best_val_acc = val_acc
        pass

    tst_acc = eval_acc(sess, logits, y, num_tst_batches,
                       is_training, tst_init_op)
    logger.info("Test set accuracy: {:.05f}".format(tst_acc))

if __name__ == '__main__':
  tf.set_random_seed(2019)

  parser = argparse.ArgumentParser()

  parser.add_argument('--run-name', required=True, type=str)
  parser.add_argument('--train-dir', required=True, type=str)
  parser.add_argument('--val-dir', required=True, type=str)
  parser.add_argument('--test-dir', required=True, type=str)
  parser.add_argument('--model-path', required=True, type=str)
  parser.add_argument('--restore-model', default=False, action='store_true')
  parser.add_argument('--restore-path', default=None, type=str)
  parser.add_argument('--batch-size', default=64, type=int)
  parser.add_argument('--num-epochs', default=100, type=int)
  parser.add_argument('--initial-lr', default=0.045, type=float)
  parser.add_argument('--lr-decay-factor', default=0.94, type=float)
  parser.add_argument('--lr-decay-steps', default=2, type=int)
  parser.add_argument('--opt-decay', default=0.9, type=float)
  parser.add_argument('--grad-clip', default=2.0, type=float)
  parser.add_argument('--label-smth', default=1.0, type=float)
  parser.add_argument('--adv-coeff', default=0.5, type=float)
  parser.add_argument('--device', default='2', choices=['0', '1', '2'], type=str)
  parser.add_argument('--eps', default=0.0784, type=float)
  parser.add_argument('--ord', default='inf', choices=['inf', '1', '2'], type=str)

  args = parser.parse_args()

  os.environ['CUDA_VISIBLE_DEVICES'] = args.device

  main(args)


text:
# -*- coding: utf-8 -*-
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the scalar plugin summary API."""

import glob
import os

import numpy as np
import six
import tensorflow as tf

from tensorboard.compat import tf2
from tensorboard.compat.proto import summary_pb2
from tensorboard.plugins.scalar import metadata
from tensorboard.plugins.scalar import summary
from tensorboard.util import tensor_util

try:
    tf2.__version__  # Force lazy import to resolve
except ImportError:
    tf2 = None

try:
    tf.compat.v1.enable_eager_execution()
except AttributeError:
    # TF 2.0 doesn't have this symbol because eager is the default.
    pass

class SummaryBaseTest(object):
    def scalar(self, *args, **kwargs):
        raise NotImplementedError()

    def test_tag(self):
        self.assertEqual("a", self.scalar("a", 1).value[0].tag)
        self.assertEqual("a/b", self.scalar("a/b", 1).value[0].tag)

    def test_metadata(self):
        pb = self.scalar("a", 1.13)
        summary_metadata = pb.value[0].metadata
        plugin_data = summary_metadata.plugin_data
        self.assertEqual(summary_metadata.summary_description, "")
        self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME)
        content = summary_metadata.plugin_data.content
        # There's no content, so successfully parsing is fine.
        metadata.parse_plugin_metadata(content)

    def test_explicit_description(self):
        description = "The first letter of the alphabet."
        pb = self.scalar("a", 1.13, description=description)
        summary_metadata = pb.value[0].metadata
        self.assertEqual(summary_metadata.summary_description, description)
        plugin_data = summary_metadata.plugin_data
        self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME)
        content = summary_metadata.plugin_data.content
        # There's no content, so successfully parsing is fine.
        metadata.parse_plugin_metadata(content)

    def test_float_value(self):
        pb = self.scalar("a", 1.13)
        value = tensor_util.make_ndarray(pb.value[0].tensor).item()
        self.assertEqual(float, type(value))
        self.assertNear(1.13, value, 1e-6)

    def test_int_value(self):
        # ints should be valid, but converted to floats.
        pb = self.scalar("a", 113)
        value = tensor_util.make_ndarray(pb.value[0].tensor).item()
        self.assertEqual(float, type(value))
        self.assertNear(113.0, value, 1e-6)

    def test_bool_value(self):
        # bools should be valid, but converted to floats.
        pb = self.scalar("a", True)
        value = tensor_util.make_ndarray(pb.value[0].tensor).item()
        self.assertEqual(float, type(value))
        self.assertEqual(1.0, value)

    def test_string_value(self):
        # Use str.* in regex because PY3 numpy refers to string arrays using
        # length-dependent type names in the format "str%d" % (32 * len(str)).
        with six.assertRaisesRegex(
            self, (ValueError, tf.errors.UnimplementedError), r"Cast str.*float"
        ):
            self.scalar("a", np.array("113"))

    def test_requires_rank_0(self):
        with six.assertRaisesRegex(self, ValueError, r"Expected scalar shape"):
            self.scalar("a", np.array([1, 1, 3]))

class SummaryV1PbTest(SummaryBaseTest, tf.test.TestCase):
    def scalar(self, *args, **kwargs):
        return summary.pb(*args, **kwargs)

    def test_tag(self):
        self.assertEqual("a/scalar_summary", self.scalar("a", 1).value[0].tag)
        self.assertEqual(
            "a/b/scalar_summary", self.scalar("a/b", 1).value[0].tag
        )

class SummaryV1OpTest(SummaryBaseTest, tf.test.TestCase):
    def scalar(self, *args, **kwargs):
        return summary_pb2.Summary.FromString(
            summary.op(*args, **kwargs).numpy()
        )

    def test_tag(self):
        self.assertEqual("a/scalar_summary", self.scalar("a", 1).value[0].tag)
        self.assertEqual(
            "a/b/scalar_summary", self.scalar("a/b", 1).value[0].tag
        )

    def test_scoped_tag(self):
        with tf.name_scope("scope"):
            self.assertEqual(
                "scope/a/scalar_summary", self.scalar("a", 1).value[0].tag
            )

class SummaryV2PbTest(SummaryBaseTest, tf.test.TestCase):
    def scalar(self, *args, **kwargs):
        return summary.scalar_pb(*args, **kwargs)

class SummaryV2OpTest(SummaryBaseTest, tf.test.TestCase):
    def setUp(self):
        super(SummaryV2OpTest, self).setUp()
        if tf2 is None:
            self.skipTest("v2 summary API not available")

    def scalar(self, *args, **kwargs):
        return self.scalar_event(*args, **kwargs).summary

    def scalar_event(self, *args, **kwargs):
        self.write_scalar_event(*args, **kwargs)
        event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), "*")))
        self.assertEqual(len(event_files), 1)
        events = list(tf.compat.v1.train.summary_iterator(event_files[0]))
        # Expect a boilerplate event for the file_version, then the summary one.
        self.assertEqual(len(events), 2)
        # Delete the event file to reset to an empty directory for later calls.
        # TODO(nickfelt): use a unique subdirectory per writer instead.
        os.remove(event_files[0])
        return events[1]

    def write_scalar_event(self, *args, **kwargs):
        kwargs.setdefault("step", 1)
        writer = tf2.summary.create_file_writer(self.get_temp_dir())
        with writer.as_default():
            summary.scalar(*args, **kwargs)
        writer.close()

    def test_scoped_tag(self):
        with tf.name_scope("scope"):
            self.assertEqual("scope/a", self.scalar("a", 1).value[0].tag)

    def test_step(self):
        event = self.scalar_event("a", 1.0, step=333)
        self.assertEqual(333, event.step)

    def test_default_step(self):
        try:
            tf2.summary.experimental.set_step(333)
            # TODO(nickfelt): change test logic so we can just omit `step` entirely.
            event = self.scalar_event("a", 1.0, step=None)
            self.assertEqual(333, event.step)
        finally:
            # Reset to default state for other tests.
            tf2.summary.experimental.set_step(None)

class SummaryV2OpGraphTest(SummaryV2OpTest, tf.test.TestCase):
    def write_scalar_event(self, *args, **kwargs):
        kwargs.setdefault("step", 1)
        # Hack to extract current scope since there's no direct API for it.
        with tf.name_scope("_") as temp_scope:
            scope = temp_scope.rstrip("/_")

        @tf2.function
        def graph_fn():
            # Recreate the active scope inside the defun since it won't propagate.
            with tf.name_scope(scope):
                summary.scalar(*args, **kwargs)

        writer = tf2.summary.create_file_writer(self.get_temp_dir())
        with writer.as_default():
            graph_fn()
        writer.close()

if __name__ == "__main__":
    tf.test.main()


text:
#! /usr/bin/python
# -*- coding: utf-8 -*-

"""Backup old render method, from the original Python repository."""

import matplotlib.pyplot as plt
import numpy as np

try:
    from .erosion import get_downhill, get_water_flow
except ImportError:
    from erosion import _get_downhill, _get_water_flow

def _get_sinks(self, downhill):
    sinks = downhill.copy()
    water = self.elevations <= self.sea_level
    sink_list = np.where((sinks == -1) & ~water & ~self.is_edge)[0]
    sinks[sink_list] = sink_list
    sinks[water] = -1

    while True:
        new_sinks = sinks.copy()
        new_sinks[~water] = sinks[sinks[~water]]
        new_sinks[sinks == -1] = -1
        if np.all(new_sinks == sinks):
            break
        sinks = new_sinks
    return sinks

def _find_lowest_sill(self, sinks):
    h = 10000
    edges = np.where(
        (sinks != -1) &
        np.any((sinks[self.adj_mat] == -1) & self.adj_mat != -1, 1)
    )[0]

    best_uv = None, None
    for u in edges:
        for v in self.adj_vertices[u]:
            if v == -1:
                continue
            if sinks[v] == -1:
                new_h = max(self.elevations[v], self.elevations[u])
                if new_h < h:
                    h = new_h
                    best_uv = u, v
    assert h < 10000
    u, v = best_uv
    return h, u, v

def _infill(self):
    """Infill the sink vertices (lower than all neighbours)."""

    elevations = self.elevations
    downhill = self.downhill
    for _ in range(1, 11):
        print('#', _)
        sinks = _get_sinks(self, downhill)
        if np.all(sinks == -1):
            return
        h, u, v = _find_lowest_sill(self, sinks)
        sink = sinks[u]
        if downhill[v] != -1:
            elevations[v] = self.elevations[downhill[v]] + 1e-5
        sink_elevations = self.elevations[sinks == sink]
        h = np.where(sink_elevations < h, h + 0.001 * (h - sink_elevations), sink_elevations) + 1e-5
        self.elevations[sinks == sink] = h
        self.downhill = get_downhill(self, np.append(self.elevations, self.sea_level))

def render_process_old(self):
    ext_elevations = np.append(self.elevations, self.sea_level)
    self.downhill = get_downhill(self, ext_elevations)
    _infill(self)
    self.flow = get_water_flow(self, self.downhill, undersea=0.0)

def _plot_rivers_old(self):
    river_percentile = 93
    vertices = self.vertices
    downhill = self.downhill
    flow = self.flow
    flow_threshold = np.percentile(flow, river_percentile)
    flow_max = np.max(flow)
    river_indicator = flow > flow_threshold
    river_indicator &= (downhill != -1)     # Only use valid downhill.

    river_vertices = np.arange(self.n_vertices)[river_indicator]
    downhill_vertices = downhill[river_indicator]

    river_from_points = vertices[river_vertices]
    river_to_points = vertices[downhill_vertices]
    river_flows = flow[river_indicator]
    for (x1, y1), (x2, y2), river_flow in zip(river_from_points, river_to_points, river_flows):
        flow_width = 2.0 * (river_flow - flow_threshold) / (flow_max - flow_threshold)
        plt.plot([x1, x2], [y1, y2], 'b-', linewidth=flow_width)

max_stars_repo_name: fyabc/Toys

max_issues_repo_name: fyabc/Toys

max_forks_repo_name: fyabc/Toys