text:
# --------------------------------------------------------'
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# --------------------------------------------------------
# BEIT: BERT Pre-Training of Image Transformers (https://arxiv.org/abs/2106.08254)
# Github source: https://github.com/microsoft/unilm/tree/master/beit
# Copyright (c) 2021 Microsoft
# Licensed under The MIT License [see LICENSE for details]

import random
import math
import numpy as np

class MaskingGenerator(object):
    def __init__(self,
                 input_size,
                 num_masking_patches,
                 min_num_patches=4,
                 max_num_patches=None,
                 min_aspect=0.3,
                 max_aspect=None):
        if not isinstance(input_size, tuple):
            input_size = (input_size, ) * 2
        self.height, self.width = input_size

        self.num_patches = self.height * self.width
        self.num_masking_patches = num_masking_patches

        self.min_num_patches = min_num_patches
        self.max_num_patches = num_masking_patches if max_num_patches is None else max_num_patches

        max_aspect = max_aspect or 1 / min_aspect
        self.log_aspect_ratio = (math.log(min_aspect), math.log(max_aspect))

    def get_shape(self):
        return self.height, self.width

    def _mask(self, mask, max_mask_patches):
        delta = 0
        for attempt in range(10):
            target_area = random.uniform(self.min_num_patches, max_mask_patches)
            aspect_ratio = math.exp(random.uniform(*self.log_aspect_ratio))
            h = int(round(math.sqrt(target_area * aspect_ratio)))
            w = int(round(math.sqrt(target_area / aspect_ratio)))
            if w < self.width and h < self.height:
                top = random.randint(0, self.height - h)
                left = random.randint(0, self.width - w)

                num_masked = mask[top:top + h, left:left + w].sum()
                # Overlap
                if 0 < h * w - num_masked <= max_mask_patches:
                    for i in range(top, top + h):
                        for j in range(left, left + w):
                            if mask[i, j] == 0:
                                mask[i, j] = 1
                                delta += 1

                if delta > 0:
                    break
        return delta

    def __call__(self):
        mask = np.zeros(shape=self.get_shape(), dtype=np.int)
        mask_count = 0
        while mask_count < self.num_masking_patches:
            max_mask_patches = self.num_masking_patches - mask_count
            max_mask_patches = min(max_mask_patches, self.max_num_patches)

            delta = self._mask(mask, max_mask_patches)
            if delta == 0:
                break
            else:
                mask_count += delta

        return mask

class RandomMaskingGenerator(object):
    def __init__(self, input_size, mask_ratio):
        if not isinstance(input_size, tuple):
            input_size = (input_size, ) * 2

        self.height, self.width = input_size

        self.num_patches = self.height * self.width
        self.num_mask = int(mask_ratio * self.num_patches)

    def __call__(self):
        mask = np.hstack([
            np.zeros(self.num_patches - self.num_mask),
            np.ones(self.num_mask),
        ])
        np.random.shuffle(mask)
        return mask  # [196]


text:
import sympy
from sympy import Function as sympy_Function
from sympy.core import power as sympy_power
from sympy.core import mod as sympy_mod
import numpy as np
from numpy.random import randn, rand

import brian2.units.unitsafefunctions as unitsafe

from .base import Function

__all__ = ['FunctionWrapper', 'DEFAULT_FUNCTIONS']

class RandnFunction(Function):
    '''
    A specifier for the randn function, allowing its use both in Python and
    C++ code (e.g. for stochastic variables). In Python, a `randn()` call will
    return `N` random numbers (e.g. the size of the `NeuronGroup`), in C++ it
    will return a single number.

    Parameters
    ----------
    N : int
        The number of random numbers generated at a time.
    '''
    def __init__(self):
        Function.__init__(self, pyfunc=randn, arg_units=[], return_unit=1)

    def __call__(self, vectorisation_idx):
        return randn(len(vectorisation_idx))

    def code_cpp(self, language, var):

        support_code = '''
        #define BUFFER_SIZE 1024
        // A randn() function that returns a single random number. Internally
        // it asks numpy's randn function for N (e.g. the number of neurons)
        // random numbers at a time and then returns one number from this
        // buffer.
        // It needs a reference to the numpy_randn object (the original numpy
        // function), because this is otherwise only available in
        // compiled_function (where is is automatically handled by weave).
        //
        double _call_randn(py::object& numpy_randn) {
            static PyArrayObject *randn_buffer = NULL;
            static double *buf_pointer = NULL;
            static npy_int curbuffer = 0;
            if(curbuffer==0)
            {
                if(randn_buffer) Py_DECREF(randn_buffer);
                py::tuple args(1);
                args[0] = BUFFER_SIZE;
                randn_buffer = (PyArrayObject *)PyArray_FromAny(numpy_randn.call(args), NULL, 1, 1, 0, NULL);
                buf_pointer = (double*)PyArray_GETPTR1(randn_buffer, 0);
            }
            double number = buf_pointer[curbuffer];
            curbuffer = curbuffer+1;
            if (curbuffer == BUFFER_SIZE)
                // This seems to be safer then using (curbuffer + 1) % BUFFER_SIZE, we might run into
                // an integer overflow for big networks, otherwise.
                curbuffer = 0;
            return number;
        }
        '''.replace('%VAR%', var)

        hashdefine_code = '''
        #define _randn(_vectorisation_idx) _call_randn(_python_randn)
        '''

        return {'support_code': support_code,
                'hashdefine_code': hashdefine_code}

    def on_compile_cpp(self, namespace, language, var):
        pass

class RandFunction(Function):
    '''
    A specifier for the rand function, allowing its use both in Python and
    C++ code (e.g. for synaptic connectivity). In Python, a
    `rand(vectorisation_idx)` call will return `len(vectorisation_idx)` random
    numbers (e.g. the size of the `NeuronGroup`), in C++ it will return a
    single number.
    '''
    def __init__(self):
        Function.__init__(self, pyfunc=rand, arg_units=[], return_unit=1)

    def __call__(self, vectorisation_idx):
        return rand(len(vectorisation_idx))

    def code_cpp(self, language, var):

        support_code = '''
        double _rand(int vectorisation_idx)
        {
	        return (double)rand()/RAND_MAX;
        }
        '''

        return {'support_code': support_code,
                'hashdefine_code': ''}

    def on_compile_cpp(self, namespace, language, var):
        pass

class ClipFunction(Function):
    '''
    A specifier for the clip function, allowing its use both in Python and
    C++ code. Only works for scalar ``a_min`` and ``a_max`` arguments or other
    state variables.

    '''
    def __init__(self):
        Function.__init__(self, pyfunc=np.clip, arg_units=[None, None, None],
                          return_unit=lambda u1, u2, u3: u1)

    def __call__(self, array, a_min, a_max):
        return np.clip(array, a_min, a_max)

    def code_cpp(self, language, var):

        support_code = '''
        double _clip(const float value, const float a_min, const float a_max)
        {
	        if (value < a_min)
	            return a_min;
	        if (value > a_max)
	            return a_max;
	        return value;
        }
        '''

        return {'support_code': support_code,
                'hashdefine_code': ''}

    def on_compile_cpp(self, namespace, language, var):
        pass

class FunctionWrapper(Function):
    '''
    Simple wrapper for functions that have exist both in numpy and C++
    (possibly with a different name, for example ``acos`` vs. ``arccos``).

    Parameters
    ----------
    pyfunc : function
        The numpy function (or its unitsafe wrapper)
    py_name : str, optional
        The name of the python function, in case it is not unambiguously
        defined by `pyfunc`. For example, the ``abs`` function in numpy is
        actually named ``absolute`` but we want to use the name ``abs``.
    cpp_name : str, optional
        The name of the corresponding function in C++, in case it is different.
    sympy_func : sympy function, optional
        The corresponding sympy function, if it exists.
    arg_units : list of `Unit`, optional
        The expected units of the arguments, ``None`` for arguments that can
        have arbitrary units. Needs only to be specified if the `pyfunc`
        function does not specify this already (e.g. via a `check_units`
        decorator)
    return_unit : `Unit` or callable, optional
        The unit of the return value of this function. Either a fixed `Unit`,
        or a function of the units of the arguments, e.g.
        ``lambda u : u **0.5`` for a square root function. Needs only to be
        specified if the `pyfunc` function does not specify this already (e.g.
        via a `check_units` decorator)
    '''
    # TODO: How to make this easily extendable for other languages?
    def __init__(self, pyfunc, py_name=None, cpp_name=None, sympy_func=None,
                 arg_units=None, return_unit=None):
        Function.__init__(self, pyfunc, sympy_func, arg_units=arg_units,
                          return_unit=return_unit)
        if py_name is None:
            py_name = pyfunc.__name__
        self.py_name = py_name
        self.cpp_name = cpp_name

    def code_cpp(self, language, var):
        if self.cpp_name is None:
            hashdefine_code = ''
        else:
            hashdefine_code = '#define {python_name} {cpp_name}'.format(python_name=self.py_name,
                                                                        cpp_name=self.cpp_name)
        return {'support_code': '',
                'hashdefine_code': hashdefine_code}

# sympy does not have a log10 function, so let's define one
class log10(sympy_Function):
    nargs = 1

    @classmethod
    def eval(cls, args):
        return sympy.functions.elementary.exponential.log(args, 10)

def _get_default_functions():

    functions = {
                # numpy functions that have the same name in numpy and math.h
                'cos': FunctionWrapper(unitsafe.cos,
                                       sympy_func=sympy.functions.elementary.trigonometric.cos),
                'sin': FunctionWrapper(unitsafe.sin,
                                       sympy_func=sympy.functions.elementary.trigonometric.sin),
                'tan': FunctionWrapper(unitsafe.tan,
                                       sympy_func=sympy.functions.elementary.trigonometric.tan),
                'cosh': FunctionWrapper(unitsafe.cosh,
                                        sympy_func=sympy.functions.elementary.hyperbolic.cosh),
                'sinh': FunctionWrapper(unitsafe.sinh,
                                        sympy_func=sympy.functions.elementary.hyperbolic.sinh),
                'tanh': FunctionWrapper(unitsafe.tanh,
                                        sympy_func=sympy.functions.elementary.hyperbolic.tanh),
                'exp': FunctionWrapper(unitsafe.exp,
                                       sympy_func=sympy.functions.elementary.exponential.exp),
                'log': FunctionWrapper(unitsafe.log,
                                       sympy_func=sympy.functions.elementary.exponential.log),
                'log10': FunctionWrapper(unitsafe.log10,
                                         sympy_func=log10),
                'sqrt': FunctionWrapper(np.sqrt,
                                        sympy_func=sympy.functions.elementary.miscellaneous.sqrt,
                                        arg_units=[None], return_unit=lambda u: u**0.5),
                'ceil': FunctionWrapper(np.ceil,
                                        sympy_func=sympy.functions.elementary.integers.ceiling,
                                        arg_units=[None], return_unit=lambda u: u),
                'floor': FunctionWrapper(np.floor,
                                         sympy_func=sympy.functions.elementary.integers.floor,
                                         arg_units=[None], return_unit=lambda u: u),
                # numpy functions that have a different name in numpy and math.h
                'arccos': FunctionWrapper(unitsafe.arccos,
                                          cpp_name='acos',
                                          sympy_func=sympy.functions.elementary.trigonometric.acos),
                'arcsin': FunctionWrapper(unitsafe.arcsin,
                                          cpp_name='asin',
                                          sympy_func=sympy.functions.elementary.trigonometric.asin),
                'arctan': FunctionWrapper(unitsafe.arctan,
                                          cpp_name='atan',
                                          sympy_func=sympy.functions.elementary.trigonometric.atan),
                'abs': FunctionWrapper(np.abs, py_name='abs',
                                       cpp_name='fabs',
                                       sympy_func=sympy.functions.elementary.complexes.Abs,
                                       arg_units=[None], return_unit=lambda u: u),
                'mod': FunctionWrapper(np.mod, py_name='mod',
                                       cpp_name='fmod',
                                       sympy_func=sympy_mod.Mod,
                                       arg_units=[None, None], return_unit=lambda u,v : u),
                'rand': RandFunction(),
                'randn': RandnFunction(),
                'clip': ClipFunction()
                }

    return functions

DEFAULT_FUNCTIONS = _get_default_functions()

ext: py

reason: import numpy,from numpy,import sympy,from sympy


text:
'''挑选特征'''

from sklearn.linear_model import RandomizedLasso
from sklearn.ensemble import RandomForestRegressor
import numpy as np

def featureSelect(select_fun, train_data, train_label, threshold_num=0, alpha=0.000015):
    X = train_data  # train data
    Y = train_label  # train label
    feture_names = list(train_data.columns)  # 现有的特征名字
    importance_features_list = []

    if select_fun == 'MeanDecreaseImpurity':
        '''平均不纯度减少 mean decrease impurity'''
        rf = RandomForestRegressor(random_state=2019)
        rf.fit(X, Y)
        feature_score = sorted(zip(feture_names, map(lambda x: round(x, 4), rf.feature_importances_)))
    elif select_fun == 'StabilitySelection':
        '''稳定性选择 StabilitySelection'''
        rlasso = RandomizedLasso(alpha, random_state=2019)  # alpha太大会导致所有特征都会为0，为1最好
        rlasso.fit(X, Y)
        feature_score = sorted(zip(feture_names, map(lambda x: round(x, 4), rlasso.scores_)))
    else:
        importance_features_list = ['MeanDecreaseImpurity', 'StabilitySelection', 'RecursiveFeatureElimination',
                                    'MeanDecreaseAccuracy']
        print("可选挑选特征的方法名:", importance_features_list)
        return importance_features_list

    for item in feature_score:
        if item[1] > threshold_num:
            importance_features_list.append(item[0])
        else:
            continue

    return importance_features_list

max_stars_repo_name: cookcv/competition

max_issues_repo_name: sunnykai/competition

max_forks_repo_name: sunnykai/competition

reason: import numpy


text:
# Copyright (C) 2021 Jason Brown
#
# SPDX-License-Identifier: MIT
"""Class file for the motion controller"""

import numpy as np

tau = np.pi * 2

class MotionCS:
    """
    All MotionCS methods will return an array of left and right motor velocities
    To be used via robot.motors.velocities = MotionControlStrategies.some_method(*args)
    """

    # Overwritten in robot.py
    max_f_speed = None

    @staticmethod
    def combine_and_scale(forward, rotation, angle=None):
        # Reverse rotation if angle is negative for symmetric strategies
        if angle is not None:
            rotation *= np.sign(angle)

        # Combine velocities
        left_speed = forward + rotation
        right_speed = forward - rotation

        # Scale so max is +-1
        if abs(left_speed) >= 1:
            left_speed = left_speed / abs(left_speed)
            right_speed = right_speed / abs(left_speed)

        if abs(right_speed) >= 1:
            right_speed = right_speed / abs(right_speed)
            left_speed = left_speed / abs(right_speed)

        return np.array([left_speed, right_speed])

    @staticmethod
    def dual_pid(prime_quantity=None, deriv_quantity=None, prime_pid=None, derivative_pid=None,
                 req_prime_quantity=None, req_deriv_quantity=None) -> float:
        """This method uses 2 PID's to control a quantity based on a threshold. If only one of the required measurements
        is given it just uses a single PID.

        The PID takes a quantity and its derivative (i.e. distance and velocity). When close to closing the error it
        uses the prime quantity but when it's far it uses the derivative quantity

        With derivative control it acts as a feed-forward PID (open loop + closed loop) as this is a powerful and
        responsive control method for situations such as these.

        It switches quantity when the output of the prime quantity controller is equal to the output of the derivative
        quantity controller, this is found using pid.query which gets output without affecting pid state.

        Args:
            prime_quantity (float): Our current prime quantity. E.g. distance, m or angle, rad
            deriv_quantity (float): Our current derivative quantity. E.g. forward speed, m/s or
                rotational speed, rad/s
            prime_pid (PID): Prime quantity PID controller
            derivative_pid (PID): Derivative quantity PID controller
            req_prime_quantity (float): Required prime quantity
            req_deriv_quantity (float): Required derivative quantity

        Returns:
            float: The drive fraction for the given inputs
        """

        if prime_quantity is None and deriv_quantity is None:
            drive = 0
        else:
            if deriv_quantity is not None:
                derivative_based_drive = derivative_pid.query(req_deriv_quantity - deriv_quantity) + req_deriv_quantity
                if prime_quantity is not None:
                    if np.sign(derivative_based_drive) * prime_pid.query(prime_quantity - req_prime_quantity) <= abs(derivative_based_drive):
                        drive = prime_pid.step(prime_quantity - req_prime_quantity)
                    else:
                        drive = derivative_pid.step(req_deriv_quantity - deriv_quantity) + req_deriv_quantity
                else:
                    drive = derivative_pid.step(req_deriv_quantity - deriv_quantity) + req_deriv_quantity
            else:
                drive = prime_pid.step(prime_quantity - req_prime_quantity)

        return drive

    @staticmethod
    def linear_dual_pid(distance=None, forward_speed=None, distance_pid=None, forward_speed_pid=None,
                        required_distance=0, required_forward_speed=None, angle_attenuation=True, angle=None):
        """Wrapper for dual_pid to make linear control simpler"""

        # If required_forward_speed is not specified use max
        req_forward_speed = MotionCS.max_f_speed if required_forward_speed is None else required_forward_speed

        # Attenuate speeds and distance based on angle so robot doesn't zoom off when not facing target
        if angle_attenuation and angle is not None:
            attenuation_factor = (np.cos(angle) ** 5) if abs(angle) <= tau / 4 else 0
            distance *= attenuation_factor
            required_distance *= attenuation_factor
            req_forward_speed *= attenuation_factor

        # Convert from actual robot velocities to drive fraction equivalent
        required_forward_drive = req_forward_speed / MotionCS.max_f_speed
        forward_speed_drive_eq = None if forward_speed is None else forward_speed / MotionCS.max_f_speed

        return MotionCS.dual_pid(prime_quantity=distance, deriv_quantity=forward_speed_drive_eq,
                                 prime_pid=distance_pid, derivative_pid=forward_speed_pid,
                                 req_prime_quantity=required_distance, req_deriv_quantity=required_forward_drive)


text:
from astropy.io import fits
import numpy as np
import warnings

def load_header(header, extnum=0):
    """
    Attempt to load a header specified as a header, a string pointing to a FITS
    file, or a string pointing to a Header text file, or a string that contains
    the actual header
    """
    if hasattr(header,'get'):
        # dict or Header
        return fits.Header(header)
    try:
        # assume fits file first
        return fits.getheader(header, extnum=extnum)
    except IOError:
        # assume header textfile
        try:
            return fits.Header().fromtextfile(header)
        except IOError:
            return fits.Header().fromstring(header,'\n')

def load_data(data, extnum=0):
    """
    Attempt to load a header specified as a header, a string pointing to a FITS
    file, or a string pointing to a Header text file, or a string that contains
    the actual header
    """
    if isinstance(data,np.ndarray):
        return data
    try:
        # assume fits file first
        return fits.getdata(data, extnum=extnum)
    except IOError:
        # assume an HDU
        if hasattr(data,'data') and isinstance(data.data,np.ndarray):
            return data.data
        elif isinstance(data, fits.HDUList):
            warnings.warn("Using 0'th extension of HDUlist to extract data")
            return data[0].data

def get_cd(wcs, n=1):
    """
    Get the value of the change in world coordinate per pixel across a linear axis.
    Defaults to wcs.wcs.cd if present.  Does not support rotated headers (e.g.,
    with nonzero CDm_n where m!=n)
    """

    if hasattr(wcs.wcs,'cd'):
        if wcs.wcs.cd[n-1,n-1] != 0:
            return wcs.wcs.cd[n-1,n-1]
    else:
        return wcs.wcs.get_cdelt()[n-1]

reason: import numpy,from astropy


text:
import os
import time
import random
import json
import numpy as np
from collections import OrderedDict

import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt

############ record curve #####################
def init_dict(keys):
    d = {}
    for key in keys:
        d[key] = []
    return d

############ record curve #####################
def save_dict(info_dict, theme, save_dir):

    with open(os.path.join(save_dir, 'infodict-{}.json'.format(theme)), 'w') as f:
        f.write(json.dumps(info_dict))

############ record curve #####################
def curve_save(x, y, tag, yaxis, theme, save_dir):
    color = ['r', 'b', 'g', 'c', 'orange', 'lightsteelblue', 'cornflowerblue', 'indianred']
    fig = plt.figure()
    # ax = plt.subplot()
    plt.grid(linestyle='-', linewidth=0.5, alpha=0.5)
    if isinstance(tag, list):
        for i, (y_term, tag_term) in enumerate(zip(y, tag)):
            plt.plot(x, y_term, color[i], label=tag_term, alpha=0.7)
    else:
        plt.plot(x, y, color[0], label=tag, alpha=0.7)
    plt.xlabel('Epoch', fontsize=12)
    plt.ylabel(yaxis, fontsize=12)
    plt.title('curve-{}'.format(theme), fontsize=14)
    plt.legend()

    fig.savefig(os.path.join(save_dir,'curve-{}.png'.format(theme)), dpi=300)
    plt.close('all') ####

def time_mark():
    time_now = int(time.time())
    time_local = time.localtime(time_now)

    dt = time.strftime('%Y%m%d-%H%M%S', time_local)
    return(dt)

def print_cz(str, f=None):
    if f is not None:
        print(str, file=f)
        if random.randint(0, 20) < 3:
            f.flush()
    print(str)

def update_lr(lr, epoch, lr_step=20, lr_gamma=0.5):
    """Sets the learning rate to the initial LR decayed by 0.5 every 20 epochs"""
    lr = lr * (lr_gamma ** (epoch // lr_step))
    return lr

#########
def remove_oldfile(dirname, file_keyword):
    for filename in os.listdir(dirname):
        if file_keyword in filename:
            os.remove(os.path.join(dirname, filename))
#########
def metric_macro(metric_clients):
    result = np.mean(metric_clients)
    return result

def overall_performance(dirname, F1_m, AUC_m, iters, tag='_overall_'):
    f = open(
        os.path.join(
            dirname,
            '{}-F1-m{:.2f}-AUC-m{:.2f}-iters{:d}.txt'.format(
                    tag,
                    F1_m,
                    AUC_m,
                    iters
                )
        ),
        'w'
    )
    f.close()

max_stars_repo_name: CityU-AIM-Group/PRR-Imbalance

max_issues_repo_name: CityU-AIM-Group/PRR-Imbalance

max_forks_repo_name: CityU-AIM-Group/PRR-Imbalance


text:
"""
Environment for 2D car driving.
You can customize this script in a way you want.

View more on [莫烦Python] : https://morvanzhou.github.io/tutorials/

Requirement:
pyglet >= 1.2.4
numpy >= 1.12.1
"""
import numpy as np
import pyglet

pyglet.clock.set_fps_limit(10000)

class CarEnv(object):
    n_sensor = 5
    action_dim = 1
    state_dim = n_sensor
    viewer = None
    viewer_xy = (500, 500)
    sensor_max = 150.
    start_point = [450, 300]
    speed = 50.
    dt = 0.1

    def __init__(self, discrete_action=False):
        self.is_discrete_action = discrete_action
        if discrete_action:
            self.actions = [-1, 0, 1]
        else:
            self.action_bound = [-1, 1]

        self.terminal = False
        # node1 (x, y, r, w, l),
        self.car_info = np.array([0, 0, 0, 20, 40], dtype=np.float64)   # car coordination
        self.obstacle_coords = np.array([
            [120, 120],
            [380, 120],
            [380, 380],
            [120, 380],
        ])
        self.sensor_info = self.sensor_max + np.zeros((self.n_sensor, 3))  # n sensors, (distance, end_x, end_y)

    def step(self, action):
        if self.is_discrete_action:
            action = self.actions[action]
        else:
            action = np.clip(action, *self.action_bound)[0]
        self.car_info[2] += action * np.pi/30  # max r = 6 degree
        self.car_info[:2] = self.car_info[:2] + \
                            self.speed * self.dt * np.array([np.cos(self.car_info[2]), np.sin(self.car_info[2])])

        self._update_sensor()
        s = self._get_state()
        r = -1 if self.terminal else 0
        return s, r, self.terminal

    def reset(self):
        self.terminal = False
        self.car_info[:3] = np.array([*self.start_point, -np.pi/2])
        self._update_sensor()
        return self._get_state()

    def render(self):
        if self.viewer is None:
            self.viewer = Viewer(*self.viewer_xy, self.car_info, self.sensor_info, self.obstacle_coords)
        self.viewer.render()

    def sample_action(self):
        if self.is_discrete_action:
            a = np.random.choice(list(range(3)))
        else:
            a = np.random.uniform(*self.action_bound, size=self.action_dim)
        return a

    def set_fps(self, fps=30):
        pyglet.clock.set_fps_limit(fps)

    def _get_state(self):
        s = self.sensor_info[:, 0].flatten()/self.sensor_max
        return s

    def _update_sensor(self):
        cx, cy, rotation = self.car_info[:3]

        n_sensors = len(self.sensor_info)
        sensor_theta = np.linspace(-np.pi / 2, np.pi / 2, n_sensors)
        xs = cx + (np.zeros((n_sensors, ))+self.sensor_max) * np.cos(sensor_theta)
        ys = cy + (np.zeros((n_sensors, ))+self.sensor_max) * np.sin(sensor_theta)
        xys = np.array([[x, y] for x, y in zip(xs, ys)])    # shape (5 sensors, 2)

        # sensors
        tmp_x = xys[:, 0] - cx
        tmp_y = xys[:, 1] - cy
        # apply rotation
        rotated_x = tmp_x * np.cos(rotation) - tmp_y * np.sin(rotation)
        rotated_y = tmp_x * np.sin(rotation) + tmp_y * np.cos(rotation)
        # rotated x y
        self.sensor_info[:, -2:] = np.vstack([rotated_x+cx, rotated_y+cy]).T

        q = np.array([cx, cy])
        for si in range(len(self.sensor_info)):
            s = self.sensor_info[si, -2:] - q
            possible_sensor_distance = [self.sensor_max]
            possible_intersections = [self.sensor_info[si, -2:]]

            # obstacle collision
            for oi in range(len(self.obstacle_coords)):
                p = self.obstacle_coords[oi]
                r = self.obstacle_coords[(oi + 1) % len(self.obstacle_coords)] - self.obstacle_coords[oi]
                if np.cross(r, s) != 0:  # may collision
                    t = np.cross((q - p), s) / np.cross(r, s)
                    u = np.cross((q - p), r) / np.cross(r, s)
                    if 0 <= t <= 1 and 0 <= u <= 1:
                        intersection = q + u * s
                        possible_intersections.append(intersection)
                        possible_sensor_distance.append(np.linalg.norm(u*s))

            # window collision
            win_coord = np.array([
                [0, 0],
                [self.viewer_xy[0], 0],
                [*self.viewer_xy],
                [0, self.viewer_xy[1]],
                [0, 0],
            ])
            for oi in range(4):
                p = win_coord[oi]
                r = win_coord[(oi + 1) % len(win_coord)] - win_coord[oi]
                if np.cross(r, s) != 0:  # may collision
                    t = np.cross((q - p), s) / np.cross(r, s)
                    u = np.cross((q - p), r) / np.cross(r, s)
                    if 0 <= t <= 1 and 0 <= u <= 1:
                        intersection = p + t * r
                        possible_intersections.append(intersection)
                        possible_sensor_distance.append(np.linalg.norm(intersection - q))

            distance = np.min(possible_sensor_distance)
            distance_index = np.argmin(possible_sensor_distance)
            self.sensor_info[si, 0] = distance
            self.sensor_info[si, -2:] = possible_intersections[distance_index]
            if distance < self.car_info[-1]/2:
                self.terminal = True

class Viewer(pyglet.window.Window):
    color = {
        'background': [1]*3 + [1]
    }
    fps_display = pyglet.clock.ClockDisplay()
    bar_thc = 5

    def __init__(self, width, height, car_info, sensor_info, obstacle_coords):
        super(Viewer, self).__init__(width, height, resizable=False, caption='2D car', vsync=False)  # vsync=False to not use the monitor FPS
        self.set_location(x=80, y=10)
        pyglet.gl.glClearColor(*self.color['background'])

        self.car_info = car_info
        self.sensor_info = sensor_info

        self.batch = pyglet.graphics.Batch()
        background = pyglet.graphics.OrderedGroup(0)
        foreground = pyglet.graphics.OrderedGroup(1)

        self.sensors = []
        line_coord = [0, 0] * 2
        c = (73, 73, 73) * 2
        for i in range(len(self.sensor_info)):
            self.sensors.append(self.batch.add(2, pyglet.gl.GL_LINES, foreground, ('v2f', line_coord), ('c3B', c)))

        car_box = [0, 0] * 4
        c = (249, 86, 86) * 4
        self.car = self.batch.add(4, pyglet.gl.GL_QUADS, foreground, ('v2f', car_box), ('c3B', c))

        c = (134, 181, 244) * 4
        self.obstacle = self.batch.add(4, pyglet.gl.GL_QUADS, background, ('v2f', obstacle_coords.flatten()), ('c3B', c))

    def render(self):
        pyglet.clock.tick()
        self._update()
        self.switch_to()
        self.dispatch_events()
        self.dispatch_event('on_draw')
        self.flip()

    def on_draw(self):
        self.clear()
        self.batch.draw()
        # self.fps_display.draw()

    def _update(self):
        cx, cy, r, w, l = self.car_info

        # sensors
        for i, sensor in enumerate(self.sensors):
            sensor.vertices = [cx, cy, *self.sensor_info[i, -2:]]

        # car
        xys = [
            [cx + l / 2, cy + w / 2],
            [cx - l / 2, cy + w / 2],
            [cx - l / 2, cy - w / 2],
            [cx + l / 2, cy - w / 2],
        ]
        r_xys = []
        for x, y in xys:
            tempX = x - cx
            tempY = y - cy
            # apply rotation
            rotatedX = tempX * np.cos(r) - tempY * np.sin(r)
            rotatedY = tempX * np.sin(r) + tempY * np.cos(r)
            # rotated x y
            x = rotatedX + cx
            y = rotatedY + cy
            r_xys += [x, y]
        self.car.vertices = r_xys

if __name__ == '__main__':
    np.random.seed(1)
    env = CarEnv()
    env.set_fps(30)
    for ep in range(20):
        s = env.reset()
        # for t in range(100):
        while True:
            env.render()
            s, r, done = env.step(env.sample_action())
            if done:
                break


text:
#!/usr/bin/env python
"""Plot data for deepmod methylation calling"""
########################################################################
# File: plot_deepmod_methylation_calling.py
#  executable: plot_deepmod_methylation_calling.py
#
# Author: Andrew Bailey
# History: Created 04/02/19
########################################################################

import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt

from py3helpers.classification import ClassificationMetrics
from py3helpers.utils import list_dir, time_it
from py3helpers.seq_tools import ReverseComplement

class CustomAmbiguityPositions(object):
    def __init__(self, ambig_filepath):
        """Deal with ambiguous positions from a tsv ambiguity position file with the format of
        contig  position            strand  change_from change_to
        'name'  0 indexed position   +/-    C           E

        :param ambig_filepath: path to ambiguity position file"""

        self.ambig_df = self.parseAmbiguityFile(ambig_filepath)

    @staticmethod
    def parseAmbiguityFile(ambig_filepath):
        """Parses a 'ambiguity position file' that should have the format:
            contig  position    strand  change_from change_to

        :param ambig_filepath: path to ambiguity position file
        """
        return pd.read_csv(ambig_filepath, sep='\t',
                           usecols=(0, 1, 2, 3, 4),
                           names=["contig", "position", "strand", "change_from", "change_to"],
                           dtype={"contig": np.str,
                                  "position": np.int,
                                  "strand": np.str,
                                  "change_from": np.str,
                                  "change_to": np.str})

    def getForwardSequence(self, contig, raw_sequence):
        """Edit 'raw_sequence' given a ambiguity positions file. Assumes raw_sequence is forward direction( 5'-3')
        :param contig: which contig the sequence belongs (aka header)
        :param raw_sequence: raw nucleotide sequence
        :return: edited nucleotide sequence
        """
        return self._get_substituted_sequence(contig, raw_sequence, "+")

    def getBackwardSequence(self, contig, raw_sequence):
        """Edit 'raw_sequence' given a ambiguity positions file, Assumes raw_sequence is forward direction( 5'-3')
        :param contig: which contig the sequence belongs (aka header)
        :param raw_sequence: raw nucleotide sequence
        :return: edited nucleotide sequence
        """
        rc = ReverseComplement()
        raw_sequence = rc.complement(raw_sequence)
        return self._get_substituted_sequence(contig, raw_sequence, "-")

    def _get_substituted_sequence(self, contig, raw_sequence, strand):
        """Change the given raw nucleotide sequence using the edits defined in the positions file

        :param contig: name of contig to find
        :param raw_sequence: nucleotide sequence (note: this is note edited in this function)
        :param strand: '+' or '-' to indicate strand
        """
        contif_df = self._get_contig_positions(contig, strand)
        raw_sequence = list(raw_sequence)
        for _, row in contif_df.iterrows():
            if raw_sequence[row["position"]] != row["change_from"]:
                raise RuntimeError(
                    "[CustomAmbiguityPositions._get_substituted_sequence]Illegal substitution requesting "
                    "change from %s to %s, row: %s" % (raw_sequence[row["position"]], row["change_to"], row))
            raw_sequence[row["position"]] = row["change_to"]
        return "".join(raw_sequence)

    def _get_contig_positions(self, contig, strand):
        """Get all unique locations within the positions file

        :param contig: name of contig to find
        :param strand: '+' or '-' to indicate strand
        """
        df = self.ambig_df.loc[
            (self.ambig_df["contig"] == contig) & (self.ambig_df["strand"] == strand)].drop_duplicates()
        assert len(df['position']) == len(set(df['position'])), "Multiple different changes for a single position. {}" \
            .format(df['position'])
        return df

def parse_deepmod_bed(deepmod_bed_path):
    """Parse the summary chromosome bed file output from deepmod

    eg: Chromosome 2 3 C 8 + 2 3 0,0,0 8 0 0
    """

    return pd.read_csv(deepmod_bed_path, sep=" ", header=None,
                       usecols=(0, 1, 3, 5, 9, 10, 11),
                       names=["contig", "start_position", "base", "strand", "n_reads", "modification_percentage",
                              "n_mod_calls"],
                       dtype={"contig": np.str,
                              "start_position": np.int,
                              "base": np.str,
                              "strand": np.str,
                              "n_reads": np.int,
                              "modification_percentage": np.int,
                              "n_mod_calls": np.int})

def aggregate_deepmod_data(deepmod_output_dir):
    deepmod_beds = list_dir(deepmod_output_dir, ext="bed")
    deepmod_bed_data = []
    for bed in deepmod_beds:
        data = parse_deepmod_bed(bed)
        data["E"] = (data["modification_percentage"] / 100)
        data["C"] = 1 - (data["modification_percentage"] / 100)
        deepmod_bed_data.append(data)

    return pd.concat(deepmod_bed_data)

def print_confusion_matrix(tp, fp, fn, tn):
    precision = tp / (tp + fp)
    false_discovery_rate = fp / (tp + fp)

    false_omission_rate = fn / (tn + fn)
    negative_predictive_value = tn / (tn + fn)

    true_positive_rate_recall = tp / (tp + fn)
    false_negative_rate = fn / (tp + fn)

    false_positive_rate = fp / (fp + tn)
    true_negative_rate_specificity = tn / (tn + fp)

    positive_likelihood_ratio = true_positive_rate_recall / false_positive_rate
    negative_likelihood_ratio = false_negative_rate / true_negative_rate_specificity

    diagnostic_odds_ratio = positive_likelihood_ratio / negative_likelihood_ratio

    f1_score = 2 * ((precision * true_positive_rate_recall) / (precision + true_positive_rate_recall))

    return (np.asarray(
        [[tp, fp, precision, false_discovery_rate],
         [fn, tn, false_omission_rate, negative_predictive_value],
         [true_positive_rate_recall, false_positive_rate, positive_likelihood_ratio, diagnostic_odds_ratio],
         [false_negative_rate, true_negative_rate_specificity, negative_likelihood_ratio, f1_score]]))

def plot_confusion_matrix(tp, fp, fn, tn, classes=("E", "C"), title="Confusion Matrix",
                          output_path=None, normalize=False):
    """Plot the confusion matrix with the information of each box explained
    :param classes: classes to label axis
    :param normalize: option to normalize output of confusion matrix
    :param tp: true positives
    :param fp: false positives
    :param fn: false negatives
    :param tn: true negatives
    :param title: title of the plot
    :param output_path: place to save plot
    """
    data = np.asarray([[tp, fp], [fn, tn]])
    # total = sum([tp, fp, fn, tn])
    # precision = tp / (tp + fp)
    # false_discovery_rate = fp / (tp + fp)
    #
    # false_omission_rate = fn / (tn + fn)
    # negative_predictive_value = tn / (tn + fn)
    #
    # true_positive_rate_recall = tp / (tp + fn)
    # false_negative_rate = fn / (tp + fn)
    #
    # false_positive_rate = fp / (fp + tn)
    # true_negative_rate_specificity = tn / (tn + fp)
    #
    # positive_likelihood_ratio = true_positive_rate_recall / false_positive_rate
    # negative_likelihood_ratio = false_negative_rate / true_negative_rate_specificity
    #
    # diagnostic_odds_ratio = positive_likelihood_ratio / negative_likelihood_ratio
    #
    # f1_score = 2 * ((precision * true_positive_rate_recall) / (precision + true_positive_rate_recall))

    if not title:
        if normalize:
            title = 'Normalized confusion matrix'
        else:
            title = 'Confusion matrix, without normalization'

    # Compute confusion matrix
    n_data = data.astype('float') / data.sum(axis=1)[:, np.newaxis]
    fig, ax = plt.subplots()
    im = ax.imshow(data, interpolation='nearest', cmap=plt.cm.Blues)
    ax.figure.colorbar(im, ax=ax)
    # We want to show all ticks...
    ax.set(xticks=np.arange(data.shape[1]),
           yticks=np.arange(data.shape[0]),
           xticklabels=classes, yticklabels=classes,
           title=title,
           ylabel='True label',
           xlabel='Predicted label')

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")

    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = data.max() / 2.
    for i in range(data.shape[0]):
        for j in range(data.shape[1]):
            ax.text(j, i, format(data[i, j], 'd') + format(n_data[i, j], '.2f'),
                    ha="center", va="center",
                    color="white" if data[i, j] > thresh else "black")

    fig.tight_layout()

    if output_path is not None:
        plt.savefig(output_path)
    else:
        plt.show()
    return True

def main():
    cpg_positions_file = "/Users/andrewbailey/data/references/ecoli/CG_ecoli_k12_mg1655_C_E.positions"
    modified_deepmod_output_dir = "/Users/andrewbailey/CLionProjects/DeepMod/ecoli_pcr_MSssI_R9"
    canonical_deepmod_output_dir = "/Users/andrewbailey/CLionProjects/DeepMod/ecoli_pcr_MSssI_R9"

    output_dir = "/Users/andrewbailey/CLionProjects/modification_detection_pipeline/output_dir/plotting_output/"
    log_file_path = os.path.join(output_dir, "confusion_matrices_file.txt")
    cpg_positions = CustomAmbiguityPositions(cpg_positions_file)

    canonical_data = aggregate_deepmod_data(canonical_deepmod_output_dir)
    canonical_data["E_label"] = 0
    canonical_data["C_label"] = 1

    modified_data = aggregate_deepmod_data(modified_deepmod_output_dir)
    modified_data["E_label"] = 1
    modified_data["C_label"] = 0

    tps = 0
    fps = 0
    tns = 0
    fns = 0
    all_data = []
    with open(log_file_path, "w") as log_file:
        chromosomes = set(modified_data["contig"]) | set(canonical_data["contig"])
        strands = set(modified_data["strand"]) | set(canonical_data["strand"])
        for chromosome in chromosomes:
            for strand in strands:
                # get positions for strand and contig
                sc_positions = cpg_positions.ambig_df.loc[(cpg_positions.ambig_df["strand"] == strand) &
                                                          (cpg_positions.ambig_df["contig"] == chromosome)]
                # pare data sets for specific contig and strand to get positions that are cpgs
                mod_sc_data = modified_data.loc[(modified_data["contig"] == chromosome) &
                                                (modified_data["strand"] == strand)]
                mod_methylation_calls = mod_sc_data.loc[mod_sc_data["start_position"].isin(sc_positions["position"])]
                canon_sc_data = canonical_data.loc[(canonical_data["contig"] == chromosome) &
                                                   (canonical_data["strand"] == strand)]
                canon_methylation_calls = canon_sc_data.loc[
                    canon_sc_data["start_position"].isin(sc_positions["position"])]

                # per site
                n_negative_calls = sum(canon_methylation_calls["n_reads"])
                n_false_negatives = sum(canon_methylation_calls["n_mod_calls"])
                n_true_negatives = n_negative_calls - n_false_negatives

                n_positive_calls = sum(mod_methylation_calls["n_reads"])
                n_true_positives = sum(mod_methylation_calls["n_mod_calls"])
                n_false_positives = n_positive_calls - n_true_positives

                tps += n_true_positives
                fps += n_false_positives
                tns += n_true_negatives
                fns += n_false_negatives
                print("Chromosome {} strand {}:".format(chromosome, strand), file=log_file)
                print("Per-call confusion matrix", file=log_file)
                print(print_confusion_matrix(n_true_positives, n_false_positives, n_false_negatives, n_true_negatives),
                      file=log_file)
                plot_confusion_matrix(n_true_positives, n_false_positives, n_false_negatives, n_true_negatives,
                                      normalize=True,
                                      output_path=os.path.join(output_dir, "per_call_{}_{}_confusion_matrix.png".format(strand, chromosome)),
                                      title="Per call CpG Normalized Confusion Matrix {}{}".format(strand, chromosome))

                # per genomic position
                chr_strand_data = pd.concat([canon_methylation_calls, mod_methylation_calls])
                label_data = chr_strand_data.loc[:, ['C_label', "E_label"]]
                prediction_data = chr_strand_data.loc[:, ['C', "E"]]
                label_data.rename(columns={'C_label': 'C', "E_label": "E"}, inplace=True)
                cmh = ClassificationMetrics(label_data, prediction_data)
                cmh.plot_roc("E", os.path.join(output_dir, "per_genomic_site_{}_{}_roc.png".format(chromosome, strand)))
                cmh.plot_precision_recall("E", os.path.join(output_dir,
                                                            "per_genomic_site_{}_{}_"
                                                            "precision_recall.png".format(chromosome, strand)))
                print("Per-genomic-site confusion matrix", file=log_file)
                print(cmh.confusion_matrix(), file=log_file)

                all_data.append(chr_strand_data)

        print("All Chromosomes both strands:", file=log_file)
        print("Per-call confusion matrix", file=log_file)
        print(print_confusion_matrix(tps, fps, fns, tns),
              file=log_file)
        plot_confusion_matrix(tps, fps, fns, tns,
                              normalize=True,
                              output_path=os.path.join(output_dir,
                                                       "all_calls_confusion_matrix.png"),
                              title="All calls CpG "
                                    "Normalized Confusion Matrix")
        all_data = pd.concat(all_data)
        label_data = all_data.loc[:, ['C_label', "E_label"]]
        prediction_data = all_data.loc[:, ['C', "E"]]
        label_data.rename(columns={'C_label': 'C', "E_label": "E"}, inplace=True)
        cmh = ClassificationMetrics(label_data, prediction_data)
        cmh.plot_roc("E", os.path.join(output_dir, "per_genomic_site_all_chromosomes_roc.png"))
        cmh.plot_precision_recall("E", os.path.join(output_dir,
                                                    "per_genomic_site_all_chromosomes"
                                                    "precision_recall.png"))
        print("Per-genomic-site confusion matrix", file=log_file)
        print(cmh.confusion_matrix(), file=log_file)

if __name__ == '__main__':
    print(time_it(main))


text:
# Copyright (c) 2012 - 2014 the GPy Austhors (see AUTHORS.txt)
# Licensed under the BSD 3-clause license (see LICENSE.txt)

import numpy as np
from ..core import GP
from .. import likelihoods
from .. import kern

class GPRegression(GP):
    """
    Gaussian Process model for regression

    This is a thin wrapper around the models.GP class, with a set of sensible defaults

    :param X: input observations
    :param Y: observed values
    :param kernel: a GPy kernel, defaults to rbf
    :param Norm normalizer: [False]
    :param noise_var: the noise variance for Gaussian likelhood, defaults to 1.

        Normalize Y with the norm given.
        If normalizer is False, no normalization will be done
        If it is None, we use GaussianNorm(alization)

    .. Note:: Multiple independent outputs are allowed using columns of Y

    """

    def __init__(self, X, Y, kernel=None, Y_metadata=None, normalizer=True, noise_var=1., mean_function=None):

        if kernel is None:
            kernel = kern.RBF(X.shape[1])

        likelihood = likelihoods.Gaussian(variance=noise_var)

        super(GPRegression, self).__init__(X, Y, kernel, likelihood, name='GP regression', Y_metadata=Y_metadata, normalizer=normalizer, mean_function=mean_function)

    @staticmethod
    def from_gp(gp):
        from copy import deepcopy
        gp = deepcopy(gp)
        return GPRegression(gp.X, gp.Y, gp.kern, gp.Y_metadata, gp.normalizer, gp.likelihood.variance.values, gp.mean_function)

    def to_dict(self, save_data=True):
        model_dict = super(GPRegression,self).to_dict(save_data)
        model_dict["class"] = "GPy.models.GPRegression"
        return model_dict

    @staticmethod
    def _from_dict(input_dict, data=None):
        import GPy
        input_dict["class"] = "GPy.core.GP"
        m = GPy.core.GP.from_dict(input_dict, data)
        return GPRegression.from_gp(m)

    def save_model(self, output_filename, compress=True, save_data=True):
        self._save_model(output_filename, compress=True, save_data=True)

max_stars_repo_name: RaulAstudillo/bocf

max_issues_repo_name: RaulAstudillo/bocf

max_forks_repo_name: RaulAstudillo/bocf


text:
# coding: utf8
# copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import sys
import os
import math
import random
import functools
import io
import time
import codecs
import numpy as np
import paddle
import paddle.fluid as fluid
import cv2
from PIL import Image
import copy

from src.utils.config import cfg
from src.models.model_builder import ModelPhase
from .baseseg import BaseSeg

class AdeSeg(BaseSeg):
    def __init__(self,
                 file_list,
                 data_dir,
                 shuffle=False,
                 mode=ModelPhase.TRAIN, base_size=520, crop_size=520, rand_scale=True):
        super(AdeSeg, self).__init__(file_list, data_dir, shuffle, mode, base_size, crop_size, rand_scale)

    def _mask_transform(self, mask):
        target = np.array(mask).astype('int32') - 1
        return target

    def load_image(self, line, src_dir, mode=ModelPhase.TRAIN):
        # original image cv2.imread flag setting
        cv2_imread_flag = cv2.IMREAD_COLOR
        if cfg.DATASET.IMAGE_TYPE == "rgba":
            # If use RBGA 4 channel ImageType, use IMREAD_UNCHANGED flags to
            # reserver alpha channel
            cv2_imread_flag = cv2.IMREAD_UNCHANGED
        #print("line: ", line)
        parts = line.strip().split(cfg.DATASET.SEPARATOR)
        if len(parts) != 2:
            if mode == ModelPhase.TRAIN or mode == ModelPhase.EVAL:
                raise Exception("File list format incorrect! It should be"
                                " image_name{}label_name\\n".format(
                                    cfg.DATASET.SEPARATOR))
            img_name, grt_name = parts[0], None
        else:
            img_name, grt_name = parts[0], parts[1]

        img_path = os.path.join(src_dir, img_name)
        img = self.cv2_imread(img_path, cv2_imread_flag)

        if grt_name is not None:
            grt_path = os.path.join(src_dir, grt_name)
            grt = self.pil_imread(grt_path)
        else:
            grt = None

        if img is None:
            raise Exception(
                "Empty image, src_dir: {}, img: {} & lab: {}".format(
                    src_dir, img_path, grt_path))

        img_height = img.shape[0]
        img_width = img.shape[1]
        #print('img.shape',img.shape)
        if grt is not None:
            grt_height = grt.shape[0]
            grt_width = grt.shape[1]

            if img_height != grt_height or img_width != grt_width:
                raise Exception(
                    "source img and label img must has the same size")
        else:
            if mode == ModelPhase.TRAIN or mode == ModelPhase.EVAL:
                raise Exception(
                    "Empty image, src_dir: {}, img: {} & lab: {}".format(
                        src_dir, img_path, grt_path))

        if len(img.shape) < 3:
            img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)

        grt = self._mask_transform(grt)

        return img, grt, img_name, grt_name

max_stars_repo_name: pkulzb/Research

max_issues_repo_name: pkulzb/Research

max_forks_repo_name: pkulzb/Research


text:
from __future__ import print_function
#import matplotlib
#matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pickle
from matplotlib.patches import Rectangle
from PIL import Image
import numpy as np
#import sys
#sys.path.append('../lib/utils/')
import _init_paths
#from blob import prep_im_for_blob
from datasets.factory import get_imdb

# image source
#img = '/data/repo/py-faster-rcnn/data/VOCdevkit2007/VOC2007/JPEGImages/004622.jpg'

IMDB="ins_train_partial"
CLASSES = ('background',
           'foreground')

# Create figure and axes
#fig,ax = plt.subplots(1)

def resize(im):
    im, im_scale = prep_im_for_blob(im, np.array([[[0, 0, 0]]]), 600,1000)
    im = im/float(255)
    return im, im_scale

def plot():
    # first read an image from the DB
    imdb = get_imdb(IMDB)
    plt.ion() # turn on interactive mode, plt.show is non-blocking
    fig = plt.figure()
    ax = fig.add_subplot(111, aspect='equal')
    #img = imdb.get_image(0)
    # matplotlib.image.AxesImage object
    #imgobj = ax.imshow(img[:,:,::-1]) # BGR to RGB
    #import pdb;pdb.set_trace()
    for i in xrange(imdb.num_images):
        # Load the demo image
        img = imdb.get_image(i)
        label = imdb.get_label(i)
        bbox = imdb.get_bbox(i)
        boxes = bbox['boxes']
        num_boxes = boxes.shape[0]
        # draw rectangle of the image region
        #boxes = np.array([[ 38,  60, 374, 499],
        #           [ 44,  99, 364, 499]], dtype=np.uint16)
        # Display the image
        #imgobj.set_data(img[:,:,::-1])
        ax.clear()
        ax.imshow(img[:,:,::-1]) # BGR to RGB
        ax.set_title('The {}-th image, {} boxes'.format(i, num_boxes))
        plt.show()
        # draw rois
        #for roi in rois:
        #    roi_rect = Rectangle( (roi[0], roi[1]), roi[2]-roi[0], roi[3]-roi[1],
        #           facecolor='red', alpha=0.1,  edgecolor='red', linewidth=1, label='class = {}'.format(roi[0]))
        #    i += 1
        #    ax.add_patch(roi_rect)
        #ax.legend(loc=2, ncol=1, bbox_to_anchor=(1.05, 1))
        #plt.savefig("vis_rois_resize.png", bbox_inches="tight")
        for box in boxes:
            # box: x1,y1,x2,y2
            rect = Rectangle( (box[0], box[1]), box[2]-box[0], box[3]-box[1],
                   alpha=0.2, facecolor="blue", edgecolor="blue")
            patch = ax.add_patch(rect)
        _ = raw_input("Press [enter] to continue.") # wait for input from the user
        #ax.legend(loc=2, ncol=1, bbox_to_anchor=(1.05, 1))
        #plt.savefig("roi_resize.png", bbox_inches="tight")

# plt.close()    # close the figure to show the next one.
plot()

max_stars_repo_name: vincentzhang/faster-rcnn-fcn

max_issues_repo_name: vincentzhang/faster-rcnn-fcn

max_forks_repo_name: vincentzhang/faster-rcnn-fcn


text:
"""
An encoder-decoder structure tries to encode a feature map to a lower dimensional representation
using the encoder network and then reconstructs the feature map with the decoder network.

Mathematically, the encoder: z = f(h_e(x))
                the decoder: x_hat = f(h_d(z))

The encoder and the decoder can be implemented with a neural network which learns the important features
to be encoded and decoded (auto encoder).

Reference: https://medium.com/pytorch/implementing-an-autoencoder-in-pytorch-19baa22647d1
"""

import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
import numpy as np
import matplotlib.pyplot as plt

class AutoEncoder(nn.Module):
    def __init__(self, in_features):
        super(AutoEncoder, self).__init__()

        # encoder layers
        self.in_1 = nn.Linear(in_features=in_features, out_features=256)
        self.in_2 = nn.Linear(in_features=256, out_features=128)

        # decoder layers
        self.out_1 = nn.Linear(in_features=128, out_features=256)
        self.out_2 = nn.Linear(in_features=256, out_features=in_features)

    def forward(self, x):
        x = self.in_1(x)
        x = F.relu(x)
        x = self.in_2(x)
        x = F.relu(x)

        x = self.out_1(x)
        x = F.relu(x)
        x = self.out_2(x)
        x = F.relu(x)

        return x

def train_model(num_epochs):
    # prepare data
    input_size = 784
    tf = transforms.Compose([transforms.ToTensor()])

    train_data = torchvision.datasets.MNIST(
        root='../data',
        train=True,
        transform=tf,
        download=True
    )
    train_loader = torch.utils.data.DataLoader(
        train_data,
        batch_size=64,
        shuffle=True,
        num_workers=0,
        pin_memory=True
    )

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model = AutoEncoder(input_size)

    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
    criterion = nn.MSELoss()

    # train auto encoder
    for epoch in range(num_epochs):
        cum_loss = 0
        for i, (data, label) in enumerate(train_loader):
            data = data.view(-1, input_size).to(device)

            optimizer.zero_grad()
            output = model(data)

            # learn to reconstruct
            loss = criterion(output, data)
            loss.backward()

            optimizer.step()
            cum_loss += loss.item()

            if (i + 1) % 100 == 0:
                print(f"Epoch: {epoch + 1}; Iteration: {i + 1}; Avg loss: {cum_loss / (i+1)}.")

    return model

def evaluate(model):
    input_size = 784
    tf = transforms.Compose([transforms.ToTensor()])
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    test_data = torchvision.datasets.MNIST(
        root='../data',
        train=False,
        transform=tf,
        download=True
    )
    test_loader = torch.utils.data.DataLoader(
        test_data,
        batch_size=64,
        shuffle=True,
        num_workers=0,
        pin_memory=True
    )

    outputs = []

    with torch.no_grad():
        for i, (data, label) in enumerate(test_loader):
            data = data.view(-1, input_size).to(device)
            output = model(data).cpu().numpy()
            outputs.append(output)

    outputs = np.concatenate(outputs, axis=0)
    return outputs

if __name__ == '__main__':
    model = train_model(20)
    res = evaluate(model)

    # visualize a few images
    sample_img = res[0].reshape(28,28)
    plt.imshow(sample_img, interpolation='nearest')
    plt.show()

max_stars_repo_name: BobbyZhouZijian/AI-Algo-Implmentations

max_issues_repo_name: BobbyZhouZijian/AI-Algo-Implmentations

max_forks_repo_name: BobbyZhouZijian/AI-Algo-Implmentations


text:
from . import metrics
from . import matrices
from .pairwise import apply_pairwise_sq, apply_pairwise_rect
from . import numba_tools

__all__ = ['metrics',
		   'apply_pairwise_sq',
           'apply_pairwise_rect',
           'numba_tools',
           'matrices']

max_stars_repo_name: kmayerb/pwseqdist

max_issues_repo_name: kmayerb/pwseqdist

max_forks_repo_name: kmayerb/pwseqdist

reason: import numba


text:
'''
Policy class for computing action from weights and observation vector.
Horia Mania --- hmania@berkeley.edu
Aurelia Guy
Benjamin Recht
'''

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.convert_parameters import vector_to_parameters, parameters_to_vector
from filter import get_filter
import torch.optim as optim
import os
os.environ["CUDA_VISIBLE_DEVICES"] = '0'

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
class Policy(object):

    def __init__(self, policy_params):

        self.ob_dim = policy_params['ob_dim']
        self.ac_dim = policy_params['ac_dim']
        self.weights = np.empty(0)

        # a filter for updating statistics of the observations and normalizing inputs to the policies
        self.observation_filter = get_filter(policy_params['ob_filter'], shape = (self.ob_dim,))
        self.update_filter = True

    def update_weights(self, new_weights):
        self.weights[:] = new_weights[:]
        return

    def get_weights(self):
        return self.weights

    def get_observation_filter(self):
        return self.observation_filter

    def act(self, ob):
        raise NotImplementedError

    def copy(self):
        raise NotImplementedError

class LinearPolicy(Policy):
    """
    Linear policy class that computes action as <w, ob>.
    """

    def __init__(self, policy_params):
        Policy.__init__(self, policy_params)
        self.weights = np.zeros((self.ac_dim, self.ob_dim), dtype = np.float64)

    def act(self, ob):
        ob = self.observation_filter(ob, update=self.update_filter)
        return np.dot(self.weights, ob)

    def get_weights_plus_stats(self):

        mu, std = self.observation_filter.get_stats()
        aux = np.asarray([self.weights, mu, std])
        return aux

class MLP(nn.Module):

    def __init__(self, input, output):
        super(MLP, self).__init__()
        self.fc1 = nn.Linear(input, 32)
        self.fc2 = nn.Linear(32, output)

    def forward(self, x):
        x = self.fc1(x)
        x = F.relu(x)
        # print(x)
        #count = x.detach().numpy()
        #count = np.where(count==0.0)
        #print("COUNT",count[0].shape)
        #print("ZERO {}".format(128-np.sum(np.nonzero(count)[0])))
        x = self.fc2(x)
        return x

class linear(nn.Module):

    def __init__(self, input, output):
        super(linear, self).__init__()
        self.fc1 = nn.Linear(input, output)

    def forward(self, x):
        x = self.fc1(x)
        # print(x)
        #count = x.detach().numpy()
        #count = np.where(count==0.0)
        #print("COUNT",count[0].shape)
        #print("ZERO {}".format(128-np.sum(np.nonzero(count)[0])))
        return x

class MLP_probs(nn.Module):

    def __init__(self, input, output):
        super(MLP_probs, self).__init__()
        self.fc1 = nn.Linear(input, 32)
        self.fc2 = nn.Linear(32, output)

    def forward(self, x):
        x = self.fc1(x)
        x = F.relu(x)
        # print(x)
        #count = x.detach().numpy()
        #count = np.where(count==0.0)
        #print("COUNT",count[0].shape)
        #print("ZERO {}".format(128-np.sum(np.nonzero(count)[0])))
        x = self.fc2(x)
        x = F.softmax(x)
        return x

class BilayerPolicy_softmax(Policy):
    """
    Linear policy class that computes action as <w, ob>.
    """

    def __init__(self, policy_params,trained_weights= None):
        Policy.__init__(self, policy_params)
        self.net = MLP_probs(self.ob_dim, self.ac_dim)
        #lin_policy = np.load('/home/harshit/work/ARS/trained_policies/Policy_Testerbi2/bi_policy_num_plus149.npz')

        #lin_policy = lin_policy.items()[0][1]
        #self.weights=None

        self.weights = parameters_to_vector(self.net.parameters()).detach().double().numpy()
        # if trained_weights is not None:
        #     print("hieohrfoiahfoidanfkjahdfj")
        #     vector_to_parameters(torch.tensor(trained_weights), self.net.parameters())
        #     self.weights = trained_weights

    def update_weights(self, new_weights):
        vector_to_parameters(torch.tensor(new_weights), self.net.parameters())
        return

    def act(self, ob):
        ob = self.observation_filter(ob, update=self.update_filter)
        obs = torch.from_numpy(ob)
        #print(np.argmax(self.net(obs).detach().double().numpy()))
        return np.argmax(self.net(obs).detach().double().numpy())

    def get_weights_plus_stats(self):

        mu, std = self.observation_filter.get_stats()
        #aux = np.asarray([self.weights.detach().double().numpy(), mu, std])
        aux = np.asarray([self.weights, mu, std])

        return aux

class BilayerPolicy(Policy):
    """
    Linear policy class that computes action as <w, ob>.
    """

    def __init__(self, policy_params,trained_weights= None):
        Policy.__init__(self, policy_params)
        self.net = MLP(self.ob_dim, self.ac_dim)
        #lin_policy = np.load('/home/harshit/work/ARS/trained_policies/Policy_Testerbi2/bi_policy_num_plus149.npz')

        #lin_policy = lin_policy.items()[0][1]
        #self.weights=None

        self.weights = parameters_to_vector(self.net.parameters()).detach().double().numpy()
        # if trained_weights is not None:
        #     print("hieohrfoiahfoidanfkjahdfj")
        #     vector_to_parameters(torch.tensor(trained_weights), self.net.parameters())
        #     self.weights = trained_weights

    def update_weights(self, new_weights):
        print("UPDATE")
        vector_to_parameters(torch.tensor(new_weights), self.net.parameters())
        return

    def act(self, ob):
        ob = self.observation_filter(ob, update=self.update_filter)
        obs = torch.from_numpy(ob)
        return self.net(obs).detach().double().numpy()

    def get_weights_plus_stats(self):

        mu, std = self.observation_filter.get_stats()
        #aux = np.asarray([self.weights.detach().double().numpy(), mu, std])
        aux = np.asarray([self.weights, mu, std])

        return aux

class SafeBilayerPolicy(Policy):
    """
    Linear policy class that computes action as <w, ob>.
    """

    def __init__(self, policy_params,trained_weights= None):
        Policy.__init__(self, policy_params)
        self.net = MLP(self.ob_dim, self.ac_dim)
        self.safeQ = MLP(self.ob_dim+self.ac_dim, 1)
        #lin_policy = np.load('/home/harshit/work/ARS/trained_policies/Policy_Testerbi2/bi_policy_num_plus149.npz')

        #lin_policy = lin_policy.items()[0][1]
        #self.weights=None

        self.weights = parameters_to_vector(self.net.parameters()).detach().double().numpy()
        # if trained_weights is not None:
        #     print("hieohrfoiahfoidanfkjahdfj")
        #     vector_to_parameters(torch.tensor(trained_weights), self.net.parameters())
        #     self.weights = trained_weights

    def update_weights(self, new_weights):
        print("UPDATE")
        vector_to_parameters(torch.tensor(new_weights), self.net.parameters())
        return

    def getQ(self,ob,action):
        ob = self.observation_filter(ob, update=self.update_filter)
        # input_to_network = np.append(ob,action).astype(np.float64)
        ob_ = torch.from_numpy(ob).float()
        return self.safeQ(ob_).detach().double().numpy()

    def act(self, ob):
        ob = self.observation_filter(ob, update=self.update_filter)
        obs = torch.from_numpy(ob)
        return self.net(obs).detach().double().numpy()

    def get_weights_plus_stats(self):

        mu, std = self.observation_filter.get_stats()
        #aux = np.asarray([self.weights.detach().double().numpy(), mu, std])
        aux = np.asarray([self.weights, mu, std])

        return aux

class SafeBilayerExplorerPolicy(Policy):
    """
    Linear policy class that computes action as <w, ob>.
    """

    def __init__(self, policy_params,trained_weights= None):
        Policy.__init__(self, policy_params)
        # if trained_weights is not None:
        self.net = MLP(self.ob_dim, self.ac_dim)
        self.safeQ = linear(self.ob_dim,self.ac_dim).to(device)
        self.optimizer = optim.RMSprop(self.safeQ.parameters())
        # else:
        #     self.safeQ = MLP(self.ob_dim,self.ac_dim).to(device)

        #lin_policy = np.load('/home/harshit/work/ARS/trained_policies/Policy_Testerbi2/bi_policy_num_plus149.npz')

        #lin_policy = lin_policy.items()[0][1]
        #self.weights=None

        self.weights = parameters_to_vector(self.net.parameters()).detach().double().numpy()
        if trained_weights is not None:
            self.safeQ.load_state_dict(torch.load(trained_weights))
            self.safeQ.to(device)
        # if trained_weights is not None:
        #     print("hieohrfoiahfoidanfkjahdfj")
        #     vector_to_parameters(torch.tensor(trained_weights), self.net.parameters())
        #     self.weights = trained_weights

    def update_weights(self, new_weights):
        print("UPDATE")
        vector_to_parameters(torch.tensor(new_weights), self.net.parameters())
        return

    def getQ(self,ob):
        # ob = self.observation_filter(ob, update=self.update_filter)
        input_to_network = ob.astype(np.float64)
        input_to_network_ = torch.from_numpy(ob).float().to(device)
        return self.safeQ(input_to_network_).cpu().detach().double().numpy()

    def act(self, ob):
        ob = self.observation_filter(ob, update=self.update_filter)
        obs = torch.from_numpy(ob)
        #print(np.argmax(self.net(obs).detach().double().numpy()))
        return self.net(obs).detach().double().numpy()

    def get_weights_plus_stats(self):

        mu, std = self.observation_filter.get_stats()
        #aux = np.asarray([self.weights.detach().double().numpy(), mu, std])
        aux = np.asarray([self.weights, mu, std])

        return aux

class SafeBilayerDiscretePolicy(Policy):
    """
    Linear policy class that computes action as <w, ob>.
    """

    def __init__(self, policy_params,trained_weights= None):
        Policy.__init__(self, policy_params)
        # if trained_weights is not None:
        self.net = MLP_probs(self.ob_dim, self.ac_dim)
        self.safeQ = MLP(self.ob_dim,self.ac_dim).to(device)
        self.target_net = MLP(self.ob_dim,self.ac_dim).to(device)
        self.target_net.load_state_dict(self.safeQ.state_dict())
        self.target_net.eval()
        self.optimizer = optim.RMSprop(self.safeQ.parameters())
        # else:
        #     self.safeQ = MLP(self.ob_dim,self.ac_dim).to(device)

        #lin_policy = np.load('/home/harshit/work/ARS/trained_policies/Policy_Testerbi2/bi_policy_num_plus149.npz')

        #lin_policy = lin_policy.items()[0][1]
        #self.weights=None

        self.weights = parameters_to_vector(self.net.parameters()).detach().double().numpy()
        # if trained_weights is not None:
        #     print("hieohrfoiahfoidanfkjahdfj")
        #     vector_to_parameters(torch.tensor(trained_weights), self.net.parameters())
        #     self.weights = trained_weights

    def update_weights(self, new_weights):
        print("UPDATE")
        vector_to_parameters(torch.tensor(new_weights), self.net.parameters())
        return

    def getQ(self,ob,action):
        ob = self.observation_filter(ob, update=self.update_filter)
        #input_to_network = np.append(ob,action).astype(np.float64)
        input_to_network_ = torch.from_numpy(ob).float()
        return self.safeQ(input_to_network_).detach().double().numpy()[action]

    def act(self, ob):
        ob = self.observation_filter(ob, update=self.update_filter)
        obs = torch.from_numpy(ob)
        #print(np.argmax(self.net(obs).detach().double().numpy()))
        return self.net(obs).detach().double().numpy()

    def get_weights_plus_stats(self):

        mu, std = self.observation_filter.get_stats()
        #aux = np.asarray([self.weights.detach().double().numpy(), mu, std])
        aux = np.asarray([self.weights, mu, std])

        return aux
def check_implementation():
    policy_params={'type':'linear',
                   'ob_filter':'MeanStdFilter',
                   'ob_dim': 24,
                   'ac_dim': 4}
    policy = BilayerPolicy(policy_params)
    print(policy.net)

#check_implementation()

max_stars_repo_name: hari-sikchi/safeRL

max_issues_repo_name: hari-sikchi/HCOPE

max_forks_repo_name: hari-sikchi/HCOPE


text:
import os
import random

import numpy as np

from matbench.constants import CLF_KEY, REG_KEY

MB_TEST_RANDOM_SEED = 1001
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
FULL_TEST = os.environ.get("MB_FULL_TESTS", False)

def model_random(training_outputs, test_inputs, response_type):
    r = random.Random(MB_TEST_RANDOM_SEED)

    r = np.random.RandomState(MB_TEST_RANDOM_SEED)

    l = len(test_inputs)

    if response_type == CLF_KEY:
        return r.choice([True, False], size=l)

    # Regression: simply sample from random distribution bounded by max
    # and min training samples
    if response_type == REG_KEY:
        pred = r.uniform(max(training_outputs), min(training_outputs), size=l)
        return pred

max_stars_repo_name: sparks-baird/matbench

max_issues_repo_name: sparks-baird/matbench

max_forks_repo_name: ardunn/matbench


text:
#!/usr/bin/python
#
# Copyright (c) 2017, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
#
# All rights reserved.
#
# The Astrobee platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import scipy.spatial.transform

class Orientations:
    def __init__(self):
        self.yaws = []
        self.pitches = []
        self.rolls = []

    def get_euler(self, index):
        return [self.yaws[index], self.pitches[index], self.rolls[index]]

    def get_rotation(self, index):
        return scipy.spatial.transform.Rotation.from_euler(
            "ZYX", self.get_euler(index), degrees=True
        )

    def add_euler(self, euler_angles):
        self.add(euler_angles[0], euler_angles[1], euler_angles[2])

    def add(self, yaw, pitch, roll):
        self.yaws.append(yaw)
        self.pitches.append(pitch)
        self.rolls.append(roll)

max_stars_repo_name: limenutt/astrobee

max_issues_repo_name: limenutt/astrobee

max_forks_repo_name: limenutt/astrobee

reason: import scipy


text:
from numpy import nan, isnan, arange, corrcoef, mean
from matplotlib import pyplot as plt
import pandas
from hydromet_forecasting.timeseries import FixedIndexTimeseries
from string import Template
import base64
import tempfile
from os import path

class Evaluator(object):
    """Evaluator class for a predicted timeseries that is given as an FixedIndexTimeseries instance and has annual seasonality.

    This class enables to evaluate the performance of an forecast by means of comparing the observed and the forecasted
    timeseries. Those series need to be instances of the FixedIndexTimeseries class. The Evaluator class implements
    various methods to analyse the series statistically. The method write_html writes a report to a specified path.

    Attributes:
        forecast: the forecasted timeseries
        y_adj: the observed timeseries, reduced to the index of the forecasted timeseries
    """

    def __init__(self, y, forecast):
        """Initialising the Evaluator Instance

            Args:
                y: A FixedIndexTimeseries instance of the observed data
                forecast:  A FixedIndexTimeseries instance of the forecasted data of the same mode as y.

            Returns:
                The Evaluator instance

            Raises:
                ValueError: When the two timeseries are not of the same mode.
                InsufficientData: If the length of the forecasted timeseries is not sufficient for evalaution. At least two
                                    complete years of data must be provided in order to compute standard deviations etc.
            """

        if not y.mode == forecast.mode:
            raise ValueError("The target timeseries is not of the same mode as the forecasted timeseries.")
        self.y = y
        self.forecast = forecast

        self.y_adj=FixedIndexTimeseries(self.y.timeseries[self.forecast.timeseries.index], mode=self.y.mode)

        datagroups = [len(self.forecast.data_by_index(i)) for i in range(1,self.forecast.maxindex+1)]
        if min(datagroups)<2:
            raise self.__InsufficientData("The length of the forecasted timeseries is not sufficient.")

    def computeP(self, annualindex=None):
        """ Returns the P value (Percentage of forecasts with error/stdev > 0.674)

            Args:
                annualindex (int): default None

            Returns:
                A list of values for each period of the year of the forecasted timeseries if annualindex is None, else one value.
                    NaN is retuned of the value could not be determined (e.g. not enough data)

            Raises:
                None
            """
        P = []
        indexrange = range(1, self.y_adj.maxindex+1) if annualindex is None else [annualindex]

        for index in indexrange:
            allowed_error =  0.674 * self.y.stdev_s(index)
            try:
                error = abs(self.forecast.data_by_index(index) - self.y_adj.data_by_index(index))
                error = error.dropna()
                good = sum(error <= allowed_error)
                P.append(float(good) / len(error))
            except:
                P.append(nan)
        return P

    def computeRelError(self, annualindex=None):
        """ Returns the relative error value of the forecast (error / stdev.sample)

                    Args:
                        None

                    Returns:
                        A list of values for each period of the year of the forecasted timeseries if annualindex is None, else one value.
                    NaN is retuned of the value could not be determined (e.g. not enough data)

                    Raises:
                        None
                    """
        relerror = []
        indexrange = range(1, self.y_adj.maxindex + 1) if annualindex is None else [annualindex]

        for index in indexrange:
            stdev = self.y.stdev_s(index)
            try:
                error = abs(self.forecast.data_by_index(index) - self.y_adj.data_by_index(index))
                error = error.dropna()
                relerror.append(mean(error.values/stdev))
            except:
                relerror.append(nan)
        return relerror

    def trainingdata_count(self, annualindex = None):
        """ Returns the number of training data for each period of the year

                    Args:
                        annualindex (int): the annualindex for which trainingdata shall be counted. default=None

                    Returns:
                        A list of values for each period of the year of the forecasted timeseries if annualindex is None, else a single integer
                    Raises:
                        None
                    """

        if annualindex is None:
            count = list()
            indexrange = range(1, self.y_adj.maxindex + 1)
            for index in indexrange:
                count.append(len(self.forecast.data_by_index(index)))
        else:
            count = len(self.forecast.data_by_index(annualindex))
        return count

    def __prepare_figure(self, width=12, height=3):

        fig, ax = plt.subplots(1, 1)
        fig.set_figwidth(width)
        fig.set_figheight(height)

        nr_bars=self.y.maxindex
        periods_in_month=nr_bars/12.0
        monthly_labels_pos=[p*nr_bars/12.0+(0.0416667*nr_bars-0.5) for p in range(0,12)]
        ax.set_xticks(monthly_labels_pos)
        ax.set_xticklabels(["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"])

        monthly_dividers=[p*nr_bars/12.0+(0.0416667*nr_bars-0.5)-1.5 for p in range(0,13)]
        ax.set_xticks(monthly_dividers, minor=True)

        ax.grid(True, which="minor", axis="x", color="black", linestyle='--')
        ax.tick_params(axis="x", which="major", length=0)
        return fig, ax

    def plot_y_stats(self):
        norm = self.y.norm()
        stdev = self.y.stdev_s()
        upper = [norm[i]+stdev[i] for i in range(0,len(stdev))]
        lower = [norm[i]-stdev[i] for i in range(0,len(stdev))]
        fig, ax = self.__prepare_figure()
        [ax.plot(self.y.data_by_year(year).values, label='individual years', color='blue', alpha=.2) for year in
         range(self.y.timeseries.index[0].year, self.y.timeseries.index[-1].year + 1)]
        ax.plot(upper, color='black')
        ax.plot(lower, color='black',label="+/- STDEV")
        ax.plot(norm,label="NORM", color='red')
        handles, labels = ax.get_legend_handles_labels()
        ax.legend(handles[-3:], labels[-3:])
        plt.ylabel(self.y.label)
        return fig

    def plot_P(self):
        P = self.computeP()
        fig, ax = self.__prepare_figure()
        ax.bar(range(0, len(P)), P, width=0.7, color="black")
        plt.ylabel("P%")
        ax.set_ylim([0, 1])
        return fig

    def plot_RelError(self):
        relerror = self.computeRelError()
        fig, ax = self.__prepare_figure()
        ax.boxplot(relerror)
        ax.plot([0, ax.get_xlim()[1]], [0.674, 0.674], color='red', linestyle='dashed')
        plt.ylabel("Error/STDEV")
        ax.set_ylim([0, 1])
        return fig

    def plot_trainingdata(self):
        count = self.trainingdata_count()
        fig, ax = self. __prepare_figure()
        ax.bar(range(0, len(count)), count, width=0.7)
        ax.bar(range(0, len(count)), count, width=0.7)
        plt.ylabel("Number of training data")

        return fig

    def plot_ts_comparison(self):
        fig, ax = plt.subplots(1, 1)
        fig.set_figwidth(8)
        fig.set_figheight(8)
        ax.scatter(self.y_adj.timeseries, self.forecast.timeseries, marker=".", color='black')
        ax.set_ylabel("predicted")
        ax.set_xlabel("observed")
        maxval = max(ax.get_xlim()[1],ax.get_ylim()[1])
        ax.set_ylim([0,maxval])
        ax.set_xlim([0,maxval])
        ax.plot([0, maxval], [0, maxval],color='green', linestyle='dashed')
        r_corr = round(corrcoef(self.y_adj.timeseries, self.forecast.timeseries)[0, 1], 3)
        ax.text(0.5 * maxval, 0.9 * maxval, ("R = %s" % (r_corr)))
        return fig

    def __table_summary(self):
        data =dict({
            'Number of training data': self.trainingdata_count(),
            'Minimum':self.y.min(),
            'Norm':self.y.norm(),
            'Maximum':self.y.max(),
            '+/- d': self.y.stdev_s(),
            'P%': self.computeP()
        })
        df=pandas.DataFrame(data)
        return df.to_html()

    def write_html(self, filename):
        """ writes an evaluation report to the specified filepath as an html

            Args:
                filename: path to the html file to be created

            Returns:
                None

            Raises:
                None
            """
        templatefilepath = path.join(path.dirname(__file__),'template')
        with open(templatefilepath, 'r') as htmltemplate:
            page=Template(htmltemplate.read())

        encoded1 = self.__encode_figure(self.plot_y_stats())
        encoded2 = self.__encode_figure(self.plot_P())
        encoded3 = self.__encode_figure(self.plot_ts_comparison())
        encoded4 = self.__encode_figure(self.plot_RelError())

        table = self.__table_summary()

        htmlpage = open(filename, 'w')
        htmlpage.write(page.safe_substitute(TABLE=table,IMAGE1=encoded1,IMAGE2=encoded2,IMAGE3=encoded3,IMAGE4=encoded4))
        htmlpage.close()
        return filename

    def __encode_figure(self, fig):

        with tempfile.TemporaryFile(suffix=".png") as tmpfile:
            fig.savefig(tmpfile, format="png")
            tmpfile.seek(0)
            encoded = base64.b64encode(tmpfile.read())
            tmpfile.close()
        return encoded

    class __InsufficientData(Exception):
        pass

class SeasonalEvaluator(object):
    def __init__(self, featurenames,selectedfeatures,modelEvaluators, score):
        self.featurenames = featurenames
        self.selectedfeatures = selectedfeatures
        self.modelEvaluators = modelEvaluators
        self.score = score

    def __prepare_figure(self, width=12, height=3):
        fig, ax = plt.subplots(1, 1)
        fig.set_figwidth(width)
        fig.set_figheight(height)
        return fig, ax

    def __table_summary(self):
        index_best = self.score.index(min(self.score))
        index_worst = self.score.index(max(self.score))
        data =dict({
            'Number of training data': int(mean([CV.trainingdata_count()[0] for CV in self.modelEvaluators])),
            'Minimum':self.modelEvaluators[0].y.min(),
            'Norm':self.modelEvaluators[0].y.norm(),
            'Maximum':self.modelEvaluators[0].y.max(),
            '+/- d': self.modelEvaluators[0].y.stdev_s(),
            'STDEV/ERROR': mean([mean(CV.computeRelError()) for CV in self.modelEvaluators]),
            'P%': mean([CV.computeP() for CV in self.modelEvaluators])
        })
        df=pandas.DataFrame(data)
        return df.to_html()

    def plot_timeseries(self):
        fig, ax = self.__prepare_figure()
        [ax.plot(CV.forecast.timeseries, color='red', label="individual forecasts", alpha=.2) for CV in self.modelEvaluators]
        df_concat = pandas.concat(([CV.forecast.timeseries for CV in self.modelEvaluators]))
        by_row_index = df_concat.groupby(df_concat.index)
        df_means = by_row_index.mean()
        ax.plot(df_means, color='black', label='mean forecast')
        ax.plot(self.modelEvaluators[0].y_adj.timeseries, color='green', label='observed')
        handles, labels = ax.get_legend_handles_labels()
        ax.legend(handles[-3:], labels[-3:])
        return fig

    def write_html(self, filename):
        """ writes an evaluation report to the specified filepath as an html

            Args:
                filename: path to the html file to be created

            Returns:
                None

            Raises:
                None
            """
        templatefilepath = path.join(path.dirname(__file__),'template')
        with open(templatefilepath, 'r') as htmltemplate:
            page=Template(htmltemplate.read())

        encoded1=self.__encode_figure(self.plot_timeseries())

        table = self.__table_summary()

        htmlpage = open(filename, 'w')
        htmlpage.write(page.safe_substitute(TABLE=table, IMAGE1=encoded1))
        htmlpage.close()
        return filename

    def __encode_figure(self, fig):

        with tempfile.TemporaryFile(suffix=".png") as tmpfile:
            fig.savefig(tmpfile, format="png")
            tmpfile.seek(0)
            encoded = base64.b64encode(tmpfile.read())
            tmpfile.close()
        return encoded

max_stars_repo_name: julesair/hydromet-forecasting

max_issues_repo_name: julesair/hydromet-forecasting

max_forks_repo_name: julesair/hydromet-forecasting

reason: from numpy