text:
import gym
from gym import spaces
import numpy as np
import matplotlib.pyplot as plt

class TicTacToe3DEnv(gym.Env):
    metadata = {'render.modes': ['human']}

    def custom_rewards(self, reward_p1=None, pen_p2=None, reward_p2=None,
                 pen_p1=None):
        if pen_p1 and len(pen_p1) != 5:
            print("ERROR: pen_p1 must be of size 5 only")
        if pen_p2 and len(pen_p2) != 5:
            print("ERROR: pen_p2 must be of size 5 only")
        if reward_p1 and len(reward_p1) != 5:
            print("ERROR: reward_p1 must be of size 5 only")
        if reward_p2 and len(reward_p2) != 5:
            print("ERROR: reward_p2 must be of size 5 only")
        if pen_p1 is None:
            pen_p1 = [-20, -18, -16, -14, -10]
        if reward_p2 is None:
            reward_p2 = [40, 36, 32, 28, 20]
        if pen_p2 is None:
            pen_p2 = [-10, -9, -8, -7, -5]
        if reward_p1 is None:
            reward_p1 = [20, 18, 16, 14, 10]
        self.pen_p1 = pen_p1
        self.reward_p2 = reward_p2
        self.pen_p2 = pen_p2
        self.reward_p1 = reward_p1

    def __init__(self):
        self.pen_p1 = [-20, -18, -16, -14, -10]
        self.reward_p2 = [40, 36, 32, 28, 20]
        self.pen_p2 = [-10, -9, -8, -7, -5]
        self.reward_p1 = [20, 18, 16, 14, 10]
        self.state = []
        super().__init__()
        self.action_space = spaces.Discrete(27)
        self.board = np.asarray([0] * 27).reshape(1, 27)
        self.turn_count = 0
        self.done = False
        self.ax = None
        self.axes = [5, 5, 5]
        self.data = np.ones([3, 3, 3], dtype=np.bool)
        self.colors = np.empty(self.axes + [4], dtype=np.float32)
        self.alpha = 0.2
        self.colors[0] = [1, 1, 1, self.alpha]
        self.colors[1] = [1, 1, 1, self.alpha]
        self.colors[2] = [1, 1, 1, self.alpha]
        self.colors[3] = [1, 1, 1, self.alpha]
        self.colors[4] = [1, 1, 1, self.alpha]
        self.previous_action = None
        self.start = 0
        self.indices_win = None

    def set_up_render(self):

        fig = plt.figure(1)
        self.ax = fig.add_subplot(111, projection='3d')

        def explode(data):
            shape_arr = np.array(data.shape)
            size = shape_arr[:3] * 2 - 1
            exploded = np.zeros(np.concatenate([size, shape_arr[3:]]), dtype=data.dtype)
            exploded[::2, ::2, ::2] = data
            return exploded

        self.data = explode(self.data)

        plt.ion()
        self.ax.voxels(self.data, facecolors=self.colors, edgecolors='grey', shade=False)
        self.ax.set_xticks([0, 2, 4])
        self.ax.set_xticklabels(["X0", "X1", "X2"])
        self.ax.set_yticks([0, 2, 4])
        self.ax.set_yticklabels(["Y0", "Y1", "Y2"])
        self.ax.set_zticks([0, 2, 4])
        self.ax.set_zticklabels(["Z0", "Z1", "Z2"])
        plt.suptitle("Tic-Tac-Toe3D")
        plt.show()

    def get_indices(self, action):
        if action < 9:
            i3 = 0
            if action < 3:
                i1 = 0
            elif action < 6:
                i1 = 2
            else:
                i1 = 4
        elif action < 18:
            i3 = 2
            if action < 12:
                i1 = 0
            elif action < 15:
                i1 = 2
            else:
                i1 = 4
        else:
            i3 = 4
            if action < 21:
                i1 = 0
            elif action < 24:
                i1 = 2
            else:
                i1 = 4
        i2 = (action % 3) * 2
        return i1, i2, i3

    def render(self, mode='human', close=False, player=1):

        if self.start == 0:
            self.set_up_render()
            return

        if self.done:
            t = np.where(np.sum(self.colors == np.asarray([1., 0, 0, 1.]), 3) == 4)
            t = [t[0], t[1], t[2]]
            for i in range(0, len(t[0])):
                self.colors[t[0][i], t[1][i], t[2][i]] = [1., 0.5, 0.5, 1.]

            t = np.where(np.sum(self.colors == np.asarray([0, 0, 1, 1.]), 3) == 4)
            t = [t[0], t[1], t[2]]
            for i in range(0, len(t[0])):
                self.colors[t[0][i], t[1][i], t[2][i]] = [0.5, 0.5, 1, 1.]

            if player == 1:
                color = [1, 0, 0, 1]
            else:
                color = [0, 0, 1, 1]
            for index_win in self.indices_win:
                i1, i2, i3 = self.get_indices(index_win)
                self.colors[i1, i2, i3] = color
        else:
            i1, i2, i3 = self.get_indices(self.previous_action)
            if player == 1:
                self.colors[i1, i2, i3] = [1, 0, 0, 1]
            else:
                self.colors[i1, i2, i3] = [0, 0, 1, 1]
        self.ax.voxels(self.data, facecolors=self.colors, edgecolors='grey', shade=False)

    def step(self, action, player=1):
        self.start = 1
        if self.done:
            print("Game Already Over")
            return None

        if self.board[0][action] != 0:
            print("Position Already Taken")
            return self.board[0], [0, 0], self.done, {"draw": False,
                                                   "winning_player": -1,
                                                   "turn_count": self.turn_count}

        viable_moves = np.where(self.board[0] == 0)[0].tolist()
        if len(viable_moves) == 0:
            self.done = True
            return self.board[0], [0, 0], self.done, {"draw": True,
                                                   "winning_player": -1,
                                                   "turn_count": self.turn_count}
        if player == 1:
            self.board[0][action] = 1
        else:
            self.board[0][action] = -1

        self.previous_action = action
        win = self.check_win(player=player)
        if win:
            self.done = True
            if player == 1:
                w = self.reward_p1
                l = self.pen_p2
            else:
                w = self.reward_p2
                l = self.pen_p1

            if self.turn_count <= 3:
                reward = [w[0], l[0]]
            elif self.turn_count <= 5:
                reward = [w[1], l[1]]
            elif self.turn_count <= 7:
                reward = [w[2], l[2]]
            elif self.turn_count <= 9:
                reward = [w[3], l[3]]
            else:
                reward = [w[4], l[4]]

            if player == 2:
                reward = reward[::-1]

            return self.board[0], reward, self.done, {"draw": False,
                                                   "winning_player": player,
                                                   "turn_count": self.turn_count}
        else:
            if player == 2:
                self.turn_count += 1
            temp = self.board
            if player == 1:
                temp = np.negative(temp)
            return temp[0], [0, 0], self.done, {"draw": False,
                                             "winning_player": -1,
                                             "turn_count": self.turn_count}

    def reset(self):
        self.state = []
        self.board = np.asarray([0] * 27).reshape(1, 27)
        self.turn_count = 0
        self.done = False
        self.ax = None
        self.axes = [5, 5, 5]
        self.data = np.ones([3, 3, 3], dtype=np.bool)
        self.colors = np.empty(self.axes + [4], dtype=np.float32)
        self.alpha = 0.2
        self.colors[0] = [1, 1, 1, self.alpha]
        self.colors[1] = [1, 1, 1, self.alpha]
        self.colors[2] = [1, 1, 1, self.alpha]
        self.colors[3] = [1, 1, 1, self.alpha]
        self.colors[4] = [1, 1, 1, self.alpha]
        self.previous_action = None
        self.start = 0
        self.indices_win = None
        return self.board[0]

    def check_win(self, player=1):
        winning_states_layers = [[1, 1, 1, 0, 0, 0, 0, 0, 0],
                                 [0, 0, 0, 1, 1, 1, 0, 0, 0],
                                 [0, 0, 0, 0, 0, 0, 1, 1, 1],
                                 [1, 0, 0, 1, 0, 0, 1, 0, 0],
                                 [0, 1, 0, 0, 1, 0, 0, 1, 0],
                                 [0, 0, 1, 0, 0, 1, 0, 0, 1],
                                 [1, 0, 0, 0, 1, 0, 0, 0, 1],
                                 [0, 0, 1, 0, 1, 0, 1, 0, 0]]
        if player == 2:
            layers = np.split(np.negative(self.board[0]), 3)
        else:
            layers = np.split(self.board[0], 3)
        layer_count = 0
        for layer in layers:
            row = 0
            t = layer_count*9
            for winning_layer in winning_states_layers:
                temp = layer * winning_layer
                if np.sum(temp == winning_layer) == len(temp):
                    if row == 0:
                        self.indices_win = [t+0, t+1, t+2]
                    elif row == 1:
                        self.indices_win = [t+3, t+4, t+5]
                    elif row == 2:
                        self.indices_win = [t+6, t+7, t+8]
                    elif row == 3:
                        self.indices_win = [t+0, t+3, t+6]
                    elif row == 4:
                        self.indices_win = [t+1, t+4, t+7]
                    elif row == 5:
                        self.indices_win = [t+2, t+5, t+8]
                    elif row == 6:
                        self.indices_win = [t+0, t+4, t+8]
                    elif row == 7:
                        self.indices_win = [t+2, t+4, t+6]
                    return True
                row += 1
            layer_count += 1
        for i in range(0, 9):
            if layers[0][i] == layers[1][i] == layers[2][i] == 1:
                self.indices_win = [i, 9+i, 18+i]
                return True
        if layers[0][0] == layers[1][3] == layers[2][6] == 1:
            self.indices_win = [0, 12, 24]
            return True
        elif layers[0][1] == layers[1][4] == layers[2][7] == 1:
            self.indices_win = [1, 13, 25]
            return True
        elif layers[0][2] == layers[1][5] == layers[2][8] == 1:
            self.indices_win = [2, 14, 26]
            return True
        elif layers[0][6] == layers[1][3] == layers[2][0] == 1:
            self.indices_win = [6 , 12 , 18]
            return True
        elif layers[0][7] == layers[1][4] == layers[2][1] == 1:
            self.indices_win = [7 , 13 , 19]
            return True
        elif layers[0][8] == layers[1][5] == layers[2][2] == 1:
            self.indices_win = [8 , 14, 20]
            return True
        elif layers[0][0] == layers[1][1] == layers[2][2] == 1:
            self.indices_win = [0 ,10 , 20]
            return True
        elif layers[0][3] == layers[1][4] == layers[2][5] == 1:
            self.indices_win = [3 , 13 , 23]
            return True
        elif layers[0][6] == layers[1][7] == layers[2][8] == 1:
            self.indices_win = [6 , 16 , 26]
            return True
        elif layers[0][2] == layers[1][1] == layers[2][0] == 1:
            self.indices_win = [2 , 10 , 18]
            return True
        elif layers[0][5] == layers[1][4] == layers[2][3] == 1:
            self.indices_win = [5 , 13 , 21]
            return True
        elif layers[0][8] == layers[1][7] == layers[2][6] == 1:
            self.indices_win = [8 ,16 , 24]
            return True
        elif layers[0][0] == layers[1][4] == layers[2][8] == 1:
            self.indices_win = [0 , 13, 26]
            return True
        elif layers[0][2] == layers[1][4] == layers[2][6] == 1:
            self.indices_win = [2 , 13 , 24]
            return True
        elif layers[0][6] == layers[1][4] == layers[2][2] == 1:
            self.indices_win = [6, 13, 20]
            return True
        elif layers[0][8] == layers[1][4] == layers[2][0] == 1:
            self.indices_win = [8 ,13 ,18]
            return True
        else:
            return False


text:
import einops
import numpy as np
from pydantic import BaseModel

from ..utils.typing import Array

class Pose(BaseModel):
    """Pose object modelling a set of poses in 3D

    Attributes
    ----------
    positions : (n, 3, 1) np.ndarray
        Positions in 3D represented as column vectors
    orientations : (n, 3, 3) np.ndarray
        Orientations in 3D described as rotation matrices which premultiply
        column vectors (v -> v' == Rv = v')
    """
    positions: Array[float, (-1, 3, 1)]
    orientations: Array[float, (-1, 3, 3)]

    @property
    def count(self):
        return self.positions.shape[0]

class Transform(BaseModel):
    """Transform object modelling a set of transforms in 3D

    Attributes
    ----------
    shifts : (n, 3, 1) np.ndarray
        Shifts in 3D
    rotations : (n, 3, 3) np.ndarray
        Rotations in 3D described as rotation matrices which premultiply
        column vectors ( v -> v' | Rv == v' )
    """
    shifts: Array[float, (-1, 3, 1)]
    rotations: Array[float, (-1, 3, 3)]

    @property
    def count(self):
        return self.shifts.shape[0]

    def apply(self, pose: Pose) -> tuple[Array, Array]:
        """Apply transformations on a set of poses

        Parameters
        ----------
        pose: Pose
            A set of poses on which transforms should be applied

        Returns
        -------
        transformed_poses: (transformed_positions, transformed_orientations)
            Transformed poses as a tuple of (m, n, 3) positions and
            (m, n, 3, 3) orientations where n is the number of poses and m is
            the number of transforms
        """
        # transformation rotations     (m, 3, 3)
        # broadcastable             (m, 1, 3, 3)
        # pose orientations            (n, 3, 3)
        # final rotations           (m, n, 3, 3)
        broadcastable_rotations = einops.rearrange(
            self.rotations, 'm i j -> m 1 i j'
        )
        final_rotations = broadcastable_rotations @ pose.orientations

        # transformation shifts        (m, 3, 1)
        # broadcastable             (m, 1, 3, 1)
        # pose orientations            (n, 3, 3)
        # oriented                  (m, n, 3, 1)
        # pose positions               (n, 3, 1)
        # final positions           (m, n, 3, 1)
        broadcastable_shifts = einops.rearrange(
            self.shifts, 'm i j -> m 1 i j'
        )
        oriented_shifts = pose.orientations @ broadcastable_shifts
        final_positions = pose.positions + oriented_shifts

        return final_positions.squeeze(), final_rotations.squeeze()

ext: py

max_stars_repo_name: alisterburt/eraldatu

max_issues_repo_name: alisterburt/eraldatu

max_forks_repo_name: alisterburt/eraldatu

reason: import numpy


text:
# Create first network with Keras
from keras.models import Sequential
from keras.layers import Dense
import numpy

def handler(event, context):
    # fix random seed for reproducibility
    seed = 7
    numpy.random.seed(seed)
    # load pima indians dataset
    dataset = numpy.loadtxt("pima-indians-diabetes.csv", delimiter=",")
    # split into input (X) and output (Y) variables
    X = dataset[:,0:8]
    Y = dataset[:,8]
    # create model
    model = Sequential()
    model.add(Dense(12, input_dim=8, init='uniform', activation='relu'))
    model.add(Dense(8, init='uniform', activation='relu'))
    model.add(Dense(1, init='uniform', activation='sigmoid'))
    # Compile model
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    # Fit the model
    model.fit(X, Y, nb_epoch=150, batch_size=10,  verbose=2)
    # calculate predictions
    predictions = model.predict(X)
    # round predictions
    rounded = [round(x[0]) for x in predictions]
    print(rounded)
    return rounded

max_stars_repo_name: Con-Mi/lambda-packs

max_issues_repo_name: Con-Mi/lambda-packs

max_forks_repo_name: Con-Mi/lambda-packs


text:
# -*- coding: utf-8 -*-
import re
import sys
import time
from unittest.case import SkipTest

import mock
import pytest
import six

from ddtrace.constants import ANALYTICS_SAMPLE_RATE_KEY
from ddtrace.constants import ENV_KEY
from ddtrace.constants import ERROR_MSG
from ddtrace.constants import ERROR_STACK
from ddtrace.constants import ERROR_TYPE
from ddtrace.constants import SERVICE_VERSION_KEY
from ddtrace.constants import SPAN_MEASURED_KEY
from ddtrace.constants import VERSION_KEY
from ddtrace.ext import SpanTypes
from ddtrace.span import Span
from tests.utils import TracerTestCase
from tests.utils import assert_is_measured
from tests.utils import assert_is_not_measured
from tests.utils import override_global_config

class SpanTestCase(TracerTestCase):
    def test_ids(self):
        s = Span(tracer=None, name="span.test")
        assert s.trace_id
        assert s.span_id
        assert not s.parent_id

        s2 = Span(tracer=None, name="t", trace_id=1, span_id=2, parent_id=1)
        assert s2.trace_id == 1
        assert s2.span_id == 2
        assert s2.parent_id == 1

    def test_tags(self):
        s = Span(tracer=None, name="test.span")
        s.set_tag("a", "a")
        s.set_tag("b", 1)
        s.set_tag("c", "1")
        d = s.to_dict()
        assert d["meta"] == dict(a="a", c="1")
        assert d["metrics"] == dict(b=1)

    def test_numeric_tags(self):
        s = Span(tracer=None, name="test.span")
        s.set_tag("negative", -1)
        s.set_tag("zero", 0)
        s.set_tag("positive", 1)
        s.set_tag("large_int", 2 ** 53)
        s.set_tag("really_large_int", (2 ** 53) + 1)
        s.set_tag("large_negative_int", -(2 ** 53))
        s.set_tag("really_large_negative_int", -((2 ** 53) + 1))
        s.set_tag("float", 12.3456789)
        s.set_tag("negative_float", -12.3456789)
        s.set_tag("large_float", 2.0 ** 53)
        s.set_tag("really_large_float", (2.0 ** 53) + 1)

        d = s.to_dict()
        assert d["meta"] == dict(
            really_large_int=str(((2 ** 53) + 1)),
            really_large_negative_int=str(-((2 ** 53) + 1)),
        )
        assert d["metrics"] == {
            "negative": -1,
            "zero": 0,
            "positive": 1,
            "large_int": 2 ** 53,
            "large_negative_int": -(2 ** 53),
            "float": 12.3456789,
            "negative_float": -12.3456789,
            "large_float": 2.0 ** 53,
            "really_large_float": (2.0 ** 53) + 1,
        }

    def test_set_tag_bool(self):
        s = Span(tracer=None, name="test.span")
        s.set_tag("true", True)
        s.set_tag("false", False)

        d = s.to_dict()
        assert d["meta"] == dict(true="True", false="False")
        assert "metrics" not in d

    def test_set_tag_metric(self):
        s = Span(tracer=None, name="test.span")

        s.set_tag("test", "value")
        assert s.meta == dict(test="value")
        assert s.metrics == dict()

        s.set_tag("test", 1)
        assert s.meta == dict()
        assert s.metrics == dict(test=1)

    def test_set_valid_metrics(self):
        s = Span(tracer=None, name="test.span")
        s.set_metric("a", 0)
        s.set_metric("b", -12)
        s.set_metric("c", 12.134)
        s.set_metric("d", 1231543543265475686787869123)
        s.set_metric("e", "12.34")
        d = s.to_dict()
        expected = {
            "a": 0,
            "b": -12,
            "c": 12.134,
            "d": 1231543543265475686787869123,
            "e": 12.34,
        }
        assert d["metrics"] == expected

    def test_set_invalid_metric(self):
        s = Span(tracer=None, name="test.span")

        invalid_metrics = [None, {}, [], s, "quarante-douze", float("nan"), float("inf"), 1j]

        for i, m in enumerate(invalid_metrics):
            k = str(i)
            s.set_metric(k, m)
            assert s.get_metric(k) is None

    def test_set_numpy_metric(self):
        try:
            import numpy as np
        except ImportError:
            raise SkipTest("numpy not installed")
        s = Span(tracer=None, name="test.span")
        s.set_metric("a", np.int64(1))
        assert s.get_metric("a") == 1
        assert type(s.get_metric("a")) == float

    def test_tags_not_string(self):
        # ensure we can cast as strings
        class Foo(object):
            def __repr__(self):
                1 / 0

        s = Span(tracer=None, name="test.span")
        s.set_tag("a", Foo())

    def test_finish(self):
        # ensure span.finish() marks the end time of the span
        s = Span(None, "test.span")
        sleep = 0.05
        time.sleep(sleep)
        s.finish()
        assert s.duration >= sleep, "%s < %s" % (s.duration, sleep)

    def test_finish_no_tracer(self):
        # ensure finish works with no tracer without raising exceptions
        s = Span(tracer=None, name="test.span")
        s.finish()

    def test_finish_called_multiple_times(self):
        # we should only record a span the first time finish is called on it
        s = Span(self.tracer, "bar")
        s.finish()
        s.finish()

    def test_finish_set_span_duration(self):
        # If set the duration on a span, the span should be recorded with this
        # duration
        s = Span(tracer=None, name="test.span")
        s.duration = 1337.0
        s.finish()
        assert s.duration == 1337.0

    def test_setter_casts_duration_ns_as_int(self):
        s = Span(tracer=None, name="test.span")
        s.duration = 3.2
        s.finish()
        assert s.duration == 3.2
        assert s.duration_ns == 3200000000
        assert isinstance(s.duration_ns, int)

    def test_get_span_returns_none_by_default(self):
        s = Span(tracer=None, name="test.span")
        assert s.duration is None

    def test_traceback_with_error(self):
        s = Span(None, "test.span")
        try:
            1 / 0
        except ZeroDivisionError:
            s.set_traceback()
        else:
            assert 0, "should have failed"

        assert s.error
        assert "by zero" in s.get_tag(ERROR_MSG)
        assert "ZeroDivisionError" in s.get_tag(ERROR_TYPE)

    def test_traceback_without_error(self):
        s = Span(None, "test.span")
        s.set_traceback()
        assert not s.error
        assert not s.get_tag(ERROR_MSG)
        assert not s.get_tag(ERROR_TYPE)
        assert "in test_traceback_without_error" in s.get_tag(ERROR_STACK)

    def test_ctx_mgr(self):
        s = Span(self.tracer, "bar")
        assert not s.duration
        assert not s.error

        e = Exception("boo")
        try:
            with s:
                time.sleep(0.01)
                raise e
        except Exception as out:
            assert out == e
            assert s.duration > 0, s.duration
            assert s.error
            assert s.get_tag(ERROR_MSG) == "boo"
            assert "Exception" in s.get_tag(ERROR_TYPE)
            assert s.get_tag(ERROR_STACK)

        else:
            assert 0, "should have failed"

    def test_span_type(self):
        s = Span(tracer=None, name="test.span", service="s", resource="r", span_type=SpanTypes.WEB)
        s.set_tag("a", "1")
        s.set_meta("b", "2")
        s.finish()

        d = s.to_dict()
        assert d
        assert d["span_id"] == s.span_id
        assert d["trace_id"] == s.trace_id
        assert d["parent_id"] == s.parent_id
        assert d["meta"] == {"a": "1", "b": "2"}
        assert d["type"] == "web"
        assert d["error"] == 0
        assert type(d["error"]) == int

    def test_span_to_dict(self):
        s = Span(tracer=None, name="test.span", service="s", resource="r")
        s.span_type = "foo"
        s.set_tag("a", "1")
        s.set_meta("b", "2")
        s.finish()

        d = s.to_dict()
        assert d
        assert d["span_id"] == s.span_id
        assert d["trace_id"] == s.trace_id
        assert d["parent_id"] == s.parent_id
        assert d["meta"] == {"a": "1", "b": "2"}
        assert d["type"] == "foo"
        assert d["error"] == 0
        assert type(d["error"]) == int

    def test_span_to_dict_sub(self):
        parent = Span(tracer=None, name="test.span", service="s", resource="r")
        s = Span(tracer=None, name="test.span", service="s", resource="r")
        s._parent = parent
        s.span_type = "foo"
        s.set_tag("a", "1")
        s.set_meta("b", "2")
        s.finish()

        d = s.to_dict()
        assert d
        assert d["span_id"] == s.span_id
        assert d["trace_id"] == s.trace_id
        assert d["parent_id"] == s.parent_id
        assert d["meta"] == {"a": "1", "b": "2"}
        assert d["type"] == "foo"
        assert d["error"] == 0
        assert type(d["error"]) == int

    def test_span_boolean_err(self):
        s = Span(tracer=None, name="foo.bar", service="s", resource="r")
        s.error = True
        s.finish()

        d = s.to_dict()
        assert d
        assert d["error"] == 1
        assert type(d["error"]) == int

    @mock.patch("ddtrace.span.log")
    def test_numeric_tags_none(self, span_log):
        s = Span(tracer=None, name="test.span")
        s.set_tag(ANALYTICS_SAMPLE_RATE_KEY, None)
        d = s.to_dict()
        assert d
        assert "metrics" not in d

        # Ensure we log a debug message
        span_log.debug.assert_called_once_with(
            "ignoring not number metric %s:%s",
            ANALYTICS_SAMPLE_RATE_KEY,
            None,
        )

    def test_numeric_tags_true(self):
        s = Span(tracer=None, name="test.span")
        s.set_tag(ANALYTICS_SAMPLE_RATE_KEY, True)
        d = s.to_dict()
        assert d
        expected = {ANALYTICS_SAMPLE_RATE_KEY: 1.0}
        assert d["metrics"] == expected

    def test_numeric_tags_value(self):
        s = Span(tracer=None, name="test.span")
        s.set_tag(ANALYTICS_SAMPLE_RATE_KEY, 0.5)
        d = s.to_dict()
        assert d
        expected = {ANALYTICS_SAMPLE_RATE_KEY: 0.5}
        assert d["metrics"] == expected

    def test_numeric_tags_bad_value(self):
        s = Span(tracer=None, name="test.span")
        s.set_tag(ANALYTICS_SAMPLE_RATE_KEY, "Hello")
        d = s.to_dict()
        assert d
        assert "metrics" not in d

    def test_set_tag_none(self):
        s = Span(tracer=None, name="root.span", service="s", resource="r")
        assert s.meta == dict()

        s.set_tag("custom.key", "100")

        assert s.meta == {"custom.key": "100"}

        s.set_tag("custom.key", None)

        assert s.meta == {"custom.key": "None"}

    def test_duration_zero(self):
        s = Span(tracer=None, name="foo.bar", service="s", resource="r", start=123)
        s.finish(finish_time=123)
        assert s.duration_ns == 0
        assert s.duration == 0

    def test_start_int(self):
        s = Span(tracer=None, name="foo.bar", service="s", resource="r", start=123)
        assert s.start == 123
        assert s.start_ns == 123000000000

        s = Span(tracer=None, name="foo.bar", service="s", resource="r", start=123.123)
        assert s.start == 123.123
        assert s.start_ns == 123123000000

        s = Span(tracer=None, name="foo.bar", service="s", resource="r", start=123.123)
        s.start = 234567890.0
        assert s.start == 234567890
        assert s.start_ns == 234567890000000000

    def test_duration_int(self):
        s = Span(tracer=None, name="foo.bar", service="s", resource="r")
        s.finish()
        assert isinstance(s.duration_ns, int)
        assert isinstance(s.duration, float)

        s = Span(tracer=None, name="foo.bar", service="s", resource="r", start=123)
        s.finish(finish_time=123.2)
        assert s.duration_ns == 200000000
        assert s.duration == 0.2

        s = Span(tracer=None, name="foo.bar", service="s", resource="r", start=123.1)
        s.finish(finish_time=123.2)
        assert s.duration_ns == 100000000
        assert s.duration == 0.1

        s = Span(tracer=None, name="foo.bar", service="s", resource="r", start=122)
        s.finish(finish_time=123)
        assert s.duration_ns == 1000000000
        assert s.duration == 1

    def test_set_tag_version(self):
        s = Span(tracer=None, name="test.span")
        s.set_tag(VERSION_KEY, "1.2.3")
        assert s.get_tag(VERSION_KEY) == "1.2.3"
        assert s.get_tag(SERVICE_VERSION_KEY) is None

        s.set_tag(SERVICE_VERSION_KEY, "service.version")
        assert s.get_tag(VERSION_KEY) == "service.version"
        assert s.get_tag(SERVICE_VERSION_KEY) == "service.version"

    def test_set_tag_env(self):
        s = Span(tracer=None, name="test.span")
        s.set_tag(ENV_KEY, "prod")
        assert s.get_tag(ENV_KEY) == "prod"

@pytest.mark.parametrize(
    "value,assertion",
    [
        (None, assert_is_measured),
        (1, assert_is_measured),
        (1.0, assert_is_measured),
        (-1, assert_is_measured),
        (True, assert_is_measured),
        ("true", assert_is_measured),
        # DEV: Ends up being measured because we do `bool("false")` which is `True`
        ("false", assert_is_measured),
        (0, assert_is_not_measured),
        (0.0, assert_is_not_measured),
        (False, assert_is_not_measured),
    ],
)
def test_set_tag_measured(value, assertion):
    s = Span(tracer=None, name="test.span")
    s.set_tag(SPAN_MEASURED_KEY, value)
    assertion(s)

def test_set_tag_measured_not_set():
    # Span is not measured by default
    s = Span(tracer=None, name="test.span")
    assert_is_not_measured(s)

def test_set_tag_measured_no_value():
    s = Span(tracer=None, name="test.span")
    s.set_tag(SPAN_MEASURED_KEY)
    assert_is_measured(s)

def test_set_tag_measured_change_value():
    s = Span(tracer=None, name="test.span")
    s.set_tag(SPAN_MEASURED_KEY, True)
    assert_is_measured(s)

    s.set_tag(SPAN_MEASURED_KEY, False)
    assert_is_not_measured(s)

    s.set_tag(SPAN_MEASURED_KEY)
    assert_is_measured(s)

@mock.patch("ddtrace.span.log")
def test_span_key(span_log):
    # Span tag keys must be strings
    s = Span(tracer=None, name="test.span")

    s.set_tag(123, True)
    span_log.warning.assert_called_once_with("Ignoring tag pair %s:%s. Key must be a string.", 123, True)
    assert s.get_tag(123) is None
    assert s.get_tag("123") is None

    span_log.reset_mock()

    s.set_tag(None, "val")
    span_log.warning.assert_called_once_with("Ignoring tag pair %s:%s. Key must be a string.", None, "val")
    assert s.get_tag(123.32) is None

def test_span_finished():
    span = Span(None, None)
    assert span.finished is False
    assert span.duration_ns is None

    span.finished = True
    assert span.finished is True
    assert span.duration_ns is not None
    duration = span.duration_ns

    span.finished = True
    assert span.finished is True
    assert span.duration_ns == duration

    span.finished = False
    assert span.finished is False

    span.finished = True
    assert span.finished is True
    assert span.duration_ns != duration

def test_span_unicode_set_tag():
    span = Span(None, None)
    span.set_tag("key", u"😌")
    span.set_tag("😐", u"😌")
    span._set_str_tag("key", u"😌")
    span._set_str_tag(u"😐", u"😌")

@pytest.mark.skipif(sys.version_info.major != 2, reason="This test only applies Python 2")
@mock.patch("ddtrace.span.log")
def test_span_binary_unicode_set_tag(span_log):
    span = Span(None, None)
    span.set_tag("key", "🤔")
    span._set_str_tag("key_str", "🤔")
    # only span.set_tag() will fail
    span_log.warning.assert_called_once_with("error setting tag %s, ignoring it", "key", exc_info=True)
    assert "key" not in span.meta
    assert span.meta["key_str"] == u"🤔"

@pytest.mark.skipif(sys.version_info.major == 2, reason="This test does not apply to Python 2")
@mock.patch("ddtrace.span.log")
def test_span_bytes_string_set_tag(span_log):
    span = Span(None, None)
    span.set_tag("key", b"\xf0\x9f\xa4\x94")
    span._set_str_tag("key_str", b"\xf0\x9f\xa4\x94")
    assert span.meta["key"] == "b'\\xf0\\x9f\\xa4\\x94'"
    assert span.meta["key_str"] == "🤔"
    span_log.warning.assert_not_called()

@mock.patch("ddtrace.span.log")
def test_span_encoding_set_str_tag(span_log):
    span = Span(None, None)
    span._set_str_tag("foo", u"/?foo=bar&baz=정상처리".encode("euc-kr"))
    span_log.warning.assert_not_called()
    assert span.meta["foo"] == u"/?foo=bar&baz=����ó��"

def test_span_nonstring_set_str_tag_exc():
    span = Span(None, None)
    with pytest.raises(TypeError):
        span._set_str_tag("foo", dict(a=1))
    assert "foo" not in span.meta

@mock.patch("ddtrace.span.log")
def test_span_nonstring_set_str_tag_warning(span_log):
    with override_global_config(dict(_raise=False)):
        span = Span(None, None)
        span._set_str_tag("foo", dict(a=1))
        span_log.warning.assert_called_once_with(
            "Failed to set text tag '%s'",
            "foo",
            exc_info=True,
        )

def test_span_ignored_exceptions():
    s = Span(None, None)
    s._ignore_exception(ValueError)

    with pytest.raises(ValueError):
        with s:
            raise ValueError()

    assert s.error == 0
    assert s.get_tag(ERROR_MSG) is None
    assert s.get_tag(ERROR_TYPE) is None
    assert s.get_tag(ERROR_STACK) is None

    s = Span(None, None)
    s._ignore_exception(ValueError)

    with pytest.raises(ValueError):
        with s:
            raise ValueError()

    with pytest.raises(RuntimeError):
        with s:
            raise RuntimeError()

    assert s.error == 1
    assert s.get_tag(ERROR_MSG) is not None
    assert "RuntimeError" in s.get_tag(ERROR_TYPE)
    assert s.get_tag(ERROR_STACK) is not None

def test_span_ignored_exception_multi():
    s = Span(None, None)
    s._ignore_exception(ValueError)
    s._ignore_exception(RuntimeError)

    with pytest.raises(ValueError):
        with s:
            raise ValueError()

    with pytest.raises(RuntimeError):
        with s:
            raise RuntimeError()

    assert s.error == 0
    assert s.get_tag(ERROR_MSG) is None
    assert s.get_tag(ERROR_TYPE) is None
    assert s.get_tag(ERROR_STACK) is None

def test_span_ignored_exception_subclass():
    s = Span(None, None)
    s._ignore_exception(Exception)

    with pytest.raises(ValueError):
        with s:
            raise ValueError()

    with pytest.raises(RuntimeError):
        with s:
            raise RuntimeError()

    assert s.error == 0
    assert s.get_tag(ERROR_MSG) is None
    assert s.get_tag(ERROR_TYPE) is None
    assert s.get_tag(ERROR_STACK) is None

def test_on_finish_single_callback():
    m = mock.Mock()
    s = Span(None, "test", on_finish=[m])
    m.assert_not_called()
    s.finish()
    m.assert_called_once_with(s)

def test_on_finish_multi_callback():
    m1 = mock.Mock()
    m2 = mock.Mock()
    s = Span(None, "test", on_finish=[m1, m2])
    s.finish()
    m1.assert_called_once_with(s)
    m2.assert_called_once_with(s)

@pytest.mark.parametrize("arg", ["span_id", "trace_id", "parent_id"])
def test_span_preconditions(arg):
    Span(None, "test", **{arg: None})
    with pytest.raises(TypeError):
        Span(None, "test", **{arg: "foo"})

def test_span_pprint():
    root = Span(None, "test.span", service="s", resource="r", span_type=SpanTypes.WEB)
    root.set_tag("t", "v")
    root.set_metric("m", 1.0)
    root.finish()
    actual = root.pprint()
    assert "name='test.span'" in actual
    assert "service='s'" in actual
    assert "resource='r'" in actual
    assert "type='web'" in actual
    assert "error=0" in actual
    assert ("tags={'t': 'v'}" if six.PY3 else "tags={'t': u'v'}") in actual
    assert "metrics={'m': 1.0}" in actual
    assert re.search("id=[0-9]+", actual) is not None
    assert re.search("trace_id=[0-9]+", actual) is not None
    assert "parent_id=None" in actual
    assert re.search("duration=[0-9.]+", actual) is not None
    assert re.search("start=[0-9.]+", actual) is not None
    assert re.search("end=[0-9.]+", actual) is not None

    root = Span(None, "test.span", service="s", resource="r", span_type=SpanTypes.WEB)
    actual = root.pprint()
    assert "duration=None" in actual
    assert "end=None" in actual

    root = Span(None, "test.span", service="s", resource="r", span_type=SpanTypes.WEB)
    root.error = 1
    actual = root.pprint()
    assert "error=1" in actual

    root = Span(None, "test.span", service="s", resource="r", span_type=SpanTypes.WEB)
    root.set_tag(u"😌", u"😌")
    actual = root.pprint()
    assert (u"tags={'😌': '😌'}" if six.PY3 else "tags={u'\\U0001f60c': u'\\U0001f60c'}") in actual

    root = Span(None, "test.span", service=object())
    actual = root.pprint()
    assert "service=<object object at" in actual

def test_manual_context_usage():
    span1 = Span(None, "span1")
    span2 = Span(None, "span2", context=span1.context)

    span2.context.sampling_priority = 2
    assert span1.context.sampling_priority == 2

    span1.context.sampling_priority = 1
    assert span2.context.sampling_priority == 1
    assert span1.context.sampling_priority == 1

def test_set_exc_info_with_unicode():
    def get_exception_span(exception):
        span = Span(None, "span1")
        try:
            raise exception
        except Exception:
            type_, value_, traceback_ = sys.exc_info()
            span.set_exc_info(type_, value_, traceback_)
        return span

    exception_span = get_exception_span(Exception(u"DataDog/水"))
    assert u"DataDog/水" == exception_span.get_tag(ERROR_MSG)

    if six.PY3:
        exception_span = get_exception_span(Exception("DataDog/水"))
        assert "DataDog/水" == exception_span.get_tag(ERROR_MSG)

max_stars_repo_name: melancholy/dd-trace-py

max_issues_repo_name: melancholy/dd-trace-py

max_forks_repo_name: melancholy/dd-trace-py


text:
"""Run decoding analyses in sensors space accross memory content and
visual perception for the working memory task and save decoding performance"""

# Authors: Romain Quentin <rom.quentin@gmail.com>
#          Jean-Remi King <jeanremi.king@gmail.com>
#
# License: BSD (3-clause)

import os
import os.path as op
import numpy as np
import mne
from h5io import read_hdf5
from mne.decoding import GeneralizingEstimator, LinearModel
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Ridge
from sklearn.metrics import make_scorer
from sklearn.model_selection import StratifiedKFold
from jr.gat import (AngularRegression, scorer_spearman,
                    scorer_angle)
from base import (complete_behavior, get_events_interactions)
from config import path_data
import sys
subject = sys.argv[1]  # read a swarm file for parralel computing on biowulf

output_folder = '/sensors_accross_epochs_and_conditions/'
# Create result folder
results_folder = op.join(path_data + 'results/' + subject + output_folder)
if not os.path.exists(results_folder):
    os.makedirs(results_folder)

# read behavior
fname = op.join(path_data, subject, 'behavior_Target.hdf5')
events = read_hdf5(fname)
events = complete_behavior(events)
events = get_events_interactions(events)
# read stimulus epochs
fname = op.join(path_data, subject, 'epochs_Target.fif')
epochs_target = mne.read_epochs(fname)
epochs_target.pick_types(meg=True, ref_meg=False)
epochs_target.crop(-0.2, 0.9)
# read cue epochs
fname = op.join(path_data, subject, 'epochs_Cue.fif')
epochs_cue = mne.read_epochs(fname)
epochs_cue.pick_types(meg=True, ref_meg=False)
epochs_cue.crop(0, 1.5)
# read probe epochs
fname = op.join(path_data, subject, 'epochs_Probe.fif')
epochs_probe = mne.read_epochs(fname)
epochs_probe.pick_types(meg=True, ref_meg=False)
epochs_probe.crop(0, 0.9)
# Concatenate the data of the three epochs
X0 = epochs_target._data
X1 = epochs_cue._data
X2 = epochs_probe._data
X = np.concatenate((X0, X1, X2), axis=2)

# Define pair of analyses (train on the 2nd and test on the 1st )
paired_analyses = [['target_sfreq_cue_left_sfreq', 'left_sfreq'],
                   ['target_sfreq_cue_right_sfreq', 'right_sfreq'],
                   ['left_sfreq', 'target_sfreq_cue_left_sfreq'],
                   ['right_sfreq', 'target_sfreq_cue_right_sfreq'],
                   ['target_angle_cue_left_angle', 'left_angle'],
                   ['target_angle_cue_right_angle', 'right_angle'],
                   ['left_angle', 'target_angle_cue_left_angle'],
                   ['right_angle', 'target_angle_cue_right_angle']]
# Loop across each pair of analyses
for paired_analysis in paired_analyses:
    y_test = np.array(events[paired_analysis[0]])
    y_train = np.array(events[paired_analysis[1]])
    # Define estimators depending on the analysis
    if 'angle' in paired_analysis[0][:14]:
        clf = make_pipeline(StandardScaler(),
                            LinearModel(AngularRegression(Ridge(),
                                                          independent=False)))
        scorer = scorer_angle
        kwargs = dict()
        gat = GeneralizingEstimator(clf, scoring=make_scorer(scorer),
                                    n_jobs=24, **kwargs)
        y_test = np.array(y_test, dtype=float)
        y_train = np.array(y_train, dtype=float)
    elif 'sfreq' in paired_analysis[0][:14]:
        clf = make_pipeline(StandardScaler(), LinearModel(Ridge()))
        scorer = scorer_spearman
        kwargs = dict()
        gat = GeneralizingEstimator(clf, scoring=make_scorer(scorer),
                                    n_jobs=24, **kwargs)
        y_test = np.array(y_test, dtype=float)
        y_train = np.array(y_train, dtype=float)
    # only consider trials with correct fixation
    sel = np.where(events['is_eye_fixed'] == 1)[0]
    y_train = y_train[sel]
    y_test = y_test[sel]
    X = np.concatenate((X0, X1, X2), axis=2)
    X = X[sel]
    # only consider non NaN values
    # Run decoding accross condition
    cv = StratifiedKFold(7)
    scores = list()
    scs = list()
    if np.isnan(y_train).any():
        sel = np.where(~np.isnan(y_train))[0]
        for train, test in cv.split(X[sel], y_train[sel]):
            gat.fit(X[sel][train], y_train[sel][train])
            score = gat.score(X[sel][test], y_test[sel][test])
            sc = gat.score(X[sel][test], y_train[sel][test])  # test on same
            scores.append(score)
            scs.append(sc)
        scores = np.mean(scores, axis=0)
        scs = np.mean(scs, axis=0)
    else:
        for train, test in cv.split(X, y_train):
            y_te = y_test[test]
            X_te = X[test]
            y_te = y_te[np.where(~np.isnan(y_te))[0]]
            X_te = X_te[np.where(~np.isnan(y_te))[0]]
            y_tr = y_train[train]
            X_tr = X[train]
            y_tr = y_tr[np.where(~np.isnan(y_tr))[0]]
            X_tr = X_tr[np.where(~np.isnan(y_tr))[0]]
            y_tr_te = y_train[test]
            X_tr_te = X[test]
            y_tr_te = y_tr_te[np.where(~np.isnan(y_tr_te))[0]]
            X_tr_te = X_tr_te[np.where(~np.isnan(y_tr_te))[0]]
            gat.fit(X_tr, y_tr)
            score = gat.score(X_te, y_te)
            sc = gat.score(X_tr_te, y_tr_te)   # test on same
            scores.append(score)
            scs.append(sc)
        scores = np.mean(scores, axis=0)
        scs = np.mean(scs, axis=0)

    # save cross-validated scores
    fname = results_folder +\
        '%s_scores_%s_cross_%s.npy' % (subject,
                                       paired_analysis[0],
                                       paired_analysis[1])
    np.save(fname, np.array(scores))  # save accross condition scores
    fname = results_folder +\
        '%s_scores_%s.npy' % (subject, paired_analysis[1])
    np.save(fname, np.array(scs))  # save scores test/train on same condition


text:
from mlapp.managers import DataManager, pipeline
from mlapp.utils.exceptions.base_exceptions import DataManagerException
import pandas as pd
import numpy as np

class FlowRegressionDataManager(DataManager):
    @pipeline
    def load_train_data(self,*args):
        print(args)

        return
    @pipeline
    def clean_train_data(self, data):
        return data

    @pipeline
    def transform_train_data(self, data):

        return data

    @pipeline
    def load_forecast_data(self, *args):
        try:
            models_outputs = args[0]
        except Exception as e:
            raise DataManagerException("Error: No data input from flow")

        # read data for forecasting
        features = {}
        data = pd.DataFrame()
        df_name = self.data_settings.get('flow_return_data')
        for model_output in models_outputs:
            index = self.data_settings.get('data_index')
            res = model_output.get(df_name, pd.DataFrame(columns=index))
            if len(res) > 0:
                df = res[0]
                # df.set_index(self.data_settings.get('data_index'))
                data = pd.concat([data, df])
            else:
                continue
        for feature_name in self.data_settings.get('flow_return_features'):
            features[feature_name] = []
            for model_output in models_outputs:
                feature_data = model_output.get(feature_name, pd.DataFrame(columns=index))
                if len(feature_data)>0:
                    if isinstance(feature_data, pd.DataFrame):
                        if not isinstance(features[feature_name], pd.DataFrame):
                            features[feature_name]=pd.DataFrame()
                        features[feature_name] = pd.concat([feature_data, features[feature_name]])
                    else:
                        # assume it is a list
                        features[feature_name].append(feature_data)
                else:
                    continue

        for key in features:
            self.save_metadata(key, features[key])

        return data

    @pipeline
    def clean_forecast_data(self, data):
        return data

    @pipeline
    def transform_forecast_data(self, data):
        index = self.data_settings.get('data_index')
        data.reset_index(inplace=True)
        agg_function = self.data_settings['data_handling'].get('agg_function_dataframe')
        values = self.data_settings['data_handling'].get('agg_on_columns')
        predictions_pivoted = pd.pivot_table(data, index=index, values=values, aggfunc=eval(agg_function))

        return predictions_pivoted

    @pipeline
    def load_target_data(self, *args):
        raise NotImplementedError()

max_stars_repo_name: kerenleibovich/mlapp

max_issues_repo_name: kerenleibovich/mlapp

max_forks_repo_name: kerenleibovich/mlapp


text:
# Precision/Recall (PR)
import os
import numpy as np
import tensorflow as tf
import dnnlib
import dnnlib.tflib as tflib
import glob
import PIL.Image

from metrics import metric_base
from training import misc

def batch_pairwise_distances(U, V):
    # Compute pairwise distances between two batches of feature vectors
    with tf.variable_scope("pairwise_dist_block"):
        # Squared norms of each row in U and V
        norm_u = tf.reduce_sum(tf.square(U), 1)
        norm_v = tf.reduce_sum(tf.square(V), 1)

        # norm_u as a row and norm_v as a column vectors
        norm_u = tf.reshape(norm_u, [-1, 1])
        norm_v = tf.reshape(norm_v, [1, -1])

        # Pairwise squared Euclidean distances
        D = tf.maximum(norm_u - 2*tf.matmul(U, V, False, True) + norm_v, 0.0)

    return D

class DistanceBlock():
    # Distance block
    def __init__(self, num_features, num_gpus):
        self.num_features = num_features
        self.num_gpus = num_gpus

        # Initialize TF graph to compute pairwise distances
        with tf.device("/cpu:0"):
            self._features_batch1 = tf.placeholder(tf.float16, shape=[None, self.num_features])
            self._features_batch2 = tf.placeholder(tf.float16, shape=[None, self.num_features])
            features_split2 = tf.split(self._features_batch2, self.num_gpus, axis = 0)
            distances_split = []
            for gpu_idx in range(self.num_gpus):
                with tf.device("/gpu:%d" % gpu_idx):
                    distances_split.append(batch_pairwise_distances(self._features_batch1, features_split2[gpu_idx]))
            self._distance_block = tf.concat(distances_split, axis = 1)

    def pairwise_distances(self, U, V):
        # Evaluate pairwise distances between two batches of feature vectors
        return self._distance_block.eval(feed_dict = {self._features_batch1: U, self._features_batch2: V})

class ManifoldEstimator():
    # Finds an estimate for the manifold of given feature vectors
    def __init__(self, distance_block, features, row_batch_size, col_batch_size, nhood_sizes, clamp_to_percentile = None):
        # Find an estimate of the manifold of given feature vectors
        num_imgs = features.shape[0]
        self.nhood_sizes = nhood_sizes
        self.num_nhoods = len(nhood_sizes)
        self.row_batch_size = row_batch_size
        self.col_batch_size = col_batch_size
        self._ref_features = features
        self._distance_block = distance_block

        # Estimate manifold of features by calculating distances to kth nearest neighbor of each sample
        self.D = np.zeros([num_imgs, self.num_nhoods], dtype = np.float16)
        distance_batch = np.zeros([row_batch_size, num_imgs], dtype = np.float16)
        seq = np.arange(max(self.nhood_sizes) + 1, dtype = np.int32)

        for begin1 in range(0, num_imgs, row_batch_size):
            end1 = min(begin1 + row_batch_size, num_imgs)
            row_batch = features[begin1:end1]

            for begin2 in range(0, num_imgs, col_batch_size):
                end2 = min(begin2 + col_batch_size, num_imgs)
                col_batch = features[begin2:end2]

                # Compute distances between batches
                distance_batch[0:end1-begin1, begin2:end2] = self._distance_block.pairwise_distances(row_batch, col_batch)

            # Find the kth nearest neighbor from the current batch
            self.D[begin1:end1, :] = np.partition(distance_batch[0:end1-begin1, :], seq, axis = 1)[:, self.nhood_sizes]

        if clamp_to_percentile is not None:
            max_distances = np.percentile(self.D, clamp_to_percentile, axis = 0)
            self.D[self.D > max_distances] = 0

    def evaluate(self, eval_features, return_realism = False, return_neighbors = False):
        # Evaluate if new feature vectors are in the estimated manifold
        num_eval_imgs = eval_features.shape[0]
        num_ref_imgs = self.D.shape[0]
        distance_batch = np.zeros([self.row_batch_size, num_ref_imgs], dtype = np.float16)
        batch_predictions = np.zeros([num_eval_imgs, self.num_nhoods], dtype = np.int32)
        realism_score = np.zeros([num_eval_imgs,], dtype = np.float32)
        nearest_indices = np.zeros([num_eval_imgs,], dtype = np.int32)

        for begin1 in range(0, num_eval_imgs, self.row_batch_size):
            end1 = min(begin1 + self.row_batch_size, num_eval_imgs)
            feature_batch = eval_features[begin1:end1]

            for begin2 in range(0, num_ref_imgs, self.col_batch_size):
                end2 = min(begin2 + self.col_batch_size, num_ref_imgs)
                ref_batch = self._ref_features[begin2:end2]

                distance_batch[0:end1-begin1, begin2:end2] = self._distance_block.pairwise_distances(feature_batch, ref_batch)

            # From the minibatch of new feature vectors, determine if they are in the estimated manifold.
            # If a feature vector is inside a hypersphere of some reference sample, then the new sample lies on the estimated manifold.
            # The radii of the hyperspheres are determined from distances of neighborhood size k.
            samples_in_manifold = distance_batch[0:end1-begin1, :, None] <= self.D
            batch_predictions[begin1:end1] = np.any(samples_in_manifold, axis = 1).astype(np.int32)

            nearest_indices[begin1:end1] = np.argmin(distance_batch[0:end1-begin1, :], axis = 1)
            realism_score[begin1:end1] = self.D[nearest_indices[begin1:end1], 0] / np.min(distance_batch[0:end1-begin1, :], axis = 1)

        if return_realism and return_neighbors:
            return batch_predictions, realism_score, nearest_indices
        elif return_realism:
            return batch_predictions, realism_score
        elif return_neighbors:
            return batch_predictions, nearest_indices

        return batch_predictions

def knn_precision_recall_features(ref_features, eval_features, feature_net, nhood_sizes,
                                  row_batch_size, col_batch_size, num_imgs, num_gpus):
    # Computes k-NN precision and recall for two sets of feature vectors
    state = dnnlib.EasyDict()
    num_features = feature_net.output_shape[1]
    state.ref_features = ref_features
    state.eval_features = eval_features

    # Initialize DistanceBlock and ManifoldEstimators.
    distance_block = DistanceBlock(num_features, num_gpus)
    state.ref_manifold = ManifoldEstimator(distance_block, state.ref_features, row_batch_size, col_batch_size, nhood_sizes)
    state.eval_manifold = ManifoldEstimator(distance_block, state.eval_features, row_batch_size, col_batch_size, nhood_sizes)

    # Evaluate precision and recall using k-nearest neighbors
    print("Evaluating k-NN precision and recall with %i samples..." % num_imgs)
    start = time.time()

    # Precision: How many points from eval_features are in ref_features manifold
    state.precision, state.realism_scores, state.nearest_neighbors = state.ref_manifold.evaluate(state.eval_features, return_realism = True, return_neighbors = True)
    state.knn_precision = state.precision.mean(axis = 0)

    # Recall: How many points from ref_features are in eval_features manifold
    state.recall = state.eval_manifold.evaluate(state.ref_features)
    state.knn_recall = state.recall.mean(axis = 0)

    elapsed_time = time.time() - start
    print("Done evaluation in: %gs" % elapsed_time)
    return state

class PR(metric_base.MetricBase):
    def __init__(self, nhood_size, minibatch_per_gpu, row_batch_size, col_batch_size, **kwargs):
        super().__init__(**kwargs)
        self.nhood_size = nhood_size
        self.minibatch_per_gpu = minibatch_per_gpu
        self.row_batch_size = row_batch_size
        self.col_batch_size = col_batch_size

    def _evaluate(self, Gs, Gs_kwargs, num_gpus, num_imgs, paths = None, **kwargs):
        minibatch_size = num_gpus * self.minibatch_per_gpu
        feature_net = misc.load_pkl("http://d36zk2xti64re0.cloudfront.net/stylegan1/networks/metrics/vgg16.pkl")

        # Compute features for reals
        cache_file = self._get_cache_file_for_reals(num_imgs)
        os.makedirs(os.path.dirname(cache_file), exist_ok = True)
        if os.path.isfile(cache_file):
            ref_features = misc.load_pkl(cache_file)
        else:
            imgs_iter = self._iterate_reals(minibatch_size = minibatch_size)
            ref_features = self._get_feats(imgs_iter, feature_net, minibatch_size, num_gpus, num_imgs)

            misc.save_pkl(ref_features, cache_file)

        if paths is not None:
            # Extract features for local sample image files (paths)
            eval_features = self._paths_to_feats(paths, feature_net, minibatch_size, num_gpus, num_imgs)
        else:
            # Extract features for newly generated fake imgs
            eval_features = self._gen_feats(Gs, feature_net, minibatch_size, num_imgs, num_gpus, Gs_kwargs)

        # Compute precision and recall
        state = knn_precision_recall_features(ref_features = ref_features, eval_features = eval_features,
            feature_net = feature_net, nhood_sizes = [self.nhood_size], row_batch_size = self.row_batch_size,
            col_batch_size = self.row_batch_size, num_imgs = num_imgs, num_gpus = num_gpus)
        self._report_result(state.knn_precision[0], suffix = "_precision")
        self._report_result(state.knn_recall[0], suffix = "_recall")

max_stars_repo_name: gperdrizet/gansformer

max_issues_repo_name: gperdrizet/gansformer

max_forks_repo_name: gperdrizet/gansformer


text:
import fileinput
import math
import os
import tempfile
from typing import Callable

import numpy as np
import pandas as pd
import pytest
from sctools.metrics.gatherer import GatherGeneMetrics, GatherCellMetrics, MetricGatherer
from sctools.metrics.merge import MergeCellMetrics, MergeGeneMetrics
from sctools.platform import TenXV2

"""
Testing Data Definition & Acquisition

The data hardcoded into this file come from the two notebooks associated with these metrics:
characterize-cell-testing-data.ipynb and characterize-gene-testing-data.ipynb. In these
notebooks, the testing .bam files are loaded into memory and interrogated for each of the
metrics in question using pandas and numpy commands. These independent calculation provide the
hard-coded data found in these tests. When testing data is changed, the notebook can be updated
to re-calculate the values found in this file.
"""

# set the input and output directories, using a tempdir to automatically clean up generated files
_data_dir = os.path.split(__file__)[0] + '/data'
_test_dir = tempfile.mkdtemp()
os.makedirs(_test_dir, exist_ok=True)

# note, to inspect these testing files, please install samtools and use the following command:
# samtools view <filename> | less

# set the input files
_gene_sorted_bam = os.path.join(_data_dir, 'small-gene-sorted.bam')
_cell_sorted_bam = os.path.join(_data_dir, 'small-cell-sorted.bam')
_cell_sorted_bam_missing_cell_barcodes = os.path.join(_data_dir, 'cell-sorted-missing-cb.bam')

# specify filenames for temporary metrics outputs that are used in the following tests
_gene_metric_output_file = os.path.join(_test_dir, 'gene_metrics.csv.gz')
_cell_metric_output_file = os.path.join(_test_dir, 'cell_metrics.csv.gz')
_cell_metric_output_file_missing_cell_barcodes = os.path.join(_test_dir, 'cell_metrics_missing_cb.csv.gz')

# run the gene metrics suite
gene_gatherer = GatherGeneMetrics(_gene_sorted_bam, _gene_metric_output_file)
gene_gatherer.extract_metrics()
_gene_metrics = pd.read_csv(_gene_metric_output_file, index_col=0)

# run the cell metrics suite
cell_gatherer = GatherCellMetrics(_cell_sorted_bam, _cell_metric_output_file)
cell_gatherer.extract_metrics()
_cell_metrics = pd.read_csv(_cell_metric_output_file, index_col=0)

# run the cell metrics suite
cell_gatherer_missing_cbs = GatherCellMetrics(_cell_sorted_bam_missing_cell_barcodes, _cell_metric_output_file_missing_cell_barcodes)
cell_gatherer_missing_cbs.extract_metrics()
_cell_metrics_missing_cbs = pd.read_csv(_cell_metric_output_file_missing_cell_barcodes, index_col=0)

def test_calculate_cell_metrics_cli():
    """test the sctools cell metrics CLI invocation"""
    cell_metrics_csv = os.path.join(_test_dir, 'cell_metrics.csv')
    return_call = TenXV2.calculate_cell_metrics(
        args=['-i', _cell_sorted_bam, '-o', cell_metrics_csv])
    assert return_call == 0

def test_calculate_gene_metrics_cli():
    """test the sctools gene metrics CLI invocation"""
    gene_metrics_csv = os.path.join(_test_dir, 'gene_metrics.csv')
    return_call = TenXV2.calculate_gene_metrics(
        args=['-i', _gene_sorted_bam, '-o', gene_metrics_csv])
    assert return_call == 0

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 300),
    (_cell_metrics, 656),
])
def test_metrics_n_reads(metrics, expected_value):
    """test that the metrics identify the correct read number"""
    assert metrics['n_reads'].sum() == expected_value

def test_cell_metrics_mean_n_genes_observed():
    """
    test that the GatherCellMetrics method identifies the correct number of genes per cell, on
    average.
    """
    genes_observed = _cell_metrics['n_genes'].mean()
    assert math.isclose(genes_observed, 1.9827, abs_tol=1e-4), '%f != %f' % (genes_observed, 1.9827)

def test_gene_metrics_n_genes():
    """Test that GatherGeneMetrics identifies the total number of genes in the test file"""
    genes_observed = _gene_metrics.shape[0]
    assert genes_observed == 8

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 88),
    (_cell_metrics, 249),
])
def test_metrics_n_molecules(metrics, expected_value):
    """Test that each metric identifies the total number of molecules in the test file

    Molecules are defined as a unique combination of {cell barcode, molecule barcode, gene}
    """
    molecules_observed = metrics['n_molecules'].sum()
    assert molecules_observed == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 217),
    (_cell_metrics, 499),
])
def test_metrics_n_fragments(metrics, expected_value):
    """Test that each metric identifies the total number of fragments in the test file.

    Fragments are defined as a unique combination of {cell barcode, molecule barcode, strand,
    position, chromosome}
    """
    fragments_observed = metrics['n_fragments'].sum()
    assert fragments_observed == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 'AL627309.7'),
    (_cell_metrics, 'AAACCTGGTAGAAGGA'),
])
def test_metrics_highest_expression_class(metrics, expected_value):
    """
    for gene metrics, this is the highest expression gene. For cell metrics, this is the highest
    expression cell.
    """
    observed_max_gene = metrics['n_reads'].idxmax()
    assert observed_max_gene == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 245),
    (_cell_metrics, 94),
])
def test_metrics_highest_read_count(metrics, expected_value):
    """
    Test that each metric identifies the what the highest read count associated with any single
    entity
    """
    observed_max_gene_reads = metrics['n_reads'].max()
    assert observed_max_gene_reads == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 300),  # todo this is 100%, we should mangle a few in the testing data
    (_cell_metrics, 655),
])
def test_metrics_number_perfect_molecule_barcodes(metrics, expected_value):
    """Test that each metric correctly identifies the number of perfect molecule barcodes where UB == UR"""
    observed_perfect_barcodes = metrics['perfect_molecule_barcodes'].sum()
    assert observed_perfect_barcodes == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_cell_metrics, 650),
    (_cell_metrics_missing_cbs, 12861)
])
def test_metrics_number_perfect_cell_barcodes(metrics, expected_value):
    """Test that each metric correctly identifies the number of perfect cell barcodes where CB == CR"""
    observed_perfect_cell_barcodes = metrics['perfect_cell_barcodes'].sum()
    assert observed_perfect_cell_barcodes == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 300),  # todo this is 100%, should get some intronic or other reads
    (_cell_metrics, 609),
])
def test_reads_mapped_exonic(metrics, expected_value):
    """Test that each metric identifies the number of reads mapped to an exon (XF=='CODING')"""
    observed = metrics['reads_mapped_exonic'].sum()
    assert observed == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 0),  # todo null case
    (_cell_metrics, 28),
])
def test_reads_mapped_intronic(metrics, expected_value):
    """Test that each metric identifies the number of reads mapped to an intron (XF=='INTRONIC')"""
    observed = metrics['reads_mapped_intronic'].sum()
    assert observed == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 0),  # todo null case
    (_cell_metrics, 19),
])
def test_reads_mapped_utr(metrics, expected_value):
    """Test that each metric identifies the number of reads mapped to a UTR (XF=='UTR')"""
    observed = metrics['reads_mapped_utr'].sum()
    assert observed == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 300),  # todo need to include at least 1 multi-mapper
    (_cell_metrics, 656),
])
def test_reads_mapped_uniquely(metrics, expected_value):
    """Uniquely mapping reads will be tagged with NH==1"""
    observed = metrics['reads_mapped_uniquely'].sum()
    assert observed == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 90),
    (_cell_metrics, 107),
])
def test_duplicate_records(metrics, expected_value):
    """Duplicate records are identified by the 1024 bit being set in the sam flag"""
    observed = metrics['duplicate_reads'].sum()
    assert observed == expected_value

@pytest.mark.parametrize('metrics, expected_value', [
    (_gene_metrics, 29),
    (_cell_metrics, 2),
])
def test_spliced_reads(metrics, expected_value):
    """
    This pipeline defines spliced reads as containing an N segment of any length in the cigar string
    """
    observed = metrics['spliced_reads'].sum()
    assert observed == expected_value

# todo failing
# @pytest.mark.parametrize('metrics', [_gene_metrics, _cell_metrics])
# def test_relationship_of_duplicates_and_fragments(metrics):
#     """
#     We expect the number of duplicates and fragments to add up to the total number of reads. The
#     rationale is that any read that is not a duplicate should be a distinct fragment, under our
#     definitions.
#
#     This fails because of (1) N-base and 2-base cell barcode correction errors and (2)
#     fragment calculationes currently do not account for soft clipping. Fixing these will cause
#     this test to pass
#     """
#     dup_and_fragments = metrics['duplicate_reads'].sum() + metrics['n_fragments'].sum()
#     reads = metrics['n_reads'].sum()
#     assert reads == dup_and_fragments

@pytest.mark.parametrize('metrics', [_gene_metrics, _cell_metrics])
def test_fragments_number_is_greater_than_molecule_number(metrics):
    """
    There should always be more fragments than molecules, as the minimum definition of a molecule is
    a fragment covered by a single read
    """
    assert np.all(metrics['n_molecules'] >= 1)
    assert np.all(metrics['n_fragments'] >= 1)
    assert np.all(metrics['n_fragments'] >= metrics['n_molecules'])

@pytest.mark.parametrize('metrics, key, expected_value', [
    (_cell_metrics, 'molecule_barcode_fraction_bases_above_30_mean',
     np.array([1.0000, 0.9500, 1.0000, 1.0000, 0.9778, 1.0000, 1.0000, 1.0000, 0.9833, 1.0000,
               1.0000, 1.0000, 1.0000, 1.0000, 0.9759, 1.0000, 1.0000, 0.9830, 1.0000, 1.0000,
               1.0000, 0.9778, 0.9783, 1.0000, 0.9800, 1.0000, 1.0000, 1.0000, 1.0000, 0.9500,
               1.0000, 0.9895, 1.0000, 0.9760, 1.0000, 1.0000, 1.0000, 0.9889, 1.0000, 0.9600,
               1.0000, 0.9909, 1.0000, 1.0000, 0.9556, 0.9800, 1.0000, 0.9000, 1.0000, 0.9588,
               1.0000, 1.0000, 0.9889, 0.8000, 0.9538, 0.9909, 0.9929, 0.9571])),
    # todo failing. Odd because mean is passing; catastrophic cancellation in the online method?
    # other methods that use the variance estimator work just fine. Something about the gene issue
    # that is identified by other methods below?
    # (_cell_metrics, 'molecule_barcode_fraction_bases_above_30_variance',
    #  np.array(
    #      [np.nan, 0.0050, np.nan, np.nan, 0.0019, 0.0000, 0.0000, np.nan, 0.0015, np.nan, 0.0000,
    #       0.0000, np.nan, 0.0000, 0.0048, 0.0000, 0.0000, 0.0029, 0.0000, np.nan, 0.0000, 0.0044,
    #       0.0109, 0.0000, 0.0020, 0.0000, 0.0000, np.nan, 0.0000, 0.0100, np.nan, 0.0010, 0.0000,
    #       0.0052, 0.0000, 0.0000, 0.0000, 0.0011, 0.0000, 0.0162, 0.0000, 0.0016, 0.0000, np.nan,
    #       0.0178, 0.0020, np.nan, np.nan, 0.0000, 0.0163, np.nan, np.nan, 0.0011, np.nan, 0.0147,
    #       0.0018, 0.0007, 0.0306])),
    (_cell_metrics, 'genomic_reads_fraction_bases_quality_above_30_mean',
     np.array(
         [0.3980, 0.6786, 0.5000, 0.9796, 0.7800, 0.7811, 0.9337, 0.8469, 0.6743, 0.4565, 0.8622,
          0.9762, 0.4925, 0.7857, 0.7478, 0.8561, 0.6327, 0.7948, 0.8405, 0.4286, 0.7735, 0.6445,
          0.7291, 0.8520, 0.6711, 0.6123, 0.8238, 0.5000, 0.8376, 0.5137, 0.7526, 0.7584, 0.7574,
          0.8379, 0.8490, 0.5000, 0.5983, 0.7489, 0.7755, 0.8107, 0.6963, 0.8363, 0.8896, 0.6186,
          0.7549, 0.7151, 1.0000, 0.5306, 0.8347, 0.7340, 0.8367, 0.8878, 0.7347, 0.4592, 0.7718,
          0.7583, 0.8439, 0.7576])),
    (_cell_metrics, 'genomic_reads_fraction_bases_quality_above_30_variance',
     np.array(
         [np.nan, 0.1812, np.nan, np.nan, 0.0266, 0.0461, 0.0042, np.nan, 0.0387, np.nan, 0.0178,
          0.0000, np.nan, 0.0002, 0.0455, 0.0342, 0.0588, 0.0359, 0.0247, np.nan, 0.0400, 0.0436,
          0.0754, 0.0005, 0.1140, 0.0617, 0.0400, np.nan, 0.0230, 0.0491, np.nan, 0.0608, 0.0556,
          0.0367, 0.0215, 0.0860, 0.2182, 0.0564, 0.0008, 0.0395, 0.0330, 0.0433, 0.0063, np.nan,
          0.0366, 0.0778, np.nan, np.nan, 0.0114, 0.0391, np.nan, np.nan, 0.0193, np.nan, 0.0288,
          0.0444, 0.0311, 0.0558])),
    (_cell_metrics, 'genomic_read_quality_mean',
     np.array(
         [25.3776, 32.5051, 27.7755, 39.9184, 34.3639, 34.5969, 37.4592, 35.9490, 31.6345, 26.5870,
          36.7500, 39.5374, 28.0896, 33.7041, 33.6079, 36.2787, 30.8472, 34.8402, 35.9327, 24.7755,
          34.3603, 31.0934, 33.2880, 36.7092, 31.9647, 30.2158, 35.3956, 27.6837, 35.8674, 27.4527,
          34.3918, 33.7323, 33.6425, 35.9552, 35.5694, 27.4184, 30.0479, 33.4621, 34.6633, 35.2128,
          32.4619, 35.7690, 36.9963, 30.0722, 33.6353, 32.6708, 39.8721, 28.0510, 35.9388, 33.1278,
          35.8265, 36.6633, 32.7188, 26.6429, 34.1053, 34.0012, 36.0956, 33.7704])),
    (_cell_metrics, 'genomic_read_quality_variance',
     np.array([np.nan, 92.5078, np.nan, np.nan, 18.9818, 29.9521, 6.6724, np.nan, 25.4164, np.nan,
               12.8541, 0.3790, np.nan, 0.0019, 28.7815, 24.6669, 37.7402, 22.8765, 16.5399, np.nan,
               22.9679, 26.2414, 44.8249, 0.5740, 70.4607, 42.5318, 24.9536, np.nan, 14.0772,
               32.6389, np.nan, 38.1213, 34.4094, 23.2517, 13.9110, 48.9622, 117.2337, 32.9814,
               0.3850, 24.3135, 17.8765, 26.5847, 5.2099, np.nan, 22.5846, 48.2133, np.nan, np.nan,
               5.6775, 23.9395, np.nan, np.nan, 12.9322, np.nan, 18.1475, 29.6960, 20.7504,
               34.9055])),
    # todo right now the metrics count reads that have no 'gene' towards molecules, whereas
    # the calculations in the notebook exclude them. We should decide which method we prefer.
    # there may be further problems.
    # (_cell_metrics, 'reads_per_molecule',
    #  np.array(
    #      [1.0000, 2.0000, np.nan, 1.0000, 9.0000, 2.4000, 2.0000, 1.0000, 3.0000, 1.0000, 3.0000,
    #       3.0000, 1.0000, np.nan, 2.4167, 4.3333, 1.2222, 5.8750, 1.3333, 1.0000, 1.2000, 1.5000,
    #       4.6000, 2.0000, 2.5000, 1.2000, 2.1429, 1.0000, 2.6364, 4.0000, 1.0000, 2.1111, 1.7273,
    #       6.2500, 5.0000, 1.3333, 2.0000, 2.2500, np.nan, 2.0000, 4.3333, 3.9286, 2.2000, 1.0000,
    #       1.5000, 1.6667, np.nan, 1.0000, 1.6667, 1.8889, 1.0000, 1.0000, 2.2500, 1.0000, 9.7500,
    #       11.0000, 4.0000, 1.5000])),
    (_cell_metrics, 'reads_per_fragment',
     np.array(
         [1.0000, 1.0000, 1.0000, 1.0000, 1.1250, 1.3333, 2.0000, 1.0000, 1.2000, 1.0000, 1.2000,
          3.0000, 1.0000, 2.0000, 1.3182, 1.4444, 1.1000, 1.4688, 1.1429, 1.0000, 1.2000, 1.2857,
          1.5333, 2.0000, 1.2500, 1.0000, 1.1538, 1.0000, 1.3182, 1.0000, 1.0000, 1.4615, 1.3571,
          1.3158, 1.2500, 1.3333, 1.0000, 1.1250, 1.0000, 1.1765, 1.0833, 1.4103, 1.1000, 1.0000,
          1.2857, 1.2500, 1.0000, 1.0000, 1.2500, 1.3077, 1.0000, 1.0000, 1.2857, 1.0000, 1.3929,
          1.5714, 1.4737, 1.1053])),
    # (_cell_metrics, 'fragments_per_molecule',  # todo failure depends on above reads_per_molecule
    #  np.array(
    #      [1.0000, 2.0000, np.nan, 1.0000, 8.0000, 1.8000, 1.0000, 1.0000, 2.5000, 1.0000, 2.5000,
    #       1.0000, 1.0000, np.nan, 1.8333, 3.0000, 1.1111, 4.0000, 1.1667, 1.0000, 1.0000, 1.1667,
    #       3.0000, 1.0000, 2.0000, 1.2000, 1.8571, 1.0000, 2.0000, 4.0000, 1.0000, 1.4444, 1.2727,
    #       4.7500, 4.0000, 1.0000, 2.0000, 2.0000, np.nan, 1.7000, 4.0000, 2.7857, 2.0000, 1.0000,
    #       1.1667, 1.3333, np.nan, 1.0000, 1.3333, 1.4444, 1.0000, 1.0000, 1.7500, 1.0000, 7.0000,
    #       7.0000, 2.7143, 1.3571])),
    (_gene_metrics, 'molecule_barcode_fraction_bases_above_30_mean',
     np.array([1.0000, 1.0000, 0.8000, 0.9885, 0.9833, 0.9857, 0.7000, 0.9444])),
    (_gene_metrics, 'molecule_barcode_fraction_bases_above_30_variance',
     np.array([np.nan, np.nan, np.nan, 0.0011, 0.0051, 0.0014, np.nan, 0.0120])),
    (_gene_metrics, 'genomic_reads_fraction_bases_quality_above_30_mean',
     np.array([0.8878, 0.3980, 0.4271, 0.8148, 0.7681, 0.7216, 0.1546, 0.5089])),
    (_gene_metrics, 'genomic_reads_fraction_bases_quality_above_30_variance',
     np.array([np.nan, np.nan, np.nan, 0.0282, 0.0346, 0.0537, np.nan, 0.0849])),
    (_gene_metrics, 'genomic_read_quality_mean',
     np.array([36.2143, 24.8469, 25.4792, 35.3664, 34.0956, 33.0364, 20.7423, 27.3078])),
    (_gene_metrics, 'genomic_read_quality_variance',
     np.array([np.nan, np.nan, np.nan, 18.4553, 21.6745, 33.6572, np.nan, 53.5457])),
    (_gene_metrics, 'reads_per_molecule',
     np.array([1.0000, 1.0000, 1.0000, 3.2500, 4.1525, 1.7500, 1.0000, 1.3846])),
    (_gene_metrics, 'reads_per_fragment',
     np.array([1.0000, 1.0000, 1.0000, 1.7333, 1.3920, 1.4000, 1.0000, 1.0588])),
    (_gene_metrics, 'fragments_per_molecule',
     np.array([1.0000, 1.0000, 1.0000, 1.8750, 2.9831, 1.2500, 1.0000, 1.3077])),
])
def test_higher_order_metrics_by_gene(metrics, key, expected_value):
    """Test metrics that depend on other metrics

    This class tests a very large number of higher-order metrics that examine the functionality of
    the test suite across all measured instances of the metric class. E.g. for cell metrics (class),
    each test will verify the value for each cell (instance).

    Parameters
    ----------
    metrics : pd.DataFrame
        Output from subclass of sctools.metrics.MetricAggregator
    key : str
        The column of metrics to interrogate in the parametrized test
    expected_value : np.ndarray
        An array of expected values

    """
    # need to sort, metrics are not always in same order as results.
    observed = sorted(np.nan_to_num(metrics[key].values).round(4))
    expected_value = sorted(np.nan_to_num(expected_value))
    assert observed == expected_value

@pytest.mark.parametrize('metrics, key, expected_value', [
    # todo failing; suspect related to problem with how fragments are defined
    # (_cell_metrics, 'fragments_with_single_read_evidence', 345),
    # todo failing. Does not make sense that this would also be a fragment issue.
    # (_cell_metrics, 'molecules_with_single_read_evidence', 130),
    (_gene_metrics, 'fragments_with_single_read_evidence', 155),
    (_gene_metrics, 'molecules_with_single_read_evidence', 42),
])
def test_single_read_evidence(metrics, key, expected_value):
    """
    We want to determine how many molecules and fragments are covered by only one read, as reads
    covered by multiple reads have much lower probabilities of being the result of error processes.
    """
    observed = metrics[key].sum()
    assert observed == expected_value

def split_metrics_file(metrics_file):
    """
    produces two mergeable on-disk metric files from a single file that contain the first 3/4
    of the file in the first output and the last 3/4 of the file in the second output, such that
    1/2 of the metrics in the two files overlap
    """
    with fileinput.FileInput([metrics_file], mode='r', openhook=fileinput.hook_compressed) as f:
        data = [line for line in f]

    header, data = data[0], data[1:]

    low_split, high_split = round(len(data) * .25), round(len(data) * .75)
    file_1, file_2 = [_test_dir + 'metrics_for_merging_%d.csv' % i for i in (1, 2)]

    with open(file_1, 'wb') as f:
        f.write(header + b'\n')
        for line in data[:high_split]:
            f.write(line + b'\n')

    with open(file_2, 'wb') as f:
        f.write(header + b'\n')
        for line in data[low_split:]:
            f.write(line + b'\n')

    return file_1, file_2

@pytest.fixture
def mergeable_cell_metrics():
    return split_metrics_file(_cell_metric_output_file)

@pytest.fixture
def mergeable_gene_metrics():
    return split_metrics_file(_gene_metric_output_file)

def test_merge_cell_metrics_cli(mergeable_cell_metrics):
    """test the sctools merge cell metrics CLI invocation"""
    return_call = TenXV2.merge_cell_metrics(
        args=['-o', _test_dir + '/merged-cell-metrics.csv.gz'] + list(mergeable_cell_metrics))
    assert return_call == 0

def test_merge_gene_metrics_cli(mergeable_gene_metrics):
    """test the sctools merge gene metrics CLI invocation"""
    return_call = TenXV2.merge_gene_metrics(
        args=['-o', _test_dir + '/merged-gene-metrics.csv.gz'] + list(mergeable_gene_metrics))
    assert return_call == 0

def test_merge_cell_metrics_does_not_correct_duplicates(mergeable_cell_metrics):
    """
    test takes offset cell metrics outputs and merges them. Cell metrics does not check for
    duplication, so should return a 2x length file.
    """
    output_file = os.path.join(_test_dir, 'merged_metrics.csv.gz')
    m = MergeCellMetrics(mergeable_cell_metrics, output_file)
    m.execute()

    merged_data = pd.read_csv(output_file, index_col=0)

    input_sizes = []
    for f in mergeable_cell_metrics:
        input_sizes.append(pd.read_csv(f, index_col=0).shape)
    target_rows = sum(row for row, col in input_sizes)

    target_cols = input_sizes[0][1]  # cols will always be the same

    assert merged_data.shape == (target_rows, target_cols)

def test_merge_gene_metrics_averages_over_multiply_detected_genes(mergeable_gene_metrics):
    output_file = os.path.join(_test_dir, 'merged_metrics.csv.gz')
    m = MergeGeneMetrics(mergeable_gene_metrics, output_file)
    m.execute()

    merged_data = pd.read_csv(output_file, index_col=0)

    input_data = pd.read_csv(mergeable_gene_metrics[0], index_col=0)
    target_cols = input_data.shape[1]

    input_genes = input_data.index
    for f in mergeable_gene_metrics[1:]:
        input_genes = input_genes.union(pd.read_csv(f, index_col=0).index)
    target_rows = len(input_genes)

    assert merged_data.shape == (target_rows, target_cols), '%s' % repr(merged_data)

@pytest.mark.parametrize('bam, gatherer', [
    (_gene_sorted_bam, GatherGeneMetrics),
    (_cell_sorted_bam, GatherCellMetrics),
])
def test_gzip_compression(bam: str, gatherer: Callable):
    """
    gzip compression should produce a .gz file which is identical when uncompressed to the
    uncompressed version
    """

    gz_fout = _test_dir + 'test_bam.csv.gz'
    g: MetricGatherer = gatherer(bam, gz_fout, compress=True)
    g.extract_metrics()
    gz_metrics = pd.read_csv(gz_fout, index_col=0)

    fout = _test_dir + 'test_bam.csv'
    g: MetricGatherer = gatherer(bam, fout, compress=False)
    g.extract_metrics()
    metrics = pd.read_csv(fout, index_col=0)

    assert np.allclose(gz_metrics.fillna(0).values, metrics.fillna(0).values)

max_stars_repo_name: benjamincarlin/sctools

max_issues_repo_name: benjamincarlin/sctools

max_forks_repo_name: benjamincarlin/sctools


text:
import textwrap
from math import sqrt

from pytest import approx

def magnitude(x, y):
    return sqrt(x * x + y * y)

def test_simple_math():
    assert abs(0.1 + 0.2) - 0.3 < 0.0001

def test_simple_math2():
    assert (-0.1 - 0.2) + 0.3 < 0.0001

def test_approx_simple():
    assert 0.1 + 0.2 == approx(0.3)

def test_approx_list():
    assert [0.1 + 1.2, 0.2 + 0.8] == approx([1.3, 1.0])

def test_approx_dict():
    values = {"v1": 0.1 + 1.2, "v2": 0.2 + 0.8}
    assert values == approx(dict(v1=1.3, v2=1.0))

def test_approx_simple_fail():
    assert 0.1 + 0.2 == approx(0.35)

def test_approx_list_fail():
    assert [0.1 + 1.2, 0.2 + 0.8] == approx([1.3, 1.1])

def test_approx_numpy():
    import numpy as np

    values = np.array([0.1, 0.2]) + np.array([1.2, 0.8])
    assert values == approx(np.array([1.3, 1.0]))

def test_magnitude_plain():
    assert abs(magnitude(8.0, 20.0) - 21.540659) < 0.00001

def test_magnitude():
    assert magnitude(8.0, 20.0) == approx(21.540659)

def get_default_health(class_name):
    assert class_name == "warrior"
    return 80

def test_default_health():
    health = get_default_health("warrior")
    assert health == 95

def get_default_player_class():
    return "warrior"

def test_default_player_class():
    x = get_default_player_class()
    assert x == "sorcerer"

def get_short_class_description(class_name):
    assert class_name == "warrior"
    return "A battle-hardened veteran, favors heavy armor and weapons."

def test_warrior_short_description():
    desc = get_short_class_description("warrior")
    assert (
        desc
        == "A battle-hardened veteran, can equip heavy armor and weapons."
    )

def get_long_class_description(class_name):
    assert class_name == "warrior"
    return textwrap.dedent(
        """\
    A seasoned veteran of many battles. High Strength and Dexterity
    allow to yield heavy armor and weapons, as well as carry
    more equipment while keeping a light roll. Weak in magic.
    """
    )

def test_warrior_long_description():
    desc = get_long_class_description("warrior")
    assert (
        desc
        == textwrap.dedent(
            """\
        A seasoned veteran of many battles. Strength and Dexterity
        allow to yield heavy armor and weapons, as well as carry
        more equipment. Weak in magic.
        """
        )
    )

def get_starting_equipment(class_name):
    assert class_name == "warrior"
    return ["long sword", "warrior set", "shield"]

def test_get_starting_equiment():
    expected = ["long sword", "shield"]
    assert get_starting_equipment("warrior") == expected

def test_long_list():
    x = [str(i) for i in range(100)]
    y = [str(i) for i in range(0, 100, 2)]
    assert x == y

def get_classes_starting_health():
    return {"warrior": 85, "sorcerer": 55, "knight": 95}

def test_starting_health():
    expected = {"warrior": 85, "sorcerer": 50}
    assert get_classes_starting_health() == expected

def get_player_classes():
    return {"warrior", "knight", "sorcerer"}

def test_player_classes():
    assert get_player_classes() == {"warrior", "sorcerer"}


text:
import os
import urllib.request
import torch
import numpy as np
from pathlib import Path
from cellpose import utils, models, io, transforms, dynamics

class CellPoseInteractiveModel:
    def __init__(
        self,
        model_dir=None,
        type="cellpose",
        resume=True,
        pretrained_model=None,
        save_freq=None,
        use_gpu=True,
        diam_mean=30.0,
        residual_on=1,
        learning_rate=0.001,
        batch_size=2,
        channels=(1, 2),
        resample=True,
        flow_threshold=0.4,
        cellprob_threshold=0.0,
        interp=True,
        default_diameter=30,
        style_on=0,
        disable_mkldnn=True,
    ):
        assert type == "cellpose"
        assert model_dir is not None
        device, gpu = models.assign_device(True, use_gpu)
        self.learning_rate = learning_rate
        self.channels = channels
        self.batch_size = batch_size
        self.model_dir = model_dir
        self.resample = resample
        self.cellprob_threshold = cellprob_threshold
        self.flow_threshold = flow_threshold
        self.interp = interp
        self.default_diameter = default_diameter
        if save_freq is None:
            if gpu:
                self.save_freq = 2000
            else:
                self.save_freq = 300
        else:
            self.save_freq = save_freq
        self.model = models.CellposeModel(
            gpu=gpu,
            device=device,
            torch=True,
            pretrained_model=pretrained_model,
            diam_mean=diam_mean,
            residual_on=residual_on,
            style_on=style_on,
            concatenation=0,
            disable_mkldnn=disable_mkldnn,
        )
        os.makedirs(self.model_dir, exist_ok=True)
        if resume:
            resume_weights_path = os.path.join(self.model_dir, "snapshot")
            if os.path.exists(resume_weights_path):
                print("Resuming model from " + resume_weights_path)
                self.load(resume_weights_path)
                # disable pretrained model
                pretrained_model = False
            else:
                print("Skipping resume, snapshot does not exist")
        # load pretrained model weights if not specified
        if pretrained_model is None:
            cp_model_dir = Path.home().joinpath(".cellpose", "models")
            os.makedirs(cp_model_dir, exist_ok=True)
            weights_path = cp_model_dir / "cytotorch_0"
            if not weights_path.exists():
                urllib.request.urlretrieve(
                    "https://www.cellpose.org/models/cytotorch_0", str(weights_path)
                )
            if not (cp_model_dir / "size_cytotorch_0.npy").exists():
                urllib.request.urlretrieve(
                    "https://www.cellpose.org/models/size_cytotorch_0.npy",
                    str(cp_model_dir / "size_cytotorch_0.npy"),
                )

            print("loading pretrained cellpose model from " + str(weights_path))
            if gpu:
                self.model.net.load_state_dict(
                    torch.load(str(weights_path)), strict=False
                )
            else:
                self.model.net.load_state_dict(
                    torch.load(str(weights_path), map_location=torch.device("cpu")),
                    strict=False,
                )
        self._iterations = 0
        momentum = 0.9
        weight_decay = 0.00001
        # Note: we are using Adam for adaptive learning rate which is different from the SDG used by cellpose
        # this support to make the training more robust to different settings
        self.model.optimizer = torch.optim.Adam(
            self.model.net.parameters(), lr=self.learning_rate, weight_decay=1e-5
        )
        self.model._set_criterion()

    def get_config(self):
        """augment the images and labels
        Parameters
        --------------
        None

        Returns
        ------------------
        config: dict
            a dictionary contains the following keys:
            1) `batch_size` the batch size for training
        """
        return {"batch_size": self.batch_size}

    def transform_labels(self, label_image):
        """transform the labels which will be used as training target
        Parameters
        --------------
        label_image: array [width, height, channel]
            a label image

        Returns
        ------------------
        array [width, height, channel]
            the transformed label image
        """
        assert label_image.ndim == 3 and label_image.shape[2] == 1
        label_image = label_image[:, :, 0]
        veci = dynamics.masks_to_flows(label_image)[0]
        # concatenate flows with cell probability
        flows = np.concatenate(
            (np.stack([label_image, label_image > 0.5], axis=0), veci), axis=0
        ).astype(np.float32)
        return flows.transpose(1, 2, 0)

    def augment(self, images, labels):
        """augment the images and labels
        Parameters
        --------------
        images: array [batch_size, width, height, channel]
            a batch of input images

        labels: array [batch_size, width, height, channel]
            a batch of labels

        Returns
        ------------------
        (images, labels) both are: array [batch_size, width, height, channel]
            augmented images and labels
        """
        images = [images[i].transpose(2, 0, 1) for i in range(images.shape[0])]
        labels = [labels[i].transpose(2, 0, 1) for i in range(labels.shape[0])]

        nimg = len(images)
        # check that arrays are correct size
        if nimg != len(labels):
            raise ValueError("train images and labels not same length")
        if labels[0].ndim < 2 or images[0].ndim < 2:
            raise ValueError(
                "training images or labels are not at least two-dimensional"
            )

        # make data correct shape and normalize it so that 0 and 1 are 1st and 99th percentile of data
        images, _, _ = transforms.reshape_and_normalize_data(
            images, test_data=None, channels=self.channels, normalize=True
        )

        # compute average cell diameter
        diam_train = np.array(
            [utils.diameters(labels[k][0])[0] for k in range(len(labels))]
        )
        diam_train[diam_train < 5] = 5.0
        scale_range = 0.5
        rsc = diam_train / self.model.diam_mean
        imgi, lbl, scale = transforms.random_rotate_and_resize(
            images,
            Y=[label[1:] for label in labels],
            rescale=rsc,
            scale_range=scale_range,
            unet=False,
        )
        return imgi, lbl

    def train(
        self,
        images,
        labels,
        iterations=1,
        rescale=True,
    ):
        imgi, lbl = self.augment(images, labels)
        train_loss = self.model._train_step(imgi, lbl)
        self._iterations += len(images)
        if self._iterations % self.save_freq == 0:
            self.save(os.path.join(self.model_dir, "snapshot"))
        return train_loss

    def train_on_batch(self, X, y):
        """train the model for one iteration
        Parameters
        --------------
        X: array [batch_size, width, height, channel]
            the input image with 2 channels

        y: array [batch_size, width, height, channel]
            the mask (a.k.a label) image with one unique pixel value for one object
            if the shape is [1, width, height], then y is the label image
            otherwise, it should have channel=4 where the 1st channel is the label image
            and the other 3 channels are the precomputed flow image

        Returns
        ------------------
        loss value
        """
        assert X.shape[0] == y.shape[0] and X.ndim == 4

        return self.train(X, y)

    def predict(self, X):
        """predict the model for one input image
        Parameters
        --------------
        X: array [batch_size, width, height, channel]
            the input image with 2 channels

        Returns
        ------------------
        array [batch_size, width, height, channel]
            the predicted label image
        """
        assert X.ndim == 4
        X = [X[i].transpose(2, 0, 1) for i in range(X.shape[0])]
        masks, flows, diams = self.model.eval(
            X,
            channels=self.channels,
            diameter=self.default_diameter,
            do_3D=False,
            net_avg=False,
            augment=False,
            resample=self.resample,
            flow_threshold=self.flow_threshold,
            cellprob_threshold=self.cellprob_threshold,
            batch_size=self.batch_size,
            interp=self.interp,
        )
        # io.masks_flows_to_seg(image, masks, flows, diams, image_name, channels)
        # io.save_masks(path, masks, flows, image_name, png=True, tif=False)
        return np.stack([np.expand_dims(mask, axis=2) for mask in masks], axis=0)

    def save(self, file_path):
        """save the model
        Parameters
        --------------
        file_path: string
            the model file path

        Returns
        ------------------
        None
        """
        self.model.net.save_model(file_path)

    def load(self, file_path):
        """load the model
        Parameters
        --------------
        file_path: string
            the model file path

        Returns
        ------------------
        None
        """
        self.model.net.load_state_dict(torch.load(file_path), strict=True)

    def export(self, format, file_path):
        """export the model into different format
        Parameters
        --------------
        format: string
            the model format to be exported
        file_path: string
            the file path to the exported model

        Returns
        ------------------
        None
        """
        raise NotImplementedError


text:
# -*- coding: utf-8 -*-
"""
Created on Sun Jul 12 14:48:23 2020

@author: Rafael Arenhart
"""

import os

import numpy as np
from numba import njit, prange

CURRENT_PATH = os.path.dirname(os.path.realpath(__file__))

isotropic_templates = np.zeros(256, dtype = np.uint8)
with open(os.path.join(CURRENT_PATH, 'minkowsky_templates.csv'), mode = 'r') as file:
    for line in file:
        index, n, *templates = [int(i.strip()) for i in line.strip().split(',')]
        for template in templates:
            isotropic_templates[template] = index

indexing_template = np.zeros((2,2,2), dtype = np.uint8)
for i, j, k in [(a, b, c) for a in range(2) for b in range(2) for c in range(2)]:
    indexing_template[i, j, k] = 2 ** (i + 2*j + 4*k)

with open(os.path.join(CURRENT_PATH, 'minkowsy_values.csv'), mode = 'r') as file:
    header = file.readline().strip().split(',')
    minkowsky_names = [i.strip() for i in header[1:]]
    divisors = file.readline().strip().split(',')
    minkowsky_divisors = np.array([int(i) for i in divisors[1:]], dtype=np.int8)
    minkowsky_values = np.zeros((22,6), dtype=np.int64)
    for line in file:
        index, *vals = [int(i.strip()) for i in line.split(',')]
        minkowsky_values[index] = vals

@njit
def cube2index(img):
    return (img * indexing_template).sum()

@njit
def get_minkowsky_functionals(img):

    x, y, z = img.shape

    results = np.zeros(6, dtype=np.int64)
    for i in range(x-1):
        for j in range(y-1):
            for k in range(x-1):
                template_index = cube2index(img[i:i+2,j:j+2,k:k+2])
                minkowsky_template = isotropic_templates[template_index]
                results += minkowsky_values[minkowsky_template]
    results = results / minkowsky_divisors

    return results

@njit(parallel = True)
def get_minkowsky_functionals_parallel(img):

    x, y, z = img.shape

    results = np.zeros(6, dtype=np.int64)
    for i in prange(x-1):
        for j in range(y-1):
            for k in range(x-1):
                template_index = cube2index(img[i:i+2,j:j+2,k:k+2])
                minkowsky_template = isotropic_templates[template_index]
                results += minkowsky_values[minkowsky_template]
    results = results / minkowsky_divisors

    return results

max_stars_repo_name: Arenhart/Tools

max_issues_repo_name: Arenhart/Tools

max_forks_repo_name: Arenhart/Tools

reason: import numpy,from numba


text:
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import numpy as np
import tensorflow as tf
from code.model import get_model
from code.train_single_client import train_single_client
from code.dataloader import preprocessing
import pandas as pd

def train(epochs=5):

	x, _ = preprocessing('./dataset_hand_movement/user_a.csv')
	x_mean = np.mean(x, axis=0)
	x_std = np.std(x, axis=0)

	model = get_model()
	aggregate_weights = model.get_weights()
	client_performances = []
	test_performances = []
	for _ in range(epochs):
		all_weights = []
		performances = []
		for client in ['./dataset_hand_movement/'+i+'.csv' for i in ['user_a', 'user_b', 'user_c', 'user_d']]:
			weights, history = train_single_client(aggregate_weights, client, x_mean, x_std)
			loss = history.history['val_loss'][-1]
			accuracy = history.history['val_accuracy'][-1]
			performances.append([loss, accuracy])
			all_weights.append(weights)
		aggregate_weights = np.stack(all_weights)
		aggregate_weights = np.mean(aggregate_weights, axis=0)
		model.set_weights(aggregate_weights)
		client_performances.append(performances)

	client_performances = np.array(client_performances)
	names = np.array([['client_id_'+str(client_id)+'_loss', 'client_id_'+str(client_id)+'_acc'] for client_id in range(client_performances.shape[1])]).flatten()
	client_performances = client_performances.reshape(client_performances.shape[0], client_performances.shape[1]*client_performances.shape[2])
	client_performances = pd.DataFrame(client_performances)
	client_performances.columns = [i for i in names]

	return aggregate_weights, client_performances

if __name__ == '__main__':
	weights, performances = train()
	model = get_model()
	model.set_weights(weights)
	model.save('trained_model.h5')
	performances.to_csv('./history.csv', index=False)

max_stars_repo_name: AmanPriyanshu/FedEEG

max_issues_repo_name: AmanPriyanshu/FedEEG

max_forks_repo_name: AmanPriyanshu/FedEEG