text:
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import tensorflow as tf

from tensorflow.keras import datasets, layers, models
import numpy as np
import time, sys, math, random

from src import tensorfi2 as tfi

(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
train_images, test_images = train_images / 255.0, test_images / 255.0

model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))

model.add(layers.Flatten())
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10))

model.compile(optimizer='adam',
    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    metrics=['accuracy'])

'''
# Change to True if you want to train from scratch
train = False

if(train):
    # Save the untrained weights for future training with modified dataset
    model.save_weights('h5/cnnc-untrained.h5')

    model.fit(train_images, train_labels, batch_size=100, epochs=5,
        validation_data=(test_images, test_labels))

    model.save_weights('h5/cnnc-trained.h5')

else:
    model.load_weights('h5/cnnc-trained.h5')

    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)
    print("Accuracy before faults:", test_acc)

    tfi.inject(model=model, confFile="confFiles/sample.yaml")

    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)
    print("Accuracy after faults:", test_acc)
'''
conf = sys.argv[1]
filePath = sys.argv[2]
filePath = os.path.join(filePath, "res.csv")

f = open(filePath, "w")
numFaults = int(sys.argv[3])
numInjections = int(sys.argv[4])
offset = 10
num = test_images.shape[0]

totsdc = 0.0

ind = []
init = random.sample(range(num), numInjections+offset)
model.load_weights('h5/cnnc-trained.h5')

for i in init:
        test_loss, test_acc = model.evaluate(test_images[i:i+1], test_labels[i:i+1], verbose=0)
        if(test_acc == 1.):
                ind.append(i)
ind = ind[:numInjections]

start = time.time()
for i in range(numFaults):
    model.load_weights('h5/cnnc-trained.h5')

    tfi.inject(model=model, confFile=conf)

    sdc = 0.
    for i in ind:
        test_loss, test_acc = model.evaluate(test_images[i:i+1], test_labels[i:i+1], verbose=0)
        if(test_acc == 0.):
            sdc = sdc + 1.

    f.write(str(sdc/numInjections))
    f.write("\n")
    totsdc = totsdc + sdc
f.write("\n")
f.write(str(totsdc/(numFaults*numInjections)))
f.write("\n")
f.write("Time for %d injections: %f seconds" % (numFaults*numInjections, time.time() - start))
f.close()

ext: py

reason: import numpy


text:
import tensorflow as tf
import numpy as np

class Actor(tf.keras.Model):
    def __init__(self):
        ...

    def call(self, x):
        ...

max_stars_repo_name: dreinq/DeepQ

max_issues_repo_name: dreinq/DeepQ

max_forks_repo_name: dreinq/DeepQ


text:
# Copyright (c) 2017 Sony Corporation. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest
import numpy as np
import nnabla.functions as F
from nbla_test_utils import list_context, function_tester

ctxs = list_context('Pad')

def ref_pad_constant(x, pad_width, mode, constant_value):
    pad_width = list(zip(pad_width[::2], pad_width[1::2]))
    pad_width = (x.ndim - len(pad_width)) * [(0, 0)] + pad_width
    return np.pad(x, pad_width, mode, constant_values=constant_value)

@pytest.mark.parametrize("seed", [313])
@pytest.mark.parametrize("ctx, func_name", ctxs)
@pytest.mark.parametrize("inshape, pad_width", [
    ((4,), (2, 9)),
    ((5,), (4, 3)),
    ((4, 5), (5, 4)),
    ((3, 6), (1, 2, 3, 4)),
    ((3, 5, 7), (2, 3)),
    ((5, 4, 6), (2, 3, 1, 4)),
    ((2, 3, 5), (1, 2, 3, 4, 5, 6)),
    ((1, 2, 3, 4), (2, 3, 4, 5)),
    ((2, 2, 3, 3), (2, 2, 2, 2, 2, 2, 2, 2)),
])
@pytest.mark.parametrize("constant_value", [0.11, -1.1])
def test_pad_constant_forward_backward(seed, ctx, func_name, inshape,
                                       pad_width, constant_value):
    rng = np.random.RandomState(seed)
    inputs = [rng.randn(*inshape).astype(np.float32)]
    func_args = [pad_width, "constant", constant_value]
    function_tester(rng, F.pad, ref_pad_constant, inputs, ctx=ctx, dstep=1e-1,
                    func_name=func_name, func_args=func_args)

def ref_pad_reflect(x, pad_width, mode):
    pad_width = list(zip(pad_width[::2], pad_width[1::2]))
    pad_width = (x.ndim - len(pad_width)) * [(0, 0)] + pad_width
    return np.pad(x, pad_width, mode)

@pytest.mark.parametrize("seed", [313])
@pytest.mark.parametrize("ctx, func_name", ctxs)
@pytest.mark.parametrize("inshape, pad_width", [
    ((4,), (2, 9)),
    ((5,), (4, 3)),
    ((4, 5), (5, 4)),
    ((3, 6), (1, 2, 3, 4)),
    ((3, 5, 7), (2, 3)),
    ((5, 4, 6), (2, 3, 1, 4)),
    ((2, 3, 5), (1, 2, 3, 4, 5, 6)),
    ((1, 2, 3, 4), (2, 3, 4, 5)),
    ((2, 2, 3, 3), (2, 2, 2, 2, 2, 2, 2, 2)),
])
def test_pad_reflect_forward_backward(seed, ctx, func_name, inshape,
                                      pad_width):
    rng = np.random.RandomState(seed)
    inputs = [rng.randn(*inshape).astype(np.float32)]
    func_args = [pad_width, "reflect"]
    function_tester(rng, F.pad, ref_pad_reflect, inputs, ctx=ctx, dstep=1e-1,
                    func_name=func_name, func_args=func_args)

@pytest.mark.parametrize("seed", [313])
@pytest.mark.parametrize("ctx, func_name", ctxs)
@pytest.mark.parametrize("inshape, pad_width", [
    ((4,), (2, 9)),
    ((5,), (4, 3)),
    ((4, 5), (5, 4)),
    ((3, 6), (1, 2, 3, 4)),
    ((3, 5, 7), (2, 3)),
    ((5, 4, 6), (2, 3, 1, 4)),
    ((2, 3, 5), (1, 2, 3, 4, 5, 6)),
    ((1, 2, 3, 4), (2, 3, 4, 5)),
    ((2, 2, 3, 3), (2, 2, 2, 2, 2, 2, 2, 2)),
])
@pytest.mark.parametrize("constant_value", [0.11, -1.1])
def test_pad_constant_double_backward(seed, ctx, func_name, inshape,
                                      pad_width, constant_value):
    from nbla_test_utils import backward_function_tester
    rng = np.random.RandomState(seed)
    inputs = [rng.randn(*inshape).astype(np.float32)]
    func_args = [pad_width, "constant", constant_value]
    backward_function_tester(rng, F.pad, ref_pad_constant, inputs, ctx=ctx,
                             dstep=1e-3, atol_b=1e-2, atol_accum=1e-2,
                             func_name=func_name, func_args=func_args)

@pytest.mark.parametrize("seed", [313])
@pytest.mark.parametrize("ctx, func_name", ctxs)
@pytest.mark.parametrize("inshape, pad_width", [
    ((4,), (2, 9)),
    ((5,), (4, 3)),
    ((4, 5), (5, 4)),
    ((3, 6), (1, 2, 3, 4)),
    ((3, 5, 7), (2, 3)),
    ((5, 4, 6), (2, 3, 1, 4)),
    ((2, 3, 5), (1, 2, 3, 4, 5, 6)),
    ((1, 2, 3, 4), (2, 3, 4, 5)),
    ((2, 2, 3, 3), (2, 2, 2, 2, 2, 2, 2, 2)),
])
def test_pad_reflect_double_backward(seed, ctx, func_name, inshape,
                                     pad_width):
    from nbla_test_utils import backward_function_tester
    rng = np.random.RandomState(seed)
    inputs = [rng.randn(*inshape).astype(np.float32)]
    func_args = [pad_width, "reflect"]
    backward_function_tester(rng, F.pad, ref_pad_reflect, inputs, ctx=ctx,
                             dstep=1e-3, atol_b=5e-2, atol_accum=5e-2,
                             func_name=func_name, func_args=func_args)

max_stars_repo_name: sdonatti/nnabla


text:
# #!C:\Users\Piotr\AppData\Local\Programs\Python\Python35-32\python.exe
#
# '''A simple OpenGL viewer for C3D files.'''
#
# import c3d
# import climate
# import collections
# import contextlib
# import numpy as np
# import pyglet
#
# from pyglet.gl import *
#
# climate.add_arg('inputs', nargs='+', metavar='FILE', help='show these c3d files')
#
# BLACK = (0, 0, 0)
# WHITE = (1, 1, 1)
# RED = (1, 0.2, 0.2)
# YELLOW = (1, 1, 0.2)
# ORANGE = (1, 0.7, 0.2)
# GREEN = (0.2, 0.9, 0.2)
# BLUE = (0.2, 0.3, 0.9)
# COLORS = (WHITE, RED, YELLOW, GREEN, BLUE, ORANGE)
#
#
# @contextlib.contextmanager
# def gl_context(scale=None, translate=None, rotate=None, mat=None):
#     glPushMatrix()
#     if mat is not None:
#         glMultMatrixf(vec(*mat))
#     if translate is not None:
#         glTranslatef(*translate)
#     if rotate is not None:
#         glRotatef(*rotate)
#     if scale is not None:
#         glScalef(*scale)
#     yield
#     glPopMatrix()
#
#
# def vec(*args):
#     return (GLfloat * len(args))(*args)
#
#
# def sphere_vertices(n=2):
#     idx = [[0, 1, 2], [0, 5, 1], [0, 2, 4], [0, 4, 5],
#            [3, 2, 1], [3, 4, 2], [3, 5, 4], [3, 1, 5]]
#     vtx = list(np.array([
#         [ 1, 0, 0], [0,  1, 0], [0, 0,  1],
#         [-1, 0, 0], [0, -1, 0], [0, 0, -1]], 'f'))
#     for _ in range(n):
#         idx_ = []
#         for ui, vi, wi in idx:
#             u, v, w = vtx[ui], vtx[vi], vtx[wi]
#             d, e, f = u + v, v + w, w + u
#             di = len(vtx)
#             vtx.append(d / np.linalg.norm(d))
#             ei = len(vtx)
#             vtx.append(e / np.linalg.norm(e))
#             fi = len(vtx)
#             vtx.append(f / np.linalg.norm(f))
#             idx_.append([ui, di, fi])
#             idx_.append([vi, ei, di])
#             idx_.append([wi, fi, ei])
#             idx_.append([di, ei, fi])
#         idx = idx_
#     vtx = np.array(vtx, 'f').flatten()
#     return np.array(idx).flatten(), vtx, vtx
#
#
# class Viewer(pyglet.window.Window):
#     def __init__(self, c3d_reader, trace=None, paused=False):
#         platform = pyglet.window.get_platform()
#         display = platform.get_default_display()
#         screen = display.get_default_screen()
#         try:
#             config = screen.get_best_config(Config(
#                 alpha_size=8,
#                 depth_size=24,
#                 double_buffer=True,
#                 sample_buffers=1,
#                 samples=4))
#         except pyglet.window.NoSuchConfigException:
#             config = screen.get_best_config(Config())
#
#         super(Viewer, self).__init__(
#             width=800, height=450, resizable=True, vsync=False, config=config)
#
#         self._frames = c3d_reader.read_frames(copy=False)
#         self._frame_rate = c3d_reader.header.frame_rate
#
#         self._maxlen = 16
#         self._trails = [[] for _ in range(c3d_reader.point_used)]
#         self._reset_trails()
#
#         self.trace = trace
#         self.paused = paused
#
#         self.zoom = 5
#         self.ty = 0
#         self.tz = -1
#         self.ry = 30
#         self.rz = 30
#
#         #self.fps = pyglet.clock.ClockDisplay()
#
#         self.on_resize(self.width, self.height)
#
#         glEnable(GL_BLEND)
#         glEnable(GL_COLOR_MATERIAL)
#         glEnable(GL_CULL_FACE)
#         glEnable(GL_DEPTH_TEST)
#         glEnable(GL_LIGHTING)
#         glEnable(GL_NORMALIZE)
#         glEnable(GL_POLYGON_SMOOTH)
#
#         glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
#         glDepthFunc(GL_LEQUAL)
#         glCullFace(GL_BACK)
#         glFrontFace(GL_CCW)
#         glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST)
#         glHint(GL_POLYGON_SMOOTH_HINT, GL_NICEST)
#         glShadeModel(GL_SMOOTH)
#
#         glLightfv(GL_LIGHT0, GL_AMBIENT, vec(0.2, 0.2, 0.2, 1.0))
#         glLightfv(GL_LIGHT0, GL_DIFFUSE, vec(1.0, 1.0, 1.0, 1.0))
#         glLightfv(GL_LIGHT0, GL_POSITION, vec(3.0, 3.0, 10.0, 1.0))
#         glEnable(GL_LIGHT0)
#
#         BLK = [100, 100, 100] * 6
#         WHT = [150, 150, 150] * 6
#         N = 10
#         z = 0
#         vtx = []
#         for i in range(N, -N, -1):
#             for j in range(-N, N, 1):
#                 vtx.extend((j,   i, z, j, i-1, z, j+1, i,   z,
#                             j+1, i, z, j, i-1, z, j+1, i-1, z))
#
#         self.floor = pyglet.graphics.vertex_list(
#             len(vtx) // 3,
#             ('v3f/static', vtx),
#             ('c3B/static', ((BLK + WHT) * N + (WHT + BLK) * N) * N),
#             ('n3i/static', [0, 0, 1] * (len(vtx) // 3)))
#
#         idx, vtx, nrm = sphere_vertices()
#         self.sphere = pyglet.graphics.vertex_list_indexed(
#             len(vtx) // 3, idx, ('v3f/static', vtx), ('n3f/static', nrm))
#
#     def on_mouse_scroll(self, x, y, dx, dy):
#         if dy == 0: return
#         self.zoom *= 1.1 ** (-1 if dy < 0 else 1)
#
#     def on_mouse_drag(self, x, y, dx, dy, buttons, modifiers):
#         if buttons == pyglet.window.mouse.LEFT:
#             # pan
#             self.ty += 0.03 * dx
#             self.tz += 0.03 * dy
#         else:
#             # roll
#             self.ry += 0.2 * -dy
#             self.rz += 0.2 * dx
#         #print('z', self.zoom, 't', self.ty, self.tz, 'r', self.ry, self.rz)
#
#     def on_resize(self, width, height):
#         glViewport(0, 0, width, height)
#         glMatrixMode(GL_PROJECTION)
#         glLoadIdentity()
#         glu.gluPerspective(45, float(width) / height, 1, 100)
#
#     def on_key_press(self, key, modifiers):
#         k = pyglet.window.key
#         if key == k.ESCAPE:
#             pyglet.app.exit()
#         elif key == k.SPACE:
#             self.paused = False if self.paused else True
#         elif key == k.PLUS or key == k.EQUAL:
#             self._maxlen *= 2
#             self._reset_trails()
#         elif key == k.UNDERSCORE or key == k.MINUS:
#             self._maxlen = max(1, self._maxlen / 2)
#             self._reset_trails()
#         elif key == k.RIGHT:
#             skip = int(self._frame_rate)
#             if modifiers:
#                 skip *= 10
#             [self._next_frame() for _ in range(skip)]
#
#     def on_draw(self):
#         self.clear()
#
#         # http://njoubert.com/teaching/cs184_fa08/section/sec09_camera.pdf
#         glMatrixMode(GL_MODELVIEW)
#         glLoadIdentity()
#         gluLookAt(1, 0, 0, 0, 0, 0, 0, 0, 1)
#         glTranslatef(-self.zoom, 0, 0)
#         glTranslatef(0, self.ty, self.tz)
#         glRotatef(self.ry, 0, 1, 0)
#         glRotatef(self.rz, 0, 0, 1)
#
#         self.floor.draw(GL_TRIANGLES)
#
#         for t, trail in enumerate(self._trails):
#             glColor4f(*(COLORS[t % len(COLORS)] + (0.7, )))
#             point = None
#             glBegin(GL_LINES)
#             for point in trail:
#                 glVertex3f(*point)
#             glEnd()
#             with gl_context(translate=point, scale=(0.02, 0.02, 0.02)):
#                 self.sphere.draw(GL_TRIANGLES)
#
#     def _reset_trails(self):
#         self._trails = [collections.deque(t, self._maxlen) for t in self._trails]
#
#     def _next_frame(self):
#         try:
#             return self._frames.next()
#         except StopIteration:
#             pyglet.app.exit()
#
#     def update(self, dt):
#         if self.paused:
#             return
#         for trail, point in zip(self._trails, self._next_frame()[1]):
#             if point[3] > -1 or not len(trail):
#                 trail.append(point[:3] / 1000.)
#             else:
#                 trail.append(trail[-1])
#
#     def mainloop(self):
#         pyglet.clock.schedule_interval(self.update, 0.1 / self._frame_rate)
#         pyglet.app.run()
#
#
# def main(args):
#     for filename in args.inputs:
#         try:
#             Viewer(c3d.Reader(open(filename, 'rb'))).mainloop()
#         except StopIteration:
#             pass
#
#
# if __name__ == '__main__':
#     climate.call(main)

import c3d
with open('G8.c3d', 'rb') as handle:
    reader = c3d.Reader(handle)
    for points in reader.read_frames():
        print(points)
    for i, (points, analog) in enumerate(reader.read_frames()):
        print('Frame {}: {}'.format(i, points.round(2)))

max_stars_repo_name: piotrbla/pyExamples

max_issues_repo_name: piotrbla/pyExamples

max_forks_repo_name: piotrbla/pyExamples


text:
#  Copyright 2020 Rikai Authors
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""Mixins
"""
from __future__ import annotations

from abc import ABC, abstractmethod
from io import BytesIO
from pathlib import Path
from typing import BinaryIO, Mapping, Optional, Union

# Third Party
import numpy as np

# Rikai
from rikai.internal.uri_utils import uri_equal
from rikai.io import open_uri

__all__ = ["ToNumpy", "Asset", "Displayable", "Drawable", "ToDict"]

class ToNumpy(ABC):
    """ToNumpy Mixin."""

    @abstractmethod
    def to_numpy(self) -> np.ndarray:
        """Returns the content as a numpy ndarray."""

class ToPIL(ABC):
    """ToPIL Mixin."""

    @abstractmethod
    def to_pil(self) -> "PIL.Image.Image":
        pass

class ToDict(ABC):
    """ToDict Mixin"""

    @abstractmethod
    def to_dict(self) -> dict:
        pass

class Displayable(ABC):
    """Mixin for notebook viz"""

    @abstractmethod
    def display(self, **kwargs) -> "IPython.display.DisplayObject":
        """Return an IPython.display.DisplayObject"""

class Drawable(ABC):
    """Mixin for a class that is drawable"""

    @abstractmethod
    def _render(self, render: "rikai.viz.Renderer", **kwargs) -> None:
        """Render the object using render."""

    def __matmul__(self, style: Union[dict, "rikai.viz.Style"]) -> Drawable:
        from rikai.viz import Style

        if not isinstance(style, (Mapping, Style)):
            raise ValueError(
                f"Must decorate drawable with a dict or style object"
                " but got {style}"
            )
        if isinstance(style, dict):
            style = Style(**style)
        return style(self)

class Asset(ABC):
    """cloud asset Mixin.

    Rikai uses asset to store certain blob on the cloud storage, to facilitate
    the functionality like fast query, example inspections, and etc.

    An asset is also a cell in a DataFrame for analytics. It offers both fast
    query on columnar format and easy tooling to access the actual data.

    Attributes
    ----------
    data : bytes, optional
        Embedded data
    uri : str
        URI of the external storage.
    """

    def __init__(
        self, data: Optional[bytes] = None, uri: Union[str, Path] = None
    ) -> None:
        assert (data is None) ^ (uri is None)
        assert data is None or isinstance(
            data, (bytes, bytearray)
        ), f"Expect bytes got {type(data)}"
        assert uri is None or isinstance(
            uri, (str, Path)
        ), f"Expect str/Path for uri, got {type(uri)}"
        self.data = data
        self.uri = str(uri) if uri is not None else None

    def __eq__(self, o: object) -> bool:
        return isinstance(o, Asset) and uri_equal(self.uri, o.uri)

    @property
    def is_embedded(self) -> bool:
        """Returns True if this Asset has embedded data."""
        return self.data is not None

    def open(self, mode="rb") -> BinaryIO:
        """Open the asset and returned as random-accessible file object."""
        if self.is_embedded:
            return BytesIO(self.data)
        return open_uri(self.uri, mode=mode)

max_stars_repo_name: changhiskhan/rikai

max_issues_repo_name: askintution/rikai

max_forks_repo_name: askintution/rikai


text:
#!/usr/bin/env python

import sys
import json
from os.path import exists, splitext

from enum import Enum
from collections import OrderedDict
from argparse import ArgumentParser

import networkx as nx
import matplotlib.pyplot as plt

# data parsing
class NodeLabel(Enum):
    ROOT_CFG 		= 1
    ROOT_SLICE		= 2
    HOST_FETCH 		= 3

class FlowNode(object):
    def __init__(self, nid, info):
        self.nid = nid
        self.info = info

        self.pred = []
        self.succ = []

        self.label = set()

    def addPred(self, node):
        self.pred.append(node)

    def addSucc(self, node):
        self.succ.append(node)

    def addLabel(self, lab):
        self.label.add(lab)

    def hasPred(self, nid):
        for n in self.pred:
            if n.nid == nid:
                return True

        return False

    def hasSucc(self, nid):
        for n in self.succ:
            if n.nid == nid:
                return True

        return False

class FlowItem(object):
    def __init__(self, nid, perm):
        self.nid = nid

        self.perm = perm
        self.temp = None

    def update(self, temp):
        self.temp = temp

    def restore(self):
        self.temp = None

class FlowGraph(object):
    def __init__(self, cfg):
        self.repo = OrderedDict()

        # init all nodes
        for rec in cfg["item"]:
            nid = rec["name"]
            self.repo[nid] = FlowItem(nid, FlowNode(nid, rec["size"]))

        # add links
        for rec in cfg["item"]:
            node = self.repo[rec["name"]].perm

            for elem in rec["pred"]:
                node.addPred(self.repo[elem].perm)

            for elem in rec["succ"]:
                node.addSucc(self.repo[elem].perm)

        # set root
        self.repo[cfg["root"]].perm.addLabel(NodeLabel.ROOT_CFG)

    def update(self, cfg, host):
        # update all nodes
        for rec in cfg["item"]:
            nid = rec["name"]
            assert nid in self.repo
            self.repo[nid].update(FlowNode(nid, rec["size"]))

        # add links
        for rec in cfg["item"]:
            node = self.repo[rec["name"]].temp

            for elem in rec["pred"]:
                node.addPred(self.repo[elem].temp)

            for elem in rec["succ"]:
                node.addSucc(self.repo[elem].temp)

        # set root
        self.repo[cfg["root"]].temp.addLabel(NodeLabel.ROOT_SLICE)

        # set host
        self.repo[host].temp.addLabel(NodeLabel.HOST_FETCH)

    def restore(self):
        for k in self.repo:
            self.repo[k].restore()

# visualizer
class NodeFormat(object):
    def __init__(self, node):
        self.alpha = 0.75
        self.width = 1.0
        self.color = "red"
        self.shape = "o"

        if NodeLabel.ROOT_CFG in node.perm.label:
            self.shape = "s"

        if node.temp is not None:
            self.color = "blue"

            if NodeLabel.ROOT_SLICE in node.temp.label:
                self.shape = "s"

            if NodeLabel.HOST_FETCH in node.temp.label:
                self.color = "yellow"

class EdgeFormat(object):
    def __init__(self, mode):
        self.alpha = 1.0
        self.width = 2.0

        if mode == 0:
            self.color = "black"
            self.style = "solid"
        elif mode == 1:
            self.color = "green"
            self.style = "dashed"
        elif mode == 2:
            self.color = "blue"
            self.style = "solid"

class Grapher(object):
    def __init__(self):
        self.graph = nx.DiGraph()

    def addNode(self, nid, val, fmt):
        self.graph.add_node(nid, val = val, fmt = fmt)

    def addEdge(self, src, dst, val, fmt):
        self.graph.add_edge(src, dst, val = val, fmt = fmt)

    def draw(self):
        # layout
        try:
            pos = nx.nx_pydot.pydot_layout(self.graph, prog = "dot")
        except Exception as ex:
            print >> sys.stderr, "Unable to use DOT layout: %s" % ex
            pos = nx.spring_layout(self.graph)

        # draw nodes
        for n in self.graph.nodes(data = True):
            fmt = n[1]["fmt"]
            nx.draw_networkx_nodes(self.graph, pos, [n[0]],
                    cmap = plt.cm.Oranges, vmin = 0.0, vmax = 1.0,
                    node_color = fmt.color,
                    node_shape = fmt.shape,
                    alpha = fmt.alpha,
                    linewidths = fmt.width)

        # draw node text
        node_labels = nx.get_node_attributes(self.graph, "val")
        nx.draw_networkx_labels(self.graph, pos, labels = node_labels)

        # draw edges
        for e in self.graph.edges(data = True):
            fmt = e[2]["fmt"]
            nx.draw_networkx_edges(self.graph, pos, [(e[0], e[1])],
                    edge_color = fmt.color,
                    style = fmt.style,
                    alpha = fmt.alpha,
                    width = fmt.width)

        # show
        plt.show()

# analysis
class Fetch(object):
    def __init__(self, rec):
        self.inst = rec["inst"]
        self.host = rec["host"]
        self.slice = rec["slice"]

class Func(object):
    def __init__(self, fid, rec):
        self.fid = fid

        # flow graph
        self.graph = FlowGraph(rec["cfg"])

        # fetches
        self.fetches = []
        for v in rec["fetch"]:
            self.fetches.append(Fetch(v))

    def draw(self, fetch):
        self.graph.restore()
        self.graph.update(fetch.slice, fetch.host)

        print >> sys.stdout, "Fetch: %s" % fetch.inst

        g = Grapher()

        for k, v in self.graph.repo.items():
            if v.temp is None:
                info = "%d" % (v.perm.info)
            else:
                info = "%d/%d" % (v.temp.info, v.perm.info)

            g.addNode(k, info, NodeFormat(v))

        for k, v in self.graph.repo.items():
            for s in v.perm.succ:
                if v.temp is not None and v.temp.hasSucc(s.nid):
                    g.addEdge(v.nid, s.nid, "", EdgeFormat(2))
                else:
                    g.addEdge(v.nid, s.nid, "", EdgeFormat(0))

        for k, v in self.graph.repo.items():
            if v.temp is None:
                continue

            for s in v.temp.succ:
                if not v.perm.hasSucc(s.nid):
                    g.addEdge(v.nid, s.nid, "", EdgeFormat(1))

        g.draw()

    def show(self):
        if len(self.fetches) == 0:
            return

        print >> sys.stdout, "Function: %s" % self.fid

        for fetch in self.fetches:
            self.draw(fetch)

class Module(object):
    def __init__(self, fn):
        if not exists(fn):
            print >> sys.stderr, "%s does not exist" % fn
            sys.exit(-1)

        self.funcs = OrderedDict()

        with open(fn, "r") as fp:
            mrec = json.load(fp)
            for k in mrec:
                self.funcs[k] = Func(k, mrec[k])

# entry point
if __name__ == "__main__":
    # init
    parser = ArgumentParser()
    parser.add_argument("files", action="append", default=[])
    parser.add_argument("-f", "--func", action="append", default=None)

    # parse
    args = parser.parse_args()

    # exec
    for fn in args.files:
        toks = splitext(fn)
        if toks[1] == ".c" and "/srcs/" in toks[0]:
            fn = toks[0].replace("/srcs/", "/syms/") + ".sym"

        mod = Module(fn)
        for fid in mod.funcs:
            if args.func is not None and fid not in args.func:
                continue

            mod.funcs[fid].show()

max_stars_repo_name: oslab-swrc/deadline

max_issues_repo_name: oslab-swrc/deadline

max_forks_repo_name: oslab-swrc/deadline

reason: import networkx


text:
# coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

"""
JahnTeller distortion analysis.
"""

import os
import warnings
from typing import Any, Dict, Optional, Tuple, Union

import numpy as np

from pymatgen.analysis.bond_valence import BVAnalyzer
from pymatgen.analysis.local_env import (
    LocalStructOrderParams,
    get_neighbors_of_site_with_index,
)
from pymatgen.core.periodic_table import Species, get_el_sp
from pymatgen.core.structure import Structure
from pymatgen.symmetry.analyzer import SpacegroupAnalyzer

MODULE_DIR = os.path.dirname(os.path.abspath(__file__))

class JahnTellerAnalyzer:
    """
    Will attempt to classify if structure *may* be Jahn-Teller active.
    Class currently uses datafile of hard-coded common Jahn-Teller
    active ions.
    If structure is annotated with magnetic moments, will estimate
    if structure may be high-spin or low-spin.
    Class aims for more false-positives than false-negatives.
    """

    def __init__(self):
        """
        Init for JahnTellerAnalyzer.
        """

        self.spin_configs = {
            "oct": {  # key is number of d electrons
                0: {"high": {"e_g": 0, "t_2g": 0}, "default": "high"},
                1: {"high": {"e_g": 0, "t_2g": 1}, "default": "high"},  # weak J-T
                2: {"high": {"e_g": 0, "t_2g": 2}, "default": "high"},  # weak
                3: {"high": {"e_g": 0, "t_2g": 3}, "default": "high"},  # no J-T
                4: {
                    "high": {"e_g": 1, "t_2g": 3},
                    "low": {"e_g": 0, "t_2g": 4},
                    "default": "high",
                },  # strong high, weak low
                5: {
                    "high": {"e_g": 2, "t_2g": 3},
                    "low": {"e_g": 0, "t_2g": 5},
                    "default": "low",
                },  # no high, weak low
                6: {
                    "high": {"e_g": 2, "t_2g": 4},
                    "low": {"e_g": 0, "t_2g": 6},
                    "default": "high",
                },  # weak high, no low
                7: {
                    "high": {"e_g": 2, "t_2g": 5},
                    "low": {"e_g": 1, "t_2g": 6},
                    "default": "low",
                },  # weak high, strong low
                8: {"high": {"e_g": 2, "t_2g": 6}, "default": "high"},  # no
                9: {"high": {"e_g": 3, "t_2g": 6}, "default": "high"},  # strong
                10: {"high": {"e_g": 4, "t_2g": 6}, "default": "high"},
            },
            "tet": {  # no low spin observed experimentally in tetrahedral, all weak J-T
                0: {"high": {"e": 0, "t_2": 0}, "default": "high"},
                1: {"high": {"e": 1, "t_2": 0}, "default": "high"},
                2: {"high": {"e": 2, "t_2": 0}, "default": "high"},
                3: {"high": {"e": 2, "t_2": 1}, "default": "high"},
                4: {"high": {"e": 2, "t_2": 2}, "default": "high"},
                5: {"high": {"e": 2, "t_2": 3}, "default": "high"},
                6: {"high": {"e": 3, "t_2": 3}, "default": "high"},
                7: {"high": {"e": 4, "t_2": 3}, "default": "high"},
                8: {"high": {"e": 4, "t_2": 4}, "default": "high"},
                9: {"high": {"e": 4, "t_2": 5}, "default": "high"},
                10: {"high": {"e": 4, "t_2": 6}, "default": "high"},
            },
        }

    def get_analysis_and_structure(
        self,
        structure: Structure,
        calculate_valences: bool = True,
        guesstimate_spin: bool = False,
        op_threshold: float = 0.1,
    ) -> Tuple[Dict, Structure]:
        """Obtain an analysis of a given structure and if it may be Jahn-Teller
        active or not. This is a heuristic, and may give false positives and
        false negatives (false positives are preferred).

        Args:
            structure: input structure
            calculate_valences: whether to attempt to calculate valences or not, structure
                should have oxidation states to perform analysis (Default value = True)
            guesstimate_spin: whether to guesstimate spin state from magnetic moments
                or not, use with caution (Default value = False)
            op_threshold: threshold for order parameter above which to consider site
                to match an octahedral or tetrahedral motif, since Jahn-Teller structures
                can often be
                quite distorted, this threshold is smaller than one might expect

        Returns:
            analysis of structure, with key 'strength' which may be 'none', 'strong',
            'weak', or 'unknown' (Default value = 0.1) and decorated structure

        """

        structure = structure.get_primitive_structure()

        if calculate_valences:
            bva = BVAnalyzer()
            structure = bva.get_oxi_state_decorated_structure(structure)

        # no point testing multiple equivalent sites, doesn't make any difference to analysis
        # but makes returned
        symmetrized_structure = SpacegroupAnalyzer(structure).get_symmetrized_structure()

        # to detect structural motifs of a given site
        op = LocalStructOrderParams(["oct", "tet"])

        # dict of site index to the Jahn-Teller analysis of that site
        jt_sites = []
        non_jt_sites = []

        for indices in symmetrized_structure.equivalent_indices:

            idx = indices[0]
            site = symmetrized_structure[idx]

            # only interested in sites with oxidation states
            if isinstance(site.specie, Species) and site.specie.element.is_transition_metal:

                # get motif around site
                order_params = op.get_order_parameters(symmetrized_structure, idx)

                if order_params[0] > order_params[1] and order_params[0] > op_threshold:
                    motif = "oct"
                    motif_order_parameter = order_params[0]
                elif order_params[1] > op_threshold:
                    motif = "tet"
                    motif_order_parameter = order_params[1]
                else:
                    motif = "unknown"
                    motif_order_parameter = None

                if motif in ["oct", "tet"]:

                    # guess spin of metal ion
                    if guesstimate_spin and "magmom" in site.properties:

                        # estimate if high spin or low spin
                        magmom = site.properties["magmom"]
                        spin_state = self._estimate_spin_state(site.specie, motif, magmom)
                    else:
                        spin_state = "unknown"

                    magnitude = self.get_magnitude_of_effect_from_species(site.specie, spin_state, motif)

                    if magnitude != "none":

                        ligands = get_neighbors_of_site_with_index(structure, idx, approach="min_dist", delta=0.15)
                        ligand_bond_lengths = [ligand.distance(structure[idx]) for ligand in ligands]
                        ligands_species = list({str(ligand.specie) for ligand in ligands})
                        ligand_bond_length_spread = max(ligand_bond_lengths) - min(ligand_bond_lengths)

                        def trim(f):
                            """
                            Avoid storing to unreasonable precision, hurts readability.
                            """
                            return float("{:.4f}".format(f))

                        # to be Jahn-Teller active, all ligands have to be the same
                        if len(ligands_species) == 1:
                            jt_sites.append(
                                {
                                    "strength": magnitude,
                                    "motif": motif,
                                    "motif_order_parameter": trim(motif_order_parameter),
                                    "spin_state": spin_state,
                                    "species": str(site.specie),
                                    "ligand": ligands_species[0],
                                    "ligand_bond_lengths": [trim(length) for length in ligand_bond_lengths],
                                    "ligand_bond_length_spread": trim(ligand_bond_length_spread),
                                    "site_indices": indices,
                                }
                            )

                    # store reasons for not being J-T active
                    else:
                        non_jt_sites.append(
                            {
                                "site_indices": indices,
                                "strength": "none",
                                "reason": "Not Jahn-Teller active for this " "electronic configuration.",
                            }
                        )
                else:
                    non_jt_sites.append(
                        {
                            "site_indices": indices,
                            "strength": "none",
                            "reason": "motif is {}".format(motif),
                        }
                    )

        # perform aggregation of all sites
        if jt_sites:
            analysis = {"active": True}  # type: Dict[str, Any]
            # if any site could exhibit 'strong' Jahn-Teller effect
            # then mark whole structure as strong
            strong_magnitudes = [site["strength"] == "strong" for site in jt_sites]
            if any(strong_magnitudes):
                analysis["strength"] = "strong"
            else:
                analysis["strength"] = "weak"
            analysis["sites"] = jt_sites
            return analysis, structure
        return {"active": False, "sites": non_jt_sites}, structure

    def get_analysis(
        self,
        structure: Structure,
        calculate_valences: bool = True,
        guesstimate_spin: bool = False,
        op_threshold: float = 0.1,
    ) -> Dict:
        """
        Convenience method, uses get_analysis_and_structure method.

        Obtain an analysis of a given structure and if it may be Jahn-Teller
        active or not. This is a heuristic, and may give false positives and
        false negatives (false positives are preferred).

        Args:
            structure: input structure
            calculate_valences: whether to attempt to calculate valences or not, structure
                should have oxidation states to perform analysis (Default value = True)
            guesstimate_spin: whether to guesstimate spin state from magnetic moments
                or not, use with caution (Default value = False)
            op_threshold: threshold for order parameter above which to consider site
                to match an octahedral or tetrahedral motif, since Jahn-Teller structures
                can often be
                quite distorted, this threshold is smaller than one might expect

        Returns:
            analysis of structure, with key 'strength' which may be 'none', 'strong',
            'weak', or 'unknown' (Default value = 0.1)
        """
        return self.get_analysis_and_structure(
            structure,
            calculate_valences=calculate_valences,
            guesstimate_spin=guesstimate_spin,
            op_threshold=op_threshold,
        )[0]

    def is_jahn_teller_active(
        self,
        structure: Structure,
        calculate_valences: bool = True,
        guesstimate_spin: bool = False,
        op_threshold: float = 0.1,
    ) -> bool:
        """
        Convenience method, uses get_analysis_and_structure method.
        Check if a given structure and if it may be Jahn-Teller
        active or not. This is a heuristic, and may give false positives and
        false negatives (false positives are preferred).

        Args:
            structure: input structure
            calculate_valences: whether to attempt to calculate valences or not, structure
                should have oxidation states to perform analysis (Default value = True)
            guesstimate_spin: whether to guesstimate spin state from magnetic moments
                or not, use with caution (Default value = False)
            op_threshold: threshold for order parameter above which to consider site
                to match an octahedral or tetrahedral motif, since Jahn-Teller structures
                can often be
                quite distorted, this threshold is smaller than one might expect

        Returns:
            boolean, True if might be Jahn-Teller active, False if not
        """

        active = False

        try:
            analysis = self.get_analysis(
                structure,
                calculate_valences=calculate_valences,
                guesstimate_spin=guesstimate_spin,
                op_threshold=op_threshold,
            )
            active = analysis["active"]
        except Exception as e:
            warnings.warn("Error analyzing {}: {}".format(structure.composition.reduced_formula, e))

        return active

    def tag_structure(
        self,
        structure: Structure,
        calculate_valences: bool = True,
        guesstimate_spin: bool = False,
        op_threshold: float = 0.1,
    ) -> Structure:
        """
        Convenience method, uses get_analysis_and_structure method.
        Add a "possible_jt_active" site property on Structure.

        Args:
            structure: input structure
            calculate_valences: whether to attempt to calculate valences or not, structure
                should have oxidation states to perform analysis (Default value = True)
            guesstimate_spin: whether to guesstimate spin state from magnetic moments
                or not, use with caution (Default value = False)
            op_threshold: threshold for order parameter above which to consider site
                to match an octahedral or tetrahedral motif, since Jahn-Teller structures
                can often be
                quite distorted, this threshold is smaller than one might expect

        Returns:
            Decorated Structure, will be in primitive setting.

        """
        try:
            analysis, structure = self.get_analysis_and_structure(
                structure,
                calculate_valences=calculate_valences,
                guesstimate_spin=guesstimate_spin,
                op_threshold=op_threshold,
            )
            jt_sites = [False] * len(structure)
            if analysis["active"]:
                for site in analysis["sites"]:
                    for index in site["site_indices"]:
                        jt_sites[index] = True
                        structure.add_site_property("possible_jt_active", jt_sites)
            return structure
        except Exception as e:
            warnings.warn("Error analyzing {}: {}".format(structure.composition.reduced_formula, e))
            return structure

    @staticmethod
    def _get_number_of_d_electrons(species: Species) -> float:
        """
        Get number of d electrons of a species.

        Args:
          species: Species object

        Returns: Number of d electrons.
        """

        # TODO: replace with more generic Hund's rule algorithm?

        # taken from get_crystal_field_spin
        elec = species.full_electronic_structure
        if len(elec) < 4 or elec[-1][1] != "s" or elec[-2][1] != "d":
            raise AttributeError("Invalid element {} for crystal field calculation.".format(species.symbol))
        nelectrons = int(elec[-1][2] + elec[-2][2] - species.oxi_state)
        if nelectrons < 0 or nelectrons > 10:
            raise AttributeError("Invalid oxidation state {} for element {}".format(species.oxi_state, species.symbol))

        return nelectrons

    def get_magnitude_of_effect_from_species(self, species: Union[str, Species], spin_state: str, motif: str) -> str:
        """
        Get magnitude of Jahn-Teller effect from provided species, spin state and motif.

        Args:
          species: e.g. Fe2+
          spin_state: "high" or "low"
          motif: "oct" or "tet"

        Returns: "none", "weak" or "strong

        """

        magnitude = "none"

        sp = get_el_sp(species)

        # has to be Species; we need to know the oxidation state
        if isinstance(sp, Species) and sp.element.is_transition_metal:

            d_electrons = self._get_number_of_d_electrons(sp)

            if motif in self.spin_configs:
                if spin_state not in self.spin_configs[motif][d_electrons]:
                    spin_state = self.spin_configs[motif][d_electrons]["default"]
                spin_config = self.spin_configs[motif][d_electrons][spin_state]
                magnitude = JahnTellerAnalyzer.get_magnitude_of_effect_from_spin_config(motif, spin_config)
        else:
            warnings.warn("No data for this species.")

        return magnitude

    @staticmethod
    def get_magnitude_of_effect_from_spin_config(motif: str, spin_config: Dict[str, float]) -> str:
        """
        Roughly, the magnitude of Jahn-Teller distortion will be:
        * in octahedral environments, strong if e_g orbitals
        unevenly occupied but weak if t_2g orbitals unevenly
        occupied
        * in tetrahedral environments always weaker

        Args:
          motif: "oct" or "tet"
          spin_config: dict of 'e' (e_g) and 't' (t2_g)
            with number of electrons in each state

        Returns:  "none", "weak" or "strong"
        """
        magnitude = "none"
        if motif == "oct":
            e_g = spin_config["e_g"]
            t_2g = spin_config["t_2g"]
            if (e_g % 2 != 0) or (t_2g % 3 != 0):
                magnitude = "weak"
                if e_g % 2 == 1:
                    magnitude = "strong"
        elif motif == "tet":
            e = spin_config["e"]
            t_2 = spin_config["t_2"]
            if (e % 3 != 0) or (t_2 % 2 != 0):
                magnitude = "weak"
        return magnitude

    @staticmethod
    def _estimate_spin_state(species: Union[str, Species], motif: str, known_magmom: float) -> str:
        """Simple heuristic to estimate spin state. If magnetic moment
        is sufficiently close to that predicted for a given spin state,
        we assign it that state. If we only have data for one spin
        state then that's the one we use (e.g. we assume all tetrahedral
        complexes are high-spin, since this is typically the case).

        Args:
          species: str or Species
          motif: "oct" or "tet"
          known_magmom: magnetic moment in Bohr magnetons

        Returns: "undefined" (if only one spin state possible), "low",
        "high" or "unknown"
        """
        mu_so_high = JahnTellerAnalyzer.mu_so(species, motif=motif, spin_state="high")
        mu_so_low = JahnTellerAnalyzer.mu_so(species, motif=motif, spin_state="low")
        if mu_so_high == mu_so_low:
            return "undefined"  # undefined or only one spin state possible
        if mu_so_high is None:
            return "low"
        if mu_so_low is None:
            return "high"
        diff = mu_so_high - mu_so_low
        # WARNING! this heuristic has not been robustly tested or benchmarked
        # using 'diff*0.25' as arbitrary measure, if known magmom is
        # too far away from expected value, we don't try to classify it
        if known_magmom > mu_so_high or abs(mu_so_high - known_magmom) < diff * 0.25:
            return "high"
        if known_magmom < mu_so_low or abs(mu_so_low - known_magmom) < diff * 0.25:
            return "low"
        return "unknown"

    @staticmethod
    def mu_so(species: Union[str, Species], motif: str, spin_state: str) -> Optional[float]:
        """Calculates the spin-only magnetic moment for a
        given species. Only supports transition metals.

        Args:
          species: Species
          motif: "oct" or "tet"
          spin_state: "high" or "low"

        Returns:
          Spin-only magnetic moment in Bohr magnetons or None if
          species crystal field not defined
        """
        try:
            sp = get_el_sp(species)
            n = sp.get_crystal_field_spin(coordination=motif, spin_config=spin_state)
            # calculation spin-only magnetic moment for this number of unpaired spins
            return np.sqrt(n * (n + 2))
        except AttributeError:
            return None

max_stars_repo_name: wangyusu/pymatgen

max_issues_repo_name: wangyusu/pymatgen

max_forks_repo_name: wangyusu/pymatgen


text:
# -*- coding: utf-8 -*-
# vim: set fileencoding=utf-8 :
# vim: set foldmethod=marker commentstring=\ \ #\ %s :
#
# Author:    Taishi Matsumura
# Created:   2019-12-02
#
# Copyright (C) 2019 Taishi Matsumura
#
import os
import argparse

parser = argparse.ArgumentParser(description='Inference.')

parser.add_argument('inference_dataset_path',
        type=str, help='Put a path to a target dataset.')

parser.add_argument('trained_network_path',
        type=str, help='Put a path to a trained network.')

parser.add_argument('-g', '--gpu',
        type=int, help='Put an ID of GPU.')

args = parser.parse_args()

assert os.path.exists(args.inference_dataset_path),  \
        'The given path does not exists.'

assert os.path.exists(args.trained_network_path),  \
        'The given path does not exists.'

inference_dataset_path = args.inference_dataset_path
trained_network_path = args.trained_network_path
gpu_id = args.gpu

# 訓練済みネットワークがあるディレクトリの名前（target_dir）
target_dir = os.path.basename(os.path.dirname(trained_network_path))

# target_dir があるディレクトリの名前（morpho）
morpho = os.path.basename(os.path.dirname(os.path.dirname(trained_network_path)))

# 利用する GPU の ID を設定する
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)

import PIL
import json
import glob
import numpy as np
from PIL import Image
from tqdm import tqdm

import keras
from keras import backend as K
from keras.utils import to_categorical

def TP(y_true, y_pred):
    y_true = K.argmax(y_true, axis=-1)
    y_pred = K.argmax(y_pred, axis=-1)
    return K.sum(K.cast(K.equal(y_true * y_pred, 1), K.floatx()))

def TN(y_true, y_pred):
    y_true = K.argmax(y_true, axis=-1)
    y_pred = K.argmax(y_pred, axis=-1)
    return K.sum(K.cast(K.equal(y_true + y_pred, 0), K.floatx()))

def FP(y_true, y_pred):
    y_true = K.argmax(y_true, axis=-1)
    y_pred = K.argmax(y_pred, axis=-1)
    return K.sum(K.cast(K.less(y_true, y_pred), K.floatx()))

def FN(y_true, y_pred):
    y_true = K.argmax(y_true, axis=-1)
    y_pred = K.argmax(y_pred, axis=-1)
    return K.sum(K.cast(K.greater(y_true, y_pred), K.floatx()))

def IoU(y_true, y_pred):
    return TP(y_true, y_pred)  \
            / (FN(y_true, y_pred) + TP(y_true, y_pred) + FP(y_true, y_pred))

def tf_normalize(images):
    images = images / 127.5
    return images - 1.0

def zeropadding(img, shape=(100, 100)):
    '''
    Input
    -----
    img : ndarray (height, width, n_channels)

    Output
    ------
    result : ndarray (height, width, n_channels)
    '''
    assert len(img.shape) == 2 or len(img.shape) == 3, 'Image must be 2D or 3D.'
    #assert img.shape[0] < shape[0], 'Image is too small.'
    #assert img.shape[1] < shape[1], 'Image is too small.'

    r0, c0 = int((shape[0] - img.shape[0]) / 2), int((shape[1] - img.shape[1]) / 2)
    if len(img.shape) == 3:
        result = np.zeros((shape[0], shape[1], img.shape[2]))
        result[r0:img.shape[0]+r0, c0:img.shape[1]+c0, :] = img
    elif len(img.shape) == 2:
        result = np.zeros((shape[0], shape[1]))
        result[r0:img.shape[0]+r0, c0:img.shape[1]+c0] = img
    return result

def split(image_path, n_wells, mask):
    org_image = np.array(PIL.Image.open(image_path).convert('L'), dtype=np.uint8)

    well_images = []
    for well_idx in range(n_wells):
        # Cut out a well image from the original image
        r, c = np.where(mask == well_idx)
        well_image = org_image[r.min():r.max(), c.min():c.max()]
		# Up sampling
        img = Image.fromarray(np.uint8(well_image))
        well_image = np.asarray(img.resize((56,56)))
        height, width = well_image.shape
        well_images.append(well_image)

    return np.array(well_images)

def get_well_imgs(img, mask, n_wells):
    well_imgs = []
    for i in range(n_wells):
        rows, clms = np.where(mask == i)
        well_imgs.append(img[min(rows):max(rows), min(clms):max(clms)])
    well_imgs = np.array(well_imgs)

    return well_imgs.reshape(
            well_imgs.shape[0], well_imgs.shape[1], well_imgs.shape[2], 1)

def inference(orgimg_path):
    # ウェル画像の切り出し
    well_images = split(orgimg_path, n_wells, mask)

    # (n_wells, height, width) -> (n_wells, height, width, 1)
    well_images = np.expand_dims(well_images, axis=-1)

    # Zeropadding
    well_images = np.array(
            [zeropadding(well_image, (56, 56)) for well_image in well_images])

    # データの正規化
    well_images = tf_normalize(well_images)

    # Inference
    return (100 * model.predict(well_images)).astype(np.uint8)

# ============
#  推論
# ============
# マスクのロード
mask = np.load(os.path.join(inference_dataset_path, 'mask.npy'))

# ウェル数のロード
with open(os.path.join(inference_dataset_path, 'mask_params.json')) as f:
    params = json.load(f)
n_wells = params['n-rows'] * params['n-clms'] * params['n-plates']

# 訓練済みネットワークのロード
model = keras.models.load_model(trained_network_path,
        custom_objects={'IoU': IoU, 'TP': TP, 'FP': FP, 'FN': FN, 'TN': TN})

# オリジナル画像のファイルパス
orgimg_paths = sorted(glob.glob(os.path.join(
        glob.escape(inference_dataset_path), 'original', '*.jpg')))

probs = []
# for each frame
for orgimg_path in tqdm(orgimg_paths):
    probs.append(inference(orgimg_path))

# ==============
#  データの作成
# ==============
# Tranform float probs into percentage
probs = np.array(probs).reshape(len(orgimg_paths), n_wells, 56, 56, 2)

# only probs which the animal is in the pixel
probs = probs[:, :, :, :, 1]

# Wells first (wells, org_imgs, height, width)
probs = probs.transpose(1, 0, 2, 3)

# シグナルの作成
signals = (np.diff((probs >= 50).astype(np.int8), axis=1)**2).sum(-1).sum(-1).T

# ==============
#  保存
# ==============
out_dir = os.path.join(inference_dataset_path, 'inference', morpho, target_dir)
os.makedirs(os.path.join(out_dir, 'probs'), exist_ok=True)

# probs の保存
for well_idx in range(n_wells):
    np.savez_compressed(os.path.join(
        out_dir, 'probs', '{:03d}.npz'.format(well_idx)), probs[well_idx])

np.savez_compressed(os.path.join(out_dir, 'probs.npz'), probs)

# シグナルの保存
np.save(os.path.join(out_dir, 'signals.npy'), signals)

max_stars_repo_name: kanglab/Sapphire

max_issues_repo_name: kanglab/Sapphire

max_forks_repo_name: kanglab/Sapphire


text:
#
# clouds.py
#
# Author(s):
# Matteo Spallanzani <spmatteo@iis.ee.ethz.ch>
#
# Copyright (c) 2018-2021 Matteo Spallanzani
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import numpy as np
from scipy.stats import ortho_group

def get_maxVs(nG, nC, conditioning, maxV, epsV):

    assert 0.0 < epsV <= 1.0  # so that the actual `maxV` for each Gaussian will be in the range `(0, 2 * maxV]`

    if conditioning == 'F':  # each Gaussian is fully dependent on the group/class variable combination; i.e., it will likely have a different volume from other Gaussians
        maxVs = np.array([maxV] * (nG * nC))
        if epsV != 0.0:
            maxVs *= epsV * (2. * np.random.rand(len(maxVs)) - 1.) + 1.
        maxVs = np.reshape(maxVs, (nG, nC))

    elif conditioning == 'G':  # the Gaussians in each cluster of corresponding homogeneous groups will have the same volume
        maxVs= np.array([maxV] * nG)
        if epsV != 0.0:
            maxVs *= epsV * (2. * np.random.rand(nG) - 1.) + 1.
        maxVs = np.tile(maxVs, (nC, 1))
        maxVs = np.transpose(maxVs, (1, 0))

    elif conditioning == 'C':  # the Gaussians composing a specific segment mixture will have the same volume
        maxVs = np.array([maxV] * nC)
        if epsV != 0.0:
            maxVs *= epsV * (2. * np.random.rand(nC) - 1.) + 1.
        maxVs = np.tile(maxVs, (nG, 1))

    elif conditioning == 'I':  # all Gaussians will have the same volume, independently of homogeneous group and segment
        if epsV != 0.0:
            maxV *= epsV * (2. * np.random.rand() - 1.) + 1.
        maxVs = np.array([maxV] * (nG * nC))
        maxVs = np.reshape(maxVs, (nG, nC))

    return maxVs

def get_spectra(nD, nG, nC, conditioning, spectrumK, epsK, spherical=False):

    assert spectrumK > 0
    assert 0.0 < epsK <= 1.0  # so that the actual spectra will be in the range `(e^{-2 * k}, e^{0} = 1]`

    if spherical:  # note that either all te Gaussians are forced to be spherical (even if they belong to different conditioning groups), ore none is forced to
        spectrumK = 0

    _base_eig_exps = - np.linspace(0.0, 1.0, nD)

    if conditioning == 'F':
        spectraK = np.array([spectrumK] * (nG * nC))
        if epsK != 0.0:
            spectraK *= epsK * (2. * np.random.rand(len(spectraK)) - 1.) + 1.
        spectraK = np.reshape(spectraK, (nG, nC))

    elif conditioning == 'G':
        spectraK = np.array([spectrumK] * nG)
        if epsK != 0.0:
            spectraK *= epsK * (2. * np.random.rand(nG) - 1.) + 1.
        spectraK = np.tile(spectraK, (nC, 1))
        spectraK = np.transpose(spectraK, (1, 0))

    elif conditioning == 'C':
        spectraK = np.array([spectrumK] * nC)
        if epsK != 0.0:
            spectraK *= epsK * (2. * np.random.rand(nC) - 1.) + 1.
        spectraK = np.tile(spectraK, (nG, 1))

    elif conditioning == 'I':
        if epsK != 0.0:
            spectrumK *= epsK * (2. * np.random.rand() - 1.) + 1.
        spectraK = np.array([spectrumK] * (nG * nC))
        spectraK = np.reshape(spectraK, (nG, nC))

    spectra = np.exp(spectraK[:, :, None] * _base_eig_exps[None, None, :])

    return spectra

def get_orientations(nD, nG, nC, conditioning, axis_aligned=False):

    def get_rotation_matrix(nD, axis_aligned=False):
        if axis_aligned:
            rot_matrix = np.eye(nD)
        else:
            rot_matrix = ortho_group.rvs(nD)
        return rot_matrix

    rot_matrices = np.zeros((nG, nC, nD, nD))

    if conditioning == 'F':
        for ig in range(0, nG):
            for ic in range(0, nC):
                rot_matrix = get_rotation_matrix(nD, axis_aligned=axis_aligned)
                np.random.shuffle(rot_matrix)  # shuffle rows
                rot_matrices[ig, ic] = rot_matrix

    elif conditioning == 'G':
        for ig in range(0, nG):
            rot_matrix = get_rotation_matrix(nD, axis_aligned=axis_aligned)
            np.random.shuffle(rot_matrix)
            rot_matrices[ig, :, :, :] = rot_matrix

    elif conditioning == 'C':
        for ic in range(0, nC):
            rot_matrix = get_rotation_matrix(nD, axis_aligned=axis_aligned)
            np.random.shuffle(rot_matrix)
            rot_matrices[..., ic, :, :] = rot_matrix

    elif conditioning == 'I':
        rot_matrix = get_rotation_matrix(nD, axis_aligned=axis_aligned)
        np.random.shuffle(rot_matrix)
        rot_matrices[..., :, :] = rot_matrix

    return rot_matrices

def create_covariance_matrices(nD, nG, nC,
                            maxV_cond, maxV, epsV,
                            spectrumK_cond, spectrumK, epsK, spherical,
                            orientation_cond, ax_align):

    cov_matrices = np.zeros((nG, nC, nD, nD))

    maxVs = get_maxVs(nG, nC, maxV_cond, maxV, epsV)
    spectra = get_spectra(nD, nG, nC, spectrumK_cond, spectrumK, epsK, spherical=spherical)
    rot_matrices = get_orientations(nD, nG, nC, orientation_cond, axis_aligned=ax_align)

    for ig in range(0, nG):
        for ic in range(0, nC):
            cov_matrix = np.dot(np.dot(rot_matrices[ig, ic], np.diag(maxVs[ig, ic] * spectra[ig, ic])), rot_matrices[ig, ic].T)
            cov_matrices[ig, ic] = cov_matrix

    return cov_matrices

max_stars_repo_name: spallanzanimatteo/Fingerprint

max_issues_repo_name: spallanzanimatteo/Fingerprint

max_forks_repo_name: spallanzanimatteo/Fingerprint

reason: import numpy,from scipy


text:
from __future__ import print_function

import os
import sys
import numpy as np
import torch
import networkx as nx
import random
from torch.autograd import Variable
from torch.nn.parameter import Parameter
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from tqdm import tqdm
from copy import deepcopy

from q_net import NStepQNet, QNet, greedy_actions
sys.path.append('%s/../common' % os.path.dirname(os.path.realpath(__file__)))
from cmd_args import cmd_args
from graph_embedding import S2VGraph

from rl_common import GraphEdgeEnv, load_graphs, test_graphs, attackable, load_base_model

sys.path.append('%s/../graph_classification' % os.path.dirname(os.path.realpath(__file__)))
from graph_common import loop_dataset, load_er_data

def propose_attack(model, s2v_g, num_added=1):
    g = s2v_g.to_networkx()
    comps = [c for c in nx.connected_component_subgraphs(g)]
    set_id = {}

    for i in range(len(comps)):
        for j in comps[i].nodes():
            set_id[j] = i

    cand = []
    for i in range(len(g) - 1):
        for j in range(i + 1, len(g)):
            if set_id[i] != set_id[j] or i == j:
                continue
            cand.append('%d %d' % (i, j))

    if cmd_args.rand_att_type == 'random':
        added = np.random.choice(cand, num_added)
        added = [(int(w.split()[0]), int(w.split()[1])) for w in added]
        g.add_edges_from(added)
        return S2VGraph(g, s2v_g.label)
    elif cmd_args.rand_att_type == 'exhaust':
        g_list = []
        for c in cand:
            x, y = [int(w) for w in c.split()]
            g2 = g.copy()
            g2.add_edge(x, y)
            g_list.append(S2VGraph(g2, s2v_g.label))
        _, _, acc = model(g_list)
        ans = g_list[0]
        for i in range(len(g_list)):
            if acc.numpy()[i] < 1:
                ans = g_list[i]
                break
        return ans
    else:
        raise NotImplementedError

if __name__ == '__main__':
    random.seed(cmd_args.seed)
    np.random.seed(cmd_args.seed)
    torch.manual_seed(cmd_args.seed)

    label_map, train_glist, test_glist = load_er_data()

    base_classifier = load_base_model(label_map, test_glist)

    new_test_list = []
    for g in tqdm(test_glist):
        new_test_list.append(propose_attack(base_classifier, g))

    test_graphs(base_classifier, new_test_list)

reason: import numpy,import networkx


text:
import numpy as np
import time

import torch

from cluster.selfrepresentation import ElasticNetSubspaceClustering, SparseSubspaceClusteringOMP
from decomposition.dim_reduction import dim_reduction
from kymatio import Scattering2D
from metrics.cluster.accuracy import clustering_accuracy
from sklearn import cluster
from sklearn.metrics import normalized_mutual_info_score, adjusted_rand_score
from sklearn.preprocessing import normalize
from torchvision import datasets, transforms

# =================================================
# Prepare MNIST dataset
# =================================================

print('Prepare scattering...')
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")

scattering = Scattering2D(J=3, shape=(32, 32))
if use_cuda:
    scattering = scattering.cuda()

print('Prepare MNIST...')
transforms_MNIST = transforms.Compose([transforms.Resize((32, 32)),
                                       transforms.ToTensor()])
MNIST_train = datasets.MNIST('./data', train=True, download=True, transform=transforms_MNIST)
MNIST_test = datasets.MNIST('./data', train=False, download=True, transform=transforms_MNIST)
MNIST_train_loader = torch.utils.data.DataLoader(MNIST_train, batch_size=60000, shuffle=False)
MNIST_test_loader = torch.utils.data.DataLoader(MNIST_test, batch_size=10000, shuffle=False)

raw_train_data, label_train = next(iter(MNIST_train_loader))  # data shape: torch.Size([60000, 1, 28, 28])
raw_test_data, label_test = next(iter(MNIST_test_loader))  # data shape: torch.Size([10000, 1, 28, 28])
label = torch.cat((label_train, label_test), 0)

print('Computing scattering on MNIST...')
if use_cuda:
    raw_train_data = raw_train_data.cuda()
    raw_test_data = raw_test_data.cuda()

train_data = scattering(raw_train_data)
test_data = scattering(raw_test_data)
data = torch.cat((train_data, test_data), 0)

print('Data preprocessing....')
n_sample = data.shape[0]

# scattering transform normalization
data = data.cpu().numpy().reshape(n_sample, data.shape[2], -1)
image_norm = np.linalg.norm(data, ord=np.inf, axis=2, keepdims=True)  # infinity norm of each transform
data = data / image_norm  # normalize each scattering transform to the range [-1, 1]
data = data.reshape(n_sample, -1)  # fatten and concatenate all transforms

# dimension reduction
data = dim_reduction(data, 500)  # dimension reduction by PCA

label = label.numpy()

# =================================================
# Create cluster objects
# =================================================
print('Begin clustering...')

# Baseline: non-subspace clustering methods
model_kmeans = cluster.KMeans(n_clusters=10)  # k-means as baseline

model_spectral = cluster.SpectralClustering(n_clusters=10,affinity='nearest_neighbors',n_neighbors=5)  # spectral clustering as baseline

# Our work: elastic net subspace clustering (EnSC)
# You et al., Oracle Based Active Set Algorithm for Scalable Elastic Net Subspace Clustering, CVPR 2016
model_ensc = ElasticNetSubspaceClustering(n_clusters=10,affinity='nearest_neighbors',algorithm='spams',active_support=True,gamma=200,tau=0.9)

# Our work: sparse subspace clusterign by orthogonal matching pursuit (SSC-OMP)
# You et al., Scalable Sparse Subspace Clustering by Orthogonal Matching Pursuit, CVPR 2016
model_ssc_omp = SparseSubspaceClusteringOMP(n_clusters=10,affinity='symmetrize',n_nonzero=5,thr=1.0e-5)

clustering_algorithms = (
    ('KMeans', model_kmeans),
    ('Spectral Clustering', model_spectral),
    ('EnSC via active support solver', model_ensc),
    ('SSC-OMP', model_ssc_omp),
)

for name, algorithm in clustering_algorithms:
    t_begin = time.time()
    algorithm.fit(data)
    t_end = time.time()
    acc = clustering_accuracy(label, algorithm.labels_)
    nmi = normalized_mutual_info_score(label, algorithm.labels_, average_method='geometric')
    ari = adjusted_rand_score(label, algorithm.labels_)

    print('Algorithm: {}. acc: {}, nmi: {}, ari: {}, Running time: {}'.format(name, acc, nmi, ari, t_end - t_begin))

max_stars_repo_name: wolhandlerdeb/clustering

max_issues_repo_name: wolhandlerdeb/clustering

max_forks_repo_name: wolhandlerdeb/clustering


text:
import numpy as np
import dynet as dy
from baseline.utils import Offsets, topk
from baseline.dy.dynety import DynetModel, Linear, WeightShareLinear, squeeze_and_transpose, attention
from baseline.dy.transformer import subsequent_mask, TransformerDecoderStack
from baseline.model import (
    register_decoder,
    register_arc_policy,
    create_seq2seq_arc_policy
)

class ArcPolicy(object):
    def __init__(self):
        super(ArcPolicy, self).__init__()

    def get_state(self, encoder_outputs):
        pass

    def __call__(self, encoder_output, hsz, beam_width=1):
        h_i = self.get_state(encoder_output)
        context = encoder_output.output
        if beam_width > 1:
            # To vectorize, we need to expand along the batch dimension, K times
            context = [dy.concatenate_to_batch([c] * beam_width) for c in context]
            h_i = [dy.concatenate_to_batch([h] * beam_width) for h in h_i]
        _, batchsz = context[0].dim()
        init_zeros = dy.zeros((hsz,), batch_size=batchsz)
        return h_i, init_zeros, context

@register_arc_policy(name='default')
class TransferLastHiddenPolicy(ArcPolicy):
    def __init__(self):
        super(TransferLastHiddenPolicy, self).__init__()

    def get_state(self, encoder_outputs):
        return encoder_outputs.hidden

@register_arc_policy(name='no_arc')
class NoArcPolicy(ArcPolicy):
    def __init__(self):
        super(NoArcPolicy, self).__init__()

    def get_state(self, encoder_outputs):
        final_state = encoder_outputs.hidden
        shape, batchsz = final_state[0].dim()
        return [dy.zeros(shape, batch_size=batchsz) for _ in len(final_state)]

class DecoderBase(DynetModel):
    def __init__(self, pc):
        super(DecoderBase, self).__init__(pc)

    def __call__(self, encoder_output, dst, train):
        pass

    def predict_one(self, src, encoder_outputs, **kwargs):
        pass

@register_decoder(name='vanilla')
class RNNDecoder(DecoderBase):
    def __init__(self, tgt_embeddings, **kwargs):
        pc = kwargs.pop('pc').add_subcollection(name=kwargs.get('name', 'rnn-decoder'))
        super(RNNDecoder, self).__init__(pc)
        self.hsz = kwargs['hsz']
        self.arc_policy = create_seq2seq_arc_policy(**kwargs)
        self.tgt_embeddings = tgt_embeddings
        rnntype = kwargs.get('rnntype', 'lstm')
        layers = kwargs['layers']
        feed_input = kwargs.get('feed_input', True)
        dsz = tgt_embeddings.get_dsz()
        if feed_input:
            self.input_i = self._feed_input
            dsz += self.hsz
        else:
            self.input_i = self._basic_input
        self.pdrop = kwargs.get('dropout', 0.5)
        self.decoder_rnn = dy.VanillaLSTMBuilder(layers, dsz, self.hsz, self.pc)
        self.init_attn(**kwargs)

        do_weight_tying = bool(kwargs.get('tie_weights', False))

        if do_weight_tying:
            if self.hsz == tgt_embeddings.get_dsz():
                self.preds = WeightShareLinear(tgt_embeddings.get_vsz(), tgt_embeddings.embeddings, self.pc, transform=squeeze_and_transpose, name=tgt_embeddings.pc.name())
            else:
                raise ValueError("weight tying only valid when prediction projection \
layer's hidden size == embedding weight dimensions")
        else:
            self.preds = Linear(self.tgt_embeddings.get_vsz(), self.hsz, self.pc)

    @staticmethod
    def _basic_input(dst_embed_i, _):
        return dst_embed_i

    @staticmethod
    def _feed_input(dst_embed_i, attn_output_i):
        return dy.concatenate([dst_embed_i, attn_output_i])

    def init_attn(self, **kwargs):
        pass

    def attn_cache(self, context):
        pass

    def attn(self, query, mask):
        return query

    def __call__(self, encoder_outputs, dst, train=False):
        src_mask = encoder_outputs.src_mask
        h_i, output_i, context = self.arc_policy(encoder_outputs, self.hsz)
        output = self.decode_rnn(context, h_i, output_i, dst, src_mask, train)
        return self.output(output)

    def decode_rnn(self, context, h_i, output_i, dst, src_mask, train):
        if train:
            self.decoder_rnn.set_dropout(self.pdrop)
        else:
            self.decoder_rnn.disable_dropout()
        embed_out = self.tgt_embeddings.encode(dst, train)
        outputs = []
        self.attn_cache(context)
        rnn_state = self.decoder_rnn.initial_state(h_i)
        for embed_i in embed_out:
            embed_i = self.input_i(embed_i, output_i)
            rnn_state = rnn_state.add_input(embed_i)
            rnn_output_i = rnn_state.output()
            output_i = self.attn(rnn_output_i, src_mask)
            outputs.append(output_i)
        return outputs

    def output(self, x):
        return [self.preds(y) for y in x]

    def prediction(self, x):
        return [dy.log_softmax(y) for y in self.output(x)]

    def predict_one(self, src, encoder_outputs, **kwargs):
        K = int(kwargs.get('beam', 5))
        mxlen = int(kwargs.get('mxlen', 100))

        paths = [[Offsets.GO] for _ in range(K)]
        # Which beams are done?
        done = np.array([False] * K)
        scores = np.array([0.0]*K)
        hidden, output_i, context = self.arc_policy(encoder_outputs, self.hsz, beam_width=K)
        num_states = len(hidden)
        rnn_state = self.decoder_rnn.initial_state(hidden)
        self.attn_cache(context)
        src_mask = encoder_outputs.src_mask

        for i in range(mxlen):
            dst_last = np.array([path[-1] for path in paths]).reshape(1, K)
            embed_i = self.tgt_embeddings.encode(dst_last)[-1]
            embed_i = self.input_i(embed_i, output_i)
            rnn_state = rnn_state.add_input(embed_i)
            rnn_output_i = rnn_state.output()
            output_i = self.attn(rnn_output_i, src_mask)
            wll = self.prediction([output_i])[-1].npvalue()  # (V,) K
            V = wll.shape[0]
            if i > 0:
                # expanded_history = np.expand_dims(scores, -1)
                # done_mask = np.expand_dims((done == False).astype(np.uint8), -1)
                # sll = np.multiply(wll.T, done_mask) + expanded_history

                wll = wll.T
                expanded_history = np.expand_dims(scores, -1)
                done_mask = np.expand_dims((done == False).astype(np.uint8), -1)
                done_mask_inv = (done_mask != 1).astype(np.uint8)
                eos_mask = np.zeros((1, V)).astype(np.uint8)
                mask = ((done_mask & eos_mask) != 1).astype(np.uint8)
                masked_wll = np.multiply(done_mask, wll)
                negged_wll = masked_wll + (done_mask_inv * -1e4)
                removed_eos = np.multiply(mask, negged_wll)
                sll = removed_eos + expanded_history
            else:
                sll = wll.T

            flat_sll = sll.reshape(-1)

            bests = topk(K, flat_sll)
            best_idx_flat = np.array(list(bests.keys()))
            best_beams = best_idx_flat // V
            best_idx = best_idx_flat % V

            new_paths = []
            new_done = []

            hidden = rnn_state.s()
            new_hidden = [[] for _ in range(num_states)]
            for j, best_flat in enumerate(best_idx_flat):
                beam_id = best_beams[j]
                best_word = best_idx[j]
                if done[j]:
                    new_paths.append(paths[beam_id] + [Offsets.EOS])
                else:
                    new_paths.append(paths[beam_id] + [best_word])
                if best_word == Offsets.EOS:
                    done[j] = True
                new_done.append(done[beam_id])
                scores[j] = bests[best_flat]
                # For each path, we need to pick that out and add it to the hiddens
                # This will be (c1, c2, ..., h1, h2, ...)
                for h_i, h in enumerate(hidden):
                    new_hidden[h_i].append(dy.pick_batch_elem(h, beam_id))

            done = np.array(new_done)
            new_hidden = [dy.concatenate_to_batch(new_h) for new_h in new_hidden]
            paths = new_paths
            rnn_state = self.decoder_rnn.initial_state(new_hidden)

        paths = np.stack([p[1:] for p in paths])
        return paths, scores

@register_decoder(name='default')
class RNNDecoderWithAttn(RNNDecoder):
    def __init__(self, tgt_embeddings, **kwargs):
        super(RNNDecoderWithAttn, self).__init__(tgt_embeddings, **kwargs)

    def init_attn(self, **kwargs):
        attn_type = kwargs.get('attn_type', 'bahdanau')
        self.attn_module = attention(attn_type, self.hsz, self.pc)

    def attn_cache(self, context):
        self.attn_module.cache_encoder(context)

    def attn(self, query, mask):
        return self.attn_module(query, mask)

@register_decoder(name='transformer')
class TransformerDecoderWrapper(DecoderBase):
    def __init__(self, tgt_embedding, dropout=0.5, layers=1, hsz=None, num_heads=4, scale=True, name='transformer-decoder-wrapper', **kwargs):
        pc = kwargs['pc'].add_subcollection(name=name)
        super(TransformerDecoderWrapper, self).__init__(pc)
        self.tgt_embedding = tgt_embedding
        dsz = self.tgt_embedding.get_dsz()
        hsz = dsz if hsz is None else hsz
        self.transformer_decoder = TransformerDecoderStack(num_heads, d_model=hsz, pdrop=dropout, scale=scale, layers=layers, pc=self.pc)
        self.proj_to_hsz = Linear(hsz, dsz, self.pc) if dsz != hsz else lambda x: x
        self.proj_to_dsz = WeightShareLinear(dsz, self.proj_to_hsz.weight, self.pc, transform=dy.transpose, name=self.proj_to_hsz.pc.name()) if dsz != hsz else lambda x: x
        self.preds = Linear(self.tgt_embedding.get_vsz(), dsz, self.pc)

    def output(self, x):
        return [self.preds(y) for y in dy.transpose(x)]

    def __call__(self, encoder_output, dst, train):
        embed_out_th_b = self.tgt_embedding.encode(dst)
        embed_out_ht_b = dy.transpose(embed_out_th_b)
        embed_out_ht_b = self.proj_to_hsz(embed_out_ht_b)
        context = dy.concatenate_cols(encoder_output.output)
        T = embed_out_ht_b.dim()[0][1]
        dst_mask = subsequent_mask(T)
        src_mask = encoder_output.src_mask
        output = self.transformer_decoder(embed_out_ht_b, context, src_mask, dst_mask, train)
        output = self.proj_to_dsz(output)
        return self.output(output)

    def predict_one(self, src, encoder_outputs, **kwargs):
        mxlen = int(kwargs.get('mxlen', 100))
        ys = np.full((1, 1), Offsets.GO)
        for i in range(mxlen - 1):
            probs = self(encoder_outputs, ys, False)
            next_word = np.argmax(probs[-1].npvalue())
            ys = np.concatenate([ys, np.full((1, 1), next_word)], axis=0)
            if next_word == Offsets.EOS:
                break
        return ys.transpose(), None

max_stars_repo_name: domyounglee/baseline

max_issues_repo_name: domyounglee/baseline

max_forks_repo_name: domyounglee/baseline


text:
import numpy as np
from collections import Counter
from sklearn import tree
from sklearn.datasets import load_iris
from copy import deepcopy

def emperical_entropy(y_train,train_no):
    y_dict=Counter(y_train)
    prob={key:y_dict[key]/train_no for key in y_dict.keys()}
    H=0
    for key in prob.keys():
        H-=prob[key]*np.log2(prob[key])
    return H

def emperical_conditional_entropy(y_train,A,train_no):
    A_value=list(set(A))
    A_no=len(A_value)
    A_dict=Counter(A)
    H_A=0
    for i in range(A_no):
        H_A+=A_dict[A_value[i]]/train_no*emperical_entropy(y_train[A==A_value[i]],A_dict[A_value[i]])
    return H_A

def find_feature_no(X_train,y_train,epsilon,method):
#    print(1)
    train_no,feature_no=np.shape(X_train)
    H=emperical_entropy(y_train,train_no)
    max_feature_no=0
    maximum=0

    for i in range(feature_no):
        H_A=emperical_conditional_entropy(y_train,X_train[:,i],train_no)
        if method=='ID3':
            g_A=H-H_A
        if method=='C4.5':
            g_A=(H-H_A)/H
#        print(g_A)

#        print(i,H,H_A,g_A)
        if g_A>maximum:
            maximum=g_A
            max_feature_no=i
    if maximum<epsilon:
        max_feature_no=0;length=0;set_A=[];leaf_node=True
#        print(maximum)

    else:
        set_A=list(set(X_train[:,max_feature_no]))
        length=len(set_A)
        leaf_node=False
#        print(max_feature_no,set_A)
    return max_feature_no,length,set_A,leaf_node

def generate(X_train,y_train,father_no,epsilon,method):
#    print(2)
    train_no,feature_no=np.shape(X_train)

    y_dict=Counter(y_train)
    C=max(y_dict,key=y_dict.get)
    if len(y_dict)==1 or train_no==0:
        max_feature_no=0;length=0;set_A=[];leaf_node=True
#        print('a')
    else:
        max_feature_no,length,set_A,leaf_node=find_feature_no(X_train,y_train,epsilon,method)
#        print('b')

    return (X_train,y_train,father_no,C,max_feature_no,length,set_A,leaf_node)

def pruning(node,leaf):

    for key in leaf:
        layer_no,node_no=key
#        print(layer_no,node_no)

    i=0
    while True:
        if node_no+1-node[(layer_no-1,i)][5]>0:
#            print("check:",node[(layer_no-1,i)][5])
            node_no=node_no-node[(layer_no-1,i)][5]
            i+=1

        else:
            node_no=i
            break

    X_train,y_train,father_no,C,max_feature_no,length,set_A,leaf_node=node[(layer_no-1,node_no)]
    node[(layer_no-1,node_no)] = X_train,y_train,father_no,C,max_feature_no,length,set_A,True
    print(layer_no-1,node_no)

    father_node_list=[(layer_no-1,node_no)]

    father_node_list,node=delete(father_node_list,node)

    maximum=0
    for key in node:
        if key[0]>maximum:
            maximum=key[0]
        layer_no=maximum

    return layer_no, node

def delete(father_node_list,node):

    if len(father_node_list)==0:
        return [],node

    else:
        temp_list=[]
        temp_node=deepcopy(node)
        for key,value in temp_node.items():
            if value[2] in father_node_list:

                temp_list.append(key)
                node.pop(key)

        node_list=deepcopy(temp_list)
        return delete(node_list,node)

def Loss_function(nobe,alpha):
    leaf_dict=find_leaf(nobe)
    L=0
    for key,value in leaf_dict.items():
        train_no=len(value[1])
        L+=train_no*emperical_entropy(value[1],train_no)
    return L+alpha*len(leaf_dict)

def find_leaf(tree):
    leaf={}
#    print(tree)
    for key,value in tree.items():
        if value[7]==True:
            leaf[key]=value
    return leaf

class DecisionTree():
    def __init__(self,X_train,y_train,X_test,epsilon=10**-3,alpha=1,criterion='entropy',method='ID3'):
        self.X_train=X_train
        self.y_train=y_train
        self.X_test=X_test
        train_no,feature_no=np.shape(self.X_train)
        test_no,_=np.shape(self.X_test)
        self.train_no=train_no
        self.feature_no=feature_no
        self.test_no=test_no
        self.epsilon=epsilon
        self.alpha=alpha
        self.criterion=criterion
        self.method=method

    def train(self):
        self.node={}
        self.layer_no=0
        node_no=1

        self.node[(0,0)]=(self.X_train,self.y_train,(np.nan,np.nan))

        while True:
            temp_no=0
            for i in range(node_no):
                self.node[(self.layer_no,i)]=generate(self.node[(self.layer_no,i)][0],self.node[(self.layer_no,i)][1],self.node[self.layer_no,i][2],self.epsilon,self.method)
                X_train,y_train,father_no,C,max_feature_no,length,set_A,leaf_node=self.node[(self.layer_no,i)]
#                print(length,max_feature_no)
                if leaf_node==False:
                    for j in range(length):
                        X_train_son=np.delete(X_train[X_train[:,max_feature_no]==set_A[j]],max_feature_no,1)
                        y_train_son=y_train[X_train[:,max_feature_no]==set_A[j]]
                        if y_train_son.size==0:
                            y_train_son=y_train
#                            X_train_son=X_train

                        self.node[(self.layer_no+1,temp_no+j)]=(X_train_son,y_train_son,(self.layer_no,i))
                    temp_no+=length

            if temp_no==0:
                break
#            print('layer_no:',self.layer_no)
            node_no=temp_no
            self.layer_no+=1
#        print(self.node.keys())

        minimum=Loss_function(self.node,self.alpha)
        print(minimum)
        flag=True
        while flag:
            leaf_dict=find_leaf(self.node)
            flag=False
            leaf={}
            print(leaf_dict.keys())
            for key in leaf_dict:
                print(key)
                if key[0]==0:
                    flag=True
                    break
                leaf[key]=leaf_dict[key]
                temp_node=deepcopy(self.node)
                layer_no,temp_node=pruning(temp_node, leaf)
                L=Loss_function(temp_node,self.alpha)
                print(L)
                leaf={}
                if L<minimum:
                    minimum=L
                    self.node=deepcopy(temp_node)
                    self.layer_no=layer_no
                    flag=True
                    break
        for key,value in self.node.items():
            if value[7]==True:
               print(key,value[2],value[3],value[6],value[7])

    def predict(self):
        y_test=np.zeros((self.test_no))

        for row_no in range(self.test_no):
            i=0
            node_no=0
            X_test=self.X_test[row_no,:]
            while i<=self.layer_no:
                X_train,y_train,father_no,C,max_feature_no,length,set_A,leaf_node=self.node[(i,node_no)]
#                print(i,node_no)
                if leaf_node==True:
                    y_test[row_no]=C
#                    print(y_test)
                    break

                index=set_A.index(X_test[max_feature_no])
                X_test=np.delete(X_test,max_feature_no)
                temp_no=0
                if node_no>=1:
                    for j in range(node_no):
                        temp_no+=self.node[(i,j)][5]
                    node_no=temp_no+index
                else:
                    node_no=index

                i+=1
        return y_test

if __name__=='__main__':
#    X_train=np.array([[0,0,0,0],[0,0,0,1],[0,1,0,1],[0,1,1,0],[0,0,0,0],[1,0,0,0],[1,0,0,1],[1,1,1,1],[1,0,1,2],[1,0,1,2],[2,0,1,2],[2,0,1,1],[2,1,0,1],[2,1,0,2],[2,0,0,0]])
#    y_train=np.array([0,0,1,1,0,0,0,1,1,1,1,1,1,1,0])
#    X_test=X_train
    iris=load_iris()
    X_train=iris.data[:,2:]
    y_train=iris.target
    X_train=X_train.astype(int)
    X_test=X_train

#    print(X_train)
#    print(y_train)

    model=DecisionTree(X_train,y_train,X_test,method='ID3')
    model.train()
    y_test=model.predict()
    print('Decision Tree', y_test)
    print(sum(y_test==y_train))
    model = tree.DecisionTreeClassifier(criterion='entropy')
    model.fit(X_train, y_train)
    y_test=model.predict(X_test)
    print('sklearn', y_test)
    print(sum(y_test==y_train))

max_stars_repo_path: Supervised Machine Learning Algorithms/Decision_Tree.py


text:
"""A video decoder processor that uses OpenCV to decode video into frames.

A wrapper around the OpenCV VideoCapture class to provide video frames to
downstream processors.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy

from video_processing import stream_processor
import cv2

class OpenCVVideoDecoderProcessor(stream_processor.ProcessorBase):
  """Processor for decoding video using OpenCV."""

  def __init__(self, configuration):
    self._input_video_file = configuration.get('input_video_file', '')
    self._video_stream_name = configuration.get('video_stream_name', 'video')
    self._index = 0

  def open(self, stream_set):
    self._capture = cv2.VideoCapture(self._input_video_file)
    frame_rate_hz = self._capture.get(cv2.CAP_PROP_FPS)
    if stream_set.frame_rate_hz < 0:
      # If the frame rate wasn't set, then we can assume that we're the top
      # of the processing chain and create a new stream set object to
      # contain the video stream.
      stream_set = stream_processor.StreamSet(frame_rate_hz=frame_rate_hz)
    else:
      if stream_set.frame_rate_hz != frame_rate_hz:
        raise Exception('Frame rate in additional video stream is not equal to '
                        'that of the original stream set.')
    image_width = int(self._capture.get(cv2.CAP_PROP_FRAME_WIDTH))
    image_height = int(self._capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
    num_frames = int(self._capture.get(cv2.CAP_PROP_FRAME_COUNT))
    stream_set.stream_headers[
        self._video_stream_name] = stream_processor.StreamHeader(
            frame_data_type=numpy.ndarray,
            header_data=stream_processor.VideoStreamHeader(
                image_width, image_height, num_frames))
    return stream_set

  def process(self, frame_set):
    if not self._capture.isOpened():
      return frame_set
    ret, frame_data = self._capture.read()
    if not ret:
      return frame_set
    frame_set[self._video_stream_name] = stream_processor.Frame(self._index,
                                                                frame_data)
    self._index += 1
    return frame_set

  def close(self):
    self._capture.release()
    return []

max_stars_repo_name: learningequality/video-vectorization

max_issues_repo_name: learningequality/video-vectorization

max_forks_repo_name: learningequality/video-vectorization


text:
import typing as t
from typing import TYPE_CHECKING

import numpy as np
import psutil
import pytest
import catboost as cbt
from catboost.core import CatBoost
from catboost.core import CatBoostRegressor
from catboost.core import CatBoostClassifier

import bentoml
import bentoml.models
from bentoml.exceptions import BentoMLException
from tests.utils.helpers import assert_have_file_extension
from tests.utils.frameworks.sklearn_utils import test_df

if TYPE_CHECKING:
    from bentoml._internal.store import Tag

def create_catboost_model() -> cbt.core.CatBoostClassifier:
    from sklearn.datasets import load_breast_cancer

    cancer = load_breast_cancer()

    X = cancer.data
    y = cancer.target

    clf = CatBoostClassifier(
        iterations=2,
        depth=2,
        learning_rate=1,
        loss_function="Logloss",
        verbose=False,
    )

    # train the model
    clf.fit(X, y)

    return clf

def save_procedure(
    model_params: t.Dict[str, t.Any],
    metadata: t.Dict[str, t.Any],
) -> "Tag":
    catboost_model = create_catboost_model()
    tag_info = bentoml.catboost.save(
        "test_catboost_model",
        catboost_model,
        model_params=model_params,
        metadata=metadata,
    )
    return tag_info

def forbidden_procedure():
    catboost_model = create_catboost_model()
    with bentoml.models.create(
        "invalid_module",
        module=__name__,
        labels=None,
        options=None,
        context=None,
        metadata=None,
    ) as ctx:
        catboost_model.save_model(ctx.path_of("saved_model.model"))
        return ctx.tag

@pytest.mark.parametrize(
    "model_params, metadata", [(dict(model_type="classifier"), {"acc": 0.876})]
)
def test_catboost_save_load(
    model_params: t.Dict[str, t.Any],
    metadata: t.Dict[str, t.Any],
) -> None:
    tag = save_procedure(model_params, metadata)
    _model = bentoml.models.get(tag)
    assert _model.info.metadata is not None
    assert_have_file_extension(_model.path, ".cbm")

    cbt_loaded = bentoml.catboost.load(_model.tag, model_params=model_params)
    assert isinstance(cbt_loaded, CatBoostClassifier)
    assert cbt_loaded.predict(test_df) == np.array([1])

def test_catboost_load_exc() -> None:
    tag = forbidden_procedure()
    with pytest.raises(BentoMLException):
        _ = bentoml.catboost.load(tag)

@pytest.mark.parametrize(
    "model_type, expected_model",
    [
        ("regressor", CatBoostRegressor),
        ("classifier", CatBoostClassifier),
        ("", CatBoost),
    ],
)
def test_catboost_model_type(
    model_type: str,
    expected_model: t.Union[CatBoost, CatBoostClassifier, CatBoostRegressor],
) -> None:
    model_params = {"model_type": model_type}
    info = save_procedure(model_params, {})
    cbt_loaded = bentoml.catboost.load(info, model_params=model_params)

    assert isinstance(cbt_loaded, expected_model)

def test_catboost_runner_setup_run_batch() -> None:
    tag = save_procedure(
        {},
        {},
    )
    runner = bentoml.catboost.load_runner(tag)

    assert tag in runner.required_models
    assert runner.num_replica == psutil.cpu_count()
    assert runner.run_batch(test_df) == np.array([1])


text:
import numpy as np

#  The first two functions are modified from MNE surface project. LIcense follows
#  This software is OSI Certified Open Source Software. OSI Certified is a certification mark of the Open Source Initiative.
#
#  Copyright (c) 2011-2019, authors of MNE-Python. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
#
#  Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
#  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
#  Neither the names of MNE-Python authors nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission.
#
#  This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage.

def triangle_neighbors(tris, npts):
    """Efficiently compute vertex neighboring triangles.
    Returns the triangles in the 1-ring of a given vertex
    """
    # this code replaces the following, but is faster (vectorized):
    #
    # this['neighbor_tri'] = [list() for _ in xrange(this['np'])]
    # for p in xrange(this['ntri']):
    #     verts = this['tris'][p]
    #     this['neighbor_tri'][verts[0]].append(p)
    #     this['neighbor_tri'][verts[1]].append(p)
    #     this['neighbor_tri'][verts[2]].append(p)
    # this['neighbor_tri'] = [np.array(nb, int) for nb in this['neighbor_tri']]
    #
    verts = tris.ravel()
    counts = np.bincount(verts, minlength=npts)
    reord = np.argsort(verts)
    tri_idx = np.unravel_index(reord, (len(tris), 3))[0]
    idx = np.cumsum(np.r_[0, counts])
    # the sort below slows it down a bit, but is needed for equivalence
    neighbor_tri = np.array([np.sort(tri_idx[v1:v2])
                    for v1, v2 in zip(idx[:-1], idx[1:])])
    return neighbor_tri

def get_surf_neighbors(tris,neighbor_tri, k):
    """Get vertices of 1-ring
    """
    verts = tris[neighbor_tri[k]]
    verts = np.setdiff1d(verts, [k], assume_unique=False)
    nneighbors = len(verts)
    return verts

def GetVertexNormals(vertices,faces,FaceNormals,e0,e1,e2):
    """
    INPUT:
    Vertices       : vertices
    Faces          : vertex connectivity
    FaceNormals    :     Outer Normal per face, having magnitude equal to area of face
    e0,e1,e2       : edge vectors

    OUTPUT:
    VertNormals    :       Unit normal at the vertex
    """

    VertNormals   =np.zeros(vertices.shape)

    #edge lengths
    de0=np.sqrt(e0[:,0]**2+e0[:,1]**2+e0[:,2]**2)
    de1=np.sqrt(e1[:,0]**2+e1[:,1]**2+e1[:,2]**2)
    de2=np.sqrt(e2[:,0]**2+e2[:,1]**2+e2[:,2]**2)

    L2=np.c_[de0**2,de1**2,de2**2]

    #Calculate weights according to N.Max [1999] for normals
    wfv1=FaceNormals/(L2[:,1]*L2[:,2])[:,np.newaxis]
    wfv2=FaceNormals/(L2[:,2]*L2[:,0])[:,np.newaxis]
    wfv3=FaceNormals/(L2[:,0]*L2[:,1])[:,np.newaxis]

#    #Calculate the weights according to MWA for normals
#    wfv1=FaceNormals*np.arcsin(2*Af/(de1*de2))[:,np.newaxis]
#    wfv2=FaceNormals*np.arcsin(2*Af/(de2*de0))[:,np.newaxis]
#    wfv3=FaceNormals*np.arcsin(2*Af/(de0*de1))[:,np.newaxis]

    verts=faces.T[0]
    for j in [0,1,2]:
      VertNormals[:,j]+=np.bincount(verts,minlength=vertices.shape[0],weights=wfv1[:,j])
    verts=faces.T[1]
    for j in [0,1,2]:
      VertNormals[:,j]+=np.bincount(verts,minlength=vertices.shape[0],weights=wfv2[:,j])
    verts=faces.T[2]
    for j in [0,1,2]:
      VertNormals[:,j]+=np.bincount(verts,minlength=vertices.shape[0],weights=wfv3[:,j])

    VertNormals=normr(VertNormals)

    return VertNormals

def fastcross(x, y):
    """Compute cross product between list of 3D vectors
    Input
    x       :  Mx3 array
    y       :  Mx3 array

    Output
    z       : Mx3 array Cross product of x and y.
    """
    if max([x.shape[0], y.shape[0]]) >= 500:
        return np.c_[x[:, 1] * y[:, 2] - x[:, 2] * y[:, 1],
                        x[:, 2] * y[:, 0] - x[:, 0] * y[:, 2],
                        x[:, 0] * y[:, 1] - x[:, 1] * y[:, 0]]
    else:
        return np.cross(x, y)

def normr(vec):
    """
    Normalizes an array of vectors. e.g. to convert a np array of vectors to unit vectors
    """
    return vec/np.sqrt((vec**2).sum(axis=1))[:,np.newaxis]

max_stars_repo_name: AbhilashReddyM/curvpack

max_issues_repo_name: AbhilashReddyM/curvpack

max_forks_repo_name: AbhilashReddyM/curvpack