text:
"""
callbackUsingNumPy.py

Demonstrates how to use callbacks with Callback.PREVIEW, using a NumPy image
The callback function will modify the preview buffer supplied by the API.
"""

from pixelinkWrapper import*
from ctypes import*
import ctypes.wintypes
import time
import threading
import numpy as np

def get_pixel_format_as_string(dataFormat):
    switcher = {
        PxLApi.PixelFormat.MONO8: "MONO8",
        PxLApi.PixelFormat.MONO16: "MONO16",
        PxLApi.PixelFormat.YUV422: "YUV422",
        PxLApi.PixelFormat.BAYER8_GRBG: "BAYER8_GRBG",
        PxLApi.PixelFormat.BAYER16_GRBG: "BAYER16_GRBG",
        PxLApi.PixelFormat.RGB24: "RGB24",
        PxLApi.PixelFormat.RGB48: "RGB48",
        PxLApi.PixelFormat.BAYER8_RGGB: "BAYER8_RGGB",
        PxLApi.PixelFormat.BAYER8_GBRG: "BAYER8_GBRG",
        PxLApi.PixelFormat.BAYER8_BGGR: "BAYER8_BGGR",
        PxLApi.PixelFormat.BAYER16_RGGB: "BAYER16_RGGB",
        PxLApi.PixelFormat.BAYER16_GBRG: "BAYER16_GBRG",
        PxLApi.PixelFormat.BAYER16_BGGR: "BAYER16_BGGR",
        PxLApi.PixelFormat.MONO12_PACKED: "MONO12_PACKED",
        PxLApi.PixelFormat.BAYER12_GRBG_PACKED: "BAYER12_GRBG_PACKED",
        PxLApi.PixelFormat.BAYER12_RGGB_PACKED: "BAYER12_RGGB_PACKED",
        PxLApi.PixelFormat.BAYER12_GBRG_PACKED: "BAYER12_GBRG_PACKED",
        PxLApi.PixelFormat.BAYER12_BGGR_PACKED: "BAYER12_BGGR_PACKED",
        PxLApi.PixelFormat.RGB24_NON_DIB: "RGB24_NON_DIB",
        PxLApi.PixelFormat.RGB48_DIB: "RGB48_DIB",
        PxLApi.PixelFormat.MONO12_PACKED_MSFIRST: "MONO12_PACKED_MSFIRST",
        PxLApi.PixelFormat.BAYER12_GRBG_PACKED_MSFIRST: "BAYER12_GRBG_PACKED_MSFIRST",
        PxLApi.PixelFormat.BAYER12_RGGB_PACKED_MSFIRST: "BAYER12_RGGB_PACKED_MSFIRST",
        PxLApi.PixelFormat.BAYER12_GBRG_PACKED_MSFIRST: "BAYER12_GBRG_PACKED_MSFIRST",
        PxLApi.PixelFormat.BAYER12_BGGR_PACKED_MSFIRST: "BAYER12_BGGR_PACKED_MSFIRST",
        PxLApi.PixelFormat.MONO10_PACKED_MSFIRST: "MONO10_PACKED_MSFIRST",
        PxLApi.PixelFormat.BAYER10_GRBG_PACKED_MSFIRST: "BAYER10_GRBG_PACKED_MSFIRST",
        PxLApi.PixelFormat.BAYER10_RGGB_PACKED_MSFIRST: "BAYER10_RGGB_PACKED_MSFIRST",
        PxLApi.PixelFormat.BAYER10_GBRG_PACKED_MSFIRST: "BAYER10_GBRG_PACKED_MSFIRST",
        PxLApi.PixelFormat.BAYER10_BGGR_PACKED_MSFIRST: "BAYER10_BGGR_PACKED_MSFIRST",
        PxLApi.PixelFormat.STOKES4_12: "STOKES4_12",
        PxLApi.PixelFormat.POLAR4_12: "POLAR4_12",
        PxLApi.PixelFormat.POLAR_RAW4_12: "POLAR_RAW4_12",
        PxLApi.PixelFormat.HSV4_12: "HSV4_12",
        PxLApi.PixelFormat.BGR24_NON_DIB: "BGR24_NON_DIB"
        }
    return switcher.get(dataFormat, "Unknown data format")

"""
Creates a NumPy 2D array representation of a byte pointer used the the Pixelink API.
    frameData: Byte pointer to the image provided by the Pixelink API
    width:     Width of the image (in pixels)
    height:    Height of the image (in pixels)
    bytesPerPixel: the number of bytes per pixel
"""
def numPy_image (frameData, width, height, bytesPerPixel):
    buffer_from_memory = pythonapi.PyMemoryView_FromMemory
    buffer_from_memory.restype = py_object
    pBuffer = buffer_from_memory(frameData, width * height * bytesPerPixel, 0x200) # 0x200 == writable
    return np.frombuffer(pBuffer, np.uint8).reshape(height, width * bytesPerPixel)

"""
Callback function called by the API just before an image is displayed in the preview window.
    N.B. This is called by the API on a thread created in the API.
"""
@PxLApi._dataProcessFunction
def callback_format_preview(hCamera, frameData, dataFormat, frameDesc, userData):
    # Copy frame descriptor information
    frameDescriptor = frameDesc.contents
    # Find image dimensions
    width = int(frameDescriptor.Roi.fWidth / frameDescriptor.PixelAddressingValue.fHorizontal)
    height = int(frameDescriptor.Roi.fHeight / frameDescriptor.PixelAddressingValue.fVertical)
    bytesPerPixel = PxLApi.getBytesPerPixel(dataFormat)

    # Recast the returned image as a NumPy 2-Darray, that we can modify
    npFrame = numPy_image (frameData, width, height, bytesPerPixel)

    print("callback_format_image: hCamera = {0}, frameData = {1}".format(hex(hCamera),
                                                                         hex(id(frameData))))
    print("    dataFormat = {0} {1}, FrameDesc = {2}".format(dataFormat,
                                                             get_pixel_format_as_string(dataFormat),
                                                             hex(id(frameDesc))))
    print("    userData = {0}, threadId = {1}".format(hex(userData), hex(id(threading.current_thread()))))
    print("    imageData = {0} {1} {2} {3} {4} {5} {6} {7}\n".format(hex(frameData[0]), hex(frameData[1]), hex(frameData[2]),
                                                                   hex(frameData[3]), hex(frameData[4]), hex(frameData[5]),
                                                                   hex(frameData[6]), hex(frameData[7])))

    # Just to see the effect of the callback, increase intensity of the middle 20% of the pixels, to 100%
    startRow = int((height/5)*2)
    endRow = int((height/5)*3)
    startCol = int(((width*bytesPerPixel)/5)*2)
    endCol = int(((width*bytesPerPixel)/5)*3)

    npFrame[startRow:endRow,startCol:endCol] = 0xff

    return 0

"""
Runs preview for a number of seconds by polling the message pump on Windows.
"""
def run_preview_window_for(hCamera, seconds):

    previewTime = 0
    startPreview = time.time()

    # The preview window will go 'Not Responding' if we do not poll the message pump, and
    # forward events onto it's handler on Windows.
    user32 = windll.user32
    msg = ctypes.wintypes.MSG()
    pMsg = ctypes.byref(msg)

    # Start preview
    ret = PxLApi.setPreviewState(hCamera, PxLApi.PreviewState.START)
    if(not(PxLApi.apiSuccess(ret[0]))):
        print("ERROR setting preview state function: {0}".format(ret[0]))
        return

    # Run preview for a number of seconds by polling the message pump
    while seconds > previewTime:
        if user32.PeekMessageW(pMsg, 0, 0, 0, 1) != 0:
            user32.TranslateMessage(pMsg)
            user32.DispatchMessageW(pMsg)
        previewTime = time.time() - startPreview

    # Stop preview
    ret = PxLApi.setPreviewState(hCamera, PxLApi.PreviewState.STOP)

def do_callback_on_preview(hCamera):
    # Set the callback function
    print("=====================================================\n")
    print("do_callback_on_preview\n")
    userData = 3735928559
    print("Registering PREVIEW callback with userData {0}\n".format(hex(userData)))
    ret = PxLApi.setCallback(hCamera, PxLApi.Callback.PREVIEW, userData, callback_format_preview)
    if(not(PxLApi.apiSuccess(ret[0]))):
        print("ERROR setting callback function: {0}".format(ret[0]))
        return

    ret = PxLApi.setStreamState(hCamera, PxLApi.StreamState.START)
    if(not(PxLApi.apiSuccess(ret[0]))):
        print("ERROR setting stream state function: {0}".format(ret[0]))
        return

    # Set the number of seconds to run preview for
    runPreviewFor = 10
    # We will start getting our callback called after we start previewing
    # Run preview for a number of seconds
    run_preview_window_for(hCamera, runPreviewFor)

    ret = PxLApi.setStreamState(hCamera, PxLApi.StreamState.STOP)

    # Disable callback on preview by setting the callback function to 0 or None
    ret = PxLApi.setCallback(hCamera, PxLApi.Callback.PREVIEW, userData, 0)

def main():

    ret = PxLApi.initialize(0)
    if(not(PxLApi.apiSuccess(ret[0]))):
        print("ERROR: {0}\n".format(ret[0]))
        return 1
    hCamera = ret[1]
    print("\nMain thread id = {}\n".format(hex(id(threading.current_thread()))))

    # do_callback_on_format_image(hCamera) /* Callback.FORMAT_IMAGE is not supported */
    do_callback_on_preview(hCamera)

    PxLApi.uninitialize(hCamera)
    return 0

if __name__ == "__main__":
    main()

max_stars_repo_name: pixelink-support/PixelinkPythonWrapper

max_issues_repo_name: pixelink-support/PixelinkPythonWrapper

max_forks_repo_name: pixelink-support/PixelinkPythonWrapper


text:
import numpy
import numba
import awkward1

@numba.njit
def _distinctParent_kernel(allpart_parent, allpart_pdg):
    out = numpy.empty(len(allpart_pdg), dtype=numpy.int64)
    for i in range(len(allpart_pdg)):
        parent = allpart_parent[i]
        if parent < 0:
            out[i] = -1
            continue
        thispdg = allpart_pdg[i]
        while parent >= 0 and allpart_pdg[parent] == thispdg:
            if parent >= len(allpart_pdg):
                raise RuntimeError("parent index beyond length of array!")
            parent = allpart_parent[parent]
        out[i] = parent
    return out

def distinctParent(parents, pdg):
    def gen():
        return _distinctParent_kernel(
            awkward1.Array(parents.array), awkward1.Array(pdg.array)
        )

    form = awkward1.forms.Form.fromjson('"int64"')
    return awkward1.layout.ArrayGenerator(
        gen, (), {}, form=form, length=parents.generator.length,
    )

@numba.njit
def _children_kernel(offsets_in, parentidx):
    offsets1_out = numpy.empty(len(parentidx) + 1, dtype=numpy.int64)
    content1_out = numpy.empty(len(parentidx), dtype=numpy.int64)
    offsets1_out[0] = 0

    offset0 = 0
    offset1 = 0
    for record_index in range(len(offsets_in) - 1):
        start_src, stop_src = offsets_in[record_index], offsets_in[record_index + 1]

        for index in range(start_src, stop_src):
            for possible_child in range(index, stop_src):
                if parentidx[possible_child] == index:
                    content1_out[offset1] = possible_child
                    offset1 = offset1 + 1
                    if offset1 >= len(content1_out):
                        raise RuntimeError("offset1 went out of bounds!")
            offsets1_out[offset0 + 1] = offset1
            offset0 = offset0 + 1
            if offset0 >= len(offsets1_out):
                raise RuntimeError("offset0 went out of bounds!")

    return offsets1_out, content1_out[:offset1]

def children(offsets, parents):
    def gen():
        coffsets, ccontent = _children_kernel(
            awkward1.Array(offsets.array), awkward1.Array(parents.array)
        )
        return awkward1.layout.ListOffsetArray64(
            awkward1.layout.Index64(coffsets), awkward1.layout.NumpyArray(ccontent),
        )

    form = awkward1.forms.Form.fromjson(
        '{"class": "ListOffsetArray64", "offsets": "i64", "content": "int64"}'
    )
    return awkward1.layout.ArrayGenerator(
        gen, (), {}, form=form, length=parents.generator.length,
    )

ext: py

max_stars_repo_name: dntaylor/coffea

max_issues_repo_name: dntaylor/coffea

max_forks_repo_name: dntaylor/coffea

reason: import numpy,import numba


text:
import rospy
import tf
import numpy
from sensor_msgs.msg import LaserScan, Image
from geometry_msgs.msg import Twist, PoseStamped, Quaternion, Point
from nav_msgs.msg import Odometry
import actionlib
from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal
from convert import image_to_numpy
from tf.transformations import euler_from_quaternion, quaternion_from_euler
import numpy as np
from math import floor, pi,isnan, copysign, sin, cos, asin, acos
import argparse
from collections import defaultdict
sign = lambda x: copysign(1, x)

R_SPEED = 0.2
M_SPEED = 0.08

def rotate_point(point, angle):
    return (cos(angle) * point[0] - sin(angle) * point[1],
        sin(angle) *  point[0] + cos(angle) * point[1])

class Controller:
    def __init__(self):
        self.imageTransform = tf.TransformListener()
        self.laserScanListener = rospy.Subscriber("/scan", LaserScan, callback = self._receiveLaserScan)
        self._velocityPublisher = rospy.Publisher("/cmd_vel_mux/input/navi", Twist, queue_size = 10)
        self._odomSubscriber = rospy.Subscriber("/odom", Odometry, callback = self._receiveOdometry)
        self._laserScan = None
        self._pendingMovement = False
        self._image = None
        self.visualize = None
        self._startPose = None
        self._depth_image = None
        self._tangle = None
        self._positionTransform2 = None
        self._odom = None
        def collectImage(msg):
            self._image = msg
        def collectDepth(msg):
            self._depth_image = msg
        rospy.Subscriber("/camera/rgb/image_raw", Image, callback = collectImage)
        rospy.Subscriber("/camera/depth/image_raw", Image, callback = collectDepth)

    def _receiveLaserScan(self, msg):
        self._laserScan = msg

    def _receiveOdometry(self, msg):
        pos = msg.pose.pose.position
        quat = msg.pose.pose.orientation
        self._odom = ([pos.x, pos.y, pos.z], [quat.x, quat.y, quat.z, quat.w])

    def _lookupOdometry(self):
        if not self._odom:
            raise tf.LookupException()
        return self._odom

    def start(self):
        rate = rospy.Rate(10.0)
        while not rospy.is_shutdown():
            if not self._receiveSensorInput():
                continue
            # if self._pendingMovement:
            #     _, duration = rate.remaining()
            #     if self._baseController.wait_for_result(timeout = duration):
            #         self._pendingMovement = False
            # else:
            rate.sleep()
            return

    def collect(self, observation, posx, posy):
        print("Collecting")

    def _printPosition(self):
        print("position: %s, rotation: %s" % (self._position, self._rotation))

    def _receiveSensorInput(self):
        try:
            (trans, quat) = self._lookupOdometry()
            (trans2, quat2) = self.imageTransform.lookupTransform("/map", "/base_link",rospy.Time(0))
            if self._laserScan is None:
                return False
            self._rotation = euler_from_quaternion(quat)
            self._position = trans[:2]
            self._positionTransform2 = tuple(trans2[:2]) + (euler_from_quaternion(quat2)[2],)
            #self._position = rotate_point(trans, self._rotation[2])

            if not self._startPose:
                self._startPose = (self._position, self._rotation,)

        except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException):
            return False
        return True

    def move_to(self, target, rotate_final = True):
        while not rospy.is_shutdown() and not self._receiveSensorInput():
            pass

        rotStart = self._rotation[2]
        pos = np.array(self._position, np.float32)
        target = np.array([target[0], target[1]], np.float32)
        direction = target - pos
        direction_angle = np.arctan2(direction[1], direction[0])
        distance = np.linalg.norm(direction)

        print("navigating to [%s, %s]" % (target[0], target[1]))
        self.rotate_to(direction_angle)
        self.move_by(distance)
        if rotate_final:
            self.rotate_to(rotStart)

    def move_by(self, distance):
        rate = rospy.Rate(100.0)
        while not rospy.is_shutdown() and not self._receiveSensorInput():
            pass

        start = np.array(self._position, np.float32)
        angle = self._rotation[2]
        ndir = np.array([cos(angle), sin(angle)], np.float32)
        target = start + distance * ndir
        while not rospy.is_shutdown():
            if not self._receiveSensorInput():
                continue

            diff = target - np.array(self._position, np.float32)
            dir_distance = np.dot(diff, ndir)
            if abs(dir_distance) <= 0.05:
                break

            msg = Twist()
            speed = M_SPEED
            msg.linear.x = sign(dir_distance) * speed
            self._velocityPublisher.publish(msg)
            rate.sleep()
        self._printPosition()

    def rotate_to(self, angle):
        while not rospy.is_shutdown() and not self._receiveSensorInput():
            pass
        # TODO: test this line
        #angle = (angle + self._startPose[1][2] + pi) % (2 * pi) - pi
        angle = (angle + pi) % (2 * pi) - pi
        rate = rospy.Rate(100.0)
        while not rospy.is_shutdown():
            if not self._receiveSensorInput():
                continue

            diff = self._rotation[2] - angle
            diff = (diff + pi) % (2 * pi) - pi
            if abs(diff) < 0.05:
                return

            msg = Twist()
            speed = R_SPEED
            msg.angular.z = -sign(diff) * speed
            self._velocityPublisher.publish(msg)
            rate.sleep()

        self._printPosition()
        msg = Twist()
        self._velocityPublisher.publish(msg)

    def rotate_by(self, angle):
        if self._tangle is None:
            while not rospy.is_shutdown() and not self._receiveSensorInput():
                pass

            self._tangle = round(self._rotation[2] / 1.57) * 1.57

        self._tangle = (self._tangle + angle + pi) % (2 * pi) - pi
        self.rotate_to(self._tangle)

    def is_occupied(self):
        assert self._laserScan is not None

        arange = self._laserScan.angle_max - self._laserScan.angle_min
        real_arange = min(arange, pi/6)
        pointCount = len(self._laserScan.ranges)
        stripPoints = int(floor((pointCount - pointCount * real_arange / arange) / 2))
        points = [0.2 if isnan(p) else p for p in self._laserScan.ranges[stripPoints:-stripPoints]]
        lbound = np.quantile(points, 0.1)
        return lbound < 0.5

    def observe(self):
        rospy.sleep(rospy.Duration(0.7))
        while not rospy.is_shutdown() and not self._receiveSensorInput():
            pass
        assert self._image is not None
        assert self._depth_image is not None
        return [
            image_to_numpy(self._image),
            image_to_numpy(self._depth_image),
            self._position,
            self._rotation,
            self._positionTransform2
        ]

    def position(self):
        while not rospy.is_shutdown() and not self._receiveSensorInput():
            continue

        return self._position

    def position_and_rotation(self):
        while not rospy.is_shutdown() and not self._receiveSensorInput():
            continue
        return tuple(self._position) + (self._rotation[2],)

if __name__ == "__main__":
    p = argparse.ArgumentParser("map_collector")
    p.add_argument("--angle", type = float, default=float("NaN"))
    p.add_argument("--goal", type = float, nargs = 2, metavar = ('x','y'), default=None)
    p.add_argument("--by", type = float, nargs = 2, action ="append", metavar = ('x','y'), default=None)
    p.add_argument("--jump", default=False, action="store_true")
    p.add_argument("--distance", type = float, default=float("NaN"))
    p = p.parse_args()
    rospy.init_node("map_collector")
    controller = Controller()
    controller.start()

    pos = controller.position()
    if p.by and len(p.by) > 0:
        for pp in p.by:
            controller.move_to(pp, rotate_final=False)

    if p.goal:
        controller.move_to(p.goal)
    if p.jump:
        controller.move_to(pos, rotate_final=False)

    if not isnan(p.angle):
        controller.rotate_to(p.angle)
    if not isnan(p.distance):
        controller.move_by(p.distance)

max_stars_repo_name: jkulhanek/robot-visual-navigation

max_issues_repo_name: leefree-GIT/robot-visual-navigation

max_forks_repo_name: leefree-GIT/robot-visual-navigation

reason: import numpy


text:
import libmist as pld
import numpy as np
import pytest
from multiprocessing import cpu_count

def test_parallel_2():
    filename = "sample_data.csv"
    cpus = cpu_count()
    total_ranks = cpus * 2
    ranks_per = cpus

    # non-parallel
    mist = pld.Search()
    mist.load_file_column_major(filename)
    mist.tuple_size = 3
    res = mist.start()

    # first ranks
    mist = pld.Search()
    mist.load_file_column_major(filename)
    mist.tuple_size = 3
    mist.ranks = ranks_per
    mist.total_ranks = total_ranks
    res1 = mist.start()

    # second ranks
    mist = pld.Search()
    mist.load_file_column_major(filename)
    mist.tuple_size = 3
    mist.ranks = ranks_per
    mist.total_ranks = total_ranks
    mist.start_rank = ranks_per
    res2 = mist.start()

    res_sort = np.sort(res, axis=0)
    res_parallel = np.concatenate([res1, res2])
    res_parallel_sort = np.sort(res_parallel, axis=0)
    assert(res.shape[0] == res_parallel_sort.shape[0])
    assert(res.shape[1] == res_parallel_sort.shape[1])
    np.testing.assert_array_equal(res_sort, res_parallel_sort)

def test_parallel_3_same_obj():
    filename = "sample_data.csv"
    cpus = cpu_count()
    total_ranks = cpus * 3
    ranks_per = cpus

    # non-parallel
    mist = pld.Search()
    mist.load_file_column_major(filename)
    mist.tuple_size = 3
    res = mist.start()

    # first ranks
    mist.ranks = ranks_per
    mist.total_ranks = total_ranks
    res1 = mist.start()

    # second ranks
    mist.ranks = ranks_per
    mist.total_ranks = total_ranks
    mist.start_rank = ranks_per
    res2 = mist.start()

    # third ranks
    mist.ranks = ranks_per
    mist.total_ranks = total_ranks
    mist.start_rank = ranks_per * 2
    res3 = mist.start()

    res_sort = np.sort(res, axis=0)
    res12 = np.concatenate([res1, res2])
    res_parallel = np.concatenate([res12, res3])
    res_parallel_sort = np.sort(res_parallel, axis=0)

    assert(res.shape[0] == res_parallel_sort.shape[0])
    assert(res.shape[1] == res_parallel_sort.shape[1])
    np.testing.assert_array_equal(res_sort, res_parallel_sort)

max_stars_repo_name: andbanman/mist

max_issues_repo_name: andbanman/mist

max_forks_repo_name: andbanman/mist


text:
import json
import numpy
from PIL import Image, ImageOps
import os
import torch

class SegmentationDataset:
    def __init__(self, json_file_name):

        with open(json_file_name) as json_file:
            self.json_config = json.load(json_file)

        self.channels   = int(self.json_config["input_channels"])
        self.height     = int(self.json_config["input_height"])
        self.width      = int(self.json_config["input_width"])

        self.target_channels   = int(self.json_config["target_channels"])

        self.input_shape     = (self.channels, self.height, self.width)
        self.output_shape    = (self.target_channels, self.height, self.width)

        self.input_path     = str(self.json_config["input_path"])
        self.target_path    = str(self.json_config["target_path"])

        augmentations_count = int(self.json_config["augmentations_count"]) + 1

        input_files     = self._find_files(self.input_path)
        target_files    = self._find_files(self.target_path)

        self.count = len(input_files)

        self.input  = numpy.zeros((self.count*augmentations_count, self.channels,        self.height, self.width))
        self.target = numpy.zeros((self.count*augmentations_count, self.target_channels, self.height, self.width))

        for i in range(self.count):
            for augmentation in range(augmentations_count):
                if numpy.random.randint(2) == 0:
                    flip    = True
                else:
                    flip    = False

                if numpy.random.randint(2) == 0:
                    mirror  = True
                else:
                    mirror  = False

                crop_area_level = float(self.json_config["crop_area_level"])

                crop_area   = crop_area_level + (1.0 - crop_area_level)*numpy.random.rand()
                noise_level = numpy.random.rand()*float(self.json_config["noise_level"])

                if augmentation == 0:
                    flip        = False
                    mirror      = False
                    crop_area   = 1.0
                    noise_level = 0.0

                print("loading ", input_files[i], target_files[i])

                input_img    = self._load_image(input_files[i], True, flip, mirror, crop_area)
                target_img   = self._load_image(target_files[i], False, flip, mirror, crop_area)

                if self.channels == 3:
                    to_rgb = True
                else:
                    to_rgb = False

                input_np     = self._image_to_numpy(input_img, to_rgb, True, noise_level)

                target_np    = self._image_to_numpy(target_img, False, False)
                target_np    = numpy.clip(target_np, 0, self.target_channels)

                self.input[i]   = input_np.copy()
                self.target[i]  = target_np.copy()

    def get_count(self):
        return self.count

    def get_batch(self, batch_size = 32):
        input   = torch.zeros((batch_size, self.channels, self.height, self.width))
        target  = torch.zeros((batch_size, self.target_channels, self.height, self.width))

        for i in range(batch_size):
            idx = numpy.random.randint(self.count)
            input[i]  = torch.from_numpy(self.input[idx])
            target[i] = torch.from_numpy(self.target[idx])

        return input, target

    def _find_files(self, path):
        files_list = []

        for file in os.listdir(path):
            if file.endswith(".png"):
                files_list.append(str(file))

        result = []
        for file_name in files_list:
            result.append(path + file_name)

        result.sort()

        return result

    def _load_image(self, file_name, to_rgb, flip = False, mirror = False, crop_area = 1.0 ):
        image = Image.open(file_name)

        image = image.crop((image.width*(1 - crop_area), image.height*(1 - crop_area), image.width*crop_area, image.height*crop_area))

        image = image.resize((self.width, self.height), Image.LANCZOS)

        if flip:
            image = ImageOps.flip(image)

        if mirror:
            image = ImageOps.mirror(image)

        if to_rgb:
            image       = image.convert('RGB')
        else:
            image       = image.convert('L')

        return image

    def _image_to_numpy(self, image, to_rgb, normalize, noise_level = 0):

        if to_rgb:
            image       = image.convert('RGB')
            image_np    = numpy.array(image).astype(numpy.uint8)
            image_np    = numpy.rollaxis(image_np, 2, 0)
        else:
            image       = image.convert('L')
            image_np    = numpy.array(image).astype(numpy.uint8)
            image_np    = numpy.expand_dims(image_np, axis = 0)

        if normalize:
            image_np    = image_np/255.0
            noise_mul   = 1
        else:
            noise_mul   = 255

        if noise_level > 0:
            rnd      = noise_mul*noise_level*numpy.random.randn(*image_np.shape)
            image_np = (1.0 - noise_level)*image_np + noise_level*rnd

        if normalize:
            image_np = numpy.clip(image_np, 0, 1)
        else:
            image_np = numpy.clip(image_np, 0, 255)

        return image_np


text:
import numpy as np
import pandas as pd
import hydrostats.ens_metrics as em
import unittest
from compress_netcdf import compress_netcfd
import os
import xarray as xr

class TestCompressNetcdf(unittest.TestCase):
    """
    Tests the functions included in compress_netcdf.py to make sure that they are working correctly with
    your python version.
    """

    def setUp(self):
        pass

    def test_compress_netcdf(self):
        start_date = "20190104"
        folder_path = os.path.join(os.getcwd(), 'Test_files/Individual_Ensembles')
        out_path = os.path.join(os.getcwd(), 'Test_files')
        compress_netcfd(folder_path, start_date, out_path, "Qout_south_america_continental", 10)

        ds = xr.open_dataset(os.path.join(os.getcwd(), out_path, start_date + ".nc"))

        flow_array = ds["Qout"].data
        flow_array_high_res = ds["Qout_high_res"].data
        initialization_values = ds["initialization_values"].data
        date = ds["date"].data
        date_high_res = ds["date_high_res"].data
        rivids = ds["rivid"].data

        benchmark_flow_array = np.load("Test_files/Comparison_Files/benchmark_flow_array.npy")
        benchmark_array_high_res = np.load("Test_files/Comparison_Files/benchmark_array_high_res.npy")
        benchmark_initialization_values = np.array([1.9156765937805176, 1.7540310621261597, 1.5276174545288086,
                                                    2.0937132835388184, 3.4932050704956055, 3.678524971008301,
                                                    0.854943037033081, 1.688418984413147, 13.452759742736816,
                                                    3.7613487243652344], dtype=np.float32)
        benchmark_date = np.array(['2019-01-05', '2019-01-06', '2019-01-07', '2019-01-08',
                                   '2019-01-09', '2019-01-10', '2019-01-11', '2019-01-12',
                                   '2019-01-13', '2019-01-14', '2019-01-15', '2019-01-16',
                                   '2019-01-17', '2019-01-18', '2019-01-19'],
                                  dtype='datetime64[ns]')
        benchmark_date_high_res = np.array(['2019-01-05', '2019-01-06', '2019-01-07', '2019-01-08',
                                            '2019-01-09', '2019-01-10', '2019-01-11', '2019-01-12',
                                            '2019-01-13', '2019-01-14'], dtype='datetime64[ns]')
        benchmark_rivids = np.array([192474, 192473, 192470, 192469, 192454, 192452, 192448, 192446, 192451, 192450],
                                    dtype=np.int32)

        print("Testing")
        self.assertTrue(np.all(np.isclose(flow_array, benchmark_flow_array)))
        self.assertTrue(np.all(np.isclose(flow_array_high_res, benchmark_array_high_res)))
        self.assertTrue(np.all(np.isclose(initialization_values, benchmark_initialization_values)))
        self.assertTrue(np.all(date == benchmark_date))
        self.assertTrue(np.all(date_high_res == benchmark_date_high_res))
        self.assertTrue(np.all(rivids == benchmark_rivids))

        ds.close()

    def tearDown(self):
        os.remove("Test_files/20190104.nc")

class TestValidateForecasts(unittest.TestCase):

    def setUp(self):
        pass

    def test_compress_netcdf(self):
        work_dir = "/Test_files/Forecast_Validation_Files"
        starting_date = "2018-08-19"
        ending_date = "2018-12-16"

        dates_range = pd.date_range(starting_date, ending_date)
        date_strings = dates_range.strftime("%Y%m%d").tolist()

        files = [os.path.join(work_dir, i + ".nc") for i in date_strings]

        data_sets = [xr.open_dataset(file) for file in files]

        for rivid in range(10):
            for forecast_day in range(15):
                dates = []
                init_dates = []
                data = []
                init_data = []

                for ds in data_sets:
                    dates.append(ds["date"].data[forecast_day])
                    data.append(ds["Qout"].data[rivid, forecast_day, :])

                    init_data.append(ds["initialization_values"].data[rivid])
                    init_dates.append(ds["start_date"].data)

                pd_dates = pd.to_datetime(dates)
                np_data = np.array(data)

                df = pd.DataFrame(np_data, index=pd_dates,
                                  columns=["Ensemble {}".format(str(i).zfill(2)) for i in range(51)])

                init_df = pd.DataFrame(init_data, index=pd.to_datetime(init_dates), columns=["Water Balance"])

                benchmark_df = init_df.copy()
                benchmark_df.index = init_df.index + pd.DateOffset(1)
                benchmark_df.columns = ["Benchmark"]

                big_df = pd.DataFrame.join(init_df, [benchmark_df, df]).dropna()

                obs = big_df.iloc[:, 0].values.astype(np.float64)
                benchmark = big_df.iloc[:, 1].values
                forecasts = big_df.iloc[:, 2:].values.astype(np.float64)

                crps = em.ens_crps(obs, forecasts)["crpsMean"]
                crps_bench = np.mean(np.abs(obs - benchmark))  # MAE
                crpss = em.skill_score(crps, crps_bench, perf_score=0)
                print("River: {}, Forecast Day: {}".format(rivid, forecast_day))
                print("crps: {}, crps bench: {}, crpss: {}".format(crps, crps_bench, crpss["skillScore"]))

        for ds in data_sets:
            ds.close()

    def tearDown(self):
        pass

if __name__ == '__main__':
    unittest.main(verbosity=2)


text:
import utilities
import export_jmol
import argparse

import shutil
import os
import warnings

from PIL import Image
from matplotlib.animation import FuncAnimation
from mpl_toolkits.mplot3d import Axes3D  # noqa
from scipy import stats
from tqdm import tqdm
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
sns.set_style(style='white')
sns.set(
    style="ticks",
    rc={
        "font.family": "Liberation Sans",
        "font.size": 40,
        "axes.linewidth": 14,
        "lines.linewidth": 5,
    },
    font_scale=2.5,
    palette=sns.color_palette("Set2")
)

def dump_data(
    file,
    folder_name,
    planes,
    jmol_planes,
    max_pos_iso,
    max_neg_iso,
    angles,
    x_cross,
    y_cross,
    atom_info,
    xyz_vec,
    fitted_x,
    fitted_y,
    phis,
    max_iso_val,
    min_iso_val,
):

    if not os.path.exists(folder_name):
        os.mkdir(folder_name)
    np.save(folder_name + "/planes.npy", planes)
    np.save(folder_name + "/jmol_plane_data.npy", jmol_planes)
    np.save(folder_name + "/max_pos_iso.npy", max_pos_iso)
    np.save(folder_name + "/max_neg_iso.npy", max_neg_iso)
    np.save(folder_name + "/angles.npy", angles)
    np.save(folder_name + "/x_cross.npy", x_cross)
    np.save(folder_name + "/y_cross.npy", y_cross)
    np.save(folder_name + "/atom_info", atom_info)
    np.save(folder_name + "/xyz_vec.npy", xyz_vec)
    np.save(folder_name + "/fitted_x.npy", fitted_x)
    np.save(folder_name + "/fitted_y.npy", fitted_y)
    np.save(folder_name + "/phis.npy", phis)
    np.save(folder_name + "/min_iso_val.npy", min_iso_val)
    np.save(folder_name + "/max_iso.npy", max_iso_val)

    shutil.copy(file, folder_name + "/" + file.split("/")[-1])

def cart2pol(x, y):
    rho = np.sqrt(x**2 + y**2)
    phi = np.arctan2(y, x)

    return rho, phi

def pol2cart(rho, phi):
    x = rho * np.cos(phi)
    y = rho * np.sin(phi)

    return x, y

def perpendicular_vector(v: list) -> list:
    thres = 0.000005
    v = np.array(v)
    if v[1] != thres:
        v -= thres

    if v[1] == 0:
        if v[0] == 0:
            raise ValueError('zero vector')
        else:
            warnings.warn(
                "Warning: y-component = 0. Might result in wrongly assigned direction leading to wrong angles.", RuntimeWarning
            )
            return [0, -1]

    return [-v[1], v[0]]

def center_atoms(atoms, center_atom):
    x = center_atom[0]
    y = center_atom[1]
    z = center_atom[2]

    # Centering atoms around given atom
    for idx, atom in enumerate(atoms):
        atoms[idx].position[0] = atom.position[0] - x
        atoms[idx].position[1] = atom.position[1] - y
        atoms[idx].position[2] = atom.position[2] - z

    return atoms

def center_data(datapoints, center):
    return datapoints - center

def find_nearest_idx(array, value: float) -> int:
    print((np.abs(array - value)))
    return (np.abs(array - value)).argmin()

def _p(x: np.array, *coeffs: np.array) -> np.array:
    return coeffs*x

def _check_bad_fit(r_val: float, p_val: float, point_count: int) -> None:
    # If the fit of this particular point is
    # very poor, warn the user. Thresholds are arbitrary
    if (np.abs(r_val) < 0.5) and p_val > 0.5:
        warnings.warn(
            f'R^2 = {r_val} and p-value = {p_val} for point #{point_count}. This fit might be very poor',
            RuntimeWarning
        )

def draw_prev_angle(vec, angles, ax):
    # Draw angle
    angle_phi = np.linspace(0, -np.deg2rad(angles[-1]))
    angle_rho = 1.8
    _, line_phi = cart2pol(vec[0], vec[1])
    angle_phi += line_phi
    angle_x, angle_y = pol2cart(angle_rho, angle_phi)

    ax.plot(angle_x, angle_y, color='purple')

def fit_points(x, y, point_count: int):
    x = np.array(x)
    y = np.array(y)

    if np.var(x) > 0.03:
        xp = np.linspace(-1, 1, len(y))

        slope, _, r_value, p_value, _ = stats.linregress(x, y)
        _check_bad_fit(r_value, p_value, point_count)

        x, y = xp, _p(xp, slope)

    else:
        # Rotate points if most has x = 0
        rot = np.deg2rad(45)*len(x)  # 45 degrees in radians
        rho, phi = cart2pol(x, y)
        phi -= rot
        x_cart_rot, y_cart_rot = pol2cart(rho, phi)

        slope, _, r_value, p_value, _ = stats.linregress(
            x_cart_rot,
            y_cart_rot,
        )
        _check_bad_fit(r_value, p_value, point_count)

        xp = np.linspace(-1, 1, len(y_cart_rot))
        y_fit = _p(xp, slope)

        # Rotate fitted points to original position
        fit_rho, fit_phi = cart2pol(xp, y_fit)
        fit_phi += rot

        x, y = pol2cart(fit_rho, fit_phi)

    return x, y

def find_zero_crossings(plane, axis: int, x_cross, y_cross):
    _, indices = np.unique(plane[:, axis], return_index=True)

    for value in plane[indices, axis]:
        line_indices = np.where(plane[:, axis] == value)
        temp_line = plane[line_indices]

        zero_crossings = np.where(np.diff(np.sign(temp_line[:, 3])))[0]

        for item in zero_crossings:
            x_cross.append((temp_line[item, 0] + temp_line[item + 1, 0])/2)
            y_cross.append((temp_line[item, 1] + temp_line[item + 1, 1])/2)

def plot_slices(
    ax,
    high_iso,
    low_iso,
    x_cross,
    y_cross,
    perp_vec,
    vec,
    angles,
    x,
    y,
):
    ax.plot(
        [0, low_iso[0] - high_iso[0]],
        [0, low_iso[1] - high_iso[1]],
        color='orange'
    )

    ax.scatter(high_iso[0], high_iso[1], marker='x', color='green')
    ax.scatter(low_iso[0], low_iso[1], marker='x', color='black')

    ax.plot([0, perp_vec[0]], [0, perp_vec[1]], color='black')

    # plot the midway points at which the
    # isovalues goes from positive to negative
    ax.scatter(x_cross, y_cross, marker='v', color='purple')

    # which direction the fitted line points to
    ax.scatter(vec[0], vec[1], color='black', marker='x')

    draw_prev_angle(vec, angles, ax)

    ax.plot(x, y, color='black', label=f'{angles[-1]}')
    ax.legend()

def export_as_gif(imgs, x, y, fig, ax, folder_name):
    # First export the slices as a gif
    frames = []
    for i in imgs:
        new_frame = Image.open(i)
        frames.append(new_frame)

    frames[0].save(
        "slices.gif",
        format="GIF",
        append_images=frames[1:],
        save_all=True,
        duration=400,
        loop=0,
    )

    # Next, export the change in angles
    ax.axis("off")
    fig.set_size_inches(10, 6)

    def update(i):
        if i != 0:
            ax.plot(x[:i], y[:i], marker='o', color='black')
        return ax

    anim = FuncAnimation(
        fig,
        update,
        frames=np.arange(len(x)),
        interval=400,
    )
    anim.save(folder_name + "/angles.gif", dpi=100, writer="imagemagick")

    # Remove the .png's
    for file in imgs:
        os.remove(file)

    print("Exported slices.gif and angles.gif!")

def generate_xy_shift(bottom_carbon, top_carbon, carbon_chain, planes, atoms):
    zaxis = np.linspace(bottom_carbon[2], top_carbon[2], len(planes))
    x_correction = [pos.position[0] for pos in atoms[carbon_chain]]
    y_correction = [pos.position[1] for pos in atoms[carbon_chain]]

    old_axis = np.linspace(bottom_carbon[2], top_carbon[2], len(carbon_chain))
    interp_x = np.interp(zaxis, old_axis, x_correction)
    interp_y = np.interp(zaxis, old_axis, y_correction)

    return interp_x, interp_y

def main():
    parser = argparse.ArgumentParser(
        description='Analyze helicity of .cube file'
    )

    parser.add_argument(
        'file', metavar=".cube_file", help='Path to .cube file to analyze'
    )

    parser.add_argument(
        'center',
        metavar='center_atom',
        help=(
            'The atom to center data points around. '
            'Can be obtained by opening the file in e.g. Avogadro. '
            'First atom is 1'
        ),
    )

    parser.add_argument(
        'bottom_atom',
        help='"bottom" atom of the linear carbon chain',
    )

    parser.add_argument(
        'top_atom',
        help='"top" atom of the linear carbon chain',
    )

    parser.add_argument(
        '--carbon-chain',
        help=(
            '<Required> indices of carbon atoms ',
            'that defines the helix chain. Example "5,1,2,3,13"',
        ),
        required=True
    )

    parser.add_argument(
        '--open-jmol',
        dest='open_jmol',
        default=False,
        action='store_true',
        help=(
            'Open the generated .spt file in jmol after analysis. '
            'Default is False'
        )
    )

    parser.add_argument(
        '--show-slices',
        default=False,
        dest='show_slices',
        action='store_true',
        help=(
            'Show each slice that has been generated from the .cube file. '
            'Default is False'
        ),
    )

    parser.add_argument(
        '--export-gif',
        default=False,
        dest='export_gif',
        action='store_true',
        help='Export the slices as a .gif. Default is False'
    )
    args = parser.parse_args()

    file = os.getcwd() + '/' + args.file
    center_idx = int(args.center) - 1
    open_jmol = args.open_jmol
    show_slices = args.show_slices
    export_gif = args.export_gif
    carbon_chain = [int(item) for item in str(args.carbon_chain).split(",")]

    atoms, all_info, xyz_vec = utilities.read_cube(file)
    center_atom = atoms[int(center_idx)].position

    # carbons that define the start and end of the linear carbon chain.
    # Used to narrow the amount of slices along the z-axis
    # as we don't care about the orbitals on the substituents etc.
    top_carbon = atoms[int(args.top_atom)].position
    bottom_carbon = atoms[int(args.bottom_atom)].position
    if top_carbon[2] < bottom_carbon[2]:
        raise ValueError(
            "Top atom is lower on the z-axis than the bottom atom"
        )

    center_x = center_atom[0]
    center_y = center_atom[1]
    center_z = center_atom[2]

    all_info[:, :3] = center_data(all_info[:, :3], center_atom)
    atoms = center_atoms(atoms, center_atom)

    atom_info = [
        atoms, top_carbon, bottom_carbon, [center_x, center_y, center_z]
    ]

    planes = []
    plane = []
    # sorting is done so we can slice in planes along the z-axis
    all_info = all_info[all_info[:, 2].argsort()]

    prev_coord = all_info[0]
    for coordinate in tqdm(all_info, desc="Finding planes.."):
        if coordinate[2] == prev_coord[2]:
            # we're in the same plane so add the coordinate
            plane.append(
                [coordinate[0], coordinate[1], coordinate[2], coordinate[3]]
            )
        else:
            plane = np.array(plane)
            # Drop coordinates with isovalues == 0.0
            # plane = plane[np.where(plane[:, 3] != 0.0)]
            #
            # if plane.size != 0:
            planes.append(plane)

            plane = []

        prev_coord = coordinate

    planes = np.array(planes)
    # Throw away first `planes` as it has 1 extra data point
    planes = planes[1:]

    pos_iso, neg_iso, plane_data = [], [], []
    max_iso_val, min_iso_val = [], []
    angles = []
    prev_vec = None
    count = 1
    z_value = []
    imgs = []
    x_cross_collect = []
    y_cross_collect = []
    fitted_x = []
    fitted_y = []
    phis = []

    shift_x, shift_y = generate_xy_shift(
        bottom_carbon, top_carbon, carbon_chain, planes, atoms
    )

    print('Analyzing..')
    for idx, plane in enumerate(planes):
        if top_carbon[2] >= plane[0, 2] >= bottom_carbon[2]:
            if idx < len(planes) - 1:
                z_value.append(plane[0, 2])

                # Impose a shift on the coordinates to center the nodal plane
                plane[:, 0] -= shift_x[idx]
                plane[:, 1] -= shift_y[idx]

                if show_slices or export_gif:
                    _, ax = plt.subplots(figsize=(8, 6), dpi=100)
                    color = ['red' if x < 0 else 'blue' for x in plane[:, 3]]
                    plt.scatter(
                        plane[:, 0],
                        plane[:, 1],
                        s=abs(plane[:, 3])*800,
                        c=color,
                        alpha=0.9,
                    )
                    plt.title(f"Z-value: {plane[0, 2]:.3}")

                # Constrict number of points by a radius-filter
                r_indices = np.where(
                    cart2pol(plane[:, 0], plane[:, 1])[0] < .5
                )
                plane = plane[r_indices]

                # Find highest and lowest isovalue
                maximum = np.amax(plane[:, 3])
                max_iso_val.append(maximum)
                max_index = np.where(plane[:, 3] == maximum)

                minimum = np.amin(plane[:, 3])
                min_iso_val.append(minimum)
                min_index = np.where(plane[:, 3] == minimum)

                max_iso = [plane[max_index, 0], plane[max_index, 1]]
                max_iso = np.array(max_iso).ravel()

                phis.append(cart2pol(max_iso[0], max_iso[1])[1])

                min_iso = [plane[min_index, 0], plane[min_index, 1]]
                min_iso = np.array(min_iso).ravel()

                plane = plane[np.lexsort((plane[:, 0], plane[:, 1]))]

                perp_vec = perpendicular_vector(
                    [max_iso[0] - min_iso[0], max_iso[1] - min_iso[1], 0])
                perp_vec = perp_vec/np.linalg.norm(perp_vec)

                # Sort plane - first by x- then by y-values
                plane = plane[np.lexsort((plane[:, 0], plane[:, 1]))]

                # Find each line of the plot + zero crossings
                x_cross = []
                y_cross = []
                find_zero_crossings(plane, 0, x_cross, y_cross)
                find_zero_crossings(plane, 1, x_cross, y_cross)
                x_cross_collect.append(x_cross)
                y_cross_collect.append(y_cross)

                x, y = fit_points(x_cross, y_cross, count)
                fitted_x.append(x)
                fitted_y.append(y)

                def unit_vector(vector):
                    return vector / np.linalg.norm(vector)

                def angle_between(v1, v2):
                    v1_u = unit_vector(v1)
                    v2_u = unit_vector(v2)
                    rad = np.arctan2(np.linalg.det(
                        [v1_u, v2_u]), np.dot(v1_u, v2_u))

                    return np.rad2deg(rad)

                vec = [x[-1], y[-1]]
                if np.dot(perp_vec, vec) < 0:
                    vec = [x[0], y[0]]
                else:
                    vec = [x[-1], y[-1]]

                if count == 1:
                    angles.append(0)
                else:
                    angle = angle_between(prev_vec, vec)
                    angles.append(angle)

                prev_vec = vec
                count += 1

                pos_iso.append([max_iso[0], max_iso[1], plane[0, 2]])
                neg_iso.append([min_iso[0], min_iso[1], plane[0, 2]])
                plane_data.append([vec[0], vec[1], plane[0, 2]])

                if show_slices or export_gif:
                    plot_slices(
                        ax,
                        max_iso,
                        min_iso,
                        x_cross,
                        y_cross,
                        perp_vec,
                        vec,
                        angles,
                        x,
                        y,
                    )
                    ax.set_xlim([-3.8, 3.8])
                    ax.set_ylim([-4, 4])

                    ax.set_xlabel('X axis')
                    ax.set_ylabel('Y axis')

                    if export_gif and idx % 2 == 0:
                        file_name = str(idx) + "_slice.png"
                        plt.savefig(file_name, bbox_inches="tight")
                        imgs.append(file_name)

    data_dic = {
        'pos_iso': pos_iso,
        'neg_iso': neg_iso,
        'plane_data': plane_data
    }

    # new figure to plot the changes in angle
    fig, ax = plt.subplots()

    # Overlay atoms on angle-plot
    y_value = np.sum(angles)/2
    for idx, atom in enumerate(atoms):
        if atom.symbol == 'C':
            ax.scatter(
                atom.position[2] + bottom_carbon[2],
                y_value,
                color='black',
                marker='o',
                s=10
            )
            ax.annotate(idx, (atom.position[2] + bottom_carbon[2], y_value))

        if atom.symbol == 'Ru':
            ax.scatter(
                atom.position[2] + bottom_carbon[2],
                y_value,
                color='turquoise',
                marker='o',
                s=10
            )

    x = np.array(z_value) + bottom_carbon[2]
    y = np.cumsum(angles)

    ax.set_xlim([x[0] - 1, x[-1] + 1])
    ax.set_ylim([y[0], y[-1]])
    ax.set_xlabel('X axis')
    ax.set_ylabel('Y axis')

    ax.plot(x, y, marker='o', color='blue')
    ax.plot(x, np.gradient(y), marker='o')

    names = file.split("/")
    folder_name = names[-3] + "_" + names[-1][:-5]

    if open_jmol:
        cmd = 'jmol ' + folder_name + '/jmol_export.spt &'
        os.system(cmd)

    # show all figures at the same time
    if show_slices:
        plt.show()

    # Save data
    dump_data(
        file,
        folder_name,
        planes,
        data_dic["plane_data"],
        data_dic["pos_iso"],
        data_dic["neg_iso"],
        angles,
        x_cross_collect,
        y_cross_collect,
        atom_info,
        xyz_vec,
        fitted_x,
        fitted_y,
        phis,
        max_iso_val,
        min_iso_val,
    )

    if export_gif:
        export_as_gif(imgs, x, y, fig, ax, folder_name)

    export_jmol.export_jmol(
            data_dic, [center_x, center_y, center_z], file, folder_name
        )

    plt.savefig(folder_name + "/angles.png", bbox_inches="tight")

if __name__ == '__main__':
    exit(main())

reason: import numpy,from scipy


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats
import time
import datetime
import sys
import os

protein_dilution_factor = 75
filename = input("Enter file name (excluding '.xlsx'): ")
filename = filename + '.xlsx'

def load_file(filename):
    """
    Loads excel file output from BSA absorbance assay.
    Returns dataframe containing absorbance values.
    """

    df = pd.read_excel(filename, skiprows=28, nrows=8, sheet_name='Sheet2', index_col=None, header=None)#, u#secols='A:H')
    df.dropna(inplace=True, axis=1)
    lane_names = [f'Lane {str(i)}' for i in range(1, len(df.columns) - 2)]
    col_names = ['well', 'standard_1'] + lane_names + ['standard_2']
    df.columns = col_names
    df = df.set_index('well')
    df['standard_absorbance'] = (df.standard_1 + df.standard_2) / 2
    df = df.drop(['standard_1', 'standard_2'], axis=1)
    return df

def get_concentrations(df):
    # Prompt for concentration values
    print("The default concentration values are [0, 20, 40, 60, 80, 100, 150, 200].")
    print("To use a new set of values, enter space-separated concentration values. Otherwise, press enter.")
    conc = np.array([0, 20, 40, 60, 80, 100, 150, 200])
    inp = input()
    while True:
        if inp == "":
            #df['conc'] = [0, 20, 40, 60, 80, 100, 150, 200]
            break
        conc = np.array([int(i) for i in inp.split()])
        if len(conc) == 8:
            break
        print("Your input values are:\n")
        print(conc)
        print("Error: there are {} expected values, please try again. Press enter again to use default values".format(len(df.standard_absorbance)))
        inp = input()
    df = df.reset_index().drop('well', axis=1)
    s_abs = df.pop('standard_absorbance')
    samples = np.concatenate([df[column].values for column in df.columns])
    return samples, s_abs, conc

def calc_protein(samples, factor, m, c):
    #os.system('cls' if os.name == 'nt' else 'clear')
    sample_concentrations = (samples - c) / m * factor
    df = pd.DataFrame()
    df['Sample_index'] = np.arange(1, len(sample_concentrations) + 1)
    df['sample_conc'] = sample_concentrations
    df["10µg (µl)"] = np.round(10 / (sample_concentrations / 1000), 1)# volume in µl containing 10 µg of protein
    df["15µg (µl)"] = np.round(15 / (sample_concentrations / 1000), 1)
    df["20µg (µl)"] = np.round(20 / (sample_concentrations / 1000), 1)
    return df

def plot_standard(x, y, m, c, output_dir):
    plt.figure()
    plt.plot(x, y, 'bo')
    plt.plot(x, m * x + c, 'r-')
    plt.xlabel("Concentration")
    plt.ylabel("Absorbance")
    plt.title("Best fit line with intercept")
    print("Line equation: y = {}x + {}".format(m, c))
    plt.savefig(os.path.join(output_dir, 'plot.png'))
    print(f"Plot saved in {output_dir}")

if __name__ == "__main__":
    df = load_file(filename)
    samples, sample_absorbance, concentration = get_concentrations(df)
    x = concentration
    y = sample_absorbance
    m, c, _, _, _ = stats.linregress(x,y)
    df = calc_protein(samples, protein_dilution_factor, m, c)
    save_name = datetime.datetime.strftime(datetime.datetime.now(), "%d%b%y_BSA.csv")
    output_dir = os.path.join(os.getcwd(), datetime.datetime.strftime(datetime.datetime.now(), "%d%b%y_BSA"))
    os.makedirs(output_dir, exist_ok=True)
    df.to_csv(os.path.join(output_dir, save_name), index=False)
    print('Results saved as in {}.'.format(output_dir))
    plot_standard(x, y, m, c, output_dir)


text:
import numpy as np
from chordkit.hearing_models import cbw_volk as cbw, bark_zwicker as bar
from chordkit.pair_constants import SETHARES_CONSTANTS as sc, AUDITORY_CONSTANTS as ac, pair_volume, pair_distance
from chordkit.chord_utils import MergedSpectrum

# Returns overlap contribution of two partials, based on an indicator
# function on the overlap zone, scaled to the amplitude of the partial.
# This is called "cbw" because it pairs with the pair-roughness function that
# uses an indicator function on the critical bandwidth, but the CBW is not used.
def cbw_overlap_pair(x_hz, ref_hz, v_x, v_ref, options={
    'amp_type': 'MIN'
}):
    distance = pair_distance(x_hz, ref_hz)

    if distance < ac['slow_beat_limit']:
        return pair_volume(v_x, v_ref, options['amp_type'])
    else:
        return 0

# Uses a bump function from a cosine wave to measure overlap
def cos_overlap_pair(x_hz, ref_hz, v_x, v_ref, options={
    'amp_type': 'MIN'
}):
    distance = pair_distance(x_hz, ref_hz)
    flat = cbw_overlap_pair(x_hz, ref_hz, v_x, v_ref, options)

    return flat * 0.5 * (1 + np.cos(np.pi * distance/ac['slow_beat_limit']))

# An imitation of the Sethares (1993) roughness function, but for overlap,
# using a bell-like function
def bell_overlap_pair(x_hz, ref_hz, v_x, v_ref, options={
    'amp_type': 'MIN',
    'K': -2.374,
    'cutoff': False
}):
    s = sc['s_star'] / (sc['s1'] * min([x_hz, ref_hz]) + sc['s2'])
    v12 = pair_volume(v_x, v_ref, options['amp_type'])

    # The following scaling factor is introduced to ensure that the maximum
    # value obtained by the pairwise roughness function is approximately 1.
    # 'K' can be overriden in the options dictionary
    K = -2.374
    if 'K' in options:
        K = options['K']

    distance = pair_distance(x_hz, ref_hz)

    # Cutoff: Following Hutchinson and Knopoff 1978, cuts off the overlap
    # function at 1.2 CBW, which prevents too-remote partials from
    # contributing to the final score.
    if options['cutoff'] == True:
        cbw_limit = 1.2 * cbw(max[x_hz, ref_hz]) / 2
        if distance < ac['slow_beat_limit'] or distance >= cbw_limit:
            v12 = 0

    try:
        return v12 * np.exp(K * sc['b'] * s * distance)
    except OverflowError:
        print(f'Overflow in computing overlap: b == {sc["b"]}, s == {s}, distance == {distance}')

###################
# SUMMATION MODEL #
###################

# The following function is based on Sethares' 1993 model, adapted for summing
# pairwise overlap functions.
def overlap_complex(
    spectrum: MergedSpectrum,
    function_type: str = 'BELL',
    overlap_limit: float = 0.1,
    *,
    options={
        'amp_type': 'MIN',
        'cutoff': False,
        'original': False,
        'show_partials': False
    }
):
    n = len(spectrum.partials['hz'])
    overlap_partials = []
    overlap_vals = np.zeros((n, n))

    if function_type.upper() == 'BELL':
        pair_assess = bell_overlap_pair
    elif function_type.upper() == 'CBW':
        pair_assess = cbw_overlap_pair
    elif function_type.upper() == 'COS':
        pair_assess = cos_overlap_pair
    else:
        raise ValueError(f'Invalid assessment function type: {function_type.upper()}')

    # Assess all pairs for overlap
    for i in range(n - 1):
        for j in range(i + 1, n):
            overlap_vals[i][j] = pair_assess(
                spectrum.partials['hz'][i],
                spectrum.partials['hz'][j],
                spectrum.partials['amp'][i],
                spectrum.partials['amp'][j],
                options=options
            )
        if options['show_partials'] == True and overlap_vals[i][j] > overlap_limit:
            overlap_partials.append((i, j))

    if options['show_partials']:
        return {
            'overlap': np.sum(overlap_vals),
            'overlap_partials': overlap_partials
        }

    return np.sum(overlap_vals)

max_stars_repo_name: bradley-gersh/chordkit

max_issues_repo_name: bradley-gersh/chordkit

max_forks_repo_name: bradley-gersh/chordkit


text:
#!/usr/bin/env python
# coding=utf-8
import numpy as np
import tensorflow as tf
import tqdm
import time
import os
from Network import Model
from Train import Train
from Utils import add_name_scope
from InputPipeline import InputPipeline

config = tf.ConfigProto(allow_soft_placement=True)
print_fn = tf.logging.info  # print
tf.logging.set_verbosity(tf.logging.INFO)

class Main(Train, Model, InputPipeline):
    def __init__(self, *args, **kwargs):
        super(Main, self).__init__(*args, **kwargs)
        print_fn(Main.__mro__)
        #----------------------------------------
        # default setting
        self.opt = tf.train.AdamOptimizer
        self.Is_summary = True
        self.update_log_step = 100
        self.save_per_steps = 1000
        self.print_log = True
        self.valid_size = 1024
        self.config = config
        #----------------------------------------

    def loaddata(self, repeat=1, shuffle=True):
        self.filenames = tf.placeholder(tf.string, shape=[None], name='filenames')
        features, labels = self.input_fn(self.filenames,
                                         repeat=repeat,
                                         shuffle=shuffle)
        return features, labels

    def inference(self, *args, **kwargs):
        self.model()
        net = self.Network(self, *args, **kwargs)
        return net

    def eval(self, data):
        BS = self.FLAGS.batch_size
        flux = data['flux']
        label = data['label'].astype(np.int32)
        print flux.dtype, flux.shape
        print label.dtype, label.shape
        if label.shape[0]%BS == 0:
            loop_num = label.shape[0]//BS
        else:
            loop_num = label.shape[0]//BS + 1
        print_fn("loop number: %d"%loop_num)
        #------------------------------------------------------------
        x = tf.placeholder(tf.float32, shape=[None, 4000])
        y_ = tf.placeholder(tf.int32, shape=[None])
        net = self.inference(x, is_training=False)
        loss = self.Loss_fn(self, net=net, y_=y_)
        prob = tf.get_collection('prob')[0]
        LOSS = []
        PROB = []
        with tf.Session(config=self.config) as sess:
            self.Saver = tf.train.Saver(max_to_keep=1)
            self.init_model(sess)
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess=sess, coord=coord)
            for i in tqdm.tqdm(xrange(loop_num)):
                el, ep, = sess.run([loss, prob],
                                   feed_dict={x:flux[i*BS:(i+1)*BS],
                                              y_:label[i*BS:(i+1)*BS]})
                LOSS.append(el)
                PROB.append(ep)
            coord.request_stop()
            coord.join(threads)
        LOSS = np.array(LOSS)
        PROB = np.vstack(PROB)
        return LOSS, PROB

    def train(self, loss, train_op):
        """it is an example, should rewriten in main function"""
        summary_train_op = tf.summary.merge_all('train')
        summary_valid_op = tf.summary.merge_all('validation')
        filenames = self.filenames
    #   iterator_handle = tf.get_collection('iterator')[0].initializer
        iterator_handle = self.iterator.initializer
        with tf.Session(config=self.config) as sess:
            self.Saver = tf.train.Saver(max_to_keep=20)
            local_step = 0
            global_step = tf.get_collection('global_step')[0]
            self.init_model(sess)
            writer_train = tf.summary.FileWriter(
                os.path.join(self.FLAGS.log_dir, "train"), sess.graph)
            writer_valid = tf.summary.FileWriter(
                os.path.join(self.FLAGS.log_dir, "valid"), sess.graph)
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess=sess, coord=coord)
            #------------------------------------------------------------
            sess.run(iterator_handle,
                     feed_dict={filenames: self.train_filenames})
            pbar = tqdm.tqdm(total=self.FLAGS.train_steps)
            is_first_step = 0
            epoch = 1
            while not coord.should_stop():
                try:
                    _, step = sess.run([train_op, global_step],
                                       feed_dict={self.Is_training: True})
                except tf.errors.OutOfRangeError:
                    sess.run(iterator_handle,
                             feed_dict={filenames: self.valid_filenames})
                    valid_losses = []
                    loop_num = self.valid_size//self.FLAGS.batch_size
                    for i in xrange(loop_num):
                        summary_valid, l_valid = sess.run(
                            [summary_valid_op, tf.get_collection('MES_l')[0]],
                            feed_dict={self.Is_training: False})
                        if i == 0:
                            writer_valid.add_summary(summary_valid, step)
                        valid_losses.append(l_valid)
                    valid_loss = np.mean(valid_losses)
                    print "epoch %d: %f"%(epoch, valid_loss)
                    sess.run(iterator_handle,
                             feed_dict={filenames: self.train_filenames})
                    epoch += 1
                #   self.Saver.save(sess, os.path.join(
                #       self.FLAGS.model_dir,
                #       self.FLAGS.model_basename+'_epoch'),
                #       global_step=epoch)
                    continue
                if local_step % (self.FLAGS.train_steps //
                                 self.update_log_step) == 0:
                    if is_first_step == 0:
                        pbar.update(0)
                        is_first_step += 1
                    else:
                        pbar.update(self.FLAGS.train_steps//self.update_log_step)

                if local_step % self.update_log_step == 0:
                    summary_train, loss_train = sess.run(
                        [summary_train_op, loss], feed_dict={self.Is_training: False})
                    if self.print_log:
                        print " training step %d done (global step: %d):" \
                              % (local_step, step), loss_train
                    writer_train.add_summary(summary_train, step)
            #------------------------------------------------------------
                local_step += 1
                if local_step == 1:
                    continue
                if (step + 1) % self.save_per_steps == 0:
                    self.save_model(sess=sess)
                if local_step > self.FLAGS.train_steps:
                    coord.request_stop()
            coord.request_stop()
            coord.join(threads)
            pbar.close()
            writer_train.close()
            writer_valid.close()

    def __call__(self):
        """an example for constructing network model"""
        self.train_filenames = ["/data/dell5/userdir/maotx/DSC/data/training.tfrecords"]
        self.valid_filenames = ["/data/dell5/userdir/maotx/DSC/data/valid.tfrecords"]
        self.Is_training = tf.placeholder(tf.bool,shape=[],name='is_train')
        x, y_ = self.loaddata()
        net = self.inference(x, is_training=self.Is_training)
        loss = self.Loss_fn(self, net=net, y_=y_)
        train_op = self.optimizer(loss, gclip=None)
        self.train(loss, train_op)

max_stars_repo_name: POFK/tftool

max_issues_repo_name: POFK/tftool

max_forks_repo_name: POFK/tftool


text:
import pandas as pd
import numpy as np
import nltk
import os
import nltk.corpus
from nltk.tokenize import word_tokenize
from nltk.tokenize import sent_tokenize
from nltk.probability import FreqDist
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.stem.wordnet import WordNetLemmatizer

# sample text for performing tokenization

def print_text_tokenize(text):
    # Passing the string text into word tokenize for breaking the sentences
    token = word_tokenize(text)
    print(token)
    return token

def print_text_sentence_tokenization(text):
    sentence_tokenized_text = sent_tokenize(text)
    print(sentence_tokenized_text)
    fdist = FreqDist(sentence_tokenized_text)
    print(fdist)
    print(fdist.most_common(2))

    # fdist.plot(30, cumulative=False)
    # plt.show()

if __name__ == "__main__":
    text = "In Brazil they drive on the right-hand side of the road. Brazil has a large coastline on the eastern \
    side of South America"
    text = "Download individual packages from http://nltk.org/nltk_data/ (see the “download” links). Unzip them \
    to the appropriate subfolder. For example"
    token = print_text_tokenize(text)
    tokenized_sent = set(stopwords.words("english"))
    print(tokenized_sent)

    filtered_sent = []
    for w in tokenized_sent:
        if w not in tokenized_sent:
            filtered_sent.append(w)
    print("Tokenized Sentence:", tokenized_sent)
    print("Filterd Sentence:", filtered_sent)

    ps = PorterStemmer()
    stemmed_words = []
    for w in filtered_sent:
        stemmed_words.append(ps.stem(w))

    print("Filtered Sentence:", filtered_sent)
    print("Stemmed Sentence:", stemmed_words)

    lem = WordNetLemmatizer()
    stem = PorterStemmer()
    word = "flying"
    print("Lemmatized Word:", lem.lemmatize(word, "v"))
    print("Stemmed Word:", stem.stem(word))

    text = """Hello Mr. Smith, how are you doing today? The weather is great, and city is awesome.
    The sky is pinkish-blue. You shouldn't eat cardboard"""
    print_text_sentence_tokenization(text)

    sent = "Albert Einstein was born in Ulm, Germany in 1879."
    tokens = nltk.word_tokenize(sent)
    print(tokens)
    print(nltk.pos_tag(tokens))

max_stars_repo_name: qsunny/python

max_issues_repo_name: qsunny/python

max_forks_repo_name: qsunny/python


text:
# Copyright (c) 2018, NVIDIA CORPORATION.

"""
Test related to Index
"""
import numpy as np
import pandas as pd
import pytest

import cudf
from cudf.core import DataFrame
from cudf.core.index import (
    CategoricalIndex,
    DatetimeIndex,
    GenericIndex,
    RangeIndex,
    as_index,
)
from cudf.tests.utils import assert_eq

def test_df_set_index_from_series():
    df = DataFrame()
    df["a"] = list(range(10))
    df["b"] = list(range(0, 20, 2))

    # Check set_index(Series)
    df2 = df.set_index(df["b"])
    assert list(df2.columns) == ["a", "b"]
    sliced_strided = df2.loc[2:6]
    print(sliced_strided)
    assert len(sliced_strided) == 3
    assert list(sliced_strided.index.values) == [2, 4, 6]

def test_df_set_index_from_name():
    df = DataFrame()
    df["a"] = list(range(10))
    df["b"] = list(range(0, 20, 2))

    # Check set_index(column_name)
    df2 = df.set_index("b")
    print(df2)
    # 1 less column because 'b' is used as index
    assert list(df2.columns) == ["a"]
    sliced_strided = df2.loc[2:6]
    print(sliced_strided)
    assert len(sliced_strided) == 3
    assert list(sliced_strided.index.values) == [2, 4, 6]

def test_df_slice_empty_index():
    df = DataFrame()
    assert isinstance(df.index, RangeIndex)
    assert isinstance(df.index[:1], RangeIndex)
    with pytest.raises(IndexError):
        df.index[1]

def test_index_find_label_range():
    # Monotonic Index
    idx = GenericIndex(np.asarray([4, 5, 6, 10]))
    assert idx.find_label_range(4, 6) == (0, 3)
    assert idx.find_label_range(5, 10) == (1, 4)
    assert idx.find_label_range(0, 6) == (0, 3)
    assert idx.find_label_range(4, 11) == (0, 4)

    # Non-monotonic Index
    idx_nm = GenericIndex(np.asarray([5, 4, 6, 10]))
    assert idx_nm.find_label_range(4, 6) == (1, 3)
    assert idx_nm.find_label_range(5, 10) == (0, 4)
    # Last value not found
    with pytest.raises(ValueError) as raises:
        idx_nm.find_label_range(0, 6)
    raises.match("value not found")
    # Last value not found
    with pytest.raises(ValueError) as raises:
        idx_nm.find_label_range(4, 11)
    raises.match("value not found")

def test_index_comparision():
    start, stop = 10, 34
    rg = RangeIndex(start, stop)
    gi = GenericIndex(np.arange(start, stop))
    assert rg.equals(gi)
    assert gi.equals(rg)
    assert not rg[:-1].equals(gi)
    assert rg[:-1].equals(gi[:-1])

@pytest.mark.parametrize(
    "func", [lambda x: x.min(), lambda x: x.max(), lambda x: x.sum()]
)
def test_reductions(func):
    x = np.asarray([4, 5, 6, 10])
    idx = GenericIndex(np.asarray([4, 5, 6, 10]))

    assert func(x) == func(idx)

def test_name():
    idx = GenericIndex(np.asarray([4, 5, 6, 10]), name="foo")
    assert idx.name == "foo"

def test_index_immutable():
    start, stop = 10, 34
    rg = RangeIndex(start, stop)
    with pytest.raises(TypeError):
        rg[1] = 5
    gi = GenericIndex(np.arange(start, stop))
    with pytest.raises(TypeError):
        gi[1] = 5

def test_categorical_index():
    pdf = pd.DataFrame()
    pdf["a"] = [1, 2, 3]
    pdf["index"] = pd.Categorical(["a", "b", "c"])
    initial_df = DataFrame.from_pandas(pdf)
    pdf = pdf.set_index("index")
    gdf1 = DataFrame.from_pandas(pdf)
    gdf2 = DataFrame()
    gdf2["a"] = [1, 2, 3]
    gdf2["index"] = pd.Categorical(["a", "b", "c"])
    assert_eq(initial_df.index, gdf2.index)
    gdf2 = gdf2.set_index("index")

    assert isinstance(gdf1.index, CategoricalIndex)
    assert_eq(pdf, gdf1)
    assert_eq(pdf.index, gdf1.index)
    assert_eq(pdf.index.codes, gdf1.index.codes.to_array())

    assert isinstance(gdf2.index, CategoricalIndex)
    assert_eq(pdf, gdf2)
    assert_eq(pdf.index, gdf2.index)
    assert_eq(pdf.index.codes, gdf2.index.codes.to_array())

def test_pandas_as_index():
    # Define Pandas Indexes
    pdf_int_index = pd.Int64Index([1, 2, 3, 4, 5])
    pdf_float_index = pd.Float64Index([1.0, 2.0, 3.0, 4.0, 5.0])
    pdf_datetime_index = pd.DatetimeIndex(
        [1000000, 2000000, 3000000, 4000000, 5000000]
    )
    pdf_category_index = pd.CategoricalIndex(["a", "b", "c", "b", "a"])

    # Define cudf Indexes
    gdf_int_index = as_index(pdf_int_index)
    gdf_float_index = as_index(pdf_float_index)
    gdf_datetime_index = as_index(pdf_datetime_index)
    gdf_category_index = as_index(pdf_category_index)

    # Check instance types
    assert isinstance(gdf_int_index, GenericIndex)
    assert isinstance(gdf_float_index, GenericIndex)
    assert isinstance(gdf_datetime_index, DatetimeIndex)
    assert isinstance(gdf_category_index, CategoricalIndex)

    # Check equality
    assert_eq(pdf_int_index, gdf_int_index)
    assert_eq(pdf_float_index, gdf_float_index)
    assert_eq(pdf_datetime_index, gdf_datetime_index)
    assert_eq(pdf_category_index, gdf_category_index)

    assert_eq(pdf_category_index.codes, gdf_category_index.codes.to_array())

def test_index_rename():
    pds = pd.Index([1, 2, 3], name="asdf")
    gds = as_index(pds)

    expect = pds.rename("new_name")
    got = gds.rename("new_name")

    assert_eq(expect, got)
    """
    From here on testing recursive creation
    and if name is being handles in recursive creation.
    """
    pds = pd.Index(expect)
    gds = as_index(got)

    assert_eq(pds, gds)

    pds = pd.Index(pds, name="abc")
    gds = as_index(gds, name="abc")
    assert_eq(pds, gds)

def test_set_index_as_property():
    cdf = DataFrame()
    col1 = np.arange(10)
    col2 = np.arange(0, 20, 2)
    cdf["a"] = col1
    cdf["b"] = col2

    # Check set_index(Series)
    cdf.index = cdf["b"]

    np.testing.assert_array_equal(cdf.index.values, col2)

    with pytest.raises(ValueError):
        cdf.index = [list(range(10))]

    idx = np.arange(0, 1000, 100)
    cdf.index = idx
    np.testing.assert_array_equal(cdf.index.values, idx)

    df = cdf.to_pandas()
    np.testing.assert_array_equal(df.index.values, idx)

    head = cdf.head().to_pandas()
    np.testing.assert_array_equal(head.index.values, idx[:5])

@pytest.mark.parametrize(
    "idx",
    [
        cudf.core.index.RangeIndex(1, 5),
        cudf.core.index.DatetimeIndex(["2001", "2003", "2003"]),
        cudf.core.index.StringIndex(["a", "b", "c"]),
        cudf.core.index.GenericIndex([1, 2, 3]),
        cudf.core.index.CategoricalIndex(["a", "b", "c"]),
    ],
)
@pytest.mark.parametrize("deep", [True, False])
def test_index_copy(idx, deep):
    idx_copy = idx.copy(deep=deep)
    assert_eq(idx, idx_copy)
    assert type(idx) == type(idx_copy)

@pytest.mark.parametrize("idx", [[1, None, 3, None, 5]])
def test_index_isna(idx):
    pidx = pd.Index(idx, name="idx")
    gidx = cudf.core.index.GenericIndex(idx, name="idx")
    assert_eq(gidx.isna().to_array(), pidx.isna())

@pytest.mark.parametrize("idx", [[1, None, 3, None, 5]])
def test_index_notna(idx):
    pidx = pd.Index(idx, name="idx")
    gidx = cudf.core.index.GenericIndex(idx, name="idx")
    assert_eq(gidx.notna().to_array(), pidx.notna())

max_stars_repo_name: mythrocks/cudf

max_issues_repo_name: mythrocks/cudf

max_forks_repo_name: mythrocks/cudf


text:
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.init as init
from collections import Iterable
from torch.autograd import Variable
import torch
import numpy as np
from torchdiffeq import odeint
from models.bnn import BNN
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class ODE(nn.Module):
    def __init__(self, num_channels=3, img_size=(64,64), method='naive language', batch_size=5):
        super(ODE, self).__init__()
        self.encoder = Encoder(num_channels=num_channels)
        self.rnn = RNN_FeatEncoder(batch_size=batch_size)
        self.decoder = Decoder(num_channels=num_channels)
        self.bnn = BNN(256+16, 256, act='tanh', n_hidden=128, bnn=False)
        self.bnn.draw_f()
        self.language_encoder = Language_Encoder()
        self.img_size = img_size
        self.method = method
        self.lan_mu_proj = nn.Linear(128, 128)
        self.lan_var_proj = nn.Linear(128, 128)
        self.vid_feat_proj = nn.Linear(256, 128)
        self.batch_size = batch_size

    def forward(self, input, pos_language=None, neg_language=None):
      """
        input is video trajectory in training, or single image in testing.
        language is language feature from VisualCommet
      """
      if pos_language is None:
        batchsize = input.shape[0]
        latent_total= self.encoder(input).view([batchsize, -1])
        state = torch.randn(batchsize, 128).to(device)
        init_state = latent_total
      else:
        batchsize = input.shape[0]
        if input.shape[1]==1:
          # Only given the first image
          if self.method == "naive language":
            latent_total= self.encoder(input.squeeze(1)).view([batchsize, 1, -1])
            language_feature = self.language_encoder(pos_language)
            state = torch.randn(batchsize, 128).to(device) * language_feature + language_feature
            init_state = latent_total[:,0]
          elif self.method == 'nce':
            latent_total= self.encoder(input.squeeze(1)).view([batchsize, 1, -1])
            language_feature = self.language_encoder(pos_language)
            mu = self.lan_mu_proj(language_feature)
            logvar = self.lan_var_proj(language_feature)
            state = self.decoder.reparametrize(mu, logvar)
            init_state = latent_total[:,0]

        else:
          contrastive_loss = 0
          # Training time, given a sequence
          latent_total= self.encoder(input.reshape([batchsize*10, 3, 64, 64])).view([batchsize, 10, -1])
          h = self.rnn.init_h.to(device)
          c = self.rnn.init_c.to(device)
          for i in reversed(range(10)):
              latent, h, c = self.rnn(latent_total[:, i:i+1, :].permute(1, 0, 2), h, c)
          state_dist = self.rnn.linearlay(latent.view([batchsize, 256]))

          if self.method == 'naive language':
            mu = state_dist[:, :128]
            logvar = state_dist[:, 128:]
            language_feature = self.language_encoder(pos_language)
            video_latent = self.decoder.reparametrize(mu, logvar)
            state = video_latent * language_feature + language_feature
            init_state = latent_total[:, 0]
          elif self.method == 'nce':
            video_feature = self.vid_feat_proj(state_dist)
            pos_language_feature = self.language_encoder(pos_language)
            neg_language_feature = self.language_encoder(neg_language)
            pos_sim = torch.sum(video_feature * pos_language_feature, dim=1)
            neg_sim = torch.sum(video_feature * neg_language_feature, dim=1)
            contrastive_loss = torch.mean(torch.exp(neg_sim) / (torch.exp(pos_sim) + 1e-10))
            mu = self.lan_mu_proj(pos_language_feature)
            logvar = self.lan_var_proj(neg_language_feature)
            state = self.decoder.reparametrize(mu, logvar)
            init_state = latent_total[:, 0]

      concat_state = torch.cat([state, init_state], 1)
      ts1 = torch.tensor(np.linspace(1, 10, 10)).to(device)
      output_latent = odeint(self.bnn, concat_state, ts1)
      recon = self.decoder(output_latent.reshape([batchsize*10, 256])).reshape([batchsize, 10, 3, 64, 64])
      if input.shape[1]==1:
        return recon
      else:
        return recon, mu, logvar, contrastive_loss

class Language_Encoder(nn.Module):
    def __init__(self):
        super(Language_Encoder, self).__init__()
        self.fc1 = nn.Linear(512, 256)
        self.fc2 = nn.Linear(256, 128)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, input):
      out = self.relu(self.fc1(input))
      out = self.fc2(out)
      return out

class Encoder(nn.Module):
    def __init__(self, num_channels=3):
        super(Encoder, self).__init__()
        self.cnn1 = nn.Conv2d(num_channels, 4, kernel_size=4, stride=2, padding=1)
        self.cnn2 = nn.Conv2d(4, 8, kernel_size=4, stride=2, padding=1)
        self.cnn3 = nn.Conv2d(8, 16, kernel_size=4, stride=2, padding=1)
        self.cnn4 = nn.Conv2d(16, 32, kernel_size=4, stride=2, padding=1)

        #self.fc1 = nn.Linear(15*20*32, 256)
        self.fc1 = nn.Linear(4*4*32, 256)
        self.fc2 = nn.Linear(256, 128)
        self.bn1 = nn.BatchNorm2d(4)
        self.bn2 = nn.BatchNorm2d(8)
        self.bn3 = nn.BatchNorm2d(16)
        self.bn4 = nn.BatchNorm2d(32)

        #self.relu = nn.ReLU(inplace=True)
        self.relu1 = nn.ReLU()
        self.relu2 = nn.ReLU()
        self.relu3 = nn.ReLU()
        self.relu4 = nn.ReLU()
        self.relu5 = nn.ReLU()
        # self.weight_init()

    def forward(self, input):
        out = self.relu1(self.bn1(self.cnn1(input)))
        out = self.relu2(self.bn2(self.cnn2(out)))
        out = self.relu3(self.bn3(self.cnn3(out)))
        out = self.relu4(self.bn4(self.cnn4(out)))
        out = self.relu5(self.fc1(out.reshape([-1, 4*4*32])))
        return self.fc2(out)

    def weight_init(self):
        for block in self._modules:
            if isinstance(self._modules[block], Iterable):
                for m in self._modules[block]:
                    if self.args.init_method == 'kaiming':
                        m.apply(kaiming_init)
                    elif self.args.init_method == 'xavier':
                        m.apply(xavier_uniform_init)
                    else:
                        m.apply(normal_init)
            else:
                if self.args.init_method == 'kaiming':
                    self._modules[block].apply(kaiming_init)
                elif self.args.init_method == 'xavier':
                    self._modules[block].apply(xavier_uniform_init)
                else:
                    self._modules[block].apply(normal_init)

class RNN_FeatEncoder(nn.Module):
    def __init__(self, batch_size=5):
        super(RNN_FeatEncoder, self).__init__()
        latent_size = 256
        self.lstm = nn.LSTM(latent_size//2, latent_size, 1)
        self.batch = batch_size
        self.init_h = torch.zeros([1, self.batch, latent_size], device=device)
        self.init_c = torch.zeros([1, self.batch, latent_size], device=device)
        self.fc3 = nn.Linear(latent_size, latent_size)
        self.fc4 = nn.Linear(latent_size, latent_size)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, input, h, c):
        out, (h, c) = self.lstm(input, (h, c))
        return out, h, c

    def linearlay(self, input):
        temp = self.relu(self.fc3(input))
        return self.fc4(temp)

class Decoder(nn.Module):
    def __init__(self, num_channels=3):
        super(Decoder, self).__init__()
        self.cnn2 = nn.ConvTranspose2d(32, 16, kernel_size=4, stride=2, padding=1)
        self.cnn3 = nn.ConvTranspose2d(16, 8, kernel_size=4, stride=2, padding=1)
        self.cnn4 = nn.ConvTranspose2d(8, 4, kernel_size=4, stride=2, padding=1)
        self.cnn5 = nn.ConvTranspose2d(4, num_channels, kernel_size=4, stride=2, padding=1)
        self.fc1 = nn.Linear(256, 256)
        #self.fc2 = nn.Linear(256, 15*20*32)
        self.fc2 = nn.Linear(256, 4*4*32)
        self.bn2 = nn.BatchNorm2d(16)
        self.bn3 = nn.BatchNorm2d(8)
        self.bn4 = nn.BatchNorm2d(4)
        self.relu1 = nn.ReLU()
        self.relu2 = nn.ReLU()
        self.relu3 = nn.ReLU()
        self.relu4 = nn.ReLU()
        self.relu5 = nn.ReLU()
        self.relu6 = nn.ReLU()

    def forward(self, z):
        out = self.relu1(self.fc1(z))
        out = self.relu2(self.fc2(out)).reshape([-1, 32, 4, 4])
        out = self.relu3(self.bn2(self.cnn2(out)))
        out = self.relu4(self.bn3(self.cnn3(out)))
        out = self.relu5(self.bn4(self.cnn4(out)))
        out = self.relu6(self.cnn5(out))
        return out

    def reparametrize(self, mu, logvar):
        std = logvar.div(2).exp_()
        eps = Variable(std.data.new(std.size()).normal_())
        return mu + std * eps

def kaiming_init(m):
    if isinstance(m, (nn.Linear, nn.Conv3d, nn.ConvTranspose3d)):
        init.kaiming_normal_(m.weight)
        if m.bias is not None:
            m.bias.data.fill_(0)
    elif isinstance(m, (nn.BatchNorm1d, nn.BatchNorm3d)):
        m.weight.data.fill_(1)
        if m.bias is not None:
            m.bias.data.fill_(0)

def normal_init(m, mean=0, std=0.1):
    if isinstance(m, (nn.Linear, nn.Conv3d)):
        m.weight.data.normal_(mean, std)
        if m.bias.data is not None:
            m.bias.data.zero_()
    elif isinstance(m, (nn.BatchNorm3d, nn.BatchNorm1d)):
        m.weight.data.fill_(1)
        if m.bias.data is not None:
            m.bias.data.zero_()

def xavier_uniform_init(m):
    if isinstance(m, (nn.Linear, nn.Conv3d, nn.ConvTranspose3d)):
        nn.init.xavier_normal_(m.weight)
        if m.bias.data is not None:
            m.bias.data.fill_(0.01)

    elif isinstance(m, (nn.BatchNorm3d, nn.BatchNorm1d)):
        m.weight.data.fill_(1)
        if m.bias.data is not None:
            m.bias.data.zero_()


text:
import json
from os import sys, path

import numpy as np
import requests

sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))

from lib.dac.dac import DAC

if __name__ == '__main__':
    headers = {"Content-Type": "application/json"}
    bn1 = np.array(np.ones(DAC.Ncarriers) * DAC.bps, dtype=int).tolist()
    bn2 = np.array(np.ones(DAC.Ncarriers), dtype=int).tolist()
    En1 = np.array(np.ones(DAC.Ncarriers)).tolist()
    En2 = np.round(np.array(np.ones(DAC.Ncarriers) / np.sqrt(2)), 3).tolist()
    eq1 = eq2 = "MMSE"
    params = [{'id': 1, 'dac_out': 1, 'osc_in': 2, 'bn': bn1, 'En': En1, 'eq': eq1}]
    print(bn1)
    # configure
    request = requests.post('http://10.1.1.10:5000/api/dac_osc', headers=headers, data=json.dumps(params))
    print(request.json())

    # get
    # request = requests.get('http://10.1.1.10:5000/api/dac_osc', headers=headers)
    # print(request.json())
    # request = requests.get('http://10.1.1.10:5000/api/dac_osc/1', headers=headers)
    # print(request.json())
    # request = requests.get('http://10.1.1.10:5000:5000/api/dac_osc/2', headers=headers)
    # print(request.json())

    # delete
    # request = requests.delete('hhttp://10.1.1.10:5000:5000/api/dac_osc/1', headers=headers)
    # print(request.json())
    # request = requests.get('http://10.1.1.10:5000:5000/api/dac_osc', headers=headers)
    # print(request.json())

max_stars_repo_name: lrodrin/transceivers

max_issues_repo_name: lrodrin/transceivers

max_forks_repo_name: lrodrin/transceivers


text:
# Author: Wei-Ning Hsu
import h5py
import io
import json
import numpy as np
import os
import torch
import torch.nn.functional
import torchvision.transforms as transforms
from collections import defaultdict
from PIL import Image
from torch.utils.data import Dataset
import pprint

def _process_resize_crop_norm_funcs(image_conf: dict):
    crop_size = image_conf.get('crop_size', 224)
    center_crop = image_conf.get('center_crop', False)
    if center_crop:
        ret_image_resize_and_crop = transforms.Compose(
            [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor()])
    else:
        ret_image_resize_and_crop = transforms.Compose(
            [transforms.RandomResizedCrop(crop_size), transforms.ToTensor()])

    RGB_mean = image_conf.get('RGB_mean', [0.485, 0.456, 0.406])
    RGB_std = image_conf.get('RGB_std', [0.229, 0.224, 0.225])
    ret_image_normalize = transforms.Normalize(mean=RGB_mean, std=RGB_std)
    return ret_image_resize_and_crop, ret_image_normalize

def _check_num_samples(english_audios, hindi_audios, j_audios, images):
    assert english_audios['melspec'].shape[0] == images['image'].shape[0]
    assert hindi_audios['melspec'].shape[0]   == images['image'].shape[0]
    assert j_audios['melspec'].shape[0]       == images['image'].shape[0]

class ImageCaptionDatasetHDF5(Dataset):
    def __init__(self, json_path, audio_conf={}, image_conf={}):
        """
        Dataset that manages a set of paired images and audio recordings

        :param audio_hdf5 (str): path to audio hdf5
        :param image_hdf5 (str): path to audio hdf5
        """
        print(f"DATALOADER: loading metadata from {json_path}")
        with open(json_path, 'r') as f:
            data = json.load(f)
        self.english_audio_hdf5_path = data['english_audio_hdf5_path']
        self.hindi_audio_hdf5_path = data['hindi_audio_hdf5_path']
        self.j_audio_hdf5_path = data['japanese_audio_hdf5_path']
        self.image_hdf5_path = data['image_hdf5_path']

        print("DATALOADER: Using the following file paths:")
        pp = pprint.PrettyPrinter(indent=4)
        pp.pprint(data)
        print("", flush=True)

        # delay creation until __getitem__ is first called
        self.images = None
        self.audios = None # This will eventually store all lang audios

        # audio features are pre-computed with default values.
        self.audio_conf = audio_conf
        self.image_conf = image_conf

        # load image/audio dataset size and check if number matches
        images = h5py.File(self.image_hdf5_path, 'r')
        # audios = h5py.File(self.english_audio_hdf5_path, 'r')

        english_audios = h5py.File(self.english_audio_hdf5_path, 'r')
        hindi_audios = h5py.File(self.hindi_audio_hdf5_path, 'r')
        j_audios = h5py.File(self.j_audio_hdf5_path, 'r')

        # Assure that the number of datapoints match
        _check_num_samples(english_audios, hindi_audios, j_audios, images)
        self.n_samples = english_audios['melspec'].shape[0]

        english_audios.close()
        hindi_audios.close()
        j_audios.close()
        images.close()

        self.image_resize_and_crop, self.image_normalize = _process_resize_crop_norm_funcs(
            image_conf)

    def _get_lang_audio(self, index, lang_key):
        audio = defaultdict(dict)
        audio[lang_key]['lmspecs'] = self.audios[lang_key]['melspec'][index]
        audio[lang_key]['nframes'] = self.audios[lang_key]['melspec_len'][index]
        if self.audio_conf.get('normalize', False):
                n_frames = audio[lang_key]['nframes']
                logspec = audio[lang_key]['lmspecs']
                mean = logspec[:, 0:n_frames].mean()
                std = logspec[:, 0:n_frames].std()
                logspec[:, 0:n_frames].add_(-mean)
                logspec[:, 0:n_frames].div_(std)
        return audio

    def _get_audio(self, index, offset=0):
        if self.audios is None:
            self._LoadAudio()

        audio = defaultdict(dict)
        for lang_id in ['english', 'hindi', 'japanese']:
            # Returns dict: {lang_id: {'lmspecs': np.ndarray[#logmel_feats, max_frame_length], 'nframes', np.ndarray[1]}}
            # nframes is just a scalar for each index
            lang_audio = self._get_lang_audio(index+offset,lang_id)
            audio.update(lang_audio)

        return audio

    def _LoadAudio(self):
        english_audios = h5py.File(self.english_audio_hdf5_path, 'r')
        hindi_audios = h5py.File(self.hindi_audio_hdf5_path, 'r')
        j_audios = h5py.File(self.j_audio_hdf5_path, 'r')
        self.audios = {
            "english"  : english_audios,
            "hindi"    : hindi_audios,
            "japanese" : j_audios
        }

    def _get_image(self, index, offset=0):
        if self.images is None:
            self._LoadImage()
        binary_img = self.images['image'][index+offset]
        img = Image.open(io.BytesIO(binary_img)).convert('RGB')
        img = self.image_resize_and_crop(img)
        img = self.image_normalize(img)
        return img

    def _LoadImage(self):
            self.images = h5py.File(self.image_hdf5_path, 'r')

    def __getitem__(self, index):
        """
        returns: image, audio
        where image is a FloatTensor of size (3, H, W)
        audio is a dict:
            {
                "english":{
                    "lmspecs":numpy.ndarray #shape: (N_freq, max_frames)
                    "nframes"::numpy.ndarray #shape: (1)
                },
                "japanese":{
                ...
            }
        """
        audio = self._get_audio(index)
        image = self._get_image(index)
        return image, audio

    def __len__(self):
        return self.n_samples

    def _create_dev_set_arg_check(self, n, p, dev_set_confs):
        error_prefix_str = str(self.__class__)+": ARGUMENT ERROR: "
        assert (n is None or p is None) and (n is not None or p is not None), \
                error_prefix_str + "Only one of 'n' and 'p' can be given"

        assert p is None or (isinstance(p, float) and 0.0 < p < 1.0), \
                error_prefix_str + "'p' must be a float between 0.0 and 1.0 exclusive"

        assert n is None or (isinstance(n, int) and 0 < n < self.n_samples), \
                error_prefix_str + f"'n' must be an int greater than 0 and less than n_samples ({self.n_samples})"

    def create_dev_set(self, n:int=None, p:float=None, dev_set_confs = dict()):
        print("DATA_LOADER: Creating dev set")
        self._create_dev_set_arg_check(n, p, dev_set_confs)
        if n is not None:
            dev_n_samples = n
        elif p is not None:
            dev_n_samples = self.n_sample
        else:
            raise ValueError("Either 'n' or 'p' must be given")

        orig_train_samples = self.n_samples
        self.n_samples = self.n_samples - dev_n_samples
        print(f"DATA_LOADER: Dev set size {dev_n_samples}")
        print(f"DATA_LOADER: Train set size reduced from {orig_train_samples} to {self.n_samples}")
        dev_audio_conf = dev_set_confs.get("audio_conf", dict())
        dev_image_conf = dev_set_confs.get("image_conf", dict())

        self._LoadAudio()
        self._LoadImage()
        dev_set = DevImageCaptionDatasetHDF5(self.audios, self.images,
                                             dev_audio_conf, dev_image_conf, self.n_samples)
        return self, dev_set

class DevImageCaptionDatasetHDF5(ImageCaptionDatasetHDF5):
    def __init__(self, audios: dict, images: h5py.File,
                 audio_conf: dict, image_conf: dict, train_n_sample):
        self.audios = audios
        self.images = images

        # audio features are pre-computed with default values.
        self.audio_conf = audio_conf
        self.image_conf = image_conf

        # Assure that the number of datapoints match
        _check_num_samples(self.audios["english"], self.audios["hindi"], self.audios["japanese"], self.images)
        self.n_samples = audios["english"]['melspec'].shape[0] - train_n_sample
        self.train_n_sample = train_n_sample

        self.image_resize_and_crop, self.image_normalize = _process_resize_crop_norm_funcs(
            self.image_conf)

    def __getitem__(self, index):
        """
        returns: image, audio, nframes
        where image is a FloatTensor of size (3, H, W)
        audio is a FloatTensor of size (N_freq, N_frames)
        nframes is an integer
        """
        audio = self._get_audio(index, offset=self.train_n_sample)
        image = self._get_image(index, offset=self.train_n_sample)
        return image, audio