text:
"""
.. module:: sensors
    :platform: Windows
    :synopsis: Module containing the various sensors one can attach to a
               vehicle.

.. moduleauthor:: Marc Müller <mmueller@beamng.gmbh>
.. moduleauthor:: Pascale Maul <pmaul@beamng.gmbh>
.. moduleauthor:: Sedonas <https://github.com/Sedonas>

This module implements various sensors that can be attached to vehicles to
extract data from simulations.
"""
import base64
import mmap
import os

from abc import ABC, abstractmethod
from xml.dom import minidom
from xml.etree import ElementTree
from xml.etree.ElementTree import Element, SubElement

import numpy as np
from PIL import Image, ImageDraw, ImageFont
from logging import DEBUG as DBG_LOG_LEVEL
from logging import getLogger

from .beamngcommon import BNGValueError, LOGGER_ID

NEAR = 0.01
FAR = 1000

LIDAR_POINTS = 2000000

class AbstractSensor(ABC):
    """
    Abstract Sensor class declaring properties common to the ordinary and noise
    sensors.
    """

    @property
    @abstractmethod
    def data(self):
        pass

    @data.setter
    @abstractmethod
    def data(self, data):
        pass

    @data.deleter
    @abstractmethod
    def data(self):
        pass

class Sensor(AbstractSensor):
    """
    Sensor meta-class declaring methods common to them.
    """

    def __init__(self):
        self._data = dict()

    @property
    def data(self):
        """
        Property used to store sensor readings.
        """
        return self._data

    @data.setter
    def data(self, data):
        self._data = data

    @data.deleter
    def data(self):
        self._data = None

    def attach(self, vehicle, name):
        """
        Called when the sensor is attached to a :class:`.Vehicle` instance.
        Used to perform sensor setup code before the simulation is started.
        This is called *after* the sensor has been entered into the vehicle's
        map of sensors under the given name.

        Args:
            vehicle (:class:`.Vehicle`): The vehicle instance the sensor is
                                         being attached to.
            name (str): The name the sensor is known under to the vehicle.
        """
        pass

    def detach(self, vehicle, name):
        """
        Called when the sensor is detached from a :class:`.Vehicle` instance.
        Used to perform sensor teardown code after the simulation is finished.
        This is called *after* the sensor has been removed from the
        vehicle's map of sensors under the given name.

        Args:
            vehicle (:class:`.Vehicle`): The vehicle instance the sensor is
                                         being detached from.
            name (str): The name the sensor was known under to the vehicle.
        """
        pass

    def encode_engine_request(self):
        """
        Called to retrieve this sensor's data request to the engine as a
        dictionary. The dictionary returned by this method will be bundled
        along the vehicle's other sensors' requests as a SensorRequest to the
        game's engine.

        Note:
            Sensors require corresponding code in the simulator to handle
            requests.

        Example:
            Consult the implementation of the :class:`.Camera` sensor for a
            good example of an engine request.

        Returns:
            The request to send to the engine as a dictionary.
        """
        return None

    def encode_vehicle_request(self):
        """
        Called to retrieve this sensor's request to the vehicle as a
        dictionary. The dictionary returned by this method will be bundled
        along the vehicle's other sensors' requests as a SensorRequest to the
        attached vehicle.

        Note:
            Sensors require corresponding code in the simulator to handle
            requests.

        Example:
            Consult the implementation of the :class:`.Electrics` sensor for a
            good example of a vehicle request.

        Returns:
            The request to send to the vehicle as a dictionary.
        """
        return None

    def decode_response(self, resp):
        """
        Called to do post-processing on sensor data obtained from the
        simulation. This method is called after raw simulation data is received
        and the resulting processed data is considered the result of a sensor
        request.

        Example:
            Consult the implementation of the :class:`.Camera` sensor for a
            good example of decoding sensor data.
        """
        return resp

    def connect(self, bng, vehicle):
        """
        Called when the attached vehicle is being initialised in the
        simulation. This method is used to perform setup code that requires the
        simulation to be running.
        """
        pass

    def disconnect(self, bng, vehicle):
        """
        Called when the attached vehicle is being removed from simulation. This
        method is used to perform teardown code after the simulation.
        """
        pass

    def get_engine_flags(self):
        """
        Called to retrieve a dictionary of settings in the engine this sensor
        requires.

        Returns:
            A dictionary of flags to set in the engine for this sensor to
            function.
        """
        return dict()

class Camera(Sensor):
    """
    A camera sensor provides several types of image data from a user-defined
    perspective relative to the vehicle. It can provide the following types of
    data:

    * Colour images
    * Pixel-wise depth
    * Pixel-wise object annotation
    * Pixel-wise instance annotation to separate overlapping objects of the
      same type

    A single camera sensor can be configured to provide any or all of these
    data at once, ensuring they all align to the same perspective.
    """

    @staticmethod
    def extract_bboxes(semantic, instance, classes):
        """
        Analyzes the given semantic annotation and instance annotation images
        for its object bounding boxes. The identified objects are returned as
        a list of dictionaries containing their bounding box corners, class
        of object according to the corresponding color in the semantic
        annotations and the provided class mapping, and the color of the
        object in the instance annotation.

        Args:
            semantic (:class:`.Image`): The image containing semantic
                                        annotation information
            instance (:class:`.Image`): The image containing instance
                                        annotation information
            classes (dict): A mapping of colors to their class names to
                            identify object types based on the semantic
                            annotation information. The keys in this dictionary
                            are the respective colors expressed as a 24bit
                            integer, i.e. r * 256^2 + g * 256 + b.

        Returns:
            A list of bounding boxes specified as dictionaries. One example
            bounding box dictionary contains:

            ```
            {
                'bbox': [minx, miny, maxx, maxy],
                'color': [233, 11, 15],
                'class': ['CAR'],
            }
            ```

            Where minx, miny, maxx, maxy specify the corners of the bounding
            box, color contains the RGB color of the object in the instance
            annotations, and class the object type identified through the
            given class mapping.
        """
        semantic = np.array(semantic)
        instance = np.array(instance)

        if semantic.shape != instance.shape:
            raise BNGValueError('Provided semantic and instance annotation '
                                'images have different sizes.')

        bboxes = {}

        for y in range(semantic.shape[0]):
            for x in range(semantic.shape[1]):
                color = instance[y, x]
                color_key = color[0] * 256 * 256 + color[1] * 256 + color[2]
                if color_key == 0:
                    continue

                clazz = semantic[y, x]
                clazz_key = clazz[0] * 256 * 256 + clazz[1] * 256 + clazz[2]

                if classes[clazz_key] == 'BACKGROUND':
                    continue

                if color_key in bboxes:
                    entry = bboxes[color_key]
                    bbox = entry['bbox']
                    bbox[0] = min(bbox[0], x)
                    bbox[1] = min(bbox[1], y)
                    bbox[2] = max(bbox[2], x)
                    bbox[3] = max(bbox[3], y)
                else:
                    entry = {
                        'bbox': [x, y, x, y],
                        'class': classes[clazz_key],
                        'color': [*color],
                    }
                    bboxes[color_key] = entry

        ret = []
        for _, v in bboxes.items():
            ret.append(v)
        return ret

    @staticmethod
    def export_bbox_xml(bboxes, folder=None, filename=None, path=None,
                        database=None, size=None):
        """
        Exports the given list of bounding boxes to the Pascal-VOC XML
        standard. The bounding box list is expected to be in the format that
        `extract_bboxes` returns. Additional properties to this function
        correspond to tags in the Pascal-VOC standard.

        Args:
            bboxes (list): The list of bounding boxes the export. Consult the
                           documentation of `extract_bboxes` for information
                           on its expected structure.
            folder (str): Contents of the `<folder>` tag. Optional.
            filename (str): Contents of the `<filename>` tag. Optional.
            path (str): Contents of the `<path>` tag. Optional.
            database (str): Contents of the `<database>` tag. Optional.
            size (tuple): Contents of the `<size>` tag. It's expected to be a
                          tuple of the image width, height, and depth.
                          Optional.

        Returns:
            XML string encoding the given list of bounding boxes according to
            Pascal-VOC.
        """
        root = Element('annotation')

        if folder:
            folder_elem = SubElement(root, 'folder')
            folder_elem.text = folder

        if filename:
            file_elem = SubElement(root, 'filename')
            file_elem.text = filename

        if path:
            path_elem = SubElement(root, 'path')
            path_elem.text = path

        if database:
            source = SubElement(root, 'source')
            database_elem = SubElement(source, 'database')
            database_elem.text = database

        if size:
            size_elem = SubElement(root, 'size')

            width = SubElement(size_elem, 'width')
            width.text = str(size[0])

            height = SubElement(size_elem, 'height')
            height.text = str(size[1])

            depth = SubElement(size_elem, 'depth')
            depth.text = str(size[2])

        segmented = SubElement(root, 'segmented')
        segmented.text = '0'

        for idx, bbox in enumerate(bboxes):
            object_elem = SubElement(root, 'object')
            name = SubElement(object_elem, 'name')
            name.text = '{}_{}'.format(bbox['class'], idx)

            pose = SubElement(object_elem, 'pose')
            pose.text = 'Unspecified'

            truncated = SubElement(object_elem, 'truncated')
            truncated.text = '0'

            difficult = SubElement(object_elem, 'difficult')
            difficult.text = '0'

            bndbox = SubElement(object_elem, 'bndbox')

            xmin = SubElement(bndbox, 'xmin')
            xmin.text = str(bbox['bbox'][0])

            ymin = SubElement(bndbox, 'ymin')
            ymin.text = str(bbox['bbox'][1])

            xmax = SubElement(bndbox, 'xmax')
            xmax.text = str(bbox['bbox'][2])

            ymax = SubElement(bndbox, 'ymax')
            ymax.text = str(bbox['bbox'][3])

        ret = ElementTree.tostring(root, 'utf-8')
        ret = minidom.parseString(ret)
        return ret.toprettyxml(indent='  ')

    @staticmethod
    def draw_bboxes(bboxes, color, width=3, font='arial.ttf', fontsize=14):
        """
        Draws the given list of bounding boxes onto the given image. The boxes
        are drawn with the given width of outlines in pixels and the given font
        and size configuration. The given image is not directly modified and
        the boxes are drawn onto a copy. The bboxes are expected to be in the
        format returned by `extract_bboxes`.

        Args:
            bboxes (list): List of bounding boxes to draw. Consult the
                           documentation of `extract_bboxes` for information
                           on the structure of this list.
            color (:class:`.Image`): The image to draw bounding boxes on. Will
                                     be copied and not modified in place.
            width (int): The width of bounding box outlines in pixels.
            font (str): A string specifying the font bounding box labels are
                        supposed to have.
            fontsize (int): The font size to draw labels with.

        Returns:
            A `:class:`.Image` that is a copy of the given image with bounding
            boxes drawn onto it.
        """

        color = color.copy()
        draw = ImageDraw.Draw(color)

        font = ImageFont.truetype(font=font, size=fontsize)

        for idx, bbox in enumerate(bboxes):
            bbox_color = bbox['color']
            bbox_color = (bbox_color[0], bbox_color[1], bbox_color[2])
            bbox_corners = bbox['bbox']
            draw.rectangle(bbox_corners, outline=bbox_color, width=width)

            text = '{}_{}'.format(bbox['class'], idx)
            text_pos = (bbox_corners[0], bbox_corners[3])
            text_anchor = 'lt'

            if text_pos[1] > color.size[1] * 0.9:
                text_pos = (bbox_corners[0], bbox_corners[1])
                text_anchor = 'lb'

            draw.text(text_pos, text, fill='#FFFFFF', stroke_width=2,
                      stroke_fill='#000000', font=font, anchor=text_anchor)

        return color

    def __init__(self, pos, direction, fov, resolution, near_far=(NEAR, FAR),
                 colour=False, depth=False, depth_distance=(NEAR, FAR),
                 depth_inverse=False, annotation=False, instance=False,
                 shmem=True):
        """
        The camera sensor is set up with a fixed offset position and
        directional vector to face relative to the vehicle. This means as the
        vehicle moves and rotates, the camera is moved and rotated accordingly.

        Besides position and orientation, the image can further be customised
        with the FoV angle the camera should have, the resolution of the
        image(s) it outputs, and the near/far plane at which objects get
        clipped from view.

        Which sensor data to provide can be indicated using boolean flags for
        the corresponding type. The way data is transmitted can be chosen via
        the `shmem` flag. When true, the data is exchanged via a region of
        memory shared with the simulator. This is a lot faster, but has the
        downside of requiring the Python process and the simulator to be on the
        same machine. When false, data is sent via a socket and thus possible
        to be transmitted over network.

        Args:
            pos (tuple): (x,y,z) tuple of the camera's position offset relative
                         to the vehicle it's attached to.
            direction (tuple): (x,y,z) tuple expressing the direction vector
                               the camera is facing.
            fov (float): The Field of View of the camera.
            resolution (tuple): (width,height) tuple encoding the camera's
                                output resolution.
            near_far (tuple): (near,far) tuple of the distance below which and
                              after which geometry gets clipped. Usually
                              does not need to be changed.
            colour (bool): Whether to output colour information.
            depth (bool): Whether to output depth information.
            depth_distance (tuple): (near,far) tuple of the distance range
                                    depth values should be mapped between.
                                    For example, a distance_scale of (10, 50)
                                    would mean geometry closer than 10 would
                                    be mapped to black and geometry further
                                    than 50 would be mapped to white. All
                                    distances in-between are interpolated
                                    accordingly.
            depth_inverse (bool): If true, depth values are inversed so that
                                  geometry that is closer is white instead
                                  of black, and geometry that is further is
                                  black instead of white. This is more
                                  typical behaviour of real-life stereo
                                  cameras that output depth images.
            annotation (bool): Whether to output annotation information.
            instance (bool): Whether to output instance annotation information.
            shmem (bool): Whether to use shared memory for sensor data
                          transmission.
        """
        super().__init__()
        self.logger = getLogger(f'{LOGGER_ID}.Camera')
        self.logger.setLevel(DBG_LOG_LEVEL)
        self.pos = pos
        self.direction = direction
        self.fov = fov
        self.resolution = resolution
        self.near_far = near_far

        self.colour = colour
        self.depth = depth
        self.depth_distance = depth_distance
        self.depth_inverse = depth_inverse
        self.annotation = annotation
        self.instance = instance
        id = ['colour', 'depth', 'annotation', 'instance']
        properties = [self.colour, self.depth, self.annotation, self.instance]
        properties = [id for id, prop in zip(id, properties) if properties]
        properties = ', '.join(properties)
        self.logger.debug(f'Set up camera to render {properties} images.')

        self.shmem = shmem
        self.colour_handle = None
        self.colour_shmem = None
        self.depth_handle = None
        self.depth_shmem = None
        self.annotation_handle = None
        self.annotation_shmem = None
        self.instance_handle = None
        self.instance_shmem = None

    def attach(self, vehicle, name):
        """
        This method is called when the camera is attached and allocates
        shared memory space to house the sensor data the camera is supposed
        to provide.

        Args:
            vehicle (:class:`.Vehicle`): The vehicle the camera is being
                                         attached to.
            name (str): The name of the camera.
        """
        if self.shmem:
            self.logger.debug('Initializing shared memory.')
            pid = os.getpid()
            prefix = ''
            if vehicle:
                prefix = vehicle.vid
            size = self.resolution[0] * self.resolution[1] * 4
            self.colour_handle = '{}.{}.{}.colour'
            self.colour_handle = self.colour_handle.format(pid, prefix, name)
            self.colour_shmem = mmap.mmap(0, size, self.colour_handle)
            self.logger.debug('Bound shmem for colour: {self.colour_handle}')

            self.depth_handle = '{}.{}.{}.depth'
            self.depth_handle = self.depth_handle.format(pid, prefix, name)
            self.depth_shmem = mmap.mmap(0, size, self.depth_handle)
            self.logger.debug(f'Bound shmem for depth: {self.depth_handle}')

            self.annotation_handle = '{}.{}.{}.annotate'
            self.annotation_handle = self.annotation_handle.format(pid, prefix,
                                                                   name)
            self.annotation_shmem = mmap.mmap(0, size, self.annotation_handle)
            self.logger.debug('Bound shmem for annotation: '
                              f'{self.annotation_handle}')

            if self.instance:
                self.instance_handle = '{}.{}.{}.instance'
                self.instance_handle = self.instance_handle.format(pid, prefix,
                                                                   name)
                self.instance_shmem = mmap.mmap(0, size, self.instance_handle)
                self.logger.debug('Bound shmem for instance: '
                                  f'{self.instance_handle}')

    def detach(self, vehicle, name):
        """
        This method is called when the camera is detached from the vehicle. It
        de-allocates the shared memory space obtained for sensor data.

        Args:
            vehicle (:class:`.Vehicle`): The vehicle the camera is being
                                         detached from.
            name (str): The name of the camera.
        """
        if self.colour_shmem:
            self.logger.debug('Unbinding shmem for color: '
                              f'{self.colour_handle}')
            self.colour_shmem.close()

        if self.depth_shmem:
            self.logger.debug('Unbinding shmem for depth: '
                              f'{self.depth_handle}')
            self.depth_shmem.close()

        if self.annotation_shmem:
            self.logger.debug('Unbinding shmem for annotation: ',
                              f'{self.annotation_handle}')
            self.annotation_shmem.close()

        if self.instance_shmem:
            self.logger.debug('Unbinding shmem for instance: '
                              f'{self.instance_handle}')
            self.instance_shmem.close()

    def connect(self, bng, vehicle):
        """
        This method is called when the vehicle is set up in the simulation.
        It's used to inform the simulation about the shared memory used to
        exchange sensor data for this camera.

        Args:
            bng (:class:`.BeamNGpy`): Running instance of BeamNGpy.
            vehicle (:class:`.Vehicle`): The vehicle being connected.
        """
        size = self.resolution[0] * self.resolution[1] * 4  # RGBA / L are 4bbp

        if self.colour_shmem:
            bng.open_shmem(self.colour_handle, size)

        if self.depth_shmem:
            bng.open_shmem(self.depth_handle, size)

        if self.annotation_shmem:
            bng.open_shmem(self.annotation_handle, size)

        if self.instance_shmem:
            bng.open_shmem(self.instance_handle, size)

    def disconnect(self, bng, vehicle):
        """
        This method is called when the vehicle is disconnected from the
        simulation. It's used to tell the simulation to close the shared memory
        used to exchange sensor data for this camera.

        Args:
            bng (:class:`.BeamNGpy`): Running instance of BeamNGpy.
            vehicle (:class:`.Vehicle`): The vehicle being disconnected.
        """
        if self.colour_shmem:
            bng.close_shmem(self.colour_handle)

        if self.depth_shmem:
            bng.close_shmem(self.depth_handle)

        if self.annotation_shmem:
            bng.close_shmem(self.annotation_handle)

        if self.instance_shmem:
            bng.close_shmem(self.instance_handle)

    def encode_engine_request(self):
        """
        This method encodes a render request to the simulation engine along
        with the properties this camera is configured with.

        Returns:
            The request to the engine as a dictionary. This dictionary contains
            fields for each property of the requested render and which modes
            (colour, depth, annotation) to render in.
        """
        req = dict(type='Camera')

        if self.colour_shmem:
            req['color'] = self.colour_handle
        else:
            req['color'] = self.colour

        if self.depth_shmem:
            req['depth'] = self.depth_handle
        else:
            req['depth'] = self.depth

        if self.annotation_shmem:
            req['annotation'] = self.annotation_handle
        else:
            req['annotation'] = self.annotation

        if self.instance_shmem:
            req['instance'] = self.instance_handle
        else:
            req['instance'] = self.instance

        req['pos'] = [float(f) for f in self.pos]
        req['direction'] = [float(f) for f in self.direction]
        req['fov'] = self.fov
        req['resolution'] = [int(i) for i in self.resolution]
        req['near_far'] = [float(f) for f in self.near_far]
        req['shmem'] = self.shmem

        return req

    def decode_image(self, buffer, width, height, channels, dtype=np.uint8):
        img_d = base64.b64decode(buffer)
        img_d = np.frombuffer(img_d, dtype=dtype)
        if channels > 1:
            img_d = img_d.reshape(height, width, channels)
        else:
            img_d = img_d.reshape(height, width)
        return Image.fromarray(img_d)

    def decode_b64_response(self, resp):
        decoded = dict(type='Camera')
        img_w = resp['width']
        img_h = resp['height']

        if self.colour:
            decoded['colour'] = self.decode_image(resp['colorRGB8'],
                                                  img_w, img_h, 4)

        if self.annotation:
            decoded['annotation'] = self.decode_image(resp['annotationRGB8'],
                                                      img_w, img_h, 4)

        if self.instance:
            decoded['instance'] = self.decode_image(resp['instanceRGB8'],
                                                    img_w, img_h, 4)

        if self.depth:
            decoded['depth'] = self.decode_image(resp['depth32F'],
                                                 img_w, img_h, 1,
                                                 dtype=np.float32)
            # TODO: More needs to be done here to scale the lightness values
            # between 0-255. Please see decode_shmem_response(). Currently
            # this would generate an invalid image where each pixel's value
            # would actually be a raw distance. I am unable to complete this
            # as using shmem=False currently does not work for me.

        return decoded

    def decode_shmem_response(self, resp):
        """
        This method obtains sensor data written to shared memory and decodes
        them as images. The resulting data is returned as a dictionary. This
        dictionary contains an entry for each requested image type that is
        mapped to a :class:`PIL.Image` instance.

        Args:
            resp (dict): The raw sensor data as a dictionary that was returned
                         by the simulation.

        Returns:
            The decoded response as a dictionary.
        """
        decoded = dict(type='Camera')
        img_w = resp['width']
        img_h = resp['height']

        size = img_w * img_h * 4

        if self.colour_shmem:
            if 'color' in resp.keys():
                self.colour_shmem.seek(0)
                colour_d = self.colour_shmem.read(size)
                colour_d = np.frombuffer(colour_d, dtype=np.uint8)
                colour_d = colour_d.reshape(img_h, img_w, 4)
                decoded['colour'] = Image.fromarray(colour_d)
            else:
                msg = 'Color buffer failed to render. '\
                      'Check that you aren\'t running on low settings.'
                self.logger.error(msg)

        if self.annotation_shmem:
            if 'annotation' in resp.keys():
                self.annotation_shmem.seek(0)
                annotate_d = self.annotation_shmem.read(size)
                annotate_d = np.frombuffer(annotate_d, dtype=np.uint8)
                annotate_d = annotate_d.reshape(img_h, img_w, 4)
                decoded['annotation'] = Image.fromarray(annotate_d)
            else:
                msg = 'Color buffer failed to render. '\
                      'Check that you aren\'t running on low settings.'
                self.logger.error(msg)

        if self.depth_shmem:
            if 'depth' in resp.keys():
                self.depth_shmem.seek(0)
                depth_d = self.depth_shmem.read(size)
                depth_d = np.frombuffer(depth_d, dtype=np.float32)
                # Use linear interpolation to map the depth values
                # between lightness values 0-255. Any distances outside
                # of the scale are clamped to either 0 or 255
                # respectively.
                if self.depth_inverse:
                    depth_dist = [self.depth_distance[0],
                                  self.depth_distance[1]]
                    depth_d = np.interp(depth_d,
                                        depth_dist,
                                        [255, 0],
                                        left=255,
                                        right=0)
                else:
                    depth_d = np.interp(depth_d, [self.depth_distance[0],
                                        self.depth_distance[1]], [0, 255],
                                        left=0,
                                        right=255)
                depth_d = depth_d.reshape(img_h, img_w)
                depth_d = np.uint8(depth_d)
                decoded['depth'] = Image.fromarray(depth_d)
            else:
                self.logger.error('Depth buffer failed to render. '
                                  'Check that you aren\'t running '
                                  'on low settings.')

        if self.instance_shmem:
            if 'instance' in resp.keys():
                self.instance_shmem.seek(0)
                instance_d = self.instance_shmem.read(size)
                instance_d = np.frombuffer(instance_d, dtype=np.uint8)
                instance_d = instance_d.reshape(img_h, img_w, 4)
                decoded['instance'] = Image.fromarray(instance_d)
            else:
                self.logger.error('Instance buffer failed to render. '
                                  'Check that you aren\'t running on low settings.')

        return decoded

    def decode_response(self, resp):
        if self.shmem:
            return self.decode_shmem_response(resp)
        else:
            return self.decode_b64_response(resp)

    def get_engine_flags(self):
        """
        Called to retrieve settings for the simulation engine. Depending on the
        types of data this camera is supposed to provide, this method returns
        a dictionary enabling certain render modes in the engine.
        """
        flags = dict()
        if self.annotation:
            flags['annotations'] = True
        return flags

class Lidar(Sensor):
    max_points = LIDAR_POINTS
    shmem_size = LIDAR_POINTS * 3 * 4

    def __init__(self, offset=(0, 0, 1.7), direction=(0, -1, 0), vres=32,
                 vangle=26.9, rps=2200000, hz=20, angle=360, max_dist=200,
                 visualized=True, shmem=True):
        """
        The Lidar sensor provides 3D point clouds representing the environment
        as detected by a pulsing laser emitted from the vehicle. The range,
        position, and refresh rate of this sensor can be customised.
        """
        super().__init__()
        self.logger = getLogger(f"{LOGGER_ID}.Lidar")
        self.logger.setLevel(DBG_LOG_LEVEL)
        self.use_shmem = shmem
        self.handle = None
        self.shmem = None

        self.offset = offset
        self.direction = direction

        self.vres = vres
        self.vangle = vangle
        self.rps = rps
        self.hz = hz
        self.angle = angle
        self.max_dist = max_dist

        self.visualized = visualized

    def attach(self, vehicle, name):
        """
        Called when the lidar sensor is attached to a vehicle. This method
        allocates shared memory space to exchange lidar data with the engine.

        Args:
            vehicle (:class:`.Vehicle`): The vehicle the sensor is being
                                         attached to.
            name (str): The name of the sensor.
        """
        pid = os.getpid()
        self.handle = '{}.{}.{}.lidar'.format(pid, vehicle.vid, name)
        if self.use_shmem:
            self.shmem = mmap.mmap(0, Lidar.shmem_size, self.handle)
            self.logger.debug(f'Bound memory for lidar: {self.handle}')

    def detach(self, vehicle, name):
        """
        Called when the lidar sensor is detached from a vehicle. This method
        de-allocates the shared memory used to exchange lidar data with the
        engine.

        Args:
            vehicle (:class:`.Vehicle`): The vehicle the sensor is being
                                         detached from.
            name (str): The name of the sensor.
        """
        if self.use_shmem:
            self.logger.debug('Closing shmem.')
            self.shmem.close()

    def connect(self, bng, vehicle):
        if self.use_shmem:
            bng.open_lidar(self.handle, vehicle, self.handle, Lidar.shmem_size,
                           offset=self.offset, direction=self.direction,
                           vres=self.vres, vangle=self.vangle, rps=self.rps,
                           hz=self.hz, angle=self.angle,
                           max_dist=self.max_dist, visualized=self.visualized)
        else:
            bng.open_lidar(self.handle, vehicle, '', 0, offset=self.offset,
                           direction=self.direction, vres=self.vres,
                           vangle=self.vangle, rps=self.rps, hz=self.hz,
                           angle=self.angle, max_dist=self.max_dist,
                           visualized=self.visualized)

    def disconnect(self, bng, vehicle):
        bng.close_lidar(self.handle)

    def encode_engine_request(self):
        """
        Called to obtain the engine request for this lidar sensor. Encodes the
        properties of this lidar to obtain data according to them.

        Returns:
            The engine request containing the settings of this lidar sensor as
            a dictionary.
        """
        req = dict(type='Lidar')
        req['name'] = self.handle
        req['shmem'] = self.use_shmem
        return req

    def decode_response(self, resp):
        """
        Reads the raw point cloud the simulation wrote to the shared memory and
        creates a numpy array of points from them. The recoded response is
        returned as a dictionary with the numpy array in the ``points`` entry.

        Returns:
            The decoded response as a dictionary with the point cloud as a
            numpy array in the ``points`` entry. The numpy array is a linear
            sequence of coordinate triplets in the form of [x0, y0, z0, x1,
            y1, z1, ..., xn, yn, zn].
        """
        points_buf = None

        if self.use_shmem:
            size = resp['size']
            self.shmem.seek(0)
            points_buf = self.shmem.read(size)
            points_buf = np.frombuffer(points_buf, dtype=np.float32)
        else:
            points_buf = resp['points']
            points_buf = np.array(points_buf, dtype=np.float32)

        assert points_buf.size % 3 == 0
        resp = dict(type='Lidar')
        resp['points'] = points_buf
        return resp

    def get_engine_flags(self):
        """
        Returns: a dictionary with the engine flag to enable Lidar.
        """
        flags = dict(lidar=True)
        return flags

class GForces(Sensor):
    """
    This sensor is used to obtain the GForces acting on a vehicle.

    # TODO: GForce sensor for specific points on/in the vehicle
    """

    def __init__(self):
        super().__init__()

    def encode_vehicle_request(self):
        req = dict(type='GForces')
        return req

class Electrics(Sensor):
    """
    This sensor is used to retrieve various values made available by the car's
    eletrics systems. These values include:

    # TODO: List all the electrics.lua values.
    - abs (int): ABS state
    - abs_active (bool):
    - airspeed (float): Airspeed
    - airflowspeed (float):
    - altitude (float): Z axis position
    - avg_wheel_av (float):
    - brake (int): Brake value
    - brake_lights (int):
    - brake_input (int): Brake input value
    - check_engine (bool): Check engine light state.
    - clutch (int): Clutch value
    - clutch_input (int): Clutch input value
    - clutch_ratio (int):
    - driveshaft (float): Driveshaft
    - engine_load (float):
    - engine_throttle (int): Engine throttle state
    - esc (int): ESC state. 0 = not present/inactive, 1 = disabled,
                 Blink = active
    - esc_active (bool):
    - exhaust_flow (float):
    - fog_lights (int): Fog light state
    - fuel (float): Percentage of fuel remaining.
    - fuel_capacity (int): Total Fuel Capacity [L].
    - fuel_volume (float):
    - gear (int):
    - gear_a (int): Gear selected in automatic mode.
    - gear_index (int):
    - gear_m (int): Gear selected in manual mode.
    - hazard (int): Hazard light state
    - hazard_signal (bool):
    - headlights (int):
    - highbeam (int): High beam state
    - horn (int):
    - ignition (bool): Engine state
    - left_signal (bool):
    - lightbar (int): Lightbar state
    - lights (int): General light state. 1 = low, 2 = high
    - lowbeam (int): Low beam state
    - lowfuel (bool): Low fuel indicator
    - lowhighbeam (int): Low-high beam state
    - lowpressure (int): Low fuel pressure indicator
    - oil (int):
    - oil_temperature (float): Oil temperature [C].
    - parking (int): Parking lights on/off (not implemented yet)
    - parkingbrake (float): Parking brake state. 0.5 = halfway on
    - parkingbrake_input (int): Parking brake input state
    - radiator_fan_spin (int):
    - reverse (int): Reverse gear state
    - right_signal (bool):
    - rpm (float): Engine RPM
    - rpmspin (float):
    - rpm_tacho (float):
    - running (bool): Engine running state
    - signal_l (int): Left signal state. 0.5 = halfway to full blink
    - signal_r (int): Right signal state. 0.5 = halfway to full blink
    - steering (int): Steering state
    - steering_input (int): Steering input state
    - tcs (int): TCS state. 0 = not present/inactive, 1 = disabled,
                 Blink = active
    - tcs_active (bool):
    - throttle (int): Throttle state
    - throttle_factor (int):
    - throttle_input (int): Throttle input state
    - turnsignal (int): Turn signal value. -1 = Left, 1 = Right,
                        gradually 'fades' between values. Use "signal_L" and
                        "signal_R" for flashing indicators.
    - two_step (bool):
    - water_temperature (float): Water temperature [C].
    - wheelspeed (float): Wheel speed [m/s].
    """
    name_map = {
        'absActive': 'abs_active',
        'avgWheelAV': 'avg_wheel_av',
        'brakelights': 'brake_lights',
        'checkengine': 'check_engine',
        'clutchRatio': 'clutch_ratio',
        'engineLoad': 'engine_load',
        'engineThrottle': 'engine_throttle',
        'escActive': 'esc_active',
        'exhaustFlow': 'exhaust_flow',
        'fog': 'fog_lights',
        'fuelVolume': 'fuel_volume',
        'fuelCapacity': 'fuel_capacity',
        'gear_A': 'gear_a',
        'gearIndex': 'gear_index',
        'gear_M': 'gear_m',
        'hazard_enabled': 'hazard_signal',
        'isShifting': 'is_shifting',
        'lights_state': 'headlights',
        'oiltemp': 'oil_temperature',
        'radiatorFanSpin': 'radiator_fan_spin',
        'rpmTacho': 'rpm_tacho',
        'signal_L': 'signal_l',
        'signal_left_input': 'left_signal',
        'signal_R': 'signal_r',
        'signal_right_input': 'right_signal',
        'tcsActive': 'tcs_active',
        'throttleFactor': 'throttle_factor',
        'twoStep': 'two_step',
        'watertemp': 'water_temperature',
    }

    def __init__(self):
        super().__init__()

    def _rename_values(self, vals):
        """
        The values returned from the game often don't follow any naming
        convention and especially don't follow this library's, so we rename
        some of them here to be more consistent.
        """

        for k, v in Electrics.name_map.items():
            if k in vals:
                vals[v] = vals[k]
                del vals[k]
        return vals

    def _reassign_values(self, vals):
        if 'left_signal' in vals:
            vals['left_signal'] = vals['left_signal'] == 1
        if 'right_signal' in vals:
            vals['right_signal'] = vals['right_signal'] == 1
        if 'hazard_signal' in vals:
            vals['hazard_signal'] = vals['hazard_signal'] == 1
        return vals

    def encode_vehicle_request(self):
        req = dict(type='Electrics')
        return req

    def decode_response(self, resp):
        if 'values' in resp:
            ret = self._rename_values(resp['values'])
            ret = self._reassign_values(ret)
            return ret
        return None

class Damage(Sensor):
    """
    The damage sensor retrieves information about how damaged the structure
    of the vehicle is. It's important to realise that this is a sensor that has
    no analogue in real life as it returns a perfect knowledge overview of how
    deformed the vehicle is. It's therefore more of a ground truth than
    simulated sensor data.
    """

    def __init__(self):
        super().__init__()

    def encode_vehicle_request(self):
        req = dict(type='Damage')
        if 'part_damage' in req.keys():
            req['part_damage'] = req['part_damage'] if req['part_damage'] else {}
        return req

class Timer(Sensor):
    """
    The timer sensor keeps track of the time that has passed since the
    simulation started. It provides that information in seconds relative to the
    scenario start and does not represent something like a day time or date. It
    properly handles pausing the simulation, meaning the value of the timer
    sensor does not progress while the simulation is paused.

    When polled, this sensor provides the time in seconds since the start of
    the scenario in a dictionary under the 'time' key.
    """

    def __init__(self):
        super().__init__()

    def encode_engine_request(self):
        req = dict(type='Timer')
        return req

class State(Sensor):
    """
    The state sensor monitors general stats of the vehicle, such as position,
    direction, velocity, etc. It is a default sensor every vehicle has and is
    used to update the vehicle.state attribute.
    """

    def __init__(self):
        super().__init__()
        self.connected = False

    def connect(self, bng, vehicle):
        self.connected = True

    def disconnect(self, bng, vehicle):
        self.connected = False

    def encode_vehicle_request(self):
        req = dict(type='State')
        return req

    def decode_response(self, resp):
        if 'state' in resp:
            return resp['state']

        return None

class IMU(Sensor):
    """
    An IMU measures forces and rotational acceleration at a certain point on a
    vehicle. This can be used to analyze forces acting on certain areas of the
    car (like the driver's position) or estimate the trajectory of a vehicle
    from its rotation and acceleration.
    """

    def __init__(self, pos=None, node=None, name=None, debug=False):
        super().__init__()

        if pos is not None and node is not None:
            raise BNGValueError('Cannot specify both position and node for '
                                'an IMU')

        self._pos = pos
        self._node = node

        self._name = name
        if self._name is None:
            self._name = str(hash(self))

        self._debug = debug

    def connect(self, bng, vehicle):
        if self._pos is not None:
            vehicle.add_imu_position(self._name, self._pos, self._debug)

        if self._node is not None:
            vehicle.add_imu_node(self._name, self._node, self._debug)

    def disconnect(self, bng, vehicle):
        vehicle.remove_imu(self._name)

    def encode_vehicle_request(self):
        req = dict(type='IMU')
        req['name'] = self._name
        return req

class Ultrasonic(Sensor):
    """
    An ultrasonic sensor (aka parking sensor) that can be placed at
    any point outside the vehicle.
    This is not an ideal sensor but one whose output is simulated based on
    depth information in images.
    """

    def __init__(self,
                 pos,
                 rot,
                 fov=(70, 35),
                 min_resolution=256,
                 near_far=(0.15, 5.5)):
        self.logger = getLogger(f'{LOGGER_ID}.Ultrasonic')
        self.logger.setLevel(DBG_LOG_LEVEL)
        self.pos = pos
        self.rot = rot
        self.fov = fov[0]
        res_height = int(min_resolution/fov[0]*fov[1])
        self.resolution = (min_resolution, res_height)
        self.near_far = near_far
        self.vis_spec = None

    def encode_engine_request(self):
        req = dict(type='Ultrasonic')
        req['pos'] = self.pos
        req['rot'] = self.rot
        req['fov'] = self.fov
        req['resolution'] = self.resolution
        req['near_far'] = self.near_far
        return req

    def startVisualization(self, bng, vehicle_id, color, radius=.1):
        """
        Called, after opening BeamNG, this will start the visualization
        of a sphere at the sensor position. This functionality is intended
        for sensors attached to vehicles and not for world sensors.

        Args:
            bng(:class:`.BeamNGpy`): instance of BeamNGpy
            vehicle_id(string): ID of the vehicle the sensor belongs to
            color(tuple): four floats (RGBA) defining the color of the sphere
            radius(float): radius of the sphere
        """
        req = dict(type='StartUSSensorVisualization')
        req['vehicle'] = vehicle_id
        req['pos'] = self.pos
        req['rot'] = self.rot
        req['color'] = color
        req['radius'] = radius
        req['lineLength'] = self.near_far[1]
        bng.send(req)
        resp = bng.recv()
        self.vis_spec = resp['sphereID']

    def stopVisualization(self, bng):
        """
        Stops the sensor visualization.
        """
        if self.vis_spec is not None:
            data = dict(type='StopUSSensorVisualization')
            data['dynSphereID'] = self.vis_spec
            bng.send(data)
        else:
            self.logger.debug('Cannot stop visualization '
                              'since no sphere ID is available.')

max_stars_repo_name: wau/BeamNGpy

max_issues_repo_name: BeamNG/BeamNGpy

max_forks_repo_name: wau/BeamNGpy


text:
# Noaa weather radar downloader
#import needed things
__author__ ='Amos Manneschmidt'

from workflow import Workflow
import sys, os, urllib, signal, time
import subprocess as sp
from subprocess import Popen

loc=sys.argv[1] # set weather station code to query from alfred
anmdir = "animations/"
staticdir = "static/"
#Also do some BS to make libaries work in remote installations
def main(wf):
	import numpy, imageio
	from PIL import Image
	from bs4 import BeautifulSoup as BS

	#####################################################
	#setup initial "quick" static image
	print("Getting images...")
	urllib.urlretrieve("http://radar.weather.gov/Warnings/Short/"+loc+"_Warnings_0.gif","static/wn.gif") #warnings layer
	urllib.urlretrieve("http://radar.weather.gov/Legend/N0R/"+loc+"_N0R_Legend_0.gif","static/lg.gif") #legends
	urllib.urlretrieve("http://radar.weather.gov/RadarImg/N0R/"+loc+"_N0R_0.gif","static/rd.gif") #static radar

	bg=Image.open(staticdir+"grayscale.gif").convert('RGBA')	#bg=background image
	cnty=Image.open(staticdir+"overlay.gif").convert('RGBA')	#cnty=county, roads, state, city overlay
	wn=Image.open(staticdir+"wn.gif").convert('RGBA')# wn=warnings layer
	cnty.paste(wn,(0,0),wn)
	rd=Image.open(staticdir+"rd.gif").convert('RGBA')
	lg=Image.open(staticdir+"lg.gif").convert('RGBA')
	temp=Image.open(staticdir+"grayscale.gif").convert('RGBA')
	temp.paste(rd,(0,0),rd)
	temp.paste(cnty,(0,0),cnty)
	temp.paste(lg,(0,0),lg)
	temp.save(staticdir+'radar static.gif')

	#open saved static image with quicklook silencing output
	null = open(os.devnull,'wb')
	p=sp.Popen(['qlmanage', '-p', staticdir+'radar static.gif'],stderr=null,stdout=null)

	######################################################################
	#Parse html for links and download the 6 most recent images for region
	html = urllib.urlopen("http://radar.weather.gov/RadarImg/N0R/"+loc+"/?C=M;O=D")
	soup = BS(html,"html.parser")

	names=['']*6
	for i in range(5,11):
		n=i-5
		names[n]=soup.find_all('a')[i]['href'][:-4]
		print("downloading image "+names[n]+".gif")
		urllib.urlretrieve("http://radar.weather.gov/RadarImg/N0R/"+loc+"/"+names[n]+".gif",anmdir+names[n]+".gif")
		urllib.urlretrieve("http://radar.weather.gov/Legend/N0R/"+loc+"/"+names[n]+"_Legend.gif",anmdir+names[n]+"_Legend.gif")
		print("     downloaded image "+names[n]+".gif")

	print("   done")

	########################################################
	#Composite together overlay and underlay for each images
	print("Compositing..")
	images=['']*6
	for i,n in enumerate(names):
	 	im=Image.open(anmdir+n+".gif").convert('RGBA')
	 	lg=Image.open(anmdir+n+"_Legend.gif").convert('RGBA')
	 	temp=Image.open(staticdir+"grayscale.gif").convert('RGBA')
	 	temp.paste(im,(0,0),im)
	 	temp.paste(cnty,(0,0),cnty)
	 	temp.paste(lg,(0,0),lg)
	 	temp.save(anmdir+n+".gif")
	 	images[5-i]=imageio.imread(anmdir+n+".gif")
	print("   done")

	print("Saving/Compiling...")
	imageio.mimwrite("radar.gif",images,duration=[.25,.25,.25,.25,.25,1.5])
	print("   done")
	if not p.poll() and p.poll() != 0:
		print("Displaying...")
		np=sp.Popen(['qlmanage', '-p', 'radar.gif'],stderr=null,stdout=null)

		time.sleep(1)
		p.kill()
		sp.Popen("rm animations/*",shell=True)
		print("   done")
	print("Alfred Script done")

if __name__ == '__main__':
	wf = Workflow(libraries=['./lib'])
	sys.exit(wf.run(main))

ext: py

reason: import numpy


text:
from typing import Dict
import numpy as np
import scipy
from typing import List, Tuple
from tqdm import tqdm
import random
import warnings
import inlp_dataset_handler
import inlp_linear_model

def get_rowspace_projection(W: np.ndarray) -> np.ndarray:
    """
    :param W: the matrix over its nullspace to project
    :return: the projection matrix over the rowspace
    """

    if np.allclose(W, 0):
        w_basis = np.zeros_like(W.T)
    else:
        w_basis = scipy.linalg.orth(W.T) # orthogonal basis

    w_basis = w_basis * np.sign(w_basis[0][0]) # handle sign ambiguity
    P_W = w_basis.dot(w_basis.T) # orthogonal projection on W's rowspace

    return P_W

def get_projection_to_intersection_of_nullspaces(rowspace_projection_matrices: List[np.ndarray], input_dim: int):
    """
    Given a list of rowspace projection matrices P_R(w_1), ..., P_R(w_n),
    this function calculates the projection to the intersection of all nullspasces of the matrices w_1, ..., w_n.
    uses the intersection-projection formula of Ben-Israel 2013 http://benisrael.net/BEN-ISRAEL-NOV-30-13.pdf:
    N(w1)∩ N(w2) ∩ ... ∩ N(wn) = N(P_R(w1) + P_R(w2) + ... + P_R(wn))
    :param rowspace_projection_matrices: List[np.array], a list of rowspace projections
    :param dim: input dim
    """

    I = np.eye(input_dim)
    Q = np.sum(rowspace_projection_matrices, axis = 0)
    P = I - get_rowspace_projection(Q)

    return P

def debias_by_specific_directions(directions: List[np.ndarray], input_dim: int):
    """
    the goal of this function is to perform INLP on a set of user-provided directiosn (instead of learning those directions).
    :param directions: list of vectors, as numpy arrays.
    :param input_dim: dimensionality of the vectors.
    """

    rowspace_projections = []

    for v in directions:
        P_v = get_rowspace_projection(v)
        rowspace_projections.append(P_v)

    P = get_projection_to_intersection_of_nullspaces(rowspace_projections, input_dim)

    return P

def run_INLP(num_classifiers: int, input_dim: int, is_autoregressive: bool, min_accuracy: float,
                             dataset_handler: inlp_dataset_handler.DatasetHandler, model: inlp_linear_model.LinearModel) -> Tuple[np.ndarray]:
    """

    :param num_classifiers: number of iterations (equivalent to number of dimensions to remove)
    :param input_dim: size of input vectors
    :param is_autoregressive: whether to train the ith classiifer on the data projected to the nullsapces of w1,...,wi-1
    :param min_accuracy: above this threshold, ignore the learned classifier
    :param dataset_handler: an isntance of DatasetHandler, in charge of storing the data & projecting it
    :param inlp_linear_model: an instance of LinearModel, in charge of training the model and returning its weights
    :return: P, the debiasing projection; rowspace_projections, the list of all rowspace projection; Ws, the list of all calssifiers.
    """

    #if dropout_rate > 0 and is_autoregressive:
    #    warnings.warn("Note: when using dropout with autoregressive training, the property w_i.dot(w_(i+1)) = 0 no longer holds.")

    I = np.eye(input_dim)
    rowspace_projections = []
    Ws = []
    dataset_handler.reinitialize()

    pbar = tqdm(range(num_classifiers))

    for i in pbar:

        # initialize models & dataset

        model.initialize_model()

        # train model, record accuracy

        acc = model.train_model(dataset_handler)
        pbar.set_description("iteration: {}, accuracy: {}".format(i, acc))
        if acc < min_accuracy: continue

        # collect parameters and rowspace projections

        W = model.get_weights()
        Ws.append(W)
        P_rowspace_wi = get_rowspace_projection(W) # projection to W's rowspace
        rowspace_projections.append(P_rowspace_wi)

        if is_autoregressive:

            """
            to ensure numerical stability, explicitly project to the intersection of the nullspaces found so far (instaed of doing X = P_iX,
            which is problematic when w_i is not exactly orthogonal to w_i-1,...,w1, due to e.g inexact argmin calculation).
            """
            # use the intersection-projection formula of Ben-Israel 2013 http://benisrael.net/BEN-ISRAEL-NOV-30-13.pdf:
            # N(w1)∩ N(w2) ∩ ... ∩ N(wn) = N(P_R(w1) + P_R(w2) + ... + P_R(wn))

            P = get_projection_to_intersection_of_nullspaces(rowspace_projections, input_dim)
            # project
            dataset_handler.apply_projection(P)

    """
    calculae final projection matrix P=PnPn-1....P2P1
    since w_i.dot(w_i-1) = 0, P2P1 = I - P1 - P2 (proof in the paper); this is more stable.
    by induction, PnPn-1....P2P1 = I - (P1+..+PN). We will use instead Ben-Israel's formula to increase stability and also generalize to the non-orthogonal case (e.g. with dropout),
    i.e., we explicitly project to intersection of all nullspaces (this is not critical at this point; I-(P1+...+PN) is roughly as accurate as this provided no dropout & regularization)
    """

    P = get_projection_to_intersection_of_nullspaces(rowspace_projections, input_dim)

    return P, rowspace_projections, Ws

if __name__ == '__main__':

    from sklearn.linear_model import SGDClassifier, Perceptron, LogisticRegression

    N = 10000
    d = 300
    X = np.random.rand(N, d) - 0.5
    Y = np.array([1 if sum(x) > 0 else 0 for x in X]) #X < 0 #np.random.rand(N) < 0.5 #(X + 0.01 * (np.random.rand(*X.shape) - 0.5)) < 0 #np.random.rand(5000) < 0.5
    #Y = np.array(Y, dtype = int)

    num_classifiers = 200
    classifier_class = SGDClassifier #Perceptron
    input_dim = d
    is_autoregressive = True
    min_accuracy = 0.0

    P, rowspace_projections, Ws = get_debiasing_projection(classifier_class, {}, num_classifiers, input_dim, is_autoregressive, min_accuracy, X, Y, X, Y, by_class = False)

    I = np.eye(P.shape[0])
    P_alternative = I - np.sum(rowspace_projections, axis = 0)
    P_by_product = I.copy()

    for P_Rwi in rowspace_projections:

        P_Nwi = I - P_Rwi
        P_by_product = P_Nwi.dot(P_by_product)

    """testing"""

    # validate that P = PnPn-1...P2P1 (should be true only when w_i.dot(w_(i+1)) = 0, in autoregressive training)

    if is_autoregressive:
        assert np.allclose(P_alternative, P)
        assert np.allclose(P_by_product, P)

    # validate that P is a projection

    assert np.allclose(P.dot(P), P)

    # validate that P projects to N(w1)∩ N(w2) ∩ ... ∩ N(wn)

    x = np.random.rand(d) - 0.5
    for w in Ws:

        assert np.allclose(np.linalg.norm(w.dot(P.dot(x))), 0.0)

    # validate that each two classifiers are orthogonal (this is expected to be true only with autoregressive training)

    if is_autoregressive:
        for i,w in enumerate(Ws):

            for j, w2 in enumerate(Ws):

                if i == j: continue

                assert np.allclose(np.linalg.norm(w.dot(w2.T)), 0)

max_stars_repo_name: gonenhila/mbert

max_issues_repo_name: gonenhila/mbert

max_forks_repo_name: gonenhila/mbert

reason: import numpy,import scipy


text:
# 倒山峰形
import numpy as np
from matplotlib import pyplot as plt
fig = plt.figure()
ax = plt.axes(projection="3d")
x = y = np.arange(start=-4, stop=4, step=0.1)
X, Y = np.meshgrid(x, y)
Z = X**2+Y**2
ax.plot_surface(X,Y,Z,alpha=0.9, cstride=1, rstride = 1, cmap='rainbow')
plt.show()
exit()

#峰值函数上半轴
import numpy as np
from matplotlib import pyplot as plt
fig = plt.figure()
ax = plt.axes(projection="3d")
x = y = np.arange(start=-4, stop=4, step=0.1)
X, Y = np.meshgrid(x, y)
Z = 3 * (1-X)**2 * np.exp(-X**2-(Y+1)**2)- 10 * (X/5- X**3 - Y**5)*np.exp(-X**2-Y**2)-1/3*np.exp(-(X+1)**2-Y**2)
Z[Z<0]=0
ax.plot_surface(X,Y,Z,alpha=0.9, cstride=1, rstride = 1, cmap='rainbow')
plt.show()
exit()

# 峰值函数全貌
import numpy as np
from matplotlib import pyplot as plt
fig = plt.figure()
ax = plt.axes(projection="3d")
x = y = np.arange(start=-4, stop=4, step=0.1)
X, Y = np.meshgrid(x, y)
Z = 3 * (1-X)**2 * np.exp(-X**2-(Y+1)**2)- 10 * (X/5- X**3 - Y**5)*np.exp(-X**2-Y**2)-1/3*np.exp(-(X+1)**2-Y**2)
ax.plot_surface(X,Y,Z,alpha=0.9, cstride=1, rstride = 1, cmap='rainbow')
plt.show()
exit()

max_stars_repo_name: zhangwenniu/Statistic-Learning-Theory

max_issues_repo_name: zhangwenniu/Statistic-Learning-Theory

max_forks_repo_name: zhangwenniu/Statistic-Learning-Theory


text:
"""
Collection of pre-processing methods for aligning images
"""

#import csaps
from scipy.interpolate import Akima1DInterpolator
from skimage import exposure
import numpy as np
import cv2
from skimage import color as skcolor
from . import slide_io
import colour

# DEFAULT_COLOR_STD_C = 0.01 # jzazbz
DEFAULT_COLOR_STD_C = 0.2 # cam16-ucs

class ImageProcesser(object):
    """Process images for registration

    `ImageProcesser` sub-classes processes images to single channel
    images which are then used in image registration.

    Each `ImageProcesser` is initialized with an image, the path to the
    image, the pyramid level, and the series number. These values will
    be set during the registration process.

    `ImageProcesser` must also have a `process_image` method, which is
    called during registration. As `ImageProcesser` has the image and
    and its relevant information (filename, level, series) as attributes,
    it should be able to access and modify the image as needed. However,
    one can also pass extra args and kwargs to `process_image`. As such,
    `process_image` will also need to accept args and kwargs.

    Attributes
    ----------
    image : ndarray
        Image to be processed

    src_f : str
        Path to slide/image.

    level : int
        Pyramid level to be read.

    series : int
        The series to be read.

    """

    def __init__(self, image, src_f, level, series):
        """
        Parameters
        ----------
        image : ndarray
            Image to be processed

        src_f : str
            Path to slide/image.

        level : int
            Pyramid level to be read.

        series : int
            The series to be read.

        """

        self.image = image
        self.src_f = src_f
        self.level = level
        self.series = series

    def process_image(self,  *args, **kwargs):
        """Pre-process image for registration

        Pre-process image for registration. Processed image should
        be a single channel uint8 image.

        Returns
        -------
        processed_img : ndarray
            Single channel processed copy of `image`

        """

class ChannelGetter(ImageProcesser):
    """Select channel from image

    """

    def __init__(self, image, src_f, level, series, *args, **kwargs):
        super().__init__(image=image, src_f=src_f, level=level,
                         series=series, *args, **kwargs)

    def process_image(self, channel="dapi", adaptive_eq=True, *args, **kwaargs):
        reader_cls = slide_io.get_slide_reader(self.src_f, series=self.series)
        reader = reader_cls(self.src_f)
        chnl = reader.get_channel(channel=channel, level=self.level, series=self.series).astype(float)
        chnl /= chnl.max()
        if adaptive_eq:
            chnl = exposure.equalize_adapthist(chnl)

        chnl = exposure.rescale_intensity(chnl, in_range="image", out_range=(0, 255)).astype(np.uint8)

        return chnl

class ColorfulStandardizer(ImageProcesser):
    """Standardize the colorfulness of the image

    """

    def __init__(self, image, src_f, level, series, *args, **kwargs):
        super().__init__(image=image, src_f=src_f, level=level,
                         series=series, *args, **kwargs)

    def process_image(self, c=DEFAULT_COLOR_STD_C, invert=True, *args, **kwargs):
        std_rgb = standardize_colorfulness(self.image, c)
        std_g = skcolor.rgb2gray(std_rgb)

        if invert:
            std_g = 255 - std_g
        processed_img = exposure.rescale_intensity(std_g, in_range="image", out_range=(0, 255)).astype(np.uint8)

        return processed_img

class Luminosity(ImageProcesser):
    """Get luminosity of an RGB image

    """

    def __init__(self, image, src_f, level, series, *args, **kwargs):
        super().__init__(image=image, src_f=src_f, level=level,
                         series=series, *args, **kwargs)

    def process_image(self,  *args, **kwaargs):
        lum = get_luminosity(self.image)
        inv_lum = 255 - lum
        processed_img = exposure.rescale_intensity(inv_lum, in_range="image", out_range=(0, 255)).astype(np.uint8)

        return processed_img

def standardize_colorfulness(img, c=DEFAULT_COLOR_STD_C, h=0):
    """Give image constant colorfulness and hue

    Image is converted to cylindrical CAM-16UCS assigned a constant
    hue and colorfulness, and then coverted back to RGB.

    Parameters
    ----------
    img : ndarray
        Image to be processed
    c : int
        Colorfulness
    h : int
        Hue, in radians (-pi to pi)

    Returns
    -------
    rgb2 : ndarray
        `img` with constant hue and colorfulness

    """
    # Convert to CAM16 #
    eps = np.finfo("float").eps
    with colour.utilities.suppress_warnings(colour_usage_warnings=True):
        if 1 < img.max() <= 255 and np.issubdtype(img.dtype, np.integer):
            cam = colour.convert(img/255 + eps, 'sRGB', 'CAM16UCS')
        else:
            cam = colour.convert(img + eps, 'sRGB', 'CAM16UCS')

    lum = cam[..., 0]
    cc = np.full_like(lum, c)
    hc = np.full_like(lum, h)
    new_a, new_b = cc * np.cos(hc), cc * np.sin(hc)
    new_cam = np.dstack([lum, new_a+eps, new_b+eps])
    with colour.utilities.suppress_warnings(colour_usage_warnings=True):
        rgb2 = colour.convert(new_cam, 'CAM16UCS', 'sRGB')
        rgb2 -= eps

    rgb2 = (np.clip(rgb2, 0, 1)*255).astype(np.uint8)

    return rgb2

def get_luminosity(img, **kwargs):
    """Get luminosity of an RGB image
        Converts and RGB image to the CAM16-UCS colorspace, extracts the
        luminosity, and then scales it between 0-255

    Parameters
    ---------
    img : ndarray
        RGB image

    Returns
    -------
    lum : ndarray
        CAM16-UCS luminosity

    """

    with colour.utilities.suppress_warnings(colour_usage_warnings=True):
        if 1 < img.max() <= 255 and np.issubdtype(img.dtype, np.integer):
            cam = colour.convert(img/255, 'sRGB', 'CAM16UCS')
        else:
            cam = colour.convert(img, 'sRGB', 'CAM16UCS')

    lum = exposure.rescale_intensity(cam[..., 0], in_range=(0, 1), out_range=(0, 255))

    return lum

def match_histograms(src_image, ref_histogram, bins=256):
    """
    Source: https://automaticaddison.com/how-to-do-histogram-matching-using-opencv/

    This method matches the source image histogram to the
    reference signal
    :param image src_image: The original source image
    :param image  ref_image: The reference image
    :return: image_after_matching
    :rtype: image (array)
    """
    def calculate_cdf(histogram):
        """
        This method calculates the cumulative distribution function
        :param array histogram: The values of the histogram
        :return: normalized_cdf: The normalized cumulative distribution function
        :rtype: array
        """
        # Get the cumulative sum of the elements
        cdf = histogram.cumsum()

        # Normalize the cdf
        normalized_cdf = cdf / float(cdf.max())

        return normalized_cdf

    def calculate_lookup(src_cdf, ref_cdf):
        """
        This method creates the lookup table
        :param array src_cdf: The cdf for the source image
        :param array ref_cdf: The cdf for the reference image
        :return: lookup_table: The lookup table
        :rtype: array
        """
        lookup_table = np.zeros(256)
        lookup_val = 0
        for src_pixel_val in range(len(src_cdf)):
            lookup_val
            for ref_pixel_val in range(len(ref_cdf)):
                if ref_cdf[ref_pixel_val] >= src_cdf[src_pixel_val]:
                    lookup_val = ref_pixel_val
                    break
            lookup_table[src_pixel_val] = lookup_val
        return lookup_table

    # Split the images into the different color channels
    src_hist,  _ = np.histogram(src_image.flatten(), bins)

    # Compute the normalized cdf for the source and reference image
    src_cdf = calculate_cdf(src_hist)
    ref_cdf = calculate_cdf(ref_histogram)

    # Make a separate lookup table for each color
    lookup_table = calculate_lookup(src_cdf, ref_cdf)

    # Use the lookup function to transform the colors of the original
    # source image
    src_after_transform = cv2.LUT(src_image, lookup_table)
    image_after_matching = cv2.convertScaleAbs(src_after_transform)

    return image_after_matching

def get_channel_stats(img):
    img_stats = [None] * 5
    img_stats[0] = np.percentile(img, 1)
    img_stats[1] = np.percentile(img, 5)
    img_stats[2] = np.mean(img)
    img_stats[3] = np.percentile(img, 95)
    img_stats[4] = np.percentile(img, 99)

    return np.array(img_stats)

def norm_img_stats(img, target):
    """Normalize an image

    Image will be normalized to have same stats as `target_stats`

    Based on method in
    "A nonlinear mapping approach to stain normalization in digital histopathology
    images using image-specific color deconvolution.", Khan et al. 2014

    Assumes that `img` values range between 0-255

    """

    target_stats = get_channel_stats(target)
    src_stats_flat = get_channel_stats(img)

    # Avoid duplicates and keep in ascending order
    lower_knots = np.array([0])
    upper_knots = np.array([300, 350, 400, 450])
    src_stats_flat = np.hstack([lower_knots, src_stats_flat, upper_knots]).astype(float)
    target_stats_flat = np.hstack([lower_knots, target_stats, upper_knots]).astype(float)

    # Add epsilon to avoid duplicate values
    eps = 10*np.finfo(float).resolution
    eps_array = np.arange(len(src_stats_flat)) * eps
    src_stats_flat = src_stats_flat + eps_array
    target_stats_flat = target_stats_flat + eps_array

    # Make sure src stats are in ascending order
    src_order = np.argsort(src_stats_flat)
    src_stats_flat = src_stats_flat[src_order]
    target_stats_flat = target_stats_flat[src_order]

    cs = Akima1DInterpolator(src_stats_flat, target_stats_flat)
    normed_img = cs(img.reshape(-1)).reshape(img.shape)

    if img.dtype == np.uint8:
        normed_img = np.clip(normed_img, 0, 255)

    return normed_img

# def norm_img_stats(img, target_stats):
#     """Normalize an image

#     Image will be normalized to have same stats as `target_stats`

#     Based on method in
#     "A nonlinear mapping approach to stain normalization in digital histopathology
#     images using image-specific color deconvolution.", Khan et al. 2014

#     """

#     eps = np.finfo(float).resolution
#     target_stats = get_channel_stats(target_stats)
#     src_stats_flat = get_channel_stats(img)

#     # Avoid duplicates and keep in ascending order
#     lower_knots = np.array([0])
#     upper_knots = np.array([300, 350, 400, 450])
#     src_stats_flat = np.hstack([lower_knots, src_stats_flat, upper_knots])
#     target_stats_flat = np.hstack([lower_knots, target_stats, upper_knots])

#     eps_array = np.arange(len(src_stats_flat)) * eps
#     src_stats_flat = np.sort(src_stats_flat + eps_array)
#     target_stats_flat = np.sort(target_stats_flat + eps_array)

#     # sp = csaps.CubicSmoothingSpline(src_stats_flat, target_stats_flat, smooth=0.995)
#     cs = Akima1DInterpolator(src_stats_flat, target_stats_flat)
#     normed_img = cs(img.reshape(-1)).reshape(img.shape)

#     if img.dtype == np.uint8:
#         normed_img = np.clip(normed_img, 0, 255)

#     return normed_img

max_stars_repo_name: MathOnco/valis

max_issues_repo_name: MathOnco/valis

max_forks_repo_name: MathOnco/valis

reason: import numpy,from scipy


text:
from __future__ import absolute_import, division, print_function

class small_cell_orientation:
 """ Class for determining an orientation matrix given a set of reflections """

 def __init__(self,miller_indices, u_vectors, sym):
  """
  @param miller_indices indexed miller indices
  @param u_vectors reciprocal space vectors for the given miller indices
  @sym cctbx symmetry object
  """
  from libtbx import adopt_init_args
  adopt_init_args(self,locals())
  # assumes miller_indices and u-vectors are flex.vec3_doubles
  # sym is a cctbx.crystal.symmetry

 def unrestrained_setting(self):
  """ Calculate the basis vectors from N spots using numpy. This is equation 5 in
  Brewster et. al 2015.
  @return a cctbx crystal_orientation object
  """
  from dials.array_family import flex
  from scitbx.matrix import sqr
  import numpy as np

  if len(self.miller_indices) < 3:
    return None

  N = self.miller_indices.size()

  hkl = self.miller_indices.as_double()
  hkl.reshape(flex.grid((N,3)))
  hkl = hkl.as_numpy_array()

  xyz = self.u_vectors.as_double()
  xyz.reshape(flex.grid((N,3)))
  xyz = xyz.as_numpy_array()

  try:
    result = np.linalg.lstsq(hkl,xyz)
  except Exception as e:
    print("Exception while calculating basis vectors: %s"%e.message)
    return None

  solution,self.residuals,rank,singular = result[0],result[1],result[2],result[3]
  if len(self.residuals) == 0:
    self.residuals = [0,0,0] # happens when only 3 spots in the max clique

  print("Summed squared residuals of x,y,z for %d spots in 1/angstroms: %.7f, %.7f, %.7f"%(N,self.residuals[0],self.residuals[1],self.residuals[2]))
  Amatrix = sqr(solution.flatten()).transpose()

  from cctbx import crystal_orientation
  ori = crystal_orientation.crystal_orientation(Amatrix, crystal_orientation.basis_type.reciprocal)
  return ori

if __name__=="__main__":
  import libtbx.easy_pickle as ep
  data = ep.load("r0013_shot-s00-20130311223605649.example")
  miller_indices,u_vectors,symmetry = data[0],data[1],data[2]
  S = small_cell_orientation(miller_indices, u_vectors, symmetry)
  print(S.unrestrained_setting())


text:
import json
import os
import argparse

import numpy as np

if __name__ == '__main__':

    os.chdir(os.path.join(os.path.dirname(__file__), '..'))
    parser = argparse.ArgumentParser()
    parser.add_argument('--experiment')
    parser.add_argument('--start')
    parser.add_argument('--end')
    parser.add_argument('--step')
    parser.add_argument('--down')
    args = parser.parse_args()
    print args

    exp = args.experiment
    start = float(args.start)
    end = float(args.end)
    step = float(args.step)
    down = float(args.down)

    POWERS=np.arange(start, end+step/2., step)
    F1 = {}
    EM = {}
    best_f1 = 0.0
    best_em = 0.0
    #for p in POWERS:
    #    print 'Testing power={}'.format(p)
    #    if p not in F1:
    #        os.system(('python code/main.py --ans_len_dist_power={} '
    #                   '--mode=official_eval '
    #                   '--json_in_path=data/dev-v1.1.json '
    #                   '--ckpt_load_dir=experiments/{}/best_checkpoint')
    #                   .format(p, exp))
    #        os.system('python code/evaluate.py data/dev-v1.1.json '
    #                  'predictions.json > performance.json')
    #        with open('performance.json', 'r') as f:
    #            scores = json.load(f)
    #    F1[p] = scores['f1']
    #    EM[p] = scores['exact_match']
    #    if F1[p] > F1[best_f1]:
    #        best_f1 = p
    #    if EM[p] > EM[best_em]:
    #        best_em = p
    #    print 'Best F1: {} @ {}'.format(F1[best_f1], best_f1)
    #    print 'Best EM: {} @ {}'.format(EM[best_em], best_em)
    #with open('f1-1.json', 'w') as f:
    #    json.dump(F1, f)
    #with open('em-1.json', 'w') as f:
    #    json.dump(EM, f)

    POWERS=np.arange(best_f1-step+step/down, best_f1+step, step/down)
    for p in POWERS:
        print 'Testing power={}'.format(p)
        if p in F1: continue
        os.system(('python code/main.py --ans_len_dist_power={} '
                   '--mode=official_eval '
                   '--json_in_path=data/dev-v1.1.json '
                   '--ckpt_load_dir=experiments/{}/best_checkpoint')
                   .format(p, exp))
        os.system('python code/evaluate.py data/dev-v1.1.json '
                  'predictions.json > performance.json')
        with open('performance.json', 'r') as f:
            scores = json.load(f)
        F1[p] = scores['f1']
        EM[p] = scores['exact_match']
        if F1[p] > F1[best_f1]:
            best_f1 = p
        if EM[p] > EM[best_em]:
            best_em = p
        print 'Best F1: {} @ {}'.format(F1[best_f1], best_f1)
        print 'Best EM: {} @ {}'.format(EM[best_em], best_em)
    with open('f1-2.json', 'w') as f:
        json.dump(F1, f)
    with open('em-2.json', 'w') as f:
        json.dump(EM, f)

    POWERS=np.arange(best_em-step+step/down, best_em+step, step/down)
    for p in POWERS:
        print 'Testing power={}'.format(p)
        if p in EM: continue
        os.system(('python code/main.py --ans_len_dist_power={} '
                   '--mode=official_eval '
                   '--json_in_path=data/dev-v1.1.json '
                   '--ckpt_load_dir=experiments/{}/best_checkpoint')
                   .format(p, exp))
        os.system('python code/evaluate.py data/dev-v1.1.json '
                  'predictions.json > performance.json')
        with open('performance.json', 'r') as f:
            scores = json.load(f)
        F1[p] = scores['f1']
        EM[p] = scores['exact_match']
        if F1[p] > F1[best_f1]:
            best_f1 = p
        if EM[p] > EM[best_em]:
            best_em = p
        print 'Best F1: {} @ {}'.format(F1[best_f1], best_f1)
        print 'Best EM: {} @ {}'.format(EM[best_em], best_em)
    with open('f1-3.json', 'w') as f:
        json.dump(F1, f)
    with open('em-3.json', 'w') as f:
        json.dump(EM, f)


text:
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.widgets import Slider
from dataset import read_dataset

def visualize_covisibility(matched_frames):
    def on_click(event):
        if event.xdata is None or event.ydata is None:
            return

        i = int(event.ydata)
        j = int(event.xdata)
        visualize_matches(matched_frames[i], matched_frames[j])

    num_frames = len(matched_frames)

    covisibility_mat = np.zeros([num_frames, num_frames])

    for i in range(num_frames):
        for j in range(i, num_frames):
            covisibility_mat[i, j] = matched_frames[i].number_of_matches(matched_frames[j])
            covisibility_mat[j, i] = matched_frames[i].number_of_matches(matched_frames[j])

    # Use Qt 5 backend in visualisation.
    matplotlib.use('qt5agg')

    # Create axis.
    fig = plt.figure()
    ax = plt.axes()
    artist = ax.imshow(covisibility_mat)
    fig.colorbar(artist, ax=ax)
    fig.canvas.mpl_connect('button_press_event', on_click)
    ax.set_title("Number of matches between frames.\nPress an element to show matches")
    plt.show()

def visualize_matches(frame0, frame1):
    kp0, _, kp1, _ = frame0.extract_correspondences_with_frame(frame1)
    draw_matches(frame0.load_image(), kp0, frame1.load_image(), kp1)

def draw_matches(img1, kp1, img2, kp2):
    if len(img1.shape) == 3:
        new_shape = (max(img1.shape[0], img2.shape[0]), img1.shape[1] + img2.shape[1], img1.shape[2])
    else:
        new_shape = (max(img1.shape[0], img2.shape[0]), img1.shape[1] + img2.shape[1])

    new_img = np.zeros(new_shape, type(img1.flat[0]))
    # Place images onto the new image.
    new_img[0:img1.shape[0], 0:img1.shape[1]] = img1
    new_img[0:img2.shape[0], img1.shape[1]:img1.shape[1] + img2.shape[1]] = img2

    # Create matches axis.
    fig, ax = plt.subplots()
    plt.subplots_adjust(left=0, right=1, top=1, bottom=0.1, hspace=0, wspace=0)
    ax.imshow(new_img)

    match_artists = []
    for i in range(kp1.shape[1]):
        match_line = np.c_[kp1[:, i], kp2[:, i] + np.array([img1.shape[1], 0])]
        match_artists += ax.plot(match_line[0, :], match_line[1, :], '-o', fillstyle='none')

    ax.set_axis_off()

    # Add slider.
    widget_color = 'lightgoldenrodyellow'
    slider = Slider(plt.axes([0.2, 0.05, 0.7, 0.03], facecolor=widget_color),
                    'Sample every ...', 1, 100, valinit=1, valstep=5)

    def update_matches(val=None):
        n = int(slider.val)
        for i, artist in enumerate(match_artists):
            artist.set_visible((i % n) == 0)

    slider.on_changed(update_matches)
    fig.show()

if __name__ == '__main__':
    matched_frames, _ = read_dataset()
    visualize_covisibility(matched_frames)


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#===============================================================================
#
# Copyright (c) 2017 <> All Rights Reserved
#
#
# File: /Users/hain/ai/book-of-qna-code/ch2/crf/crf_sample.py
# Author: Hai Liang Wang
# Date: 2018-06-15:12:31:38
# Borrow from
# https://github.com/TeamHG-Memex/sklearn-crfsuite/blob/master/docs/CoNLL2002.ipynb
#
#===============================================================================

"""

"""
from __future__ import print_function
from __future__ import division

__copyright__ = "Copyright (c) 2017 . All Rights Reserved"
__author__    = "Hai Liang Wang"
__date__      = "2018-06-15:12:31:38"

import os
import sys
curdir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(curdir)

if sys.version_info[0] < 3:
    sys.setdefaultencoding("utf-8")
    reload(sys)
    # raise "Must be using Python 3"
else:
    unicode = str

from itertools import chain

import nltk
import sklearn
import scipy.stats
from sklearn.metrics import make_scorer
from sklearn.grid_search import RandomizedSearchCV

import sklearn_crfsuite
from sklearn_crfsuite import scorers
from sklearn_crfsuite import metrics

train_sents = list(nltk.corpus.conll2002.iob_sents('esp.train'))
test_sents = list(nltk.corpus.conll2002.iob_sents('esp.testb'))

def word2features(sent, i):
    word = sent[i][0]
    postag = sent[i][1]

    features = {
        'bias': 1.0,
        'word.lower()': word.lower(),
        'word[-3:]': word[-3:],
        'word[-2:]': word[-2:],
        'word.isupper()': word.isupper(),
        'word.istitle()': word.istitle(),
        'word.isdigit()': word.isdigit(),
        'postag': postag,
        'postag[:2]': postag[:2],
    }
    if i > 0:
        word1 = sent[i-1][0]
        postag1 = sent[i-1][1]
        features.update({
            '-1:word.lower()': word1.lower(),
            '-1:word.istitle()': word1.istitle(),
            '-1:word.isupper()': word1.isupper(),
            '-1:postag': postag1,
            '-1:postag[:2]': postag1[:2],
        })
    else:
        features['BOS'] = True

    if i < len(sent)-1:
        word1 = sent[i+1][0]
        postag1 = sent[i+1][1]
        features.update({
            '+1:word.lower()': word1.lower(),
            '+1:word.istitle()': word1.istitle(),
            '+1:word.isupper()': word1.isupper(),
            '+1:postag': postag1,
            '+1:postag[:2]': postag1[:2],
        })
    else:
        features['EOS'] = True

    return features

def sent2features(sent):
    return [word2features(sent, i) for i in range(len(sent))]

def sent2labels(sent):
    return [label for token, postag, label in sent]

def sent2tokens(sent):
    return [token for token, postag, label in sent]

X_train = [sent2features(s) for s in train_sents]
y_train = [sent2labels(s) for s in train_sents]

X_test = [sent2features(s) for s in test_sents]
y_test = [sent2labels(s) for s in test_sents]

crf = sklearn_crfsuite.CRF(
    algorithm='lbfgs',
    c1=0.1,
    c2=0.1,
    max_iterations=100,
    all_possible_transitions=True
)
crf.fit(X_train, y_train)

labels = list(crf.classes_)
labels.remove('O')

y_pred = crf.predict(X_test)
metrics.flat_f1_score(y_test, y_pred,
                      average='weighted', labels=labels)

# group B and I results
sorted_labels = sorted(
    labels,
    key=lambda name: (name[1:], name[0])
)
print(metrics.flat_classification_report(
    y_test, y_pred, labels=sorted_labels, digits=3
))

# define fixed parameters and parameters to search
crf = sklearn_crfsuite.CRF(
    algorithm='lbfgs',
    max_iterations=100,
    all_possible_transitions=True
)
params_space = {
    'c1': scipy.stats.expon(scale=0.5),
    'c2': scipy.stats.expon(scale=0.05),
}

# use the same metric for evaluation
f1_scorer = make_scorer(metrics.flat_f1_score,
                        average='weighted', labels=labels)

# search
rs = RandomizedSearchCV(crf, params_space,
                        cv=3,
                        verbose=1,
                        n_jobs=-1,
                        n_iter=50,
                        scoring=f1_scorer)
rs.fit(X_train, y_train)

# crf = rs.best_estimator_
print('best params:', rs.best_params_)
print('best CV score:', rs.best_score_)
print('model size: {:0.2f}M'.format(rs.best_estimator_.size_ / 1000000))

crf = rs.best_estimator_
y_pred = crf.predict(X_test)
print(metrics.flat_classification_report(
    y_test, y_pred, labels=sorted_labels, digits=3
))

from collections import Counter

def print_transitions(trans_features):
    for (label_from, label_to), weight in trans_features:
        print("%-6s -> %-7s %0.6f" % (label_from, label_to, weight))

print("Top likely transitions:")
print_transitions(Counter(crf.transition_features_).most_common(20))

print("\nTop unlikely transitions:")
print_transitions(Counter(crf.transition_features_).most_common()[-20:])

def print_state_features(state_features):
    for (attr, label), weight in state_features:
        print("%0.6f %-8s %s" % (weight, label, attr))

print("Top positive:")
print_state_features(Counter(crf.state_features_).most_common(30))

print("\nTop negative:")
print_state_features(Counter(crf.state_features_).most_common()[-30:])

max_stars_repo_name: thundercrawl/book-of-qna-code

max_issues_repo_name: helloqorld/book-of-qna-code

max_forks_repo_name: helloqorld/book-of-qna-code

reason: import scipy


text:
import cv2
import numpy as np

img = cv2.imread("./data/pup.jpg")
num_rows, num_cols = img.shape[:2]

print(num_rows, num_cols)

translation_matrix = np.float32([[1,0,70], [0,1,110]])

img_translation = cv2.warpAffine(img, translation_matrix, (num_cols,num_rows))

cv2.imshow("Translation", img_translation)
cv2.waitKey()


text:
import errno
import os
import sqlite3
import warnings
from enum import Enum

import numpy as np

class ReplayBuffer(object):

    def __init__(self, maxlen, dim_observations=None, dim_actions=None,
                 path_to_db_read=None, path_to_db_write=None, write=False, save_interval=None):
        self.maxlen = maxlen
        if not save_interval:
            self.save_interval = maxlen
        else:
            self.save_interval = min(save_interval, maxlen)
            if save_interval > maxlen:
                warnings.warn("You can not set the buffer's saving interval greater than the buffer size.")
        self.number_of_samples_seen = 0
        self.last_saved_at = 0
        self.index = 0
        self.length = 0
        self.write = write
        self.buffer_name = ""
        self.path_to_db_read = path_to_db_read
        self.path_to_db_write = path_to_db_write
        self.dim_observations = dim_observations
        self.dim_actions = dim_actions
        self.dim_one_sample = 2 * self.dim_observations + self.dim_actions + 4
        self.buf = np.empty(shape=(maxlen, self.dim_one_sample), dtype=np.float)

        if self.write and not self.path_to_db_write:
            raise TypeError("You can not write buffer data if not path to the database is defined.")

        if (self.dim_observations is None or self.dim_actions is None) and self.path_to_db_read:
            raise TypeError("You can not read, if no dimensions for the observations and actions are specified.")

        # check if the db exists before reading
        if self.path_to_db_read:
            self.path_to_db_read = os.path.join(self.path_to_db_read, self.__class__.__name__)
            if not os.path.exists(self.path_to_db_read):
                raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.path_to_db_read)

        # read data into the buffer if needed
        if self.path_to_db_read:
            self.read_buffer(n_samples=maxlen)

        # check if we need to save the buffer during training
        if self.write:
            self.path_to_db_write = os.path.join(self.path_to_db_write,  self.__class__.__name__)

            # create connection to database and create if it does not exist yet
            self.conn = sqlite3.connect(self.path_to_db_write)
            self.c = self.conn.cursor()

            # create the table "data" of it does not exist
            sql_create_data_table = """ CREATE TABLE IF NOT EXISTS data (%s); """ % self.create_column_names()
            self.c.execute(sql_create_data_table)
            self.conn.commit()

    def __del__(self):
        # save the buffer one last time and close connection
        if self.write:
            self.save_buffer()
            self.conn.close()

    def append(self, state, action, reward, next_state, done, idx_episode):
        """ Appends the replay buffer with a single sample """
        data = np.concatenate((np.array([self.number_of_samples_seen]), np.array([idx_episode]), np.array(state),
                               np.array(action), np.array([reward]), np.array(next_state), np.array([done])))
        self.buf[self.index] = data
        self.length = min(self.length + 1, self.maxlen)
        self.index = (self.index + 1) % self.maxlen
        self.number_of_samples_seen += 1
        # check if buffer needs to be saved
        if self.write and self.number_of_samples_seen % self.save_interval == 0:
            self.save_buffer()

    def sample(self, batch_size, with_replacement=True):
        """ Samples a random batch of size batch_size from the replay buffer """
        if with_replacement:
            indices = np.random.randint(self.length, size=batch_size)  # faster
        else:
            indices = np.random.permutation(self.length)[:batch_size]
        return self.buf[indices]

    def sample_batch(self, batch_size):
        """ Splits a randomly sampled batch into states, actions, rewards, next_states and dones """
        data = self.sample(batch_size)
        ind = 2     # first entry is sample_id and second is episode_id
        states = data[:, ind:(self.dim_observations+ind)]
        ind += self.dim_observations
        actions = data[:, ind:(ind+self.dim_actions)]
        ind += self.dim_actions
        rewards = data[:, ind:(ind+1)]
        ind += 1
        next_states = data[:, ind:(ind+self.dim_observations)]
        ind += self.dim_observations
        dones = data[:, ind:(ind+1)]
        return states, actions, rewards.reshape(-1, 1), next_states, dones.reshape(-1, 1)

    def save_buffer(self):
        """ Saves all samples in the replay buffer, which were appended since last saving """
        print("\nSaving replay buffer ...")
        start = self.last_saved_at % self.maxlen
        end = self.number_of_samples_seen % self.maxlen
        if start >= end:
            start = start - end
            end = self.maxlen
        data = self.buf[start:end]
        data_list_of_tuples = map(tuple, data.tolist())
        self.c.executemany('INSERT INTO data VALUES %s' % self.create_input_str(), data_list_of_tuples)
        self.conn.commit()
        self.last_saved_at = self.number_of_samples_seen
        print("Finished saving.")

    def read_buffer(self, n_samples):
        """ Returns the last n_samples samples from the replay buffer """
        print("\nReading Buffer from %s ..." % self.path_to_db_read)
        connection = sqlite3.connect(self.path_to_db_read)
        cursor = connection.cursor()
        column_names = self.get_column_names()
        input_str = "SELECT %s FROM (SELECT %s FROM data ORDER BY oid DESC Limit %d) ORDER BY RANDOM()" %\
                    (column_names, column_names, min(n_samples, self.maxlen))
        cursor.execute(input_str)
        data = np.array(cursor.fetchall())
        if data.shape[0] == 0:
            raise ValueError("Buffer is empty! Please check.")
        self.buf[0:data.shape[0], :] = data
        self.length = data.shape[0]
        self.number_of_samples_seen = self.length
        self.index = self.length - 1
        self.number_of_samples_seen = self.length
        connection.close()
        print("Finished reading buffer --> %d samples imported.\n" % self.length)

    def create_column_names(self):
        """ Creates a string for defining the columns in a table of a sql database """
        column_names = "Sample_ID int,"
        column_names += "Episode_ID int,"
        for var in range(self.dim_observations):
            column_names += "State_" + str(var) + " float,"
        for var in range(self.dim_actions):
            column_names += "Action_" + str(var) + " float,"
        column_names += "Reward float,"
        for var in range(self.dim_observations):
            column_names += "Next_State_" + str(var) + " float,"
        column_names += "Done float"
        return column_names

    def get_column_names(self):
        """ Creates a string for reading the columns in a table of a sql database
        Note: Priorities are not included! """
        column_names = "Sample_ID, "
        column_names += "Episode_ID, "
        for var in range(self.dim_observations):
            column_names += "State_" + str(var) + ", "
        for var in range(self.dim_actions):
            column_names += "Action_" + str(var) + ", "
        column_names += "Reward, "
        for var in range(self.dim_observations):
            column_names += "Next_State_" + str(var) + ", "
        column_names += "Done "
        return column_names

    def create_input_str(self):
        """ Creates a string for inserting the buffer data into a sql table """
        input_str = "("
        for var in range(self.dim_one_sample - 1):
            input_str += "?,"
        input_str += "?)"
        return input_str