text:
from __future__ import annotations
from abc import ABC
from datetime import datetime, timedelta
from typing import Union, List, Any
import logging
from copy import deepcopy
import numpy as np
from collections import OrderedDict

class MeasurementBase(ABC):
    """
    Base class for all measurement types (Counts and Spectrum so far).

    Attributes
    ----------
    live_time : float
        The live time of the measurement
    real_time : float
        The real time of the measurement
    name : str
        Optional name to attach to the measurement.
    comment : str
        Optional comment to attach to the measurement.
    start_time : datetime
        Optional time at which the measurement began.
    log : logging.Logger
        Logger for logging loggable log logs.

    Methods
    -------
    end_time() -> datetime or None
        Property returning the end time of the measurement. "The end times are nigh."
    copy_me() -> Measurement
        Method which returns a deep copy of the measurement.
    resample(factor: float = 1.0) -> Measurement
        Must be implemented by subclasses.
    subtract_background(background: Measurement) -> Measurement
        Method which subtracts a background signal from the measurement.
    consecutive_addition(other: MeasurementBase) -> Measurement
        Must be implemented by subclasses.
    simultaneous_addition(other: MeasurementBase) -> Measurement
        Must be implemented by subclasses.
    consecutive_subtraction(other: MeasurementBase) -> Measurement
        Must be implemented by subclasses.
    simultaneous_subtraction(other: MeasurementBase) -> Measurement
        Must be implemented by subclasses.
    """
    def __init__(self,
                 live_time: float = 1.0,
                 real_time: float = 1.0,
                 name: str = '',
                 comment: str = '',
                 start_time: Union[datetime, None] = None):
        """
        Parameters
        ----------
        live_time : float
            The live time of the measurement
        real_time : float
            The real time of the measurement
        name : str
            Optional name to attach to the measurement.
        comment : str
            Optional comment to attach to the measurement.
        start_time : datetime
            Optional time at which the measurement began.
        """
        self.live_time = live_time
        self.real_time = real_time
        self.name = name
        self.comment = comment
        self.start_time = start_time
        self.log = logging.getLogger(f"{self.__class__.__name__}:{self.name}")
        self.log.debug(f"Created {self.__class__.__name__}({self.name})")

    @property
    def end_time(self) -> Union[datetime, None]:
        """
        Property which returns the time at which the measurement ended, if the start time is known, otherwise it
        returns None.

        Returns
        -------
        datetime
        """
        return None if self.start_time is None else self.start_time + timedelta(seconds=self.real_time)

    def copy_me(self):
        """
        Method which returns a deep copy of the measurement.

        Returns
        -------
        CopiedMeasurement
        """
        return deepcopy(self)

    def resample(self, factor: float = 1.0) -> MeasurementBase:
        raise NotImplementedError

    def subtract_background(self, background: MeasurementBase):
        """
        Method which subtracts a background signal from the measurement.

        Parameters
        ----------
        background : MeasurementBase

        Returns
        -------
        BackgroundSubtractedMeasurement
        """
        return self.simultaneous_subtraction(background)

    def consecutive_addition(self, other: MeasurementBase):
        raise NotImplementedError

    def simultaneous_addition(self, other: MeasurementBase):
        raise NotImplementedError

    def consecutive_subtraction(self, other: MeasurementBase):
        raise NotImplementedError

    def simultaneous_subtraction(self, other: MeasurementBase):
        raise NotImplementedError

class TickedMeasurement(MeasurementBase, ABC):
    """
    Base class for all ticked measurement types (TickedCounts and TickedSpectrum so far).

    Attributes
    ----------
    tick_times : Array[datetime]
        Numpy array of datetime objects, labelling the times at which the ticks occurred.
    live_times : Array[float]
        Numpy array of floats, determining the live times of each tick.
    real_times : Array[float]
        Numpy array of floats, determining the real times of each tick.
    live_time : float
        The total live time of the measurement
    real_time : float
        The total real time of the measurement
    name : str
        Optional name to attach to the measurement.
    comment : str
        Optional comment to attach to the measurement.
    start_time : datetime
        Optional time at which the measurement began.
    log : logging.Logger
        Logger for logging loggable log logs.

    Methods
    -------
    n_ticks() -> int
        Property which returns the number of ticks in the measurement.
    end_time() -> datetime or None
        Property returning the end time of the measurement. "The end times are nigh."
    copy_me() -> Measurement
        Method which returns a deep copy of the measurement.
    resample(factor: float = 1.0) -> Measurement
        Must be implemented by subclasses.
    subtract_background(background: Measurement) -> Measurement
        Method which subtracts a background signal from the measurement.
    consecutive_addition(other: MeasurementBase) -> Measurement
        Must be implemented by subclasses.
    simultaneous_addition(other: MeasurementBase) -> Measurement
        Must be implemented by subclasses.
    consecutive_subtraction(other: MeasurementBase) -> Measurement
        This method cannot be used for ticked types. Raises TypeError always.
    simultaneous_subtraction(other: MeasurementBase) -> Measurement
        Must be implemented by subclasses.
    get_single_tick(tick_number: int) -> Measurement
        Must be implemented by subclasses.
    sum_ticks(start: datetime = None, end: datetime = None) -> Measurement
        Must be implemented by subclasses.
    """
    def __init__(self,
                 tick_times: np.ndarray,
                 live_times: np.ndarray,
                 real_times: np.ndarray,
                 name: str = '',
                 comment: str = '',
                 start_time: Union[datetime, None] = None):
        """
        Parameters
        ----------
        tick_times : Array[datetime]
            Numpy array of datetime objects, labelling the times at which the ticks occurred.
        live_times : Array[float]
            Numpy array of floats, determining the live times of each tick.
        real_times : Array[float]
            Numpy array of floats, determining the real times of each tick.
        name : str
            Optional name to attach to the measurement.
        comment : str
            Optional comment to attach to the measurement.
        start_time : datetime
            Optional time at which the measurement began.
        """
        if start_time is None:
            start_time = tick_times[0]
        super().__init__(live_time=live_times.sum(),
                         real_time=real_times.sum(),
                         name=name,
                         comment=comment,
                         start_time=start_time)
        self.tick_times = tick_times
        self.live_times = live_times
        self.real_times = real_times

    @property
    def n_ticks(self) -> int:
        """
        Property which returns the number of ticks in the measurement.

        Returns
        -------
        int
        """
        return self.tick_times.size

    def get_single_tick(self, tick_number: int):
        raise NotImplementedError

    def sum_ticks(self, start: Union[datetime, None] = None, end: Union[datetime, None] = None):
        raise NotImplementedError

    def consecutive_subtraction(self, other: MeasurementBase):
        raise TypeError('Consecutive subtraction makes no sense for ticked types.')

class ArrayBase:
    """
    Base class for all Array types (CountsArray, SpectrumArray, TickedCountsArray and TickedSpectrumArray so far).

    Attributes
    ----------
    elements : OrderedDict[str, Measurement]
        OrderedDict, mapping measurements by their names.

    Methods
    -------
    names() -> List[str]
        Property which returns a list of the measurement names which are keys to the mapping.
    number_of_members() -> int
        Property which returns the number of members of the array.
    start_times() -> List[datetime]
        Property which returns a list of datetime instances which give the times at which each member of the array
        began to be collected.
    contains_all(names: List[str]) -> bool
        Method which returns True if all the strings in the argument match the names of elements in the array.
    names_match(other: ArrayBase) -> bool
        Method which returns True if all the names of the other array are present in this array, and vice-versa.
    consecutive_addition(other: ArrayBase) -> ArrayBase
        Method which returns the consecutive sum of the CountsArray with another Array.
    simultaneous_addition(other: ArrayBase) -> ArrayBase
        Method which returns the simultaneous sum of the CountsArray with another Array.
    consecutive_subtraction(other: ArrayBase) -> ArrayBase
        Method which returns the consecutive subtraction of another CountsArray from the Array.
    simultaneous_subtraction(other: ArrayBase) -> ArrayBase:
        Method which returns the simultaneous subtraction of another CountsArray from the Array.
    subtract_backgrounds(self, backgrounds: ArrayBase) -> ArrayBase:
        Method which subtracts the background from each member of the array, using the array of background counts
        supplied as the argument.
    simultaneous_sum() -> MeasurementBase
        Must be implemented by subclasses.
    consecutive_sum() -> MeasurementBase
        Must be implemented by subclasses.
    resample(factor: float) -> ArrayBase
        Method which returns a new array with each member resampled after scaling by the factor parameter.
    masked_array(names: List[str]) -> ArrayBase
        Method which returns a new array consisting only of members with the names in the given list of names.
    """
    def __init__(self, items):
        self.elements = OrderedDict([(i.name, i) for i in items])

    def __getitem__(self, item: str):
        return self.elements[item]

    def get(self, key: str, default_val: Any = None):
        return self.elements.get(key, default_val)

    def __setitem__(self, key: str, value):
        self.elements[key] = value

    def __contains__(self, item: str) -> bool:
        return item in self.elements

    def __iter__(self):
        for k in self.elements:
            yield self.elements[k]

    @property
    def names(self) -> List[str]:
        """
        Property which returns a list of the 'names' attributes for each member of the array.

        Returns
        -------
        List[str]
        """
        return list(self.elements.keys())

    @property
    def number_of_members(self) -> int:
        """
        Property which returns the number of members of the array.

        Returns
        -------
        int
        """
        return len(self.elements)

    @property
    def start_times(self) -> List[datetime]:
        """
        Property which returns a list of the 'start_time' attributes for each member of the array. The values are
        datetime objects or None.

        Returns
        -------
        List[Union[str, None]]
        """
        return [e.start_time for e in self]

    def contains_all(self, names: List[str]) -> bool:
        """
        Method which returns True if all the strings in the argument match the names of elements in the array.

        Parameters
        ----------
        names : List[str]
            List of names to match.

        Returns
        -------
        bool
        """
        return all([name in self for name in names])

    def names_match(self, other: ArrayBase):
        """
        Method which returns True if all the names of the other array are present in this array, and vice-versa.

        Parameters
        ----------
        other : Array

        Returns
        -------
        bool
        """
        return self.contains_all(other.names) and other.contains_all(self.names)

    def consecutive_addition(self, other: ArrayBase):
        """
        Method which returns the consecutive sum of the CountsArray with another Array.

        Parameters
        ----------
        other : Array
            The Array instance to add to this one.

        Returns
        -------
        Array
        """
        assert self.names_match(other)
        return self.__class__([d.consecutive_addition(other[d.name]) for d in self])

    def simultaneous_addition(self, other: ArrayBase):
        """
        Method which returns the simultaneous sum of the CountsArray with another Array.

        Parameters
        ----------
        other : Array
            The Array instance to add to this one.

        Returns
        -------
        Array
        """
        assert self.names_match(other)
        return self.__class__([d.simultaneous_addition(other[d.name]) for d in self])

    def consecutive_subtraction(self, other: ArrayBase):
        """
        Method which returns the consecutive subtraction of another CountsArray from the Array.

        Parameters
        ----------
        other : Array
            The Array instance to subtract from this one.

        Returns
        -------
        Array
        """
        assert self.names_match(other)
        return self.__class__([d.consecutive_subtraction(other[d.name]) for d in self])

    def simultaneous_subtraction(self, other: ArrayBase):
        """
        Method which returns the simultaneous subtraction of another CountsArray from the Array.

        Parameters
        ----------
        other : Array
            The Array instance to subtract from this one.

        Returns
        -------
        Array
        """
        assert self.names_match(other)
        return self.__class__([d.simultaneous_subtraction(other[d.name]) for d in self])

    def subtract_backgrounds(self, backgrounds: ArrayBase):
        """
        Method which subtracts the background from each member of the array, using the array of background counts
        supplied as the argument.

        Parameters
        ----------
        backgrounds : ArrayBase

        Returns
        -------
        Array
        """
        return self.simultaneous_subtraction(backgrounds)

    def simultaneous_sum(self):
        raise NotImplementedError

    def consecutive_sum(self):
        raise NotImplementedError

    def resample(self, factor: float):
        """
        Method which returns a new array with each member resampled after scaling by the factor parameter.

        Parameters
        ----------
        factor : float
            Scale by which the measurements will be scaled before resampling.

        Returns
        -------
        Array
        """
        return self.__class__([e.resample(factor) for e in self])

    def masked_array(self, names: List[str]):
        """
        Method which returns a new array consisting only of members with the names in the given list of names.

        Parameters
        ----------
        names : List[str]

        Returns
        -------
        Array
        """
        assert self.contains_all(names)
        return self.__class__([self[name] for name in names])

class TickedArrayBase(ArrayBase, ABC):
    """
    Base class for all TickedArray types (TickedCountsArray and TickedSpectrumArray so far).

    Attributes
    ----------
    elements : OrderedDict[str, Measurement]
        OrderedDict, mapping measurements by their names.

    Methods
    -------
    tick_times() -> Array[datetime]
        Property which returns a list of datetime instances labelling the time at which each tick occurred.
    n_ticks() -> int
        Property which returns the number of ticks in the first member of the array.
    n_ticks_all() -> List[int]
        Property which, like n_ticks, returns the number of ticks, but returns a list of values for each member, in
        case you suspect they may not all contain equal numbers.
    names() -> List[str]
        Property which returns a list of the measurement names which are keys to the mapping.
    number_of_members() -> int
        Property which returns the number of members of the array.
    start_times() -> List[datetime]
        Property which returns a list of datetime instances which give the times at which each member of the array
        began to be collected.
    contains_all(names: List[str]) -> bool
        Method which returns True if all the strings in the argument match the names of elements in the array.
    names_match(other: ArrayBase) -> bool
        Method which returns True if all the names of the other array are present in this array, and vice-versa.
    consecutive_addition(other: ArrayBase) -> ArrayBase
        Method which returns the consecutive sum of the CountsArray with another Array.
    simultaneous_addition(other: ArrayBase) -> ArrayBase
        Method which returns the simultaneous sum of the CountsArray with another Array.
    consecutive_subtraction(other: ArrayBase) -> ArrayBase
        This method makes no sense...
    simultaneous_subtraction(other: ArrayBase) -> ArrayBase:
        Method which returns the simultaneous subtraction of another CountsArray from the Array.
    subtract_backgrounds(self, backgrounds: ArrayBase) -> ArrayBase:
        Method which subtracts the background from each member of the array, using the array of background counts
        supplied as the argument.
    simultaneous_sum() -> MeasurementBase
        Must be implemented by subclasses.
    consecutive_sum() -> MeasurementBase
        Must be implemented by subclasses.
    resample(factor: float) -> ArrayBase
        Method which returns a new array with each member resampled after scaling by the factor parameter.
    masked_array(names: List[str]) -> ArrayBase
        Method which returns a new array consisting only of members with the names in the given list of names.
    get_single_tick(tick_num: int) -> ArrayBase
        Must be implemented by subclasses.
    sum_ticks(start: datetime, end: datetime) -> ArrayBase
        Must be implemented by subclasses.
    """
    def __init__(self, items: List[TickedMeasurement]):
        super(TickedArrayBase, self).__init__(items)

    @property
    def tick_times(self) -> np.ndarray:
        """
        Property which returns a list of datetime instances labelling the time at which each tick occurred (for the
        first member of the array).

        Returns
        -------
        Array[datetime]
        """
        return self[self.names[0]].tick_times

    @property
    def n_ticks(self) -> int:
        """
        Property which returns the number of ticks in the first member of the array.

        Returns
        -------
        int
        """
        return self.tick_times.size

    @property
    def n_ticks_all(self) -> List[int]:
        """
        Property which, like n_ticks, returns the number of ticks, but returns a list of values for each member, in
        case you suspect they may not all contain equal numbers.

        Returns
        -------
        List[int]
        """
        return [self[name].n_ticks for name in self.names]

    def get_single_tick(self, tick_num: int):
        raise NotImplementedError

    def sum_ticks(self, start: Union[datetime, None] = None, end: Union[datetime, None] = None):
        raise NotImplementedError

max_stars_repo_name: iandellarocca/geiger

max_issues_repo_name: iandellarocca/geiger

max_forks_repo_name: iandellarocca/geiger


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

from types import SimpleNamespace

import uuid
import numpy as np
import tensorflow as tf
from tensorflow.python.keras.backend import gradients
from tensorflow.python.keras.backend import set_session

from arch.api.utils import log_utils

LOGGER = log_utils.getLogger()

def _init_session():
    from tensorflow.python.keras import backend
    sess = backend.get_session()
    tf.get_default_graph()
    set_session(sess)
    return sess

class DenseModel(object):
    def __init__(self):
        self.input = None
        self.model_weight = None
        self.model_shape = None
        self.bias = None
        self.model = None
        self.lr = 1.0
        self.layer_config = None
        self.sess = _init_session()
        self.role = "host"
        self.activation_placeholder_name = "activation_placeholder" + str(uuid.uuid1())
        self.activation_gradient_func = None
        self.activation_func = None
        self.is_empty_model = False
        self.activation_input = None
        self.model_builder = None

    def forward_dense(self, x):
        pass

    def apply_update(self, delta):
        pass

    def get_weight_gradient(self, delta):
        pass

    def set_sess(self, sess):
        self.sess = sess

    def build(self, input_shape=None, layer_config=None, model_builder=None, restore_stage=False):
        if not input_shape:
            if self.role == "host":
                raise ValueError("host input is empty!")
            else:
                self.is_empty_model = True
                return

        self.model_builder = model_builder

        self.layer_config = layer_config

        self.model = model_builder(input_shape=input_shape,
                                   nn_define=layer_config,
                                   optimizer=SimpleNamespace(optimizer="SGD", kwargs={}),
                                   loss="keep_predict_loss",
                                   metrics=None)

        dense_layer = self.model.get_layer_by_index(0)
        if not restore_stage:
            self._init_model_weight(dense_layer)

        if self.role == "host":
            self.activation_func = dense_layer.activation
            self.__build_activation_layer_gradients_func(dense_layer)

    def export_model(self):
        if self.is_empty_model:
            return ''.encode()

        layer_weights = [self.model_weight]
        if self.bias is not None:
            layer_weights.append(self.bias)

        self.model.set_layer_weights_by_index(0, layer_weights)
        return self.model.export_model()

    def restore_model(self, model_bytes):
        if self.is_empty_model:
            return

        LOGGER.debug("model_bytes is {}".format(model_bytes))
        self.model = self.model.restore_model(model_bytes)
        self._init_model_weight(self.model.get_layer_by_index(0), restore_stage=True)

    def _init_model_weight(self, dense_layer, restore_stage=False):
        if not restore_stage:
            self.sess.run(tf.initialize_all_variables())

        trainable_weights = self.sess.run(dense_layer.trainable_weights)
        self.model_weight = trainable_weights[0]
        self.model_shape = dense_layer.get_weights()[0].shape

        if dense_layer.use_bias:
            self.bias = trainable_weights[1]

    def __build_activation_layer_gradients_func(self, dense_layer):
        shape = dense_layer.output_shape
        dtype = dense_layer.get_weights()[0].dtype

        input_data = tf.placeholder(shape=shape,
                                    dtype=dtype,
                                    name=self.activation_placeholder_name)

        self.activation_gradient_func = gradients(dense_layer.activation(input_data), input_data)

    def forward_activation(self, input_data):
        self.activation_input = input_data

        output = self.activation_func(input_data)
        if not isinstance(output, np.ndarray):
            output = self.sess.run(output)

        return output

    def backward_activation(self):
        placeholder = tf.get_default_graph().get_tensor_by_name(":".join([self.activation_placeholder_name, "0"]))
        return self.sess.run(self.activation_gradient_func,
                             feed_dict={placeholder: self.activation_input})

    def get_weight(self):
        return self.model_weight

    def get_bias(self):
        return self.bias

    def set_learning_rate(self, lr):
        self.lr = lr

    @property
    def empty(self):
        return self.is_empty_model

    @property
    def output_shape(self):
        return self.model_weight.shape[1:]

class GuestDenseModel(DenseModel):
    def __init__(self):
        super(GuestDenseModel, self).__init__()
        self.role = "guest"

    def forward_dense(self, x):
        if self.empty:
            return None

        self.input = x

        output = np.matmul(x, self.model_weight)

        return output

    def get_input_gradient(self, delta):
        if self.empty:
            return None

        error = np.matmul(delta, self.model_weight.T)

        return error

    def get_weight_gradient(self, delta):
        if self.empty:
            return None

        delta_w = np.matmul(delta.T, self.input) / self.input.shape[0]

        return delta_w

    def apply_update(self, delta):
        if self.empty:
            return None

        self.model_weight -= self.lr * delta.T

class HostDenseModel(DenseModel):
    def __init__(self):
        super(HostDenseModel, self).__init__()
        self.role = "host"

    def forward_dense(self, x):
        self.input = x
        output = x * self.model_weight

        if self.bias is not None:
            output += self.bias

        return output

    def get_input_gradient(self, delta, acc_noise):
        error = delta * (self.model_weight + acc_noise).T

        return error

    def get_weight_gradient(self, delta):
        delta_w = self.input.fast_matmul_2d(delta) / self.input.shape[0]

        return delta_w

    def update_weight(self, delta):
        self.model_weight -= delta * self.lr

    def update_bias(self, delta):
        self.bias -= np.mean(delta, axis=0) * self.lr

ext: py

max_stars_repo_name: fqiang/FATE

reason: import numpy


text:
##############################################################
### Copyright (c) 2018-present, Xuanyi Dong                ###
### Style Aggregated Network for Facial Landmark Detection ###
### Computer Vision and Pattern Recognition, 2018          ###
##############################################################
from PIL import Image
from scipy.ndimage.interpolation import zoom
from utils.file_utils import load_txt_file
import numpy as np
import copy, math

def pil_loader(path):
  # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
  with open(path, 'rb') as f:
    with Image.open(f) as img:
      return img.convert('RGB')

def remove_item_from_list(list_to_remove, item):
  assert isinstance(list_to_remove, list), 'input list is not a list'

  try:
    list_to_remove.remove(item)
  except ValueError:
    print('Warning!!!!!! Item to remove is not in the list. Remove operation is not done.')

  return list_to_remove

def anno_parser(anno_path, num_pts):
  data, num_lines = load_txt_file(anno_path)
  if data[0].find('version: ') == 0: # 300-W
    return anno_parser_v0(anno_path, num_pts)
  else:
    return anno_parser_v1(anno_path, num_pts)

def anno_parser_v0(anno_path, num_pts):
  '''
  parse the annotation for 300W dataset, which has a fixed format for .pts file
  return:
    pts: 3 x num_pts (x, y, oculusion)
  '''
  data, num_lines = load_txt_file(anno_path)
  assert data[0].find('version: ') == 0, 'version is not correct'
  assert data[1].find('n_points: ') == 0, 'number of points in second line is not correct'
  assert data[2] == '{' and data[-1] == '}', 'starting and end symbol is not correct'

  assert data[0] == 'version: 1' or data[0] == 'version: 1.0', 'The version is wrong : {}'.format(data[0])
  n_points = int(data[1][len('n_points: '):])

  assert num_lines == n_points + 4, 'number of lines is not correct'    # 4 lines for general information: version, n_points, start and end symbol
  assert num_pts == n_points, 'number of points is not correct'

  # read points coordinate
  pts = np.zeros((3, n_points), dtype='float32')
  line_offset = 3    # first point starts at fourth line
  point_set = set()
  for point_index in range(n_points):
    try:
      pts_list = data[point_index + line_offset].split(' ')       # x y format
      if len(pts_list) > 2:    # handle edge case where additional whitespace exists after point coordinates
        pts_list = remove_item_from_list(pts_list, '')
      pts[0, point_index] = float(pts_list[0])
      pts[1, point_index] = float(pts_list[1])
      pts[2, point_index] = float(1)      # oculusion flag, 0: oculuded, 1: visible. We use 1 for all points since no visibility is provided by 300-W
      point_set.add( point_index )
    except ValueError:
      print('error in loading points in %s' % anno_path)
  return pts, point_set

def anno_parser_v1(anno_path, NUM_PTS, one_base=True):
  '''
  parse the annotation for MUGSY-Full-Face dataset, which has a fixed format for .pts file
  return: pts: 3 x num_pts (x, y, oculusion)
  '''
  data, n_points = load_txt_file(anno_path)
  assert n_points <= NUM_PTS, '{} has {} points'.format(anno_path, n_points)
  # read points coordinate
  pts = np.zeros((3, NUM_PTS), dtype='float32')
  point_set = set()
  for line in data:
    try:
      idx, point_x, point_y, oculusion = line.split(' ')
      idx, point_x, point_y, oculusion = int(idx), float(point_x), float(point_y), oculusion == 'True'
      if one_base==False: idx = idx+1
      assert idx >= 1 and idx <= NUM_PTS, 'Wrong idx of points : {:02d}-th in {:s}'.format(idx, anno_path)
      pts[0, idx-1] = point_x
      pts[1, idx-1] = point_y
      pts[2, idx-1] = float( oculusion )
      point_set.add(idx)
    except ValueError:
      raise Exception('error in loading points in {}'.format(anno_path))
  return pts, point_set

def PTSconvert2str(points):
  assert isinstance(points, np.ndarray) and len(points.shape) == 2, 'The points is not right : {}'.format(points)
  assert points.shape[0] == 2 or points.shape[0] == 3, 'The shape of points is not right : {}'.format(points.shape)
  string = ''
  num_pts = points.shape[1]
  for i in range(num_pts):
    ok = False
    if points.shape[0] == 3 and bool(points[2, i]) == True:
      ok = True
    elif points.shape[0] == 2:
      ok = True

    if ok:
      string = string + '{:02d} {:.2f} {:.2f} True\n'.format(i+1, points[0, i], points[1, i])
  string = string[:-1]
  return string

def PTSconvert2box(points, expand_ratio=None):
  assert isinstance(points, np.ndarray) and len(points.shape) == 2, 'The points is not right : {}'.format(points)
  assert points.shape[0] == 2 or points.shape[0] == 3, 'The shape of points is not right : {}'.format(points.shape)
  if points.shape[0] == 3:
    points = points[:2, points[-1,:].astype('bool') ]
  elif points.shape[0] == 2:
    points = points[:2, :]
  else:
    raise Exception('The shape of points is not right : {}'.format(points.shape))
  assert points.shape[1] >= 2, 'To get the box of points, there should be at least 2 vs {}'.format(points.shape)
  box = np.array([ points[0,:].min(), points[1,:].min(), points[0,:].max(), points[1,:].max() ])
  W = box[2] - box[0]
  H = box[3] - box[1]
  assert W > 0 and H > 0, 'The size of box should be greater than 0 vs {}'.format(box)
  if expand_ratio is not None:
    box[0] = int( math.floor(box[0] - W * expand_ratio) )
    box[1] = int( math.floor(box[1] - H * expand_ratio) )
    box[2] = int( math.ceil(box[2] + W * expand_ratio) )
    box[3] = int( math.ceil(box[3] + H * expand_ratio) )
  return box

def for_generate_box_str(anno_path, num_pts, extend):
  if isinstance(anno_path, str):
    points, _ = anno_parser(anno_path, num_pts)
  else:
    points = anno_path.copy()
  box = PTSconvert2box(points, extend)
  return '{:.2f} {:.2f} {:.2f} {:.2f}'.format(box[0], box[1], box[2], box[3])

def resize_heatmap(maps, height, width, order=3):
  # maps  = np.ndarray with shape [height, width, channels]
  # order = 0 Nearest
  # order = 1 Bilinear
  # order = 2 Cubic
  assert isinstance(maps, np.ndarray) and len(maps.shape) == 3, 'maps type : {}'.format(type(maps))

  scale = tuple(np.array([height,width], dtype=float) / np.array(maps.shape[:2]))
  return zoom(maps, scale + (1,), order=order)

def analysis_dataset(dataset):
  all_values = np.zeros((3,len(dataset.datas)), dtype=np.float64)
  hs = np.zeros((len(dataset.datas),), dtype=np.float64)
  ws = np.zeros((len(dataset.datas),), dtype=np.float64)

  for index, image_path in enumerate(self.datas):
    img = pil_loader(image_path)
    ws[index] = image.size[0]
    hs[index] = image.size[1]
    img = np.array(img)
    all_values[:, index] = np.mean(np.mean(img, axis=0), axis=0).astype('float64')
  mean = np.mean(all_values, axis=1)
  std  = np.std (all_values, axis=1)
  return mean, std, ws, hs

def split_datasets(dataset, point_ids):
  sub_dataset = copy.deepcopy(dataset)
  assert len(point_ids) > 0
  assert False, 'un finished'

def convert68to49(points):
  points = points.copy()
  assert len(points.shape) == 2 and (points.shape[0] == 3 or points.shape[0] == 2) and points.shape[1] == 68, 'The shape of points is not right : {}'.format(points.shape)
  out = np.ones((68,)).astype('bool')
  out[[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,60,64]] = False
  cpoints = points[:, out]
  assert len(cpoints.shape) == 2 and cpoints.shape[1] == 49
  return cpoints

def convert68to51(points):
  points = points.copy()
  assert len(points.shape) == 2 and (points.shape[0] == 3 or points.shape[0] == 2) and points.shape[1] == 68, 'The shape of points is not right : {}'.format(points.shape)
  out = np.ones((68,)).astype('bool')
  out[[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]] = False
  cpoints = points[:, out]
  assert len(cpoints.shape) == 2 and cpoints.shape[1] == 51
  return cpoints

reason: import numpy,from scipy


text:
import numpy as np
from scipy.signal import savgol_filter
if __name__ == '__main__':
    import sys
    from os import path
    sys.path.append( path.dirname(path.dirname( path.abspath(__file__) ) ))

from lib.OpenChromFile import FileOpenClass
from lib.Smoother import Moving_average, data_Bunching
from lib import AirPLS
from lib.peakDetectionAlgorithm import adjustPeaksBoundary, peakSearchAlgorithm
from lib.thresholdCalc import threshold

from lib.ChromAlgorithm import SmoothData, Detrand, data_Bunching
from lib.Peak import Peak
from lib.PeakSymmetry import calcPeakSymmetry

'''
class Peak:
    # ------------ Property ------------
    # Peak 邊界 (front x, front y, back x, back y)
    boundary = (0,0,0,0)
    boundary_index = (-1,-1) # ( front index, back index)
    # Peak 峰值位置 (x,y)
    Apex = (0,0)
    Apex_index = -1
    # Peak 基本參數
    height,tr, width ,area = 0,0,0,0
    # peak 的非對稱資訊
    w05,w01,w005,tf,As,N, sigma= 0,0,0,1,1,0,0
    # implied Peak 資訊 ( 三次微分所找的peak )
    co_peak = []
    #-----------------------------------
    def __init__(self):
        pass
'''
class Chromatograph:
    # ------------ Property ------------
    time=[]
    signal=[]
    baseline=[]
    # 微分圖譜, 分別為一次微分, 二次微分, 三次微分
    dydx=[]
    dydx2=[]
    dydx3=[]
    # 儲存 Peaks 的容器 , 為 list(Class Peak)
    peak_list=[]
    # -----------------------------------

    def __init__(self, time=[], signal=[]):
        self.time = time
        self.signal = signal

    def DataBunch(self, bunching_point=3):
        self.time, self.signal = data_Bunching( self.time, self.signal, bunching_point )

    def CalcBaseline(self, method="AirPLS", *args, **kwargs):
        lambda_ = kwargs.get('lambda_') if 'lambda_' in kwargs else 10000
        self.baseline = Detrand( method, self.signal, lambda_= lambda_ )

    def Smooth( self, method='Savitzky_Golay_Smooth', window_size=13 , *args, **kwargs ):
        order = kwargs.get('order') if 'order' in kwargs else 3
        mode = kwargs.get('mode') if 'mode' in kwargs else 'nearest'
        sigma = kwargs.get('sigma') if 'sigma' in kwargs else 7

        self.time, self.signal = SmoothData(
            method,
            self.time,
            self.signal,
            window_size,
            order=order,
            mode= mode,
            sigma=sigma
        )

    def SmoothDerivative( self, window_size=21, alpha=1.5):
        # ------ 把 window size 處理為奇數 ------
        window_size = window_size+1 if window_size%2 == 0 else window_size
        # ------ ------ ------ ------ ----- -----
        dx = self.time[1] - self.time[0]

        self.dydx = savgol_filter(
            self.signal, window_length=window_size, polyorder=2, deriv=1, delta=dx
          )  # 一次微分
        self.dydx2 = savgol_filter(
            self.signal, window_length=int(window_size*alpha), polyorder=2, deriv=2, delta=dx
            ) # 二次微分
        self.dydx3 = savgol_filter(
            self.signal, window_length=int(window_size*alpha*alpha), polyorder=3, deriv=3, delta=dx
            ) # 三次微分

    def PeakDetection(self, k = 10, tail_factor = 0.3, MIN_PEAK_INTERVAL=0.5 ):
        dydx_threshold = threshold(self.dydx.copy(), k)
        dydx2_threshold = threshold(self.dydx2.copy(), k)
        #///////////////////////////////////////////////////////////////////////
        peak_index_list = peakSearchAlgorithm(
            [self.time,self.signal],
            [self.dydx,self.dydx2,self.dydx3],
            dydx_threshold[1],
            dydx_threshold[0] * tail_factor,
            dydx2_threshold[0],
            offset=10,
            MAX_TIME_WIDTH=0.5
            )

        # peak_index_list : list  , 2D-array
        # array( [ start index, highest index, end index, apex index ] )
        # index 為 time array 或 signal array的編號
        # start index : peak 起始點的 index
        # highest index : peak 最高點的 index
        # end index : peak 終點的 index
        # apex index : 峰值位置的 index, 為 list, 為利用三次微分圖譜所找到的 peak
        #   為單一peak時, len(apex index)=1且應為 apex index[0] == highest index
        #   若有高度共析狀況發生, 圖譜與一次微分無發區分該共析peaks,
        #   此時 len(apex index) > 1, 其 list內容則是共析peak的apex位置
        #///////////////////////////////////////////////////////////////////////

        boundary_table = adjustPeaksBoundary(
            self.time, self.signal, peak_index_list, self.baseline,
            MIN_PEAK_INTERVAL
            )
        self.__build_peak_list( peak_index_list, boundary_table )

        #-----------------------

        temp = []
        for p in self.peak_list:
            temp.append(
                calcPeakSymmetry(self.time, self.signal, p)
                )
        self.peak_list = temp.copy()

        #------------------------

    def __build_peak_list(self, peak_index_list, boundary_table):
        self.peak_list = []
        for i in range( len(peak_index_list) ):
            peak = Peak()
            peak.boundary_index = (
                peak_index_list[i][0], peak_index_list[i][2]
                )
            peak.boundary= ( boundary_table[i] )
            peak.Apex = (
                self.time[ peak_index_list[i][1]], self.signal[ peak_index_list[i][1] ]
                )
            peak.Apex_index = peak_index_list[i][1]

            peak.height = peak.Apex[1]
            peak.tr = peak.Apex[0]
            peak.width = peak.boundary[2] - peak.boundary[0]
            # ----- 計算積分 -----
            area = np.trapz(
                self.signal[ peak.boundary_index[0]: peak.boundary_index[1] ],
                self.time[ peak.boundary_index[0]: peak.boundary_index[1] ]
                )

            # ----- 計算基線 -----
            # 因為計算出來的 baseline可能會大於原始訊號, 這會造成積分值會比預期低
            # 因此多一到工序  min( signal, baseline ), 保證積分時的基線必 <= signal
            base = []
            for i in range( peak.boundary_index[0], peak.boundary_index[1], 1 ):
                base.append( min( self.signal[i], self.baseline[i] ) )

            base_area = np.trapz(
                base ,
                self.time[ peak.boundary_index[0]: peak.boundary_index[1] ]
                )
            peak.area = area - base_area

            # ------ ------ ------ ------
            peak.co_peak = peak_index_list[ 3 ]

            # ----- 寫入資料 -----
            self.peak_list.append( peak )
            # ------ ------ ------ ------

    def peakHeightFilter( self, SN_rate=3.0 ):
        # 利用 peak height 過濾"已找尋到"的 peaks
        new_peak_list = []
        for peak in self.peak_list:
            if (peak.height / self.baseline[ peak.Apex_index ] ) >= SN_rate :
                new_peak_list.append( peak )
        self.peak_list = new_peak_list

max_stars_repo_name: WenShaoPang/ChromatographAPI

max_issues_repo_name: WenShaoPang/ChromatographAPI

max_forks_repo_name: WenShaoPang/ChromatographAPI


text:
import tensorflow as tf
import numpy as np
import scipy.io
from matplotlib import pyplot as plt
import time

# Load data
dof = 'Ux' # Type the desired degree of freedom (Ux, Uy, Uz, Rx, Ry, Rz)
data = scipy.io.loadmat(dof + '.mat')
solutions = data[dof]
del data
parameters = scipy.io.loadmat('parameters.mat')
alpha = parameters['alpha']
E = parameters['E']/2e+08 # Scale by dividing with the mean value
Fy = parameters['Fy']/2.75e+05 # Scale by dividing with the mean value
del parameters

# Split the data to training and test set
split_ratio = 0.70 # train/test
total_samples = solutions.shape[0]
N_time_steps = solutions.shape[1]
total_dofs = solutions.shape[2]
training_samples = int(split_ratio * total_samples)
test_samples = total_samples - training_samples
test_solutions = solutions[training_samples:total_samples,:,:]
solutions = solutions[0:training_samples,:,:]
parameters = np.concatenate([E,alpha,Fy], axis=1)
test_parameters = parameters[training_samples:total_samples,:]
parameters= parameters[0:training_samples,:]

# Set hyperparameters
latent_space_dimension = 64
epochs = 5000
batch_size = 16
learning_rate = 1e-4
hidden_size = 256

# Scale the data by dividing with max absolute value
u_abs_max = np.load(dof + '_max.npy')
solutions = solutions / u_abs_max
test_solutions = test_solutions / u_abs_max

# Load the trained Encoder
solutions_encoder = tf.keras.models.load_model('Encoder_' + dof)

# Feed Forward Neural Network (FFNN)
FFNN  = tf.keras.Sequential([
(tf.keras.Input(shape = 3)),
tf.keras.layers.Dense(hidden_size),
tf.keras.layers.LeakyReLU(),
tf.keras.layers.Dense(hidden_size),
tf.keras.layers.LeakyReLU(),
tf.keras.layers.Dense(hidden_size),
tf.keras.layers.LeakyReLU(),
tf.keras.layers.Dense(hidden_size),
tf.keras.layers.LeakyReLU(),
tf.keras.layers.Dense(hidden_size),
tf.keras.layers.LeakyReLU(),
tf.keras.layers.Dense(hidden_size),
tf.keras.layers.LeakyReLU(),
tf.keras.layers.Dense(latent_space_dimension)
])

FFNN.compile(optimizer= tf.keras.optimizers.Adam(learning_rate = learning_rate), loss='mse')
FFNN_output = solutions_encoder(solutions)
test_FFNN_output = solutions_encoder(test_solutions)
start_time = time.time()
FFNN_history = FFNN.fit(parameters, FFNN_output, validation_data=(test_parameters, test_FFNN_output), batch_size = batch_size, epochs = epochs, shuffle=True)
training_time = time.time() - start_time
print('Training time: ',   training_time, ' sec')
FFNN.save('FFNN_' + dof)

# FFNN L2-norm error
FFNN_results = FFNN(test_parameters)
encoded_results = FFNN(test_parameters)
encoded_solutions = solutions_encoder(test_solutions)
error_nn = 0
for sample in range(test_samples):
    error_nn =  error_nn + (1/test_samples) * (np.linalg.norm(encoded_solutions[sample,:] - encoded_results[sample,:],2 ) / np.linalg.norm(encoded_solutions[sample,:],2 ))
print('L2 norm error - FFNN = ', error_nn)

# Summarize history for loss
plt.plot(FFNN_history.history['loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train'], loc='upper left')
plt.show()

max_stars_repo_path: Nonlinear Time History Analysis of Steel Building/2 - Train_FFNN.py

reason: import numpy,import scipy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2009-2011 Rosen Diankov (rosen.diankov@gmail.com)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Check collision calls, use collision reports, and do distance queries.

.. examplepre-block:: collision
  :image-width: 300

.. examplepost-block:: collision
"""
from __future__ import with_statement # for python 2.5
__author__ = 'Rosen Diankov'
__copyright__ = '2009-2010 Rosen Diankov (rosen.diankov@gmail.com)'
__license__ = 'Apache License, Version 2.0'

import time
import openravepy
if not __openravepy_build_doc__:
    from numpy import *
    from openravepy import *

def collisioncallback(report,fromphysics):
    """Whenever a collision or physics detects a collision, this function is called"""
    s = 'collision callback with '
    if report.plink1 is not None:
        s += '%s:%s '%(report.plink1.GetParent().GetName(),report.plink1.GetName())
    else:
        s += '(NONE)'
    s += ' x '
    if report.plink2 is not None:
        s += ' %s:%s'%(report.plink2.GetParent().GetName(),report.plink2.GetName())
    else:
        s += '(NONE)'
    print s
    print 'from physics: ',fromphysics
    return CollisionAction.DefaultAction

def main(env,options):
    """Main example code.
    """
    env.Load('robots/barrettwam.robot.xml')
    # register an optional collision callback
    handle = env.RegisterCollisionCallback(collisioncallback)
    robot1 = env.GetRobots()[0]

    try:
        # when doing fast ray collision checking, can specify multiple rays where each column is one ray
        ray1 = array((0,0,-10,0,0,100)) # specify dir*range, pos
        ray2 = array((0,0,10,0,0,-100)) # specify dir*range, pos
        inliers,hitpoints = env.CheckCollisionRays(r_[[ray1],[ray2]],robot1)
        print 'rays hit:',inliers,'hit points:',hitpoints

        # can get more fine grained information
        report1 = CollisionReport()
        inlier1 = env.CheckCollision(Ray(ray1[0:3],ray1[3:6]))
        inlier1 = env.CheckCollision(Ray(ray1[0:3],ray1[3:6]),report1)
        print 'numcontacts: ',len(report1.contacts),' pos: ', report1.contacts[0].pos,' norm: ',report1.contacts[0].norm
    except openrave_exception,e:
        print e

    robot2 = env.ReadRobotXMLFile('robots/mitsubishi-pa10.zae')
    env.Add(robot2)
    body1 = env.ReadKinBodyXMLFile('data/mug1.kinbody.xml')
    env.Add(body1)

    env.CheckCollision(robot1,robot2)
    env.CheckCollision(robot1,body1)
    env.CheckCollision(robot2,body1)

    print 'checking self collision'
    robot1.SetDOFValues([2.98],[3])
    if not robot1.CheckSelfCollision():
        print 'no collision detected (bad)!!, ',env.CheckCollision(robot1.GetLinks()[1],robot1.GetLinks()[13])

    handle.close()
    # test distance queries
    T = eye(4)
    T[0,3] = 0.5
    robot1.SetTransform(T)
    T = eye(4)
    T[1,3] = 0.5
    body1.SetTransform(T)

    print 'move the robots to update the closest distance'
    if not env.GetCollisionChecker().SetCollisionOptions(CollisionOptions.Distance|CollisionOptions.Contacts):
        print 'current checker does not support distance, switching to pqp...'
        collisionChecker = RaveCreateCollisionChecker(env,'pqp')
        collisionChecker.SetCollisionOptions(CollisionOptions.Distance|CollisionOptions.Contacts)
        env.SetCollisionChecker(collisionChecker)
    report = CollisionReport()
    while True:
        contacts = []
        with env:
            check = env.CheckCollision(robot1, report)
            print 'mindist: ',report.minDistance
            contacts += report.contacts
            check = env.CheckCollision(robot2, report)
            print 'mindist: ',report.minDistance
            contacts += report.contacts
            check = env.CheckCollision(body1, report)
            print 'mindist: ',report.minDistance
            contacts += report.contacts
        handles = [env.drawlinestrip(points=array((c.pos,c.pos-c.depth*c.norm)), linewidth=3.0, colors=array((1,0,0,1))) for c in contacts]
        time.sleep(0.1)

from optparse import OptionParser
from openravepy.misc import OpenRAVEGlobalArguments

@openravepy.with_destroy
def run(args=None):
    """Command-line execution of the example.

    :param args: arguments for script to parse, if not specified will use sys.argv
    """
    parser = OptionParser(description='Example shows how to query collision detection information using openravepy')
    OpenRAVEGlobalArguments.addOptions(parser)
    (options, leftargs) = parser.parse_args(args=args)
    OpenRAVEGlobalArguments.parseAndCreateThreadedUser(options,main,defaultviewer=True)

if __name__ == "__main__":
    run()

max_issues_repo_name: jdsika/holy

max_forks_repo_name: jdsika/holy

reason: from numpy


text:
"""Module containing examples of report builder functions and classes."""

from collections import OrderedDict
import numpy as np

def example_fn_build_report(report, pvarray):
    """Example function that builds a report when used in the
    :py:class:`~pvfactors.engine.PVEngine` with full mode simulations.
    Here it will be a dictionary with lists of calculated values.

    Parameters
    ----------
    report : dict
        Initially ``None``, this will be passed and updated by the function
    pvarray : PV array object
        PV array with updated calculation values

    Returns
    -------
    report : dict
        Report updated with newly calculated values
    """
    # Initialize the report
    if report is None:
        list_keys = ['qinc_front', 'qinc_back', 'iso_front', 'iso_back']
        report = OrderedDict({key: [] for key in list_keys})
    # Add elements to the report
    if pvarray is not None:
        pvrow = pvarray.pvrows[1]  # use center pvrow
        report['qinc_front'].append(
            pvrow.front.get_param_weighted('qinc'))
        report['qinc_back'].append(
            pvrow.back.get_param_weighted('qinc'))
        report['iso_front'].append(
            pvrow.front.get_param_weighted('isotropic'))
        report['iso_back'].append(
            pvrow.back.get_param_weighted('isotropic'))
    else:
        # No calculation was performed, because sun was down
        report['qinc_front'].append(np.nan)
        report['qinc_back'].append(np.nan)
        report['iso_front'].append(np.nan)
        report['iso_back'].append(np.nan)

    return report

class ExampleReportBuilder(object):
    """A class is required to build reports when running calculations with
    multiprocessing because of python constraints"""

    @staticmethod
    def build(report, pvarray):
        """Method that will build the simulation report. Here we're using the
        previously defined
        :py:function:`~pvfactors.report.example_fn_build_report`.

        Parameters
        ----------
        report : dict
            Initially ``None``, this will be passed and updated by the function
        pvarray : PV array object
            PV array with updated calculation values

        Returns
        -------
        report : dict
            Report updated with newly calculated values
        """
        return example_fn_build_report(report, pvarray)

    @staticmethod
    def merge(reports):
        """Method used to merge multiple reports together. Here it simply
        concatenates the lists of values saved in the different reports.

        Parameters
        ----------
        reports : list of dict
            List of reports that need to be concatenated together

        Returns
        -------
        report : dict
            Final report with all concatenated values
        """
        report = reports[0]
        # Merge only if more than 1 report
        if len(reports) > 1:
            keys_report = list(reports[0].keys())
            for other_report in reports[1:]:
                for key in keys_report:
                    report[key] += other_report[key]
        return report

max_stars_repo_name: tcapelle/pvfactors

max_issues_repo_name: tcapelle/pvfactors

max_forks_repo_name: tcapelle/pvfactors


text:
#!/usr/bin/env python
"""
Writes dax files or tiff files. This is mostly used
by the simulator.

We try and follow a convention were the first dimension (slow
axis) is the image height and the second dimension (fast axis)
is the image width, so image.shape = [height, width]

Hazen 1/18
"""

import numpy
import os
import tifffile

def inferWriter(filename, width = None, height = None):
    """
    Given a file name this will try to return the appropriate
    writer based on the file extension.
    """
    ext = os.path.splitext(filename)[1]
    if (ext == ".dax"):
        return DaxWriter(filename, width = width, height = height)
    elif (ext == ".fits"):
        return FITSWriter(filename, width = width, height = height)
    elif (ext == ".tif") or (ext == ".tiff"):
        return TiffWriter(filename, width = width, height = height)
    else:
        print(ext, "is not a recognized file type")
        raise IOError("only .dax and .tif are supported (case sensitive..)")

def dummyDaxFile(name, x_size, y_size):
    ddax = DaxWriter(name, width = x_size, height = y_size)
    frame = numpy.ones((x_size, y_size))
    ddax.addFrame(frame)
    ddax.close()

def singleFrameDax(name, frame):
    [fx, fy] = frame.shape
    dax_file = DaxWriter(name, width = fy, height = fx)
    dax_file.addFrame(frame)
    dax_file.close()

class Writer(object):

    def __init__(self, width = None, height = None, **kwds):
        super(Writer, self).__init__(**kwds)
        self.w = width
        self.h = height

    def frameToU16(self, frame):
        frame = frame.copy()
        frame[(frame < 0)] = 0
        frame[(frame > 65535)] = 65535

        return numpy.round(frame).astype(numpy.uint16)

class DaxWriter(Writer):

    def __init__(self, name, **kwds):
        super(DaxWriter, self).__init__(**kwds)

        self.name = name
        if len(os.path.dirname(name)) > 0:
            self.root_name = os.path.dirname(name) + "/" + os.path.splitext(os.path.basename(name))[0]
        else:
            self.root_name = os.path.splitext(os.path.basename(name))[0]
        self.fp = open(self.name, "wb")
        self.l = 0

    def addFrame(self, frame):
        frame = self.frameToU16(frame)

        if (self.w is None) or (self.h is None):
            [self.h, self.w] = frame.shape
        else:
            assert(self.h == frame.shape[0])
            assert(self.w == frame.shape[1])

        frame.tofile(self.fp)
        self.l += 1

    def close(self):
        self.fp.close()

        self.w = int(self.w)
        self.h = int(self.h)

        inf_fp = open(self.root_name + ".inf", "w")
        inf_fp.write("binning = 1 x 1\n")
        inf_fp.write("data type = 16 bit integers (binary, little endian)\n")
        inf_fp.write("frame dimensions = " + str(self.w) + " x " + str(self.h) + "\n")
        inf_fp.write("number of frames = " + str(self.l) + "\n")
        inf_fp.write("Lock Target = 0.0\n")
        if True:
            inf_fp.write("x_start = 1\n")
            inf_fp.write("x_end = " + str(self.w) + "\n")
            inf_fp.write("y_start = 1\n")
            inf_fp.write("y_end = " + str(self.h) + "\n")
        inf_fp.close()

class FITSWriter(Writer):
    """
    This is mostly for testing. It will store all the movie data in
    memory, then dump it when the file is closed.
    """
    def __init__(self, filename, **kwds):
        super(FITSWriter, self).__init__(**kwds)

        self.filename = filename
        self.frames = []

    def addFrame(self, frame):
        frame = self.frameToU16(frame)

        if (self.w is None) or (self.h is None):
            [self.h, self.w] = frame.shape
        else:
            assert(self.h == frame.shape[0])
            assert(self.w == frame.shape[1])

        self.frames.append(frame)

    def close(self):
        # Import here to avoid making astropy mandatory for everybody.
        from astropy.io import fits

        # Remove old file, if any.
        if os.path.exists(self.filename):
            os.remove(self.filename)

        data = numpy.zeros((len(self.frames), self.h, self.w), dtype = numpy.uint16)
        for i in range(len(self.frames)):
            data[i,:,:] = self.frames[i]

        hdu = fits.PrimaryHDU(data)
        hdu.writeto(self.filename)

class TiffWriter(Writer):

    def __init__(self, filename, **kwds):
        super(TiffWriter, self).__init__(**kwds)
        self.tif_fp = tifffile.TiffWriter(filename)

    def addFrame(self, frame):
        frame = self.frameToU16(frame)

        # Enforce that all the frames are the same size.
        if (self.h is None) or (self.w is None):
            [self.h, self.w] = frame.shape
        else:
            assert(self.h == frame.shape[0])
            assert(self.w == frame.shape[1])

        self.tif_fp.save(frame)

    def close(self):
        self.tif_fp.close()

max_stars_repo_name: oxfordni/storm-analysis

max_issues_repo_name: oxfordni/storm-analysis

max_forks_repo_name: oxfordni/storm-analysis

reason: import numpy,from astropy


text:
from functools import partial
import gym
from gym.spaces import Box
from gym.wrappers import TimeLimit
import numpy as np

from .multiagentenv import MultiAgentEnv
from .obsk import get_joints_at_kdist, get_parts_and_edges, build_obs

# using code from https://github.com/ikostrikov/pytorch-ddpg-naf
class NormalizedActions(gym.ActionWrapper):

    def _action(self, action):
        action = (action + 1) / 2
        action *= (self.action_space.high - self.action_space.low)
        action += self.action_space.low
        return action

    def action(self, action_):
        return self._action(action_)

    def _reverse_action(self, action):
        action -= self.action_space.low
        action /= (self.action_space.high - self.action_space.low)
        action = action * 2 - 1
        return action

class MujocoMulti(MultiAgentEnv):

    def __init__(self, batch_size=None, **kwargs):
        super().__init__(batch_size, **kwargs)
        self.scenario = kwargs["env_args"]["scenario"] # e.g. Ant-v2
        self.agent_conf = kwargs["env_args"]["agent_conf"] # e.g. '2x3'

        self.agent_partitions, self.mujoco_edges, self.mujoco_globals  = get_parts_and_edges(self.scenario,
                                                                                self.agent_conf)

        self.n_agents = len(self.agent_partitions)
        self.n_actions = max([len(l) for l in self.agent_partitions])
        self.obs_add_global_pos = kwargs["env_args"].get("obs_add_global_pos", False)

        self.agent_obsk = kwargs["env_args"].get("agent_obsk", None) # if None, fully observable else k>=0 implies observe nearest k agents or joints
        self.agent_obsk_agents = kwargs["env_args"].get("agent_obsk_agents", False)  # observe full k nearest agents (True) or just single joints (False)

        if self.agent_obsk is not None:
            self.k_categories_label = kwargs["env_args"].get("k_categories")
            if self.k_categories_label is None:
                if self.scenario in ["Ant-v2", "manyagent_ant"]:
                    self.k_categories_label = "qpos,qvel,cfrc_ext|qpos"
                elif self.scenario in ["Humanoid-v2", "HumanoidStandup-v2"]:
                    self.k_categories_label = "qpos,qvel,cfrc_ext,cvel,cinert,qfrc_actuator|qpos"
                elif self.scenario in ["Reacher-v2"]:
                    self.k_categories_label = "qpos,qvel,fingertip_dist|qpos"
                elif self.scenario in ["coupled_half_cheetah"]:
                    self.k_categories_label = "qpos,qvel,ten_J,ten_length,ten_velocity|"
                else:
                    self.k_categories_label = "qpos,qvel|qpos"

            k_split = self.k_categories_label.split("|")
            self.k_categories = [k_split[k if k < len(k_split) else -1].split(",") for k in range(self.agent_obsk+1)]

            self.global_categories_label = kwargs["env_args"].get("global_categories")
            self.global_categories = self.global_categories_label.split(",") if self.global_categories_label is not None else []

        if self.agent_obsk is not None:
            self.k_dicts = [get_joints_at_kdist(agent_id,
                                                self.agent_partitions,
                                                self.mujoco_edges,
                                                k=self.agent_obsk,
                                                kagents=False,) for agent_id in range(self.n_agents)]

        # load scenario from script
        self.episode_limit = self.args.episode_limit

        self.env_version = kwargs["env_args"].get("env_version", 2)
        if self.env_version == 2:
            try:
                self.wrapped_env = NormalizedActions(gym.make(self.scenario))
            except gym.error.Error:
                from envs import REGISTRY as env_REGISTRY
                self.wrapped_env = NormalizedActions(TimeLimit(partial(env_REGISTRY[self.scenario],**kwargs["env_args"])(), max_episode_steps=self.episode_limit))
        else:
            assert False,  "not implemented!"
        self.timelimit_env = self.wrapped_env.env
        self.timelimit_env._max_episode_steps = self.episode_limit
        self.env = self.timelimit_env.env
        self.dt = self.env.dt
        self.timelimit_env.reset()
        self.obs_size = self.get_obs_size()

        # COMPATIBILITY
        self.n = self.n_agents
        self.observation_space = [Box(low=np.array([-10]*self.n_agents), high=np.array([10]*self.n_agents)) for _ in range(self.n_agents)]

        acdims = [len(ap) for ap in self.agent_partitions]
        self.action_space = tuple([Box(self.env.action_space.low[sum(acdims[:a]):sum(acdims[:a+1])],
                                       self.env.action_space.high[sum(acdims[:a]):sum(acdims[:a+1])]) for a in range(self.n_agents)])
        pass

    def step(self, actions):

        # need to remove dummy actions that arise due to unequal action vector sizes across agents
        flat_actions = np.concatenate([actions[i][:self.action_space[i].low.shape[0]] for i in range(self.n_agents)])
        obs_n, reward_n, done_n, info_n = self.wrapped_env.step(flat_actions) #obs_n is the state
        self.steps += 1
        obs_agents = self.get_obs_from_state(obs_n)#obs_agents are the obs with agents

        info = {}
        info.update(info_n)

        if done_n:
            if self.steps < self.episode_limit:
                info["episode_limit"] = False   # the next state will be masked out
            else:
                info["episode_limit"] = True    # the next state will not be masked out

        return obs_agents, reward_n, done_n, info

    def get_obs(self):
        """ Returns all agent observat3ions in a list """
        obs_n = []
        for a in range(self.n_agents):
            obs_n.append(self.get_obs_agent(a))
        return obs_n

    def get_obs_agent(self, agent_id):
        if self.agent_obsk is None:
            return self.env._get_obs()
        else:
            return build_obs(self.env,
                                  self.k_dicts[agent_id],
                                  self.k_categories,
                                  self.mujoco_globals,
                                  self.global_categories,
                                  vec_len=getattr(self, "obs_size", None))

    def get_obs_size(self):
        """ Returns the shape of the observation """
        if self.agent_obsk is None:
            return self.get_obs_agent(0).size
        else:
            return max([len(self.get_obs_agent(agent_id)) for agent_id in range(self.n_agents)])

    def get_obs_from_state(self, state):
        # for cheeta environment
        if self.scenario == "HalfCheetah-v2":
            qpos = np.zeros((9,))
            qpos[1:] = state[:8]
            qvel = state[8:]
            state = {'qpos':qpos, 'qvel':qvel}
        if self.scenario == "Swimmer-v2":
            s1 = 5
            s2 = 2
            qpos = np.zeros((s1,))
            qpos[s2:] = state[:s1-s2]
            qvel = state[s1-s2:]
            state = {'qpos':qpos, 'qvel':qvel}

        if self.scenario == "Walker2d-v2":
            s1 = 9
            s2 = 1
            qpos = np.zeros((s1,))
            qpos[s2:] = state[:s1-s2]
            qvel = state[s1-s2:]
            state = {'qpos':qpos, 'qvel':qvel}

        if self.scenario == "Ant-v2": #15-2+14+14*6
            s1 = 15
            s2 = 2
            s3 = 14
            qpos = np.zeros((s1,))
            qpos[s2:] = state[:s1-s2]
            qvel = state[s1-s2:s1-s2+s3]
            cfrc_ext = state[s1-s2+s3:]
            cftc_ext = np.array(cfrc_ext).reshape((14,6))
            state = {'qpos':qpos, 'qvel':qvel, 'cftc_ext':cftc_ext}

        obs = []
        for agent_id in range(self.n_agents):
            ob = build_obs(self.env,
                    self.k_dicts[agent_id],
                    self.k_categories,
                    self.mujoco_globals,
                    self.global_categories,
                    vec_len=getattr(self, "obs_size", None), query_state = state)
            obs.append(ob)
        return obs

    def get_state(self, team=None):
        # TODO: May want global states for different teams (so cannot see what the other team is communicating e.g.)
        return self.env._get_obs()

    def get_state_size(self):
        """ Returns the shape of the state"""
        return len(self.get_state())

    def get_avail_actions(self): # all actions are always available
        return np.ones(shape=(self.n_agents, self.n_actions,))

    def get_avail_agent_actions(self, agent_id):
        """ Returns the available actions for agent_id """
        return np.ones(shape=(self.n_actions,))

    def get_total_actions(self):
        """ Returns the total number of actions an agent could ever take """
        return self.n_actions # CAREFUL! - for continuous dims, this is action space dim rather
        # return self.env.action_space.shape[0]

    def get_stats(self):
        return {}

    # TODO: Temp hack
    def get_agg_stats(self, stats):
        return {}

    def reset(self, **kwargs):
        """ Returns initial observations and states"""
        self.steps = 0
        self.timelimit_env.reset()
        return self.get_obs()

    def render(self, **kwargs):
        self.env.render(**kwargs)

    def close(self):
        # raise NotImplementedError
        self.env.close()

    def seed(self, args):
        pass

    def get_env_info(self):

        env_info = {"state_shape": self.get_state_size(),
                    "obs_shape": self.get_obs_size(),
                    "n_actions": self.get_total_actions(),
                    "n_agents": self.n_agents,
                    "episode_limit": self.episode_limit,
                    "action_spaces": self.action_space,
                    "actions_dtype": np.float32,
                    "normalise_actions": False
                    }
        return env_info


text:
import cv2
import numpy as np
import matplotlib.pyplot as plt
import os
import subprocess
import sys
from matplotlib import colors

img = cv2.imread('gray_rose.jpg',0)
dft = cv2.dft(np.float32(img),flags = cv2.DFT_COMPLEX_OUTPUT)
reales = []
complejos = []
for i in range(0,220):
    for j in range(0,220):
        r,c = dft[i][j]
        reales.append(r)
        complejos.append(c)

def img_real_frec(y,real):
    font = {'family': 'serif',
        'color':  'darkred',
        'weight': 'normal',
        'size': 16,
        }
    if(real):
        fig = plt.figure()
        x = np.arange(48400)

        plt.subplot(111)
        plt.plot(x,y,linestyle='-')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.legend(loc=2, prop={'size': 4})
        plt.title("Real part value per frecuency",fontdict=font)
        plt.xlabel('Frecuencies',fontsize=7)
        plt.ylabel('Real part value',fontsize=7)

        plt.savefig("real_frec.pdf")
    else:
        fig = plt.figure()
        x = np.arange(48400)

        plt.subplot(111)
        plt.plot(x,y,linestyle='-')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.legend(loc=2, prop={'size': 4})
        plt.title("Imaginary part value per frecuency",fontdict=font)
        plt.xlabel('Frecuencies',fontsize=7)
        plt.ylabel('Imaginary part value',fontsize=7)

        plt.savefig("imaginary_frec.pdf")

def img_real_sections(y,real):
    x = np.arange(48400)
    font = {'family': 'serif',
        'color':  'darkred',
        'weight': 'normal',
        'size': 5,
        }
    if(real):
        fig = plt.figure()
        plt.subplot(321)
        plt.ylim(-100,100)
        y = np.array(y)
        #print y[np.abs(y)<10]
        plt.scatter(x[np.abs(y)<10],y[np.abs(y)<10],s=1,c='red',label=r'$368 (0.76 \%)$')
        plt.scatter(x[np.abs(y)>=10],y[np.abs(y)>=10],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec real part value less than 10",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Real part value',fontsize=5)

        plt.subplot(322)
        plt.ylim(-1000,1000)
        #print np.shape(y[np.abs(y)<100])
        plt.scatter(x[np.abs(y)<100],y[np.abs(y)<100],s=1,c='red',label=r'$3284 (6.78 \%)$')
        plt.scatter(x[np.abs(y)>=100],y[np.abs(y)>=100],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec real part value less than 100",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Real part value',fontsize=5)

        plt.subplot(323)
        plt.ylim(-10000,10000)
        #print np.shape(y[np.abs(y)<1000])
        plt.scatter(x[np.abs(y)<1000],y[np.abs(y)<1000],s=1,c='red',label=r'$25651 (52.99\%)$')
        plt.scatter(x[np.abs(y)>=1000],y[np.abs(y)>=1000],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec real part value less than 1000",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Real part value',fontsize=5)

        plt.subplot(324)
        plt.ylim(-100000,100000)
        #print np.shape(y[np.abs(y)<10000])
        plt.scatter(x[np.abs(y)<10000],y[np.abs(y)<10000],s=1,c='red',label=r'$ 47111 (97.33 \%)$')
        plt.scatter(x[np.abs(y)>=10000],y[np.abs(y)>=10000],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec real part value less than 10.000",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Real part value',fontsize=5)

        plt.subplot(325)
        plt.ylim(-1000000,1000000)
        #print np.shape(y[np.abs(y)<100000])
        plt.scatter(x[np.abs(y)<100000],y[np.abs(y)<100000],s=1,c='red',label=r'$48369 (99.93 \%)$')
        plt.scatter(x[np.abs(y)>=100000],y[np.abs(y)>=100000],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec real part value less than 100.000",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Real part value',fontsize=5)

        plt.subplot(326)
        plt.ylim(-10000000,10000000)
        #print np.shape(y[np.abs(y)<1000000])
        plt.scatter(x[np.abs(y)<1000000],y[np.abs(y)<1000000],s=1,c='red',label=r'$48397 (99.99 \%)$')
        plt.scatter(x[np.abs(y)>=1000000],y[np.abs(y)>=1000000],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec real part value less than 1.000.000",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Real part value',fontsize=5)

        plt.tight_layout()

        plt.savefig("real_sections.pdf")
    else:
        fig = plt.figure()
        plt.subplot(321)
        plt.ylim(-100,100)
        y = np.array(y)
        plt.scatter(x[np.abs(y)<10],y[np.abs(y)<10],s=1,c='red',label=r'$306 (0.63 \%)$')
        plt.scatter(x[np.abs(y)>=10],y[np.abs(y)>=10],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec imaginary part value less than 10",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Imaginary part value',fontsize=5)

        plt.subplot(322)
        plt.ylim(-1000,1000)
        plt.scatter(x[np.abs(y)<100],y[np.abs(y)<100],s=1,c='red',label=r'$3190 (6.59 \%)$')
        plt.scatter(x[np.abs(y)>=100],y[np.abs(y)>=100],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec imaginary part value less than 100",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Imaginary part value',fontsize=5)

        plt.subplot(323)
        plt.ylim(-10000,10000)
        plt.scatter(x[np.abs(y)<1000],y[np.abs(y)<1000],s=1,c='red',label=r'$25494 (52.67\%)$')
        plt.scatter(x[np.abs(y)>=1000],y[np.abs(y)>=1000],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec imaginary part value less than 1000",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Imaginary part value',fontsize=5)

        plt.subplot(324)
        plt.ylim(-100000,100000)
        plt.scatter(x[np.abs(y)<10000],y[np.abs(y)<10000],s=1,c='red',label=r'$ 47076 (97.26 \%)$')
        plt.scatter(x[np.abs(y)>=10000],y[np.abs(y)>=10000],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec imaginary part value less than 10.000",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Imaginary part value',fontsize=5)

        plt.subplot(325)
        plt.ylim(-1000000,1000000)
        plt.scatter(x[np.abs(y)<100000],y[np.abs(y)<100000],s=1,c='red',label=r'$48364 (99.92 \%)$')
        plt.scatter(x[np.abs(y)>=100000],y[np.abs(y)>=100000],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec imaginary part value less than 100.000",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Imaginary part value',fontsize=5)

        plt.subplot(326)
        plt.ylim(-10000000,10000000)
        plt.scatter(x[np.abs(y)<1000000],y[np.abs(y)<1000000],s=1,c='red',label=r'$48400 (100 \%)$')
        plt.scatter(x[np.abs(y)>=1000000],y[np.abs(y)>=1000000],s=1,c='gray')
        plt.plot(np.arange(48400),np.zeros(48400),color='black',linewidth=0.01,label='zero line')
        plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
        plt.title("Frec imaginary part value less than 1.000.000",fontdict=font)
        plt.legend(loc=2, prop={'size': 4})
        plt.xlabel('Frecuencies',fontsize=5)
        plt.ylabel('Imaginary part value',fontsize=5)

        plt.tight_layout()

        plt.savefig("imaginary_sections.pdf")

def img_real_hist(y,real):
    font = {'family': 'serif',
        'color':  'darkred',
        'weight': 'normal',
        'size': 16,
        }
    if(real):
        fig = plt.figure()
        plt.subplot(111)
        y = np.array(y)
        bins = np.arange(0,7)
        plt.xticks(bins, [r"$10^%s$" % i for i in bins])
        plt.hist(np.log10(abs(y)),log=True, bins='auto')
        plt.axhline(10,c='b',linewidth=0.1)
        plt.axhline(100,c='b',linewidth=0.1)
        plt.axhline(1000,c='b',linewidth=0.1)
        plt.axhline(1,c='b',linewidth=0.1)
        plt.xlabel('Real part value')
        plt.ylabel('Amount of frecuencies')
        plt.title("Histogram of real part values per frecuency",fontdict=font)

        plt.savefig("real_hist.pdf")
    else:
        fig = plt.figure()
        plt.subplot(111)
        y = np.array(y)
        bins = np.arange(0,10)
        plt.xticks(bins, [r"$10^%s$" % i for i in bins])
        y[0] = 10**-10
        y[110] = 10**-10
        plt.hist(np.log10(abs(y)),log=True, bins='auto')
        plt.axhline(10,c='b',linewidth=0.1)
        plt.axhline(100,c='b',linewidth=0.1)
        plt.axhline(1000,c='b',linewidth=0.1)
        plt.axhline(1,c='b',linewidth=0.1)
        plt.xlabel('Imaginary part value')
        plt.ylabel('Amount of frecuencies')
        plt.title("Histogram of imaginary part values per frecuency",fontdict=font)

        plt.savefig("imaginary_hist.pdf")

img_real_frec(reales,True)
img_real_frec(complejos,False)

img_real_sections(reales,True)
img_real_sections(complejos,False)

img_real_hist(reales,True)
img_real_hist(complejos,False)


text:
"""
this module implements convolutional transformer
for transducer decoder
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

from trainer.model.modules.transformer import TransformerEncoderLayer

class Net(nn.Module):
    """
    Class implements convolutonal transformer used for transducer
    decoder in "Minimum Bayes Risk Training of RNN-Transducer for
    End-to-End Speech Recognition", InterSpeech2020

    Args:
        embeddings (nn.module): embedding layer
        output_dim (int): output dimension
        d_model (int): d_model dimension used in transformer
        num_layers (int): number of convlution and transformer layers
        heads (int): number of heads in multi-head self-attention
        d_ff (int): size of the inner FF layer in transformer
        dropout (float): dropout probilities
        max_relative_positions (int): relative position encoding
        max_size (int): maximum input sequence length

    """
    def __init__(self, embeddings, output_dim, d_model,
                 num_layers, heads=8, d_ff=2048, dropout=0.1,
                 max_relative_positions=0, max_size=5000):
        super(Net, self).__init__()
        self.embeddings = embeddings
        self.output_dim = output_dim
        self.conv = nn.ModuleList(
            [nn.Conv1d(in_channels=embeddings.embedding_dim,
                       out_channels=d_model,
                       kernel_size=5,
                       padding=(5-1)*1)] + \
            [nn.Conv1d(in_channels=d_model,
                       out_channels=d_model,
                       kernel_size=5,
                       padding=(5-1)*1) for i in range(num_layers-1)]
        )
        self.transformer = nn.ModuleList(
            [TransformerEncoderLayer(
                d_model, heads, d_ff, dropout,
                max_relative_positions=max_relative_positions)
             for i in range(num_layers)])
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)

        self.linear_out = nn.Linear(d_model, output_dim)

        mask = self._get_attn_subsequent_mask(max_size)
        # Register self.mask as a buffer .
        self.register_buffer('mask', mask)

    def forward(self, src, softmax=False):
        #src: batch, frame
        out = self.embeddings(src)
        # getting mask for current batch
        # Run the forward pass of every layer of the tranformer.
        src_batch, src_len = src.size()
        padding_idx = self.embeddings.padding_idx
        pad_mask = src.data.eq(padding_idx).unsqueeze(1)\
                   .expand(src_batch, src_len, src_len)
        mask = torch.gt(pad_mask + self.mask[:, :pad_mask.size(1), :pad_mask.size(1)]\
                                            .expand_as(pad_mask), 0)

        for conv, trans in zip(self.conv, self.transformer):
            #causal convolution
            out = F.relu(conv(out.transpose(1, 2))[:, :, :-conv.padding[0]])
            out = out.transpose(1, 2)
            out = trans(out, mask=mask)
        out = self.layer_norm(out)
        out = self.linear_out(out)
        if softmax:
            out = F.log_softmax(out, dim=-1)
        return out

    def _get_attn_subsequent_mask(self, size):
        ''' Get an attention mask to avoid using the subsequent info.'''
        attn_shape = (1, size, size)
        subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
        subsequent_mask = torch.from_numpy(subsequent_mask)
        return subsequent_mask


text:
# -*- coding: utf-8 -*-

from dash import Dash
from dash.dependencies import Input, Output, ALL, State, MATCH, ALLSMALLER, ClientsideFunction
from Dashapps.Dash_fun import apply_layout_with_auth, load_object, save_object
import dash_core_components as dcc
import dash_html_components as html
import dash_bootstrap_components as dbc
import pandas as pd
import plotly.express as px
import plotly.graph_objs as go
from Dashapps.Dash_base import warning_card, colors, cite_card, description_card, draft_template
import dash_table
from datetime import datetime
import numpy as np
from flask import request
import locale

url_base = '/dash/app8/'

data_sources = [
    "https://data.cdc.gov/NCHS/Weekly-Counts-of-Deaths-by-State-and-Select-Causes/3yf8-kanr",
    "https://data.cdc.gov/NCHS/Weekly-Counts-of-Deaths-by-State-and-Select-Causes/muzy-jte6",
    "https://www.reddit.com/r/dataisbeautiful/comments/mjdmln/oc_weekly_deaths_from_all_causes_by_year_in_the/"
]

data_licenses = [
    "https://www.usa.gov/government-works"
]

sourced_date = "03/20/2021"

cite_text = '"I will not say: do not weep; for not all tears are an evil."'
cite_author = "J.R.R. Tolkien"
cite_link = "https://en.wikipedia.org/wiki/J._R._R._Tolkien"
description_text = '''What you see below are the deaths in the USA in year 2020 in comparison to all the years before. We see how the pandamic affected the number of deaths per week in 2020.'''

df = pd.read_csv('app_data/processed/0008.csv', dtype={'Jurisdiction of Occurrence': str,'Year': int,'Week': int,'Cause': int})

chart_groups = df.groupby(by='Year')

chart_data = []

chart_colors=['orange', 'blue', 'green', 'black', 'grey', 'purple', 'red']

for group, dataframe in chart_groups:
    dataframe = dataframe.sort_values(by=['Week'])

    trace = go.Scatter(x=dataframe.Week.tolist(),
                       y=dataframe.Cause.tolist(),
                       marker=dict(color=chart_colors[len(chart_data)]),
                       mode='lines+markers',
                       name=group)
    chart_data.append(trace)

chart_layout =  go.Layout(xaxis={'title': 'Week of the Year'},
                    yaxis={'title': 'Deaths per Week'},
                    margin={'l': 0, 'b': 50, 't': 50, 'r': 0},
                    legend = dict(
                            orientation="h", # Looks much better horizontal than vertical
                            y=-0.15
                            ),
                            template=draft_template,
                            annotations=[
                                dict(
                                    templateitemname="draft watermark",
                                    text="www.blackandwhitedata.com",
                                )
                            ],
                    hovermode='closest')

fig = go.Figure(data=chart_data, layout=chart_layout)

# The Layout
layout = html.Div(style={'font-family':'"Poppins", sans-serif', 'backgroundColor': colors['background']}, children=[
    html.H1(
        children='Deaths in the USA in 2020',
        style={
            'textAlign': 'center',
            'color': colors['text'],
            'backgroundColor': colors['background']
        }
    ),
    html.Div(children=description_card(description_text), style={
        'textAlign': 'center',
        'color': colors['text'],
        'backgroundColor': colors['background']
    }),
    html.Div(children=cite_card(cite_text,cite_author,cite_link), style={
        'textAlign': 'center',
        'color': colors['text'],
        'backgroundColor': colors['background']
    }),
    html.Br(),
    dcc.Graph(
        id='example-graph-2',
        figure=fig
    ),

    html.Br(),
    html.Hr(className="my-2"),
    html.Br(),
    html.Div(children=warning_card(data_sources,data_licenses,sourced_date), style={
        'textAlign': 'left',
        'color': colors['text'],
        'backgroundColor': colors['background']
    })
])

def Add_Dash(server):
    app = Dash(server=server, url_base_pathname=url_base, external_stylesheets = [dbc.themes.BOOTSTRAP], external_scripts = ["https://cdn.plot.ly/plotly-locale-de-latest.js"], meta_tags=[{"name": "viewport", "content": "width=device-width, initial-scale=1"}])

    apply_layout_with_auth(app, layout)

    return app.server

max_stars_repo_name: SchneiderDaniel/data

max_issues_repo_name: SchneiderDaniel/data

max_forks_repo_name: SchneiderDaniel/data


text:
#############################################################################################
# Append the path to ummon3 to PATH-Variable so that ummon can be imported during development
import sys
sys.path.insert(0,'../../ummon3')
sys.path.insert(0,'../ummon3')
#############################################################################################

import unittest
import logging
import os
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from math import log
from torch.autograd import Variable
from torch.utils.data.dataset import TensorDataset
from torch.utils.data import DataLoader
from ummon import *

# set fixed seed for reproducible results
torch.manual_seed(4)
np.random.seed(4)

class TestExamples(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        super(TestExamples, self).__init__(*args, **kwargs)

        # BACKUP files
        backup_dir = "_backup"
        files = os.listdir(".")
        dir = "."
        for file in files:
            if file.endswith(Trainingstate().extension) or file.endswith(".log"):
                if not os.path.exists(backup_dir):
                    os.makedirs(backup_dir)
                os.rename(os.path.join(dir,file), os.path.join(backup_dir,file))

    def test_sine(self):
        import examples.sine

        ts = examples.sine.example()
        assert ts["best_validation_loss"][1] < 1.2

         # Clean up
        files = os.listdir(".")
        dir = "."
        for file in files:
            if file.endswith(Trainingstate().extension) or file.endswith(".log"):
                os.remove(os.path.join(dir,file))

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--test', default="", metavar="",
                        help="Execute a specific test")
    argv = parser.parse_args()
    sys.argv = [sys.argv[0]]
    if argv.test is not "":
        eval(str("TestExamples()." + argv.test + '()'))
    else:
        unittest.main()


text:
"""Same as CostBinary, but provides access to full Data"""

import typing
import numpy as np
import datatable as dt
from h2oaicore.metrics import CustomScorer
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import confusion_matrix

class CostBinaryWithData(CustomScorer):
    _description = "Calculates cost per row in binary classification: `(fp_cost*FP + fn_cost*FN) / N`"
    _binary = True
    _maximize = False
    _perfect_score = 0
    _display_name = "CostWithData"
    _threshold = 0.5  # Example only, should be adjusted based on domain knowledge and other experiments

    _needs_X = True  # This assures that X is passed in

    # The cost of false positives and negatives will vary by data set, we use the rules from the below as an example
    # https://www.kaggle.com/uciml/aps-failure-at-scania-trucks-data-set
    _fp_cost = 10
    _fn_cost = 500

    def score(self,
              actual: np.array,
              predicted: np.array,
              sample_weight: typing.Optional[np.array] = None,
              labels: typing.Optional[np.array] = None,
              X: typing.Optional[dt.Frame] = None,
              **kwargs) -> float:
        # can compute arbitrary cost from all original features
        if X is not None:
            assert X.nrows == len(actual)
            assert X.ncols >= 1
            X_pd = X.to_pandas()

        # label actuals as 1 or 0
        lb = LabelEncoder()
        labels = lb.fit_transform(labels)
        actual = lb.transform(actual)

        # label predictions as 1 or 0
        predicted = predicted >= self._threshold

        # use sklearn to get fp and fn
        cm = confusion_matrix(actual, predicted, sample_weight=sample_weight, labels=labels)
        tn, fp, fn, tp = cm.ravel()

        # calculate`$1*FP + $2*FN`
        return ((fp * self.__class__._fp_cost) + (fn * self.__class__._fn_cost)) / (
                tn + fp + fn + tp)  # divide by total weighted count to make loss invariant to data size

max_stars_repo_name: pragnesh-ai/driverlessai-recipes

max_issues_repo_name: pragnesh-ai/driverlessai-recipes

max_forks_repo_name: pragnesh-ai/driverlessai-recipes


text:
import numpy as np
import matplotlib.pyplot as plt
x=np.array([1,2,3,4,5,-1])
y=np.array([12,45,4,22,30,22])
plt.plot(x,y)

max_stars_repo_path: 100 Curso aulapharos/008 BibliotecasModulos/003 Spyder matplotlib/Matplotlib004.py

max_stars_repo_name: malcabaut/AprendiendoPython

max_issues_repo_name: malcabaut/AprendiendoPython

max_forks_repo_name: malcabaut/AprendiendoPython