text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 30 14:13:12 2020

@author: fabian geiger
"""

import numpy as np
import scipy.io as sio
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
from imblearn import under_sampling
from pathlib import Path

def mix_real_MIT(kfold=3, seed=333, undersample=True, split='random'):
    """
    Utility function to mix the MIT dataset with the self-acquired dataset
    """
    #realData_path = Path('/home/fabian/Documents/Master_thesis/Data_Collection/3kOhm_FB/data_MT_FabianGeiger_5sess.mat')
    realData_path = Path('../../Data_Collection/3kOhm_FB/data_MT_FabianGeiger_5sess.mat')
    #MITdata_path = Path('/home/fabian/Documents/Master_thesis/Research/STAG_MIT/classification_lite/metadata.mat')
    MITdata_path = Path('../../Research/STAG_MIT/classification_lite/metadata.mat')

    # These two lists will contain valid data split into recording sessions
    x = []
    y = []

    realData = sio.loadmat(realData_path, squeeze_me=True)
    real_valid_mask = realData['valid_flag'] == 1
    # Scale all data to the range [0, 1]
    realPressure = realData['tactile_data'][real_valid_mask].astype(np.float32)
    realPressure = np.clip((realPressure - 1510)/(3000 - 1510), 0.0, 1.0)
    realObject_id = realData['object_id'][real_valid_mask]
    realSession_id = realData['session_id'][real_valid_mask]
    # Split into sessions
    num_sessions = len(np.unique(realSession_id))
    for i in range(num_sessions):
        session_mask = realSession_id == i
        x.append(realPressure[session_mask])
        y.append(realObject_id[session_mask])

    MITdata = sio.loadmat(MITdata_path, squeeze_me=True)
    MIT_valid_mask = MITdata['hasValidLabel'] == 1
    MITpressure = MITdata['pressure'].reshape((-1, 32*32)).astype(np.float32)
    MITpressure = MITpressure[MIT_valid_mask]
    MITpressure = np.clip((MITpressure - 500)/(650 - 500), 0.0, 1.0)
    MITobject_id = MITdata['objectId'][MIT_valid_mask]
    # Only use the same objects as in the real data set
    MITobjects = list(MITdata['objects'])
    used_objects = list(map(str.strip, realData['objects']))
    # Each class has three recording IDs that correspond to the different
    # experiment days. There are 81 recording IDs (3*27)
    # 0  - 26 belong to the first recording
    # 27 - 53 belong to the second recording
    # 54 - 81 belong to the third recording
    MITrecording_id = MITdata['recordingId'][MIT_valid_mask]
    for i in range(3):
        # Find valid samples from the different recording days
        recording_mask = np.logical_and(MITrecording_id >= i*27,
                                        MITrecording_id < (i+1)*27)

        used_pressure = []
        used_object_id = []
        for i, obj in enumerate(used_objects):
            idx = MITobjects.index(obj)
            used_mask = np.logical_and(MITobject_id == idx, recording_mask)
            used_pressure.append(MITpressure[used_mask])
            used_object_id.append(np.full(len(MITobject_id[used_mask]), i))

        x.append(np.concatenate(used_pressure))
        y.append(np.concatenate(used_object_id))

    if kfold is not None:
        # Decrease the test size if cross validation is used
        test_size = 0.15
    else:
        kfold = 3
        test_size = 0.33

    if(split == 'random'):
        pressure = np.concatenate(x)
        object_id = np.concatenate(y)
        if(undersample):
            us = under_sampling.RandomUnderSampler(random_state=seed,
                                               sampling_strategy='not minority')
            us_pressure, us_object_id = us.fit_resample(pressure, object_id)

            pressure, object_id = us_pressure, us_object_id

        # Split the already balanced dataset in a stratified way -> training
        # and test set will still be balanced
        train_data, test_data,\
            train_labels, test_labels = train_test_split(pressure, object_id,
                                                         test_size=test_size,
                                                         random_state=seed,
                                                         shuffle=True,
                                                         stratify=object_id)
        #print(train_data.shape, train_labels.shape)
        # This generates a k fold split in a stratified way.
        # Easy way to do k fold cross validation
        skf = StratifiedKFold(n_splits=kfold, shuffle=True,
                              random_state=seed)
        # train_ind, val_ind = skf.split(train_data, train_labels)
        # skf_gen = skf.split(train_data, train_labels)

        return train_data, train_labels, test_data, test_labels, skf

    elif(split == 'session'):
        return x, y

max_stars_repo_name: geigerf/utils

max_issues_repo_name: geigerf/utils

max_forks_repo_name: geigerf/utils

reason: import numpy,import scipy


text:
#!/usr/bin/env python
# coding: utf-8

r"""Bspline demo"""

import matplotlib.pyplot as plt
import numpy as np

from corelib.geometry.bspline import bspline

colors = ('b', 'g', 'r', 'c', 'm', 'y', 'k')

cv = np.array([[50., 25.],
               [59., 12.],
               [50., 10.],
               [57.,  2.],
               [40.,  4.],
               [40., 14.]])

plt.plot(cv[:, 0], cv[:, 1], 'o-', label='Control Points')

for d in range(1, 21):
    p = bspline(cv, n=200, degree=d, periodic=False)
    x, y = p.T
    plt.plot(x, y, 'k-', label='Degree %s' % d, color=colors[d % len(colors)])

plt.minorticks_on()
plt.legend()
plt.xlabel('x')
plt.ylabel('y')
plt.xlim(35, 70)
plt.ylim(0, 30)
plt.gca().set_aspect('equal', adjustable='box')
plt.show()

ext: py

max_stars_repo_name: guillaume-florent/corelib

max_issues_repo_name: guillaume-florent/corelib

max_forks_repo_name: guillaume-florent/corelib

reason: import numpy


text:
"""
    Mask Cam
    Copyright (c) 2020 by SilentByte <https://www.silentbyte.com/>
"""

import json
import logging
import base64
import boto3
import sagemaker

from uuid import UUID, uuid4
from datetime import datetime

from abc import abstractmethod
from typing import (
    Any,
    Dict,
    Optional,
    Union,
)

from maskcam import (
    httpstatus,
    schemas,
    settings,
)

from maskcam.db import Repo
from maskcam.collections import CaseInsensitiveDict
from maskcam.test_sagemaker_responses import responses

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)

def _jsonify(data: Union[dict, list]) -> str:
    class ExtendedEncoder(json.JSONEncoder):
        def default(self, o):
            if isinstance(o, UUID):
                return str(o)
            elif isinstance(o, datetime):
                return o.isoformat()
            else:
                return json.JSONEncoder.default(self, o)

    return json.dumps(
        data,
        indent=None,
        separators=(',', ':'),
        sort_keys=True,
        cls=ExtendedEncoder,
    )

def _generate_s3_photo_key(id: UUID) -> str:
    return settings.PHOTO_KEY_PREFIX + str(id)

class Event:
    method: str
    headers: CaseInsensitiveDict
    query_params: CaseInsensitiveDict
    body: Any
    is_base64: bool

    def __init__(
            self,
            method: str = '',
            headers: Optional[Dict[str, str]] = None,
            query_params: Optional[Dict[str, str]] = None,
            body: Any = None,
            is_base64: bool = False
    ):
        self.method = method
        self.headers = CaseInsensitiveDict(headers)
        self.query_params = CaseInsensitiveDict(query_params)
        self.body = body
        self.is_base64 = is_base64

class Context:
    pass

class GatewayException(Exception):
    status_code = httpstatus.HTTP_500_INTERNAL_SERVER_ERROR

class BadRequestException(GatewayException):
    status_code = httpstatus.HTTP_400_BAD_REQUEST

class Response:
    status_code: int = httpstatus.HTTP_200_OK
    headers: Dict[str, str] = None
    body: Optional[str] = None

    def __init__(
            self,
            status_code: int = httpstatus.HTTP_200_OK,
            headers: Optional[Dict[str, str]] = None,
            body: Optional[str] = None,
    ):
        self.status_code = status_code
        self.headers = headers if headers is not None else {}
        self.body = body

class JsonResponse(Response):
    def __init__(
            self,
            data: Union[dict, list],
            status_code=httpstatus.HTTP_200_OK,
            headers: Optional[Dict[str, str]] = None,
    ):
        super().__init__(status_code, headers, _jsonify(data))

        if 'Content-Type' not in self.headers:
            self.headers['Content-Type'] = 'application/json'

class Lambda:
    @abstractmethod
    def handle(self, event: Event, context: Context) -> Response:
        raise NotImplementedError()

    @staticmethod
    def translate_event(lambda_event: dict) -> Event:
        """
        Translates an event from an AWS service to a simplified internal structure.
        For available fields, see: <https://docs.aws.amazon.com/lambda/latest/dg/lambda-services.html>.
        """

        method = lambda_event.get('httpMethod', '').upper()
        headers = lambda_event.get('headers', {})
        query_params = lambda_event.get('queryStringParameters', {})
        body = lambda_event.get('body', '')
        is_base64 = lambda_event.get('isBase64Encoded', False)

        return Event(method, headers, query_params, body, is_base64)

    @staticmethod
    def translate_context(_lambda_context: dict) -> Context:
        return Context()

    def bind(self) -> callable:
        # noinspection PyBroadException
        def handler(event, context) -> dict:
            try:
                response = self.handle(
                    event=Lambda.translate_event(event),
                    context=Lambda.translate_context(context),
                )

                return {
                    'statusCode': response.status_code,
                    'headers': response.headers,
                    'body': response.body,
                }

            except GatewayException as e:
                log.warning(f'[GatewayException] {e}')
                return {
                    'statusCode': e.status_code,
                    'body': None,
                }

            except:
                log.exception('Unhandled exception')
                return {
                    'statusCode': httpstatus.HTTP_500_INTERNAL_SERVER_ERROR,
                    'body': None,
                }

        return handler

class UploadLambda(Lambda):
    @staticmethod
    def parse_sagemaker_output(output: dict, person_threshold):
        def _extract_fields(predict_dict):
            return dict(name=predict_dict['name'], person_prob=float(predict_dict['percentage_probability']),
                        no_mask_prob=float(predict_dict['classes']['no_mask']))

        # If it came in as a string convert it to a dict
        people_in_frame = 0
        mask_prediction = []

        for prediction in output:
            try:
                data = _extract_fields(prediction)
                if data['name'] == "person" and data['person_prob'] > person_threshold:
                    people_in_frame += 1
                    mask_prediction.append(100 - data['no_mask_prob'])
            except (ValueError, KeyError) as e:
                log.critical(
                    "It looks like the format of the sagemaker response has changed")
                log.exception(e)

        if people_in_frame is 0:
            return None
        else:
            return dict(min_mask=min(mask_prediction), people_in_frame=people_in_frame)

    @staticmethod
    def _parse_body(event: Event) -> Optional[dict]:
        try:
            return schemas.apply_schema(schemas.UploadSchema, event.body)
        except schemas.ValidationError:
            log.debug(event.body)
            log.exception("Invalid payload received")
            return None

    def handle(self, event: Event, context: Context) -> Response:
        data = UploadLambda._parse_body(event)
        if data is None:
            return JsonResponse(
                status_code=httpstatus.HTTP_400_BAD_REQUEST,
                data={'message': 'Invalid payload'},
            )
        # Do sagemaker prediction
        sagemaker = boto3.client(
            'runtime.sagemaker',
            region_name='us-east-1'
        )
        if len(settings.SAGEMAKER_ENDPOINT) != 0:  # If a non blank sagemaker endpoint has been put in
            result = sagemaker.invoke_endpoint(
                EndpointName=settings.SAGEMAKER_ENDPOINT,
                ContentType='image/jpeg',
                Body=bytearray(base64.b64decode(data['photo_data'])))
            body_response = json.loads(result['Body'].read().decode('utf-8'))
            log.info(body_response)
            sagemaker_output = self.parse_sagemaker_output(
                body_response, data['person_threshold'])

        else:  # Otherwise run it in testing mode
            log.debug("mocking output because a blank sagemaker endpoint was put in")
            from random import randint
            body_response = responses[randint(0, 2)]

            sagemaker_output = self.parse_sagemaker_output(
                body_response, data['person_threshold'])
            log.debug(sagemaker_output)

        # If there was no person detected don't send to bucket
        if sagemaker_output is None:
            return JsonResponse([], headers={
                'Access-Control-Allow-Origin': settings.ACCESS_CONTROL_ALLOW_ORIGIN,
            })

        # Get the activity type. Always save the image.
        if data['override'] == "True":
            activity = "override"
        else:
            if sagemaker_output['min_mask'] < data['mask_threshold']:
                activity = "violation"
            else:
                activity = "compliant"

        record_id = uuid4()

        s3 = boto3.client('s3', region_name='us-east-1')
        s3.put_object(
            Bucket=settings.PHOTO_BUCKET_NAME,
            Key=_generate_s3_photo_key(record_id),
            Body=base64.b64decode(data['photo_data']),
            ContentType='image/jpeg',
        )

        repo = Repo(
            host=settings.DB_HOST,
            database=settings.DB_NAME,
            user=settings.DB_USER,
            password=settings.DB_PASSWORD,
        )

        repo.insert_record(
            id=record_id,
            device_serial=data['device_serial'],
            recorded_on=data['timestamp'],
            min_confidence=sagemaker_output['min_mask'],
            people_in_frame=sagemaker_output['people_in_frame'],
            activity=activity
        )
        prepared_response = dict(activity=activity, sagemaker_response=body_response
                                 )

        # return the raw sagemaker output for the RPI to make the decisions
        return JsonResponse(prepared_response, headers={
            'Access-Control-Allow-Origin': settings.ACCESS_CONTROL_ALLOW_ORIGIN,
        })

class FetchActivitiesLambda(Lambda):
    def handle(self, event: Event, context: Context) -> Response:
        repo = Repo(
            host=settings.DB_HOST,
            database=settings.DB_NAME,
            user=settings.DB_USER,
            password=settings.DB_PASSWORD,
        )

        s3 = boto3.client('s3', region_name='us-east-1')

        data = [
            {
                'id': a['id'],
                'type': a['activity'],
                'timestamp': a['recorded_on'],
                'camera': a['device_name'],
                'minConfidence': a['min_confidence'],
                'peopleInFrame': a['people_in_frame'],
                'photoUrl': s3.generate_presigned_url(
                    'get_object',
                    Params={
                        'Bucket': settings.PHOTO_BUCKET_NAME,
                        'Key': _generate_s3_photo_key(a['id']),
                    },
                    ExpiresIn=60 * 60 * 24,
                )
            }
            for a in repo.get_all_activities()
        ]

        return JsonResponse(data, headers={
            'Access-Control-Allow-Origin': settings.ACCESS_CONTROL_ALLOW_ORIGIN,
        })

class RateActivityLambda(Lambda):
    @staticmethod
    def _parse_body(event: Event) -> Optional[dict]:
        try:
            return schemas.apply_schema(schemas.RatingSchema, event.body)
        except schemas.ValidationError:
            log.debug(event.body)
            log.exception("Invalid payload received")
            return None

    def handle(self, event: Event, context: Context) -> Response:
        data = RateActivityLambda._parse_body(event)
        if data is None:
            return JsonResponse(
                status_code=httpstatus.HTTP_400_BAD_REQUEST,
                data={'message': 'Invalid payload'},
            )

        repo = Repo(
            host=settings.DB_HOST,
            database=settings.DB_NAME,
            user=settings.DB_USER,
            password=settings.DB_PASSWORD,
        )

        repo.rate_activity(
            activity_id=data['activity_id'],
            rating=data['rating'],
        )

        return JsonResponse({}, headers={
            'Access-Control-Allow-Origin': settings.ACCESS_CONTROL_ALLOW_ORIGIN,
        })

class FetchStatsLambda(Lambda):
    def handle(self, event: Event, context: Context) -> Response:
        repo = Repo(
            host=settings.DB_HOST,
            database=settings.DB_NAME,
            user=settings.DB_USER,
            password=settings.DB_PASSWORD,
        )

        cameras = repo.get_all_cameras()
        stats = repo.get_stats()

        data = {
            'cameras': [
                {
                    'id': c['id'],
                    'deviceName': c['device_name'],
                    'pingedOn': c['pinged_on'],
                    'compliantCount': c['compliant_count'],
                    'violationCount': c['violation_count'],
                    'overrideCount': c['override_count'],
                }
                for c in cameras
            ],

            # TODO: Implement.
            'activityHistory': {
                'compliant': stats['compliant_count'],
                'violation': stats['violation_count'],
                'override': stats['override_count'],
            }
        }

        return JsonResponse(data, headers={
            'Access-Control-Allow-Origin': settings.ACCESS_CONTROL_ALLOW_ORIGIN,
        })

class PingLambda(Lambda):
    @staticmethod
    def _parse_body(event: Event) -> Optional[dict]:
        try:
            return schemas.apply_schema(schemas.PingSchema, event.body)
        except schemas.ValidationError:
            log.debug(event.body)
            log.exception("Invalid payload received")
            return None

    def handle(self, event: Event, context: Context) -> Response:
        data = PingLambda._parse_body(event)
        if data is None:
            return JsonResponse(
                status_code=httpstatus.HTTP_400_BAD_REQUEST,
                data={'message': 'Invalid payload'},
            )

        repo = Repo(
            host=settings.DB_HOST,
            database=settings.DB_NAME,
            user=settings.DB_USER,
            password=settings.DB_PASSWORD,
        )

        repo.insert_ping(
            device_serial=data['device_serial'],
            device_name=data['device_name'],
        )

        return JsonResponse({}, headers={
            'Access-Control-Allow-Origin': settings.ACCESS_CONTROL_ALLOW_ORIGIN,
        })

upload_handler = UploadLambda().bind()
fetch_activities_handler = FetchActivitiesLambda().bind()
rate_activity_handler = RateActivityLambda().bind()
fetch_stats_handler = FetchStatsLambda().bind()
ping_handler = PingLambda().bind()

max_stars_repo_name: SilentByte/healthcam

max_issues_repo_name: SilentByte/healthcam

max_forks_repo_name: SilentByte/healthcam

reason: import sage


text:
import os
import sys
import time
import json
import torch

import numpy as np
import vegans.utils.utils as utils
import matplotlib.pyplot as plt

from torch.nn import MSELoss
from datetime import datetime
from abc import ABC, abstractmethod
from torch.utils.data import DataLoader
from torchvision.utils import make_grid
from vegans.utils.utils import plot_losses
from torch.utils.tensorboard import SummaryWriter

class AbstractGenerativeModel(ABC):
    """The AbstractGenerativeModel is the most basic building block of vegans. All GAN implementation should
    at least inherit from this class. If a conditional version is implemented look at `AbstractConditionalGenerativeModel`.

    Parameters
    ----------
    x_dim : list, tuple
        Number of the output dimensions of the generator and input dimension of the discriminator / critic.
        In the case of images this will be [nr_channels, nr_height_pixels, nr_width_pixels].
    z_dim : int, list, tuple
        Number of the latent dimensions for the generator input. Might have dimensions of an image.
    optim : dict or torch.optim
        Optimizer used for each network. Could be either an optimizer from torch.optim or a dictionary with network
        name keys and torch.optim as value, i.e. {"Generator": torch.optim.Adam}.
    optim_kwargs : dict
        Optimizer keyword arguments used for each network. Must be a dictionary with network
        name keys and dictionary with keyword arguments as value, i.e. {"Generator": {"lr": 0.0001}}.
    feature_layer : torch.nn.*
        Output layer used to compute the feature loss. Should be from either the discriminator or critic.
        If `feature_layer` is not None, the original generator loss is replaced by a feature loss, introduced
        [here](https://arxiv.org/abs/1606.03498v1).
    fixed_noise_size : int
        Number of images shown when logging. The fixed noise is used to produce the images in the folder/images
        subdirectory, the tensorboard images tab and the samples in get_training_results().
    device : string
        Device used while training the model. Either "cpu" or "cuda".
    ngpu : int
        Number of gpus used during training if device == "cuda".
    folder : string
        Creates a folder in the current working directory with this name. All relevant files like summary, images, models and
        tensorboard output are written there. Existing folders are never overwritten or deleted. If a folder with the same name
        already exists a time stamp is appended to make it unique.
    """

    #########################################################################
    # Actions before training
    #########################################################################
    def __init__(self, x_dim, z_dim, optim, optim_kwargs, feature_layer, fixed_noise_size, device, ngpu, folder, secure):
        self.x_dim = tuple([x_dim]) if isinstance(x_dim, int) else tuple(x_dim)
        self.z_dim = tuple([z_dim]) if isinstance(z_dim, int) else tuple(z_dim)
        self.ngpu = ngpu if ngpu is not None else 0
        self.secure = secure
        self.fixed_noise_size = fixed_noise_size
        self.device = device
        if self.device is None:
            self.device = "cuda" if torch.cuda.is_available() else "cpu"
        elif device not in ["cuda", "cpu"]:
            raise ValueError("device must be cuda or cpu.")

        if feature_layer is not None:
            assert isinstance(feature_layer, torch.nn.Module), (
                "`feature_layer` must inherit from nn.Module. Is: {}.".format(type(feature_layer))
            )
        self.feature_layer = feature_layer
        if self.feature_layer is not None:
            self.feature_layer.to(self.device)
        if not hasattr(self, "folder"):
            if folder is None:
                self.folder = folder
            else:
                self.folder = folder
                if os.path.exists(folder):
                    now = datetime.now()
                    now = now.strftime("%Y%m%d_%H%M%S")
                    self.folder = folder + "_" + now
                os.makedirs(self.folder)

        self.loss_functions = self._define_loss()
        self.optimizers = self._define_optimizers(optim=optim, optim_kwargs=optim_kwargs)
        self.to(self.device)

        self.fixed_noise = self.sample(n=fixed_noise_size)
        self._check_attributes()
        self.hyperparameters = {
            "x_dim": x_dim, "z_dim": z_dim, "ngpu": ngpu, "folder": folder, "optimizers": self.optimizers,
            "device": self.device, "loss_functions": self.loss_functions
        }
        self._init_run = True

        if hasattr(self, "generator") and hasattr(self, "adversary"):
            self.TYPE = "GAN"
            self._Z_transformer = self.generator
            self._X_transformer = self.adversary
        elif hasattr(self, "encoder") and hasattr(self, "decoder"):
            self.TYPE = "VAE"
            self._Z_transformer = self.decoder
            self._X_transformer = self.encoder
        else:
            raise NotImplementedError(
                "Model should either have self.generator and self.adversary (self.TYPE = 'GAN') or \n"+
                "self.decoder and self.encoder (self.TYPE = 'VAE')."
            )
        self.images_produced = True if len(self._Z_transformer.output_size) == 3 else False
        self.eval()

    def _define_optimizers(self, optim, optim_kwargs):
        """ Define the optimizers dictionary.

        After this step self.optimizers must exist and be of the form
        {"Network1": optimizer1, "Network2": optimizer2, ...}

        Parameters
        ----------
        optim : torch.optim or dict
            If dict, must be of form {"Network1": torch.optim1, "Network2": torch.optim2, ...}. If one network is
            not specified as a key of the dictionary the default optimizer (self._default_optimizer()) will be
            fetched.
        optim_kwargs : dict
            If dict, must be of form {"Network1": optim_kwargs1, "Network2": optim_kwargs1, ...}. If one network is
            not specified as a key of the dictionary no arguments = {} are passed.
        """
        dict_optim_kwargs = {}
        for name, _ in self.neural_nets.items():
            dict_optim_kwargs[name] = {}
        if isinstance(optim_kwargs, dict):
            for name, opt_kwargs in optim_kwargs.items():
                dict_optim_kwargs[name] = opt_kwargs
        self._check_dict_keys(param_dict=dict_optim_kwargs, where="_define_optimizers_kwargs")

        dict_optim = {}
        if optim is None:
            for name, _ in self.neural_nets.items():
                dict_optim[name] = self._default_optimizer()
        elif isinstance(optim, dict):
            for name, opt in optim.items():
                dict_optim[name] = opt
            for name, _ in self.neural_nets.items():
                if name not in dict_optim:
                    dict_optim[name] = self._default_optimizer()
        else:
            for name, _ in self.neural_nets.items():
                dict_optim[name] = optim
        self._check_dict_keys(param_dict=dict_optim, where="_define_optimizers")

        optimizers = {}
        for name, network in self.neural_nets.items():
            optimizers[name] = dict_optim[name](params=network.parameters(), **dict_optim_kwargs[name])
        return optimizers

    def _default_optimizer(self):
        return torch.optim.Adam

    def _check_dict_keys(self, param_dict, where):
        """ Checks if `param_dict` has the correct form.

        The correct form is {"Network1": value1, "Network2": value2, ...} for every network.
        This utility function checks if every network in `self.neural_nets` has an entry in `param_dict`
        and if every entry in `param_dict` exits in `self.neural_nets`.

        Parameters
        ----------
        param_dict : TYPE
            Parameter dictionary
        where : TYPE
            Specifies the location from which the function is called for meaningful `KeyError`.

        Raises
        ------
        KeyError
            If `param_dict` is not of the specified form.
        """
        for name, _ in param_dict.items():
            if name not in self.neural_nets:
                available_nets = [name for name, _ in self.neural_nets.items()]
                raise KeyError("Error in {}: `{}` not in self.neural_nets. Must be one of: {}.".format(
                    where, name, available_nets)
                )
        for name, _ in self.neural_nets.items():
            if name not in param_dict:
                raise KeyError("Error in {}: self.{} not in param_dict.".format(
                    where, name)
                )

    def _check_attributes(self):
        assert hasattr(self, "neural_nets"), "Model must have attribute 'neural_nets'."
        assert hasattr(self, "device"), "Model must have attribute 'device'."
        assert hasattr(self, "optimizers"), "Model must have attribute 'optimizers'."
        assert isinstance(self.ngpu, int) and self.ngpu >= 0, "ngpu must be positive integer. Given: {}.".format(ngpu)
        assert len(self.z_dim) == 1 or len(self.z_dim) == 3, (
            "z_dim must either have length 1 (for vector input) or 3 (for image input). Given: {}.".format(z_dim)
        )
        assert len(self.x_dim) == 1 or len(self.x_dim) == 3, (
            "x_dim must either have length 1 (for vector input) or 3 (for image input). Given: {}.".format(x_dim)
        )
        assert isinstance(self.neural_nets, dict), "'neural_nets' attribute of AbstractGenerativeModel must be dictionary."
        self._check_dict_keys(self.optimizers, where="_define_optimizer_kwargs")

    @abstractmethod
    def _define_loss(self):
        pass

    def _set_up_training(self, X_train, y_train, X_test, y_test, epochs, batch_size, steps,
        print_every, save_model_every, save_images_every, save_losses_every, enable_tensorboard):
        """ Create the dataloaders, SummaryWriters for tensorboard and transform the saving indicators.

        This function creates all data needed during training like the data loaders and save steps.
        It also creates the hyperparameter dictionary and the `steps` dictionary.
        """
        train_dataloader, test_dataloader = self._set_up_data(
            X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test, batch_size=batch_size
        )
        nr_test = 0 if X_test is None else len(X_test)

        writer_train = writer_test = None
        if enable_tensorboard:
            assert self.folder is not None, (
                "`folder` argument in constructor was set to `None`. `enable_tensorboard` must be False or `folder` needs to be specified."
            )
            writer_train = SummaryWriter(os.path.join(self.folder, "tensorboard/train/"))
            if X_test is not None:
                writer_test = SummaryWriter(os.path.join(self.folder, "tensorboard/test/"))

        self.steps = self._create_steps(steps=steps)
        save_periods = self._set_up_saver(
            print_every=print_every, save_model_every=save_model_every, save_images_every=save_images_every,
            save_losses_every=save_losses_every, nr_batches=len(train_dataloader)
        )
        self.hyperparameters.update({
            "epochs": epochs, "batch_size": batch_size, "steps": self.steps,
            "print_every": print_every, "save_model_every": save_model_every, "save_images_every": save_images_every,
            "enable_tensorboard": enable_tensorboard, "nr_train": len(X_train), "nr_test": nr_test
        })
        return train_dataloader, test_dataloader, writer_train, writer_test, save_periods

    def _set_up_data(self, X_train, y_train, X_test, y_test, batch_size):
        """ If `X_train` / `X_test` are not data loaders, create them.

        Also asserts their input shapes for consistency.
        """
        x_train_batch, y_train_batch, x_test_batch, y_test_batch = self._get_batch(
            X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test, batch_size=batch_size
        )

        if self.secure:
            self._assert_shapes(X_train=x_train_batch, y_train=y_train_batch, X_test=x_test_batch, y_test=y_test_batch)

        train_dataloader = X_train
        if not isinstance(X_train, DataLoader):
            train_data = utils.DataSet(X=X_train, y=y_train)
            train_dataloader = DataLoader(train_data, batch_size=batch_size)

        test_dataloader = X_test
        if X_test is not None and not isinstance(X_test, DataLoader):
            test_data = utils.DataSet(X=X_test, y=y_test)
            test_dataloader = DataLoader(test_data, batch_size=batch_size)

        return train_dataloader, test_dataloader

    def _get_batch(self, X_train, y_train, X_test, y_test, batch_size):
        """ Returns the first batch of the training and testing data for the consistency checks.
        """
        if isinstance(X_train, DataLoader):
            assert y_train is None, (
                "If `X_train` is of type torch.utils.data.DataLoader, `y_train` must be None. The dataloader must " +
                "return values for X and y when iterating."
            )
            try:
                x_train_batch, y_train_batch = iter(X_train).next()
            except ValueError:
                x_train_batch = iter(X_train).next()
                y_train_batch = None
        else:
            x_train_batch = X_train[:batch_size]
            y_train_batch = y_train[:batch_size] if y_train is not None else None

        if isinstance(X_test, DataLoader):
            assert y_test is None, (
                "If `X_test` is of type torch.utils.data.DataLoader, `y_test` must be None. The dataloader must " +
                "return values for X and y when iterating."
            )
            if X_test is not None:
                try:
                    x_test_batch, y_test_batch = iter(X_test).next()
                except ValueError:
                    x_test_batch = iter(X_test).next()
                    y_test_batch = None
        else:
            x_test_batch = X_test[:batch_size] if X_test is not None else None
            y_test_batch = y_test[:batch_size] if y_test is not None else None

        return x_train_batch, y_train_batch, x_test_batch, y_test_batch

    def _assert_shapes(self, X_train, y_train, X_test, y_test):
        assert len(X_train.shape) == 2 or len(X_train.shape) == 4, (
            "X_train must be either have 2 or 4 shape dimensions. Given: {}.".format(X_train.shape) +
            "Try to use X_train.reshape(-1, 1) or X_train.reshape(-1, 1, height, width)."
        )
        assert X_train.shape[1:] == self.x_dim, (
            "Wrong input shape for adversary / encoder. Given: {}. Needed: {}.".format(X_train.shape, self.x_dim)
        )

        if X_test is not None:
            assert X_train.shape[1:] == X_test.shape[1:], (
                "X_train and X_test must have same dimensions. Given: {} and {}.".format(X_train.shape[1:], X_test.shape[1:])
            )

    def _create_steps(self, steps):
        """ Creates the `self.steps` dictionary.

        The created dictionary must be of the form {"Network1": steps1, "Network2": steps2, ...}. During training
        "Network1" will be trained for `steps1` steps, and so on.
        Functionality similar to `self_define_optimizers()`.

        Parameters
        ----------
        steps : None, dict
            If not None a dictionary of the form {"Network1": steps1, "Network2": steps2, ...} is expected.
            This dictionary might also be partially filled.
        """
        steps = {}
        for name, neural_net in self.neural_nets.items():
            steps[name] = 1
        if steps is not None:
            assert isinstance(steps, dict), "steps parameter must be of type dict. Given: {}.".format(type(steps))
            steps = steps
            for name, _ in self.neural_nets.items():
                if name not in steps:
                    steps[name] = 1
            self._check_dict_keys(steps, where="_create_steps")
        return steps

    def _set_up_saver(self, print_every, save_model_every, save_images_every, save_losses_every, nr_batches):
        """ Calculates saving indicators if strings are passed. Additionally corresponding folders are created
        in `self.folder`.

        Returns
        -------
        Returns all saving indicators as integers.
        """
        print_every = self._string_to_batchnr(log_string=print_every, nr_batches=nr_batches, name="print_every")
        if save_model_every is not None:
            save_model_every = self._string_to_batchnr(log_string=save_model_every, nr_batches=nr_batches, name="save_model_every")
            assert self.folder is not None, (
                "`folder` argument in constructor was set to `None`. `save_model_every` must be None or `folder` needs to be specified."
            )
            os.mkdir(os.path.join(self.folder, "models/"))
        if save_images_every is not None:
            save_images_every = self._string_to_batchnr(log_string=save_images_every, nr_batches=nr_batches, name="save_images_every")
            assert self.folder is not None, (
                "`folder` argument in constructor was set to `None`. `save_images_every` must be None or `folder` needs to be specified."
            )
            os.mkdir(os.path.join(self.folder, "images/"))
        save_losses_every = self._string_to_batchnr(log_string=save_losses_every, nr_batches=nr_batches, name="save_losses_every")
        self.total_training_time = 0
        self.current_timer = time.perf_counter()
        self.batch_training_times = []

        return print_every, save_model_every, save_images_every, save_losses_every

    def _string_to_batchnr(self, log_string, nr_batches, name):
        """ Transforms string of the form "0.2e" into 0.2 and performs basic sanity checks.
        """
        if isinstance(log_string, str):
            assert log_string.endswith("e"), "If `{}` is string, must end with 'e' (for epoch), e.g. 0.25e.".format(name)
            save_epochs = float(log_string.split("e")[0])
            log_string = max([int(save_epochs*nr_batches), 1])
        return log_string

    #########################################################################
    # Actions during training
    #########################################################################
    def fit(self, X_train, X_test=None, epochs=5, batch_size=32, steps=None,
        print_every="1e", save_model_every=None, save_images_every=None, save_losses_every="1e", enable_tensorboard=False):
        """ Trains the model, iterating over all contained networks.

        Parameters
        ----------
        X_train : np.array or torch.utils.data.DataLoader
            Training data for the generative network. Usually images.
        X_test : np.array, optional
            Testing data for the generative network. Must have same shape as X_train.
        epochs : int, optional
            Number of epochs (passes over the training data set) performed during training.
        batch_size : int, optional
            Batch size used when creating the data loader from X_train. Ignored if torch.utils.data.DataLoader is passed
            for X_train.
        steps : dict, optional
            Dictionary with names of the networks to indicate how often they should be trained, i.e. {"Generator": 5} indicates
            that the generator is trained 5 times while all other networks are trained once.
        print_every : int, string, optional
            Indicates after how many batches the losses for the train data should be printed to the console. Can also be a string
            of the form "0.25e" (4 times per epoch), "1e" (once per epoch) or "3e" (every third epoch).
        save_model_every : int, string, optional
            Indicates after how many batches the model should be saved. Can also be a string
            of the form "0.25e" (4 times per epoch), "1e" (once per epoch) or "3e" (every third epoch).
        save_images_every : int, string, optional
            Indicates after how many batches the images for the losses and fixed_noise should be saved. Can also be a string
            of the form "0.25e" (4 times per epoch), "1e" (once per epoch) or "3e" (every third epoch).
        save_losses_every : int, string, optional
            Indicates after how many batches the losses for the train and test data should be calculated. Can also be a string
            of the form "0.25e" (4 times per epoch), "1e" (once per epoch) or "3e" (every third epoch).
        enable_tensorboard : bool, optional
            Flag to indicate whether subdirectory folder/tensorboard should be created to log losses and images.
        """
        if not self._init_run:
            raise ValueError("Run initializer of the AbstractGenerativeModel class is your subclass!")
        train_dataloader, test_dataloader, writer_train, writer_test, save_periods = self._set_up_training(
            X_train, y_train=None, X_test=X_test, y_test=None, epochs=epochs, batch_size=batch_size, steps=steps,
            print_every=print_every, save_model_every=save_model_every, save_images_every=save_images_every,
            save_losses_every=save_losses_every, enable_tensorboard=enable_tensorboard
        )
        max_batches = len(train_dataloader)
        test_x_batch = iter(test_dataloader).next().to(self.device).float() if X_test is not None else None
        print_every, save_model_every, save_images_every, save_losses_every = save_periods
        train_x_batch = iter(train_dataloader).next()
        if len(train_x_batch) != batch_size:
            raise ValueError(
                "Return value from train_dataloader has wrong shape. Should return object of size batch_size. " +
                "Did you pass a dataloader to `X_train` containing labels as well?"
            )

        self.train()
        if save_images_every is not None:
            self._log_images(images=self.generate(z=self.fixed_noise), step=0, writer=writer_train)
        for epoch in range(epochs):
            print("---"*20)
            print("EPOCH:", epoch+1)
            print("---"*20)
            for batch, X in enumerate(train_dataloader):
                batch += 1
                step = epoch*max_batches + batch
                X = X.to(self.device).float()
                Z = self.sample(n=len(X))
                for name, _ in self.neural_nets.items():
                    for _ in range(self.steps[name]):
                        self._losses = self.calculate_losses(X_batch=X, Z_batch=Z, who=name)
                        self._zero_grad(who=name)
                        self._backward(who=name)
                        self._step(who=name)

                if print_every is not None and step % print_every == 0:
                    self._losses = self.calculate_losses(X_batch=X, Z_batch=Z)
                    self._summarise_batch(
                        batch=batch, max_batches=max_batches, epoch=epoch,
                        max_epochs=epochs, print_every=print_every
                    )

                if save_model_every is not None and step % save_model_every == 0:
                    self.save(name="models/model_{}.torch".format(step))

                if save_images_every is not None and step % save_images_every == 0:
                    self._log_images(images=self.generate(z=self.fixed_noise), step=step, writer=writer_train)
                    self._save_losses_plot()

                if save_losses_every is not None and step % save_losses_every == 0:
                    self._log_losses(X_batch=X, Z_batch=Z, mode="Train")
                    if enable_tensorboard:
                        self._log_scalars(step=step, writer=writer_train)
                    if test_x_batch is not None:
                        self._log_losses(X_batch=test_x_batch, Z_batch=self.sample(n=len(test_x_batch)), mode="Test")
                        if enable_tensorboard:
                            self._log_scalars(step=step, writer=writer_test)

        self.eval()
        self._clean_up(writers=[writer_train, writer_test])

    @abstractmethod
    def calculate_losses(self, X_batch, Z_batch, who=None):
        pass

    def _calculate_feature_loss(self, X_real, X_fake):
        """ Calculates feature loss if `self.feature_layer` is not None.

        Every network takes the `feature_layer` argument in its constructor.
        If it is not None, a layer of the discriminator / critic should be specified.
        A feature loss will be calculated which is the MSE between the output for real
        and fake samples of the specified `self.feature_layer`.

        Parameters
        ----------
        X_real : torch.Tensor
            Real samples.
        X_fake : torch.Tensor
            Fake samples.
        """
        X_real_features = self.feature_layer(X_real)
        X_fake_features = self.feature_layer(X_fake)
        feature_loss = MSELoss()(X_real_features, X_fake_features)
        return feature_loss

    def _zero_grad(self, who=None):
        if who is not None:
            self.optimizers[who].zero_grad()
        else:
            [optimizer.zero_grad() for _, optimizer in self.optimizers.items()]

    def _backward(self, who=None):
        assert len(self._losses) != 0, "'self._losses' empty when performing '_backward'."
        if who is not None:
            self._losses[who].backward(retain_graph=True)
        else:
            [loss.backward(retain_graph=True) for _, loss in self._losses.items()]

    def _step(self, who=None):
        if who is not None:
            self.optimizers[who].step()
        else:
            [optimizer.step() for _, optimizer in self.optimizers.items()]

    #########################################################################
    # Logging during training
    #########################################################################
    def _summarise_batch(self, batch, max_batches, epoch, max_epochs, print_every):
        """ Print summary statistics after a specified amount of batches.

        Parameters
        ----------
        batch : int
            Current batch.
        max_batches : int
            Maximum number of total batches.
        epoch : int
            Current epoch.
        max_epochs : int
            Maximum number of total epochs.
        print_every : int
            After how many batches the summary should be printed.
        """
        step = epoch*max_batches + batch
        max_steps = max_epochs*max_batches
        remaining_batches = max_epochs*max_batches - step
        print("Step: {} / {} (Epoch: {} / {}, Batch: {} / {})".format(
            step, max_steps, epoch+1, max_epochs, batch, max_batches)
        )
        print("---"*20)
        for name, loss in self._losses.items():
            print("{}: {}".format(name, loss.item()))

        self.batch_training_times.append(time.perf_counter() - self.current_timer)
        self.total_training_time = np.sum(self.batch_training_times)
        time_per_batch = np.mean(self.batch_training_times) / print_every

        print("\n")
        print("Time left: ~{} minutes (Steps remaining: {}).".format(
            np.round(remaining_batches*time_per_batch/60, 3), remaining_batches
            )
        )
        self.current_timer = time.perf_counter()

    def _log_images(self, images, step, writer):
        if self.images_produced:
            if writer is not None:
                grid = make_grid(images)
                writer.add_image('images', grid, step)

            fig, axs = self._build_images(images)
            if fig is not None:
                plt.savefig(os.path.join(self.folder, "images/image_{}.png".format(step)))
                plt.close()
            print("Images logged.")

    @staticmethod
    def _build_images(images):
        """ Build matplotlib figure containing all images.

        Parameters
        ----------
        images : torch.Tensor()
            A torch Tensor containing the images to be plotted by matplotlib.

        Returns
        -------
        fig, axs: plt.Figure, np.array
            Objects containing the figure as well as all separate axis objects.
        """
        images = images.detach().cpu().numpy()
        if images.shape[1] == 1:
            images = images[:, 0, :, :]
        elif images.shape[1] == 3:
            images = images.reshape((-1, images.shape[2], images.shape[3], 3))
        else:
            return None, None
        nrows = int(np.sqrt(len(images)))
        ncols = len(images) // nrows
        fig, axs = plt.subplots(nrows=nrows, ncols=ncols, figsize=(10, 10))

        for i, (ax, image) in enumerate(zip(np.ravel(axs), images)):
            ax.imshow(image)
            ax.axis("off")
        return fig, axs

    def _log_losses(self, X_batch, Z_batch, mode):
        self._losses = self.calculate_losses(X_batch=X_batch, Z_batch=Z_batch)
        self._append_losses(mode=mode)

    def _append_losses(self, mode):
        if not hasattr(self, "logged_losses"):
            self.logged_losses = self._create_logged_losses()
        for name, loss in self._losses.items():
            self.logged_losses[mode][name].append(self._losses[name].item())

    def _create_logged_losses(self):
        with_test = self.hyperparameters["nr_test"] is not None
        logged_losses = {"Train": {}}
        if with_test:
            logged_losses["Test"] = {}

        for name, _ in self._losses.items():
            logged_losses["Train"][name] = []
            if with_test:
                logged_losses["Test"][name] = []

        return logged_losses

    def _save_losses_plot(self):
        """ Creates the `losses.png` plot in the `self.folder` path.
        """
        if hasattr(self, "logged_losses"):
            fig, axs = plot_losses(self.logged_losses, show=False, share=False)
            plt.savefig(os.path.join(self.folder, "losses.png"))
            plt.close()

    def _log_scalars(self, step, writer):
        """ Log all scalars with tensorboard.
        """
        if writer is not None:
            for name, loss in self._losses.items():
                writer.add_scalar("Loss/{}".format(name), loss.item(), step)
            writer.add_scalar("Time/Total", self.total_training_time / 60, step)
            writer.add_scalar("Time/Batch", np.mean(self.batch_training_times) / 60, step)

    #########################################################################
    # After training
    #########################################################################
    def _clean_up(self, writers=None):
        [writer.close() for writer in writers if writer is not None]

    def get_training_results(self, by_epoch=False, agg=None):
        """ Call after training to get fixed_noise samples and losses.

        Parameters
        ----------
        by_epoch : bool, optional
            If true one loss value per epoch is returned for every logged_loss. Otherwise frequency is given
            by `save_losses_every` argument of `fit`, i.e. `save_losses_every=10` saves losses every 10th batch,
            `save_losses_every="0.25e` saves losses 4 times per epoch.
        agg : None, optional
            Aggregation function used if by_epoch is true, otherwise ignored. Default is np.mean for all batches
            in one epoch.

        Returns
        -------
        samples : np.array
            Images produced with the final model for the fixed_noise.
        losses_dict : dict
            Dictionary containing all loss types logged during training
        """
        samples = self.generate(z=self.fixed_noise)
        losses = self.get_losses(by_epoch=by_epoch, agg=agg)
        return samples, losses

    def get_losses(self, by_epoch=False, agg=None):
        """ Get losses logged during training

        Parameters
        ----------
        by_epoch : bool, optional
            If true one loss value per epoch is returned for every logged_loss. Otherwise frequency is given
            by `save_losses_every` argument of `fit`, i.e. `save_losses_every=10` saves losses every 10th batch,
            `save_losses_every="0.25e` saves losses 4 times per epoch.
        agg : None, optional
            Aggregation function used if by_epoch is true, otherwise ignored. Default is np.mean for all batches
            in one epoch.

        Returns
        -------
        losses_dict : dict
            Dictionary containing all loss types logged during training
        """
        if agg is None:
            agg = np.mean
        assert callable(agg), "agg: Aggregation function must be callable."
        losses_dict = self.logged_losses.copy()
        if by_epoch:
            epochs = self.get_hyperparameters()["epochs"]
            for mode, loss_dict in losses_dict.items():
                for key, losses in loss_dict.items():
                    batches_per_epoch = len(losses) // epochs
                    loss_dict[key] = [losses[epoch*batches_per_epoch:(epoch+1)*batches_per_epoch] for epoch in range(epochs)]
                    loss_dict[key] = [agg(loss_epoch) for loss_epoch in loss_dict[key]]

        return losses_dict

    #########################################################################
    # Saving and loading
    #########################################################################
    def save(self, name=None):
        """ Saves model in the model folder as torch / pickle object.

        Parameters
        ----------
        name : str, optional
            name of the saved file. folder specified in the constructor used
            in absolute path.
        """
        if name is None:
            name = "model.torch"
        if self.folder is not None:
            torch.save(self, os.path.join(self.folder, name))
        else:
            torch.save(self, os.path.join("", name))

        print("Model saved to {}.".format(os.path.join(self.folder, name)))

    @staticmethod
    def load(path):
        """ Load an already trained model.

        Parameters
        ----------
        path : TYPE
            path to the saved file.

        Returns
        -------
        AbstractGenerativeModel
            Trained model
        """
        return torch.load(path)

    #########################################################################
    # Utility functions
    #########################################################################
    def sample(self, n):
        """ Sample from the latent distribution.

        Parameters
        ----------
        n : int
            Number of samples drawn from the latent distribution.

        Returns
        -------
        torch.tensor
            Random numbers with shape of [n, *z_dim]
        """
        return torch.randn(size=(n, *self.z_dim), requires_grad=True, device=self.device)

    def generate(self, z=None, n=None):
        """ Generate output with generator / decoder.

        Parameters
        ----------
        z : None, optional
            Latent input vector to produce an output from.
        n : None, optional
            Number of outputs to be generated.

        Returns
        -------
        np.array
            Output produced by generator / decoder.
        """
        return self(z=z, n=n)

    def predict(self, x):
        """ Use the critic / discriminator to predict if input is real / fake.

        Parameters
        ----------
        x : np.array
            Images or samples to be predicted.

        Returns
        -------
        np.array
            Array with one output per x indicating the realness of an input.
        """
        if not isinstance(x, torch.Tensor):
            x = torch.from_numpy(x).to(self.device)
        predictions = self._X_transformer(x)
        return predictions

    def get_hyperparameters(self):
        """ Returns a dictionary containing all relevant hyperparameters.

        Returns
        -------
        dict
            Dictionary containing all relevant hyperparameters.
        """
        return self.hyperparameters

    def summary(self, save=False):
        """ Print summary of the model in Keras style way.

        Parameters
        ----------
        save : bool, optional
            If true summary is saved in model folder, printed to console otherwise.
        """
        if save:
            assert self.folder is not None, (
                "`folder` argument in constructor was set to `None`. `enable_tensorboard` must be False or `folder` needs to be specified."
            )
            sys_stdout_temp = sys.stdout
            sys.stdout = open(os.path.join(self.folder, 'summary.txt'), 'w')
        for name, neural_net in self.neural_nets.items():
            neural_net.summary()
            print("\n\n")
        print("Hyperparameters\n---------------")
        for key, value in self.get_hyperparameters().items():
            print("{}: ---> {}".format(str(key), str(value)))
        if save:
            sys.stdout = sys_stdout_temp
            sys_stdout_temp

    def get_number_params(self):
        """ Returns the number of parameters in the model.

        Returns
        -------
        dict
            Dictionary containing the number of parameters per network.
        """
        nr_params_dict = {}
        for name, neural_net in self.neural_nets.items():
            nr_params_dict[name] = neural_net.get_number_params()
        return nr_params_dict

    def eval(self):
        """ Set all networks to evaluation mode.
        """
        self.training = False
        [network.eval() for name, network in self.neural_nets.items()]

    def train(self):
        """ Set all networks to training mode.
        """
        self.training = True
        [network.train() for name, network in self.neural_nets.items()]

    def to(self, device):
        """ Map all networks to device.
        """
        [network.to(device) for name, network in self.neural_nets.items()]

    def __call__(self, z=None, n=None):
        if z is not None and n is not None:
            raise ValueError("Only one of 'z' and 'n' is needed.")
        elif z is None and n is None:
            raise ValueError("Either 'z' or 'n' must be not None.")
        elif n is not None:
            z = self.sample(n=n)
        sample = self._Z_transformer(z)
        if self.training:
            return sample
        return sample.detach().cpu().numpy()

    def __str__(self):
        self.summary()


text:
import dataset
import matplotlib.pyplot as plt
import numpy as np
import preprocess

from logging import getLogger
from loss import original_loss
from tensorflow.keras.applications import MobileNetV2
from tensorflow.python.keras.engine.network import Network
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import SGD

logger = getLogger(__name__)

input_shape = (96, 96, 3)
classes = 10
batch_size = 128
feature_out = 1280
alpha = 0.5
lambda_ = 0.1

def main() -> None:
    (x_train_s, x_ref, y_ref), _ = dataset.get_fasion_mnist()
    x_train_s = preprocess.resize(x_train_s)
    x_ref = preprocess.resize(x_ref)

    np.random.seed(0)
    train(x_train_s, x_ref, y_ref, 40)

def train(x_target, x_ref, y_ref, epoch_num):
    """ 学習
    """
    logger.info("bulid model...")
    network = MobileNetV2(
        include_top=True,
        input_shape=input_shape,
        alpha=alpha,
        # depth_multiplier=1,
        weights="imagenet",
    )
    network.layers.pop()
    for layer in network.layers:
        if layer.name == "block_13_expand":
            break
        else:
            layer.trainable = False

    model_t = Model(inputs=network.input, outputs=network.layers[-1].output)
    model_r = Network(inputs=model_t.input, outputs=model_t.output, name="shared_layer")
    prediction = Dense(classes, activation="softmax")(model_t.output)
    model_r = Model(inputs=model_r.input, outputs=prediction)

    optimizer = SGD(lr=5e-5, decay=0.00005)
    model_r.compile(optimizer=optimizer, loss="categorical_crossentropy")
    model_t.compile(optimizer=optimizer, loss=original_loss)

    model_t.summary()
    model_r.summary()

    logger.info(f"x_target shape is {x_target.shape}")
    logger.info(f"x_ref shape is {x_ref.shape}")

    logger.info("training...")
    ref_samples = np.arange(x_ref.shape[0])
    loss, loss_c = [], []
    for epoch in range(epoch_num):
        x_r, y_r, lc, ld = [], [], [], []
        np.random.shuffle(x_target)
        np.random.shuffle(ref_samples)
        for i in range(len(x_target)):
            x_r.append(x_ref[ref_samples[i]])
            y_r.append(y_ref[ref_samples[i]])
        x_r = np.array(x_r)
        y_r = np.array(y_r)
        for i in range(len(x_target) // batch_size):
            batch_target = x_target[i * batch_size : i * batch_size + batch_size]
            batch_ref = x_r[i * batch_size : i * batch_size + batch_size]
            batch_y = y_r[i * batch_size : i * batch_size + batch_size]

            lc.append(
                model_t.train_on_batch(
                    batch_target, np.zeros((batch_size, feature_out))
                )
            )
            ld.append(model_r.train_on_batch(batch_ref, batch_y))
        loss.append(np.mean(ld))
        loss_c.append(np.mean(lc))

        if (epoch + 1) % 1 == 0:
            logger.info(f"epoch: {epoch}")
            logger.info(f"Descriptive loss: {loss[-1]}")
            logger.info(f"Compact loss: {loss_c[-1]}")
    model_t.save("_data/model.h5")

    plt.plot(loss, label="Descriptive loss")
    plt.xlabel("epoch")
    plt.legend()
    plt.show()
    plt.savefig("_data/descriptive_loss.png")

    plt.plot(loss_c, label="Compact loss")
    plt.xlabel("epoch")
    plt.legend()
    plt.show()
    plt.savefig("_data/compact_loss.png")

if __name__ == "__main__":
    import logging

    logging.basicConfig(level=logging.DEBUG)

    main()

max_stars_repo_name: iimuz/til

max_issues_repo_name: iimuz/til

max_forks_repo_name: iimuz/til


text:
from typing import Dict
from typing import Tuple
from typing import Union

import numpy as np
import pandas as pd
import pytest

from etna.datasets import TSDataset

frequencies = ["D", "15min"]
DistributionDict = Dict[str, pd.DataFrame]

@pytest.fixture(params=frequencies, ids=frequencies)
def date_range(request) -> pd.DatetimeIndex:
    """Create pd.Series with range of dates."""
    freq = request.param
    dtr = pd.date_range(start="2020-01-01", end="2020-03-01", freq=freq)
    return dtr

@pytest.fixture
def all_date_present_df(date_range: pd.Series) -> pd.DataFrame:
    """Create pd.DataFrame that contains some target on given range of dates without gaps."""
    df = pd.DataFrame({"timestamp": date_range})
    df["target"] = [i for i in range(len(df))]
    df.set_index("timestamp", inplace=True)
    return df

@pytest.fixture
def all_date_present_df_two_segments(all_date_present_df: pd.Series) -> pd.DataFrame:
    """Create pd.DataFrame that contains two segments with some targets on given range of dates without gaps."""
    df_1 = all_date_present_df.reset_index()
    df_2 = all_date_present_df.copy().reset_index()

    df_1["segment"] = "segment_1"
    df_2["segment"] = "segment_2"

    classic_df = pd.concat([df_1, df_2], ignore_index=True)
    df = TSDataset.to_dataset(classic_df)
    return df

@pytest.fixture
def df_with_missing_value_x_index(random_seed, all_date_present_df: pd.DataFrame) -> Tuple[pd.DataFrame, int]:
    """Create pd.DataFrame that contains some target on given range of dates with one gap."""
    # index cannot be first or last value,
    # because Imputer should know starting and ending dates
    timestamps = sorted(all_date_present_df.index)[1:-1]
    idx = np.random.choice(timestamps)
    df = all_date_present_df
    df.loc[idx, "target"] = np.NaN
    return df, idx

@pytest.fixture
def df_with_missing_range_x_index(all_date_present_df: pd.DataFrame) -> Tuple[pd.DataFrame, list]:
    """Create pd.DataFrame that contains some target on given range of dates with range of gaps."""
    timestamps = sorted(all_date_present_df.index)
    rng = timestamps[2:7]
    df = all_date_present_df
    df.loc[rng, "target"] = np.NaN
    return df, rng

@pytest.fixture
def df_with_missing_range_x_index_two_segments(
    df_with_missing_range_x_index: pd.DataFrame,
) -> Tuple[pd.DataFrame, list]:
    """Create pd.DataFrame that contains some target on given range of dates with range of gaps."""
    df_one_segment, rng = df_with_missing_range_x_index
    df_1 = df_one_segment.reset_index()
    df_2 = df_one_segment.copy().reset_index()
    df_1["segment"] = "segment_1"
    df_2["segment"] = "segment_2"
    classic_df = pd.concat([df_1, df_2], ignore_index=True)
    df = TSDataset.to_dataset(classic_df)
    return df, rng

@pytest.fixture
def df_all_missing(all_date_present_df: pd.DataFrame) -> pd.DataFrame:
    """Create pd.DataFrame with all values set to nan."""
    all_date_present_df.loc[:, :] = np.NaN
    return all_date_present_df

@pytest.fixture
def df_all_missing_two_segments(all_date_present_df_two_segments: pd.DataFrame) -> pd.DataFrame:
    """Create pd.DataFrame with all values set to nan."""
    all_date_present_df_two_segments.loc[:, :] = np.NaN
    return all_date_present_df_two_segments

@pytest.fixture
def daily_exog_ts() -> Dict[str, Union[TSDataset, DistributionDict]]:
    df1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=48),
            "segment": "segment_1",
            "target": 1,
        }
    )
    df2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=48),
            "segment": "segment_2",
            "target": [1] + 23 * [0] + [1] + 23 * [0],
        }
    )
    df = pd.concat([df1, df2], ignore_index=True)

    df_exog1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="D", periods=3),
            "segment": "segment_1",
            "regressor_exog": 2,
        }
    )
    df_exog2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="D", periods=3),
            "segment": "segment_2",
            "regressor_exog": 40,
        }
    )
    df_exog = pd.concat([df_exog1, df_exog2], ignore_index=True)

    target1 = pd.DataFrame(
        {
            "fold": list(range(24)),
            "distribution": 1 / 24,
        }
    )
    target2 = pd.DataFrame(
        {
            "fold": list(range(24)),
            "distribution": [1] + 23 * [0],
        }
    )

    ts = TSDataset(df=TSDataset.to_dataset(df), freq="H", df_exog=TSDataset.to_dataset(df_exog), known_future="all")
    distribution = {"segment_1": target1, "segment_2": target2}
    return {"ts": ts, "distribution": distribution}

@pytest.fixture()
def daily_exog_ts_diff_endings(daily_exog_ts):
    ts = daily_exog_ts["ts"]
    ts.loc[ts.index[-5] :, pd.IndexSlice["segment_1", "target"]] = np.NAN
    return ts

@pytest.fixture
def inplace_resampled_daily_exog_ts() -> TSDataset:
    df1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=48),
            "segment": "segment_1",
            "target": 1,
        }
    )
    df2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=48),
            "segment": "segment_2",
            "target": [1] + 23 * [0] + [1] + 23 * [0],
        }
    )
    df = pd.concat([df1, df2], ignore_index=True)

    df_exog1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=72),
            "segment": "segment_1",
            "regressor_exog": 2 / 24,
        }
    )
    df_exog2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=72),
            "segment": "segment_2",
            "regressor_exog": [40] + 23 * [0] + [40] + 23 * [0] + [40] + 23 * [0],
        }
    )
    df_exog = pd.concat([df_exog1, df_exog2], ignore_index=True)

    ts = TSDataset(df=TSDataset.to_dataset(df), freq="H", df_exog=TSDataset.to_dataset(df_exog), known_future="all")
    return ts

@pytest.fixture
def noninplace_resampled_daily_exog_ts() -> TSDataset:
    df1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=48),
            "segment": "segment_1",
            "target": 1,
        }
    )
    df2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=48),
            "segment": "segment_2",
            "target": [1] + 23 * [0] + [1] + 23 * [0],
        }
    )
    df = pd.concat([df1, df2], ignore_index=True)

    df_exog1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=72),
            "segment": "segment_1",
            "regressor_exog": [2] + 23 * [np.NAN] + [2] + 23 * [np.NAN] + [2] + 23 * [np.NAN],
            "resampled_exog": 2 / 24,
        }
    )
    df_exog2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="H", periods=72),
            "segment": "segment_2",
            "regressor_exog": [40] + 23 * [np.NAN] + [40] + 23 * [np.NAN] + [40] + 23 * [np.NAN],
            "resampled_exog": [40] + 23 * [0] + [40] + 23 * [0] + [40] + 23 * [0],
        }
    )
    df_exog = pd.concat([df_exog1, df_exog2], ignore_index=True)

    ts = TSDataset(df=TSDataset.to_dataset(df), freq="H", df_exog=TSDataset.to_dataset(df_exog), known_future="all")
    return ts

@pytest.fixture
def weekly_exog_same_start_ts() -> Dict[str, Union[TSDataset, DistributionDict]]:
    """Target and exog columns start on Monday."""
    df1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="D", periods=14),
            "segment": "segment_1",
            "target": 1,
        }
    )
    df2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="D", periods=14),
            "segment": "segment_2",
            "target": [1] + 6 * [0] + [1] + 6 * [0],
        }
    )
    df = pd.concat([df1, df2], ignore_index=True)

    df_exog1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="W", periods=3),
            "segment": "segment_1",
            "regressor_exog": 2,
        }
    )
    df_exog2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="W", periods=3),
            "segment": "segment_2",
            "regressor_exog": 40,
        }
    )
    df_exog = pd.concat([df_exog1, df_exog2], ignore_index=True)

    target1 = pd.DataFrame(
        {
            "fold": list(range(7)),
            "distribution": 1 / 7,
        }
    )
    target2 = pd.DataFrame(
        {
            "fold": list(range(7)),
            "distribution": [1] + 6 * [0],
        }
    )
    distribution = {"segment_1": target1, "segment_2": target2}
    ts = TSDataset(df=TSDataset.to_dataset(df), freq="D", df_exog=TSDataset.to_dataset(df_exog), known_future="all")

    return {"ts": ts, "distribution": distribution}

@pytest.fixture
def weekly_exog_diff_start_ts() -> Dict[str, Union[TSDataset, DistributionDict]]:
    """Target starts on Thursday and exog starts on Monday."""
    df1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-08", freq="D", periods=14),
            "segment": "segment_1",
            "target": 1,
        }
    )
    df2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-08", freq="D", periods=14),
            "segment": "segment_2",
            "target": [1] + 6 * [0] + [1] + 6 * [0],
        }
    )
    df = pd.concat([df1, df2], ignore_index=True)

    df_exog1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="W", periods=4),
            "segment": "segment_1",
            "regressor_exog": 2,
        }
    )
    df_exog2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-05", freq="W", periods=4),
            "segment": "segment_2",
            "regressor_exog": 40,
        }
    )
    df_exog = pd.concat([df_exog1, df_exog2], ignore_index=True)

    target1 = pd.DataFrame(
        {
            "fold": list(range(7)),
            "distribution": 1 / 7,
        }
    )
    target2 = pd.DataFrame(
        {
            "fold": list(range(7)),
            "distribution": [0, 0, 0, 1, 0, 0, 0],
        }
    )

    ts = TSDataset(df=TSDataset.to_dataset(df), freq="D", df_exog=TSDataset.to_dataset(df_exog), known_future="all")
    distribution = {"segment_1": target1, "segment_2": target2}
    return {"ts": ts, "distribution": distribution}

@pytest.fixture
def incompatible_freq_ts() -> TSDataset:
    df1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-01", freq="7T", periods=20),
            "segment": "segment_1",
            "target": 1,
        }
    )
    df2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-01", freq="7T", periods=20),
            "segment": "segment_2",
            "target": 2,
        }
    )
    df = pd.concat([df1, df2], ignore_index=True)

    df_exog1 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-01", freq="H", periods=3),
            "segment": "segment_1",
            "exog": 2,
        }
    )
    df_exog2 = pd.DataFrame(
        {
            "timestamp": pd.date_range(start="2020-01-01", freq="H", periods=3),
            "segment": "segment_2",
            "exog": 40,
        }
    )
    df_exog = pd.concat([df_exog1, df_exog2], ignore_index=True)

    ts = TSDataset(df=TSDataset.to_dataset(df), freq="7T", df_exog=TSDataset.to_dataset(df_exog))
    return ts


text:
"""Amplitude statistical module."""

import numpy as np
import scipy.stats

def mean(amplitude_values):
    """Calculate the mean value of the values.

    Args:
        amplitude_values (numpy.ndarray):
            Array of floats representing signal values.

    Returns:
       float:
           `mean` value of the input array.
    """
    return np.mean(amplitude_values)

def std(amplitude_values):
    """Compute the arithmetic mean value of the values.

    Args:
        amplitude_values (numpy.ndarray):
            Array of floats representing signal values.

    Returns:
       float:
           `std` value of the input array.
    """
    return np.std(amplitude_values)

def var(amplitude_values):
    """Compute the variance value of the values.

    Args:
        amplitude_values (numpy.ndarray):
            Array of floats representing signal values.

    Returns:
       float:
           `std` value of the input array.
    """
    return np.var(amplitude_values)

def rms(amplitude_values):
    """Compute the RMS (Root Mean Square) of the values.

    Args:
        amplitude_values (numpy.ndarray):
            Array of floats representing signal values.

    Returns:
       float:
           RMS of the input array.
    """
    return np.sqrt((np.array(amplitude_values) ** 2).mean())

def crest_factor(amplitude_values):
    """Compute the ratio of the peak to the RMS.

    Used for estimating the amount of impact wear in a bearing.

    Args:
        amplitude_values (numpy.ndarray):
            Array of floats representing signal values.

    Returns:
        float:
            The crest factor of the inputted values.
    """
    peak = max(np.abs(amplitude_values))
    return peak / rms(amplitude_values)

def skew(amplitude_values):
    """Compute the sample skewness of an array of values.

    Args:
        amplitude_values (numpy.ndarray):
            Array of floats representing signal values.

    Returns:
       float:
           The skewness value of the input array.
    """
    return scipy.stats.skew(amplitude_values)

def kurtosis(amplitude_values, fisher=True, bias=True):
    """Compute the kurtosis ,Fisher or Pearson, of an array of values.

    Args:
        amplitude_values (numpy.ndarray):
            Array of floats representing signal values.
        fisher (bool):
            If ``True``, Fisher’s definition is used (normal ==> 0.0). If ``False``,
            Pearson’s definition is used (normal ==> 3.0). Defaults to ``True``.
        bias (bool):
            If ``False``, then the calculations are corrected for statistical bias.
            Defaults to ``True``.

    Returns:
       float:
           The kurtosis value of the input array. If all values are equal, return
           `-3` for Fisher's definition and `0` for Pearson's definition.
    """
    return scipy.stats.kurtosis(amplitude_values, fisher=fisher, bias=bias)

max_stars_repo_name: sintel-dev/SigPro

max_issues_repo_name: sintel-dev/SigPro

max_forks_repo_name: signals-dev/SigPro


text:
#!/usr/bin/python3

import matplotlib.pyplot as plt
import numpy as np

def sigmoid(z):
    return 1.0 / (1.0 + np.exp(-z))

z = np.arange(-7, 7, 0.01)
phi_z = sigmoid(z)

plt.plot(z, phi_z)
plt.axvline(0.0, color = 'k')
plt.axhspan(0.0, 1.0, facecolor = '1.0', alpha = 1.0, ls = 'dotted')
plt.axhline(0.5, ls = 'dotted', color = 'k')
plt.yticks([0.0, 0.5, 1.0])
plt.ylim(-0.1, 1.1)
plt.xlabel('z')
plt.ylabel('$\phi (z)$')
plt.show()

max_stars_repo_name: tyburam/python-machine-learning

max_issues_repo_name: tyburam/python-machine-learning

max_forks_repo_name: tyburam/python-machine-learning


text:
from pathlib import Path
from collections import defaultdict
import pandas as pd
import numpy as np
import argparse

class AttachmentPersonTracking(object):
    """ Person tracking Results of Infants and Mothers based on the Kinect Data

    The data is expected to be saved as a csv file with the first line as the header.
    The expected column names are
        1. "ID" for the session id;
        2. "RE" for which reunion the data is related to, either 1 or 2;
        3. "timestamp" for the time when the data point is collected;
        4. "{b, m}Pos3D_{X, Y, Z}" for the 3D tracking results of infants and mothers;
        5. "bHeightFromFloor" for the normalized height of baby from the floor;
        6. "{b, m}InterpolationType" for the indicator of missing data.
            1 for data missing in the beginning, which are filled by first seen data
            2 for normally collected data
            3 for data missing in the end, which are filled by last seen data
            999 for completely missing data
    """

    DEFAULT_CONTACT_THRESHOLD = 800
    DEFAULT_CARRY_THRESHOLD = 900
    DEFAULT_INIT_APP_TIME = 125

    def __init__(self, data_path: Path, output_path: Path):
        self.data = pd.read_csv(data_path)
        self._compute_distance()
        self._compute_approach()
        self.output_path = output_path

    @property
    def sessions(self):
        return list(sorted(set(self.data['ID'])))

    def _get_both_data(self, sess, re):
        features = ['timestamp', 'dist3d']
        for sub in ['b', 'm']:
            for pos in['X', 'Y', 'Z']:
                features.append('{}Pos3D_{}'.format(sub, pos))
            features.append('{}Approach'.format(sub))
        th = 2 if sess < 1000 else 3

        return self.data[(self.data['ID'] == sess) & (self.data['RE'] == re) &
                         (self.data['bInterpolationType'] <= th) & (self.data['mInterpolationType'] <= th)][features]

    def _get_infant_data(self, sess, re):
        features = ['timestamp', 'dist3d', 'bHeightFromFloor', 'bApproach']
        for pos in['X', 'Y', 'Z']:
            features.append('bPos3D_{}'.format(pos))
        th = 2 if sess < 1000 else 3

        return self.data[
            (self.data['ID'] == sess) & (self.data['RE'] == re) & (self.data['bInterpolationType'] <= th)
        ][features]

    def _get_mother_data(self, sess, re):
        features = ['timestamp', 'mApproach']
        for pos in['X', 'Y', 'Z']:
            features.append('mPos3D_{}'.format(pos))
        th = 2 if sess < 1000 else 3

        return self.data[
            (self.data['ID'] == sess) & (self.data['RE'] == re) & (self.data['mInterpolationType'] <= th)
        ][features]

    def _compute_distance(self):
        self.data['dist3d'] = np.sqrt(sum(
            (self.data['bPos3D_{}'.format(pos)] - self.data['mPos3D_{}'.format(pos)]) ** 2
            for pos in ['X', 'Y', 'Z']
        ))

    def _compute_approach(self):
        grouped = self.data.groupby(['ID', 'RE'])
        for gname, gdata in grouped:
            baby_movement = np.stack([
                gdata.iloc[1:]['bPos3D_{}'.format(pos)].to_numpy() -
                gdata.iloc[:-1]['bPos3D_{}'.format(pos)].to_numpy() for pos in ['X', 'Y', 'Z']
            ], axis=0)
            pos_diff = np.stack([
                gdata.iloc[1:]['mPos3D_{}'.format(pos)].to_numpy() -
                gdata.iloc[1:]['bPos3D_{}'.format(pos)].to_numpy() for pos in ['X', 'Y', 'Z']
            ], axis=0)
            self.data.loc[
                (self.data['ID'] == gname[0]) & (self.data['RE'] == gname[1]), 'bApproach'
            ] = np.concatenate((
                np.array([0]),
                np.sum(np.multiply(baby_movement, pos_diff), axis=0) / np.linalg.norm(pos_diff, axis=0) / 40.
            ))

            mother_movement = np.stack([
                gdata.iloc[1:]['mPos3D_{}'.format(pos)].to_numpy() -
                gdata.iloc[:-1]['mPos3D_{}'.format(pos)].to_numpy() for pos in ['X', 'Y', 'Z']
            ], axis=0)
            pos_diff = np.stack([
                gdata.iloc[1:]['bPos3D_{}'.format(pos)].to_numpy() -
                gdata.iloc[1:]['mPos3D_{}'.format(pos)].to_numpy() for pos in ['X', 'Y', 'Z']
            ], axis=0)
            self.data.loc[
                (self.data['ID'] == gname[0]) & (self.data['RE'] == gname[1]), 'mApproach'
            ] = np.concatenate((
                np.array([0]),
                np.sum(np.multiply(mother_movement, pos_diff), axis=0) / np.linalg.norm(pos_diff, axis=0) / 40.
            ))

    def _contact_duration(self, sess, re, th=DEFAULT_CONTACT_THRESHOLD):
        df = self._get_both_data(sess, re)
        return len(set(df[df['dist3d'] < th]['timestamp'])) * 0.04

    def _time_held(self, sess, re, th=DEFAULT_CARRY_THRESHOLD):
        df = self._get_infant_data(sess, re)
        return len(set(df[df['bHeightFromFloor'] > th]['timestamp'])) * 0.04

    def _velocity(self, sess, re, sub, direction):
        if sub == 'b':
            df = self._get_infant_data(sess, re)
        elif sub == 'm':
            df = self._get_mother_data(sess, re)
        else:
            raise NotImplemented

        if direction == 'toward':
            return np.mean(df[df['{}Approach'.format(sub)] > 0]['{}Approach'.format(sub)])
        elif direction == 'away':
            return np.mean(df[df['{}Approach'.format(sub)] < 0]['{}Approach'.format(sub)])
        else:
            raise NotImplemented

    def _initial_approach(self, sess, re, sub, th=DEFAULT_INIT_APP_TIME):
        if sub == 'b':
            df = self._get_infant_data(sess, re)
        elif sub == 'm':
            df = self._get_mother_data(sess, re)
        else:
            raise NotImplemented

        return np.mean(df[:th]['{}Approach'.format(sub)])

    def extract_features(self):
        features = defaultdict(list)

        for s in self.sessions:
            features['ID'].append(s)
            for r in [1, 2]:
                features['ContactDuration_R{}'.format(r)].append(self._contact_duration(s, r))
                features['TimeHeld_R{}'.format(r)].append(self._time_held(s, r))
                features['InfantInitialApproach_R{}'.format(r)].append(self._initial_approach(s, r, 'b'))
                features['InfantVelocityToward_R{}'.format(r)].append(self._velocity(s, r, 'b', 'toward'))
                features['InfantVelocityAway_R{}'.format(r)].append(self._velocity(s, r, 'b', 'away'))
                features['MotherInitialApproach_R{}'.format(r)].append(self._initial_approach(s, r, 'm'))
                features['MotherVelocityToward_R{}'.format(r)].append(self._velocity(s, r, 'm', 'toward'))
                features['MotherVelocityAway_R{}'.format(r)].append(self._velocity(s, r, 'm', 'away'))

        pd.DataFrame(features).to_csv(self.output_path)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Extract Features from person tracking results')
    parser.add_argument('data_path', type=str, help='Path to the person tracking result data')
    parser.add_argument('output_path', type=str, help='Path to store the extracted features')
    args = parser.parse_args()
    data = AttachmentPersonTracking(Path(args.data_path), Path(args.output_path))
    data.extract_features()


text:
from enum import Enum

import numpy as np

# Colab Training
COLAB_TRAINING = False

# Model Defaults
DEFAULT_NUM_HG = 4 # NOTE in their final design, 8 were used
DEFAULT_EPOCHS = 70
DEFAULT_DATA_BASE_DIR = 'data'
DEFAULT_MODEL_BASE_DIR = 'models'
DEFAULT_LOGS_BASE_DIR = 'logs'
DEFAULT_OUTPUT_BASE_DIR = 'output'

DEFAULT_RESUME_DIR_FLAG = '_resume_'
HPE_EPOCH_PREFIX = 'hpe_epoch'
HPE_HOURGLASS_STACKS_PREFIX = 'hpe_hourglass_stacks'

# Loss Functions
class LossFunctionOptions(Enum):
    keras_mse = 0
    weighted_mse = 1
    euclidean_loss = 2
    focal_loss = 3

DEFAULT_LOSS = LossFunctionOptions.keras_mse.name

# Image augmentation
class ImageAugmentationStrength(Enum):
    none = 0
    light = 1
    medium = 2
    heavy = 3
DEFAULT_AUGMENT = ImageAugmentationStrength.none.name

# Output activation
class OutputActivation(Enum):
    linear = 0
    sigmoid = 1
    relu = 2
DEFAULT_ACTIVATION = OutputActivation.linear.name

# Optimizer
class OptimizerType(Enum):
    rmsProp = 0
    adam = 1
DEFAULT_OPTIMIZER = OptimizerType.adam.name

DEFAULT_LEARNING_RATE = 0.005

# Dataset Constants
if COLAB_TRAINING:
    # DEFAULT_TRAIN_ANNOT_PATH = '/content/gdrive/My Drive/COCO/2017/annotations/person_keypoints_train2017.json'
    # DEFAULT_VAL_ANNOT_PATH = '/content/gdrive/My Drive/COCO/2017/annotations/person_keypoints_val2017.json'
    DEFAULT_TRAIN_ANNOT_PATH = '/content/datasets/annotations/person_keypoints_train2017.json'
    DEFAULT_VAL_ANNOT_PATH = '/content/datasets/annotations/person_keypoints_val2017.json'
    DEFAULT_TRAIN_IMG_PATH = '/content/datasets/coco'
    DEFAULT_VAL_IMG_PATH = '/content/datasets/coco'
    DEFAULT_PICKLE_PATH = '/content/gdrive/MyDrive/Colab Data/COCO-Human-Pose/Pickles'
else:
    DEFAULT_TRAIN_ANNOT_PATH = 'data/annotations/person_keypoints_train2017.json'
    DEFAULT_VAL_ANNOT_PATH = 'data/annotations/person_keypoints_val2017.json'
    DEFAULT_TRAIN_IMG_PATH = 'data/coco'
    DEFAULT_VAL_IMG_PATH = 'data/coco'
    DEFAULT_PICKLE_PATH = 'Pickles'

# Backcompatability constants.
COCO_TRAIN_ANNOT_PATH = DEFAULT_TRAIN_ANNOT_PATH
COCO_VAL_ANNOT_PATH = DEFAULT_VAL_ANNOT_PATH

# Order of keypoints in COCO dataset
COCO_KEYPOINT_LABEL_ARR = ["nose", "left_eye", "right_eye", "left_ear", "right_ear", "left_shoulder", "right_shoulder", "left_elbow",
                           "right_elbow", "left_wrist", "right_wrist", "left_hip", "right_hip", "left_knee", "right_knee", "left_ankle", "right_ankle"]

# This array was copied from https://github.com/microsoft/human-pose-estimation.pytorch/blob/master/lib/dataset/coco.py
# The original array was 1-indexed, so we subtract 1 from each element
COCO_SKELETON = np.array([
                    [16, 14],
                    [14, 12],
                    [17, 15],
                    [15, 13],
                    [12, 13],
                    [6, 12],
                    [7, 13],
                    [6, 7],
                    [6, 8],
                    [7, 9],
                    [8, 10],
                    [9, 11],
                    [2, 3],
                    [1, 2],
                    [1, 3],
                    [2, 4],
                    [3, 5],
                    [4, 6],
                    [5, 7]]) - 1

# Colouring for linking joints together
COLOUR_MAP = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd',
            #   '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf',
              '#B73CC7', '#0258C9', '#f76f8e', '#048ba8', '#16db93',
              '#f06449', '#0f7173', '#276FBF', '#791e94', '#407899',
              '#e71d36', '#FFA69E', '#678EC2', '#069E7D']

# Model parameters
INPUT_DIM = (256,256)
INPUT_CHANNELS = 3
OUTPUT_DIM = (64,64)
NUM_CHANNELS = 256

# Data Generator Constants
DEFAULT_BATCH_SIZE = 12 #NOTE need to test optimal batch size
NUM_COCO_KEYPOINTS = 17 # Number of joints to detect
NUM_COCO_KP_ATTRBS = 3 # (x,y,v) * 17 keypoints
BBOX_SLACK = 1.3 # before augmentation, increase bbox size to 130%

KP_FILTERING_GT = 4 # Greater than x keypoints

# NOTE: The effective sigma is downscaled by a factor of 4 (from (256,256) to (64,64)) on each side, so ensure the sigma is appropriately sized
HEATMAP_SIGMA = 4 # As per https://towardsdatascience.com/human-pose-estimation-with-stacked-hourglass-network-and-tensorflow-c4e9f84fd3ce
REVERSE_HEATMAP_SIGMA = 1 # Use reverse sigma when heatmap is reversed by a factor of 4 from (64,64) to (256,256)

# NOTE: Don't use, the scale is applied in the loss function.
HEATMAP_SCALE = 1 #TODO figure out if we want to scale heatmaps, set to 1 for no effect. There are 82 times background pixels to foreground pixels in 7*7 patch of 64*64 heatmap, see same link for HEATMAP_SIGMA
TRAIN_SHUFFLE = True # Can set to false for debug purposes
VAL_SHUFFLE = False

# Data filtering constants
BBOX_MIN_SIZE = 900 # Filter out images smaller than 30x30, TODO tweak

# Output filtering constants
HM_TO_KP_THRESHOLD = 0.2
HM_TO_KP_THRESHOLD_POST_FILTER = HM_TO_KP_THRESHOLD / 5

PCK_THRESHOLD = 0.2
# This default PCK threshold is used when either hip is not present.
# It is empirically chosen by taking the mean torso width in the validation set
# {'mean': 25.273791558055517, 'std': 19.27466898776274}
DEFAULT_PCK_THRESHOLD = PCK_THRESHOLD * 25

# Output save names
OUTPUT_STACKED_HEATMAP = 'heatmaps'
OUTPUT_SKELETON = 'skeleton'
OUTPUT_USER_IMG = 'user_img'

class Generator(Enum):
    train_gen = 0
    val_gen = 1
    representative_set_gen = 2

class Metrics(Enum):
    pck = 0
    oks = 1

# Data aug flip R/L probability
RL_FLIP = 0.5

max_issues_repo_name: axjing/PoseEstimate

max_forks_repo_name: axjing/PoseEstimate


text:
"""
Deriving formulas for parabolic interpolation.
"""

from sympy import *

# Premise: We have a parabola P(x), that intersects points (0, a), (1, b) and (2, c).
# We want to find the minimum point (both x and value).
a, b, c, x = symbols("a b c x")

# P(x) = k_2 * x**2 + k_1 * x + k_0
k0, k1, k2 = symbols("k_0, k_1, k_2")
P = k2 * x**2 + k1 * x + k0

# Let's solve its coefficients.
solution = solve(
    [
        P.subs({x: 0}) - a,
        P.subs({x: 1}) - b,
        P.subs({x: 2}) - c,
    ],
    (k0, k1, k2),
)
P = simplify(P.subs(solution))

print(f"P(x) =\n{P}\n")

# Find the vertex of the parabola by taking the derivative, and solving its zeros.
dP = diff(P, x)

solution, = solve(dP, x)

# Simplify and print the solution.
output = solution
print(f"minimal x =\n{output}\n")

# Find the value of the vertex by subsituting back the solution.
value = P.subs({x: solution})

output = value
print(f"minimal value =\n{output}\n")

max_stars_repo_name: ollpu/tiralabra

max_issues_repo_name: ollpu/tiralabra

max_forks_repo_name: ollpu/tiralabra

reason: from sympy


text:
#!/usr/bin/env python3.6
# -*- coding:utf-8 -*-
__author__ = 'Lu ShaoAn'
__version__ = '1.0'
__date__ = '2021.05.25'
__copyright__ = 'Copyright 2021, PI'

import numpy as np
import cv2
from einops import rearrange, repeat
import torch
from mmdet.models.builder import MODELS

print(MODELS)

imgs = np.load('./test_images.npy')
print(imgs.shape)

new = repeat(imgs[0], 'h w c -> (h 2) (w 2) c')

cv2.imshow('img', imgs[0])
cv2.imshow('new', new)
cv2.waitKey(0)

max_stars_repo_name: lushaoan/mmdet

max_issues_repo_name: lushaoan/mmdet

max_forks_repo_name: lushaoan/mmdet


text:
from unittest import TestCase

import numpy as np

import elegy

class LinearTest(TestCase):
    def test_connects(self):

        x = np.random.uniform(-1, 1, size=(4, 3))
        linear = elegy.nn.Linear(5)

        y_pred = linear.call_with_defaults(rng=elegy.RNGSeq(42))(x)

        assert y_pred.shape == (4, 5)

    def test_on_model(self):

        model = elegy.Model(module=elegy.nn.Linear(2))

        x = np.ones([3, 5])

        y_pred = model.predict(x, initialize=True)
        logs = model.evaluate(x)

max_stars_repo_name: floscha/elegy

max_issues_repo_name: floscha/elegy

max_forks_repo_name: floscha/elegy


text:
import numpy as np
import sys
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

###
numFiles = 20

N_ave_0 = np.zeros((30,30,numFiles))
Z_ave_0 = np.zeros((30,30,numFiles))

for i in np.arange(0,numFiles):
    N = np.load("N_mat_self_rec0_"+str(i)+".npy")
    Z = np.load("Z_mat_self_rec0_"+str(i)+".npy")

    N_ave_0[:,:,i]=N[:,0,-1,:,:].mean(axis=0)
    Z_ave_0[:,:,i]=N[:,0,-1,:,:].mean(axis=0)

np.save("N_ave_0.npy", N_ave_0)
np.save("Z_ave_0.npy", Z_ave_0)

###
numFiles = 20

N_ave_1 = np.zeros((30,30,numFiles))
Z_ave_1 = np.zeros((30,30,numFiles))

for i in np.arange(0,numFiles):
    N = np.load("N_mat_self_rec_"+str(i)+".npy")
    Z = np.load("Z_mat_self_rec_"+str(i)+".npy")

    N_ave_1[:,:,i]=N[:,0,-1,:,:].mean(axis=0)
    Z_ave_1[:,:,i]=N[:,0,-1,:,:].mean(axis=0)

np.save("N_ave_1.npy", N_ave_1)
np.save("Z_ave_1.npy", Z_ave_1)


text:
import gevent
import numpy
from PyMata.pymata import PyMata

def toggle(board):
    board.digital_write(2, False)
    gevent.sleep(2.0)
    board.digital_write(2, True)
    gevent.sleep(2.0)
    board.digital_write(2, False)
    gevent.sleep(2.0)
    board.digital_write(2, True)

def main():
    port = '/dev/ttyACM0'

    base = 0.6
    gust = numpy.sin(numpy.linspace(0, 2*3.14, 100))
    wind = base + (0.3)*gust
    print(wind)

    board = PyMata(port, verbose=True)

    FAN_PIN = 3
    GAUGE_PIN = 6
    POTENTIOMETER_ANALOG_PIN = 0

    board.set_pin_mode(13, board.OUTPUT, board.DIGITAL)
    board.set_pin_mode(FAN_PIN, board.PWM, board.ANALOG)
    board.servo_config(GAUGE_PIN)
    board.set_pin_mode(POTENTIOMETER_ANALOG_PIN, board.INPUT, board.ANALOG)

    for _ in range(0, 100):

        for v in list(wind):
            print(v)

            pot1024 = board.analog_read(POTENTIOMETER_ANALOG_PIN)
            pot = (1.0 / 1024.0) * pot1024

            v_scaled = pot * v

            gauge_degrees = 180 - int(180.0 * v_scaled)
            board.analog_write(GAUGE_PIN, gauge_degrees)

            fan_duty = int(255 * v_scaled)
            board.analog_write(FAN_PIN, fan_duty)

            gevent.sleep(0.025)

    #board.set_pin_mode(PUSH_BUTTON, board.INPUT, board.DIGITAL, cb_push_button)
    #board.set_pin_mode(POTENTIOMETER, board.INPUT, board.ANALOG, cb_potentiometer)

if __name__ == '__main__':
    main()

max_stars_repo_name: mastensg/windportal

max_issues_repo_name: mastensg/windportal

max_forks_repo_name: mastensg/windportal


text:
import numpy as np

def B64read(strin):
    assert len(strin)==15
    str64 = """ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"""
    k = np.zeros(15)
    for i in range(15):
        k[i] = str64.find(strin[i])/63.0
    return k

def B64write(k):
    assert len(k)==15
    str64 = """ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"""
    str=''
    for i in range(15):
        str += str64[int(round(k[i]*63))]
    return str

def foilPoints(k):
    Hmax = .3/2 # Maximum thickness t/c
    Gmax = .05/2 # Maximal TE gap
    almax = 0.35 # Maximum TE angle [rad]
    betmax = 1 # Maximum boat angle [rad]

    # Upper Side
    c = k[5-1]*Hmax
    y8 = k[1-1]*c
    PU = k[3-1]
    x10 = PU
    x11 = PU+k[4-1]*(1-PU)
    x9 = PU-k[2-1]*PU

    # Lower Side
    b = k[10-1]*Hmax
    y6 = -k[6-1]*b
    PL = k[8-1]
    x3 = PL+k[9-1]*(1-PL)
    x4 = PL
    x5 = PL-k[7-1]*(PL)

    # Trailing edge
    g = k[15-1]*Gmax
    a = g*0.5
    al = (2*k[14-1]-1)*almax
    bet = k[13-1]*betmax

    alup = al+bet*0.5
    allo = al-bet*0.5

    # First Limit: Not outside bounding box
    # c >= a+V*sin(al)
    Vmax = 1.0
    if alup>0 and c>a:
        Vmax = np.minimum(Vmax,(c-a)/np.sin(alup))
    if alup<0 and b>3*a:
        Vmax = np.minimum(Vmax,(b-3*a)/np.sin(-alup))
    # Second Limit: No further than x11
    # x11 <= 1-V*k*cos(al)
    # x11-1 <= -V*cos(al)
    # (1-x11)/cos(al) >= V
    Vmax = np.minimum(Vmax,(1-x11)/np.cos(alup))

    VUp = k[11-1]*Vmax
    x12 = 1-VUp*np.cos(alup)
    y12 = a+VUp*np.sin(alup)

    # First Limit: Not outside bounding box
    # c >= a+V*k=1*sin(al)
    Vmax = 1.0
    if allo<0 and b>a:
        Vmax = np.minimum(Vmax,(b-a)/np.sin(-allo))
    if allo>0 and c>3*a:
        Vmax = np.minimum(Vmax,(c-3*a)/np.sin(allo))
    # Second Limit: No further than x11
    # x11 <= 1-V*k*cos(al)
    # x11-1 <= -V*cos(al)
    # (1-x11)/cos(al) >= V
    Vmax = np.minimum(Vmax,(1-x3)/np.cos(allo))
    VLo = k[12-1]*Vmax
    x2 = 1-VLo*np.cos(allo)
    y2 = -a+VLo*np.sin(allo)

    #x2,-x3,-x4,-x5,-x9,-x10,-x11,x12,y2,-y6,-y8,y12,-a,-b,-c

    P = np.zeros((13,2))
    P[0,0] = 1
    P[0,1] = -a
    P[1,0] = x2
    P[1,1] = y2
    P[2,0] = x3
    P[2,1] = -b
    P[3,0] = x4
    P[3,1] = -b

    P[4,0] = x5
    P[4,1] = -b
    P[5,1] = y6
    P[6,0] = 0
    P[7,1] = y8
    P[8,0] = x9
    P[8,1] = c

    P[9,0] = x10
    P[9,1] = c
    P[10,0] = x11
    P[10,1] = c
    P[11,0] = x12
    P[11,1] = y12
    P[12,0] = 1
    P[12,1] = a

    return P

max_stars_repo_name: mranneberg/airfoilDesigner

max_issues_repo_name: mranneberg/airfoilDesigner

max_forks_repo_name: mranneberg/airfoilDesigner


text:
import pytest
import numpy as np

import riptable as rt

# TODO: Implement tests for hstack_any().