text:
# |----------------------- HDF5 DATASET -----------------------|
import copy
import logging
import os
import pickle
import shutil
import zipfile
from typing import Dict, Optional, List, Callable, Tuple, Union

import h5py
import numpy as np
import torch
from torch import Tensor

from climart.data_loading.constants import LEVELS, LAYERS, GLOBALS, EXP_TYPES
from climart.data_loading import constants
from climart.data_transform.normalization import Normalizer
from climart.data_transform.normalization import NormalizationMethod
from climart.data_transform.transforms import AbstractTransform, IdentityTransform
from climart.utils.utils import get_logger, get_target_variable_names, pressure_from_level_array

log = get_logger(__name__)

class ClimART_HdF5_Dataset(torch.utils.data.Dataset):
    """Class for working with multiple HDF5 datasets"""

    def __init__(
            self,
            years: List[int],
            name: str = "",
            data_dir: Optional[str] = None,
            exp_type: str = 'pristine',
            target_type: Union[str, List[str]] = 'shortwave',
            target_variable: Union[str, List[str]] = 'fluxes+hr',
            normalizer: Optional[Normalizer] = None,
            input_transform: Optional[AbstractTransform] = None,
            output_transform: Optional[AbstractTransform] = None,
            load_h5_into_mem: bool = False,
            verbose: bool = True
    ):
        """
        """
        self._name = str(name)
        self.normalizer = copy.deepcopy(normalizer)
        if not verbose:
            log.setLevel(logging.WARNING)

        if isinstance(years, int):
            years = [years]
        assert all([y in constants.ALL_YEARS for y in years]), 'All years must be within 1979-2014.'

        if not isinstance(input_transform, AbstractTransform):
            input_transform = IdentityTransform(exp_type)
        if not isinstance(output_transform, AbstractTransform):
            output_transform = IdentityTransform(exp_type)

        self.years = years
        self.n_years = len(years)
        self._load_h5_into_mem = load_h5_into_mem
        self._layer_mask = 45 if exp_type == constants.CLEAR_SKY else 14

        if data_dir is None:
            log.info(" No data_dir was specified, defaulting to climart.data_loading.constants.DATA_DIR")
            data_dir = constants.DATA_DIR

        self.dataset_index_to_sub_dataset: Dict[int, Tuple[int, int]] = dict()
        dataset_size = 0
        dset_kwargs = dict(
            data_dir=data_dir,
            exp_type=exp_type,
            target_type=target_type,
            target_variable=target_variable
        )
        if load_h5_into_mem:
            log.info(f" {self.name} Loading the H5 into RAM & pre-processing it!")
            dset_class = RT_HdF5_FastSingleDataset
            # normalization is done before starting training
            dset_kwargs['input_normalizer'] = self.normalizer.get_normalizers()
            self.normalizer.set_input_normalizers(new_normalizer=None)

            dset_kwargs['input_transform'] = input_transform  # batched
            dset_kwargs['output_transform'] = output_transform

            self._input_transform = IdentityTransform(exp_type)
            self._output_transform = IdentityTransform(exp_type)
        else:
            dset_class = RT_HdF5_SingleDataset
            self._input_transform: AbstractTransform = input_transform  # not batched
            self._output_transform: AbstractTransform = output_transform

        self.h5_dsets: List[dset_class] = []
        for file_num, year in enumerate(years):
            year_h5_dset = dset_class(filename=f"{str(year)}.h5", **dset_kwargs)
            n_samples_of_year = len(year_h5_dset)
            for h5_file_idx in range(n_samples_of_year):
                self.dataset_index_to_sub_dataset[h5_file_idx + dataset_size] = (file_num, h5_file_idx)
            dataset_size += n_samples_of_year
            self.h5_dsets.append(year_h5_dset)
        self.dataset_size = dataset_size

        for dset in self.h5_dsets:
            dset.copy_to_slurm_tmp_dir()

        log.info(self)

    @property
    def name(self):
        return self._name.upper()

    def set_input_transform(self, new_transform: Callable):
        self._input_transform = new_transform

    def __str__(self):
        s = f" {self.name} dataset: {self.n_years} years used, with a total size of {len(self)} examples."
        return s

    def __len__(self):
        return self.dataset_size

    def __getitem__(self, item) -> (Dict[str, Tensor], Tensor):  # Dict[str, Tensor]):
        which_h5, h5_index = self.dataset_index_to_sub_dataset[item]
        raw_Xs, raw_Ys = self.h5_dsets[which_h5][h5_index]
        if self._load_h5_into_mem:
            X = raw_Xs
            Y = raw_Ys
        else:
            Xs = self.normalizer(raw_Xs['data'])
            Xs = self._input_transform.transform(Xs)
            X = {'data': Xs,
                 'level_pressure_profile': raw_Xs['level_pressure_profile']}

            Y = self._output_transform.transform(raw_Ys)
            Y = {k: torch.from_numpy(v).float() for k, v in Y.items()}

        return X, Y

    @property
    def spatial_dim(self) -> Dict[str, int]:
        return self.h5_dsets[0].spatial_dim

    @property
    def input_dim(self) -> Dict[str, int]:
        return self.h5_dsets[0].input_dim

    @property
    def output_dim(self) -> Dict[str, int]:
        return 100  # self.h5_dsets[0].output_dim

    def close(self):
        for dset in self.h5_dsets:
            dset.close()

class RT_HdF5_SingleDataset(torch.utils.data.Dataset):
    def __init__(self,
                 filename: str,
                 data_dir: str,
                 exp_type: str = 'pristine',
                 target_type: Union[str, List[str]] = 'shortwave',
                 target_variable: Union[str, List[str]] = 'fluxes+hr'
                 ):
        assert exp_type in EXP_TYPES, f"arg `exp_type`= {exp_type} is not one of {str(EXP_TYPES)}!"
        self.exp_type = exp_type
        self._filename = filename
        self._name = self._filename.replace('.h5', '')
        self._target_variables = get_target_variable_names(target_type, target_variable)
        self._layer_mask = 45 if exp_type == constants.CLEAR_SKY else 14

        dirs = constants.get_data_subdirs(data_dir)
        in_dir, out_dir = dirs[constants.INPUTS], dirs[exp_type]
        self._in_path = os.path.join(in_dir, filename)
        self._out_path = os.path.join(out_dir, filename)
        with h5py.File(self._in_path, 'r') as h5f:
            self._num_examples = h5f[LAYERS].shape[0]
            n_levels = h5f[LEVELS].shape[1]
            n_layers = h5f[LAYERS].shape[1]
            self._spatial_dim = {GLOBALS: 0, LEVELS: n_levels, LAYERS: n_layers}

            n_glob_feats = h5f[GLOBALS].shape[1]
            n_level_feats = h5f[LEVELS].shape[2]
            n_layer_feats = h5f[LAYERS][..., :self._layer_mask].shape[2]

            self._in_dim = {GLOBALS: n_glob_feats, LEVELS: n_level_feats, LAYERS: n_layer_feats}

        with h5py.File(self._out_path, 'r') as h5f:
            self._out_dim = {
                key: h5f[key].shape[-1] for key in self._target_variables
            }

    def copy_to_slurm_tmp_dir(self):
        if 'SLURM_TMPDIR' in os.environ:
            log.info(f' Copying {self.name} h5 file to SLURM_TMPDIR')
            h5_path_new_in = os.environ['SLURM_TMPDIR'] + '/input_' + self._filename
            shutil.copyfile(self._in_path, h5_path_new_in)
            self._in_path = h5_path_new_in

            h5_path_new_out = os.environ['SLURM_TMPDIR'] + '/output_' + self._filename
            shutil.copyfile(self._out_path, h5_path_new_out)
            self._out_path = h5_path_new_out

    # def __str__(self):
    #     return get_exp_ID(self.exp_type, self.target_types, self.target_variables)

    @property
    def name(self):
        return self._name

    @property
    def input_path(self):
        return self._in_path

    @property
    def output_path(self):
        return self._out_path

    def __len__(self):
        return self._num_examples

    def __getitem__(self, index) -> (Dict[str, np.ndarray], Dict[str, np.ndarray]):
        with h5py.File(self.input_path, 'r') as h5f:
            X = {
                LAYERS: np.array(h5f[LAYERS][index, :, :self._layer_mask]),
                LEVELS: np.array(h5f[LEVELS][index]),
                GLOBALS: np.array(h5f[GLOBALS][index]),
            }
            X = {'data': X, 'level_pressure_profile': pressure_from_level_array(X[LEVELS])}
        with h5py.File(self.output_path, 'r') as h5f:
            Y = {
                output_var: np.array(h5f[output_var][index])
                for output_var in self._target_variables
            }

        return X, Y

    def get_raw_input_data(self) -> Dict[str, np.ndarray]:
        with h5py.File(self.input_path, 'r') as h5f:
            raw_data = {
                LAYERS: np.array(h5f[LAYERS][..., :self._layer_mask]),
                LEVELS: np.array(h5f[LEVELS]),
                GLOBALS: np.array(h5f[GLOBALS])
            }
        return raw_data

    def get_dataset_statistics(self, statistics: List[str] = None) -> Dict[str, np.ndarray]:
        if statistics is None:
            statistics = ['mean', 'std']
        raw_data = self.get_raw_input_data()
        stats = dict()
        axes_to_norm = {GLOBALS: 0, LEVELS: (0, 1), LAYERS: (0, 1)}
        for data_type, data in raw_data.items():
            if 'mean' in statistics:
                stats[data_type + '_mean'] = np.mean(data, axis=axes_to_norm[data_type], dtype=np.float64)
            if 'std' in statistics:
                stats[data_type + '_std'] = np.std(data, axis=axes_to_norm[data_type], dtype=np.float64)
            if 'min' in statistics:
                stats[data_type + '_min'] = np.min(data, axis=axes_to_norm[data_type])
            if 'max' in statistics:
                stats[data_type + '_max'] = np.max(data, axis=axes_to_norm[data_type])

        return stats

    @property
    def spatial_dim(self) -> Dict[str, int]:
        return self._spatial_dim

    @property
    def input_dim(self) -> Dict[str, int]:
        return self._in_dim

    @property
    def output_dim(self) -> Dict[str, int]:
        return self._out_dim

    def close(self):
        pass

class RT_HdF5_FastSingleDatasetTwo(RT_HdF5_SingleDataset):
    def __init__(self, input_preprocesser: Optional[Dict[str, Callable]], *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.input_data = h5py.File(self.input_path, 'r')
        self.output_data = h5py.File(self.output_path, 'r')

    def close(self):
        self.input_data.close()
        self.output_data.close()

    def __getitem__(self, index) -> (Dict[str, np.ndarray], Dict[str, np.ndarray]):
        X = {
            LAYERS: np.array(self.input_data[LAYERS][index, :, :self._layer_mask]),
            LEVELS: np.array(self.input_data[LEVELS][index]),
            GLOBALS: np.array(self.input_data[GLOBALS][index])
        }
        X = {'data': X,
             'level_pressure_profile': pressure_from_level_array(X[LEVELS])
             }
        Y = {
            output_var: np.array(self.output_data[output_var][index])
            for output_var in self._target_variables
        }

        return X, Y

def get_processed_fname(h5_name: str, ending='.npz', **kwargs):
    processed = h5_name.replace('.h5', '').replace('hdf5/', 'numpy/')
    for k, v in kwargs.items():
        if v is None:
            name = 'No'
        elif isinstance(v, str):
            name = v.upper()
        elif isinstance(v, NormalizationMethod) or isinstance(v, AbstractTransform):
            name = v.__class__.__name__
        else:
            try:
                name = v.__qualname__.replace('.', '_')
            except Exception as e:
                print(e)
                name = "?"
        processed += f'_{name}{k}'
    return processed + f'{ending}'

class RT_HdF5_FastSingleDataset(RT_HdF5_SingleDataset):
    def __init__(self,
                 input_normalizer: Optional[Dict[str, Callable]] = None,
                 input_transform: Optional[AbstractTransform] = None,
                 output_transform: Optional[AbstractTransform] = None,
                 write_data: bool = True,
                 reload_if_exists: bool = True,
                 *args, **kwargs):
        super().__init__(*args, **kwargs)
        pkwargs = dict(
            in_normalizer=input_normalizer[GLOBALS],
            in_transform=input_transform,
            out_transform=output_transform
        )
        ending = '.npz' if self.exp_type == 'pristine' else f"_{self.exp_type}.npz"
        input_processed_fname = get_processed_fname(self.input_path, **pkwargs, ending=ending)
        output_processed_fname = get_processed_fname(self.output_path, **pkwargs, ending='.npz')
        # if os.path.isfile(input_processed_fname) and reload_if_exists:
        #    self.input_data = self._reload_data(input_processed_fname)
        # if os.path.isfile(output_processed_fname) and reload_if_exists:
        #    self.output_data = self._reload_data(output_processed_fname)

        if os.path.isfile(input_processed_fname) and os.path.isfile(output_processed_fname) and reload_if_exists:
            self.input_data = self._reload_data(input_processed_fname)
            self.output_data = self._reload_data(output_processed_fname)

        elif os.path.isfile(input_processed_fname) and reload_if_exists:
            self.input_data = self._reload_data(input_processed_fname)
            self._preprocess_h5data(input_normalizer, input_transform, output_transform, inputs=False)
            if write_data:
                self._write_data(output_processed_fname, self.output_data)

        else:
            self._preprocess_h5data(input_normalizer, input_transform, output_transform)
            if write_data:
                self._write_data(input_processed_fname, self.input_data)
                self._write_data(output_processed_fname, self.output_data)

        if isinstance(self.input_data, dict) and 'edges' in self.input_data and self.input_data['edges'].shape[
            -1] > 14 and self.exp_type == 'pristine':
            log.info('overwriting', input_processed_fname)
            self._preprocess_h5data(input_normalizer, input_transform, output_transform, outputs=False)
            self._write_data(input_processed_fname, self.input_data)

    def _reload_data(self, fname):
        log.info(f' Reloading from {fname}')
        try:
            in_data = np.load(fname, allow_pickle=True)
        except zipfile.BadZipFile as e:
            log.warning(f"{fname} was not properly saved or has been corrupted.")
            raise e
        try:
            in_files = in_data.files
        except AttributeError:
            return in_data

        if len(in_files) == 1:
            return in_data[in_files[0]]
        else:
            return {k: in_data[k] for k in in_files}

    def _write_data(self, fname, data):
        os.makedirs(os.path.dirname(fname), exist_ok=True)

        if isinstance(data, dict) or isinstance(data, np.ndarray):
            try:
                np.savez_compressed(fname, **data) if isinstance(data, dict) else np.savez_compressed(fname, data)
            except OverflowError:
                with open(fname, "wb") as fp:
                    pickle.dump(data, fp, pickle.HIGHEST_PROTOCOL)
            except PermissionError as e:
                log.warning(f" Tried to cache data to {fname} but got error {e}. "
                            f"Consider adjusting the permissions to cache the data and make training more efficient.")
        else:
            raise ValueError(f"Data has type {type(data)}")

    def _preprocess_h5data(self,
                           input_normalizer: Optional[Dict[str, Callable]] = None,
                           input_transform: AbstractTransform = None,
                           output_transform: AbstractTransform = None,
                           inputs: bool = True,
                           outputs: bool = True,
                           ):
        if inputs:
            with h5py.File(self.input_path, 'r') as input_data_h5:
                input_data = {
                    LAYERS: np.array(input_data_h5[LAYERS][:, :, :self._layer_mask]),
                    LEVELS: np.array(input_data_h5[LEVELS]),
                    GLOBALS: np.array(input_data_h5[GLOBALS])
                }
                level_pressure_profile = pressure_from_level_array(input_data[LEVELS])

            if input_normalizer is not None:
                for k in constants.INPUT_TYPES:
                    input_data[k] = input_normalizer[k](input_data[k])

            if input_transform is not None:
                input_data = input_transform.batched_transform(input_data)
            self.input_data = {
                'data': input_data,
                'level_pressure_profile': level_pressure_profile
            }

        if outputs:
            with h5py.File(self.output_path, 'r') as output_data_h5:
                output_data = {
                    output_var: np.array(output_data_h5[output_var])
                    for output_var in self._target_variables
                }

            if output_transform is not None:
                output_data = output_transform.batched_transform(X=output_data)
            self.output_data = output_data

    def copy_to_slurm_tmp_dir(self):
        pass

    def __getitem__(self, index) -> (Dict[str, np.ndarray], Dict[str, np.ndarray]):
        if isinstance(self.input_data['data'], np.ndarray) and len(self.input_data['data'].shape) == 0:
            # Recover a nested dictionary of np arrays back, i.e. transform np.ndarray of object dtype into dict
            self.input_data['data'] = self.input_data['data'].item()

        if isinstance(self.input_data['data'], np.ndarray):
            X = torch.from_numpy(self.input_data['data'][index]).float()
        else:
            X = {k: torch.from_numpy(v[index]).float() for k, v in self.input_data['data'].items()}

        X = {'data': X,
             'level_pressure_profile': torch.from_numpy(self.input_data['level_pressure_profile'][index]).float()
             }

        if isinstance(self.output_data, np.ndarray):
            Y = torch.from_numpy(self.output_data[index]).float()
        else:
            Y = {k: torch.from_numpy(v[index]).float() for k, v in self.output_data.items()}

        return X, Y

max_stars_repo_name: RolnickLab/climart

max_issues_repo_name: RolnickLab/climart

max_forks_repo_name: RolnickLab/climart


text:
# TODO keyboard exploration to create terminal state
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import argparse
import signal
import sys

from agent.environment.ai2thor_file import \
    THORDiscreteEnvironment as THORDiscreteEnvironmentFile
from agent.utils import populate_config
import numpy as np

from agent.tools import SimpleImageViewer

#
# Navigate the scene using your keyboard
#
actions = ["MoveAhead", "RotateRight",
           "RotateLeft", "MoveBack", "LookUp", "LookDown", "MoveLeft", "MoveRight"]

def reset_env(name, target):
    # Reset to new env
    def h5_file_lambda(s): return "data_old/{scene}.h5".replace("{scene}", s)
    env = THORDiscreteEnvironmentFile(
        scene_name=name, terminal_state=target, method="random", reward="soft_goal", h5_file_path=h5_file_lambda, action_size=9)
    env.reset()
    return env

def key_press(key, mod):

    global human_agent_action, human_wants_restart, stop_requested, info
    if key == ord('R') or key == ord('r'):  # r/R
        human_wants_restart = True
    if key == ord('Q') or key == ord('q'):  # q/Q
        stop_requested = True
    if key == 0xFF52:  # up
        human_agent_action = 0
    if key == 0xFF53:  # right
        human_agent_action = 7
    if key == 0xFF51:  # left
        human_agent_action = 6
    if key == 0xFF54:  # down
        human_agent_action = 3
    if key == 105:  # i key LookUp
        human_agent_action = 4
    if key == 107:  # k key LookDown
        human_agent_action = 5
    if key == 106:  # j key RotateRight
        human_agent_action = 2
    if key == 108:  # l key RotateLeft
        human_agent_action = 1
    if key == 32:
        info = True
    print(key)

def rollout(env):

    global human_agent_action, human_wants_restart, stop_requested, info
    human_agent_action = None
    human_wants_restart = False
    state = None
    # env.reset()
    while True:
        # waiting for keyboard input
        if human_agent_action is not None:
            # move actions
            state = env.step(env.acts.index(actions[human_agent_action]))
            human_agent_action = None

        # waiting for reset command
        if human_wants_restart:
            # reset agent to random location
            env.reset()
            human_wants_restart = False
        if info:
            # print(env.get_state.instance_detections2D)
            for bbox in env.boudingbox.keys():
                print(bbox.split('|')[0])
            info = False
        # check quit command
        if stop_requested:
            break
        viewer.imshow(env.observation)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument("-s", "--scene_name", type=str, default="FloorPlan1",
                        help="AI2THOR scene name")
    args = parser.parse_args()

    print("Loading scene dump {}".format(args.scene_name))

    env = reset_env(args.scene_name, {"object":"GarbageCan"})
    human_agent_action = None
    human_wants_restart = False
    stop_requested = False
    info = False

    viewer = SimpleImageViewer()
    viewer.imshow(np.zeros((300, 400, 3), dtype=np.uint8))
    viewer.window.on_key_press = key_press

    print("Use arrow keys to move the agent.")
    print("Press R to reset agent\'s location.")
    print("Press Q to quit.")

    rollout(env)

    print("Goodbye.")

ext: py

max_stars_repo_name: norips/visual-navigation-agent-pytorch

max_issues_repo_name: norips/visual-navigation-agent-pytorch

max_forks_repo_name: norips/visual-navigation-agent-pytorch

reason: import numpy


text:
# RUN THIS FILE FROM THE TERMINAL OR COMMAND LINE

# THIS IS AN EXAMPLE TO CREATE MULTIPLE TIME SERIES

# BELOW YOU WILL FIND THE CONFIGURATION SECTION

# THIS EXAMPLE CONSIDERS PARALLEL PROCESSING TO SPEED UP THE DRIVING PROFILE GENERATION

from emobpy import Mobility, Availability, Charging, Export, DataBase
import pandas as pd
import numpy as np
import random
import psutil
from multiprocessing import Lock, Process, Queue

nr_workers = min(20, psutil.cpu_count()-1)

try:
    import ray
    is_ray = True
    deco = ray.remote
    ray.init(num_cpus=nr_workers, ignore_reinit_error=True, include_webui=False)
except ImportError:
    deco = None
    is_ray = False

def raydecorator(condition, decorator):
    return decorator if condition else lambda x: x

@raydecorator(is_ray, deco)
def evsteps(odir, s1, s2, s3, p, r, seed):
    print('evsteps seed:', seed)
    np.random.seed(seed)
    m = Mobility()
    m.setParams(p)
    m.setStats(s1, s2, s3)
    m.setRules(r)
    m.run()
    m.save_profile(odir)
    del m
    return None

def drive_iter(odir,s1,s2,s3,p,r,from_,to_):
    prequeue = []
    for i in range(from_, to_):
        prequeue.append([odir, s1, s2, s3, p, r])
    return prequeue

@raydecorator(is_ray, deco)
def gridsteps(whenAtHome,
               whenAtWork,
               obs,
               code,
               battery_capacity,
               charging_eff,
               soc_init,
               soc_min,
               db,
               fold,
               battopt):
    prob = whenAtHome[obs['whenathome']]
    if obs['g_type'] == 'commuter':
        prob['prob_charging_point']['workplace'] = whenAtWork[obs['whenatwork']]
    a = Availability(code)
    a.setScenario(prob)
    a.setVehicleFeature(battery_capacity, charging_eff)
    a.setBatteryRules(soc_init, soc_min, battopt)
    a.loadSettingDriving(db)
    a.run()
    a.save_profile(fold, obs)
    return None

def scen_solve_mov(func, queue, queue_lock, io_lock):
    while True:
        queue_lock.acquire()
        if queue.empty():
            queue_lock.release()
            return
        lsconf, seed = queue.get()
        queue_lock.release()
        func(*lsconf,seed)
    return None

def scen_solve(func, queue, queue_lock, io_lock):
    while True:
        queue_lock.acquire()
        if queue.empty():
            queue_lock.release()
            return
        lsconf = queue.get()
        queue_lock.release()
        func(*lsconf)
    return None

if __name__ == '__main__':

### EDIT THIS SECTION TO CREATE YOUR OWN SCENARIO ###

####################################################
########### Initial configuration parameters #######
####################################################

    dbdir = '/home/dbeier/git-projects/emobpy_examples/casestudy/test123' # directory to store the profiles
    run_driving = True
    run_availability = False
    run_charging = False
    run_export = False

####################################################
############# Configuration for driving ############
####################################################
    if run_driving:

        bev_n = 25  # number of driving time series
        dist = {'fulltime': 0.485, 'parttime': 0.135, 'freetime': 0.380}  # share of profiles
        hours = 24*14  # Total hours for each time series
        timestep = 0.5 # in hours. e.g. 0.5 means half hour
        consumption = 0.18  # kWh/km
        refdate = '31/12/2018'  # time series starting date. Starting time always 00:00

        seeds = list(range(bev_n))

        stat1 = pd.read_csv('vehiclestats/1_group_trips_weekdays.csv')  # importing statistics
        stat2 = pd.read_csv('vehiclestats/2_time_purpose_group_weeks_hrs.csv') # importing statistics. Time steps in the file must match the timestep variable. e.g. If the variable is 0.5 then in the file the time steps should be 0.5 as well
        stat3 = pd.read_csv('vehiclestats/3_km_range_purpose_car.csv') # importing statistics

        # param dictionary. Person key must match with the stat1 column "group" and with the stat2. Open the csv file and compare
        param_comm_fulltime = {'person':'fulltime',
                               'group':'commuter',
                               'refdate':refdate,
                               'energy_consumption':consumption,
                               'hours':hours,
                               'timestep_in_hrs':timestep}

        param_comm_parttime = {'person':'parttime',
                               'group':'commuter',
                               'refdate':refdate,
                               'energy_consumption':consumption,
                               'hours':hours,
                               'timestep_in_hrs':timestep}

        param_free = {'person':'freetime',
                      'group':'freetime',
                      'refdate':refdate,
                      'energy_consumption':consumption,
                      'hours':hours,
                      'timestep_in_hrs':timestep}

        rules_comm_fulltime={'weekday':
                                {'n_trip_out': [1],
                                 'last_trip_to':{'home':True},
                                 'at_least_one_trip':{'workplace':True},
                                 'overall_min_time_at':{'home':9,'workplace':7.0},
                                 'overall_max_time_at':{'workplace':8.0},
                                 'min_state_duration':{'workplace':3.5}
                                },
                            'weekend':
                                {'n_trip_out': [1],
                                 'last_trip_to':{'home':True},
                                 'overall_min_time_at':{'home':6,'workplace':3},
                                 'overall_max_time_at':{'workplace':4},
                                 'min_state_duration':{'workplace':3}
                                }
                            }

        rules_comm_parttime={'weekday':
                                {'n_trip_out': [1],
                                 'last_trip_to':{'home':True},
                                 'at_least_one_trip':{'workplace':True},
                                 'overall_min_time_at':{'home':9,'workplace':3.5},
                                 'overall_max_time_at':{'workplace':4},
                                 'min_state_duration':{'workplace':3.5}
                                },
                            'weekend':
                                {'n_trip_out': [1],
                                 'last_trip_to':{'home':True},
                                 'overall_min_time_at':{'home':6,'workplace':3},
                                 'overall_max_time_at':{'workplace':4},
                                 'min_state_duration':{'workplace':3}
                                }
                            }

        rules_freetime={'weekday':
                            {'n_trip_out': [1],
                             'last_trip_to':{'home':True},
                             'overall_min_time_at':{'home':9}
                            },
                        'weekend':
                            {'n_trip_out': [1],
                             'last_trip_to':{'home':True},
                             'overall_min_time_at':{'home':6}
                            }
                       }

####################################################
######## Configuration for grid availability #######
####################################################

    if run_availability:

        soc_init = 0.5   # initial state of charge. soc_init*100 = %
        soc_min = 0.01    # profile can not have hours with state of charge lower than this value. soc_min*100 = %
        battery_capacity = 40  # kWh
        battopt = list(range(battery_capacity + 5, battery_capacity*4, 5))  # in case the motor electricity demand is high (long distance trips).
                                                                            # The battery_capacity may be not enough.
                                                                            # Then this list has several battery sizes that are tested until a size is good enough to fullfil demand requirements.
                                                                            # However, it may happend that this list values are still not enough.
                                                                            # Then it creates a file whose name indicate "FAIL".
                                                                            # This has been done so to avoid stopping the creation of the rest of the profiles.
                                                                            # There is a Notebook that can be use after this run to help to identify the FAIL files,
                                                                            # to test more options to create a success profile.
        charging_eff = 0.90

        parkingathome = {'street': 0.19, 'garage': 0.81}  # from the total amount of driving profiles, this share represents the ones who have a garage or not.
        commuters_atworkplace = {'workpark': 0.5, 'publicpark': 0.25, 'none': 0.25}  # This share applies only for commuters type driving profiles

        # This configuration applies for all profiles
        whenAtHome ={'street': {'prob_charging_point' :
                                         {'errands':  {'public':0.5,'none':0.5},
                                          'escort':   {'public':0.5,'none':0.5},
                                          'leisure':  {'public':0.5,'none':0.5},
                                          'shopping': {'public':0.5,'none':0.5},
                                          'home':     {'public':0.5,'none':0.5},
                                          'workplace':{'public':0.0,'workplace':1.0,'none':0.0},
                                          'driving':  {'none':1.0}
                                          },
                                    'capacity_charging_point' :
                                          {'public':22,'home':3.7,'workplace':11,'none':0}
                                    },
                     'garage': {'prob_charging_point' :
                                         {'errands':  {'public':0.5,'none':0.5},
                                          'escort':   {'public':0.5,'none':0.5},
                                          'leisure':  {'public':0.5,'none':0.5},
                                          'shopping': {'public':0.5,'none':0.5},
                                          'home':     {'public':0.0,'home':1.0,'none':0.0},
                                          'workplace':{'public':0.0,'workplace':1.0,'none':0.0},
                                          'driving':  {'none':1.0}
                                          },
                                    'capacity_charging_point' :
                                          {'public':22,'home':3.7,'workplace':11,'none':0}
                                    }
                        }
        # this configuration applies only for commuters type and depends on the case ('workpark', 'publicpark', 'none') the above configuration gets replaced the 'workplace' key with the indicated below
        whenAtWork = {'workpark':{'public':0.0,'workplace':1.0,'none':0.0},
                      'publicpark':{'public':0.5,'workplace':0.0,'none':0.5},
                      'none':{'public':0.0,'workplace':0.0,'none':1.0}}

####################################################
########### Configuration for grid demand ##########
####################################################

    if run_charging:

        options_list = ['immediate',
                        'balanced',
                        'from_0_to_24_at_home',
                        'from_23_to_8_at_home'
                        ]

####################################################
######### Configuration for csv report file ########
####################################################

    if run_export:

        report_folder = 'csv'

####################################################
################# END CONFIGURATION ################
####################################################

############# DO NOT MODIFY THIS SECTION ###########
####################################################

    if run_driving:

        inventory = [[dbdir,stat1,stat2,stat3,param_comm_fulltime,rules_comm_fulltime,0,round(bev_n*dist['fulltime'])],
                     [dbdir,stat1,stat2,stat3,param_comm_parttime,rules_comm_parttime,0,round(bev_n*dist['parttime'])],
                     [dbdir,stat1,stat2,stat3,param_free,rules_freetime,0,round(bev_n*dist['freetime'])]]

        lista = []
        for l in inventory:
            for p in drive_iter(*l):
                lista.append(p)

        if is_ray:
            queue = []
        else:
            queue = Queue()

        for q,seed in zip(lista, seeds):
            if is_ray:
                queue.append(evsteps.remote(*q,seed))
            else:
                queue.put((q,seed))

        if is_ray:
            ray.get(queue)
        else:
            io_lock = Lock()
            queue_lock = Lock()
            processes = {}
            for i in range(nr_workers):
                processes[i] = Process(target=scen_solve_mov, args=(evsteps, queue, queue_lock, io_lock))
                processes[i].start()
            for i in range(nr_workers):
                processes[i].join()
        print('=:=:=:=:=:=:=:=:=:=:= Driving done! =:=:=:=:=:=:=:=:=:=:')

    db = DataBase(dbdir)

    if run_availability:
        db.update()
        iddrivingp = [k for k, v in db.db.items() if v['kind'] == 'driving']
        random.shuffle(iddrivingp)
        groups = {}
        init = 0
        for key, val in parkingathome.items():
            idx = init + int(round(len(iddrivingp)*val+0.01, 0))
            try:
                print('When_at_home slice:', init, ':', idx)
                for code in iddrivingp[init:idx]:
                    groups[code] = {'g_type': db.db[code]['g_type'],
                                    'per_str': db.db[code]['per_str'],
                                    'whenathome': key,
                                    'whenatwork': False}
            except:
                print('When_at_home slice:', init, ':', idx, 'out of range. Total profiles:',len(iddrivingp))
                for code in iddrivingp[init:]:
                    groups[code] = {'g_type': db.db[code]['g_type'],
                                    'per_str': db.db[code]['per_str'],
                                    'whenathome': key,
                                    'whenatwork': False}
            init = idx
        commuters = []
        for code, dict_ in groups.items():
            if 'commuter' == dict_['g_type']:
                commuters.append(code)
        random.shuffle(commuters)
        init = 0
        for key, val in commuters_atworkplace.items():
            idx = init + int(round(len(commuters)*val+0.01, 0))
            try:
                print('Commuter_at_work slice:', init, ':', idx)
                for code in commuters[init:idx]:
                    groups[code]['whenatwork'] = key
            except:
                print('Commuter_at_work slice:', init, ':', idx, 'out of range. Total profiles:',len(commuters))
                for code in commuters[init:]:
                    groups[code]['whenatwork'] = key
            init = idx
        for cod, dic in groups.items():
            print(cod, dic)
        if is_ray:
            queue = []
        else:
            queue = Queue()
        for code, obs in groups.items():
            q = [whenAtHome, whenAtWork, obs, code, battery_capacity, charging_eff, soc_init, soc_min, db, dbdir, battopt]
            if is_ray:
                queue.append(gridsteps.remote(*q))
            else:
                queue.put(q)
        if is_ray:
            ray.get(queue)
        else:
            io_lock = Lock()
            queue_lock = Lock()
            processes = {}
            for i in range(nr_workers):
                processes[i] = Process(target=scen_solve, args=(gridsteps, queue, queue_lock, io_lock))
                processes[i].start()
            for i in range(nr_workers):
                processes[i].join()
        print('=:=:=:=:=:=:=:=:=:=:= Grid availability done! =:=:=:=:=:=:=:=:=:=:')

    if run_charging:
        db.update()
        for code, v in db.db.items():
            if v['kind'] == 'availability':
                print('   Success?  Availability Profile: ', code, ' : ', v['success'])

        for option in options_list:
            db.update()
            for ids in db.db.keys():
                if db.db[ids]['kind'] == 'availability':
                    c = Charging(ids)
                    c.loadScenario(db)
                    c.setSubScenario(option)
                    c.run()
                    c.save_profile(dbdir)
        print('=:=:=:=:=:=:=:=:=:=:= Charging options done! =:=:=:=:=:=:=:=:=:=:')

    if run_export:
        db.update()
        export = Export()
        export.loaddata(db)
        print("Creating report... \nIf there is a failed availability file in the database, this process may hang up. \nIn which case check your database folder")
        export.to_csv()
        export.save_files(report_folder)
        print('=:=:=:=:=:=:=:=:=:=:= Export done! =:=:=:=:=:=:=:=:=:=:')

    print('==== Program finished ====')


text:
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
sys.path.append('../')

from auto_scan_test import AutoScanTest, IgnoreReasons
from program_config import TensorConfig, ProgramConfig, OpConfig, CxxConfig, TargetType, PrecisionType, DataLayoutType, Place
import unittest
from functools import partial
import numpy as np
import hypothesis
from hypothesis import given, settings, seed, example, assume
import hypothesis.strategies as st
import argparse

class TestRsqrtOp(AutoScanTest):
    def __init__(self, *args, **kwargs):
        AutoScanTest.__init__(self, *args, **kwargs)
        self.enable_testing_on_place(TargetType.Host, PrecisionType.FP32, DataLayoutType.NCHW, thread=[1,4])
        self.enable_testing_on_place(TargetType.X86, PrecisionType.FP32, DataLayoutType.NCHW, thread=[1,4])
        self.enable_testing_on_place(TargetType.ARM, PrecisionType.FP32, DataLayoutType.NCHW, thread=[1,4])
        opencl_places = [Place(TargetType.OpenCL, PrecisionType.FP16, DataLayoutType.ImageDefault),
                          Place(TargetType.OpenCL, PrecisionType.FP16, DataLayoutType.ImageFolder),
                          Place(TargetType.OpenCL, PrecisionType.FP32, DataLayoutType.NCHW),
                          Place(TargetType.OpenCL, PrecisionType.Any, DataLayoutType.ImageDefault),
                          Place(TargetType.OpenCL, PrecisionType.Any, DataLayoutType.ImageFolder),
                          Place(TargetType.OpenCL, PrecisionType.Any, DataLayoutType.NCHW),
                          Place(TargetType.Host, PrecisionType.FP32)]
        self.enable_testing_on_place(places=opencl_places)

    def is_program_valid(self, program_config: ProgramConfig , predictor_config: CxxConfig) -> bool:
        return False # fix arm_opencl ci error
        in_shape = list(program_config.inputs["input_data"].shape)
        if predictor_config.target() == TargetType.OpenCL:
            if len(in_shape) != 4:
                return False
        return True

    def sample_program_configs(self, draw):
        in_shape = draw(st.lists(st.integers(min_value=1, max_value=8), min_size=1, max_size=6))

        def generate_input(*args, **kwargs):
            # Make sure input data is greater than 0
            return np.random.random(in_shape).astype(np.float32) + 0.1

        rsqrt_op = OpConfig(
            type = "rsqrt",
            inputs = {"X" : ["input_data"]},
            outputs = {"Out": ["output_data"]},
            attrs = {})
        program_config = ProgramConfig(
            ops=[rsqrt_op],
            weights={},
            inputs={
                "input_data":
                TensorConfig(data_gen=partial(generate_input))
            },
            outputs=["output_data"])
        return program_config

    def sample_predictor_configs(self):
        return self.get_predictor_configs(), ["rsqrt"], (1e-5, 1e-5)

    def add_ignore_pass_case(self):
        pass

    def test(self, *args, **kwargs):
        target_str = self.get_target()
        max_examples = 25
        if target_str == "OpenCL":
            # Make sure to generate enough valid cases for OpenCL
            max_examples = 200

        self.run_and_statis(quant=False, min_success_num=25, max_examples=max_examples)

if __name__ == "__main__":
    unittest.main(argv=[''])

max_stars_repo_name: xiebaiyuan/PaddleLite

max_issues_repo_name: xiebaiyuan/PaddleLite

max_forks_repo_name: xiebaiyuan/PaddleLite


text:
from pNLsys.core.solver import solve_diffeq

import numpy as np
import matplotlib.pyplot as plt
from typing import List

__all__ = [
  "phasePlane", "DEplot"
]

def phasePlane(x1, x2, func, params):
  X1, X2 = np.meshgrid(x1, x2)  # create grid
  u, v = np.zeros(X1.shape), np.zeros(X2.shape)
  NI, NJ = X1.shape
  for i in range(NI):
    for j in range(NJ):
      x = X1[i, j]
      y = X2[i, j]
      dx = func(0, (x, y), *params.values())  # compute values on grid
      u[i, j] = dx[0]
      v[i, j] = dx[1]
  M = np.hypot(u, v)
  u /= M
  v /= M
  return X1, X2, u, v, M

def DEplot(sys: object, tspan: tuple, x0: List[List[float]],
           x: np.ndarray, y: np.ndarray, params: dict):
  if len(tspan) != 3:
    raise Exception(
        'tspan should be tuple of size 3: (min, max, number of points).')
  # Set up the figure the way we want it to look
  plt.figure(figsize=(12, 9))

  X1, X2, dx1, dx2, M = phasePlane(
      x, y, sys, params
  )

  # Quiver plot
  plt.quiver(X1, X2, dx1, dx2, M, scale=None, pivot='mid')
  plt.grid()

  if tspan[0] < 0:
    t1 = np.linspace(0, tspan[0], tspan[2])
    t2 = np.linspace(0, tspan[1], tspan[2])
    if min(tspan) < 0:
      t_span1 = (np.max(t1), np.min(t1))
    else:
      t_span1 = (np.min(t1), np.max(t1))
    t_span2 = (np.min(t2), np.max(t2))
    for x0i in x0:
      sol1 = solve_diffeq(sys, t1, t_span1, x0i, params)
      plt.plot(sol1.y[0, :], sol1.y[1, :], '-r')
      sol2 = solve_diffeq(sys, t2, t_span2, x0i, params)
      plt.plot(sol2.y[0, :], sol2.y[1, :], '-r')
  else:
    t = np.linspace(tspan[0], tspan[1], tspan[2])
    t_span = (np.min(t), np.max(t))
    for x0i in x0:
      sol = solve_diffeq(sys, t, t_span, x0i, params)
      plt.plot(sol.y[0, :], sol.y[1, :], '-r')

  plt.xlim([np.min(x), np.max(x)])
  plt.ylim([np.min(y), np.max(y)])
  plt.show()

max_stars_repo_name: smallpondtom/pNLsys

max_issues_repo_name: smallpondtom/pNLsys

max_forks_repo_name: smallpondtom/pNLsys


text:
# movieclassifier/main.py
"""The entry point for loading, training and evaluating the movieclassifier package."""

from pathlib import Path

import mlflow
import numpy as np

from movieclassifier import config, data, eval, train, utils

# Load data
movie_data_fp = Path(config.DATA_DIR, "interim", "movie_data.csv")
doc_stream = utils.stream_docs(movie_data_fp)
size = 1000
X_train, y_train = utils.get_minibatch(doc_stream, size)
X_train, y_train = np.asarray(X_train, dtype=str), np.asarray(y_train, dtype=str)
X_test, y_test = utils.get_minibatch(doc_stream, size=5000)
X_test, y_test = np.asarray(X_test, dtype=str), np.asarray(y_test, dtype=str)

# Preprocess data
X_train = data.hash_vectorize(X_train)
X_test = data.hash_vectorize(X_test)

# Tracking

# Set experiment
mlflow.set_experiment(experiment_name="online-logreg")

with mlflow.start_run() as run:
    # Train model
    clf = train.train(X_train, y_train)

    # Evaluate model
    accuracy = eval.evaluate(clf, X_test, y_test)
    mlflow.log_metric("Accuracy", accuracy)
    mlflow.sklearn.log_model(clf, "model")
    print(f"Accuracy: {accuracy}")
    print()
    print("Logged data and model in run: {}".format(run.info.run_id))

max_stars_repo_name: cmftall/movieclassifier

max_issues_repo_name: cmftall/movieclassifier

max_forks_repo_name: cmftall/movieclassifier