text:
#!/usr/bin/env python3
# Copyright (C) Alibaba Group H volding Limited.

"""
Meters.
Modifed from https://github.com/facebookresearch/SlowFast/blob/master/slowfast/utils/meters.py.
Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
"""

import datetime
import numpy as np
import os
from collections import defaultdict, deque
import torch
from utils.timer import Timer

import utils.logging as logging
import utils.metrics as metrics
import utils.misc as misc
import utils.distributed as du

logger = logging.get_logger(__name__)

class TestMeter(object):
    """
    Perform the multi-view ensemble for testing: each video with an unique index
    will be sampled with multiple clips, and the predictions of the clips will
    be aggregated to produce the final prediction for the video.
    The accuracy is calculated with the given ground truth labels.
    """

    def __init__(
        self,
        cfg,
        num_videos,
        num_clips,
        num_cls,
        overall_iters,
        ensemble_method="sum",
    ):
        """
        Construct tensors to store the predictions and labels. Expect to get
        num_clips predictions from each video, and calculate the metrics on
        num_videos videos.
        Args:
            num_videos (int): number of videos to test.
            num_clips (int): number of clips sampled from each video for
                aggregating the final prediction for the video.
            num_cls (int): number of classes for each prediction.
            overall_iters (int): overall iterations for testing.
            multi_label (bool): if True, use map as the metric.
            ensemble_method (str): method to perform the ensemble, options
                include "sum", and "max".
        """

        self.cfg = cfg
        self.iter_timer = Timer()
        self.num_clips = num_clips
        self.overall_iters = overall_iters
        self.ensemble_method = ensemble_method
        # Initialize tensors.
        self.video_preds = torch.zeros((num_videos, num_cls))

        self.video_labels = (
            torch.zeros((num_videos)).long()
        )
        self.clip_count = torch.zeros((num_videos)).long()
        self.clip_indices = torch.linspace(0, num_videos-1, num_videos).long()
        self.model_ema_enabled = False
        # Reset metric.
        self.reset()

    def reset(self):
        """
        Reset the metric.
        """
        self.clip_count.zero_()
        self.video_preds.zero_()
        self.video_labels.zero_()

    def update_stats(self, preds, labels, clip_ids):
        """
        Collect the predictions from the current batch and perform on-the-flight
        summation as ensemble.
        Args:
            preds (tensor): predictions from the current batch. Dimension is
                N x C where N is the batch size and C is the channel size
                (num_cls).
            labels (tensor): the corresponding labels of the current batch.
                Dimension is N.
            clip_ids (tensor): clip indexes of the current batch, dimension is
                N.
        """
        for ind in range(preds.shape[0]):
            vid_id = int(clip_ids[ind]) // self.num_clips
            if self.video_labels[vid_id].sum() > 0:
                assert torch.equal(
                    self.video_labels[vid_id].type(torch.FloatTensor),
                    labels[ind].type(torch.FloatTensor),
                )
            self.video_labels[vid_id] = labels[ind]
            if self.ensemble_method == "sum":
                self.video_preds[vid_id] += preds[ind]
            elif self.ensemble_method == "max":
                self.video_preds[vid_id] = torch.max(
                    self.video_preds[vid_id], preds[ind]
                )
            else:
                raise NotImplementedError(
                    "Ensemble Method {} is not supported".format(
                        self.ensemble_method
                    )
                )
            self.clip_count[vid_id] += 1

    def log_iter_stats(self, cur_iter):
        """
        Log the stats.
        Args:
            cur_iter (int): the current iteration of testing.
        """
        if (cur_iter + 1) % self.cfg.LOG_PERIOD != 0:
            return
        eta_sec = self.iter_timer.seconds() * (self.overall_iters - cur_iter)
        eta = str(datetime.timedelta(seconds=int(eta_sec)))
        stats = {
            "split": "test_iter" if not self.model_ema_enabled else "ema_test_iter",
            "cur_iter": "{}".format(cur_iter + 1),
            "eta": eta,
            "time_diff": self.iter_timer.seconds(),
        }
        logging.log_json_stats(stats)

    def iter_tic(self):
        self.iter_timer.reset()

    def iter_toc(self):
        self.iter_timer.pause()

    def finalize_metrics(self, ks=(1, 5)):
        """
        Calculate and log the final ensembled metrics.
        ks (tuple): list of top-k values for topk_accuracies. For example,
            ks = (1, 5) correspods to top-1 and top-5 accuracy.
        """
        if not all(self.clip_count == self.num_clips):
            logger.warning(
                "clip count {} ~= num clips {}".format(
                    ", ".join(
                        [
                            # "{}: {}".format(i, k)
                            # for i, k in enumerate(self.clip_count.tolist())
                            "{}: {}".format(ind, self.clip_count[ind]) for idx, ind in enumerate(self.clip_indices[self.clip_count!=self.num_clips].tolist())
                        ]
                    ),
                    self.num_clips,
                )
            )

        stats = {"split": "test_final" if not self.model_ema_enabled else "ema_test_final"}
        num_topks_correct = metrics.topks_correct(
            self.video_preds, self.video_labels, ks
        )
        topks = [
            (x / self.video_preds.size(0)) * 100.0
            for x in num_topks_correct
        ]
        assert len({len(ks), len(topks)}) == 1
        for k, topk in zip(ks, topks):
            stats["top{}_acc".format(k)] = "{:.{prec}f}".format(
                topk, prec=2
            )
        logging.log_json_stats(stats)

    def set_model_ema_enabled(self, model_ema_enabled):
        self.model_ema_enabled = model_ema_enabled

class EpicKitchenMeter(object):
    """
    Perform the multi-view ensemble for testing: each video with an unique index
    will be sampled with multiple clips, and the predictions of the clips will
    be aggregated to produce the final prediction for the video.
    The accuracy is calculated with the given ground truth labels.

    For the EpicKitchenMeter specifically, it caters to the need of the EpicKitchens
    dataset, where both verbs and nouns are predicted before actions are predicted using
    those predictions.
    """

    def __init__(
        self,
        cfg,
        num_videos,
        num_clips,
        num_cls,
        overall_iters,
        ensemble_method="sum",
    ):
        """
        Construct tensors to store the predictions and labels. Expect to get
        num_clips predictions from each video, and calculate the metrics on
        num_videos videos.
        Args:
            cfg (Config): the global config object.
            num_videos (int): number of videos to test.
            num_clips (int): number of clips sampled from each video for
                aggregating the final prediction for the video.
            num_cls (int): number of classes for each prediction.
            overall_iters (int): overall iterations for testing.
            multi_label (bool): if True, use map as the metric.
            ensemble_method (str): method to perform the ensemble, options
                include "sum", and "max".
        """

        self.cfg = cfg
        self.iter_timer = Timer()
        self.num_clips = num_clips
        self.num_videos = num_videos
        self.overall_iters = overall_iters
        self.ensemble_method = ensemble_method

        assert self.ensemble_method in ["sum", "max"], f"Ensemble Method {ensemble_method} is not supported"

        if cfg.DATA.MULTI_LABEL or not hasattr(cfg.DATA, "TRAIN_VERSION"):
            # Initialize tensors.
            self.video_preds = {
                "verb_class": torch.zeros((num_videos, self.num_clips, num_cls[0])),
                "noun_class": torch.zeros((num_videos, self.num_clips, num_cls[1])),
                "action_class_ind_pred": torch.zeros((num_videos, self.num_clips, num_cls[0]*num_cls[1]))
            }

            self.video_labels = {
                "verb_class": torch.zeros((num_videos)),  # verb
                "noun_class": torch.zeros((num_videos)),  # noun
                "action_class_ind_pred": torch.zeros((num_videos)),
            }
            self.update_stats = self.update_stats_multi_label
            self.finalize_metrics = self.finalize_metrics_multi_label
        elif hasattr(cfg.DATA, "TRAIN_VERSION") and cfg.DATA.TRAIN_VERSION in ["only_train_verb", "only_train_noun"]:
            self.video_preds = torch.zeros((num_videos, self.num_clips, num_cls))
            self.video_labels = torch.zeros((num_videos))
            self.update_stats = self.update_stats_separate_label
            self.finalize_metrics = self.finalize_metrics_separate_label
        else: raise NotImplementedError
        self.video_names = {i: "" for i in range(num_videos)}
        self.clip_count = torch.zeros((num_videos)).long()
        self.clip_indices = torch.linspace(0, num_videos-1, num_videos).long()
        # Reset metric.
        self.reset()

    def reset(self):
        """
        Reset the metric.
        """
        self.clip_count.zero_()
        if isinstance(self.video_preds, dict):
            for k, v in self.video_preds.items():
                v.zero_()
            for k, v in self.video_labels.items():
                v.zero_()
        else:
            self.video_preds.zero_()
            self.video_labels.zero_()

    def update_stats_separate_label(self, preds, labels, clip_ids):
        """
        Collect the predictions from the current batch and perform on-the-flight
        summation as ensemble, for separate verb and noun training.
        Args:
            preds (tensor): predictions from the current batch. Dimension is
                N x C where N is the batch size and C is the channel size
                (num_cls).
            labels (tensor): the corresponding labels of the current batch.
                Dimension is N.
            clip_ids (tensor): clip indexes of the current batch, dimension is
                N.
        """
        for ind in range(preds.shape[0]):
            vid_id = int(clip_ids[ind]) // self.num_clips
            view_id = int(clip_ids[ind]) % self.num_clips
            if self.video_labels[vid_id].sum() > 0:
                assert torch.equal(
                    self.video_labels[vid_id].type(torch.FloatTensor),
                    labels[ind].type(torch.FloatTensor),
                )

            self.video_labels[vid_id] = labels[ind]

            self.video_preds[vid_id][view_id] = preds[ind]

            self.clip_count[vid_id] += 1

    def update_stats_multi_label(self, preds_verb, preds_noun, labels_verb, labels_noun, clip_ids, names=[]):
        """
        Collect the predictions from the current batch and perform on-the-flight
        summation as ensemble, for joint verb and noun training.
        Args:
            preds_verb (tensor): verb predictions from the current batch. Dimension is
                N x C where N is the batch size and C is the channel size
                (num_cls[0]).
            preds_noun (tensor): noun predictions from the current batch. Dimension is
                N x C where N is the batch size and C is the channel size
                (num_cls[1]).
            labels_verb (tensor): the corresponding verb labels of the current batch.
                Dimension is N.
            labels_noun (tensor): the corresponding noun labels of the current batch.
                Dimension is N.
            clip_ids (tensor): clip indexes of the current batch, dimension is
                N.
            names (list): list of video names.
        """
        for ind in range(preds_verb.shape[0]):
            vid_id = int(clip_ids[ind]) // self.num_clips
            view_id = int(clip_ids[ind]) % self.num_clips
            if self.video_labels["verb_class"][vid_id].sum() > 0:
                assert torch.equal(
                    self.video_labels["verb_class"][vid_id].type(torch.FloatTensor),
                    labels_verb[ind].type(torch.FloatTensor),
                )
                assert torch.equal(
                    self.video_labels["noun_class"][vid_id].type(torch.FloatTensor),
                    labels_noun[ind].type(torch.FloatTensor),
                )
            if len(names) > 0:
                if self.video_names[vid_id] != "":
                    assert self.video_names[vid_id] == names[ind], \
                        f"For {vid_id}, its name {self.video_names[vid_id]} should be equal to {names[ind]}"
                else:
                    self.video_names[vid_id] = names[ind]

            self.video_labels["verb_class"][vid_id] = labels_verb[ind]
            self.video_labels["noun_class"][vid_id] = labels_noun[ind]
            self.video_labels["action_class_ind_pred"][vid_id] = labels_verb[ind] * preds_noun.shape[1] + labels_noun[ind]

            self.video_preds["verb_class"][vid_id][view_id] = preds_verb[ind]
            self.video_preds["noun_class"][vid_id][view_id] = preds_noun[ind]
            self.video_preds["action_class_ind_pred"][vid_id][view_id] = (preds_verb[ind].unsqueeze(-1) * preds_noun[ind].unsqueeze(-2)).reshape(-1)

            self.clip_count[vid_id] += 1

    def log_iter_stats(self, cur_iter):
        """
        Log the stats.
        Args:
            cur_iter (int): the current iteration of testing.
        """
        if (cur_iter + 1) % self.cfg.LOG_PERIOD != 0:
            return
        eta_sec = self.iter_timer.seconds() * (self.overall_iters - cur_iter)
        eta = str(datetime.timedelta(seconds=int(eta_sec)))
        stats = {
            "split": "test_iter" if not self.model_ema_enabled else "ema_test_iter",
            "cur_iter": "{}".format(cur_iter + 1),
            "eta": eta,
            "time_diff": self.iter_timer.seconds(),
        }
        logging.log_json_stats(stats)

    def iter_tic(self):
        self.iter_timer.reset()

    def iter_toc(self):
        self.iter_timer.pause()

    def finalize_metrics_multi_label(self, ks=(1, 5)):
        """
        Calculate and log the final ensembled metrics for joint verb and
        noun training.
        ks (tuple): list of top-k values for topk_accuracies. For example,
            ks = (1, 5) correspods to top-1 and top-5 accuracy.
        """
        if not all(self.clip_count == self.num_clips):
            logger.warning(
                "clip count {} ~= num clips {}".format(
                    ", ".join(
                        [
                            # "{}: {}".format(i, k)
                            # for i, k in enumerate(self.clip_count.tolist())
                            "{}: {}".format(ind, self.clip_count[ind]) for idx, ind in enumerate(self.clip_indices[self.clip_count!=self.num_clips].tolist())
                        ]
                    ),
                    self.num_clips,
                )
            )

        stats = {"split": "test_final" if not self.model_ema_enabled else "ema_test_final"}
        video_preds = {}
        if self.ensemble_method == "sum":
            video_preds["verb_class"] = self.video_preds["verb_class"].sum(1)
            video_preds["noun_class"] = self.video_preds["noun_class"].sum(1)
            video_preds["action_class_ind_pred"] = self.video_preds["action_class_ind_pred"].sum(1)
        elif self.ensemble_method == "max":
            video_preds["verb_class"] = self.video_preds["verb_class"].max(1)[0]
            video_preds["noun_class"] = self.video_preds["noun_class"].max(1)[0]
            video_preds["action_class_ind_pred"] = self.video_preds["action_class_ind_pred"].max(1)[0]
        num_topks_correct, b = metrics.joint_topks_correct(
            video_preds, self.video_labels, ks
        )
        for name, v in num_topks_correct.items():
            topks = [ (x / b) * 100.0 for x in v ]
            assert len({len(ks), len(topks)}) == 1
            for k, topk in zip(ks, topks):
                stats["top_{}_acc_{}".format(name, k)] = "{:.{prec}f}".format(
                    topk, prec=2
                )
        logging.log_json_stats(stats)

    def finalize_metrics_separate_label(self, ks=(1, 5)):
        """
        Calculate and log the final ensembled metrics, for separate verb
        and noun training.
        ks (tuple): list of top-k values for topk_accuracies. For example,
            ks = (1, 5) correspods to top-1 and top-5 accuracy.
        """
        if not all(self.clip_count == self.num_clips):
            logger.warning(
                "clip count {} ~= num clips {}".format(
                    ", ".join(
                        [
                            "{}: {}".format(ind, self.clip_count[ind]) for idx, ind in enumerate(self.clip_indices[self.clip_count!=self.num_clips].tolist())
                        ]
                    ),
                    self.num_clips,
                )
            )

        stats = {"split": "test_final" if not self.model_ema_enabled else "ema_test_final"}
        if self.ensemble_method == "sum":
            video_preds = self.video_preds.sum(1)
        elif self.ensemble_method == "max":
            video_preds = self.video_preds.max(1)[0]
        num_topks_correct = metrics.topks_correct(
            video_preds, self.video_labels, ks
        )
        topks = [
            (x / self.video_preds.size(0)) * 100.0
            for x in num_topks_correct
        ]
        assert len({len(ks), len(topks)}) == 1
        for k, topk in zip(ks, topks):
            stats["top{}_acc".format(k)] = "{:.{prec}f}".format(
                topk, prec=2
            )
        logging.log_json_stats(stats)

    def set_model_ema_enabled(self, model_ema_enabled):
        """
        Whether the meter logs for ema models or not.
        Args:
            model_ema_enabled (bool): indicator of whether ema model
                is enabled.
        """
        self.model_ema_enabled = model_ema_enabled

    def get_video_preds(self):
        """
        Returns the saved video predictions.
        """
        video_preds = {}
        if self.ensemble_method == "sum":
            video_preds["verb_class"] = self.video_preds["verb_class"].sum(1)
            video_preds["noun_class"] = self.video_preds["noun_class"].sum(1)
            video_preds["action_class_ind_pred"] = self.video_preds["action_class_ind_pred"].sum(1)
        elif self.ensemble_method == "max":
            video_preds["verb_class"] = self.video_preds["verb_class"].max(1)[0]
            video_preds["noun_class"] = self.video_preds["noun_class"].max(1)[0]
            video_preds["action_class_ind_pred"] = self.video_preds["action_class_ind_pred"].max(1)[0]
        return video_preds

class ScalarMeter(object):
    """
    A scalar meter uses a deque to track a series of scaler values with a given
    window size. It supports calculating the median and average values of the
    window, and also supports calculating the global average.
    """

    def __init__(self, window_size=10):
        """
        Args:
            window_size (int): size of the max length of the deque.
        """
        self.deque = deque(maxlen=window_size)
        self.total = 0.0
        self.count = 0

    def reset(self):
        """
        Reset the deque.
        """
        self.deque.clear()
        self.total = 0.0
        self.count = 0

    def add_value(self, value):
        """
        Add a new scalar value to the deque.
        """
        self.deque.append(value)
        self.count += 1
        self.total += value

    def get_win_median(self):
        """
        Calculate the current median value of the deque.
        """
        return np.median(self.deque)

    def get_win_avg(self):
        """
        Calculate the current average value of the deque.
        """
        return np.mean(self.deque)

    def get_global_avg(self):
        """
        Calculate the global mean value.
        """
        return self.total / self.count

class TrainMeter(object):
    """
    Measure training stats.
    """

    def __init__(self, epoch_iters, cfg):
        """
        Args:
            epoch_iters (int): the overall number of iterations of one epoch.
            cfg (Config): the global config object.
        """
        self._cfg = cfg
        self.epoch_iters = epoch_iters
        self.MAX_EPOCH = cfg.OPTIMIZER.MAX_EPOCH * epoch_iters
        self.iter_timer = Timer()
        self.loss = ScalarMeter(cfg.LOG_PERIOD)
        self.loss_total = 0.0
        self.lr = None
        # Current minibatch errors (smoothed over a window).
        self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD)
        self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD)
        # Number of misclassified examples.
        self.num_top1_mis = 0
        self.num_top5_mis = 0
        self.num_samples = 0
        self.opts = defaultdict(ScalarMeter)

    def reset(self):
        """
        Reset the Meter.
        """
        self.loss.reset()
        self.loss_total = 0.0
        self.lr = None
        self.mb_top1_err.reset()
        self.mb_top5_err.reset()
        self.num_top1_mis = 0
        self.num_top5_mis = 0
        self.num_samples = 0
        self.opts = defaultdict(ScalarMeter)

    def iter_tic(self):
        """
        Start to record time.
        """
        self.iter_timer.reset()

    def iter_toc(self):
        """
        Stop to record time.
        """
        self.iter_timer.pause()

    def update_stats(self, top1_err, top5_err, loss, lr, mb_size, **kwargs):
        """
        Update the current stats.
        Args:
            top1_err (float): top1 error rate.
            top5_err (float): top5 error rate.
            loss (float): loss value.
            lr (float): learning rate.
            mb_size (int): mini batch size.
        """
        self.loss.add_value(loss)
        self.lr = lr
        self.loss_total += loss * mb_size
        self.num_samples += mb_size

        for k,v in kwargs.items():
            if isinstance(v, torch.Tensor):
                v = v.item()
            assert isinstance(v, (float, int))
            self.opts[k].add_value(v)

        if not self._cfg.PRETRAIN.ENABLE and not self._cfg.LOCALIZATION.ENABLE:
            # Current minibatch stats
            self.mb_top1_err.add_value(top1_err)
            self.mb_top5_err.add_value(top5_err)
            # Aggregate stats
            self.num_top1_mis += top1_err * mb_size
            self.num_top5_mis += top5_err * mb_size
    def update_custom_stats(self, stats):
        """
        Update stats using custom keys.
        Args:
            stats (dict): additional stats to be updated.
        """
        for k,v in stats.items():
            if isinstance(v, torch.Tensor):
                v = v.item()
            assert isinstance(v, (float, int))
            self.opts[k].add_value(v)

    def log_iter_stats(self, cur_epoch, cur_iter):
        """
        log the stats of the current iteration.
        Args:
            cur_epoch (int): the number of current epoch.
            cur_iter (int): the number of current iteration.
        """
        if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0:
            return
        eta_sec = self.iter_timer.seconds() * (
            self.MAX_EPOCH - (cur_epoch * self.epoch_iters + cur_iter + 1)
        )
        eta = str(datetime.timedelta(seconds=int(eta_sec)))
        stats = {
            "_type": "train_iter",
            "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.OPTIMIZER.MAX_EPOCH),
            "iter": "{}/{}".format(cur_iter + 1, self.epoch_iters),
            "time_diff": self.iter_timer.seconds(),
            "eta": eta,
            "loss": self.loss.get_win_median(),
            "lr": self.lr,
            # "gpu_mem": "{:.2f} GB".format(misc.gpu_mem_usage()),
        }
        for k,v in self.opts.items():
            stats[k] = v.get_win_median()
        if not self._cfg.PRETRAIN.ENABLE and not self._cfg.LOCALIZATION.ENABLE:
            stats["top1_err"] = self.mb_top1_err.get_win_median()
            stats["top5_err"] = self.mb_top5_err.get_win_median()
        logging.log_json_stats(stats)

    def log_epoch_stats(self, cur_epoch):
        """
        Log the stats of the current epoch.
        Args:
            cur_epoch (int): the number of current epoch.
        """
        eta_sec = self.iter_timer.seconds() * (
            self.MAX_EPOCH - (cur_epoch + 1) * self.epoch_iters
        )
        eta = str(datetime.timedelta(seconds=int(eta_sec)))
        stats = {
            "_type": "train_epoch",
            "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.OPTIMIZER.MAX_EPOCH),
            "time_diff": self.iter_timer.seconds(),
            "eta": eta,
            "lr": self.lr,
            "gpu_mem": "{:.2f} GB".format(misc.gpu_mem_usage()),
            "RAM": "{:.2f}/{:.2f} GB".format(*misc.cpu_mem_usage()),
        }
        for k,v in self.opts.items():
            stats[k] = v.get_global_avg()
        if not self._cfg.PRETRAIN.ENABLE:
            top1_err = self.num_top1_mis / self.num_samples
            top5_err = self.num_top5_mis / self.num_samples
            avg_loss = self.loss_total / self.num_samples
            stats["top1_err"] = top1_err
            stats["top5_err"] = top5_err
            stats["loss"] = avg_loss
        logging.log_json_stats(stats)

class ValMeter(object):
    """
    Measures validation stats.
    """

    def __init__(self, max_iter, cfg):
        """
        Args:
            max_iter (int): the max number of iteration of the current epoch.
            cfg (Config): the global config object.
        """
        self._cfg = cfg
        self.max_iter = max_iter
        self.iter_timer = Timer()
        # Current minibatch errors (smoothed over a window).
        self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD)
        self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD)
        # Min errors (over the full val set).
        self.min_top1_err = 100.0
        self.min_top5_err = 100.0
        # Number of misclassified examples.
        self.num_top1_mis = 0
        self.num_top5_mis = 0
        self.num_samples = 0
        self.all_preds = []
        self.all_labels = []
        self.model_ema_enabled = False
        self.opts = defaultdict(ScalarMeter)

    def reset(self):
        """
        Reset the Meter.
        """
        self.iter_timer.reset()
        self.mb_top1_err.reset()
        self.mb_top5_err.reset()
        self.num_top1_mis = 0
        self.num_top5_mis = 0
        self.num_samples = 0
        self.all_preds = []
        self.all_labels = []
        self.opts = defaultdict(ScalarMeter)

    def iter_tic(self):
        """
        Start to record time.
        """
        self.iter_timer.reset()

    def iter_toc(self):
        """
        Stop to record time.
        """
        self.iter_timer.pause()

    def update_stats(self, top1_err, top5_err, mb_size, **kwargs):
        """
        Update the current stats.
        Args:
            top1_err (float): top1 error rate.
            top5_err (float): top5 error rate.
            mb_size (int): mini batch size.
        """
        for k,v in kwargs.items():
            if isinstance(v, torch.Tensor):
                v = v.item()
            self.opts[k].add_value(v)
        self.mb_top1_err.add_value(top1_err)
        self.mb_top5_err.add_value(top5_err)
        self.num_top1_mis += top1_err * mb_size
        self.num_top5_mis += top5_err * mb_size
        self.num_samples += mb_size

    def update_custom_stats(self, stats):
        """
        Update stats using custom keys.
        Args:
            stats (dict): additional stats to be updated.
        """
        for k,v in stats.items():
            if isinstance(v, torch.Tensor):
                v = v.item()
            assert isinstance(v, (float, int))
            self.opts[k].add_value(v)

    def update_predictions(self, preds, labels):
        """
        Update predictions and labels.
        Args:
            preds (tensor): model output predictions.
            labels (tensor): labels.
        """
        # TODO: merge update_prediction with update_stats.
        self.all_preds.append(preds)
        self.all_labels.append(labels)

    def log_iter_stats(self, cur_epoch, cur_iter):
        """
        log the stats of the current iteration.
        Args:
            cur_epoch (int): the number of current epoch.
            cur_iter (int): the number of current iteration.
        """
        if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0:
            return
        eta_sec = self.iter_timer.seconds() * (self.max_iter - cur_iter - 1)
        eta = str(datetime.timedelta(seconds=int(eta_sec)))
        stats = {
            "_type": "val_iter" if not self.model_ema_enabled else "ema_val_iter",
            "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.OPTIMIZER.MAX_EPOCH),
            "iter": "{}/{}".format(cur_iter + 1, self.max_iter),
            "time_diff": self.iter_timer.seconds(),
            "eta": eta,
            "gpu_mem": "{:.2f} GB".format(misc.gpu_mem_usage()),
        }
        for k,v in self.opts.items():
            stats[k] = v.get_win_median()
        stats["top1_err"] = self.mb_top1_err.get_win_median()
        stats["top5_err"] = self.mb_top5_err.get_win_median()
        logging.log_json_stats(stats)

    def log_epoch_stats(self, cur_epoch):
        """
        Log the stats of the current epoch.
        Args:
            cur_epoch (int): the number of current epoch.
        """
        stats = {
            "_type": "val_epoch" if not self.model_ema_enabled else "ema_val_epoch",
            "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.OPTIMIZER.MAX_EPOCH),
            "time_diff": self.iter_timer.seconds(),
            "gpu_mem": "{:.2f} GB".format(misc.gpu_mem_usage()),
            "RAM": "{:.2f}/{:.2f} GB".format(*misc.cpu_mem_usage()),
        }
        for k,v in self.opts.items():
            if "top1_err" in k or "top5_err" in k:
                stats[k] = v.get_win_median()
            else:
                stats[k] = v.get_global_avg()
        top1_err = self.num_top1_mis / self.num_samples
        top5_err = self.num_top5_mis / self.num_samples
        self.min_top1_err = min(self.min_top1_err, top1_err)
        self.min_top5_err = min(self.min_top5_err, top5_err)

        stats["top1_err"] = top1_err
        stats["top5_err"] = top5_err
        stats["min_top1_err"] = self.min_top1_err
        stats["min_top5_err"] = self.min_top5_err

        logging.log_json_stats(stats)

    def set_model_ema_enabled(self, model_ema_enabled):
        self.model_ema_enabled = model_ema_enabled


text:
# coding: utf8
# !/usr/env/python
"""Base class for common functions of all terrainbento erosion models."""

import os
import sys
import time as tm

import numpy as np
import xarray as xr
import yaml

from landlab import ModelGrid, RasterModelGrid, create_grid
from landlab.components import FlowAccumulator, NormalFault
from landlab.io.netcdf import to_netcdf, write_raster_netcdf
from terrainbento.boundary_handlers import (
    CaptureNodeBaselevelHandler,
    GenericFuncBaselevelHandler,
    NotCoreNodeBaselevelHandler,
    PrecipChanger,
    SingleNodeBaselevelHandler,
)
from terrainbento.clock import Clock
from terrainbento.precipitators import RandomPrecipitator, UniformPrecipitator
from terrainbento.runoff_generators import SimpleRunoff

_SUPPORTED_PRECIPITATORS = {
    "UniformPrecipitator": UniformPrecipitator,
    "RandomPrecipitator": RandomPrecipitator,
}
_SUPPORTED_RUNOFF_GENERATORS = {"SimpleRunoff": SimpleRunoff}

_VALID_PRECIPITATORS = (UniformPrecipitator, RandomPrecipitator)
_VALID_RUNOFF_GENERATORS = SimpleRunoff

_DEFAULT_PRECIPITATOR = {"UniformPrecipitator": {}}
_DEFAULT_RUNOFF_GENERATOR = {"SimpleRunoff": {}}

_SUPPORTED_BOUNDARY_HANDLERS = [
    "NormalFault",
    "PrecipChanger",
    "CaptureNodeBaselevelHandler",
    "NotCoreNodeBaselevelHandler",
    "SingleNodeBaselevelHandler",
    "GenericFuncBaselevelHandler",
]

_HANDLER_METHODS = {
    "NormalFault": NormalFault,
    "PrecipChanger": PrecipChanger,
    "CaptureNodeBaselevelHandler": CaptureNodeBaselevelHandler,
    "NotCoreNodeBaselevelHandler": NotCoreNodeBaselevelHandler,
    "SingleNodeBaselevelHandler": SingleNodeBaselevelHandler,
    "GenericFuncBaselevelHandler": GenericFuncBaselevelHandler,
}

def _verify_boundary_handler(handler):
    bad_name = False
    bad_instance = False
    if isinstance(handler, str):
        if handler not in _SUPPORTED_BOUNDARY_HANDLERS:
            bad_name = True
    else:  # if a dictionary {name, handler}
        for key in handler:
            name = handler[key].__class__.__name__
            if name not in _SUPPORTED_BOUNDARY_HANDLERS:
                bad_instance = True

    if bad_name:
        raise ValueError(
            (
                "Only supported boundary condition handlers are "
                "permitted. These include: {valid}".format(
                    valid="\n".join(_SUPPORTED_BOUNDARY_HANDLERS)
                )
            )
        )

    if bad_instance:
        raise ValueError(
            (
                "An invalid instance of "
                + name
                + " was passed as a boundary handler."
                + str(handler)
            )
        )

def _setup_precipitator_or_runoff(grid, params, supported):
    """"""
    if len(params) != 1:
        raise ValueError(
            "Too many values provided to set up precipitator or runoff_generator"
        )
    for name in params:
        constructor = supported[name]
        instance = constructor(grid, **params[name])
    return instance

def _setup_boundary_handlers(grid, name, params):
    """Setup BoundaryHandlers for use by a terrainbento model.

    A boundary condition handler is a class with a **run_one_step** method
    that takes the parameter ``step``. Permitted boundary condition handlers
    include the Landlab Component **NormalFault** as well as the following
    options from terrainbento: **PrecipChanger**,
    **CaptureNodeBaselevelHandler**, **NotCoreNodeBaselevelHandler**,
    **SingleNodeBaselevelHandler**.

    Parameters
    ----------
    handler : str
        Name of a supported boundary condition handler.
    """
    _verify_boundary_handler(name)
    # Instantiate handler
    handler_func = _HANDLER_METHODS[name]
    boundary_handler = handler_func(grid, **params)

    return boundary_handler

class ErosionModel(object):

    """Base class providing common functionality for terrainbento models.

    An **ErosionModel** is the skeleton for the models of terrain evolution in
    terrainbento.

    This is a base class that does not implement any processes, but rather
    simply handles I/O and setup. Derived classes are meant to include
    Landlab components to model actual erosion processes.

    It is expected that a derived model will define an **__init__** and a
    **run_one_step** method. If desired, the derived model can overwrite the
    existing **run_for**, **run**, and **finalize** methods.

    The following at-node fields must be specified in the grid:
        - ``topographic__elevation``
    """

    _required_fields = ["topographic__elevation"]

    @classmethod
    def from_file(cls, file_like):
        """Construct a terrainbento model from a file.

        Parameters
        ----------
        file_like : file_like or str
            Contents of a parameter file, a file-like object, or the path to
            a parameter file.

        Examples
        --------
        >>> from io import StringIO
        >>> filelike = StringIO('''
        ... grid:
        ...   RasterModelGrid:
        ...     - [4, 5]
        ...     - fields:
        ...         node:
        ...           topographic__elevation:
        ...             constant:
        ...               - value: 0.0
        ... clock:
        ...   step: 1
        ...   stop: 200
        ... ''')
        >>> model = ErosionModel.from_file(filelike)
        >>> model.clock.step
        1.0
        >>> model.clock.stop
        200.0
        >>> model.grid.shape
        (4, 5)
        """
        # first get contents.
        try:
            contents = file_like.read()
        except AttributeError:  # was a str
            if os.path.isfile(file_like):
                with open(file_like, "r") as fp:
                    contents = fp.read()
            else:
                contents = file_like  # not tested

        # then parse contents.
        params = yaml.safe_load(contents)

        # construct instance
        return cls.from_dict(params)

    @classmethod
    def from_dict(cls, params, output_writers=None):
        """Construct a terrainbento model from an input parameter dictionary.

        The input parameter dictionary portion associated with the "grid"
        keword will be passed directly to the Landlab
        `create_grid <https://landlab.readthedocs.io/en/master/reference/grid/create.html#landlab.grid.create.create_grid>`_.
        function.

        Parameters
        ----------
        params : dict
            Dictionary of input parameters.
        output_writers : dictionary of output writers.
            Classes or functions used to write incremental output (e.g. make a
            diagnostic plot). These should be passed in a dictionary with two
            keys: "class" and "function". The value associated with each of
            these should be a list containing the uninstantiated output
            writers. See the Jupyter notebook examples for more details.

        Examples
        --------
        >>> params = {
        ...     "grid": {
        ...         "RasterModelGrid": [
        ...             (4, 5),
        ...             {
        ...                 "fields": {
        ...                     "node": {
        ...                         "topographic__elevation": {
        ...                             "constant": [{"value": 0.0}]
        ...                         }
        ...                     }
        ...                 }
        ...             },
        ...         ]
        ...     },
        ...     "clock": {"step": 1, "stop": 200},
        ... }
        >>> model = ErosionModel.from_dict(params)
        >>> model.clock.step
        1.0
        >>> model.clock.stop
        200.0
        >>> model.grid.shape
        (4, 5)
        """
        cls._validate(params)

        # grid, clock
        grid = create_grid(params.pop("grid"))
        clock = Clock.from_dict(params.pop("clock"))

        # precipitator
        precip_params = params.pop("precipitator", _DEFAULT_PRECIPITATOR)
        precipitator = _setup_precipitator_or_runoff(
            grid, precip_params, _SUPPORTED_PRECIPITATORS
        )

        # runoff_generator
        runoff_params = params.pop(
            "runoff_generator", _DEFAULT_RUNOFF_GENERATOR
        )
        runoff_generator = _setup_precipitator_or_runoff(
            grid, runoff_params, _SUPPORTED_RUNOFF_GENERATORS
        )

        # boundary_handlers
        boundary_handlers = params.pop("boundary_handlers", {})
        bh_dict = {}
        for name in boundary_handlers:
            bh_params = boundary_handlers[name]
            bh_dict[name] = _setup_boundary_handlers(grid, name, bh_params)

        # create instance
        return cls(
            clock,
            grid,
            precipitator=precipitator,
            runoff_generator=runoff_generator,
            boundary_handlers=bh_dict,
            output_writers=output_writers,
            **params
        )

    @classmethod
    def _validate(cls, params):
        """Make sure necessary things for a model grid and a clock are here."""
        if "grid" not in params:
            raise ValueError("No grid provided as part of input parameters")
        if "clock" not in params:
            raise ValueError("No clock provided as part of input parameters")

    def __init__(
        self,
        clock,
        grid,
        precipitator=None,
        runoff_generator=None,
        flow_director="FlowDirectorSteepest",
        depression_finder=None,
        flow_accumulator_kwargs=None,
        boundary_handlers=None,
        output_writers=None,
        output_interval=None,
        save_first_timestep=True,
        output_prefix="terrainbento_output",
        fields=None,
    ):
        """
        Parameters
        ----------
        clock : terrainbento Clock instance
        grid : landlab model grid instance
            The grid must have all required fields.
        precipitator : terrainbento precipitator, optional
            An instantiated version of a valid precipitator. See the
            :py:mod:`precipitator <terrainbento.precipitator>` module for
            valid options. The precipitator creates rain. Default value is the
            :py:class:`UniformPrecipitator` with a rainfall flux of 1.0.
        runoff_generator : terrainbento runoff_generator, optional
            An instantiated version of a valid runoff generator. See the
            :py:mod:`runoff generator <terrainbento.runoff_generator>` module
            for valid options. The runoff generator converts rain into runoff.
            This runoff is then accumulated into surface water discharge
            (:math:`Q`) and used by channel erosion components. Default value
            is :py:class:`SimpleRunoff` in which all rainfall turns into
            runoff. For the drainage area version of the stream power law use
            the default precipitator and runoff_generator.

            If the default values of both the precipitator and
            runoff_generator are used, then :math:`Q` will be equal to drainage
            area.

        flow_director : str, optional
            String name of a
            `Landlab FlowDirector <https://landlab.readthedocs.io/en/master/reference/components/flow_director.html>`_.
            Default is "FlowDirectorSteepest".
        depression_finder : str, optional
            String name of a Landlab depression finder. Default is None.
        flow_accumulator_kwargs : dictionary, optional
            Dictionary of any additional keyword arguments to pass to the
            `Landlab FlowAccumulator <https://landlab.readthedocs.io/en/master/reference/components/flow_accum.html>`_.
            Default is an empty dictionary.
        boundary_handlers : dictionary, optional
            Dictionary with ``name: instance`` key-value pairs. Each entry
            must be a valid instance of a terrainbento boundary handler. See
            the :py:mod:`boundary handlers <terrainbento.boundary_handlers>`
            module for valid options.
        output_writers : dictionary of output writers.
            Classes or functions used to write incremental output (e.g. make a
            diagnostic plot). These should be passed in a dictionary with two
            keys: "class" and "function". The value associated with each of
            these should be a list containing the uninstantiated output
            writers. See the Jupyter notebook examples for more details.
        output_interval : float, optional
            Default is the Clock's stop time.
        save_first_timestep : bool, optional
            Indicates whether model output should be saved at time zero.  Default is
            True.
        output_prefix : str, optional
            String prefix for names of output netCDF files. Default is
            ``"terrainbento_output"``.
        fields : list, optional
            List of field names to write as netCDF output. Default is to only
            write out "topographic__elevation".

        Returns
        -------
        ErosionModel: object

        Examples
        --------
        This model is a base class and is not designed to be run on its own. We
        recommend that you look at the terrainbento tutorials for examples of
        usage.
        """
        flow_accumulator_kwargs = flow_accumulator_kwargs or {}
        boundary_handlers = boundary_handlers or {}
        output_writers = output_writers or {}
        fields = fields or ["topographic__elevation"]
        # type checking
        if isinstance(clock, Clock) is False:
            raise ValueError("Provided Clock is not valid.")
        if isinstance(grid, ModelGrid) is False:
            raise ValueError("Provided Grid is not valid.")

        # save the grid, clock, and parameters.
        self.grid = grid
        self.clock = clock

        # first pass of verifying fields
        self._verify_fields(self._required_fields)

        # save reference to elevation
        self.z = grid.at_node["topographic__elevation"]

        self.grid.add_zeros("node", "cumulative_elevation_change")

        self.grid.add_field(
            "node", "initial_topographic__elevation", self.z.copy()
        )

        # save output_information
        self.save_first_timestep = save_first_timestep
        self._out_file_name = output_prefix
        self.output_fields = fields
        self._output_files = []
        if output_interval is None:
            output_interval = clock.stop
        self.output_interval = output_interval

        # instantiate model time.
        self._model_time = 0.0

        # instantiate container for computational timestep:
        self._compute_time = [tm.time()]

        ###################################################################
        # address Precipitator and RUNOFF_GENERATOR
        ###################################################################

        # verify that precipitator is valid
        if precipitator is None:
            precipitator = UniformPrecipitator(self.grid)
        else:
            if isinstance(precipitator, _VALID_PRECIPITATORS) is False:
                raise ValueError("Provided value for precipitator not valid.")
        self.precipitator = precipitator

        # verify that runoff_generator is valid
        if runoff_generator is None:
            runoff_generator = SimpleRunoff(self.grid)
        else:
            if isinstance(runoff_generator, _VALID_RUNOFF_GENERATORS) is False:
                raise ValueError(
                    "Provide value for runoff_generator not valid."
                )
        self.runoff_generator = runoff_generator

        ###################################################################
        # instantiate flow direction and accumulation
        ###################################################################
        # Instantiate a FlowAccumulator, if DepressionFinder is provided
        # AND director = Steepest, then we need routing to be D4,
        # otherwise, just passing params should be sufficient.
        if (depression_finder is not None) and (
            flow_director == "FlowDirectorSteepest"
        ):
            self.flow_accumulator = FlowAccumulator(
                self.grid,
                routing="D4",
                depression_finder=depression_finder,
                **flow_accumulator_kwargs
            )
        else:
            self.flow_accumulator = FlowAccumulator(
                self.grid,
                flow_director=flow_director,
                depression_finder=depression_finder,
                **flow_accumulator_kwargs
            )

        if self.flow_accumulator.depression_finder is None:
            self._erode_flooded_nodes = True
        else:
            self._erode_flooded_nodes = False

        ###################################################################
        # Boundary Conditions and Output Writers
        ###################################################################
        _verify_boundary_handler(boundary_handlers)
        self.boundary_handlers = boundary_handlers

        if "class" in output_writers:
            instantiated_classes = []
            for ow_class in output_writers["class"]:
                instantiated_classes.append(ow_class(self))
            output_writers["class"] = instantiated_classes

        self.output_writers = output_writers

    def _verify_fields(self, required_fields):
        """Verify all required fields are present."""
        for field in required_fields:
            if field not in self.grid.at_node:
                raise ValueError(
                    "Required field {field} not present.".format(field=field)
                )

    @property
    def model_time(self):
        """Return current time of model integration in model time units."""
        return self._model_time

    def calculate_cumulative_change(self):
        """Calculate cumulative node-by-node changes in elevation."""
        self.grid.at_node["cumulative_elevation_change"][:] = (
            self.grid.at_node["topographic__elevation"]
            - self.grid.at_node["initial_topographic__elevation"]
        )

    def create_and_move_water(self, step):
        """Create and move water.

        Run the precipitator, the runoff generator, and the flow
        accumulator, in that order.
        """
        self.precipitator.run_one_step(step)
        self.runoff_generator.run_one_step(step)
        self.flow_accumulator.run_one_step()

    def write_output(self):
        """Write output to file as a netCDF.

        Filenames will have the value of ``"output_filename"`` from the
        input file or parameter dictionary as the first part of the file
        name and the model run iteration as the second part of the
        filename.
        """
        self.calculate_cumulative_change()
        filename = self._out_file_name + str(self.iteration).zfill(4) + ".nc"
        self._output_files.append(filename)
        if isinstance(self.grid, RasterModelGrid):
            write_raster_netcdf(
                filename, self.grid, names=self.output_fields, format="NETCDF4"
            )
        else:
            to_netcdf(self.grid, filename, format="NETCDF4")

        self.run_output_writers()

    def finalize__run_one_step(self, step):
        """Finalize run_one_step method.

        This base-class method increments model time and updates
        boundary conditions.
        """
        # calculate model time
        self._model_time += step

        # Update boundary conditions
        self.update_boundary_conditions(step)

    def finalize(self):
        """Finalize model.

        This base-class method does nothing. Derived classes can
        override it to run any required finalization steps.
        """
        pass

    def run_for(self, step, runtime):
        """Run model without interruption for a specified time period.

        ``run_for`` runs the model for the duration ``runtime`` with model time
        steps of ``step``.

        Parameters
        ----------
        step : float
            Model run timestep.
        runtime : float
            Total duration for which to run model.
        """
        elapsed_time = 0.0
        keep_running = True
        while keep_running:
            if elapsed_time + step >= runtime:
                step = runtime - elapsed_time
                keep_running = False
            self.run_one_step(step)
            elapsed_time += step

    def run(self):
        """Run the model until complete.

        The model will run for the duration indicated by the input file
        or dictionary parameter ``"stop"``, at a time step specified by
        the parameter ``"step"``, and create ouput at intervals of
        ``"output_duration"``.
        """
        self._itters = []

        if self.save_first_timestep:
            self.iteration = 0
            self._itters.append(0)
            self.write_output()
        self.iteration = 1
        time_now = self._model_time
        while time_now < self.clock.stop:
            next_run_pause = min(
                time_now + self.output_interval, self.clock.stop
            )
            self.run_for(self.clock.step, next_run_pause - time_now)
            time_now = self._model_time
            self._itters.append(self.iteration)
            self.write_output()
            self.iteration += 1

        # now that the model is finished running, execute finalize.
        self.finalize()

    def _ensure_precip_runoff_are_vanilla(self, vsa_precip=False):
        """Ensure only default versions of precipitator/runoff are used.

        Some models only work when the precipitator and runoff generator
        are the default versions.
        """
        if isinstance(self.precipitator, UniformPrecipitator) is False:
            raise ValueError(
                "This model must be run with a UniformPrecipitator."
            )

        if vsa_precip is False:
            if self.precipitator._rainfall_flux != 1:
                raise ValueError(
                    "This model must use a rainfall__flux value of 1.0."
                )

        # if isinstance(self.runoff_generator, SimpleRunoff) is False:
        #     raise ValueError("This model must be run with SimpleRunoff.")

        if self.runoff_generator.runoff_proportion != 1.0:
            raise ValueError("The model must use a runoff_proportion of 1.0.")

    def run_output_writers(self):
        """Run all output writers."""
        if "class" in self.output_writers:
            for ow_class in self.output_writers["class"]:
                ow_class.run_one_step()
        if "function" in self.output_writers:
            for ow_function in self.output_writers["function"]:
                ow_function(self)

    def update_boundary_conditions(self, step):
        """Run all boundary handlers forward by step.

        Parameters
        ----------
        step : float
            Timestep in unit of model time.
        """
        # Run each of the baselevel handlers.
        for name in self.boundary_handlers:
            self.boundary_handlers[name].run_one_step(step)

    def to_xarray_dataset(
        self,
        time_unit="time units",
        reference_time="model start",
        space_unit="space units",
    ):
        """Convert model output to an xarray dataset.

        If you would like to have CF compliant NetCDF make sure that your time
        and space units and reference times will work with standard decoding.

        The default time unit and reference time will give the time dimention a
        value of "time units since model start". The default space unit will
        give a value of "space unit".

        Parameters
        ----------
        time_unit: str, optional
            Name of time unit. Default is "time units".
        reference time: str, optional
            Reference tim. Default is "model start".
        space_unit: str, optional
            Name of space unit. Default is "space unit".
        """
        # open all files as a xarray dataset
        ds = xr.open_mfdataset(
            self._output_files,
            concat_dim="nt",
            engine="netcdf4",
            combine="nested",
            data_vars=self.output_fields,
        )

        # add a time dimension
        time_array = np.asarray(self._itters) * self.output_interval
        time = xr.DataArray(
            time_array,
            dims=("nt"),
            attrs={
                "units": time_unit + " since " + reference_time,
                "standard_name": "time",
            },
        )

        ds["time"] = time

        # set x and y to coordinates
        ds = ds.set_coords(["x", "y", "time"])

        # rename dimensions
        ds = ds.rename(name_dict={"ni": "x", "nj": "y", "nt": "time"})

        # set x and y units
        ds["x"] = xr.DataArray(ds.x, dims=("x"), attrs={"units": space_unit})
        ds["y"] = xr.DataArray(ds.y, dims=("y"), attrs={"units": space_unit})

        return ds

    def save_to_xarray_dataset(
        self,
        filename="terrainbento.nc",
        time_unit="time units",
        reference_time="model start",
        space_unit="space units",
    ):
        """Save model output to xarray dataset.

        If you would like to have CF compliant NetCDF make sure that your time
        and space units and reference times will work with standard decoding.

        The default time unit and reference time will give the time dimention a
        value of "time units since model start". The default space unit will
        give a value of "space unit".

        Parameters
        ----------
        filename: str, optional
            The file path where the file should be saved. The default value is
            "terrainbento.nc".
        time_unit: str, optional
            Name of time unit. Default is "time units".
        reference time: str, optional
            Reference tim. Default is "model start".
        space_unit: str, optional
            Name of space unit. Default is "space unit".
        """
        ds = self.to_xarray_dataset(
            time_unit=time_unit,
            space_unit=space_unit,
            reference_time=reference_time,
        )
        ds.to_netcdf(filename, engine="netcdf4", format="NETCDF4")
        ds.close()

    def remove_output_netcdfs(self):
        """Remove all netCDF files written by a model run."""
        try:
            for f in self._output_files:
                os.remove(f)
        except WindowsError:  # pragma: no cover
            print(
                "The Windows OS is picky about file-locks and did not permit "
                "terrainbento to remove the netcdf files."
            )  # pragma: no cover

def main():  # pragma: no cover
    """Executes model."""
    try:
        infile = sys.argv[1]
    except IndexError:
        print("Must include input file name on command line")
        sys.exit(1)

    erosion_model = ErosionModel.from_file(infile)
    erosion_model.run()

if __name__ == "__main__":
    main()

ext: py

max_stars_repo_name: alexmitchell/terrainbento

max_issues_repo_name: alexmitchell/terrainbento

max_forks_repo_name: alexmitchell/terrainbento

reason: import numpy


text:
import gzip
import numpy as np

EMBEDDINGS_PATH = 'data/word_embeddings/vectors_pruned.200.txt.gz'
EMBEDDINGS_DIM = 200

class Vectorizer():
    _VECTORS = None
    UNK = [0.]*200

    @staticmethod
    def load_pretrained_vectors():
        if Vectorizer._VECTORS is not None:
            return Vectorizer._VECTORS

        Vectorizer._VECTORS = {}

        with gzip.open(EMBEDDINGS_PATH) as f:
            lines = f.readlines()
        for line in lines:
            line = line.split()
            Vectorizer._VECTORS[line[0]] = np.array(map(float, line[1:]))
        return Vectorizer

    @staticmethod
    def vectorize_word(w):
        """Returns the 200 dim word vector of w"""
        return Vectorizer._VECTORS.get(w, Vectorizer.UNK)

    @staticmethod
    def vectorize_sentence(s):
        """
        Returns a list of vectors of the words in the sentence
        """
        return [Vectorizer.vectorize_word(w) for w in s.split()] # + [Vectorizer.vectorize_word('<eos>')]

Vectorizer.load_pretrained_vectors()

# Example usage
print(Vectorizer.vectorize_word('box'))
print(Vectorizer.vectorize_sentence('blue box'))

max_stars_repo_name: manna/musical-autocomplete

max_issues_repo_name: manna/musical-autocomplete

max_forks_repo_name: manna/musical-autocomplete


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

__author__ = 'pygeostat development team'
__date__ = '2020-01-04'
__version__ = '1.0.0'

import os, sys
try:
    import pygeostat as gs
except (ImportError, ModuleNotFoundError):
    sys.path.append(os.path.abspath(os.path.join(os.path.dirname( __file__ ), r'..')))
    import pygeostat as gs

import unittest
import warnings
import subprocess
import numpy as np

class PygeostatDataFileTest(unittest.TestCase):
    '''
     Test suite for pygeostat data file class
    '''

    def setUp(self):
        warnings.simplefilter('ignore', category=ImportWarning)

    def tearDown(self):
        for file in os.listdir('.'):
            _, extention = os.path.splitext(file)
            if extention in ['.vtk', '.h5', '.vtu', '.gsb']:
                os.remove(file)

    def test_file_not_found(self):
        with self.assertRaises(FileNotFoundError):
            _ = gs.DataFile(flname='notfound')

    def test_load_gslib_attributes(self):
        dat = gs.ExampleData('oilsands')
        self.assertEqual(dat.x.lower(), 'east')
        self.assertEqual(dat.y.lower(), 'north')
        self.assertEqual(dat.z.lower(), 'elevation')

    def test_write_read_gsb(self):
        data = gs.ExampleData('point3d_ind_mv')
        try:
            gs.write_gsb(data, 'test1.gsb')
        except Exception:
            self.fail('Unable to write gsb output')

        try:
            data.write_file('test2.gsb')
        except Exception:
            self.fail('Unable to write gsb output')

        import os
        assert os.path.isfile('test1.gsb'), 'file does not exist'
        assert os.path.isfile('test1.gsb'), 'file does not exist'

        try:
            data1 = gs.read_gsb('test1.gsb')
        except Exception:
            self.fail('Unable to read the gsb file')

        try:
            data2 = gs.DataFile('test2.gsb')
        except Exception:
            self.fail('Unable to read the gsb file')

        assert np.all(data1 == data.data)
        assert np.all(data2.data == data.data)

    def test_read_csv(self):
        try:
            dat= gs.read_file('https://people.sc.fsu.edu/~jburkardt/data/csv/addresses.csv')
        except Exception:
            self.fail('Unable to load the CSV file')

    def test_write_read_h5(self):
        data = {'East': [1000, 1200], 'North': [2000, 2500]}
        dat = gs.DataFile(data=data)

        try:
            dat.write_file('test.h5')
        except Exception:
            self.fail('Unable to write hdf5 output')

    def test_write_vtk(self):
        data = {'East': [1000, 1200], 'North': [2000, 2500], 'Elevation': [1,1]}
        dat = gs.DataFile(data=data, dftype='point')

        try:
            dat.write_file('test.vtk')
        except Exception:
            self.fail('Unable to write vtk output')

    def test_set_catdict(self):
        data = {'East': [1000, 1200], 'North': [2000, 2500], 'Elevation': [1,1], 'Categorical': [0,1]}
        dat = gs.DataFile(data=data, dftype='point', cat='Categorical')

        try:
            dat.setcatdict({0: 'Category 1', 1: 'Category 2'})
        except Exception:
            self.fail('Unable to set the categorical dictionary')

    def test_set_catdict_missing_category(self):
        '''
        A test to check a missing categorical code in a dictionary raises the correct exception
        '''

        data = {'East': [1000, 1200, 1500], 'North': [2000, 2500, 2600], 'Elevation': [1,1, 2], 'Categorical': [0,1,-99]}
        dat = gs.DataFile(data=data, dftype='point', cat='Categorical')

        with self.assertRaises(ValueError):
            dat.setcatdict({0: 'Category 1', 1: 'Category 2'})

    def test_set_catdict_typeissue(self):
        '''
        A test to check that provided non numeric codes can be handled by a proper error message
        '''
        data = {'East': [1000, 1200, 1500], 'North': [2000, 2500, 2600], 'Elevation': [1,1, 2], 'Categorical': [0,1,'-99']}
        dat = gs.DataFile(data=data, dftype='point', cat='Categorical')

        with self.assertRaises(TypeError):
            dat.setcatdict({0: 'Category 1', 1: 'Category 2', '-99': 'Unknown'})

class PygeostatDataFileInfergriddefTest(unittest.TestCase):
    '''
     Test suite for pygeostat infergriddef of data file class
    '''
    def setUp(self):
        warnings.simplefilter('ignore', category=ImportWarning)

    def compare(self, griddef1, griddef2):
        assert griddef1.nx   == griddef2.nx
        assert griddef1.ny   == griddef2.ny
        assert griddef1.nz   == griddef2.nz
        assert griddef1.xmn  == griddef2.xmn
        assert griddef1.ymn  == griddef2.ymn
        assert griddef1.zmn  == griddef2.zmn
        assert griddef1.xsiz == griddef2.xsiz
        assert griddef1.ysiz == griddef2.ysiz
        self.assertEqual(griddef1.zsiz,  griddef2.zsiz)

    def test_infergriddef(self):
        df2d = gs.ExampleData("point2d_ind")
        df3d = gs.ExampleData("point3d_ind_mv")
        #where xsiz = ysiz = zsiz, a float can also be provided.
        self.compare(df3d.infergriddef(blksize = 75),
                     df3d.infergriddef(blksize = [75,75,75]))

        #where nx = ny = nz, an int can also be provided.
        self.compare(df3d.infergriddef(nblk = 60) ,
                     df3d.infergriddef(nblk = [60,60,60]))

        #for 3D data, infergriddef() must return a 3D grid definition even if zsiz is given as None or 0 or 1.
        self.compare(df3d.infergriddef(blksize = [50,60,1]),
                     gs.GridDef('''20  135.0  50.0
                                   19  1230.0 60.0
                                   82  310.5  1.0'''))

        self.compare(df3d.infergriddef(blksize = [50,60,1]),
                     df3d.infergriddef(blksize = [50,60,None]))

        self.compare(df3d.infergriddef(blksize = [50,60,1]),
                     df3d.infergriddef(blksize = [50,60,0]))

        # nz given as None or 0 or 1 returns a 2D grid that covers the vertical extent of a 3D dataset.
        self.compare(df3d.infergriddef(nblk = [50,60,1]),
                     gs.GridDef('''50  119.8   19.6
                                   60  1209.1  18.2
                                   1   350.85  81.7'''))

        self.compare(df3d.infergriddef(nblk = [50,60,1]),
                     df3d.infergriddef(nblk = [50,60,None]))

        self.compare(df3d.infergriddef(nblk = [50,60,1]),
                     df3d.infergriddef(nblk = [50,60,0]))

        #if nblk is not integer,infergriddef(), must convert it to int and provide correct grid definition
        _nblk = [12.3, 5.7, 10.11]
        self.compare(df3d.infergriddef(nblk=_nblk),
                     df3d.infergriddef(nblk=[int(i)  for i in _nblk]))

        # If data is 2-D, zsiz must be provided as None. Otherwise it must raise exception.
        with self.assertRaises(ValueError):
            df2d.infergriddef(blksize = [50,60,4])

        with self.assertRaises(ValueError):
            df2d.infergriddef(blksize = 75)

        self.compare(df2d.infergriddef(blksize = [50,60,None]),
                     gs.GridDef('''24   25.0    50.0
                                   19   1230.0  60.0
                                   1    0.5     1.0'''))

        #If data is 2-D, nz must be provided as None. Otherwise it must raise exception.
        with self.assertRaises(ValueError):
            df2d.infergriddef(nblk = [50,60,1])

        with self.assertRaises(ValueError):
            df2d.infergriddef(nblk = 60)

        self.compare(df2d.infergriddef(nblk = [50,60,None]),
                gs.GridDef('''50   11.9    23.8
                              60   1209.1  18.2
                              1    0.5     1.0'''))

class PygeostatGridDefFromString(unittest.TestCase):
    '''
     Test suite for pygeostat GridDef class with grid string
    '''

    def setUp(self):
        warnings.simplefilter('ignore', category=ImportWarning)
        self.grid = gs.GridDef('''40  0.5  1
                     40  0.5  1
                     40  0.5  1''')
        self.testfl = gs.ExampleData('3d_grid', griddef=self.grid)

    def test_grid_count(self):
        assert self.testfl.griddef.count() == 64000
        self.assertEqual(self.grid.count(), 64000)

    def test_grid_origin(self):
        assert self.testfl.griddef.origin() == (0.0, 0.0, 0.0)
        assert self.grid.origin() == (0.0, 0.0, 0.0)

    def test_grid_blockvolume(self):
        assert self.testfl.griddef.block_volume == 1.0
        assert self.grid.block_volume == 1.0

    def test_grid_extents(self):
        assert self.testfl.griddef.extents() == [(0.0, 40.0), (0.0, 40.0), (0.0, 40.0)]
        assert self.grid.extents() == [(0.0, 40.0), (0.0, 40.0), (0.0, 40.0)]

    def test_grid_indexcoords(self):
        assert self.testfl.griddef.get_index(10.0, 5.0, 15.0) == (22569, True)
        assert self.grid.get_index(10.0, 5.0, 15.0) == (22569, True)
        # Check for coordinates outside of grid
        assert self.testfl.griddef.get_index(10.0, 5.0, 50.0) == (-1, False)
        assert self.grid.get_index(10.0, 5.0, 50.0) == (-1, False)

    def test_grid_indicescoords(self):
        assert self.testfl.griddef.get_index3d(10.0, 5.0, 15.0) == (9, 4, 14, True)
        assert self.grid.get_index3d(10.0, 5.0, 15.0) == (9, 4, 14, True)
        # Check for coordinates outside of grid
        assert self.testfl.griddef.get_index3d(10.0, 5.0, 50.0) == (-1, -1, -1, False)
        assert self.grid.get_index3d(10.0, 5.0, 50.0) == (-1, -1, -1, False)

class PygeostatGridDefFromArray(unittest.TestCase):
    '''
     Test suite for pygeostat GridDef class with grid array
    '''

    def setUp(self):
        warnings.simplefilter('ignore', category=ImportWarning)
        self.grid = gs.GridDef([40,0.5,1,40,0.5,1,40,0.5,1])
        self.testfl = gs.ExampleData('3d_grid', griddef=self.grid)

    def test_grid_count(self):
        assert self.testfl.griddef.count() == 64000
        self.assertEqual(self.grid.count(), 64000)

    def test_grid_origin(self):
        assert self.testfl.griddef.origin() == (0.0, 0.0, 0.0)
        assert self.grid.origin() == (0.0, 0.0, 0.0)

    def test_grid_blockvolume(self):
        assert self.testfl.griddef.block_volume == 1.0
        assert self.grid.block_volume == 1.0

    def test_grid_extents(self):
        assert self.testfl.griddef.extents() == [(0.0, 40.0), (0.0, 40.0), (0.0, 40.0)]
        assert self.grid.extents() == [(0.0, 40.0), (0.0, 40.0), (0.0, 40.0)]

    def test_grid_indexcoords(self):
        assert self.testfl.griddef.get_index(10.0, 5.0, 15.0) == (22569, True)
        assert self.grid.get_index(10.0, 5.0, 15.0) == (22569, True)
        # Check for coordinates outside of grid
        assert self.testfl.griddef.get_index(10.0, 5.0, 50.0) == (-1, False)
        assert self.grid.get_index(10.0, 5.0, 50.0) == (-1, False)

    def test_grid_indicescoords(self):
        assert self.testfl.griddef.get_index3d(10.0, 5.0, 15.0) == (9, 4, 14, True)
        assert self.grid.get_index3d(10.0, 5.0, 15.0) == (9, 4, 14, True)
        # Check for coordinates outside of grid
        assert self.testfl.griddef.get_index3d(10.0, 5.0, 50.0) == (-1, -1, -1, False)
        assert self.grid.get_index3d(10.0, 5.0, 50.0) == (-1, -1, -1, False)

if __name__ == '__main__':
	subprocess.call([sys.executable, '-m', 'unittest', str(__file__), '-v'])

max_stars_repo_name: haroldvelasquez/pygeostat

max_issues_repo_name: haroldvelasquez/pygeostat

max_forks_repo_name: haroldvelasquez/pygeostat


text:
"""How to implement a PAX module using python dataclass."""

from dataclasses import dataclass, field
from typing import Callable, Optional

import jax
import jax.numpy as jnp
import pax

@dataclass
class Linear(pax.Module):
    """A linear module"""

    in_dim: int
    out_dim: int
    with_bias: bool = True
    name: Optional[str] = None
    weight: jnp.ndarray = field(init=False, repr=False)
    bias: Optional[jnp.ndarray] = field(init=False, repr=False)
    counter: jnp.ndarray = field(init=False)
    w_init: Callable = field(default=jax.nn.initializers.normal(), repr=False)
    b_init: Callable = field(default=jax.nn.initializers.zeros, repr=False)
    parameters = pax.parameters_method("weight", "bias")

    def __post_init__(self):
        self.weight = self.w_init(pax.next_rng_key(), (self.in_dim, self.out_dim))
        self.bias = None
        if self.with_bias:
            self.bias = self.b_init(pax.next_rng_key(), (self.out_dim,))
        self.counter = jnp.array(0)

    def __call__(self, x: jnp.ndarray) -> jnp.ndarray:
        self.counter += 1
        x = jnp.dot(x, self.weight)
        if self.with_bias:
            x = x + self.bias
        return x

pax.seed_rng_key(42)

fc = Linear(3, 4, name="fc1")

print("Before:", fc)
dummy_x = jnp.empty((32, 3))
fc, y = pax.purecall(fc, dummy_x)
assert y.shape == (32, 4)
print("After :", fc)

reason: import jax


text:
# --------------
# Importing header files
import numpy as np

# Path of the file has been stored in variable called 'path'

#New record
new_record=[[50,  9,  4,  1,  0,  0, 40,  0]]

#Code starts here
data = np.genfromtxt(path, delimiter=',', skip_header=1)

census = np.concatenate((data, new_record), axis=0)

# --------------
#Code starts here
age = census[:,0]
print(age)

max_age = max(age)
min_age = min(age)
age_mean= np.mean(age)
age_std = np.std(age)

# --------------
#Code starts here
race_0 = census[np.where(census[:,2] == 0)]
race_1 = census[np.where(census[:,2] == 1)]
race_2 = census[np.where(census[:,2] == 2)]
race_3 = census[np.where(census[:,2] == 3)]
race_4 = census[np.where(census[:,2] == 4)]

len_0 = race_0.shape[0]
len_1 = race_1.shape[0]
len_2 = race_2.shape[0]
len_3 = race_3.shape[0]
len_4 = race_4.shape[0]

minority_race = np.argmin([len_0, len_1, len_2, len_3, len_4])
print(minority_race)

# --------------
#Code starts here
senior_citizens = census[census[:,0] > 60]

working_hours_sum = np.sum(senior_citizens[:,6])

senior_citizens_len = senior_citizens.shape[0]

avg_working_hours = working_hours_sum/ senior_citizens_len

print(avg_working_hours)

# --------------
#Code starts here
high = census[census[:,1] > 10]
low = census[census[:,1] <= 10]

avg_pay_high = np.mean(high[:,7])
avg_pay_low = np.mean(low[:,7])

print(avg_pay_high)
print('')
print(avg_pay_low)

max_stars_repo_name: NeelkanthMehta/ga-dsmp

max_issues_repo_name: NeelkanthMehta/ga-dsmp

max_forks_repo_name: NeelkanthMehta/ga-dsmp


text:
# -*- coding: utf-8 -*-
import cv2
import time
import torch
import numpy as np
from utils import get_affine_transform,_transpose_and_gather_feat, _nms, _topk, _topk_channel, multi_pose_post_process
from show_all_imgs import add_coco_hp

num_classes = 1
vis_thresh = 0.3

# mean & std value of per pictrue
mean = [0.408, 0.447, 0.470]
std = [0.289, 0.274, 0.278]

def pre_process(image, scale, meta=None):
    height, width = image.shape[0:2]
    new_height = int(height * scale)
    new_width  = int(width * scale)

    inp_height, inp_width = 416, 416
    c = np.array([new_width / 2., new_height / 2.])
    s = max(height, width) * 1.0

    trans_input = get_affine_transform(c, s, 0, [inp_width, inp_height])
    resized_image = cv2.resize(image, (new_width, new_height))
    inp_image = cv2.warpAffine(resized_image, trans_input, (inp_width, inp_height), flags=cv2.INTER_LINEAR)

    images = inp_image
    meta = {'c': c,'s': s, 'out_height': inp_height //4, 'out_width': inp_width // 4}
    return images, meta

#==================================================================================

def multi_pose_decode(heat, wh, kps, reg=None, hm_hp=None, hp_offset=None, K=100):
    #>>>:param heat: keypoint heatmap 定位目标中心点的heatmap : (1, 1, 128, 128)
    #    :param wh: object size 确定矩形宽高  : (1, 2, 128, 128)
    #>>>:param kps: joint locations 相对于目标中心的各关键点偏移 : (1, 34, 128, 128)
    #    :param reg: local offset 包围框的偏移补偿  : (1, 2, 128, 128)
    #>>>:param hm_hp: joint heatmap 一般的关键点估计heatmap : (1, 17, 128, 128)
    #    :param hp_offset: joint offset 关键点估计的偏移 : (1, 2, 128, 128)
    #    :param K: top-K :100

	# batch:1  cat:1(类别数)  height:128  width:128
    _, cat, height, width = heat.size()

    heat = heat.reshape([-1, height, width])
    wh = wh.reshape([-1, height, width])
    kps = kps.reshape([-1, height, width])
    reg = reg.reshape([-1, height, width])
    hm_hp = hm_hp.reshape([-1, height, width])
    hp_offset = hp_offset.reshape([-1, height, width])

    num_joints = 17
    heat = _nms(heat)
    scores, inds, clses, ys, xs = _topk(heat, K=K)

    kps = _transpose_and_gather_feat(kps, inds)
    kps[..., ::2] += xs.view(  K, 1).expand(  K, num_joints)
    kps[..., 1::2] += ys.view(  K, 1).expand(  K, num_joints)

    if reg is not None:
        reg = _transpose_and_gather_feat(reg, inds)
        xs = xs.view(  K, 1) + reg[ :, 0:1]
        ys = ys.view(  K, 1) + reg[ :, 1:2]
    else:
        xs = xs.view(  K, 1) + 0.5
        ys = ys.view(  K, 1) + 0.5

    wh = _transpose_and_gather_feat(wh, inds)
    wh = wh.view(  K, 2)

    scores = scores.view(  K, 1)
    bboxes = torch.cat([xs - wh[..., 0:1] / 2,
                        ys - wh[..., 1:2] / 2,
                        xs + wh[..., 0:1] / 2,
                        ys + wh[..., 1:2] / 2], dim=1)

    if hm_hp is not None:
        hm_hp_nms_time = time.time()
        hm_hp = _nms(hm_hp)
        thresh = 0.1
        kps = kps.view( K, num_joints, 2).permute( 1, 0, 2).contiguous() # J x K x 2
        reg_kps = kps.unsqueeze(2).expand(  num_joints, K, K, 2)

        hm_score, hm_inds, hm_ys, hm_xs = _topk_channel(hm_hp, K=K) # J x K

        hm_inds = hm_inds.reshape([17*K])
        if hp_offset is not None:
            hp_offset = _transpose_and_gather_feat(hp_offset, hm_inds)
            hp_offset = hp_offset.view(  num_joints, K, 2)
            hm_xs = hm_xs + hp_offset[ :, :, 0]
            hm_ys = hm_ys + hp_offset[ :, :, 1]

        else:
            hm_xs = hm_xs + 0.4
            hm_ys = hm_ys + 0.4

        mask = (hm_score > thresh).float()
        hm_score = (1 - mask) * -1 + mask * hm_score
        hm_ys = (1 - mask) * (-10000) + mask * hm_ys
        hm_xs = (1 - mask) * (-10000) + mask * hm_xs

        hm_kps = torch.stack([hm_xs, hm_ys], dim=-1).unsqueeze(1).expand(  num_joints, K, K, 2)

        dist = (((reg_kps - hm_kps) ** 2).sum(dim=3) ** 0.5)  # [17, k, k]
        min_dist, min_ind = dist.min(dim=2) # J x K  [17,8]
        hm_score = hm_score.gather(1, min_ind).unsqueeze(-1) # J x K x 1  [17, 8, 1]

        min_dist = min_dist.unsqueeze(-1)  # [17, 8, 1]
        min_ind = min_ind.view(  num_joints, K, 1, 1).expand(  num_joints, K, 1, 2)  # [17, 8, 1, 2]

        hm_kps = hm_kps.gather(2, min_ind)  #  [17, 8, 1, 2]
        hm_kps = hm_kps.view(  num_joints, K, 2)  #  [17, 8, 1, 2]

        l = bboxes[ :, 0].view(  1, K, 1).expand(  num_joints, K, 1)
        t = bboxes[ :, 1].view(  1, K, 1).expand(  num_joints, K, 1)
        r = bboxes[ :, 2].view(  1, K, 1).expand(  num_joints, K, 1)
        b = bboxes[ :, 3].view(  1, K, 1).expand(  num_joints, K, 1)

        mask = (hm_kps[..., 0:1] < l) + (hm_kps[..., 0:1] > r) + \
                (hm_kps[..., 1:2] < t) + (hm_kps[..., 1:2] > b) + \
                (hm_score < thresh) + (min_dist > (torch.max(b - t, r - l) * 0.3))

        mask = (mask > 0).float().expand(  num_joints, K, 2)
        kps = (1 - mask) * hm_kps + mask * kps
        kps = kps.permute( 1,0, 2).contiguous().view(  K, num_joints * 2)
    # detections = torch.cat([bboxes, scores, kps, clses], dim=2)
    detections = torch.cat([bboxes, scores, kps, torch.transpose(hm_score.squeeze(dim=2), 0, 1)], dim=1)

    return detections.numpy()

#===============================================================================
# post process of decode output
def post_process(dets, meta, scale=1):
    dets = multi_pose_post_process(dets.copy(), [meta['c']], [meta['s']], meta['out_height'], meta['out_width'])
    dets = np.array(dets, dtype=np.float32).reshape(-1, 56) # 0:3,x1,y1;x2,y2 4:中心点置信度 5:39,keypoint 40:56, keypoint 置信度
    return dets

# ===============================================================================
# show results
def show_results(image, results):
    for b_id, detection in enumerate(results):
        bbox = detection[:4]
        bbox_prob = detection[4]
        keypoints = detection[5:39]
        keypoints_prob = detection[39:]
        if bbox_prob > vis_thresh:
            add_coco_hp(image, keypoints, keypoints_prob)

max_stars_repo_name: wenjunyoung/multipose

max_issues_repo_name: wenjunyoung/multipose

max_forks_repo_name: wenjunyoung/multipose


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 23 16:53:22 2021

@author: Clau
"""

import numpy as np
import pandas as pd

#HI
df1 = pd.read_csv('con_IGAS.csv')
df1.columns = ['T' , 'W']
HI_Kwh = (((df1['W'].sum())/60)/1000)
print(HI_Kwh)

'''
#HMI
df2 = pd.read_csv('results/HMI_El_Espino.csv')
df2.columns = ['T' , 'W']
HMI_Kwh = (((df2['W'].sum())/60)/1000)
print(HMI_Kwh)

#LMI
df3 = pd.read_csv('results/LMI_El_Espino.csv')
df3.columns = ['T' , 'W']
LMI_Kwh = (((df3['W'].sum())/60)/1000)
print(LMI_Kwh)

#LI
df4 = pd.read_csv('results/LI_El_Espino.csv')
df4.columns = ['T' , 'W']
LI_Kwh = (((df4['W'].sum())/60)/1000)
print(LI_Kwh)

#If we consider just two categories

'''


text:
"""
This is an quantum program used for generating random number. Created on November 11, 2019
"""
import numpy as np
from qiskit import(QuantumCircuit, execute, Aer,IBMQ) # load modules for quantum computing
import time,sys
from qiskit.tools.monitor import job_monitor

def qc(n):
    # Create Quantum Circuit
    circ = QuantumCircuit(n,n) # Create a Quantum Circuit of n qubits
    circ.h(range(n))
    for i in range(n):
        circ.measure(i,i)

    # Assign the simulator
    backend_sim = Aer.get_backend('qasm_simulator') # The Aer's qasm_simulator
    job_sim = execute(circ, backend_sim, shots = 1024) # Execute the circuit on the qasm_simulator and set 1024 shots
    result_sim = job_sim.result()
    counts = result_sim.get_counts(circ)
    print(counts)
"""
Implement real Quantum Computing device
"""
    provider = IBMQ.get_provider(group = 'open')
    backend = provider.get_backend('ibmqx4')
    job = execute(circ, backend = backend, shots = 1024)
    job_monitor(job)
    result = job.result()
    counts_real = result.get_counts(circ)
    print(counts_real)

    bits = ""
    for x in counts.values():
        if x > shots/(2**n):
            bits += "1"
        else:
            bits += "0"
    return int(bits, 2)

if __name__ == '__main__':
    start_time = time.time()
    numbers = []
    size = 10
    qubits = 3 # bits = 2**qubits
    shots = 1024
    for i in range(size):
        n = qc(qubits)
        numbers.append(n)
    print("list=" + str(numbers))
    print("Calculation time:", (time.time()-start_time),"s")

max_stars_repo_name: qingxinlianshui/Quanta

max_issues_repo_name: qingxinlianshui/Quanta

max_forks_repo_name: qingxinlianshui/Quanta


text:
#!/usr/bin/env python2

import json
import numpy as np
import scipy.signal
from scipy.optimize import leastsq
import matplotlib.pyplot as plt
from copy import deepcopy

class Spectrum(object):
    def __init__(self):
        self.omega = []
        self.Z = []

    @classmethod
    def fromExperiments(cls, data):
        s = cls()
        """ data: List of Experiment-objects
            """
        if len(data) < 2:
            raise ValueError(
                "Too little experiments in spectrum: %i" %
                len(data))
        data.sort(key=lambda e: e.w)
        w, z = map(list, map(None, *[d.impedance() for d in data]))
        s.omega = np.array(w)
        s.Z = np.array(z)
        return s

    @classmethod
    def fromCircuit(cls, omega, eqc, p):
        """ omega: list of angular frequencies for this spectrum
            eqc: equivalent circuit function returning the impedance
            p: parameters to be used for the equivalent circuit function
            """
        s = cls()
        s.omega = np.array(omega)
        s.Z = np.array([eqc(w, p) for w in omega])
        s.eqc = eqc
        return s

    @classmethod
    def fromRawData(cls, omega, Z):
        """ omega: list of angular frequencies for this spectrum
            Z: list of impedances for this spectrum
            """
        s = cls()
        s.omega = deepcopy(np.array(omega))
        s.Z = deepcopy(np.array(Z))
        return s

    @classmethod
    def fromJSON(cls, jsonstring):
        d = json.loads(jsonstring)
        s = cls()
        s.omega = d['omega']
        s.Z = np.array(d['Re']) + 1j * np.array(d['Im'])
        return s

    def toJSON(self):
        return json.dumps({'omega':list(self.omega),
                           'Re':list(np.real(self.Z)),
                           'Im':list(np.imag(self.Z))})

    def fit(self, spectrum, pset):
        # TODO: utilize jacobian, if possible
        def residuals(p, spectrum, pset):
            pset.updateUnmaskedTransformedValues(np.abs(p))
            pset.applyConstraints()
            res = []
            for i, w in enumerate(spectrum.omega):
                z = self.eqc(w, pset._values)
                res.append(np.real(z) - np.real(spectrum.Z[i]))
                res.append(np.imag(z) - np.imag(spectrum.Z[i]))
            return np.array(res)

        p0 = pset.getUnmaskedTransformedValues()
        plsq = leastsq(residuals, p0, args=(spectrum, pset), \
                full_output = True,
                xtol = 1e-20,
                ftol = 1e-12,
                factor = 1)
        pset.updateUnmaskedTransformedValues(np.abs(plsq[0]))
        r = residuals(plsq[0], spectrum, pset)
        rmsd = np.sqrt(np.sum(np.power(r,2.0))/len(r))
        return pset, rmsd

    def updateParameter(self, p):
        self.p = np.array(p)
        self.Z = np.array([self.eqc(w, p) for w in self.omega])

    def interpolate(
            self, new_omega=2 * np.pi * np.power(10, np.arange(-5, 10, 0.1))):
        re = np.interp(new_omega, self.omega, np.real(self.Z))
        im = np.interp(new_omega, self.omega, np.imag(self.Z))
        self.Z = re + 1j * im
        self.omega = np.array(new_omega)

    def getImaginaryMaxima(self):
        return scipy.signal.argrelmax(np.imag(self.Z))

    def getImaginaryMinima(self):
        return scipy.signal.argrelmin(np.imag(self.Z))

    def cropFrequencyRange(self, wRange):
        newZ = []
        newOmega = []
        for i, w in enumerate(self.omega):
            if (wRange[0] == None or w > wRange[0]) and \
               (wRange[1] == None or w <= wRange[1]):
                newOmega.append(w)
                newZ.append(self.Z[i])
        self.Z = np.array(newZ)
        self.omega = np.array(newOmega)

    def plot_bode(self, style=['b+-', 'g+-']):
        re, = plt.semilogx(self.omega, np.real(self.Z), style[0])
        im, = plt.semilogx(self.omega, np.imag(self.Z), style[1])
        plt.legend([re, im], ['Re(Z)', 'Im(Z)'])
        return re, im

    def plot_nyquist(self, style='rx-'):
        plt.gca().set_aspect('equal')
        return plt.plot(np.real(self.Z), np.imag(self.Z), style)

    def do_kkr(self, change=False):
        ng = 1
        absZ = np.abs(self.Z)
        phiZ = np.angle(self.Z)
        linF = np.log(self.omega)
        int_term = np.zeros(self.Z.shape)
        diff_term = np.zeros(self.Z.shape)
        for k in xrange(ng, phiZ.shape[0] - ng):
            int_term[k] = -np.trapz(phiZ[k:], linF[k:])
            diff_term[k] = np.mean(
                np.diff(
                    phiZ[
                        k -
                        1:k +
                        2]) /
                np.diff(
                    linF[
                        k -
                        1:k +
                        2]))
        gamma = -np.pi / 6.0
        lnH = 2.0 / np.pi * int_term + gamma * diff_term
        err_lnH = np.log(absZ[ng:-ng]) - lnH[ng:-ng]
        constOffset = np.mean(err_lnH)
        abszhit = np.zeros(absZ.shape)
        abszhit[ng:-ng] = np.exp(constOffset + lnH[ng:-ng])
        z = abszhit * np.exp(1j * phiZ)
        w = self.omega
        if change:
            self.omega = w[ng:-ng]
            self.Z = z[ng:-ng]
        return Spectrum.fromRawData(w[ng:-ng], z[ng:-ng])

max_stars_repo_name: robbje/eis

max_issues_repo_name: robbje/eis

max_forks_repo_name: robbje/eis

reason: import numpy,import scipy,from scipy


text:
from math import sqrt, log

import numpy as np
from numpy.linalg import norm, lstsq

import time
import logging
logger = logging.getLogger(__name__)

def lsqr_spark( matrix_Ab, m, n, N, tol=1e-14, iter_lim=None):
    """
    A simple version of LSQR on Spark
    """

    x_iter = []
    time_iter = []
    t0 = time.time()

    logger.info('In LSQR!')
    eps  = 32*np.finfo(float).eps;      # slightly larger than eps

    if tol < eps:
        tol = eps
    elif tol >= 1:
        tol = 1-eps

    max_n_stag = 3

    u = matrix_Ab.get_b()

    beta = norm(u)
    u   /= beta

    v = np.dot( matrix_Ab.ltimes_vec(u), N ).squeeze() # v is an array

    alpha = norm(v)
    if alpha != 0:
       v    /= alpha

    w     = v.copy()
    x     = np.zeros(n)

    phibar = beta
    rhobar = alpha

    nrm_a    = 0.0
    cnd_a    = 0.0
    sq_d     = 0.0
    nrm_r    = beta
    nrm_ar_0 = alpha*beta

    if nrm_ar_0 == 0:     # alpha == 0 || beta == 0
        return x, 0, 0

    nrm_x  = 0
    sq_x   = 0
    z      = 0
    cs2    = -1
    sn2    = 0

    stag   = 0

    flag = -1
    if iter_lim is None:
        iter_lim = np.max( [20, 2*np.min([m,n])] )

    for itn in xrange(int(iter_lim)):

        u = matrix_Ab.rtimes_vec(np.dot(N,v)).squeeze() - alpha*u
        beta = norm(u)
        u   /= beta

        nrm_a = sqrt(nrm_a**2 + alpha**2 + beta**2)

        v = np.dot( matrix_Ab.ltimes_vec(u), N).squeeze() - beta*v

        alpha = norm(v)
        v    /= alpha

        rho    =  sqrt(rhobar**2+beta**2)
        cs     =  rhobar/rho
        sn     =  beta/rho
        theta  =  sn*alpha
        rhobar = -cs*alpha
        phi    =  cs*phibar
        phibar =  sn*phibar

        x      = x + (phi/rho)*w
        w      = v-(theta/rho)*w

        # estimate of norm(r)
        nrm_r   = phibar

        # estimate of norm(A'*r)
        nrm_ar  = phibar*alpha*np.abs(cs)

        # check convergence
        if nrm_ar < tol*nrm_ar_0:
            flag = 0
        #    break

        if nrm_ar < eps*nrm_a*nrm_r:
            flag = 0
        #    break

        # estimate of cond(A)
        sq_w    = np.dot(w,w)
        nrm_w   = sqrt(sq_w)
        sq_d   += sq_w/(rho**2)
        cnd_a   = nrm_a*sqrt(sq_d)

        # check condition number
        if cnd_a > 1/eps:
            flag = 1
        #    break

        # check stagnation
        if abs(phi/rho)*nrm_w < eps*nrm_x:
            stag += 1
        else:
            stag  = 0
        if stag >= max_n_stag:
            flag = 1
        #    break

        # estimate of norm(x)
        delta   =  sn2*rho
        gambar  = -cs2*rho
        rhs     =  phi - delta*z
        zbar    =  rhs/gambar
        nrm_x   =  sqrt(sq_x + zbar**2)
        gamma   =  sqrt(gambar**2 + theta**2)
        cs2     =  gambar/gamma
        sn2     =  theta /gamma
        z       =  rhs   /gamma
        sq_x   +=  z**2

        x_iter.append(x)
        time_iter.append( time.time() - t0 )

        logger.info("Finished one iteration!")

    y_iter = x_iter
    x_iter = [np.dot(N,x) for x in x_iter]
    return x_iter, y_iter, time_iter

max_stars_repo_name: chocjy/randomized-LS-solvers

max_issues_repo_name: chocjy/randomized-LS-solvers

max_forks_repo_name: chocjy/randomized-LS-solvers

reason: import numpy,from numpy