text:
# -*- encoding: utf-8 -*-
import numbers
import multiprocessing
import glob
import gzip
import os
import re
import time
import traceback
from typing import Optional, Union

import numpy as np
import pynisher
import lockfile
from sklearn.utils.validation import check_random_state

from autosklearn.util.backend import Backend
from autosklearn.constants import BINARY_CLASSIFICATION
from autosklearn.metrics import calculate_score, Scorer
from autosklearn.ensembles.ensemble_selection import EnsembleSelection
from autosklearn.ensembles.abstract_ensemble import AbstractEnsemble
from autosklearn.util.logging_ import get_logger

Y_ENSEMBLE = 0
Y_VALID = 1
Y_TEST = 2

class EnsembleBuilder(multiprocessing.Process):
    def __init__(
            self,
            backend: Backend,
            dataset_name: str,
            task_type: int,
            metric: Scorer,
            limit: int,
            ensemble_size: int = 10,
            ensemble_nbest: int = 100,
            max_models_on_disc: int = 100,
            performance_range_threshold: float = 0,
            seed: int = 1,
            shared_mode: bool = False,
            max_iterations: int = None,
            precision: str = "32",
            sleep_duration: int = 2,
            memory_limit: int = 1000,
            read_at_most: int = 5,
            random_state: Optional[Union[int, np.random.RandomState]] = None,
    ):
        """
            Constructor

            Parameters
            ----------
            backend: util.backend.Backend
                backend to write and read files
            dataset_name: str
                name of dataset
            task_type: int
                type of ML task
            metric: str
                name of metric to score predictions
            limit: int
                time limit in sec
            ensemble_size: int
                maximal size of ensemble (passed to autosklearn.ensemble.ensemble_selection)
            ensemble_nbest: int/float
                if int: consider only the n best prediction
                if float: consider only this fraction of the best models
                Both wrt to validation predictions
                If performance_range_threshold > 0, might return less models
            max_models_on_disc: int
               Defines the maximum number of models that are kept in the disc.
               If int, it must be greater or equal than 1. If None, feature is disabled.
               It defines an upper bound on the models that can be used in the ensemble.
            performance_range_threshold: float
                Keep only models that are better than:
                    dummy + (best - dummy)*performance_range_threshold
                E.g dummy=2, best=4, thresh=0.5 --> only consider models with score > 3
                Will at most return the minimum between ensemble_nbest models,
                and max_models_on_disc. Might return less
            seed: int
                random seed
                if set to -1, read files with any seed (e.g., for shared model mode)
            shared_model: bool
                auto-sklearn used shared model mode (aka pSMAC)
            max_iterations: int
                maximal number of iterations to run this script
                (default None --> deactivated)
            precision: ["16","32","64","128"]
                precision of floats to read the predictions
            sleep_duration: int
                duration of sleeping time between two iterations of this script (in sec)
            memory_limit: int
                memory limit in mb
            read_at_most: int
                read at most n new prediction files in each iteration
        """

        super(EnsembleBuilder, self).__init__()

        self.backend = backend  # communication with filesystem
        self.dataset_name = dataset_name
        self.task_type = task_type
        self.metric = metric
        self.time_limit = limit  # time limit
        self.ensemble_size = ensemble_size
        self.performance_range_threshold = performance_range_threshold

        if isinstance(ensemble_nbest, numbers.Integral) and ensemble_nbest < 1:
            raise ValueError("Integer ensemble_nbest has to be larger 1: %s" %
                             ensemble_nbest)
        elif not isinstance(ensemble_nbest, numbers.Integral):
            if ensemble_nbest < 0 or ensemble_nbest > 1:
                raise ValueError(
                    "Float ensemble_nbest best has to be >= 0 and <= 1: %s" %
                    ensemble_nbest)

        self.ensemble_nbest = ensemble_nbest

        if max_models_on_disc is not None and max_models_on_disc < 1:
            raise ValueError(
                "max_models_on_disc has to be a positive number or None"
            )
        self.max_models_on_disc = max_models_on_disc
        self.seed = seed
        self.shared_mode = shared_mode  # pSMAC?
        self.max_iterations = max_iterations
        self.precision = precision
        self.sleep_duration = sleep_duration
        self.memory_limit = memory_limit
        self.read_at_most = read_at_most
        self.random_state = check_random_state(random_state)

        # part of the original training set
        # used to build the ensemble
        self.dir_ensemble = os.path.join(
            self.backend.temporary_directory,
            '.auto-sklearn',
            'predictions_ensemble',
        )
        # validation set (public test set) -- y_true not known
        self.dir_valid = os.path.join(
            self.backend.temporary_directory,
            '.auto-sklearn',
            'predictions_valid',
        )
        # test set (private test set) -- y_true not known
        self.dir_test = os.path.join(
            self.backend.temporary_directory,
            '.auto-sklearn',
            'predictions_test',
        )
        self.dir_models = os.path.join(
            self.backend.temporary_directory,
            '.auto-sklearn',
            'models',
        )
        logger_name = 'EnsembleBuilder(%d):%s' % (self.seed, self.dataset_name)
        self.logger = get_logger(logger_name)
        if ensemble_nbest == 1:
            self.logger.debug("Behaviour depends on int/float: %s, %s (ensemble_nbest, type)" %
                              (ensemble_nbest, type(ensemble_nbest)))

        self.start_time = 0
        self.model_fn_re = re.compile(r'_([0-9]*)_([0-9]*)_([0-9]{1,3}\.[0-9]*)\.npy')

        # already read prediction files
        # {"file name": {
        #    "ens_score": float
        #    "mtime_ens": str,
        #    "mtime_valid": str,
        #    "mtime_test": str,
        #    "seed": int,
        #    "num_run": int,
        #    "deleted": bool,
        #    Y_ENSEMBLE: np.ndarray
        #    Y_VALID: np.ndarray
        #    Y_TEST: np.ndarray
        #    }
        # }
        self.read_preds = {}
        self.last_hash = None  # hash of ensemble training data
        self.y_true_ensemble = None
        self.SAVE2DISC = True

        # hidden feature which can be activated via an environment variable. This keeps all
        # models and predictions which have ever been a candidate. This is necessary to post-hoc
        # compute the whole ensemble building trajectory.
        self._has_been_candidate = set()

        self.validation_performance_ = np.inf

    def run(self):
        buffer_time = 5  # TODO: Buffer time should also be used in main!?
        while True:
            time_left = self.time_limit - buffer_time
            safe_ensemble_script = pynisher.enforce_limits(
                wall_time_in_s=int(time_left),
                mem_in_mb=self.memory_limit,
                logger=self.logger
            )(self.main)
            safe_ensemble_script()
            if safe_ensemble_script.exit_status is pynisher.MemorylimitException:
                # if ensemble script died because of memory error,
                # reduce nbest to reduce memory consumption and try it again
                if isinstance(self.ensemble_nbest, numbers.Integral) and \
                        self.ensemble_nbest == 1:
                    self.logger.critical("Memory Exception --"
                                         " Unable to escape from memory exception")
                else:
                    if isinstance(self.ensemble_nbest, numbers.Integral):
                        self.ensemble_nbest = int(self.ensemble_nbest / 2)
                    else:
                        self.ensemble_nbest = self.ensemble_nbest / 2
                    self.logger.warning("Memory Exception -- restart with "
                                        "less ensemble_nbest: %d" % self.ensemble_nbest)
                    # ATTENTION: main will start from scratch;
                    # all data structures are empty again
                    continue
            break

    def main(self, return_pred=False):
        """

        :param return_pred:
            return tuple with last valid, test predictions
        :return:
        """
        self.start_time = time.time()
        iteration = 0
        valid_pred, test_pred = None, None
        while True:

            # maximal number of iterations
            if (self.max_iterations is not None
                    and 0 < self.max_iterations <= iteration):
                self.logger.info("Terminate ensemble building because of max iterations: %d of %d",
                                 self.max_iterations,
                                 iteration)
                break
            iteration += 1

            used_time = time.time() - self.start_time
            self.logger.debug(
                'Starting iteration %d, time left: %f',
                iteration,
                self.time_limit - used_time,
            )

            # populates self.read_preds
            if not self.read_ensemble_preds():
                time.sleep(self.sleep_duration)
                continue

            # Only the models with the n_best predictions are candidates
            # to be in the ensemble
            candidate_models = self.get_n_best_preds()
            if not candidate_models:  # no candidates yet
                continue

            # populates predictions in self.read_preds
            # reduces selected models if file reading failed
            n_sel_valid, n_sel_test = self. \
                get_valid_test_preds(selected_keys=candidate_models)

            # If valid/test predictions loaded, then reduce candidate models to this set
            if len(n_sel_test) != 0 and len(n_sel_valid) != 0 \
                    and len(set(n_sel_valid).intersection(set(n_sel_test))) == 0:
                # Both n_sel_* have entries, but there is no overlap, this is critical
                self.logger.error("n_sel_valid and n_sel_test are not empty, but do "
                                  "not overlap")
                time.sleep(self.sleep_duration)
                continue

            # If any of n_sel_* is not empty and overlaps with candidate_models,
            # then ensure candidate_models AND n_sel_test are sorted the same
            candidate_models_set = set(candidate_models)
            if candidate_models_set.intersection(n_sel_valid).intersection(n_sel_test):
                candidate_models = sorted(list(candidate_models_set.intersection(
                    n_sel_valid).intersection(n_sel_test)))
                n_sel_test = candidate_models
                n_sel_valid = candidate_models
            elif candidate_models_set.intersection(n_sel_valid):
                candidate_models = sorted(list(candidate_models_set.intersection(
                    n_sel_valid)))
                n_sel_valid = candidate_models
            elif candidate_models_set.intersection(n_sel_test):
                candidate_models = sorted(list(candidate_models_set.intersection(
                    n_sel_test)))
                n_sel_test = candidate_models
            else:
                # This has to be the case
                n_sel_test = []
                n_sel_valid = []

            if os.environ.get('ENSEMBLE_KEEP_ALL_CANDIDATES'):
                for candidate in candidate_models:
                    self._has_been_candidate.add(candidate)

            # train ensemble
            ensemble = self.fit_ensemble(selected_keys=candidate_models)
            if ensemble is not None:
                # We can't use candidate_models here, as n_sel_* might be empty
                valid_pred = self.predict(set_="valid",
                                          ensemble=ensemble,
                                          selected_keys=n_sel_valid,
                                          n_preds=len(candidate_models),
                                          index_run=iteration)
                # TODO if predictions fails, build the model again during the
                #  next iteration!
                test_pred = self.predict(set_="test",
                                         ensemble=ensemble,
                                         selected_keys=n_sel_test,
                                         n_preds=len(candidate_models),
                                         index_run=iteration)
            else:
                time.sleep(self.sleep_duration)
        if return_pred:
            return valid_pred, test_pred

    def read_ensemble_preds(self):
        """
            reading predictions on ensemble building data set;
            populates self.read_preds
        """

        self.logger.debug("Read ensemble data set predictions")

        if self.y_true_ensemble is None:
            try:
                self.y_true_ensemble = self.backend.load_targets_ensemble()
            except FileNotFoundError:
                self.logger.debug(
                    "Could not find true targets on ensemble data set: %s",
                    traceback.format_exc(),
                )
                return False

        # no validation predictions so far -- no dir
        if not os.path.isdir(self.dir_ensemble):
            self.logger.debug("No ensemble dataset prediction directory found")
            return False

        if self.shared_mode is False:
            pred_path = os.path.join(
                glob.escape(self.dir_ensemble),
                'predictions_ensemble_%s_*_*.npy*' % self.seed,
            )
        # pSMAC
        else:
            pred_path = os.path.join(
                glob.escape(self.dir_ensemble),
                'predictions_ensemble_*_*_*.npy*',
            )

        y_ens_files = glob.glob(pred_path)
        y_ens_files = [y_ens_file for y_ens_file in y_ens_files
                       if y_ens_file.endswith('.npy') or y_ens_file.endswith('.npy.gz')]
        self.y_ens_files = y_ens_files
        # no validation predictions so far -- no files
        if len(self.y_ens_files) == 0:
            self.logger.debug("Found no prediction files on ensemble data set:"
                              " %s" % pred_path)
            return False

        # First sort files chronologically
        to_read = []
        for y_ens_fn in self.y_ens_files:
            match = self.model_fn_re.search(y_ens_fn)
            _seed = int(match.group(1))
            _num_run = int(match.group(2))
            _budget = float(match.group(3))
            to_read.append([y_ens_fn, match, _seed, _num_run, _budget])

        n_read_files = 0
        # Now read file wrt to num_run
        for y_ens_fn, match, _seed, _num_run, _budget in \
                sorted(to_read, key=lambda x: x[3]):
            if self.read_at_most and n_read_files >= self.read_at_most:
                # limit the number of files that will be read
                # to limit memory consumption
                break

            if not y_ens_fn.endswith(".npy") and not y_ens_fn.endswith(".npy.gz"):
                self.logger.info('Error loading file (not .npy or .npy.gz): %s', y_ens_fn)
                continue

            if not self.read_preds.get(y_ens_fn):
                self.read_preds[y_ens_fn] = {
                    "ens_score": -1,
                    "mtime_ens": 0,
                    "mtime_valid": 0,
                    "mtime_test": 0,
                    "seed": _seed,
                    "num_run": _num_run,
                    "budget": _budget,
                    Y_ENSEMBLE: None,
                    Y_VALID: None,
                    Y_TEST: None,
                    # Lazy keys so far:
                    # 0 - not loaded
                    # 1 - loaded and in memory
                    # 2 - loaded but dropped again
                    "loaded": 0
                }

            if self.read_preds[y_ens_fn]["mtime_ens"] == os.path.getmtime(y_ens_fn):
                # same time stamp; nothing changed;
                continue

            # actually read the predictions and score them
            try:
                if y_ens_fn.endswith("gz"):
                    open_method = gzip.open
                elif y_ens_fn.endswith("npy"):
                    open_method = open
                else:
                    raise ValueError("Unknown filetype %s" % y_ens_fn)
                with open_method(y_ens_fn, 'rb') as fp:
                    y_ensemble = self._read_np_fn(fp=fp)
                    score = calculate_score(solution=self.y_true_ensemble,
                                            # y_ensemble = y_true for ensemble set
                                            prediction=y_ensemble,
                                            task_type=self.task_type,
                                            metric=self.metric,
                                            all_scoring_functions=False)

                    if self.read_preds[y_ens_fn]["ens_score"] > -1:
                        self.logger.critical(
                            'Changing ensemble score for file %s from %f to %f '
                            'because file modification time changed? %f - %f',
                            y_ens_fn,
                            self.read_preds[y_ens_fn]["ens_score"],
                            score,
                            self.read_preds[y_ens_fn]["mtime_ens"],
                            os.path.getmtime(y_ens_fn),
                        )

                    self.read_preds[y_ens_fn]["ens_score"] = score
                    self.read_preds[y_ens_fn][Y_ENSEMBLE] = y_ensemble
                    self.read_preds[y_ens_fn]["mtime_ens"] = os.path.getmtime(
                        y_ens_fn
                    )
                    self.read_preds[y_ens_fn]["loaded"] = 1

                    n_read_files += 1

            except Exception:
                self.logger.warning(
                    'Error loading %s: %s',
                    y_ens_fn,
                    traceback.format_exc(),
                )
                self.read_preds[y_ens_fn]["ens_score"] = -1

        self.logger.debug(
            'Done reading %d new prediction files. Loaded %d predictions in '
            'total.',
            n_read_files,
            np.sum([pred["loaded"] > 0 for pred in self.read_preds.values()])
        )
        return True

    def get_n_best_preds(self):
        """
            get best n predictions (i.e., keys of self.read_preds)
            according to score on "ensemble set"
            n: self.ensemble_nbest

            Side effect: delete predictions of non-candidate models
        """

        sorted_keys = self._get_list_of_sorted_preds()

        # number of models available
        num_keys = len(sorted_keys)
        # remove all that are at most as good as random
        # note: dummy model must have run_id=1 (there is no run_id=0)
        dummy_scores = list(filter(lambda x: x[2] == 1, sorted_keys))
        # number of dummy models
        num_dummy = len(dummy_scores)
        dummy_score = dummy_scores[0]
        self.logger.debug("Use %f as dummy score" % dummy_score[1])
        sorted_keys = filter(lambda x: x[1] > dummy_score[1], sorted_keys)
        # remove Dummy Classifier
        sorted_keys = list(filter(lambda x: x[2] > 1, sorted_keys))
        if not sorted_keys:
            # no model left; try to use dummy score (num_run==0)
            # log warning when there are other models but not better than dummy model
            if num_keys > num_dummy:
                self.logger.warning("No models better than random - using Dummy Score!"
                                    "Number of models besides current dummy model: %d. "
                                    "Number of dummy models: %d",
                                    num_keys - 1,
                                    num_dummy)
            sorted_keys = [
                (k, v["ens_score"], v["num_run"]) for k, v in self.read_preds.items()
                if v["seed"] == self.seed and v["num_run"] == 1
            ]
        # reload predictions if scores changed over time and a model is
        # considered to be in the top models again!
        if not isinstance(self.ensemble_nbest, numbers.Integral):
            # Transform to number of models to keep. Keep at least one
            keep_nbest = max(1, min(len(sorted_keys),
                                    int(len(sorted_keys) * self.ensemble_nbest)))
            self.logger.debug(
                "Library pruning: using only top %f percent of the models for ensemble "
                "(%d out of %d)",
                self.ensemble_nbest * 100, keep_nbest, len(sorted_keys)
            )
        else:
            # Keep only at most ensemble_nbest
            keep_nbest = min(self.ensemble_nbest, len(sorted_keys))
            self.logger.debug("Library Pruning: using for ensemble only "
                              " %d (out of %d) models" % (keep_nbest, len(sorted_keys)))

        # One can only read at most max_models_on_disc models
        if self.max_models_on_disc is not None and keep_nbest > self.max_models_on_disc:
            self.logger.debug(
                "Restricting the number of models to %d instead of %d due to argument "
                "max_models_on_disc",
                self.max_models_on_disc, keep_nbest,
            )
            keep_nbest = self.max_models_on_disc

        for k, _, _ in sorted_keys[:keep_nbest]:
            if self.read_preds[k][Y_ENSEMBLE] is None:
                if k.endswith("gz"):
                    open_method = gzip.open
                elif k.endswith("npy"):
                    open_method = open
                else:
                    raise ValueError("Unknown filetype %s" % k)
                with open_method(k, 'rb') as fp:
                    self.read_preds[k][Y_ENSEMBLE] = self._read_np_fn(fp=fp)
                # No need to load valid and test here because they are loaded
                #  only if the model ends up in the ensemble
            self.read_preds[k]['loaded'] = 1

        # consider performance_range_threshold
        if self.performance_range_threshold > 0:
            best_score = sorted_keys[0][1]
            min_score = dummy_score[1]
            min_score += (best_score - min_score) * self.performance_range_threshold
            if sorted_keys[keep_nbest - 1][1] < min_score:
                # We can further reduce number of models
                # since worst model is worse than thresh
                for i in range(0, keep_nbest):
                    # Look at most at keep_nbest models,
                    # but always keep at least one model
                    current_score = sorted_keys[i][1]
                    if current_score <= min_score:
                        self.logger.debug("Dynamic Performance range: "
                                          "Further reduce from %d to %d models",
                                          keep_nbest, max(1, i))
                        keep_nbest = max(1, i)
                        break
        ensemble_n_best = keep_nbest

        # reduce to keys
        sorted_keys = list(map(lambda x: x[0], sorted_keys))

        # remove loaded predictions for non-winning models
        for k in sorted_keys[ensemble_n_best:]:
            self.read_preds[k][Y_ENSEMBLE] = None
            self.read_preds[k][Y_VALID] = None
            self.read_preds[k][Y_TEST] = None
            if self.read_preds[k]['loaded'] == 1:
                self.logger.debug(
                    'Dropping model %s (%d,%d) with score %f.',
                    k,
                    self.read_preds[k]['seed'],
                    self.read_preds[k]['num_run'],
                    self.read_preds[k]['ens_score'],
                )
                self.read_preds[k]['loaded'] = 2

        # return best scored keys of self.read_preds
        return sorted_keys[:ensemble_n_best]

    def get_valid_test_preds(self, selected_keys: list):
        """
        get valid and test predictions from disc
        and store them in self.read_preds

        Parameters
        ---------
        selected_keys: list
            list of selected keys of self.read_preds

        Return
        ------
        success_keys:
            all keys in selected keys for which we could read the valid and
            test predictions
        """
        success_keys_valid = []
        success_keys_test = []

        for k in selected_keys:
            valid_fn = glob.glob(
                os.path.join(
                    glob.escape(self.dir_valid),
                    'predictions_valid_%d_%d_%s.npy*' % (
                        self.read_preds[k]["seed"],
                        self.read_preds[k]["num_run"],
                        self.read_preds[k]["budget"],
                    )
                )
            )
            valid_fn = [vfn for vfn in valid_fn if vfn.endswith('.npy') or vfn.endswith('.npy.gz')]
            test_fn = glob.glob(
                os.path.join(
                    glob.escape(self.dir_test),
                    'predictions_test_%d_%d_%s.npy*' % (
                        self.read_preds[k]["seed"],
                        self.read_preds[k]["num_run"],
                        self.read_preds[k]["budget"]
                    )
                )
            )
            test_fn = [tfn for tfn in test_fn if tfn.endswith('.npy') or tfn.endswith('.npy.gz')]

            # TODO don't read valid and test if not changed
            if len(valid_fn) == 0:
                # self.logger.debug("Not found validation prediction file "
                #                   "(although ensemble predictions available): "
                #                   "%s" % valid_fn)
                pass
            else:
                valid_fn = valid_fn[0]
                if self.read_preds[k]["mtime_valid"] == os.path.getmtime(valid_fn) \
                        and self.read_preds[k][Y_VALID] is not None:
                    success_keys_valid.append(k)
                    continue
                try:
                    if valid_fn.endswith("gz"):
                        open_method = gzip.open
                    elif valid_fn.endswith("npy"):
                        open_method = open
                    else:
                        raise ValueError("Unknown filetype %s" % valid_fn)
                    with open_method(valid_fn, 'rb') as fp:
                        y_valid = self._read_np_fn(fp)
                        self.read_preds[k][Y_VALID] = y_valid
                        success_keys_valid.append(k)
                        self.read_preds[k]["mtime_valid"] = os.path.getmtime(valid_fn)
                except Exception:
                    self.logger.warning('Error loading %s: %s',
                                        valid_fn, traceback.format_exc())

            if len(test_fn) == 0:
                # self.logger.debug("Not found test prediction file (although "
                #                   "ensemble predictions available):%s" %
                #                   test_fn)
                pass
            else:
                test_fn = test_fn[0]
                if self.read_preds[k]["mtime_test"] == \
                        os.path.getmtime(test_fn) \
                        and self.read_preds[k][Y_TEST] is not None:
                    success_keys_test.append(k)
                    continue
                try:
                    if test_fn.endswith("gz"):
                        open_method = gzip.open
                    elif test_fn.endswith("npy"):
                        open_method = open
                    else:
                        raise ValueError("Unknown filetype %s" % test_fn)
                    with open_method(test_fn, 'rb') as fp:
                        y_test = self._read_np_fn(fp)
                        self.read_preds[k][Y_TEST] = y_test
                        success_keys_test.append(k)
                        self.read_preds[k]["mtime_test"] = os.path.getmtime(test_fn)
                except Exception:
                    self.logger.warning('Error loading %s: %s',
                                        test_fn, traceback.format_exc())

        return success_keys_valid, success_keys_test

    def fit_ensemble(self, selected_keys: list):
        """
            fit ensemble

            Parameters
            ---------
            selected_keys: list
                list of selected keys of self.read_preds

            Returns
            -------
            ensemble: EnsembleSelection
                trained Ensemble
        """
        predictions_train = np.array([self.read_preds[k][Y_ENSEMBLE] for k in selected_keys])
        include_num_runs = [
            (
                self.read_preds[k]["seed"],
                self.read_preds[k]["num_run"],
                self.read_preds[k]["budget"],
            )
            for k in selected_keys]

        # check hash if ensemble training data changed
        current_hash = hash(predictions_train.data.tobytes())
        if self.last_hash == current_hash:
            self.logger.debug(
                "No new model predictions selected -- skip ensemble building "
                "-- current performance: %f",
                self.validation_performance_,
            )

            # Delete files of non-candidate models
            if self.max_models_on_disc is not None:
                self._delete_excess_models()

            return None
        self.last_hash = current_hash

        ensemble = EnsembleSelection(
            ensemble_size=self.ensemble_size,
            task_type=self.task_type,
            metric=self.metric,
            random_state=self.random_state,
        )

        try:
            self.logger.debug(
                "Fitting the ensemble on %d models.",
                len(predictions_train),
            )
            start_time = time.time()
            ensemble.fit(predictions_train, self.y_true_ensemble,
                         include_num_runs)
            end_time = time.time()
            self.logger.debug(
                "Fitting the ensemble took %.2f seconds.",
                end_time - start_time,
            )
            self.logger.info(ensemble)
            self.validation_performance_ = min(
                self.validation_performance_,
                ensemble.get_validation_performance(),
            )

        except ValueError:
            self.logger.error('Caught ValueError: %s', traceback.format_exc())
            time.sleep(self.sleep_duration)
            return None
        except IndexError:
            self.logger.error('Caught IndexError: %s' + traceback.format_exc())
            time.sleep(self.sleep_duration)
            return None

        # Delete files of non-candidate models
        if self.max_models_on_disc is not None:
            self._delete_excess_models()

        return ensemble

    def predict(self, set_: str,
                ensemble: AbstractEnsemble,
                selected_keys: list,
                n_preds: int,
                index_run: int):
        """
            save preditions on ensemble, validation and test data on disc

            Parameters
            ----------
            set_: ["valid","test"]
                data split name
            ensemble: EnsembleSelection
                trained Ensemble
            selected_keys: list
                list of selected keys of self.read_preds
            n_preds: int
                number of prediction models used for ensemble building
                same number of predictions on valid and test are necessary
            index_run: int
                n-th time that ensemble predictions are written to disc

            Return
            ------
            y: np.ndarray
        """
        self.logger.debug("Predicting the %s set with the ensemble!", set_)

        # Save the ensemble for later use in the main auto-sklearn module!
        if self.SAVE2DISC:
            self.backend.save_ensemble(ensemble, index_run, self.seed)

        predictions = np.array([
            self.read_preds[k][Y_VALID if set_ == 'valid' else Y_TEST]
            for k in selected_keys
        ])

        if n_preds == predictions.shape[0]:
            y = ensemble.predict(predictions)
            if self.task_type == BINARY_CLASSIFICATION:
                y = y[:, 1]
            if self.SAVE2DISC:
                self.backend.save_predictions_as_txt(
                    predictions=y,
                    subset=set_,
                    idx=index_run,
                    prefix=self.dataset_name,
                    precision=8,
                )
            return y
        else:
            self.logger.info(
                "Found inconsistent number of predictions and models (%d vs "
                "%d) for subset %s",
                predictions.shape[0],
                n_preds,
                set_,
            )
            return None
        # TODO: ADD saving of predictions on "ensemble data"

    def _get_list_of_sorted_preds(self):
        """
            Returns a list of sorted predictions in descending order
            Predictions are taken from self.read_preds.

            Parameters
            ----------
            None

            Return
            ------
            sorted_keys: list
        """
        # Sort by score - higher is better!
        # First sort by num_run
        sorted_keys = list(reversed(sorted(
            [
                (k, v["ens_score"], v["num_run"])
                for k, v in self.read_preds.items()
            ],
            key=lambda x: x[2],
        )))
        # Then by score
        sorted_keys = list(reversed(sorted(sorted_keys, key=lambda x: x[1])))
        return sorted_keys

    def _delete_excess_models(self):
        """
            Deletes models excess models on disc. self.max_models_on_disc
            defines the upper limit on how many models to keep.
            Any additional model with a worst score than the top
            self.max_models_on_disc is deleted.

        """

        # Obtain a list of sorted pred keys
        sorted_keys = self._get_list_of_sorted_preds()
        sorted_keys = list(map(lambda x: x[0], sorted_keys))

        if len(sorted_keys) <= self.max_models_on_disc:
            # Don't waste time if not enough models to delete
            return

        # The top self.max_models_on_disc models would be the candidates
        # Any other low performance model will be deleted
        # The list is in ascending order of score
        candidates = sorted_keys[:self.max_models_on_disc]

        # Loop through the files currently in the directory
        for pred_path in self.y_ens_files:

            # Do not delete candidates
            if pred_path in candidates:
                continue

            if pred_path in self._has_been_candidate:
                continue

            match = self.model_fn_re.search(pred_path)
            _full_name = match.group(0)
            _seed = match.group(1)
            _num_run = match.group(2)
            _budget = match.group(3)

            # Do not delete the dummy prediction
            if int(_num_run) == 1:
                continue

            # Besides the prediction, we have to take care of three other files: model,
            # validation and test.
            model_name = '%s.%s.%s.model' % (_seed, _num_run, _budget)
            model_path = os.path.join(self.dir_models, model_name)
            pred_valid_name = 'predictions_valid' + _full_name
            pred_valid_path = os.path.join(self.dir_valid, pred_valid_name)
            pred_test_name = 'predictions_test' + _full_name
            pred_test_path = os.path.join(self.dir_test, pred_test_name)

            paths = [model_path, pred_path]
            if os.path.exists(pred_valid_path):
                paths.append(pred_valid_path)
            if os.path.exists(pred_test_path):
                paths.append(pred_test_path)

            # Lets lock all the files "at once" to avoid weird race conditions. Also,
            # we either delete all files of a model (model, prediction, validation
            # and test), or delete none. This makes it easier to keep track of which
            # models have indeed been correctly removed.
            locks = [lockfile.LockFile(path) for path in paths]
            try:
                for lock in locks:
                    lock.acquire()
            except Exception as e:
                if isinstance(e, lockfile.AlreadyLocked):
                    # If the file is already locked, we deal with it later. Not a big deal
                    self.logger.info(
                        'Model %s is already locked. Skipping it for now.', model_name)
                else:
                    # Other exceptions, however, should not occur.
                    # The message bellow is asserted in test_delete_excess_models()
                    self.logger.error(
                        'Failed to lock model %s files due to error %s', model_name, e)
                for lock in locks:
                    if lock.i_am_locking():
                        lock.release()
                continue

            # Delete files if model is not a candidate AND prediction is old. We check if
            # the prediction is old to avoid deleting a model that hasn't been appreciated
            # by self.get_n_best_preds() yet.
            original_timestamp = self.read_preds[pred_path]['mtime_ens']
            current_timestamp = os.path.getmtime(pred_path)
            if current_timestamp == original_timestamp:
                # The messages logged here are asserted in
                # test_delete_excess_models(). Edit with care.
                try:
                    for path in paths:
                        os.remove(path)
                    self.logger.info(
                        "Deleted files of non-candidate model %s", model_name)
                except Exception as e:
                    self.logger.error(
                        "Failed to delete files of non-candidate model %s due"
                        " to error %s", model_name, e)

            # If we reached this point, all locks were done by this thread. So no need
            # to check "lock.i_am_locking()" here.
            for lock in locks:
                lock.release()

    def _read_np_fn(self, fp):
        if self.precision == "16":
            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float16)
        elif self.precision == "32":
            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float32)
        elif self.precision == "64":
            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float64)
        else:
            predictions = np.load(fp, allow_pickle=True)
        return predictions

max_stars_repo_name: FelixNeutatz/auto-sklearn

max_issues_repo_name: FelixNeutatz/auto-sklearn

max_forks_repo_name: FelixNeutatz/auto-sklearn


text:
from .utils import PyKEArgumentHelpFormatter
import re
import numpy as np
import sys
from matplotlib import pyplot as plt
from astropy.io import fits as pyfits
from . import kepio, kepmsg, kepkey

__all__ = ['kepdraw']

def kepdraw(infile, outfile=None, datacol=None, ploterr=False,
            errcol='SAP_FLUX_ERR', quality=False, lcolor='#363636', lwidth=1.0,
            fcolor='#a8a7a7', falpha=0.2, labelsize=20, ticksize=14, xsize=18.,
            ysize=6., fullrange=False, chooserange=False, y1=0, y2=1e4,
            plotgrid=False, ylabel='e$^-$ s$^{-1}$', plottype='fast',
            noninteractive=False, verbose=False, logfile='kepdraw.log'):
    """
    kepdraw -- Interactive plotting of time series data

    Parameters
    ----------
    infile : str
        The name of a MAST standard format FITS file containing a Kepler light
        curve within the first data extension.
    outfile : string
        The name of the output PNG plot file.
    datacol : string
        The FITS column within extension 1 of infile to be plotted against
        time.
    ploterr : bool
        Plot error bars with the data?
    errcol : str
        The uncertainty data coupled to datacol. If ploterr is ``True`` then
        these data will be plotted as error bars
    quality : bool
        Ignore bad data and cadence gaps in datacol?
    lcolor : str
        The color of the line plot. These can be chosen in html notation, e.g.
        black is #000000 and white is #ffffff.
    lwidth : float
        The width of the plot line in arbitrary units. The plotting default
        width is 1.0.
    fcolor : str
        The color of the fill under the plot line. These can be chosen in html
        notation, e.g. black is #000000 and white is #ffffff. The html color
        scheme can be browsed in multiple online charts.
    falpha : float
        The transparency of the fill color. 0.0 is transparent, 1.0 is opaque.
    labelsize : float
        The font size of the plot labels.
    ticksize : float
        The axes tick sizes within the plot.
    xsize : float
        The length of the plotting window in arbitrary units.
        A typical value is 16.0
    ysize : float
        The height of the plotting window in arbitrary units.
        A typical value is 8.0
    fullrange : bool
        Plot the flux range between zero and the maximum and 101% of the flux
        maximum? Otherwise plot the range 99% of the flux minimum to 101% of
        the flux maximum.
    plotgrid : bool
        Plot a grid over the data on the major tick marks?
    plottype : str
        The look and feel of the plot. 'fast' provides a rapid quick look plot
        using discrete points to represent the data. 'pretty' provides a
        continuous curve between data points, except at natural data gaps or
        where quality issues occur. Pretty curves are significantly slower to
        build. Options:

            * fast
            * pretty
    non-interactive : bool
        If True, prevents the matplotlib window to pop up.
    verbose : bool
        Print informative messages and warnings to the shell and logfile?
    logfile : str
        Name of the logfile containing error and warning messages.

    Examples
    --------

    .. code-block:: bash

        $ kepdraw kplr010666592-2009131110544_slc.fits

    .. image:: ../_static/images/api/kepdraw.png
        :align: center
    """

    if outfile is None:
        outfile = infile.split('.')[0] + "-{}.png".format(__all__[0])

    hashline = '--------------------------------------------------------------'
    kepmsg.log(logfile, hashline, verbose)
    call = ('KEPDRAW -- '
            + ' infile={}'.format(infile)
            + ' outfile={}'.format(outfile)
            + ' datacol={}'.format(datacol)
            + ' ploterr={}'.format(ploterr)
            + ' errcol={}'.format(errcol)
            + ' quality={}'.format(quality)
            + ' lcolor={}'.format(lcolor)
            + ' lwidth={}'.format(lwidth)
            + ' fcolor={}'.format(fcolor)
            + ' falpha={}'.format(falpha)
            + ' labelsize={}'.format(labelsize)
            + ' ticksize={}'.format(ticksize)
            + ' xsize={}'.format(xsize)
            + ' ysize={}'.format(ysize)
            + ' fullrange={}'.format(fullrange)
            + ' chooserange={}'.format(chooserange)
            + ' ymin={}'.format(y1)
            + ' ymax={}'.format(y2)
            + ' plotgrid={}'.format(plotgrid)
            + ' ylabel={}'.format(ylabel)
            + ' plottype={}'.format(plottype)
            + ' verbose={}'.format(verbose)
            + ' logfile={}'.format(logfile))

    kepmsg.log(logfile, call+'\n', verbose)

    # start time
    kepmsg.clock('KEPDRAW started at', logfile, verbose)

    # open input file
    struct = pyfits.open(infile, 'readonly')
    tstart, tstop, bjdref, cadence = kepio.timekeys(struct, infile,
                                                    logfile, verbose)

    # read table structure
    table = kepio.readfitstab(infile, struct[1], logfile, verbose)

    # read table columns
    intime = kepio.readtimecol(infile, table, logfile, verbose) + bjdref

    if datacol is None:
        default_priority_columns = ['DETSAP_FLUX', 'PDCSAP_FLUX', 'SAP_FLUX']
        flux_columns_found = [c for c in struct[1].data.columns.names
                              if 'FLUX' in c]

        if len(flux_columns_found) == 0:
            raise ValueError("No flux columns found.")
        print("Found the following flux columns: {}".format(", ".join(flux_columns_found)))

        for colname in default_priority_columns:
            if colname in flux_columns_found:
                datacol = colname
                print("Using data column {} on the plot...".format(datacol))
                break
        else:
            datacol = flux_columns_found[0]
            print("Using data column {} on the plot...".format(datacol))

    indata = kepio.readfitscol(infile, table, datacol, logfile, verbose)
    indataerr = kepio.readfitscol(infile, table, errcol, logfile, verbose)
    qualty = kepio.readfitscol(infile, table, 'SAP_QUALITY', logfile, verbose)

    # close infile
    struct.close()

    # remove infinities and bad data
    if np.isnan(np.nansum(indataerr)):
        indataerr[:] = 1.0e-5
    work1 = np.array([intime, indata, indataerr, qualty],dtype='float64')
    work1 = np.rot90(work1, 3)
    work1 = work1[~np.isnan(work1).any(1)]
    work1 = work1[~np.isinf(work1).any(1)]
    if quality:
        work1 = work1[work1[:, 0] == 0.0]
    barytime = np.array(work1[:, 3], dtype='float64')
    data = np.array(work1[:, 2], dtype='float32')
    dataerr = np.array(work1[:, 1], dtype='float32')
    if len(barytime) == 0:
        message = 'ERROR -- KEPDRAW: Plotting arrays are full of NaN'
        kepmsg.err(logfile, message, verbose)

    # clean up x-axis unit
    barytime0 = float(int(tstart / 100) * 100.0)
    barytime -= barytime0
    xlab = 'BJD $-$ %d' % barytime0

    # clean up y-axis units
    nrm = 0
    try:
        nrm = len(str(int(np.nanmax(data))))-1
    except:
        nrm = 0
    data = data / 10**nrm
    if 'e$^-$ s$^{-1}$' in ylabel or 'default' in ylabel:
        if nrm == 0:
            ylab1 = 'e$^-$ s$^{-1}$'
        else:
            ylab1 = '10$^{%d}$ e$^-$ s$^{-1}$' % nrm
    else:
        ylab1 = re.sub('_','-',ylabel)

    # data limits
    xmin = np.nanmin(barytime)
    xmax = np.nanmax(barytime)
    ymin = np.nanmin(data)
    ymax = np.nanmax(data)
    xr = xmax - xmin
    yr = ymax - ymin
    barytime = np.insert(barytime,[0],[barytime[0]])
    barytime = np.append(barytime,[barytime[-1]])
    data = np.insert(data,[0],[-10000.0])
    data = np.append(data,-10000.0)
    # define size of plot on monitor screen
    plt.figure(figsize=[xsize,ysize])
    # delete any fossil plots in the matplotlib window
    plt.clf()
    # position axes inside the plotting window
    ax = plt.subplot(111)
    plt.subplots_adjust(0.06,0.15,0.92,0.83)
    # force tick labels to be absolute rather than relative
    plt.gca().xaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False))
    plt.gca().yaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False))
    ax.yaxis.set_major_locator(plt.MaxNLocator(5))
    # rotate y labels by 90 deg
    labels = ax.get_yticklabels()
    # ifplot type is 'fast' plot data time series as points
    if plottype == 'fast':
        plt.plot(barytime, data, 'o', color=lcolor, markersize=lwidth)
    # ifplot type is 'pretty' plot data time series as an unbroken line,
    # retaining data gaps
    else:
        ltime = np.array([], dtype='float64')
        ldata = np.array([], dtype='float32')
        dt = 0
        work1 = 2.0 * cadence / 86400
        for i in range(1, len(data) - 1):
            dt = barytime[i] - barytime[i - 1]
            if dt < work1:
                ltime = np.append(ltime, barytime[i])
                ldata = np.append(ldata, data[i])
            else:
                plt.plot(ltime, ldata, color=lcolor, linestyle='-', linewidth=lwidth)
                ltime = np.array([], dtype='float64')
                ldata = np.array([], dtype='float32')
        plt.plot(ltime, ldata, color=lcolor, linestyle='-', linewidth=lwidth)

    # plot the fill color below data time series, with no data gaps
    plt.fill(barytime, data, fc=fcolor, linewidth=0.0, alpha=falpha)

    # define plot x and y limits
    plt.xlim(xmin-xr*0.01,xmax+xr*0.01)
    if ymin-yr*0.01 <= 0.0 or fullrange:
        plt.ylim(1.0e-10,ymax+yr*0.01)
    else:
        plt.ylim(ymin-yr*0.01,ymax+yr*0.01)
    if chooserange:
        plt.ylim(y1,y2)

    # plot labels
    plt.xlabel(xlab, {'color' : 'k'})
    try:
        plt.ylabel(ylab1, {'color' : 'k'})
    except:
        ylab1 = '10**%d e-/s' % nrm
        plt.ylabel(ylab1, {'color' : 'k'})

    if plotgrid:
        plt.grid()

    ax.minorticks_on()
    # save plot to file
    if outfile is not None:
        print("Writing output file {}...".format(outfile))
        plt.savefig(outfile)

    if not noninteractive:
        plt.show()
    # end time
    kepmsg.clock('KEPDRAW completed at' , logfile, verbose)

def kepdraw_main():
    import argparse

    parser = argparse.ArgumentParser(
             description='Interactive plotting of Kepler time series data',
             formatter_class=PyKEArgumentHelpFormatter)
    parser.add_argument('infile', help='Name of input file', type=str)
    parser.add_argument('--outfile', default=None,
                        help='name of output PNG file')
    parser.add_argument('--datacol', default=None,
                        help='Name of data column to plot')
    parser.add_argument('--ploterr', action='store_true',
                        help='Plot data error bars?')
    parser.add_argument('--errcol', default='SAP_FLUX_ERR',
                        help='Name of data error column', type=str)
    parser.add_argument('--quality', action='store_true',
                        help=('Ignore cadences where the data quality is'
                              ' questionable?'))
    parser.add_argument('--lcolor', default='#363636',
                        help='HTML color of data line within plot', type=str)
    parser.add_argument('--lwidth', default=1.0,
                        help='type of image intensity scale', type=float)
    parser.add_argument('--fcolor', default='#a8a7a7',
                        help='HTML color to fill the area below the data', type=str)
    parser.add_argument('--falpha', default=0.2,
                        help='type of image intensity scale', type=float)
    parser.add_argument('--labelsize', default=20.,
                        help='Fontsize of axis labels', type=float)
    parser.add_argument('--ticksize', default=14.,
                        help='Fontsize of numeric tick labels', type=float)
    parser.add_argument('--xsize', default=18.,
                        help='X-dimension size of plot', type=float)
    parser.add_argument('--ysize', default=6.,
                        help='Y-dimension size of plot', type=float)
    parser.add_argument('--fullrange', action='store_true',
                        help='Plot flux range from 0.0 e-/sec?')
    parser.add_argument('--chooserange', action='store_true',
                        help='Choose Y-axis range?')
    parser.add_argument('--ymin', default=0.,
                        help='Low limit of the Y-axis range to plot [e-/s]',
                        type=float)
    parser.add_argument('--ymax', default=1e4,
                        help='High limit of the Y-axis range to plot [e-/s]',
                        type=float)
    parser.add_argument('--plotgrid', action='store_true',
                        help='Plot axis grid?')
    parser.add_argument('--ylabel', default='e$^-$ s$^{-1}$',
                        help='Plot axis label', type=str)
    parser.add_argument('--plottype', default='fast', help='plot type',
                        type=str, choices=['fast','pretty'])
    parser.add_argument('--non-interactive', action='store_true',
                        help='Pop up matplotlib plot window?',
                        dest='noninteractive')
    parser.add_argument('--verbose', action='store_true',
                        help='Write to a log file?')
    parser.add_argument('--logfile', '-l', help='Name of ascii log file',
                        default='kepdraw.log', type=str)
    args = parser.parse_args()
    kepdraw(args.infile, args.outfile, args.datacol, args.ploterr, args.errcol,
            args.quality, args.lcolor, args.lwidth, args.fcolor, args.falpha,
            args.labelsize, args.ticksize, args.xsize, args.ysize,
            args.fullrange, args.chooserange, args.ymin, args.ymax,
            args.plotgrid, args.ylabel, args.plottype, args.noninteractive,
            args.verbose, args.logfile)

ext: py

max_stars_repo_name: ecalifornica/pyke

max_issues_repo_name: ecalifornica/pyke

max_forks_repo_name: mirca/PyKE

reason: import numpy,from astropy


text:
import numpy as np

#Kinematics parameters

l_1 = 160
l_2 = 160

hip_T_world = np.array([
    [-1,0,0],
    [0,1,-(l_1 + l_2)],
    [0,0,1]
])

world_T_hip = np.array([
    [-1,0,0],
    [0,1,l_1 + l_2],
    [0,0,1]
])

zero_config_rotation = np.array([
    [0, -np.pi],
    [0, -np.pi],
    [np.pi, -np.pi],
    [np.pi, -np.pi]
])

joint_polarities = np.array([
    [-1,-1],
    [1,1],
    [-1,-1],
    [1,1]
])

elbows = [False, False, False, False]

def simple_walk_controller(coordinates, current_position):
    if len(coordinates) != 4:
        print("Wrong Input, coordinates should be an array of 4, 2 array of x,y coordinates")
        return

    q = np.zeros(8)
    for i in range(len(coordinates)):

        #x = coordinates[i][0]
        #y = coordinates[i][1]
        #print(f"hip coordinates: {x,y}[mm]")

        p_foot = np.append(coordinates[i], 1)
        p_hip = hip_T_world.dot(p_foot)

        x = p_hip[0]
        y = p_hip[1]

        #x = p_world[0]
        #y = p_world[1]

        if not is_within_reach(x, y):
            return None, None

        q_1, q_2 = inverse_kinematics_one_leg(x, y, elbows[i])

        print(f"Inverse kinematics of : {x, y} gives : {np.degrees(q_1), np.degrees(q_2)}")

        q_1 = q_1 + zero_config_rotation[i][0]
        q_2 = q_2 + zero_config_rotation[i][1]

        q_1 = q_1 * joint_polarities[i][0]
        q_2 = q_2 * joint_polarities[i][1]

        #print("\nBefore shortest path:")
        #print(f"Initial : q_1 = {np.degrees(current_position[2*i])}, q_2 = {np.degrees(current_position[2*i + 1])}")
        #print(f"Final : q_1 = {np.degrees(q_1)}, q_2 = {np.degrees(q_2)}\n")

        q_1 = shortest_path(current_position[2*i], q_1)
        #q_2 = shortest_path(current_position[2*i + 1], q_2)

        #q_2 = current_position[2*i+1] + shortest_path(current_position[2*i+1], q_2)

        #q_2 = neg(q_2)

        #print("\nAfter shortest path:")
        #print(f"Final : q_1 = {np.degrees(q_1)}, q_2 = {np.degrees(q_2)}\n")

        q[2*i] = q_1
        q[2*i+1] = q_2

    return q

def inverse_kinematics_one_leg(x,y, elbow_left = True):

    r = np.sqrt(abs(x**2)+abs(y**2))

    gamma = np.arctan2(y,x)
    alpha = np.arccos((l_1**2 + l_2**2 - r**2) / (2*l_1*l_2))
    if r == 0:
        beta = 0
    else:
        beta = np.arccos((r**2+l_1**2-l_2**2) / (2*r*l_1))

    if elbow_left:
        q_2 = np.pi - alpha
        q_1 = gamma - beta
    else:
        q_2 = np.pi + alpha
        print(f"q_2 = pi + {np.degrees(alpha)}")
        q_1 = gamma + beta

    return q_1, q_2

'''
def forward_kinematics(q, end_effector_index=1):
    if len(q) != 8:
        print("Wrong input. q should be an array with 8 joint angles")
        return

    coordinates = []

    print(q)

    for i in range(len(q) // 2):
        coordinates.append(forward_kinematics_one_leg(q[i], q[i+1], end_effector_index))

    return coordinates

def forward_kinematics_one_leg(q_1, q_2, end_effector_index=1):

    q_1 = q_1 + config.zero_config_rotation[0]
    q_2 = q_2 + config.zero_config_rotation[1]

    x = np.zeros(config.num_joints // 4)
    y = np.zeros(config.num_joints // 4)

    x[0] = Config.l_1 * np.cos(q_1)
    y[0] = Config.l_1 * np.sin(q_1)

    x[1] = x[0] + Config.l_2 * np.cos(q_1 + q_2)
    y[1] = y[0] + Config.l_2 * np.sin(q_1 + q_2)

    p_hip = np.array([ x[end_effector_index], y[end_effector_index], 0, 1 ])

    p_world = Config.world_T_hip.dot(p_hip)

    return p_world[0], p_world[1]
'''

def is_within_reach(x, y):
    if x**2 + y**2 > (l_1 + l_2)**2:
        print(f"({x},{y}) is not within reach")
        return False
    return True

def pos(angle):
    if angle < 0:
        return 2*np.pi + angle
    else:
        return angle

def neg(angle):
    if angle >= 0:
        return - 2 * np.pi + angle
    else:
        return angle

def shortest_path(initial, final):
    distance_to_positive = pos(final) - initial
    distance_to_negative = abs(neg(final)) + initial

    if distance_to_positive < distance_to_negative:
        return pos(final)
    else:
        return neg(final)

max_stars_repo_path: Experiments/Real Time Enabled/Updating the software stack/Changed from Threading to Multiprocessing/kinematics.py

max_stars_repo_name: kevinAlfsen/RASH-Bachelor-Thesis

max_issues_repo_name: kevinAlfsen/RASH-Bachelor-Thesis

max_forks_repo_name: kevinAlfsen/RASH-Bachelor-Thesis

reason: import numpy


text:
# --------------
# import the libraries
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings('ignore')

# Code starts here

df=pd.read_csv(path)
print(df.head(5))
X=df.drop(["insuranceclaim"],1)
y=df.insuranceclaim
X_train, X_test, y_train, y_test= train_test_split(X,y, random_state=6, test_size=0.2)

# Code ends here

# --------------
import matplotlib.pyplot as plt

# Code starts here

X_train['bmi'].plot(kind="box", )
q_value=X_train['bmi'].quantile(0.95)
y_train.value_counts(normalize=True)
# Code ends here

# --------------
# Code starts here

relation= X_train.corr()
print(relation)
sns.pairplot(X_train)

# Code ends here

# --------------
import seaborn as sns
import matplotlib.pyplot as plt

# Code starts here

cols= ['children','sex','region','smoker']
fig, axes= plt.subplots(2,2 )
for i in range(0,2):
    for j in range(0,2):
        col= cols[i*2 + j]
        sns.countplot(x=X_train[col], hue=y_train, ax=axes[i,j])
# Code ends here

# --------------
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

# parameters for grid search
parameters = {'C':[0.1,0.5,1,5]}

# Code starts here

lr=LogisticRegression()
grid= GridSearchCV(estimator=lr, param_grid=parameters)
grid.fit(X_train, y_train)
y_pred= grid.predict(X_test)
accuracy= accuracy_score(y_test, y_pred)
print(accuracy)
# Code ends here

# --------------
from sklearn.metrics import roc_auc_score
from sklearn import metrics

# Code starts here

score=roc_auc_score(y_test, y_pred)
print(score)
y_pred_proba= grid.predict_proba(X_test)[:,1]
print(y_pred_proba)

fpr, tpr, thresholds= metrics.roc_curve(y_test, y_pred_proba)

roc_auc= roc_auc_score(y_test, y_pred_proba)

plt.plot(fpr,tpr,label="Logistic model, auc="+str(roc_auc))

# Code ends here

max_stars_repo_name: sparsh-tekriwal/ga-learner-dsmp-repo

max_issues_repo_name: sparsh-tekriwal/ga-learner-dsmp-repo

max_forks_repo_name: sparsh-tekriwal/ga-learner-dsmp-repo


text:
from warnings import warn

from . import overlap

def fractional_slice(task, axes):
    """

    >>> fractional_slice(('x', 5.1), {0: 2})  # doctest: +SKIP
    (getitem, ('x', 6), (slice(0, 2),))

    >>> fractional_slice(('x', 3, 5.1), {0: 2, 1: 3})  # doctest: +SKIP
    (getitem, ('x', 3, 5), (slice(None, None, None), slice(-3, None)))

    >>> fractional_slice(('x', 2.9, 5.1), {0: 2, 1: 3})  # doctest: +SKIP
    (getitem, ('x', 3, 5), (slice(0, 2), slice(-3, None)))
    """

    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.fractional_slice.',
         Warning)

    return overlap.fractional_slice(task, axes)

def expand_key(k, dims, name=None, axes=None):
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.expand_keys.',
         Warning)

    return overlap.expand_key(k, dims, name, axes)

def ghost_internal(x, axes):
    """ Share boundaries between neighboring blocks

    Parameters
    ----------

    x: da.Array
        A dask array
    axes: dict
        The size of the shared boundary per axis

    The axes input informs how many cells to dask.array.overlap between neighboring blocks
    {0: 2, 2: 5} means share two cells in 0 axis, 5 cells in 2 axis
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.ghost_internal.',
         Warning)

    return overlap.overlap_internal(x, axes)

def trim_internal(x, axes):
    """ Trim sides from each block

    This couples well with the ghost operation, which may leave excess data on
    each block

    See also
    --------
    dask.array.chunk.trim
    dask.array.map_blocks
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.trim_internal.',
         Warning)

    return overlap.trim_internal(x, axes)

def periodic(x, axis, depth):
    """ Copy a slice of an array around to its other side

    Useful to create periodic boundary conditions for ghost
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.periodic.',
         Warning)

    return overlap.periodic(x, axis, depth)

def reflect(x, axis, depth):
    """ Reflect boundaries of array on the same side

    This is the converse of ``periodic``
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.reflect.',
         Warning)

    return overlap.reflect(x, axis, depth)

def nearest(x, axis, depth):
    """ Each reflect each boundary value outwards

    This mimics what the skimage.filters.gaussian_filter(... mode="nearest")
    does.
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.nearest.',
         Warning)

    return overlap.nearest(x, axis, depth)

def constant(x, axis, depth, value):
    """ Add constant slice to either side of array """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.constant.',
         Warning)

    return overlap.constant(x, axis, depth, value)

def _remove_ghost_boundaries(l, r, axis, depth):
    lchunks = list(l.chunks)
    lchunks[axis] = (depth,)
    rchunks = list(r.chunks)
    rchunks[axis] = (depth,)

    l = l.rechunk(tuple(lchunks))
    r = r.rechunk(tuple(rchunks))
    return l, r

def boundaries(x, depth=None, kind=None):
    """ Add boundary conditions to an array before ghosting

    See Also
    --------
    periodic
    constant
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.boundaries.',
         Warning)

    return overlap.boundaries(x, depth, kind)

def ghost(x, depth, boundary):
    """ Share boundaries between neighboring blocks

    Parameters
    ----------

    x: da.Array
        A dask array
    depth: dict
        The size of the shared boundary per axis
    boundary: dict
        The boundary condition on each axis. Options are 'reflect', 'periodic',
        'nearest', 'none', or an array value.  Such a value will fill the
        boundary with that value.

    The depth input informs how many cells to dask.array.overlap between neighboring
    blocks ``{0: 2, 2: 5}`` means share two cells in 0 axis, 5 cells in 2 axis.
    Axes missing from this input will not be overlapped.

    Examples
    --------
    >>> import numpy as np
    >>> import dask.array as da

    >>> x = np.arange(64).reshape((8, 8))
    >>> d = da.from_array(x, chunks=(4, 4))
    >>> d.chunks
    ((4, 4), (4, 4))

    >>> g = da.ghost.ghost(d, depth={0: 2, 1: 1},
    ...                       boundary={0: 100, 1: 'reflect'})
    >>> g.chunks
    ((8, 8), (6, 6))

    >>> np.array(g)
    array([[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
           [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
           [  0,   0,   1,   2,   3,   4,   3,   4,   5,   6,   7,   7],
           [  8,   8,   9,  10,  11,  12,  11,  12,  13,  14,  15,  15],
           [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
           [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
           [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
           [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
           [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
           [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
           [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
           [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
           [ 48,  48,  49,  50,  51,  52,  51,  52,  53,  54,  55,  55],
           [ 56,  56,  57,  58,  59,  60,  59,  60,  61,  62,  63,  63],
           [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
           [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.ghost.',
         Warning)

    return overlap.overlap(x, depth, boundary)

def add_dummy_padding(x, depth, boundary):
    """
    Pads an array which has 'none' as the boundary type.
    Used to simplify trimming arrays which use 'none'.

    >>> import dask.array as da
    >>> x = da.arange(6, chunks=3)
    >>> add_dummy_padding(x, {0: 1}, {0: 'none'}).compute()  # doctest: +NORMALIZE_WHITESPACE
    array([..., 0, 1, 2, 3, 4, 5, ...])
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.add_dummy_padding.',
         Warning)

    return overlap.add_dummy_padding(x, depth, boundary)

def map_overlap(x, func, depth, boundary=None, trim=True, **kwargs):
    """ Map a function over blocks of the array with some overlap

    We share neighboring zones between blocks of the array, then map a
    function, then trim away the neighboring strips.

    Parameters
    ----------
    func: function
        The function to apply to each extended block
    depth: int, tuple, or dict
        The number of elements that each block should share with its neighbors
        If a tuple or dict then this can be different per axis
    boundary: str, tuple, dict
        How to handle the boundaries.
        Values include 'reflect', 'periodic', 'nearest', 'none',
        or any constant value like 0 or np.nan
    trim: bool
        Whether or not to trim ``depth`` elements from each block after
        calling the map function.
        Set this to False if your mapping function already does this for you
    **kwargs:
        Other keyword arguments valid in ``map_blocks``

    Examples
    --------
    >>> import numpy as np
    >>> import dask.array as da

    >>> x = np.array([1, 1, 2, 3, 3, 3, 2, 1, 1])
    >>> x = da.from_array(x, chunks=5)
    >>> def derivative(x):
    ...     return x - np.roll(x, 1)

    >>> y = x.map_overlap(derivative, depth=1, boundary=0)
    >>> y.compute()
    array([ 1,  0,  1,  1,  0,  0, -1, -1,  0])

    >>> x = np.arange(16).reshape((4, 4))
    >>> d = da.from_array(x, chunks=(2, 2))
    >>> d.map_overlap(lambda x: x + x.size, depth=1).compute()
    array([[16, 17, 18, 19],
           [20, 21, 22, 23],
           [24, 25, 26, 27],
           [28, 29, 30, 31]])

    >>> func = lambda x: x + x.size
    >>> depth = {0: 1, 1: 1}
    >>> boundary = {0: 'reflect', 1: 'none'}
    >>> d.map_overlap(func, depth, boundary).compute()  # doctest: +NORMALIZE_WHITESPACE
    array([[12,  13,  14,  15],
           [16,  17,  18,  19],
           [20,  21,  22,  23],
           [24,  25,  26,  27]])
    """
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.map_overlap.',
         Warning)

    return overlap.map_overlap(x, func, depth, boundary, trim, **kwargs)

def coerce_depth(ndim, depth):
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.coerce_depth.',
         Warning)

    return overlap.coerce_depth(ndim, depth)

def coerce_boundary(ndim, boundary):
    warn('DeprecationWarning: the dask.array.ghost module has '
         'been renamed to dask.array.overlap, '
         'use dask.array.overlap.coerce_boundary.',
         Warning)

    return overlap.coerce_boundary(ndim, boundary)


text:
# -*- coding: utf-8 -*-
# *****************************************************************************
# Copyright (c) 2020, Intel Corporation All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#
#     Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# *****************************************************************************

from numba import types
from numba.extending import (
    models,
    register_model,
    make_attribute_wrapper
)

class EmptyIndexType(types.Type):

    # this index represents special case of pd.Index([]) with dtype='object'
    # for overload typing functions assume it has following dtype
    dtype = types.pyobject

    def __init__(self, is_named=False):
        self.is_named = is_named
        super(EmptyIndexType, self).__init__(
            name='EmptyIndexType({})'.format(is_named))

@register_model(EmptyIndexType)
class EmptyIndexModel(models.StructModel):
    def __init__(self, dmm, fe_type):

        name_type = types.unicode_type if fe_type.is_named else types.none
        members = [
            ('name', name_type),
        ]
        models.StructModel.__init__(self, dmm, fe_type, members)

# FIXME_Numba#3372: add into numba.types to allow returning from objmode
types.EmptyIndexType = EmptyIndexType

make_attribute_wrapper(EmptyIndexType, 'name', '_name')

reason: from numba