text:
import numpy as np
from .aggregator_base import Aggregator
from .krum import MultiKrum
from .fedavg import FedAvg

class Median(Aggregator):
    """
    Implementation of the median byzantine-robust aggregation technique.
    """
    def __init__(self, arguments, name, kwargs):
        print("MEDIAN is SELECTED")
        super().__init__(arguments, name)
        self.portion_uploaded = self.args.get_portion_uploaded()
        self.portion_downloaded = self.args.get_portion_downloaded()

        # Some assertions for restricted portions
        assert self.portion_uploaded > 0 and self.portion_uploaded <= 1, "Upload proportion should be within (0, 1]"
        assert self.portion_downloaded > 0 and self.portion_downloaded <= 1, "Download proportion should be within (0, 1]"

    def restricted_grad_sharing(self, gradients, portion):
        #Restricts upload of gradient by the stipulated portion
        #Function borrowed from MultiKrum.
        return MultiKrum.restricted_grad_sharing(self, gradients, portion)

    def get_gradient(self, idx, epoch, portion):
        #Function borrowed from MultiKrum.
        # idx - worker index
        # epoch - communication round
        # This function collects the uploaded gradients of clients for a given epoch, and converts these from Torch to numpy
        return MultiKrum.get_gradient(self, idx, epoch, portion)

    def get_uploaded_gradients(self):
        """
        Function borrowed from MultiKrum
        Collects the gradients of each worker given the epoch.

        Returns: Worker and its gradients (list)
        """
        return MultiKrum.get_uploaded_gradients(self)

    def aggregate(self, uploaded_gradients):
        """
        Computes the Median gradient"
        Step 1. Collect the gradients of selected workers for each layer.
        Step 2. Calculate the median gradient per layer
        """
        assert len(uploaded_gradients.values()) > 0, "Empty list of gradient to aggregate"
        num_layers = len(uploaded_gradients[list(uploaded_gradients.keys())[0]])  # number of layers

        median_gradient = list()
        for layer in range(num_layers):
            collect_grads = list()
            for i in uploaded_gradients.keys():
                collect_grads.append(uploaded_gradients[i][layer])
            median_layer = np.median(np.array(collect_grads), axis = 0)
            #print("Computing median gradient of layer: {}. Shape: {}".format(layer, median_layer.shape))
            median_gradient.append(median_layer)
        #print("Length of median gradient", len(median_gradient))
        print("Return: Median gradient")
        return median_gradient

    def update_global_params(self, global_client, agg_gradient):
        "Updates global parameters"
        return MultiKrum.update_global_params(self, global_client, agg_gradient)

    def update_params(self, local_clients, global_client, agg_gradient):
        #A. Update global parameters
        updated_global_params = MultiKrum.update_global_params(self, global_client, agg_gradient)

        #B. Update local parameters
        print("Portion downloaded by clients:", self.portion_downloaded)
        for client in local_clients:
            ##If download = 1.0, update all local clients with full updated_params
            ##Else, use FedAvg.simple_update_with_missing to account for restricted download of paramaters
            if self.portion_downloaded == 1:
                self.args.get_logger().info("Unrestricted download of global parameters on client #{}",
                                            str(client.get_client_index()))
                client.update_nn_parameters(updated_global_params)

            else:
                downloadable = super().get_downloadable_params()  # Restricted global parameters for download
                FedAvg.simple_update_with_missing(self, client, downloadable)

    def run_aggregation(self, epoch, client_ids, local_clients, global_client):
        MultiKrum.run_aggregation(self, epoch, client_ids, local_clients, global_client)

max_stars_repo_name: ameldocena/StratifiedAggregation

max_issues_repo_name: ameldocena/StratifiedAggregation

max_forks_repo_name: ameldocena/StratifiedAggregation


text:
import cv2
import numpy as np
import glob

images = [(file, cv2.imread(file)) for file in glob.glob("data/01_Pink_impTNT/01_pink_imptnt_01/*.jpg")]

for file, img in images:
    print(file[38:])
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    filt = cv2.bilateralFilter(img, 15, 5, 5)
    equ = cv2.equalizeHist(filt)
    th1 = cv2.adaptiveThreshold(filt, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 195, 25)
    th2 = cv2.adaptiveThreshold(equ, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 195, 60)
    th = 255 - (255-th1) - (255-th2)
    kernel_co = np.ones((15, 15), np.uint8)
    kernel_dil = np.ones((20, 25), np.uint8)
    closing = cv2.morphologyEx(th, cv2.MORPH_CLOSE, kernel_co)
    opening = cv2.morphologyEx(closing, cv2.MORPH_OPEN, kernel_co)
    dilation = cv2.erode(opening, kernel_dil, 1)
    mask = 255 - dilation
    edge = cv2.Canny(filt, 20, 255)
    edge_masked = edge * mask

    th_masked = th
    ids = np.argwhere(mask.flatten() == 0)
    th_masked = th_masked.flatten()
    th_masked[ids] = 255
    th_masked = th_masked.reshape(mask.shape[0], -1)
    cv2.imwrite("data/01_Pink_impTNT/01_pink_imptnt_01_1/{}".format(file[38:]), edge_masked)

ext: py

max_stars_repo_name: utanashati/cobar-flies

max_issues_repo_name: utanashati/cobar-flies

max_forks_repo_name: utanashati/cobar-flies

reason: import numpy


text:
import torch
import numpy as np
from utils.boxes import calculate_ious
from utils.constants import BACKGROUND_INDEX

class Mean(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.sum = 0
        self.count = 0

    def update(self, v, n):
        self.sum += v
        self.count += n

    @property
    def result(self):
        return self.sum / self.count

class AveragePrecision(object):
    def __init__(self, num_classes, recall_steps):
        self.num_classes = num_classes
        self.recall_steps = recall_steps
        self.reset()

    def reset(self):
        self.num_relevent = torch.zeros([self.num_classes])
        self.det_scores = []
        self.det_classes = []
        self.true_positives = []
        self.false_positives = []

    def update(self, det_boxes, det_scores, det_classes, true_boxes, true_classes,
               difficulties):
        iou_thres = np.linspace(0.5, 0.95, 10)

        bs = len(det_boxes)
        for i in range(bs):
            num_dets = det_boxes[i].shape[0]
            num_true = true_boxes[i].shape[0]
            if num_true == 0:
                self.true_positives.extend([False] * num_dets)
                self.false_positives.extend([True] * num_dets)
                continue

            # Count relevent elements
            class_indices, counts = torch.unique(
                true_classes[i][
                    (true_classes[i] != BACKGROUND_INDEX)
                    & (difficulties[i] == 0)
                ],
                return_counts=True
            )
            self.num_relevent[class_indices] += counts.cpu()

            if num_dets == 0:
                continue

            # Store detections
            self.det_scores.extend(det_scores[i].tolist())
            self.det_classes.extend(det_classes[i].tolist())

            # Determine if detections match ground truth in terms of IoU and class label
            ious = calculate_ious(true_boxes[i], det_boxes[i])   # [num_true, num_dets]
            mask = (   # [num_true, num_dets]
                (ious >= iou_thres[0])
                & (true_classes[i].unsqueeze(-1) == det_classes[i])
                & (true_classes[i].unsqueeze(-1) != BACKGROUND_INDEX)
            )

            is_tp = np.full([num_dets, 10], False)
            is_fp = np.full([num_dets, 10], True)
            indices = torch.where(mask)   # ground truth indices, detection indices
            if indices[0].shape[0] > 0:
                matches = torch.stack(indices, axis=1)   # [num_matches, 2]
                matches = matches.cpu().numpy()   # only np.unique() supports `return_index` option
                ious = ious.cpu().numpy()
                ious = ious[matches[:, 0], matches[:, 1]]

                # Find the ground truth with the best IoU for each detection
                matches = matches[ious.argsort()[::-1]]
                indices = np.unique(matches[:, 1], return_index=True)[1]
                matches = matches[indices]
                ious = ious[indices]

                is_difficult = difficulties[i].bool().cpu().numpy()[matches[:, 0]]
                is_fp[matches[:, 1][is_difficult]] = (
                    np.expand_dims(ious[is_difficult], axis=-1)
                    < iou_thres
                )

                # Each ground truth is assigned to the detection with the best IoU
                indices = np.unique(matches[:, 0], return_index=True)[1]
                matches = matches[indices]
                ious = ious[indices]

                is_easy = ~(difficulties[i].bool().cpu().numpy()[matches[:, 0]])
                is_tp[matches[:, 1][is_easy]] = (
                    np.expand_dims(ious[is_easy], axis=-1)
                    >= iou_thres
                )
                is_fp[is_tp] = False

            self.true_positives.extend(is_tp.tolist())
            self.false_positives.extend(is_fp.tolist())

    @property
    def result(self):
        """
        Returns:
            APs: float32 tensor. Shape: [num_classes, 10].
        """
        scores, indices = torch.sort(
            torch.FloatTensor(self.det_scores),
            descending=True
        )
        classes = torch.IntTensor(self.det_classes)[indices]
        true_positives = torch.IntTensor(self.true_positives)[indices]
        false_positives = torch.IntTensor(self.false_positives)[indices]
        recall_thres = torch.linspace(0, 1, self.recall_steps)

        APs = torch.zeros([self.num_classes, 10])
        for c in range(self.num_classes):
            if (classes == c).int().sum() == 0:
                continue

            tp_cum = torch.cumsum(true_positives[classes == c], axis=0)    # [n, 10]
            fp_cum = torch.cumsum(false_positives[classes == c], axis=0)   # [n, 10]
            num_relevent = self.num_relevent[c]

            recalls = tp_cum / num_relevent
            precisions = tp_cum / (tp_cum + fp_cum + 1e-10)

            recall_above_thres = recalls.unsqueeze(-1) >= recall_thres   # [n, 10, recall_steps]

            max_precisions, _ = torch.max(   # [10, recall_steps]
                precisions.unsqueeze(-1) * recall_above_thres.int(),
                axis=0
            )
            APs[c] = torch.mean(max_precisions, axis=-1)
        return APs

max_stars_repo_name: biyoml/PyTorch-SSD

max_issues_repo_name: biyoml/PyTorch-SSD

max_forks_repo_name: biyoml/PyTorch-SSD


text:
import torch
from torch.utils.data import Dataset
import numpy as np
import os, glob
import cv2 as cv

class DrumDataset_test(Dataset): #dtype = 'tmel', 'time', 'mel
    def __init__(self,labels,test_path,dtype='tmel'):
        folders = ['images/time','images/mel']

        path_t = os.path.join("./",test_path,folders[0])
        path_mel = os.path.join("./",test_path,folders[1])

        self.labels = labels
        self.test_data = []
        self.test_label = []

        if dtype == 'tmel':
            for path_t_name, path_mel_name in zip(glob.glob(os.path.join(path_t,'*')),glob.glob(os.path.join(path_mel,'*'))):
                label = self.label_config(path_t_name)

                img_t = cv.imread(path_t_name,0)
                img_t = np.expand_dims(img_t, axis=2)
                img_mel = cv.imread(path_mel_name)

                img = np.concatenate((img_t,img_mel),axis=2)
                img = np.transpose(img,(2,0,1))

                self.test_data.append(img)
                self.test_label.append(label)

        elif dtype == 'time':
            for path_t_name in glob.glob(os.path.join(path_t,'*')):
                label = self.label_config(path_t_name)

                img_t = cv.imread(path_t_name,0)
                img_t = np.expand_dims(img_t, axis=2)
                img = np.transpose(img_t,(2,0,1))

                self.test_data.append(img)
                self.test_label.append(label)

        elif dtype == 'mel':
            for path_mel_name in glob.glob(os.path.join(path_mel,'*')):
                label = self.label_config(path_mel_name)

                img_mel = cv.imread(path_mel_name)
                img = np.transpose(img_mel,(2,0,1))

                self.test_data.append(img)
                self.test_label.append(label)

    def label_config(self,path_name):
        data_name = path_name.split('_')
        label_n = data_name[-1][:-4]
        if label_n == '':
            label = 'X'
        else:
            label = self.labels.index(label_n)
        return label

    def __len__(self):
        return len(self.test_data)

    def __getitem__(self,idx):
        self.test_data = np.array(self.test_data)
        self.test_label = np.array(self.test_label)
        return self.test_data[idx], self.test_label[idx]

max_stars_repo_name: SoobumKim/Drum-Classification-pytorch

max_issues_repo_name: SoobumKim/Drum-Classification-pytorch

max_forks_repo_name: SoobumKim/Drum-Classification-pytorch


text:
"""
This script makes a blanket and then segments it in a similar
manner to the EU DEMO segmentation for remote maintenance.
"""

import math

import numpy as np
import paramak

def main(number_of_sections=8, gap_size=15, central_block_width=200):

    number_of_segments = 8
    gap_size = 15.
    central_block_width = 200

    offset = (360 / number_of_segments) / 2

    # a plasma shape is made and used by the BlanketFP, which builds around
    # the plasma
    plasma = paramak.Plasma(
        elongation=1.59,
        triangularity=0.33,
        major_radius=910,
        minor_radius=290)
    plasma.solid

    # this makes a cutter shape that is used to make the blanket bananna
    # segment that has parallel sides
    parallel_outboard_gaps_outer = paramak.BlanketCutterParallels(
        thickness=gap_size, azimuth_placement_angle=np.linspace(
            0, 360, number_of_segments, endpoint=False),
        gap_size=central_block_width)

    # this makes a gap that seperates the inboard and outboard blanket
    inboard_to_outboard_gaps = paramak.ExtrudeStraightShape(
        points=[(plasma.high_point[0] - (0.5 * gap_size), plasma.high_point[1]),
                (plasma.high_point[0] - (0.5 * gap_size), plasma.high_point[1] + 1000),
                (plasma.high_point[0] + (0.5 * gap_size), plasma.high_point[1] + 1000),
                (plasma.high_point[0] + (0.5 * gap_size), plasma.high_point[1]),
                ],
        distance=math.tan(math.radians(360 / (2 * number_of_segments))) * plasma.high_point[0] * 2,
        azimuth_placement_angle=np.linspace(0, 360, number_of_segments, endpoint=False)
    )

    # this makes the regular gaps (non parallel) gaps on the outboard blanket
    outboard_gaps = paramak.BlanketCutterStar(
        distance=gap_size,
        azimuth_placement_angle=np.linspace(
            0 + offset,
            360 + offset,
            number_of_segments,
            endpoint=False)
    )

    # makes the outboard blanket with cuts for all the segmentation
    outboard_blanket = paramak.BlanketFP(
        plasma=plasma,
        thickness=100,
        stop_angle=90,
        start_angle=-60,
        offset_from_plasma=30,
        rotation_angle=360,
        cut=[
            outboard_gaps,
            parallel_outboard_gaps_outer,
            inboard_to_outboard_gaps])

    # this makes the regular gaps on the outboard blanket
    inboard_gaps = paramak.BlanketCutterStar(
        distance=gap_size, azimuth_placement_angle=np.linspace(
            0, 360, number_of_segments * 2, endpoint=False))

    # makes the inboard blanket with cuts for all the segmentation
    inboard_blanket = paramak.BlanketFP(
        plasma=plasma,
        thickness=100,
        stop_angle=90,
        start_angle=260,
        offset_from_plasma=30,
        rotation_angle=360,
        cut=[inboard_gaps, inboard_to_outboard_gaps],
        union=outboard_blanket
    )

    # saves the blanket as an stp file
    inboard_blanket.export_stp('blanket.stp')

if __name__ == "__main__":
    main()

max_stars_repo_name: PullRequest-Agent/paramak

max_issues_repo_name: PullRequest-Agent/paramak

max_forks_repo_name: PullRequest-Agent/paramak


text:
import contextlib
import copy
import datetime
import functools
import importlib
import inspect
import logging
from typing import Type, Optional, Tuple, Union
import dateutil
import os
import pickle
import platform
import pprint
import sys
import tempfile
import time
import uuid
import warnings
from collections import OrderedDict
from functools import wraps
from itertools import chain

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import qcodes
import scipy.ndimage as ndimage
from matplotlib.widgets import Button
from qcodes.data.data_array import DataArray
from qcodes.data.data_set import DataSet
from qcodes.plots.qcmatplotlib import MatPlot

import qtt.pgeometry
from qtt.pgeometry import (mkdirc,  # import for backwards compatibility
                           mpl2clipboard, tilefigs)

NotGitRepositoryError: Type[Exception]

try:
    from dulwich import porcelain
    from dulwich.repo import NotGitRepository, Repo
    NotGitRepositoryError = NotGitRepository
except ModuleNotFoundError:
    warnings.warn('please install dulwich: pip install dulwich --global-option="--pure"')
    NotGitRepositoryError = Exception

# explicit import

try:
    from qcodes.plots.pyqtgraph import QtPlot
except BaseException:
    pass

# do NOT load any other qtt submodules here

try:
    import qtpy.QtCore as QtCore
    import qtpy.QtGui as QtGui
    import qtpy.QtWidgets as QtWidgets
except BaseException:
    pass

# %%

class measure_time():
        """ Create context manager that measures execution time and prints to stdout """

        def __init__(self, message: Optional[str] = 'dt: '):
            self.message = message

        def __enter__(self):
            self.start_time = time.perf_counter()
            return self

        @property
        def delta_time(self) -> float:
            """ Return time spend in the context

            Returns:
                Time in seconds
            """
            return self.dt

        def __exit__(self, exc_type, exc_value, exc_traceback):
            self.dt = time.perf_counter() - self.start_time

            if self.message is not None:
                print(f'{self.message} {self.dt:.3f} [s]')

@contextlib.contextmanager
def logging_context(level: int = logging.INFO, logger: Optional[logging.Logger] = None):
    """ A context manager that changes the logging level

    Args:
        level: Logging level to set in the context
        logger: Logger to update, if None then update the default logger

    """
    if logger is None:
        logger = logging.getLogger()
    previous_level = logger.getEffectiveLevel()
    logger.setLevel(level)

    try:
        yield
    finally:
        logger.setLevel(previous_level)

def is_spyder_environment():
    """ Return True if the process is running in a Spyder environment """
    return 'SPY_TESTING' in os.environ

def get_module_versions(modules, verbose=0):
    """ Returns the module version of the given pip packages.

    Args:
        modules ([str]): a list with pip packages, e.g. ['numpy', 'scipy'].
        verbose (int): verbosity (0 == silent).

    Returns:
        r (dict): dictionary with package names and version number for each given module.

    """
    module_versions = dict()
    for module in modules:
        try:
            package = importlib.import_module(module)
            version = getattr(package, '__version__', 'none')
            module_versions[module] = version
        except ModuleNotFoundError:
            module_versions[module] = 'none'
        if verbose:
            print('module {0} {1}'.format(package, version))
    return module_versions

def get_git_versions(repos, get_dirty_status=False, verbose=0):
    """ Returns the repository head guid and dirty status and package version number if installed via pip.
        The version is only returned if the repo is installed as pip package without edit mode.
        NOTE: currently the dirty status is not working correctly due to a bug in dulwich...

    Args:
        repos ([str]): a list with repositories, e.g. ['qtt', 'qcodes'].
        get_dirty_status (bool): selects whether to use the dulwich package and collect the local code
                            changes for the repositories.
        verbose (int): verbosity (0 == silent).

    Retuns:
        r (dict): dictionary with repo names, head guid and (optionally) dirty status for each given repository.

    """
    heads = dict()
    dirty_stats = dict()
    for repo in repos:
        try:
            package = importlib.import_module(repo)
            init_location = os.path.split(package.__file__)[0]
            repo_location = os.path.join(init_location, '..')
            repository = Repo(repo_location)
            heads[repo] = repository.head().decode('ascii')
            if get_dirty_status:
                status = porcelain.status(repository)
                is_dirty = len(status.unstaged) == 0 or any(len(item) != 0 for item in status.staged.values())
                dirty_stats[repo] = is_dirty
        except (AttributeError, ModuleNotFoundError, NotGitRepositoryError):
            heads[repo] = 'none'
            if get_dirty_status:
                dirty_stats[repo] = 'none'
        if verbose:
            print('{0}: {1}'.format(repo, heads[repo]))
    return heads, dirty_stats

def get_python_version(verbose=0):
    """ Returns the python version."""
    version = sys.version
    if verbose:
        print('Python', version)
    return version

def code_version(repository_names=None, package_names=None, get_dirty_status=False, verbose=0):
    """ Returns the python version, module version for; numpy, scipy, qupulse
        and the git guid and dirty status for; qcodes, qtt, spin-projects and pycqed,
        if present on the machine. NOTE: currently the dirty status is not working
        correctly due to a bug in dulwich...

    Args:
        repository_names ([str]): a list with repositories, e.g. ['qtt', 'qcodes'].
        package_names ([str]): a list with pip packages, e.g. ['numpy', 'scipy'].
        get_dirty_status (bool): selects whether the local code has changed for the repositories.
        verbose (int): output level.

    Returns:
        status (dict): python, modules and git repos status.

    """
    _default_git_versions = ['qcodes', 'qtt', 'projects', 'pycqed']
    _default_module_versions = ['numpy', 'scipy', 'qupulse', 'h5py', 'skimage']
    if not repository_names:
        repository_names = _default_git_versions
    if not package_names:
        package_names = _default_module_versions
    result = dict()
    repository_stats, dirty_stats = get_git_versions(repository_names, get_dirty_status, verbose)
    result['python'] = get_python_version(verbose)
    result['git'] = repository_stats
    result['version'] = get_module_versions(package_names, verbose)
    result['timestamp'] = datetime.datetime.now().isoformat()  # ISO 8601
    result['system'] = {'node': platform.node()}
    if get_dirty_status:
        result['dirty'] = dirty_stats

    return result

# %% Debugging

def deprecated(func):
    """ This is a decorator which can be used to mark functions as deprecated. It will result in a warning being
    emitted when the function is used."""

    @functools.wraps(func)
    def new_func(*args, **kwargs):
        try:
            filename = inspect.getfile(func)
        except BaseException:
            filename = '?'
        try:
            lineno = inspect.getlineno(func)
        except BaseException:
            lineno = -1
        warnings.warn_explicit(
            "Call to deprecated function {}.".format(func.__name__),
            category=UserWarning,
            filename=filename,
            lineno=lineno,
        )
        return func(*args, **kwargs)

    return new_func

def rdeprecated(txt=None, expire=None):
    """ This is a decorator which can be used to mark functions as deprecated.

    It will result in a warning being emitted when the function is used. After the expiration data the decorator
    will generate an Exception.

    Args:
        txt (str): reason for deprecation.
        expire (str): date of expiration.
    """
    import datetime

    from dateutil import parser
    if expire is not None:
        now = datetime.datetime.now()
        expiredate = parser.parse(expire)
        dt = expiredate - now
        expired = dt.total_seconds() < 0
    else:
        expired = None

    def deprecated_inner(func):
        """ This is a decorator which can be used to mark functions
        as deprecated. It will result in a warning being emitted
        when the function is used."""

        @functools.wraps(func)
        def new_func(*args, **kwargs):
            try:
                filename = inspect.getfile(func)
            except BaseException:
                filename = '?'
            try:
                lineno = inspect.getlineno(func)
            except BaseException:
                lineno = -1
            if txt is None:
                etxt = ''
            else:
                etxt = ' ' + txt

            if expire is not None:
                if expired:
                    raise Exception("Call to deprecated function {}.{}".format(func.__name__, etxt))
                else:
                    warnings.warn_explicit(
                        "Call to deprecated function {} (will expire on {}).{}".format(
                            func.__name__, expiredate, etxt),
                        category=UserWarning,
                        filename=filename,
                        lineno=lineno,
                    )
            else:
                warnings.warn_explicit(
                    "Call to deprecated function {}.{}".format(func.__name__, etxt),
                    category=UserWarning,
                    filename=filename,
                    lineno=lineno,
                )
            return func(*args, **kwargs)

        return new_func

    return deprecated_inner

# %%

def update_dictionary(alldata, **kwargs):
    """ Update elements of a dictionary.

    Args:
        alldata (dict): dictionary to be updated.
        kwargs (dict): keyword arguments.

    """
    for k in kwargs:
        alldata[k] = kwargs[k]

def stripDataset(dataset):
    """ Make sure a dataset can be pickled .

    Args:
        dataset (qcodes DataSet): TODO.

    Returns:
        dataset (qcodes DataSet): the dataset from the function argument.

    """
    dataset.sync()
    dataset.data_manager = None
    dataset.background_functions = {}
    try:
        dataset.formatter.close_file(dataset)
    except BaseException:
        pass

    if 'scanjob' in dataset.metadata:
        if 'minstrumenthandle' in dataset.metadata['scanjob']:
            dataset.metadata['scanjob']['minstrumenthandle'] = str(
                dataset.metadata['scanjob']['minstrumenthandle'])

    return dataset

# %%

def negfloat(x):
    """ Helper function """
    return -float(x)

def checkPickle(obj, verbose=0):
    """ Check whether an object can be pickled.

    Args:
        obj (object): object to be checked.
        verbose (int): verbosity (0 == silent).

    Returns:
        c (bool): True of the object can be pickled.

    """
    try:
        _ = pickle.dumps(obj)
    except Exception as ex:
        if verbose:
            print(ex)
        return False
    return True

def freezeclass(cls):
    """ Decorator to freeze a class."""
    cls.__frozen = False

    def frozensetattr(self, key, value):
        if self.__frozen and not hasattr(self, key):
            print("Class {} is frozen. Cannot set {} = {}"
                  .format(cls.__name__, key, value))
        else:
            object.__setattr__(self, key, value)

    def init_decorator(func):
        @wraps(func)
        def wrapper(self, *args, **kwargs):
            func(self, *args, **kwargs)
            self.__frozen = True

        return wrapper

    cls.__setattr__ = frozensetattr
    cls.__init__ = init_decorator(cls.__init__)

    return cls

# %%

def resampleImage(im):
    """ Resample the image so it has the similar sample rates (samples/mV) in both axis.

    Args:
        im (DataArray): input image.
    Returns:
        imr (numpy array): resampled image.
        setpoints (list of 2 numpy arrays): setpoint arrays from resampled image.
    """
    setpoints = im.set_arrays
    mVrange = [abs(setpoints[0][-1] - setpoints[0][0]),
               abs(setpoints[1][0, -1] - setpoints[1][0, 0])]
    samprates = [im.shape[0] // mVrange[0], im.shape[1] // mVrange[1]]
    factor = int(max(samprates) // min(samprates))
    if factor >= 2:
        axis = int(samprates[0] - samprates[1] < 0)
        if axis == 0:
            facrem = im.shape[0] % factor
            if facrem > 0:
                im = im[:-facrem, :]
            facrem = facrem + 1
            im = im.reshape(im.shape[0] // factor, factor, im.shape[1]).mean(1)
            spy = np.linspace(setpoints[0][0], setpoints[
                0][-facrem], im.shape[0])
            spx = np.tile(np.expand_dims(np.linspace(
                setpoints[1][0, 0], setpoints[1][0, -1], im.shape[1]), 0), im.shape[0])
            setpointy = DataArray(name='Resampled_' + setpoints[0].array_id,
                                  array_id='Resampled_' + setpoints[0].array_id, label=setpoints[0].label,
                                  unit=setpoints[0].unit, preset_data=spy, is_setpoint=True)
            setpointx = DataArray(name='Resampled_' + setpoints[1].array_id,
                                  array_id='Resampled_' + setpoints[1].array_id, label=setpoints[1].label,
                                  unit=setpoints[1].unit, preset_data=spx, is_setpoint=True)
            setpoints = [setpointy, setpointx]
        else:
            facrem = im.shape[1] % factor
            if facrem > 0:
                im = im[:, :-facrem]
            facrem = facrem + 1
            im = im.reshape(im.shape[0], im.shape[1] //
                            factor, factor).mean(-1)
            spx = np.tile(np.expand_dims(np.linspace(setpoints[1][0, 0], setpoints[
                1][0, -facrem], im.shape[1]), 0), [im.shape[0], 1])
            idx = setpoints[1].array_id
            if idx is None:
                idx = 'x'
            idy = setpoints[1].array_id
            if idy is None:
                idy = 'y'
            setpointx = DataArray(name='Resampled_' + idx, array_id='Resampled_' + idy, label=setpoints[1].label,
                                  unit=setpoints[1].unit, preset_data=spx, is_setpoint=True)
            setpoints = [setpoints[0], setpointx]

    return im, setpoints

def diffImage(im, dy, size=None):
    """ Simple differentiation of an image.

    Args:
        im (numpy array): input image.
        dy (integer or string): method of differentiation. For an integer it is the axis of differentiation.
            Allowed strings are 'x', 'y', 'xy'.
        size (str): describes the size e.g. 'same'.

    """
    if dy == 0 or dy == 'x':
        im = np.diff(im, n=1, axis=1)
        if size == 'same':
            im = np.hstack((im, im[:, -1:]))
    elif dy == 1 or dy == 'y':
        im = np.diff(im, n=1, axis=0)
        if size == 'same':
            im = np.vstack((im, im[-1:, :]))
    elif dy == -1:
        im = -np.diff(im, n=1, axis=0)
        if size == 'same':
            im = np.vstack((im, im[-1:, :]))
    elif dy == 2 or dy == 'xy':
        imx = np.diff(im, n=1, axis=1)
        imy = np.diff(im, n=1, axis=0)
        im = imx[0:-1, :] + imy[:, 0:-1]
    elif dy is None:
        pass
    else:
        raise Exception('differentiation method %s not supported' % dy)
    return im

def diffImageSmooth(im, dy='x', sigma=2):
    """ Simple differentiation of an image.

    Args:
        im (array): input image.
        dy (string or integer): direction of differentiation. can be 'x' (0) or 'y' (1) or 'xy' (2) or 'g' (3).
        sigma (float): parameter for gaussian filter kernel.

    """
    if sigma is None:
        imx = diffImage(im, dy)
        return imx

    if dy is None:
        imx = im.copy()
    elif dy == 0 or dy == 'x':
        if len(im.shape) == 1:
            raise Exception(f'invalid parameter dy={dy} for 1D image')
        else:
            imx = ndimage.gaussian_filter1d(im, axis=1, sigma=sigma, order=1, mode='nearest')
    elif dy == 1 or dy == 'y':
        imx = ndimage.gaussian_filter1d(im, axis=0, sigma=sigma, order=1, mode='nearest')
    elif dy == -1:
        imx = -ndimage.gaussian_filter1d(im, axis=0,
                                         sigma=sigma, order=1, mode='nearest')
    elif dy == 2 or dy == 3 or dy == 'xy' or dy == 'xmy' or dy == 'xmy2' or dy == 'g' or dy == 'x2my2' or dy == 'x2y2':
        if len(np.array(im).shape) != 2:
            raise Exception(f'differentiation mode {dy} cannot be combined with input shape {np.array(im).shape}')
        imx0 = ndimage.gaussian_filter1d(
            im, axis=1, sigma=sigma, order=1, mode='nearest')
        imx1 = ndimage.gaussian_filter1d(
            im, axis=0, sigma=sigma, order=1, mode='nearest')
        if dy == 2 or dy == 'xy':
            imx = imx0 + imx1
        if dy == 'xmy':
            imx = imx0 - imx1
        if dy == 3 or dy == 'g':
            imx = np.sqrt(imx0 ** 2 + imx1 ** 2)
        if dy == 'xmy2':
            warnings.warn('please do not use this option')
            imx = np.sqrt(imx0 ** 2 + imx1 ** 2)
        if dy == 'x2y2':
            imx = imx0 ** 2 + imx1 ** 2
        if dy == 'x2my2':
            imx = imx0 ** 2 - imx1 ** 2
    else:
        raise Exception('differentiation method %s not supported' % dy)
    return imx

def scanTime(dd):
    """ Return date a scan was performed."""
    w = dd.metadata.get('scantime', None)
    if isinstance(w, str):
        w = dateutil.parser.parse(w)
    return w

# %%

def showImage(im, extent=None, fig=None, title=None):
    """ Show image in figure window.

    Args:
        im (array): TODO.
        extent (list): matplotlib style image extent.
        fig (None or int): figure window to show image.
        title (None or str): figure title.

    """
    import matplotlib.pyplot as plt
    if fig is not None:
        plt.figure(fig)
        plt.clf()
        plt.imshow(im, extent=extent, interpolation='nearest')
        if extent is not None:
            if extent[0] > extent[1]:
                plt.gca().invert_xaxis()
        if title is not None:
            plt.title(title)

# %% Tools from pgeometry

def cfigure(*args, **kwargs):
    """ Create Matplotlib figure with copy to clipboard functionality.

    By pressing the 'c' key figure is copied to the clipboard.

    """
    if 'facecolor' in kwargs:
        fig = plt.figure(*args, **kwargs)
    else:
        fig = plt.figure(*args, facecolor='w', **kwargs)

    def ff(xx, figx=fig):
        return mpl2clipboard(fig=figx)

    fig.canvas.mpl_connect('key_press_event', ff)  # mpl2clipboard)
    return fig

def static_var(varname, value):
    """ Helper function to create a static variable."""

    def decorate(func):
        setattr(func, varname, value)
        return func

    return decorate

try:
    import qtpy.QtGui as QtGui
    import qtpy.QtWidgets as QtWidgets

    def monitorSizes(verbose=0):
        """ Return monitor sizes."""
        _qd = QtWidgets.QDesktopWidget()
        if sys.platform == 'win32' and _qd is None:
            import ctypes
            user32 = ctypes.windll.user32
            wa = [
                [0, 0, user32.GetSystemMetrics(0), user32.GetSystemMetrics(1)]]
        else:
            _applocalqt = QtWidgets.QApplication.instance()

            if _applocalqt is None:
                _applocalqt = QtWidgets.QApplication([])
                _qd = QtWidgets.QDesktopWidget()
            else:
                _qd = QtWidgets.QDesktopWidget()

            nmon = _qd.screenCount()
            wa = [_qd.screenGeometry(ii) for ii in range(nmon)]
            wa = [[w.x(), w.y(), w.width(), w.height()] for w in wa]

            if verbose:
                for ii, w in enumerate(wa):
                    print('monitor %d: %s' % (ii, str(w)))
        return wa
except BaseException:
    def monitorSizes(verbose=0):
        """ Dummy function for monitor sizes."""
        return [[0, 0, 1600, 1200]]

    pass

# %% Helper tools

def in_ipynb():
    try:
        cfg = get_ipython().config
        if cfg['IPKernelApp']['parent_appname'] == 'ipython-notebook':
            return True
        else:
            return False
    except NameError:
        return False

def pythonVersion():
    try:
        import IPython
        ipversion = '.'.join('%s' % x for x in IPython.version_info[:-1])
    except BaseException:
        ipversion = 'None'

    pversion = '.'.join('%s' % x for x in sys.version_info[0:3])
    print('python %s, ipython %s, notebook %s' %
          (pversion, ipversion, in_ipynb()))

# %%

def _covert_integer_to_rgb_color(integer_value):
    if integer_value < 0 or integer_value > 256 ** 3:
        raise ValueError('Integer value cannot be converted to RGB!')

    red = integer_value & 0xFF
    green = (integer_value >> 8) & 0xFF
    blue = (integer_value >> 16) & 0xFF
    return red, green, blue

def _convert_rgb_color_to_integer(rgb_color):
    if not isinstance(rgb_color, tuple) or not all(isinstance(i, int) for i in rgb_color):
        raise ValueError('Color should be an RGB integer tuple.')

    if len(rgb_color) != 3:
        raise ValueError('Color should be an RGB integer tuple with three items.')

    if any(i < 0 or i > 255 for i in rgb_color):
        raise ValueError('Color should be an RGB tuple in the range 0 to 255.')

    red = rgb_color[0]
    green = rgb_color[1] << 8
    blue = rgb_color[2] << 16
    return int(red + green + blue)

def set_ppt_slide_background(slide, color, verbose=0):
    """ Sets the background color of PPT slide.

    Args:
        slide (object): PowerPoint COM object for slide.
        color (tuple): tuple with RGB color specification.
        verbose (int): verbosity (0 == silent).
    """
    fore_color = slide.Background.Fill.ForeColor
    ppt_color = _convert_rgb_color_to_integer(color)
    if verbose > 1:
        print('Setting PPT slide background color:')
        print(' - Current color: {0}'.format(_covert_integer_to_rgb_color(fore_color.RGB)))
        print(' - Setting to {0} -> {1}'.format(color, ppt_color))

    slide.FollowMasterBackground = 0
    fore_color.RGB = ppt_color

# %%

def _ppt_determine_image_position(ppt, figsize, fname, verbose=1):
    top = 120

    if figsize is not None:
        left = (ppt.PageSetup.SlideWidth - figsize[0]) / 2
        width = figsize[0]
        height = figsize[1]
    else:
        slidewh = [ppt.PageSetup.SlideWidth, ppt.PageSetup.SlideHeight]
        width = 16 * ((slidewh[0] * .75) // 16)
        height = 16 * (((slidewh[1] - 120) * .9) // 16)
        height = min(height, 350)
        left = (slidewh[0] - width) / 2

        try:
            import cv2
            imwh = cv2.imread(fname).shape[1], cv2.imread(fname).shape[0]
        except BaseException:
            imwh = None
        if imwh is not None:
            imratio = imwh[0] / imwh[1]
            slideratio = slidewh[0] / slidewh[1]
            if verbose >= 1:
                print(' image aspect ratio %.2f, slide aspect ratio %.2f' % (imratio, slideratio))
            if slideratio > imratio:
                # wide slide, so make the image width smaller
                print('adjust width %d->%d' % (width, height * imratio))
                width = height * imratio
            else:
                # wide image, so make the image height smaller
                print('adjust height %d->%d' % (height, width / imratio))
                height = int(width / imratio)

        if verbose:
            print('slide width height: %s' % (slidewh,))
            print('image width height: %d, %d' % (width, height))
    return left, top, width, height

def create_figure_ppt_callback(fig : Optional[int] = None, title : Optional[str] = None,
                       notes : Optional[Union[str, DataSet]]=None,
                       position : Tuple[float, float, float, float] = (0.9, 0.925, 0.075, 0.05)) -> None:
    """ Create a button on a matplotlib figure to copy data to PowerPoint slide.

    The figure is copied to PowerPoint using @ref addPPTslide.

    Args:
        fig: Handle to matplotlib window. If None, then use the current figure
        title: title for the slide.
        notes: notes to add to the slide.
        position: position specified as fraction left, right, width, height.

    Example:
        >>> plt.figure(10)
        >>> plt.plot(np.arange(100), np.random.rand(100), 'o', label='input data')
        >>> create_figure_ppt_callback(fig=10, title='test')
        >>> plt.show()
    """
    if fig is None:
        fig = plt.gcf().number
    plt.figure(fig)
    ax = plt.gca()
    ppt_axis = plt.axes(position, label=f'figure_ppt_callback_axis {uuid.uuid1()}')
    ppt_button = Button(ppt_axis, 'ppt')
    ppt_axis._button = ppt_button
    ppt_axis.set_alpha(.5)
    plt.sca(ax)

    def figure_ppt_callback(event):
        print('creating PowerPoint slide for figure %d' % fig)
        ppt_axis.set_visible(False)
        addPPTslide(fig=fig, title=title, notes=notes)
        ppt_axis.set_visible(True)

    ppt_button.on_clicked(figure_ppt_callback)

try:
    import win32com
    import win32com.client

    def addPPTslide(title=None, fig=None, txt=None, notes=None, figsize=None,
                    subtitle=None, maintext=None, show=False, verbose=1,
                    activate_slide=True, ppLayout=None, extranotes=None, background_color=None,
                    maximum_notes_size: int = 10000):
        """ Add slide to current active Powerpoint presentation.

        Arguments:
            title (str): title added to slide.
            fig (matplotlib.figure.Figure or qcodes.plots.pyqtgraph.QtPlot or integer):
                figure added to slide.
            txt (str) : Deprecated, use subtitle instead.
            notes (str or QCoDeS station): notes added to slide.
            figsize (list): size (width,height) of figurebox to add to powerpoint.
            subtitle (str): text added to slide as subtitle.
            maintext (str): text in textbox added to slide.
            show (bool): shows the powerpoint application.
            verbose (int): print additional information.
            activate_slide (bool): activate the current slide.
            ppLayout (int): layout of PP-slide (TitleOnly = 11, Text = 2).
            extranotes (str): notes for slide.
            background_color (None or tuple): background color for the slide.
            maximum_notes_size: Maximum size of the notes in number of characters

        Returns:
            ppt: PowerPoint presentation.
            slide: PowerPoint slide.

        The interface to Powerpoint used is described here:
            https://msdn.microsoft.com/en-us/library/office/ff743968.aspx

        Example:
            >>> title = 'An example title'
            >>> fig = plt.figure(10)
            >>> txt = 'Some comments on the figure'
            >>> notes = 'some additional information'
            >>> addPPTslide(title,fig, subtitle = txt,notes = notes)
        """
        Application = win32com.client.Dispatch("PowerPoint.Application")

        if verbose >= 2:
            print('Number of open PPTs: %d.' % Application.presentations.Count)

        try:
            ppt = Application.ActivePresentation
        except Exception:
            print('Could not open active Powerpoint presentation, opening blank presentation.')
            try:
                ppt = Application.Presentations.Add()
            except Exception as ex:
                warnings.warn('Could not make connection to Powerpoint presentation.')
                return None, None

        if show:
            Application.Visible = True  # shows what's happening, not required, but helpful for now

        if verbose >= 2:
            print('addPPTslide: presentation name: %s' % ppt.Name)

        ppLayoutTitleOnly = 11
        ppLayoutText = 2

        if txt is not None:
            if subtitle is None:
                warnings.warn('please do not use the txt field any more')
                subtitle = txt
            else:
                raise ValueError('please do not use the txt field any more')

            txt = None

        if fig is None:
            # no figure, text box over entire page
            if ppLayout is None:
                ppLayout = ppLayoutText
        else:
            # we have a figure, assume textbox is for dataset name only
            ppLayout = ppLayoutTitleOnly

        max_slides_count_warning = 750
        max_slides_count = 950
        maximum_notes_size = int(maximum_notes_size)

        if ppt.Slides.Count > max_slides_count_warning:
            warning_message = "Your presentation has more than {} slides! \
                Please start a new measurement logbook.".format(max_slides_count_warning)
            warnings.warn(warning_message)
        if ppt.Slides.Count > max_slides_count:
            error_message = "Your presentation has more than {} slides! \
                Please start a new measurement logbook.".format(max_slides_count)
            raise MemoryError(error_message)

        if verbose:
            print('addPPTslide: presentation name: %s, adding slide %d' %
                  (ppt.Name, ppt.Slides.count + 1))

        slide = ppt.Slides.Add(ppt.Slides.Count + 1, ppLayout)

        if background_color is not None:
            set_ppt_slide_background(slide, background_color, verbose=verbose)

        if fig is None:
            titlebox = slide.shapes.Item(1)
            mainbox = slide.shapes.Item(2)
            if maintext is None:
                raise TypeError('maintext argument is None')
            mainbox.TextFrame.TextRange.Text = maintext
        else:
            titlebox = slide.shapes.Item(1)
            mainbox = None
            if maintext is not None:
                warnings.warn('maintext not implemented when figure is set')

        if title is not None:
            slide.shapes.title.textframe.textrange.text = title
        else:
            slide.shapes.title.textframe.textrange.text = 'QCoDeS measurement'

        from qtt.measurements.videomode import VideoMode  # import here, to prevent default imports of gui code

        if fig is not None:
            fname = tempfile.mktemp(prefix='qcodesimageitem', suffix='.png')
            if isinstance(fig, matplotlib.figure.Figure):
                fig.savefig(fname)
            elif isinstance(fig, int):
                fig = plt.figure(fig)
                fig.savefig(fname)
            elif isinstance(fig, VideoMode) or \
                    fig.__class__.__name__ == 'VideoMode':
                if isinstance(fig.lp, list):
                    # do NOT change this into a list comprehension
                    ff = []
                    for jj in range(len(fig.lp)):
                        ff.append(fig.lp[jj].plotwin.grab())

                    sz = ff[0].size()
                    sz = QtCore.QSize(sz.width() * len(ff), sz.height())
                    figtemp = QtGui.QPixmap(sz)
                    p = QtGui.QPainter(figtemp)
                    offset = 0
                    for ii in range(len(ff)):
                        p.drawPixmap(offset, 0, ff[ii])
                        offset += ff[ii].size().width()
                    figtemp.save(fname)
                    p.end()
                else:
                    figtemp = fig.lp.plotwin.grab()
                    figtemp.save(fname)
            elif isinstance(fig, QtWidgets.QWidget):
                # generic method
                figtemp = fig.plotwin.grab()
                figtemp.save(fname)
            elif isinstance(fig, QtWidgets.QWidget):
                try:
                    figtemp = QtGui.QPixmap.grabWidget(fig)
                except BaseException:
                    figtemp = fig.grab()
                figtemp.save(fname)
            elif isinstance(fig, qcodes.plots.pyqtgraph.QtPlot):
                fig.save(fname)
            else:
                if verbose:
                    raise TypeError('figure is of an unknown type %s' % (type(fig),))
            top = 120

            left, top, width, height = _ppt_determine_image_position(ppt, figsize, fname, verbose=verbose>=2)

            if verbose >= 2:
                print('fname %s' % fname)
            slide.Shapes.AddPicture(FileName=fname, LinkToFile=False,
                                    SaveWithDocument=True, Left=left, Top=top, Width=width, Height=height)

        if subtitle is not None:
            # add subtitle
            subtitlebox = slide.Shapes.AddTextbox(
                1, Left=100, Top=80, Width=500, Height=300)
            subtitlebox.Name = 'subtitle box'
            subtitlebox.TextFrame.TextRange.Text = subtitle

        if notes is None:
            warnings.warn(
                'please set notes for the powerpoint slide. e.g. use the station or reshape_metadata')

        if isinstance(notes, qcodes.Station):
            station = notes
            gates = getattr(station, 'gates', None)
            notes = reshape_metadata(station, printformat='s', add_scanjob=True)
            if extranotes is not None:
                notes = '\n' + extranotes + '\n' + notes
            if gates is not None:
                notes = 'gates: ' + str(gates.allvalues()) + '\n\n' + notes
        elif isinstance(notes, DataSet):
            notes = reshape_metadata(notes, printformat='s', add_gates=True)

        if not isinstance(notes, str):
            warnings.warn(f'type of notes argument is {type(notes)}, converting to string')
            notes = str(notes)

        if notes is not None:
            if notes == '':
                notes = ' '
            if len(notes) > maximum_notes_size:
                warnings.warn(f'notes for powerpoint are {len(notes)} characters, reducing to {maximum_notes_size}')
                notes = notes[:maximum_notes_size]

            slide.notespage.shapes.placeholders[
                2].textframe.textrange.insertafter(notes)

        if activate_slide:
            idx = int(slide.SlideIndex)
            if verbose >= 2:
                print('addPPTslide: goto slide %d' % idx)
            Application.ActiveWindow.View.GotoSlide(idx)
        return ppt, slide

    def addPPT_dataset(dataset, title=None, notes=None,
                       show=False, verbose=1, paramname='measured',
                       printformat='fancy', customfig=None, extranotes=None, **kwargs):
        """ Add slide based on dataset to current active Powerpoint presentation.

        Args:
            dataset (DataSet): data and metadata from DataSet added to slide.
            customfig (QtPlot): custom QtPlot object to be added to
                                slide (for dataviewer).
            notes (string): notes added to slide.
            show (bool): shows the powerpoint application.
            verbose (int): print additional information.
            paramname (None or str): passed to dataset.default_parameter_array.
            printformat (string): 'fancy' for nice formatting or 'dict'
                                  for easy copy to python.
        Returns:
            ppt: PowerPoint presentation.
            slide: PowerPoint slide.

        Example:
        -------
        >>> notes = 'some additional information'
        >>> addPPT_dataset(dataset,notes)
        """
        if len(dataset.arrays) < 2:
            raise IndexError('The dataset contains less than two data arrays')

        if customfig is None:

            if isinstance(paramname, str):
                if title is None:
                    parameter_name = dataset.default_parameter_name(paramname=paramname)
                    title = 'Parameter: %s' % parameter_name
                temp_fig = QtPlot(dataset.default_parameter_array(
                    paramname=paramname), show_window=False)
            else:
                if title is None:
                    title = 'Parameter: %s' % (str(paramname),)
                for idx, parameter_name in enumerate(paramname):
                    if idx == 0:
                        temp_fig = QtPlot(dataset.default_parameter_array(
                            paramname=parameter_name), show_window=False)
                    else:
                        temp_fig.add(dataset.default_parameter_array(
                            paramname=parameter_name))

        else:
            temp_fig = customfig

        text = 'Dataset location: %s' % dataset.location
        if notes is None:
            try:
                metastring = reshape_metadata(dataset,
                                              printformat=printformat)
            except Exception as ex:
                metastring = 'Could not read metadata: %s' % str(ex)
            notes = 'Dataset %s metadata:\n\n%s' % (dataset.location,
                                                    metastring)
            scanjob = dataset.metadata.get('scanjob', None)
            if scanjob is not None:
                s = pprint.pformat(scanjob)
                notes = 'scanjob: ' + str(s) + '\n\n' + notes

            gatevalues = dataset.metadata.get('allgatevalues', None)
            if gatevalues is not None:
                notes = 'gates: ' + str(gatevalues) + '\n\n' + notes

        ppt, slide = addPPTslide(title=title, fig=temp_fig, subtitle=text,
                                 notes=notes, show=show, verbose=verbose,
                                 extranotes=extranotes,
                                 **kwargs)
        return ppt, slide

except ImportError:
    def addPPTslide(title=None, fig=None, txt=None, notes=None, figsize=None,
                    subtitle=None, maintext=None, show=False, verbose=1,
                    activate_slide=True, ppLayout=None, extranotes=None, background_color=None,
                    maximum_notes_size: int = 10000):
        """ Add slide to current active Powerpoint presentation.

        Dummy implementation.
        """
        warnings.warn(
            'addPPTslide is not available on your system. Install win32com from https://pypi.org/project/pypiwin32/.')

    def addPPT_dataset(dataset, title=None, notes=None,
                       show=False, verbose=1, paramname='measured',
                       printformat='fancy', customfig=None, extranotes=None, **kwargs):
        """ Add slide based on dataset to current active Powerpoint presentation.

        Dummy implementation.
        """
        warnings.warn(
            'addPPT_dataset is not available on your system. Install win32com from https://pypi.org/project/pypiwin32/.')

# %%

def reshape_metadata(dataset, printformat='dict', add_scanjob=True, add_gates=True, add_analysis_results=True, verbose=0):
    """ Reshape the metadata of a DataSet.

    Args:
        dataset (DataSet or qcodes.Station): a dataset of which the metadata will be reshaped.
        printformat (str): can be 'dict' or 'txt','fancy' (text format).
        add_scanjob (bool): If True, then add the scanjob at the beginning of the notes.
        add_analysis_results (bool): If True, then add the analysis_results at the beginning of the notes.
        add_gates (bool): If True, then add the scanjob at the beginning of the notes.
        verbose (int): verbosity (0 == silent).

    Returns:
        str: the reshaped metadata.

    """
    if isinstance(dataset, qcodes.Station):
        station = dataset
        all_md = station.snapshot(update=False)['instruments']
        header = None
    else:
        tmp = dataset.metadata.get('station', None)
        if tmp is None:
            all_md = {}
        else:
            all_md = tmp['instruments']

        header = 'dataset: %s' % dataset.location

        if hasattr(dataset.io, 'base_location'):
            header += ' (base %s)' % dataset.io.base_location

    if add_gates:
        gate_values = dataset.metadata.get('allgatevalues', None)

        if gate_values is not None:
            gate_values = dict([(key, np.around(value, 3)) for key, value in gate_values.items()])
            header += '\ngates: ' + str(gate_values) + '\n'

    scanjob = dataset.metadata.get('scanjob', None)
    if scanjob is not None and add_scanjob:
        s = pprint.pformat(scanjob)
        header += '\n\nscanjob: ' + str(s) + '\n'

    analysis_results = dataset.metadata.get('analysis_results', None)
    if analysis_results is not None and add_analysis_results:
        s = pprint.pformat(analysis_results)
        header += '\n\analysis_results: ' + str(s) + '\n'

    metadata = OrderedDict()
    # make sure the gates instrument is in front
    all_md_keys = sorted(sorted(all_md), key=lambda x: x == 'gates', reverse=True)
    for x in all_md_keys:
        metadata[x] = OrderedDict()
        if 'IDN' in all_md[x]['parameters']:
            metadata[x]['IDN'] = dict({'name': 'IDN', 'value': all_md[
                x]['parameters']['IDN']['value']})
            metadata[x]['IDN']['unit'] = ''
        for y in sorted(all_md[x]['parameters'].keys()):
            try:
                if y != 'IDN':
                    metadata[x][y] = OrderedDict()
                    param_md = all_md[x]['parameters'][y]
                    metadata[x][y]['name'] = y
                    if isinstance(param_md['value'], (float, np.float64)):
                        metadata[x][y]['value'] = float(
                            format(param_md['value'], '.3f'))
                    else:
                        metadata[x][y]['value'] = str(param_md['value'])
                    metadata[x][y]['unit'] = param_md.get('unit', None)
                    metadata[x][y]['label'] = param_md.get('label', None)
            except KeyError as ex:
                if verbose:
                    print('failed on parameter %s / %s: %s' % (x, y, str(ex)))

    if printformat == 'dict':
        ss = str(metadata).replace('(', '').replace(
            ')', '').replace('OrderedDict', '')
    else:  # 'txt' or 'fancy'
        ss = ''
        for k in metadata:
            if verbose:
                print('--- %s' % k)
            s = metadata[k]
            ss += '\n## %s:\n' % k
            for p in s:
                pp = s[p]
                if verbose:
                    print('  --- %s: %s' % (p, pp.get('value', '??')))
                ss += '%s: %s (%s)' % (pp['name'],
                                       pp.get('value', '?'), pp.get('unit', ''))
                ss += '\n'

    if header is not None:
        ss = header + '\n\n' + ss
    return ss

# %%

def setupMeasurementWindows(*args, **kwargs):
    raise Exception('use qtt.gui.live_plotting.setupMeasurementWindows instead')

def updatePlotTitle(qplot, basetxt='Live plot'):
    """ Update the plot title of a QtPlot window."""
    txt = basetxt + ' (%s)' % time.asctime()
    qplot.win.setWindowTitle(txt)

# %%

def flatten(lst):
    """ Flatten a list.

    Args:
        lst (list): list to be flattened.

    Returns:
        list: flattened list.

    Example:
        >>> flatten([ [1,2], [3,4], [10] ])
        [1, 2, 3, 4, 10]
    """
    return list(chain(*lst))

# %%

def cutoffFilter(x, thr, omega):
    """ Smooth cutoff filter.

    Filter definition from: http://paulbourke.net/miscellaneous/imagefilter/

    Example
    -------
    >>> plt.clf()
    >>> x=np.arange(0, 4, .01)
    >>> _=plt.plot(x, cutoffFilter(x, 2, .25), '-r')

    """
    y = .5 * (1 - np.sin(np.pi * (x - thr) / (2 * omega)))
    y[x < thr - omega] = 1
    y[x > thr + omega] = 0
    return y

# %%

def smoothFourierFilter(fs=100, thr=6, omega=2, fig=None):
    """ Create smooth ND filter for Fourier high or low-pass filtering.

    Example
    -------
    >>> F=smoothFourierFilter([24,24], thr=6, omega=2)
    >>> _=plt.figure(10); plt.clf(); _=plt.imshow(F, interpolation='nearest')
    """
    rr = np.meshgrid(*[range(f) for f in fs])

    x = np.dstack(rr)
    x = x - (np.array(fs) / 2 - .5)
    x = np.linalg.norm(x, axis=2)

    F = cutoffFilter(x, thr, omega)

    if fig is not None:
        plt.figure(10)
        plt.clf()
        plt.imshow(F, interpolation='nearest')

    return F

F = smoothFourierFilter([36, 36])

# %%

def fourierHighPass(imx, nc=40, omega=4, fs=1024, fig=None):
    """ Implement simple high pass filter using the Fourier transform."""
    f = np.fft.fft2(imx, s=[fs, fs])  # do the fourier transform

    fx = np.fft.fftshift(f)

    if fig:
        plt.figure(fig)
        plt.clf()
        plt.imshow(np.log(np.abs(f) + 1), interpolation='nearest')
        plt.title('Fourier spectrum (real part)')
        plt.figure(fig + 1)
        plt.clf()
        plt.imshow(np.log(np.abs(fx) + 1), interpolation='nearest')
        plt.title('Fourier spectrum (real part)')

    if nc > 0 and omega == 0:
        f[0:nc, 0:nc] = 0
        f[-nc:, -nc:] = 0
        f[-nc:, 0:nc] = 0
        f[0:nc, -nc:] = 0
        img_back = np.fft.ifft2(f)  # inverse fourier transform

    else:
        # smooth filtering

        F = 1 - smoothFourierFilter(fx.shape, thr=nc, omega=omega)
        fx = F * fx
        ff = np.fft.ifftshift(fx)  # inverse shift
        img_back = np.fft.ifft2(ff)  # inverse fourier transform

    imf = img_back.real
    imf = imf[0:imx.shape[0], 0:imx.shape[1]]

    return imf

# %%

def slopeClick(drawmode='r--', **kwargs):
    """ Calculate slope for line piece of two points clicked by user. Works
    with matplotlib but not with pyqtgraph. Uses the currently active figure.

    Args:
        drawmode (string): plotting style.

    Returns:
        coords (2 x 2 array): coordinates of the two clicked points.
        signedslope (float): slope of linepiece connecting the two points.

    """
    ax = plt.gca()
    ax.set_autoscale_on(False)
    coords = qtt.pgeometry.ginput(2, drawmode, **kwargs)
    plt.pause(1e-6)
    signedslope = (coords[1, 0] - coords[1, 1]) / (coords[0, 0] - coords[0, 1])

    return coords, signedslope

def clickGatevals(plot, drawmode='ro'):
    """ Get gate values for all gates at clicked point in a heatmap.

    Args:
        plot (qcodes MatPlot object): plot of measurement data.
        drawmode (string): plotting style.

    Returns:
        gatevals (dict): values of the gates at clicked point.

    """
    # TODO: implement for virtual gates
    if not isinstance(plot, qcodes.plots.qcmatplotlib.MatPlot):
        raise Exception(
            'The plot object is not based on the MatPlot class from qcodes.')

    ax = plt.gca()
    ax.set_autoscale_on(False)
    coords = qtt.pgeometry.ginput(drawmode=drawmode)
    data_array = plot.traces[0]['config']['z']
    dataset = data_array.data_set

    gatevals = copy.deepcopy(dataset.metadata['allgatevalues'])
    if len(data_array.set_arrays) != 2:
        raise Exception('The DataArray does not have exactly two set_arrays.')

    for arr in data_array.set_arrays:
        if len(arr.ndarray.shape) == 1:
            gatevals[arr.name] = coords[1, 0]
        else:
            gatevals[arr.name] = coords[0, 0]

    return gatevals

# %%

def connect_slot(target):
    """ Create a slot by dropping signal arguments."""

    def signal_drop_arguments(*args, **kwargs):
        target()

    return signal_drop_arguments

max_stars_repo_name: eendebakpt/qtt

max_issues_repo_name: eendebakpt/qtt

max_forks_repo_name: eendebakpt/qtt

reason: import numpy,import scipy


text:
""" This script contains the implementation of the MAML algorithms designed by
Chelsea Finn et al. (https://arxiv.org/pdf/1703.03400).
Terminology:
------------
Support set : a set of training examples
    (inputs+labels: iterable of (img, label) pairs)
Query set : a set of test examples
    (inputs +labels : iterable of (img, label) pairs )
Task/Dataset : Support set + Query set.
Meta-train set: a set of datasets for meta-training
Meta-test set: a set of datasets for meta-evaluation
Meta-batch size: Number of tasks to consider for a meta-iteration
"""
import time
import copy
import logging
import datetime
import pickle
import numpy as np
import os
import subprocess

import gin
import torch
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.python import debug as tf_debug
from tensorflow.keras.models import clone_model
from tensorflow.keras.layers import Dense, Conv2D

from turtle_code import Turtle
from helper import ConvX
from metadl.api.api import MetaLearner, Learner, Predictor
from utils import create_grads_shell, reset_grads, app_custom_grads

# https://discuss.pytorch.org/t/it-there-anyway-to-let-program-select-free-gpu-automatically/17560/7
def set_device():
    """Automatically chooses the right device to run pytorch on

    Returns:
        str: device identifier which is best suited (most free GPU, or CPU in case GPUs are unavailable)
    """
    if torch.cuda.is_available():
        os.system('nvidia-smi -q -d Memory |grep -A4 GPU|grep Free >tmp')
        memory_available = [int(x.split()[2]) for x in open('tmp', 'r').readlines()]
        gpu_id = int(np.argmax(memory_available))
        print("trying gpu_id", gpu_id)
        torch.cuda.set_device(gpu_id)
        dev = torch.cuda.current_device()
    else:
        dev = "cpu"
    return dev

TURTLE_CONF = {
    "opt_fn": torch.optim.Adam,
    "T": 5,
    "T_val": 5,
    "T_test": 5,
    "lr": 0.001,
    "act": torch.nn.ReLU(),
    "beta": 0.9,
    "meta_batch_size": 2,
    "time_input": False,
    "decouple": None,
    "input_type": "raw_grads",
    "second_order": False,
    "layer_wise": False,
    "param_lr": True,
    "history": "grads",
    "grad_clip": 10,
    "dev": set_device(),
    "batching_eps": False,
    "test_adam": False,
    "baselearner_fn": ConvX,
    "layers": [20,20,20,20,20,1]
}

@gin.configurable
class MyMetaLearner(MetaLearner):
    def __init__(self,
                meta_iterations,
                meta_batch_size,
                support_batch_size,
                query_batch_size,
                img_size,
                N_ways):
        """
        Args:
            meta_iterations : number of meta-iterations to perform, i.e. the
            number of times the meta-learner's weights are updated.

            meta_batch_size : The number of (learner, task) pairs that are used
            to produce the meta-gradients, used to update the meta-learner's
            weights after each meta-iteration.

            support_batch_size : The batch size for the support set.
            query_batch_size : The batch size for the query set.
            img_size : Integer, images are considered to be
                        (img_size, img_size, 3)
        """
        super().__init__()
        self.meta_iterations = meta_iterations
        self.meta_batch_size = meta_batch_size
        self.support_batch_size = support_batch_size
        self.query_batch_size = query_batch_size
        self.img_size = img_size
        self.N_ways = N_ways
        TURTLE_CONF["dev"] = set_device()
        self.device = TURTLE_CONF["dev"]

        self.baselearner_args = {
            "dev": self.device,
            "train_classes": self.N_ways,
            "eval_classes": self.N_ways,
            "criterion":nn.CrossEntropyLoss(),
            "num_blocks": 4,
            "img_size": (1, 3, self.img_size, self.img_size)
        }

        TURTLE_CONF["baselearner_args"] = self.baselearner_args
        TURTLE_CONF["train_batch_size"] = self.N_ways * self.support_batch_size
        TURTLE_CONF["test_batch_size"] = self.N_ways * self.query_batch_size
        torch.manual_seed(1234)
        if torch.cuda.is_available():
            torch.cuda.manual_seed_all(1234)
        np.random.seed(1234)

        print("before initializing TURTLE:", subprocess.check_output(['nvidia-smi']).decode("utf-8"))
        self.turtle = Turtle(**TURTLE_CONF)
        print("-"*50)
        print("after initializing TURTLE:", subprocess.check_output(['nvidia-smi']).decode("utf-8"))

    def dataloader(self, dataset_episodic):
        to_torch_labels = lambda a: torch.from_numpy(a.numpy()).long()
        to_torch_imgs = lambda a: torch.from_numpy(np.transpose(a.numpy(), (0, 3, 1, 2)))
        # 2

        def data_loader(n_batches):
            for i, (e, _) in enumerate(dataset_episodic):
                if i == n_batches:
                    break
                logging.info('e shape: {}'.format(len(e)))
                yield (to_torch_imgs(e[0]), to_torch_labels(e[1]),
                    to_torch_imgs(e[3]), to_torch_labels(e[4]))

        datal = data_loader(n_batches=1)
        for i, batch in enumerate(datal):
            #3
            data_support, labels_support, data_query, labels_query = [x.to(device=self.device) for x in batch]
            logging.info('Supp imgs: {} | Supp labs : {} | Query imgs : {} | Query labs \n \n'.format(data_support.shape, labels_support.shape, data_query.shape, labels_query.shape))

    def process_task(self, batch):
        """
        batch : [sup_imgs, sup_labs, sup_tidx, qry_imgs, qry_labs, qry_tidx]
        sup_imgs : [batch_idx, nbr_imgs, H, W, C]
        """
        to_torch_labels = lambda a: torch.from_numpy(a.numpy()).long()
        to_torch_imgs = lambda a: torch.from_numpy(np.transpose(a.numpy(), (0, 1, 4, 2, 3)))
        return (to_torch_imgs(batch[0]), to_torch_labels(batch[1]),
                    to_torch_imgs(batch[3]), to_torch_labels(batch[4]))

    def meta_fit(self, meta_dataset_generator):
        """ Encapsulates the meta-learning procedure. In the fo-MAML case,
        the meta-learner's weights update.

        Args:
            meta_dataset_generator : a DataGenerator object. We can access
                the meta-train and meta-validation episodes via its attributes.
                Refer to the metadl/data/dataset.py for more details.
        Returns:
            A Learner object initialized with the meta-learner's weights.
        """
        # Load dataset in db
        meta_train_dataset = meta_dataset_generator.meta_train_pipeline
        meta_valid_dataset = meta_dataset_generator.meta_valid_pipeline
        meta_train_dataset = meta_train_dataset.batch(1)
        meta_valid_dataset = meta_valid_dataset.batch(1)
        mtrain_iterator = meta_train_dataset.__iter__()
        mval_iterator = meta_valid_dataset.__iter__()

        print("start of ingestion (creating dataset objects):", subprocess.check_output(['nvidia-smi']).decode("utf-8"))
        log = []
        start_time = time.time()

        for meta_iter in range(self.meta_iterations):
            if meta_iter % 2500 == 0:
                scores = []
                # Perform meta-validation and save model state
                for val_iter in range(2):
                    vbatch = next(mval_iterator)[0]
                    vbatch = self.process_task(vbatch)
                    vx_spt, vy_spt, vx_qry, vy_qry = [x[0].to(device=self.device) for x in vbatch]
                    # print("val:", vy_spt.size(), vy_qry.size())
                    vacc = self.turtle.evaluate(vx_spt, vy_spt, vx_qry, vy_qry)
                    scores.append(vacc)
                    # del vx_spt; del vy_spt; del vx_qry; del vy_qry
                    # if torch.cuda.is_available():
                    #     torch.cuda.empty_cache()
                val_accuracy = np.mean(scores)
                iter_time = time.time() - start_time
                logging.info(f'[Meta-iteration {meta_iter}] Val acc: {val_accuracy:.3f} | Time: {iter_time:.2f}')
                self.turtle.store_file(os.path.join('trained_models/feedback/turtle/models', f'{meta_iter}.pkl'))

            batch = next(mtrain_iterator)[0]
            batch = self.process_task(batch)
            x_spt, y_spt, x_qry, y_qry = [x[0].to(device=self.device) for x in batch]
            #print("train", y_spt.size(), y_qry.size())
            self.turtle.train(x_spt, y_spt, x_qry, y_qry)
            break

        return MyLearner(self.turtle, self.device)

@gin.configurable
class MyLearner(Learner):
    """ In the case of fo-MAML, encapsulates a neural network and its training
    methods.
    """
    def __init__(self,
                meta_iterations,
                meta_batch_size,
                support_batch_size,
                query_batch_size,
                img_size,
                N_ways,
                turtle=None,
                device=None):
        """
        Args:
            neural_net : a keras.Sequential object. A neural network model to
                        copy as Learner.
            num_epochs : Integer, the number of epochs to consider for the
                        training on support examples.
            lr : Float, the learning rate associated to the learning procedure
                (Adaptation).
            img_size : Integer, images are considered to be
                        (img_size,img_size,3)
        """
        super().__init__()
        self.meta_iterations = meta_iterations
        self.meta_batch_size = meta_batch_size
        self.support_batch_size = support_batch_size
        self.query_batch_size = query_batch_size
        self.img_size = img_size
        self.N_ways = N_ways

        if turtle == None:
            self.device = set_device()

            self.baselearner_args = {
                "dev": self.device,
                "train_classes": self.N_ways,
                "eval_classes": self.N_ways,
                "criterion":nn.CrossEntropyLoss(),
                "num_blocks": 4,
                "img_size": (1, 3, self.img_size, self.img_size)
            }

            TURTLE_CONF["dev"] = self.device
            TURTLE_CONF["baselearner_args"] = self.baselearner_args
            TURTLE_CONF["train_batch_size"] = self.N_ways * self.support_batch_size
            TURTLE_CONF["test_batch_size"] = self.N_ways * self.query_batch_size
            self.turtle = Turtle(**TURTLE_CONF)
        else:
            self.turtle = turtle
            self.device = self.turtle.dev

    # def __call__(self, imgs):
    #     return self.learner(imgs)

    def process_task(self, images, labels):
        """
        batch : [sup_imgs, sup_labs, sup_tidx, qry_imgs, qry_labs, qry_tidx]
        sup_imgs : [batch_idx, nbr_imgs, H, W, C]
        """
        to_torch_labels = lambda a: torch.from_numpy(a.numpy()).long()
        to_torch_imgs = lambda a: torch.from_numpy(np.transpose(a.numpy(), (0, 3, 1, 2)))
        return to_torch_imgs(images), to_torch_labels(labels)

    def fit(self, dataset_train):
        """ The learner's fit function over the train set of a task.

        Args:
            dataset_train : a tf.data.Dataset object. Iterates over the training
                            examples (support set).
        Returns:
            predictor : An instance of MyPredictor that is initilialized with
                the fine-tuned learner's weights in this case.
        """
        for images, labels in dataset_train:
            images, labels = self.process_task(images, labels)
            images, labels = images.to(self.device), labels.to(self.device)
            network, parameters = self.turtle.evaluate(images, labels, None, None, return_network=True)
            predictor = MyPredictor(network, parameters)
            break
        return predictor

    def load(self, model_dir):
        """Loads the learner model from a pickle file.

        Args:
            model_dir: the directory name in which the participant's code and
                their saved/serialized learner are stored.
        """

        if(os.path.isdir(model_dir) != True):
            raise ValueError(('The model directory provided is invalid. Please'
                + ' check that its path is valid.'))

        ckpt_path = os.path.join(model_dir, 'learner.pt')
        self.turtle.read_file(ckpt_path)

    def save(self, model_dir):
        """Saves the learner model into a pickle file.

        Args:
            model_dir: the directory name from which the participant's code and
                their saved/serialized learner are loaded.
        """

        if(os.path.isdir(model_dir) != True):
            os.mkdir(model_dir)

        ckpt_file = os.path.join(model_dir, 'learner.pt')
        self.turtle.store_file(ckpt_file)

####### Predictor ########
@gin.configurable
class MyPredictor(Predictor):
    """ The predictor is meant to predict labels of the query examples at
    meta-test time.
    """
    def __init__(self,
                 network,
                 parameters):
        """
        Args:
            learner : a MyLearner object that encapsulates the fine-tuned
                neural network.
        """
        super().__init__()
        self.network = network
        self.parameters = [x.clone().detach() for x in parameters]
        self.device = self.parameters[0].device

    def process_imgs(self, images):
        to_torch_imgs = lambda a: torch.from_numpy(np.transpose(a.numpy(), (0, 3, 1, 2)))
        return to_torch_imgs(images)

    def predict(self, dataset_test):
        """ Predicts labels of the query set examples associated to a task.
        Note that the query set is a tf.data.Dataset containing 50 examples for
        the Omniglot dataset.

        Args:
            dataset_test : a tf.data.Dataset object. An iterator over the
                unlabelled query examples.
        Returns:
            preds : tensors, shape (num_examples, N_ways). We are using the
                Sparse Categorical Accuracy to evaluate the predictions. Valid
                tensors can take 2 different forms described below.

        Case 1 : The i-th prediction row contains the i-th example logits.
        Case 2 : The i-th prediction row contains the i-th example
                probabilities.

        Since in both cases the SparseCategoricalAccuracy behaves the same way,
        i.e. taking the argmax of the row inputs, both forms are valid.

        Note : In the challenge N_ways = 5 at meta-test time.
        """
        self.network.eval()
        for images in dataset_test:
            images = self.process_imgs(images[0]).to(self.device)
            qry_logits = self.network.forward_weights(images, self.parameters).detach()
        return qry_logits

max_stars_repo_name: Intellikz/metadl

max_issues_repo_name: Intellikz/metadl

max_forks_repo_name: Intellikz/metadl


text:
# Copyright 2020 The Trieste Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import gpflow
import numpy.testing as npt
import pytest
import tensorflow as tf

from trieste.acquisition.rule import (
    AcquisitionRule,
    EfficientGlobalOptimization,
    ThompsonSampling,
    TrustRegion,
    OBJECTIVE
)
from trieste.bayesian_optimizer import BayesianOptimizer
from trieste.data import Dataset
from trieste.models import GaussianProcessRegression
from trieste.space import Box
from trieste.utils.objectives import (
    branin,
    BRANIN_GLOBAL_MINIMUM,
    BRANIN_GLOBAL_ARGMIN,
    mk_observer,
)

from tests.util.misc import random_seed

@random_seed
@pytest.mark.parametrize('num_steps, acquisition_rule', [
    (30, EfficientGlobalOptimization()),
    (22, TrustRegion()),
    (17, ThompsonSampling(500, 3)),
])
def test_optimizer_finds_minima_of_the_branin_function(
        num_steps: int, acquisition_rule: AcquisitionRule
) -> None:
    search_space = Box(tf.constant([0.0, 0.0], tf.float64), tf.constant([1.0, 1.0], tf.float64))

    def build_model(data: Dataset) -> GaussianProcessRegression:
        variance = tf.math.reduce_variance(data.observations)
        kernel = gpflow.kernels.Matern52(variance, tf.constant([0.2, 0.2], tf.float64))
        gpr = gpflow.models.GPR((data.query_points, data.observations), kernel, noise_variance=1e-5)
        gpflow.utilities.set_trainable(gpr.likelihood, False)
        return GaussianProcessRegression(gpr)

    initial_query_points = search_space.sample(5)
    observer = mk_observer(branin, OBJECTIVE)
    initial_data = observer(initial_query_points)
    model = build_model(initial_data[OBJECTIVE])

    res = BayesianOptimizer(
        observer, search_space
    ).optimize(
        num_steps, initial_data, {OBJECTIVE: model}, acquisition_rule
    )

    if res.error is not None:
        raise res.error

    dataset = res.datasets[OBJECTIVE]

    arg_min_idx = tf.squeeze(tf.argmin(dataset.observations, axis=0))

    best_y = dataset.observations[arg_min_idx]
    best_x = dataset.query_points[arg_min_idx]

    argmin = tf.cast(BRANIN_GLOBAL_ARGMIN, tf.float64)
    relative_argmin_err = tf.abs((best_x - argmin) / argmin)
    # these accuracies are the current best for the given number of optimization steps, which makes
    # this is a regression test
    assert tf.reduce_any(tf.reduce_all(relative_argmin_err < 0.03, axis=-1), axis=0)
    npt.assert_allclose(best_y, BRANIN_GLOBAL_MINIMUM, rtol=0.03)

max_stars_repo_name: st--/trieste

max_issues_repo_name: st--/trieste

max_forks_repo_name: st--/trieste


text:
import numpy as np

def collapse(j, data):
    """Collapses samples into a single row"""
    uniques = np.unique(j)
    return uniques, np.array([data[j == u].sum() for u in uniques])

def nearest(coords, shape, **kwargs):
    valid = ~(np.isnan(coords).all(1))
    valid = np.logical_and(valid, np.logical_and(coords[:,0] > -.5, coords[:,0] < shape[2]+.5))
    valid = np.logical_and(valid, np.logical_and(coords[:,1] > -.5, coords[:,1] < shape[1]+.5))
    valid = np.logical_and(valid, np.logical_and(coords[:,2] > -.5, coords[:,2] < shape[0]+.5))

    rcoords = coords[valid].round().astype(int)
    j = np.ravel_multi_index(rcoords.T[::-1], shape, mode='clip')
    #return np.nonzero(valid)[0], j, (rcoords > 0).all(1) #np.ones((valid.sum(),))
    return np.nonzero(valid)[0], j, np.ones((valid.sum(),))

def trilinear(coords, shape, **kwargs):
    #trilinear interpolation equation from http://paulbourke.net/miscellaneous/interpolation/
    valid = ~(np.isnan(coords).all(1))
    (x, y, z), floor = np.modf(coords[valid].T)
    floor = floor.astype(int)
    ceil = floor + 1
    x[x < 0] = 0
    y[y < 0] = 0
    z[z < 0] = 0

    i000 = np.array([floor[2], floor[1], floor[0]])
    i100 = np.array([floor[2], floor[1],  ceil[0]])
    i010 = np.array([floor[2],  ceil[1], floor[0]])
    i001 = np.array([ ceil[2], floor[1], floor[0]])
    i101 = np.array([ ceil[2], floor[1],  ceil[0]])
    i011 = np.array([ ceil[2],  ceil[1], floor[0]])
    i110 = np.array([floor[2],  ceil[1],  ceil[0]])
    i111 = np.array([ ceil[2],  ceil[1],  ceil[0]])

    v000 = (1-x)*(1-y)*(1-z)
    v100 = x*(1-y)*(1-z)
    v010 = (1-x)*y*(1-z)
    v110 = x*y*(1-z)
    v001 = (1-x)*(1-y)*z
    v101 = x*(1-y)*z
    v011 = (1-x)*y*z
    v111 = x*y*z

    i    = np.tile(np.nonzero(valid)[0], [1, 8]).ravel()
    j    = np.hstack([i000, i100, i010, i001, i101, i011, i110, i111])
    data = np.vstack([v000, v100, v010, v001, v101, v011, v110, v111]).ravel()
    return i, np.ravel_multi_index(j, shape, mode='clip'), data

def distance_func(func, coords, shape, renorm=True, mp=True):
    """Generates masks for seperable distance functions"""
    nZ, nY, nX = shape
    dx = coords[:,0] - np.atleast_2d(np.arange(nX)).T
    dy = coords[:,1] - np.atleast_2d(np.arange(nY)).T
    dz = coords[:,2] - np.atleast_2d(np.arange(nZ)).T

    Lx, Ly, Lz = func(dx), func(dy), func(dz)
    ix, jx = np.nonzero(Lx)
    iy, jy = np.nonzero(Ly)
    iz, jz = np.nonzero(Lz)
    ba = np.broadcast_arrays
    def func(v):
        mx, my, mz = ix[jx == v], iy[jy == v], iz[jz == v]
        idx, idy, idz = [i.ravel() for i in ba(*np.ix_(mx, my, mz))]
        vx, vy, vz = [i.ravel() for i in ba(*np.ix_(Lx[mx, v], Ly[my, v], Lz[mz, v]))]

        i = v * np.ones((len(idx,)))
        j = np.ravel_multi_index((idz, idy, idx), shape, mode='clip')
        data = vx*vy*vz
        if renorm:
            data /= data.sum()
        return i, j, data

    if mp:
        from .. import mp
        ijdata = mp.map(func, range(len(coords)))
    else:
        #ijdata = map(func, range(len(coords)))
        ijdata = [func(x) for x in range(len(coords))]

    return np.hstack(ijdata)

def gaussian(coords, shape, sigma=1, window=3, **kwargs):
    raise NotImplementedError
    def gaussian(x):
        pass
    return distance_func(gaussian, coords, shape, **kwargs)

def lanczos(coords, shape, window=3, **kwargs):
    def lanczos(x):
        out = np.zeros_like(x)
        sel = np.abs(x)<window
        selx = x[sel]
        out[sel] = np.sin(np.pi * selx) * np.sin(np.pi * selx / window) * (window / (np.pi**2 * selx**2))
        return out

    return distance_func(lanczos, coords, shape, **kwargs)

max_stars_repo_name: mvdoc/pycortex

max_issues_repo_name: mvdoc/pycortex

max_forks_repo_name: mvdoc/pycortex


text:
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import gflags  # sudo pip install python-gflags
import sys
import scipy as sp # sudo pip install spicy
from scipy import stats
import math

#gflags.DEFINE_string('cris', None, 'un parametro para testear', short_name='c')
gflags.DEFINE_string('size', None, 'The model size', short_name='s')
gflags.DEFINE_list('device_name', None, 'Broken device name', short_name='d') # pass devices name separated with ,
#gflags.DEFINE_string('size', 149, 'The model size', short_name='s')
gflags.MarkFlagAsRequired('size')
gflags.MarkFlagAsRequired('device_name')

FLAGS = gflags.FLAGS

try:
    argv = FLAGS(sys.argv)  # parse flags
except gflags.FlagsError, e:
    print '%s\nUsage: %s ARGS\n%s' % (e, sys.argv[0], FLAGS)
    sys.exit(1)

#print "anduvo", FLAGS.size

df_Command = pd.DataFrame()
df_Devices = pd.DataFrame()
df_Task = pd.DataFrame()
for prob in range(10, 100, 10):
    nameCommand = "datafiles/ModelSize"+str(FLAGS.size)+"devices"+str(len(FLAGS.device_name))
    nameTask = "datafiles/ModelSize"+str(FLAGS.size)+"devices"+str(len(FLAGS.device_name))
    nameDevices = "datafiles/ModelSize"+str(FLAGS.size)+"devices"+str(len(FLAGS.device_name))
    for i in range(len(FLAGS.device_name)):
    	nameCommand += "_"+str(prob)+str(FLAGS.device_name[i])
    	nameTask += "_"+str(prob)+str(FLAGS.device_name[i])
    	nameDevices += "_"+str(prob)+str(FLAGS.device_name[i])
    nameCommand += "command.txt"
    nameTask += "task.txt"
    nameDevices += "device.txt"
    #print nameCommand
    #print nameTask
    #print nameDevices
    df_new_Command = pd.read_csv(nameCommand, sep=" ")
    df_new_Command['probability'] = [prob for i in range(len(df_new_Command))]
    df_Command = pd.concat([df_Command, df_new_Command])
    df_new_Devices = pd.read_csv(nameDevices, sep=" ")
    df_new_Devices['probability'] = [prob for i in range(len(df_new_Devices))]
    df_Devices = pd.concat([df_Devices, df_new_Devices])
    df_new_Task = pd.read_csv(nameTask, sep=" ")
    df_new_Task['probability'] = [prob for i in range(len(df_new_Task))]
    df_Task = pd.concat([df_Task, df_new_Task])

#print df_Command
#print df_Devices
#print df_Task

###COMMANDS RECEIVED PER PERSON
CoomandNames = map(lambda x: str(x), df_Command.groupby(['Command']).indices.keys())

for Command in range(len(CoomandNames)):
	df_data = pd.DataFrame()
	data = []
	labels = []
	xtitle = ""
 	command_temp = CoomandNames[Command]
 	conf_interval = []
 	sample_size = []
 	data_mean = []
 	data_std = []
 	#z_critical = stats.norm.ppf(q = 0.995)  # Get the z-critical value* for 99%
 	z_critical = stats.norm.ppf(q = 0.975)  # Get the z-critical value* for 99%
	print("z-critical value:" + str(z_critical))              # Check the z-critical value

	for prob in range(10, 100, 10):
		df_data[["#Persons"+str(prob),"Command"+str(prob)]] = df_Command[df_Command["probability"] == prob][["#Persons","Command"]]
		data.append(df_data[df_data["Command"+str(prob)] == command_temp]["#Persons"+str(prob)])
		labels.append(prob)
		sample_size.append(len(df_data[df_data["Command"+str(prob)] == command_temp]["#Persons"+str(prob)]))
		data_mean.append((df_data[df_data["Command"+str(prob)] == command_temp]["#Persons"+str(prob)]).mean())
		data_std.append((df_data[df_data["Command"+str(prob)] == command_temp]["#Persons"+str(prob)]).std())
		#print "sample size: "
		#print sample_size
	for i in range(len(data_mean)):
		#print "data i:" +str(data_mean[i]) + " " + str(data_std[i])
		margin_of_error = z_critical * (data_std[i]/math.sqrt(sample_size[i]))
		conf = [data_mean[i] - margin_of_error, data_mean[i] + margin_of_error]
		conf_interval.append(conf)
	print conf_interval
	#print data
	plt.boxplot(data, labels = labels, showmeans = True, notch = True, conf_intervals=conf_interval)
	plt.suptitle(command_temp, fontsize=14, fontweight='bold')
	for i in range(len(FLAGS.device_name)):
 		xtitle += " " +str(FLAGS.device_name[i])
	plt.xlabel("Failure probability of devices:" +xtitle, fontsize=12)
	plt.ylabel('Total number of persons that receives the command', fontsize=12)
	plt.ylim(bottom=-1)
	#plt.show()
	for i in range(len(FLAGS.device_name)):
		figuretitle = '_'+str(FLAGS.device_name[i])
	plt.savefig(str(FLAGS.size)+'BoxPlot_Command_IC'+command_temp+figuretitle+'.png')
	plt.close()

# ###USE OF DEVICES
#
DevicesNames = map(lambda x: str(x), df_Devices.groupby(['Device']).indices.keys())
for Device in range(len(DevicesNames)):
 	device_temp = DevicesNames[Device]
 	#print device_temp
	df_data = pd.DataFrame()
	data = []
	labels = []
	xtitle = ""
	for prob in range(10, 100, 10):
		df_data[["#Uses"+str(prob),"Device"+str(prob)]] = df_Devices[df_Devices["probability"] == prob][["#Uses","Device"]]
		data.append(df_data[df_data["Device"+str(prob)] == device_temp]["#Uses"+str(prob)])
		labels.append(prob)
	#print data
	plt.boxplot(data, labels = labels, showmeans = True)
	plt.suptitle(device_temp, fontsize=14, fontweight='bold')
	for i in range(len(FLAGS.device_name)):
 		xtitle += " " +str(FLAGS.device_name[i])
	plt.xlabel("Failure probability of devices:" +xtitle, fontsize=12)
	plt.ylabel('Total number of uses of the device', fontsize=12)
	plt.ylim(bottom=-1)
	#plt.show()
	for i in range(len(FLAGS.device_name)):
		figuretitle = '_'+str(FLAGS.device_name[i])
	plt.savefig(str(FLAGS.size)+'BoxPlot_Device_'+device_temp+figuretitle+'.png')
	plt.close()

reason: import numpy,import scipy,from scipy


text:
import numpy as np
import sys

def load_and_normalize_vectors(filename, encoding=None):
	with open(filename, "r", encoding=encoding) as f:
		words = []
		vectors = []
		for line in f:
			line = line.split()
			words.append(line[0])
			vectors.append(line[1:])

		vectors = np.array(vectors).astype(np.float)
		words = np.array(words)
		# Normalize
		#row_den = np.sum(vectors ** 2, axis=1)
		#vectors = vectors / row_sums[:, np.newaxis]

		return vectors, words

def main(argv):
	words_filename = argv[1]
	contexts_filename = argv[2]
	W, words = load_and_normalize_vectors(filename=words_filename, encoding="utf-8")
	C, contexts = load_and_normalize_vectors(filename=contexts_filename, encoding="utf-8")
	# W and words are numpy arrays.
	w2i = {w: i for i, w in enumerate(words)}
	#i2c = {i: c for i, c in enumerate(contexts)}

	words_to_compare = ["car", "bus", "hospital", "hotel", "gun", "bomb", "horse", "fox", "table", "bowl", "guitar", "piano"]
	for w in words_to_compare:
		print("Analyzing word: {}".format(w))
		word_vec = W[w2i[w]]  # get the dog vector

		sims = W.dot(word_vec)  # compute similarities
		most_similar_ids = sims.argsort()[-2:-21:-1]
		sim_words = words[most_similar_ids]
		print("\nMost similar words:")
		print(sim_words)

		print("\nMost similar contexts:")
		sims = C.dot(word_vec)  # compute similarities
		most_similar_ids = sims.argsort()[-2:-11:-1]
		sim_contexts = contexts[most_similar_ids]
		print(sim_contexts)

if __name__ == "__main__":
	main(sys.argv)

max_stars_repo_name: zabatani/word-embeddings

max_issues_repo_name: zabatani/word-embeddings

max_forks_repo_name: zabatani/word-embeddings