text:
# MIT License
#
# Copyright (c) 2019 Tuomas Halvari, Juha Harviainen, Juha Mylläri, Antti Röyskö, Juuso Silvennoinen
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import json
import sys

import cv2
import matplotlib.pyplot as plt
import numpy as np
import torch
from numpy.random import RandomState
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
from tqdm import trange

from dpemu import runner
from dpemu.dataset_utils import load_coco_val_2017
from dpemu.filters.image import Brightness
from dpemu.ml_utils import load_yolov3
from dpemu.nodes import Array
from dpemu.nodes.series import Series
from dpemu.plotting_utils import print_results_by_model, visualize_scores
from dpemu.utils import generate_unique_path
from dpemu.utils import get_project_root

cv2.ocl.setUseOpenCL(False)
torch.multiprocessing.set_start_method("spawn", force="True")

def get_data(argv):
    return load_coco_val_2017(int(argv[1]), is_shuffled=True)

def get_err_root_node():
    err_node = Array()
    err_root_node = Series(err_node)
    # err_node.addfilter(GaussianNoise("mean", "std"))
    # err_node.addfilter(Blur_Gaussian("std"))
    # err_node.addfilter(Snow("snowflake_probability", "snowflake_alpha", "snowstorm_alpha"))
    # err_node.addfilter(FastRain("probability", "range"))
    # err_node.addfilter(StainArea("probability", "radius_generator", "transparency_percentage"))
    # err_node.addfilter(JPEG_Compression("quality"))
    # err_node.addfilter(Resolution("k"))
    err_node.addfilter(Brightness("tar", "rat", "range"))
    # err_node.addfilter(SaturationVectorized("tar", "rate", "range"))
    # err_node.addfilter(Identity())
    return err_root_node

def get_err_params_list():
    # err_params_list = [{"mean": 0, "std": std} for std in [10 * i for i in range(0, 4)]]
    # err_params_list = [{"std": std} for std in [i for i in range(0, 4)]]
    # err_params_list = [{"snowflake_probability": p, "snowflake_alpha": .4, "snowstorm_alpha": 0}
    #                    for p in [10 ** i for i in range(-4, 0)]]
    # err_params_list = [{"probability": p, "range": 255} for p in [10 ** i for i in range(-4, 0)]]
    # err_params_list = [
    #     {"probability": p, "radius_generator": GaussianRadiusGenerator(0, 50), "transparency_percentage": 0.2}
    #     for p in [10 ** i for i in range(-6, -2)]]
    # err_params_list = [{"quality": q} for q in [10, 20, 30, 100]]
    # err_params_list = [{"k": k} for k in [1, 2, 3, 4]]
    err_params_list = [{"tar": 1, "rat": rat, "range": 255} for rat in [0, .4, .8, 1.2]]
    # err_params_list = [{}]
    return err_params_list

class Preprocessor:
    def run(self, _, imgs, params):
        return None, imgs, {}

class YOLOv3CPUModel:

    def __init__(self):
        self.random_state = RandomState(42)
        self.coco91class = [
            1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33,
            34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
            62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90
        ]
        self.show_imgs = False

    @staticmethod
    def __draw_box(img, class_id, class_names, confidence, x, y, w, h):
        label = str(class_names[class_id]) + " " + str(confidence)
        colors = np.random.randint(0, 255, size=(len(class_names), 3), dtype="uint8")
        color = [int(c) for c in colors[class_id]]
        cv2.rectangle(img, (x, y), (x + w, y + h), color, 2)
        cv2.putText(img, label, (x, y - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)

    def __get_results_for_img(self, img, img_id, class_names, net):
        conf_threshold = 0
        nms_threshold = .4
        img_h = img.shape[0]
        img_w = img.shape[1]
        inference_size = 608
        scale = 1 / 255

        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
        blob = cv2.dnn.blobFromImage(img, scale, (inference_size, inference_size), (0, 0, 0), True)
        net.setInput(blob)
        out_layer_names = net.getUnconnectedOutLayersNames()
        outs = net.forward(out_layer_names)

        class_ids = []
        confs = []
        boxes = []
        results = []
        for out in outs:
            for detection in out:
                scores = detection[5:]
                class_id = np.argmax(scores)
                conf = float(scores[class_id])
                if conf > conf_threshold:
                    center_x = detection[0] * img_w
                    center_y = detection[1] * img_h
                    w = detection[2] * img_w
                    h = detection[3] * img_h
                    x = center_x - w / 2
                    y = center_y - h / 2
                    class_ids.append(class_id)
                    confs.append(conf)
                    boxes.append([x, y, w, h])

        indices = cv2.dnn.NMSBoxes(boxes, confs, conf_threshold, nms_threshold)
        for i in indices:
            i = i[0]
            x, y, w, h = boxes[i]

            results.append({
                "image_id": img_id,
                "category_id": self.coco91class[class_ids[i]],
                "bbox": [x, y, w, h],
                "score": confs[i],
            })

            if self.show_imgs:
                self.__draw_box(img, class_ids[i], class_names, round(confs[i], 2), int(round(x)), int(round(y)),
                                int(round(w)), int(round(h)))

        if self.show_imgs:
            cv2.imshow(str(img_id), img)
            path_to_img = generate_unique_path("out", "jpg")
            cv2.imwrite(path_to_img, img, [cv2.IMWRITE_JPEG_QUALITY, 100])
            cv2.waitKey()
            cv2.destroyAllWindows()

        return results

    def run(self, _, imgs, params):
        img_ids = params["img_ids"]
        class_names = params["class_names"]
        self.show_imgs = params["show_imgs"]
        path_to_yolov3_weights = params["path_to_yolov3_weights"]
        path_to_yolov3_cfg = params["path_to_yolov3_cfg"]

        net = cv2.dnn.readNet(path_to_yolov3_weights, path_to_yolov3_cfg)
        results = []
        for i in trange(len(imgs)):
            results.extend(self.__get_results_for_img(imgs[i], img_ids[i], class_names, net))
        if not results:
            return {"mAP-50": 0}

        path_to_results = generate_unique_path("tmp", "json")
        with open(path_to_results, "w") as fp:
            json.dump(results, fp)

        coco_gt = COCO(f"{get_project_root()}/data/annotations/instances_val2017.json")
        coco_eval = COCOeval(coco_gt, coco_gt.loadRes(path_to_results), "bbox")
        coco_eval.params.imgIds = img_ids
        coco_eval.evaluate()
        coco_eval.accumulate()
        coco_eval.summarize()
        return {"mAP-50": round(coco_eval.stats[1], 3)}

def get_model_params_dict_list(img_ids, class_names, path_to_yolov3_weights, path_to_yolov3_cfg):
    return [
        {"model": YOLOv3CPUModel, "params_list": [{
            "img_ids": img_ids,
            "class_names": class_names,
            "show_imgs": True,
            "path_to_yolov3_weights": path_to_yolov3_weights,
            "path_to_yolov3_cfg": path_to_yolov3_cfg
        }]}
    ]

def visualize(df):
    # visualize_scores(df, ["mAP-50"], [True], "std", "Object detection with Gaussian noise", x_log=False)
    # visualize_scores(df, ["mAP-50"], [True], "std", "Object detection with Gaussian blur", x_log=False)
    # visualize_scores(df, ["mAP-50"], [True], "snowflake_probability", "Object detection with snow filter", x_log=True)
    # visualize_scores(df, ["mAP-50"], [True], "probability", "Object detection with rain filter", x_log=True)
    # visualize_scores(df, ["mAP-50"], [True], "probability", "Object detection with added stains", x_log=True)
    # visualize_scores(df, ["mAP-50"], [True], "quality", "Object detection with JPEG compression", x_log=False)
    # visualize_scores(df, ["mAP-50"], [True], "k", "Object detection with reduced resolution", x_log=False)
    visualize_scores(df, ["mAP-50"], [True], "rat", "Object detection with added brightness", x_log=False)
    # visualize_scores(df, ["mAP-50"], [True], "rate", "Object detection with added saturation", x_log=False)
    plt.show()

def main(argv):
    if len(argv) != 2:
        exit(0)

    imgs, img_ids, class_names, _ = get_data(argv)
    path_to_yolov3_weights, path_to_yolov3_cfg = load_yolov3()

    df = runner.run(
        train_data=None,
        test_data=imgs,
        preproc=Preprocessor,
        preproc_params=None,
        err_root_node=get_err_root_node(),
        err_params_list=get_err_params_list(),
        model_params_dict_list=get_model_params_dict_list(img_ids, class_names, path_to_yolov3_weights,
                                                          path_to_yolov3_cfg),
        n_processes=1
    )

    print_results_by_model(df, ["img_ids", "class_names", "show_imgs", "mean", "radius_generator",
                                "transparency_percentage", "range", "snowflake_alpha", "snowstorm_alpha"])
    visualize(df)

if __name__ == "__main__":
    main(sys.argv)

max_forks_repo_name: thalvari/dpEmu-AutoML

reason: import numpy,from numpy


text:
# -*- coding: utf-8 -*-
"""
SEG-2 support for ObsPy.

A file format description is given by [Pullan1990]_.

:copyright:
    Lion Krischer (krischer@geophysik.uni-muenchen.de), 2011
:license:
    GNU Lesser General Public License, Version 3
    (https://www.gnu.org/copyleft/lesser.html)
"""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
from future.builtins import *  # NOQA
from future.utils import PY2

from copy import deepcopy
from struct import unpack
import warnings

import numpy as np

from obspy import Stream, Trace, UTCDateTime
from obspy.core import AttribDict
from obspy.core.compatibility import from_buffer
from .header import MONTHS

WARNING_HEADER = "Many companies use custom defined SEG2 header variables." + \
    " This might cause basic header information reflected in the single " + \
    "traces' stats to be wrong (e.g. recording delays, first sample " + \
    "number, station code names, ..). Please check the complete list of " + \
    "additional unmapped header fields that gets stored in " + \
    "Trace.stats.seg2 and/or the manual of the source of the SEG2 files " + \
    "for fields that might influence e.g. trace start times."

class SEG2BaseError(Exception):
    """
    Base class for all SEG-2 specific errors.
    """
    pass

class SEG2InvalidFileError(SEG2BaseError):
    """
    Will be raised if something is not correct with the SEG-2 file.
    """
    pass

class SEG2(object):
    """
    Class to read and write SEG 2 formatted files. The main reason this is
    realized as a class is for the ease of passing the various parameters from
    one function to the next.

    Do not change the file_pointer attribute while using this class. It will
    be used to keep track of which parts have been read yet and which not.
    """
    def __init__(self):
        pass

    def read_file(self, file_object):
        """
        Reads the following file and will return a Stream object. If
        file_object is a string it will be treated as a file name, otherwise it
        will be expected to be a file like object with read(), seek() and
        tell() methods.

        If it is a file_like object, file.seek(0, 0) is expected to be the
        beginning of the SEG-2 file.
        """
        # Read the file if it is a file name.
        if not hasattr(file_object, 'write'):
            self.file_pointer = open(file_object, 'rb')
        else:
            self.file_pointer = file_object
            self.file_pointer.seek(0, 0)

        self.stream = Stream()

        # Read the file descriptor block. This will also determine the
        # endianness.
        self.read_file_descriptor_block()

        # Loop over every trace, read it and append it to the Stream.
        for tr_pointer in self.trace_pointers:
            self.file_pointer.seek(tr_pointer, 0)
            self.stream.append(self.parse_next_trace())

        if not hasattr(file_object, 'write'):
            self.file_pointer.close()
        return self.stream

    def read_file_descriptor_block(self):
        """
        Handles the reading of the file descriptor block and the free form
        section following it.
        """
        file_descriptor_block = self.file_pointer.read(32)

        # Determine the endianness and check if the block id is valid.
        if unpack(b'B', file_descriptor_block[0:1])[0] == 0x55 and \
           unpack(b'B', file_descriptor_block[1:2])[0] == 0x3a:
            self.endian = b'<'
        elif unpack(b'B', file_descriptor_block[0:1])[0] == 0x3a and \
                unpack(b'B', file_descriptor_block[1:2])[0] == 0x55:
            self.endian = b'>'
        else:
            msg = 'Wrong File Descriptor Block ID'
            raise SEG2InvalidFileError(msg)

        # Check the revision number.
        revision_number = unpack(self.endian + b'H',
                                 file_descriptor_block[2:4])[0]
        if revision_number != 1:
            msg = '\nOnly SEG 2 revision 1 is officially supported. This file '
            msg += 'has revision %i. Reading it might fail.' % revision_number
            msg += '\nPlease contact the ObsPy developers with a sample file.'
            warnings.warn(msg)
        size_of_trace_pointer_sub_block = unpack(
            self.endian + b'H', file_descriptor_block[4:6])[0]
        number_of_traces = unpack(
            self.endian + b'H', file_descriptor_block[6:8])[0]

        # Define the string and line terminators.
        (size_of_string_terminator,
         first_string_terminator_char,
         second_string_terminator_char,
         size_of_line_terminator,
         first_line_terminator_char,
         second_line_terminator_char
         ) = unpack(b'BccBcc', file_descriptor_block[8:14])

        # Assemble the string terminator.
        if size_of_string_terminator == 1:
            self.string_terminator = first_string_terminator_char
        elif size_of_string_terminator == 2:
            self.string_terminator = first_string_terminator_char + \
                second_string_terminator_char
        else:
            msg = 'Wrong size of string terminator.'
            raise SEG2InvalidFileError(msg)
        # Assemble the line terminator.
        if size_of_line_terminator == 1:
            self.line_terminator = first_line_terminator_char
        elif size_of_line_terminator == 2:
            self.line_terminator = first_line_terminator_char + \
                second_line_terminator_char
        else:
            msg = 'Wrong size of line terminator.'
            raise SEG2InvalidFileError(msg)

        # Read the trace pointer sub-block and retrieve all the pointers.
        trace_pointer_sub_block = \
            self.file_pointer.read(size_of_trace_pointer_sub_block)
        self.trace_pointers = []
        for _i in range(number_of_traces):
            index = _i * 4
            self.trace_pointers.append(
                unpack(self.endian + b'L',
                       trace_pointer_sub_block[index:index + 4])[0])

        # The rest of the header up to where the first trace pointer points is
        # a free form section.
        self.stream.stats = AttribDict()
        self.stream.stats.seg2 = AttribDict()
        self.parse_free_form(
            self.file_pointer.read(self.trace_pointers[0] -
                                   self.file_pointer.tell()),
            self.stream.stats.seg2)

        # Get the time information from the file header.
        # XXX: Need some more generic date/time parsers.
        if "ACQUISITION_TIME" in self.stream.stats.seg2 \
                and "ACQUISITION_DATE" in self.stream.stats.seg2:
            time = self.stream.stats.seg2.ACQUISITION_TIME
            date = self.stream.stats.seg2.ACQUISITION_DATE
            time = time.strip().split(':')
            date = date.strip().split('/')
            hour, minute, second = int(time[0]), int(time[1]), float(time[2])
            day, month, year = int(date[0]), MONTHS[date[1].lower()], \
                int(date[2])
            self.starttime = UTCDateTime(year, month, day, hour, minute,
                                         second)
        else:
            self.starttime = UTCDateTime(0)

    def parse_next_trace(self):
        """
        Parse the next trace in the trace pointer list and return a Trace
        object.
        """
        trace_descriptor_block = self.file_pointer.read(32)
        # Check if the trace descriptor block id is valid.
        if unpack(self.endian + b'H', trace_descriptor_block[0:2])[0] != \
           0x4422:
            msg = 'Invalid trace descriptor block id.'
            raise SEG2InvalidFileError(msg)
        size_of_this_block = unpack(self.endian + b'H',
                                    trace_descriptor_block[2:4])[0]
        number_of_samples_in_data_block = \
            unpack(self.endian + b'L', trace_descriptor_block[8:12])[0]
        data_format_code = unpack(b'B', trace_descriptor_block[12:13])[0]

        # Parse the data format code.
        if data_format_code == 4:
            dtype = np.float32
            sample_size = 4
        elif data_format_code == 5:
            dtype = np.float64
            sample_size = 8
        elif data_format_code == 1:
            dtype = np.int16
            sample_size = 2
        elif data_format_code == 2:
            dtype = np.int32
            sample_size = 4
        elif data_format_code == 3:
            msg = ('\nData format code 3 (20-bit SEG-D floating point) not '
                   'supported yet.\nPlease contact the ObsPy developers with '
                   'a sample file.')
            raise NotImplementedError(msg)
        else:
            msg = 'Unrecognized data format code'
            raise SEG2InvalidFileError(msg)

        # The rest of the trace block is free form.
        header = {}
        header['seg2'] = AttribDict()
        self.parse_free_form(self.file_pointer.read(size_of_this_block - 32),
                             header['seg2'])
        header['delta'] = float(header['seg2']['SAMPLE_INTERVAL'])
        # Set to the file's start time.
        header['starttime'] = deepcopy(self.starttime)
        if 'DELAY' in header['seg2']:
            if float(header['seg2']['DELAY']) != 0:
                msg = "Non-zero value found in Trace's 'DELAY' field. " + \
                      "This is not supported/tested yet and might lead " + \
                      "to a wrong starttime of the Trace. Please contact " + \
                      "the ObsPy developers with a sample file."
                warnings.warn(msg)

        if "DESCALING_FACTOR" in header["seg2"]:
            header['calib'] = float(header['seg2']['DESCALING_FACTOR'])

        # Unpack the data.
        data = from_buffer(
            self.file_pointer.read(number_of_samples_in_data_block *
                                   sample_size),
            dtype=dtype)
        # Integrate SEG2 file header into each trace header
        tmp = self.stream.stats.seg2.copy()
        tmp.update(header['seg2'])
        header['seg2'] = tmp
        return Trace(data=data, header=header)

    def parse_free_form(self, free_form_str, attrib_dict):
        """
        Parse the free form section stored in free_form_str and save it in
        attrib_dict.
        """
        # Separate the strings.
        strings = free_form_str.split(self.string_terminator)
        # This is not fully according to the SEG-2 format specification (or
        # rather the specification only speaks about on offset of 2 bytes
        # between strings and a string_terminator between two free form
        # strings. The file I have show the following separation between two
        # strings: 'random offset byte', 'string_terminator',
        # 'random offset byte'
        # Therefore every string has to be at least 3 bytes wide to be
        # acceptable after being split at the string terminator.

        def is_good_char(c):
            return c in (b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN'
                         b'OPQRSTUVWXYZ!"#$%&\'()*+,-./:; <=>?@[\\]^_`{|}~ ')

        # A loop over a bytestring in Python 3 returns integers. This can be
        # solved with a number of imports from the python-future module and
        # all kinds of subtle changes throughout this file. Separating the
        # handling for Python 2 and 3 seems the cleaner and simpler approach.
        if PY2:
            strings = ["".join(filter(is_good_char, _i))
                       for _i in strings
                       if len(_i) >= 3]
        else:
            strings = ["".join(map(chr, filter(is_good_char, _i)))
                       for _i in strings
                       if len(_i) >= 3]

        # Every string has the structure OPTION<SPACE>VALUE. Write to
        # stream.stats attribute.
        for string in strings:
            string = string.strip()
            string = string.split(' ')
            key = string[0].strip()
            value = ' '.join(string[1:]).strip()
            setattr(attrib_dict, key, value)
        # Parse the notes string again.
        if hasattr(attrib_dict, 'NOTE'):
            notes = attrib_dict.NOTE.split(self.line_terminator.decode())
            attrib_dict.NOTE = AttribDict()
            for note in notes:
                note = note.strip()
                note = note.split(' ')
                key = note[0].strip()
                value = ' '.join(note[1:]).strip()
                setattr(attrib_dict.NOTE, key, value)

def _is_seg2(filename):
    if not hasattr(filename, 'write'):
        file_pointer = open(filename, 'rb')
    else:
        file_pointer = filename

    file_descriptor_block = file_pointer.read(4)
    if not hasattr(filename, 'write'):
        file_pointer.close()
    try:
        # Determine the endianness and check if the block id is valid.
        if unpack(b'B', file_descriptor_block[0:1])[0] == 0x55 and \
           unpack(b'B', file_descriptor_block[1:2])[0] == 0x3a:
            endian = b'<'
        elif unpack(b'B', file_descriptor_block[0:1])[0] == 0x3a and \
                unpack(b'B', file_descriptor_block[1:2])[0] == 0x55:
            endian = b'>'
        else:
            return False
    except Exception:
        return False
    # Check the revision number.
    revision_number = unpack(endian + b'H',
                             file_descriptor_block[2:4])[0]
    if revision_number != 1:
        return False
    return True

def _read_seg2(filename, **kwargs):  # @UnusedVariable
    seg2 = SEG2()
    st = seg2.read_file(filename)
    warnings.warn(WARNING_HEADER)
    return st

ext: py

reason: import numpy


text:
import inspect
import math
from typing import Any, Callable, Dict
import argparse

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

BERT_BASE_UNCASED_CONFIG = {
    "architectures": ["BertForMaskedLM"],
    "attention_probs_dropout_prob": 0.1,
    "hidden_act": "gelu",
    "hidden_dropout_prob": 0.1,
    "hidden_size": 768,
    "initializer_range": 0.02,
    "intermediate_size": 3072,
    "layer_norm_eps": 1e-12,
    "max_position_embeddings": 512,
    "model_type": "bert",
    "num_attention_heads": 12,
    "num_hidden_layers": 12,
    "pad_token_id": 0,
    "type_vocab_size": 2,
    "vocab_size": 30522,
}

BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/config.json",
    "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/config.json",
}

class PretrainedConfig(object):
    model_type: str = ""
    is_composition: bool = False

    def __init__(self, **kwargs):
        # Attributes with defaults
        self.return_dict = kwargs.pop("return_dict", True)
        self.output_hidden_states = kwargs.pop("output_hidden_states", False)
        self.output_attentions = kwargs.pop("output_attentions", False)
        self.torchscript = kwargs.pop("torchscript", False)  # Only used by PyTorch models
        self.use_bfloat16 = kwargs.pop("use_bfloat16", False)
        self.pruned_heads = kwargs.pop("pruned_heads", {})
        self.tie_word_embeddings = kwargs.pop(
            "tie_word_embeddings", True
        )  # Whether input and output word embeddings should be tied for all MLM, LM and Seq2Seq models.

        # Is decoder is used in encoder-decoder models to differentiate encoder from decoder
        self.is_encoder_decoder = kwargs.pop("is_encoder_decoder", False)
        self.is_decoder = kwargs.pop("is_decoder", False)
        self.add_cross_attention = kwargs.pop("add_cross_attention", False)
        self.tie_encoder_decoder = kwargs.pop("tie_encoder_decoder", False)

        # Parameters for sequence generation
        self.max_length = kwargs.pop("max_length", 20)
        self.min_length = kwargs.pop("min_length", 0)
        self.do_sample = kwargs.pop("do_sample", False)
        self.early_stopping = kwargs.pop("early_stopping", False)
        self.num_beams = kwargs.pop("num_beams", 1)
        self.num_beam_groups = kwargs.pop("num_beam_groups", 1)
        self.diversity_penalty = kwargs.pop("diversity_penalty", 0.0)
        self.temperature = kwargs.pop("temperature", 1.0)
        self.top_k = kwargs.pop("top_k", 50)
        self.top_p = kwargs.pop("top_p", 1.0)
        self.repetition_penalty = kwargs.pop("repetition_penalty", 1.0)
        self.length_penalty = kwargs.pop("length_penalty", 1.0)
        self.no_repeat_ngram_size = kwargs.pop("no_repeat_ngram_size", 0)
        self.bad_words_ids = kwargs.pop("bad_words_ids", None)
        self.num_return_sequences = kwargs.pop("num_return_sequences", 1)
        self.chunk_size_feed_forward = kwargs.pop("chunk_size_feed_forward", 0)
        self.output_scores = kwargs.pop("output_scores", False)
        self.return_dict_in_generate = kwargs.pop("return_dict_in_generate", False)

        # Fine-tuning task arguments
        self.architectures = kwargs.pop("architectures", None)
        self.finetuning_task = kwargs.pop("finetuning_task", None)
        self.id2label = kwargs.pop("id2label", None)
        self.label2id = kwargs.pop("label2id", None)
        if self.id2label is not None:
            kwargs.pop("num_labels", None)
            self.id2label = dict((int(key), value) for key, value in self.id2label.items())
            # Keys are always strings in JSON so convert ids to int here.
        else:
            self.num_labels = kwargs.pop("num_labels", 2)

        # Tokenizer arguments TODO: eventually tokenizer and models should share the same config
        self.tokenizer_class = kwargs.pop("tokenizer_class", None)
        self.prefix = kwargs.pop("prefix", None)
        self.bos_token_id = kwargs.pop("bos_token_id", None)
        self.pad_token_id = kwargs.pop("pad_token_id", None)
        self.eos_token_id = kwargs.pop("eos_token_id", None)
        self.sep_token_id = kwargs.pop("sep_token_id", None)

        self.decoder_start_token_id = kwargs.pop("decoder_start_token_id", None)

        # task specific arguments
        self.task_specific_params = kwargs.pop("task_specific_params", None)

        # TPU arguments
        self.xla_device = kwargs.pop("xla_device", None)

        # Name or path to the pretrained checkpoint
        self._name_or_path = str(kwargs.pop("name_or_path", ""))

        # Drop the transformers version info
        kwargs.pop("transformers_version", None)

        # Additional attributes without default values
        for key, value in kwargs.items():
            try:
                setattr(self, key, value)
            except AttributeError as err:
                print("Can't set {} with value {} for {}".format(key, value, self))
                raise err

    @property
    def use_return_dict(self) -> bool:
        """
        :obj:`bool`: Whether or not return :class:`~transformers.file_utils.ModelOutput` instead of tuples.
        """
        # If torchscript is set, force `return_dict=False` to avoid jit errors
        return self.return_dict and not self.torchscript

    @property
    def num_labels(self) -> int:
        """
        :obj:`int`: The number of labels for classification models.
        """
        return len(self.id2label)

    @num_labels.setter
    def num_labels(self, num_labels: int):
        self.id2label = {i: "LABEL_{}".format(i) for i in range(num_labels)}
        self.label2id = dict(zip(self.id2label.values(), self.id2label.keys()))

    @classmethod
    def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "PretrainedConfig":
        """
        Instantiates a :class:`~transformers.PretrainedConfig` from a Python dictionary of parameters.

        Args:
            config_dict (:obj:`Dict[str, Any]`):
                Dictionary that will be used to instantiate the configuration object. Such a dictionary can be
                retrieved from a pretrained checkpoint by leveraging the
                :func:`~transformers.PretrainedConfig.get_config_dict` method.
            kwargs (:obj:`Dict[str, Any]`):
                Additional parameters from which to initialize the configuration object.

        Returns:
            :class:`PretrainedConfig`: The configuration object instantiated from those parameters.
        """
        return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)

        config = cls(**config_dict)

        if hasattr(config, "pruned_heads"):
            config.pruned_heads = dict(
                (int(key), value) for key, value in config.pruned_heads.items()
            )

        # Update config with kwargs if needed
        to_remove = []
        for key, value in kwargs.items():
            if hasattr(config, key):
                setattr(config, key, value)
                to_remove.append(key)
        for key in to_remove:
            kwargs.pop(key, None)

        if return_unused_kwargs:
            return config, kwargs
        else:
            return config

class BertConfig(PretrainedConfig):
    r"""
    This is the configuration class to store the configuration of a :class:`~transformers.BertModel` or a
    :class:`~transformers.TFBertModel`. It is used to instantiate a BERT model according to the specified arguments,
    defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
    to that of the BERT `bert-base-uncased <https://huggingface.co/bert-base-uncased>`__ architecture.

    Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model
    outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information.

    Args:
        vocab_size (:obj:`int`, `optional`, defaults to 30522):
            Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the
            :obj:`inputs_ids` passed when calling :class:`~transformers.BertModel` or
            :class:`~transformers.TFBertModel`.
        hidden_size (:obj:`int`, `optional`, defaults to 768):
            Dimensionality of the encoder layers and the pooler layer.
        num_hidden_layers (:obj:`int`, `optional`, defaults to 12):
            Number of hidden layers in the Transformer encoder.
        num_attention_heads (:obj:`int`, `optional`, defaults to 12):
            Number of attention heads for each attention layer in the Transformer encoder.
        intermediate_size (:obj:`int`, `optional`, defaults to 3072):
            Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
        hidden_act (:obj:`str` or :obj:`Callable`, `optional`, defaults to :obj:`"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string,
            :obj:`"gelu"`, :obj:`"relu"`, :obj:`"silu"` and :obj:`"gelu_new"` are supported.
        hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1):
            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
        attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1):
            The dropout ratio for the attention probabilities.
        max_position_embeddings (:obj:`int`, `optional`, defaults to 512):
            The maximum sequence length that this model might ever be used with. Typically set this to something large
            just in case (e.g., 512 or 1024 or 2048).
        type_vocab_size (:obj:`int`, `optional`, defaults to 2):
            The vocabulary size of the :obj:`token_type_ids` passed when calling :class:`~transformers.BertModel` or
            :class:`~transformers.TFBertModel`.
        initializer_range (:obj:`float`, `optional`, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (:obj:`float`, `optional`, defaults to 1e-12):
            The epsilon used by the layer normalization layers.
        gradient_checkpointing (:obj:`bool`, `optional`, defaults to :obj:`False`):
            If True, use gradient checkpointing to save memory at the expense of slower backward pass.
        position_embedding_type (:obj:`str`, `optional`, defaults to :obj:`"absolute"`):
            Type of position embedding. Choose one of :obj:`"absolute"`, :obj:`"relative_key"`,
            :obj:`"relative_key_query"`. For positional embeddings use :obj:`"absolute"`. For more information on
            :obj:`"relative_key"`, please refer to `Self-Attention with Relative Position Representations (Shaw et al.)
            <https://arxiv.org/abs/1803.02155>`__. For more information on :obj:`"relative_key_query"`, please refer to
            `Method 4` in `Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)
            <https://arxiv.org/abs/2009.13658>`__.
        use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if ``config.is_decoder=True``.

    Examples::

        >>> from transformers import BertModel, BertConfig

        >>> # Initializing a BERT bert-base-uncased style configuration
        >>> configuration = BertConfig()

        >>> # Initializing a model from the bert-base-uncased style configuration
        >>> model = BertModel(configuration)

        >>> # Accessing the model configuration
        >>> configuration = model.config
    """
    model_type = "bert"

    def __init__(
        self,
        vocab_size=30522,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=2,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        pad_token_id=0,
        gradient_checkpointing=False,
        position_embedding_type="absolute",
        use_cache=True,
        **kwargs,
    ):
        super().__init__(pad_token_id=pad_token_id, **kwargs)

        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.hidden_act = hidden_act
        self.intermediate_size = intermediate_size
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.gradient_checkpointing = gradient_checkpointing
        self.position_embedding_type = position_embedding_type
        self.use_cache = use_cache

def apply_chunking_to_forward(
    forward_fn: Callable[..., torch.Tensor], chunk_size: int, chunk_dim: int, *input_tensors
) -> torch.Tensor:
    """
    This function chunks the :obj:`input_tensors` into smaller input tensor parts of size :obj:`chunk_size` over the
    dimension :obj:`chunk_dim`. It then applies a layer :obj:`forward_fn` to each chunk independently to save memory.

    If the :obj:`forward_fn` is independent across the :obj:`chunk_dim` this function will yield the same result as
    directly applying :obj:`forward_fn` to :obj:`input_tensors`.

    Args:
        forward_fn (:obj:`Callable[..., torch.Tensor]`):
            The forward function of the model.
        chunk_size (:obj:`int`):
            The chunk size of a chunked tensor: :obj:`num_chunks = len(input_tensors[0]) / chunk_size`.
        chunk_dim (:obj:`int`):
            The dimension over which the :obj:`input_tensors` should be chunked.
        input_tensors (:obj:`Tuple[torch.Tensor]`):
            The input tensors of ``forward_fn`` which will be chunked

    Returns:
        :obj:`torch.Tensor`: A tensor with the same shape as the :obj:`forward_fn` would have given if applied`.

    Examples::

        # rename the usual forward() fn to forward_chunk()
        def forward_chunk(self, hidden_states):
            hidden_states = self.decoder(hidden_states)
            return hidden_states

        # implement a chunked forward function
        def forward(self, hidden_states):
            return apply_chunking_to_forward(self.forward_chunk, self.chunk_size_lm_head, self.seq_len_dim, hidden_states)
    """

    assert len(input_tensors) > 0, "{} has to be a tuple/list of tensors".format(input_tensors)
    tensor_shape = input_tensors[0].shape[chunk_dim]
    assert all(
        input_tensor.shape[chunk_dim] == tensor_shape for input_tensor in input_tensors
    ), "All input tenors have to be of the same shape"

    # inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
    num_args_in_forward_chunk_fn = len(inspect.signature(forward_fn).parameters)
    assert num_args_in_forward_chunk_fn == len(
        input_tensors
    ), "forward_chunk_fn expects {} arguments, but only {} input tensors are given".format(
        num_args_in_forward_chunk_fn, len(input_tensors)
    )

    if chunk_size > 0:
        assert (
            input_tensors[0].shape[chunk_dim] % chunk_size == 0
        ), "The dimension to be chunked {} has to be a multiple of the chunk size {}".format(
            input_tensors[0].shape[chunk_dim], chunk_size
        )

        num_chunks = input_tensors[0].shape[chunk_dim] // chunk_size

        # chunk input tensor into tuples
        input_tensors_chunks = tuple(
            input_tensor.chunk(num_chunks, dim=chunk_dim) for input_tensor in input_tensors
        )
        # apply forward fn to every tuple
        output_chunks = tuple(
            forward_fn(*input_tensors_chunk) for input_tensors_chunk in zip(*input_tensors_chunks)
        )
        # concatenate output at same dimension
        return torch.cat(output_chunks, dim=chunk_dim)

    return forward_fn(*input_tensors)

def _gelu_python(x):
    """
    Original Implementation of the GELU activation function in Google BERT repo when initially created. For
    information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 +
    torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in
    torch.nn.functional Also see the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

ACT2FN = {
    "relu": F.relu,
    "gelu": _gelu_python,
    "tanh": torch.tanh,
    "sigmoid": torch.sigmoid,
}

class BertEmbeddings(nn.Module):
    """Construct the embeddings from word, position and token_type embeddings."""

    def __init__(self, config):
        super().__init__()
        self.word_embeddings = nn.Embedding(
            config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id
        )
        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
        self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)

        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
        # any TensorFlow checkpoint file
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
        self.register_buffer(
            "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))
        )
        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")

    def forward(
        self,
        input_ids=None,
        token_type_ids=None,
    ):
        input_shape = input_ids.size()
        seq_length = input_shape[1]

        position_ids = self.position_ids[:, 0:seq_length]

        inputs_embeds = self.word_embeddings(input_ids)
        token_type_embeddings = self.token_type_embeddings(token_type_ids)

        embeddings = inputs_embeds + token_type_embeddings
        if self.position_embedding_type == "absolute":
            position_embeddings = self.position_embeddings(position_ids)
            embeddings += position_embeddings
        embeddings = self.LayerNorm(embeddings)
        embeddings = self.dropout(embeddings)
        return embeddings

class BertSelfAttention(nn.Module):
    def __init__(self, config):
        super().__init__()
        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(
            config, "embedding_size"
        ):
            raise ValueError(
                "The hidden size (%d) is not a multiple of the number of attention "
                "heads (%d)" % (config.hidden_size, config.num_attention_heads)
            )

        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.query = nn.Linear(config.hidden_size, self.all_head_size)
        self.key = nn.Linear(config.hidden_size, self.all_head_size)
        self.value = nn.Linear(config.hidden_size, self.all_head_size)

        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
        if (
            self.position_embedding_type == "relative_key"
            or self.position_embedding_type == "relative_key_query"
        ):
            self.max_position_embeddings = config.max_position_embeddings
            self.distance_embedding = nn.Embedding(
                2 * config.max_position_embeddings - 1, self.attention_head_size
            )

    def transpose_for_scores(self, x):
        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
        x = x.view(*new_x_shape)
        return x.permute(0, 2, 1, 3)

    def forward(
        self,
        hidden_states,
        attention_mask=None,
    ):
        key_layer = self.transpose_for_scores(self.key(hidden_states))
        value_layer = self.transpose_for_scores(self.value(hidden_states))
        query_layer = self.transpose_for_scores(self.query(hidden_states))

        # Take the dot product between "query" and "key" to get the raw attention scores.
        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))

        if (
            self.position_embedding_type == "relative_key"
            or self.position_embedding_type == "relative_key_query"
        ):
            seq_length = hidden_states.size()[1]
            position_ids_l = torch.arange(
                seq_length, dtype=torch.long, device=hidden_states.device
            ).view(-1, 1)
            position_ids_r = torch.arange(
                seq_length, dtype=torch.long, device=hidden_states.device
            ).view(1, -1)
            distance = position_ids_l - position_ids_r
            positional_embedding = self.distance_embedding(
                distance + self.max_position_embeddings - 1
            )
            positional_embedding = positional_embedding.to(
                dtype=query_layer.dtype
            )  # fp16 compatibility

            if self.position_embedding_type == "relative_key":
                relative_position_scores = torch.einsum(
                    "bhld,lrd->bhlr", query_layer, positional_embedding
                )
                attention_scores = attention_scores + relative_position_scores
            elif self.position_embedding_type == "relative_key_query":
                relative_position_scores_query = torch.einsum(
                    "bhld,lrd->bhlr", query_layer, positional_embedding
                )
                relative_position_scores_key = torch.einsum(
                    "bhrd,lrd->bhlr", key_layer, positional_embedding
                )
                attention_scores = (
                    attention_scores + relative_position_scores_query + relative_position_scores_key
                )

        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
        if attention_mask is not None:
            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
            attention_scores = attention_scores + attention_mask

        # Normalize the attention scores to probabilities.
        attention_probs = nn.Softmax(dim=-1)(attention_scores)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs = self.dropout(attention_probs)

        context_layer = torch.matmul(attention_probs, value_layer)

        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(*new_context_layer_shape)

        outputs = (context_layer,)

        return outputs

class BertSelfOutput(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

class BertAttention(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.self = BertSelfAttention(config)
        self.output = BertSelfOutput(config)

    def forward(
        self,
        hidden_states,
        attention_mask=None,
    ):
        self_outputs = self.self(
            hidden_states,
            attention_mask,
        )
        attention_output = self.output(self_outputs[0], hidden_states)
        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
        return outputs

class BertIntermediate(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
        if isinstance(config.hidden_act, str):
            self.intermediate_act_fn = ACT2FN[config.hidden_act]
        else:
            self.intermediate_act_fn = config.hidden_act

    def forward(self, hidden_states):
        hidden_states = self.dense(hidden_states)
        hidden_states = self.intermediate_act_fn(hidden_states)
        return hidden_states

class BertOutput(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

class BertLayer(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.chunk_size_feed_forward = config.chunk_size_feed_forward
        self.seq_len_dim = 1
        self.attention = BertAttention(config)
        self.intermediate = BertIntermediate(config)
        self.output = BertOutput(config)

    def forward(self, hidden_states, attention_mask=None):
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attention_outputs = self.attention(
            hidden_states,
            attention_mask,
        )
        attention_output = self_attention_outputs[0]

        outputs = self_attention_outputs[1:]

        layer_output = apply_chunking_to_forward(
            self.feed_forward_chunk,
            self.chunk_size_feed_forward,
            self.seq_len_dim,
            attention_output,
        )
        outputs = (layer_output,) + outputs

        return outputs

    def feed_forward_chunk(self, attention_output):
        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)
        return layer_output

class BertEncoder(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])

    def forward(self, hidden_states, attention_mask=None):
        for i, layer_module in enumerate(self.layer):
            layer_outputs = layer_module(hidden_states, attention_mask)
            hidden_states = layer_outputs[0]

        return (hidden_states,)

class BertPooler(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.activation = nn.Tanh()

    def forward(self, hidden_states):
        # We "pool" the model by simply taking the hidden state corresponding
        # to the first token.
        first_token_tensor = hidden_states[:, 0]
        pooled_output = self.dense(first_token_tensor)
        pooled_output = self.activation(pooled_output)
        return pooled_output

class BertModel(nn.Module):
    def __init__(self, config, add_pooling_layer=True):
        super().__init__()
        self.config = config

        # self.embeddings = BertEmbeddings(config)
        self.encoder = BertEncoder(config)

        self.pooler = BertPooler(config) if add_pooling_layer else None

    def forward(self, embedding_output): # input_ids):
        batch_size, seqlen, hidden_size = embedding_output.size()
        input_shape = (batch_size, seqlen)
        device = embedding_output.device

        # input_shape = input_ids.size()
        # device = input_ids.device

        attention_mask = torch.ones(input_shape, device=device)
        # token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)

        # embedding_output = self.embeddings(
        #     input_ids=input_ids,
        #     token_type_ids=token_type_ids,
        # )

        extended_attention_mask = attention_mask[:, None, None, :]
        dtype = next(iter(self.parameters())).dtype
        extended_attention_mask = extended_attention_mask.to(dtype=dtype)
        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0

        encoder_outputs = self.encoder(
            embedding_output,
            attention_mask=extended_attention_mask,
        )

        sequence_output = encoder_outputs[0]
        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

        return pooled_output or sequence_output

example_text = """
    example:
        python bert_base.py --batch 16 --enable_cudnn --number 5 --repeats 5
        python bert_base.py --batch 8 --number 8 --repeats 8
"""

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="base_maker",
        description="template maker",
        epilog=example_text,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--batch', type=int, default=1)
    parser.add_argument('--enable_cudnn', action='store_true')
    parser.add_argument('--number', type=int, default=10)
    parser.add_argument('--repeats', type=int, default=10)

    args = parser.parse_args()

    if args.enable_cudnn:
        assert torch.backends.cudnn.is_available()
        torch.backends.cudnn.enabled = True
    else:
        torch.backends.cudnn.enabled = False

    batch = args.batch

    config = BertConfig.from_dict(BERT_BASE_UNCASED_CONFIG)
    model = BertModel(config, add_pooling_layer=False).cuda().half()
    model.eval()

    input_shape = [batch, config.max_position_embeddings, config.hidden_size]
    embedding_output = torch.rand(input_shape).cuda().half()

    # warm up
    out = model(embedding_output)

    number = args.number
    repeats = args.repeats
    for i in range(repeats):
        time_record = []
        for j in range(number):
            torch.cuda.synchronize()
            start = torch.cuda.Event(enable_timing=True)
            end = torch.cuda.Event(enable_timing=True)
            start.record()

            output = model(embedding_output)

            end.record()
            torch.cuda.synchronize()
            total = start.elapsed_time(end)
            time_record.append(total)
        if i == repeats - 1:
            print("Average inference latency for bert-base fp16", np.mean(time_record))
    print("batch = ", batch)

max_stars_repo_name: QinHan-Erin/AMOS

max_issues_repo_name: QinHan-Erin/AMOS

max_forks_repo_name: QinHan-Erin/AMOS


text:
import numpy as np
import pandas as pd

import io
from PIL import Image

import matplotlib
from anndata import AnnData

from typing import Optional, Union, Mapping  # Special
from typing import Sequence, Iterable  # ABCs
from typing import Tuple  # Classes

from enum import Enum

from matplotlib import rcParams, ticker, gridspec, axes
from matplotlib.axes import Axes
from abc import ABC

def get_img_from_fig(fig, dpi=180):
    buf = io.BytesIO()
    from io import BytesIO

    fig.savefig(
        buf, format="png", dpi=dpi, bbox_inches="tight", pad_inches=0, transparent=True
    )
    buf.seek(0)
    img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)
    img = np.asarray(Image.open(BytesIO(img_arr)))
    buf.close()
    # img = cv2.imdecode(img_arr, 1)
    # img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGB)

    return img

def centroidpython(x, y):
    l = len(x)
    return sum(x) / l, sum(y) / l

def get_cluster(search, dictionary):
    for (
        cl,
        sub,
    ) in (
        dictionary.items()
    ):  # for name, age in dictionary.iteritems():  (for Python 2.x)
        if search in sub:
            return cl

def get_node(node_list, split_node):
    result = np.array([])
    for node in node_list:
        result = np.append(result, np.array(split_node[node]).astype(int))
    return result.astype(int)

def check_sublist(full, sub):
    index_bool = []
    for barcode in full:
        if barcode in sub:
            index_bool.append(True)
        else:
            index_bool.append(False)
    return index_bool

max_stars_repo_name: BiomedicalMachineLearning/stLearn

max_issues_repo_name: BiomedicalMachineLearning/stLearn

max_forks_repo_name: BiomedicalMachineLearning/stLearn


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 14 20:41:09 2015

@author: oliver
"""
import numpy as np
from sympy import symbols
import mubosym as mbs

###############################################################
# general system setup example
myMBS = mbs.MBSworld('reflective_wall', connect=False, force_db_setup=False)
#myMBS1 = MBS()

body = ["A", "B", "C"]
marker = ["A_M0", "B_M0", "C_M0"]
x0 = []

I = [1.,1.,1.]

######################################
# reflective wall reloaded
#setup clean line

#b_n[0] = myMBS.add_body_3d(999, 0, 1.0, I, 'rod-1-cardanic', parameters = [1.5,0.])
myMBS.add_body_3d(body[0], 'world_M0', 1.0, I, 'y-axes', parameters = [0.])
myMBS.add_force_special(body[0], 'grav')

myMBS.add_force(body[0],'world_M0', parameters = [100.,0.,0.])
myMBS.add_marker(marker[0], body[0], 0.,0.,0.)

for ii in range(len(body))[1:]:
    myMBS.add_body_3d(body[ii], marker[ii-1], 1.0, I, 'rod-1-cardanic', parameters = [-1.5,0.])
    myMBS.add_marker(marker[ii], body[ii],  0.,0.,0.)
    myMBS.add_force_special(body[ii], 'grav')

#myMBS.add_reflective_wall(b_n[1], eqn, IF, 20. )
#myMBS.add_reflective_wall(b_n[2], eqn, IF, 0. )

x0 = np.hstack(( 1. * np.ones(myMBS.dof), 0. * np.ones(myMBS.dof)))
#x0[2] = 0.
factor = 5.
R = myMBS.get_frame('world_M0')
x = R[0]
y = R[1]
z = R[2]

for b in myMBS.bodies.keys():
    myMBS.add_damping(b,0.1)

eqn = x-0.5
myMBS.add_reflective_wall(body[2], eqn, 'world_M0', 1000, .2, +1.)
myMBS.add_reflective_wall(body[1], eqn, 'world_M0', 1000, .2, +1.)
#str_m_b, equ, str_m_b_ref, c, gamma, s):
#eqn = x-2.0
#myMBS.add_reflective_wall(b_n[1], eqn, IF, 1000, .2 , mySyms, -1.)

#equ1 = y
#myMBS.add_geometric_constaint(2, equ1, IF, factor, mySyms)
#equ2 = x-1.5
#myMBS.add_geometric_constaint(2, equ2, IF, factor, mySyms)

#another constraint
#equ2 = x
#myMBS.add_geometric_constaint(2, equ2, IF, factor, mySyms)

#################################################
# constants
g = symbols('g')
constants = [ g ]          # Parameter definitions
constants_vals = [9.81]     # Numerical value

const_dict = dict(zip(constants, constants_vals))
myMBS.set_const_dict(const_dict)

#for ii in range(len(myMBS.eq_constr)/2):
#    x0 = myMBS.correct_the_initial_state(myMBS.n_constr[ii], x0)

body_frames_in_graphics = [body[0], body[1], body[2]]
fixed_frames_in_graphics = []

myMBS.kaneify()
myMBS.prep_lambdas(body_frames_in_graphics, fixed_frames_in_graphics)

dt = 0.01  # 10 ms for a nice animation result
t_max = 30.

myMBS.inte_grate_full(x0, t_max, dt, mode = 0)

x_final = myMBS.x_t[-1]

x00 = dict(zip(myMBS.q_flat+myMBS.u_flat, x_final))

x_op = {myMBS.q_flat[0]: 0.0, myMBS.q_flat[1]: np.pi/2., myMBS.q_flat[2]: np.pi/2., myMBS.u_flat[1]: 0.0, myMBS.u_flat[0]: 0.0, myMBS.u_flat[2]: 0.0}
#x_op = {myMBS.q_flat[0]: 0.0, myMBS.q_flat[1]: 0.0, myMBS.q_flat[2]: 0.0, myMBS.u_flat[1]: 0.0, myMBS.u_flat[0]: 0.0, myMBS.u_flat[2]: 0.0}

a_op = {myMBS.a_flat[0]: 0.0, myMBS.a_flat[1]: 0., myMBS.a_flat[2]: 0.}

jac = myMBS.calc_lin_analysis_n(len(myMBS.x_t)-1)

myMBS.prepare(mbs.DATA_PATH, save=True)
myMBS.animate(t_max, dt, scale = 4, time_scale = 1, t_ani = 30.0)

max_stars_repo_name: skidzo/mubosym

max_issues_repo_name: skidzo/mubosym

max_forks_repo_name: skidzo/mubosym

reason: import numpy,from sympy


text:
__author__ = 'wperkins'
"""
Compares LMR output between two archive directories.
"""

import glob
import pickle
import numpy as np
import os.path as path

# proxy-first version vs time-first version comparison
is_proxy_first_driver_loop = False

# Use this if the two reconstructions use different state variables
# Also use the
diff_state_vectors = False

# All close testing tolerances (checks for diff > atol + rtol*desired_value)
rtol = 1e-4
atol = 1e-6

base_dir = '/home/katabatic/wperkins/data/LMR/output/testing'

sim_dir1 = 'test_proxydb_lmrdb_ref/r0'
sim_dir2 = 'test_final_proxydb_lmrdb_sparse/r0'

dir1 = path.join(base_dir, sim_dir1)
dir2 = path.join(base_dir, sim_dir2)

d1_npfiles = glob.glob(path.join(dir1, '*.npz'))
d1_pckl_files = glob.glob(path.join(dir1, '*.pckl'))
d1_files = d1_npfiles + d1_pckl_files
d1_names = [path.split(fpath)[1] for fpath in d1_files]

d2_npfiles = glob.glob(path.join(dir2, '*.npz'))
d2_pckl_files = glob.glob(path.join(dir1, '*.pckl'))
d2_files = d2_npfiles + d2_pckl_files
d2_names = [path.split(fpath)[1] for fpath in d2_files]

# Match assimilated proxies
assim_fname = 'assimilated_proxies.npy'

assim1 = np.load(path.join(dir1, assim_fname), encoding='latin1')
assim2 = np.load(path.join(dir2, assim_fname), encoding='latin1')

print('Comparing assimilated proxy files...')
assim2_idx_match_to1 = []
for proxy2 in assim2:
    key2, item2 = next(iter(proxy2.items()))
    id2 = item2[0]
    for i, proxy in enumerate(assim1):
        key, item = next(iter(proxy.items()))
        id1 = item[0]
        if id1 == id2:
            assim2_idx_match_to1.append(i)
            for obj1, obj2 in zip(item, item2):
                if isinstance(obj1, np.ndarray):
                    np.testing.assert_array_equal(obj1, obj2)
                elif isinstance(obj1, float):
                    np.testing.assert_allclose(obj1, obj2, rtol=1e-6)
                else:
                    assert obj1 == obj2
            break

nproxies = len(assim2)
assert len(assim2) == len(assim1)
print('Found and matched content of {:d} assimilated proxies'.format(nproxies))

#Test that they created the same files. NOTE: if first dir is missing files
# they will not be checked.  Reference (first directory) should be the benchmark
for name in d1_names:
    assert name in d2_names

# Test equivalence of file contents
for idx, file in enumerate(d1_files):

    idx2 = d2_names.index(d1_names[idx])
    print(d1_names[idx])

    if diff_state_vectors:
        exclude = ['Xb_one.npz', 'gmt.npz', 'gmt_ensemble.npz']

    exclude = [
        # 'ensemble_mean_tas_sfc_Amon.npz',
        # 'ensemble_variance_tas_sfc_Amon.npz',
        # 'gmt.npz',
        # 'gmt_ensemble.npz',
    ]

    if d1_names[idx] in exclude:
        continue

    if path.splitext(file)[1] == '.npz':
        f1 = np.load(file)
        f2 = np.load(d2_files[idx2])

        for key, values in f1.items():
            values2 = f2[key]

            print('\t', key, ': ', values.dtype)

            if values.dtype.kind == 'S' or 'U' in values.dtype.kind:
                values = values.astype(str)
                values2 = values.astype(str)
                for i1, i2 in zip(values, values2):
                    assert i1 == i2
            elif not values.dtype == np.object:
                if is_proxy_first_driver_loop:
                    if key == 'Xb_one_aug':
                        # Need to re arrange proxie order
                        p1 = values[:-nproxies]
                        p2 = values2[:-nproxies]
                        np.testing.assert_allclose(p1, p2)

                        ye1 = values[-nproxies:]
                        ye1_rearr = ye1[assim2_idx_match_to1]
                        ye2 = values2[-nproxies:]
                        np.testing.assert_allclose(ye1_rearr, ye2)
                    elif key == 'gmt_save':
                        # Proxies assimilated in different order check ending
                        np.testing.assert_allclose(values[-1], values2[-1],
                                                   rtol=rtol)
                elif len(values.shape) > 1:
                    np.testing.assert_allclose(values.mean(axis=1),
                                               values2.mean(axis=1),
                                               rtol=rtol)
                else:
                    np.testing.assert_allclose(values, values2, rtol=rtol)
            else:
                assert values == values2

    elif path.splitext(file)[1] == '.pckl':
        with open(file, 'r') as load_file:
            f1 = pickle.load(load_file)
        with open(d2_files[idx2], 'r') as load_file:
            f2 = pickle.load(load_file)

        for key, dict in f1.items():
            dict2 = f2[key]

            for sub_key, value in dict.items():
                value2 = dict2[sub_key]
                if is_proxy_first_driver_loop and sub_key == 'HXa':
                    # Proxy order is different between the two right now
                    continue
                elif type(value) == np.ndarray:
                    np.testing.assert_allclose(value, value2)
                else:
                    assert value == value2
    else:
        continue

max_stars_repo_name: frodre/LMR

max_issues_repo_name: mingsongli/LMR

max_forks_repo_name: mingsongli/LMR


text:
import numpy as np

"""

define error processing functions

"""

# jacobian matrix to covariance
def jac_to_cov(jac):
    from scipy.linalg import svd
    _, s, VT = svd(jac, full_matrices=False)
    threshold = np.finfo(float).eps * max(jac.shape) * s[0]
    s = s[s > threshold]
    VT = VT[:s.size]
    pcov = np.dot(VT.T / s**2, VT)
    return pcov

# covariance to standard deviation
def cov_to_err(cov):
    return np.sqrt(np.diag(cov))

# jacobian matrix to standard deviation
def jac_to_err(jac):
    from scipy.linalg import svd
    _, s, VT = svd(jac, full_matrices=False)
    threshold = np.finfo(float).eps * max(jac.shape) * s[0]
    s = s[s > threshold]
    VT = VT[:s.size]
    pcov = np.dot(VT.T / s**2, VT)
    return np.sqrt(np.diag(pcov))

# inversed hessian matrix to standard deviation
def hessinv_to_err(hess_inv):
    return np.sqrt(np.diag(hess_inv))

# least_squares[jac] --> jacobian matrix --> standard deviation
def do_post(ls_result, label_scaler=None):
    """

    Parameters
    ----------
    ls_result:
        least_squares full output
        https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.least_squares.html

    label_scaler:
        SLAM label scale for training labels

    Returns
    -------
    pp_result:
        post-processed result

    """
    pp_result = dict()
    pp_result["cost"] = ls_result["cost"]
    pp_result["grad"] = ls_result["grad"]

    pp_result["pcov"] = jac_to_cov(ls_result["jac"]) * \
                        np.dot(label_scaler.scale_.reshape(-1, 1), label_scaler.scale_.reshape(1, -1))
    pp_result["pstd"] = cov_to_err(pp_result["pcov"])

    pp_result["message"] = ls_result["message"]
    pp_result["nfev"] = ls_result["nfev"]
    pp_result["optimality"] = ls_result["optimality"]
    pp_result["status"] = ls_result["status"]
    pp_result["success"] = ls_result["success"]

    pp_result["x"] = label_scaler.inverse_transform(ls_result["x"].reshape(1, -1)).flatten()

    return pp_result

reason: import numpy,from scipy


text:
import numpy as np

##############################################################################
# Input info to solve the problem
##############################################################################
# Input list to be searched
v = [1, 15, 4, 20, 9]
S = 14  # S is the sum that is desired for the numbers in the sublist
L = 3  # L is de desired lenght for the sublist

###############################################################################
# External values used to solve the problem
###############################################################################
# STATE in one indicate that the list is to be search
STATE = 1
# Array to indicate state of next jumps to try for each position given
nj = np.ones(len(v), dtype=int)
lj = np.zeros(len(v), dtype=int)
set_sum = v[0]
pos = 0
init_pos = 0
save_list = [v[0]]

###############################################################################
# Principal traceback loop algorithm
###############################################################################
# All combinations are tried having always in the list v[init_pos]; once this
#  is done with the last element there are not more sublist to be checked.
# The algorithm follows the following procedure:
# 1. Try to check if the sum and len desired is satisfied by v[init_pos], if
#    desired result stop, other case to 2.
# 2. The algorithm move to the nearest value/position to the right from the
#    acual position such that this have not been added to the list so far, if
#    find sum and len desired stop, if not try to repeat two and in case of
#    not value to the right try with 3.
# 3. Go back to the last position that has been tried from the actual position,
#    set up tried jumps to zero, if the actual position is not the last
#    position of the list try 2; otherwise finish program saying that the
#    sublist in search does not exist.

while(init_pos != (len(v) - 1) and STATE):
    # The searched list is found to be satisfied in this condition
    if (set_sum == S and len(save_list) == L):
        print("\n\nThe list is given by: ", save_list)
        STATE = 0

    # It is possible to try the next value to be summed, at nj[pos] steps from
    #   v[pos].
    elif(set_sum < S and len(save_list) < L and
         (pos + nj[pos]) < len(v)):
        lj[pos + nj[pos]] = -nj[pos]  # Aave how to go back
        pos += nj[pos]  # Go to next position
        set_sum += v[pos]  # Actualize sum
        save_list.append(v[pos])  # Add the actal value to list
        # print("c1: save_list: ", save_list)  # debug print

    # It is not possible to try the next value to be summed at nj[pos] steps
    #  from v[pos], then we go back a position and set to try the next jump
    #  that have been not been tried there.
    elif(nj[init_pos] != (len(v) - init_pos)):
        set_sum -= v[pos]  # Actuaize sum
        del save_list[-1]  # Delete value that makes condition not possible
        # Set up jumps tried in deleted value (actual pos)/(from list) again to
        #   1
        nj[pos] = 1
        pos += lj[pos]  # Go back to the last position
        nj[pos] += 1  # Set it up to try the next jump to the right
        # print("c2: save_list: ", save_list, set_sum)  # debug print

    # All possible combinations that include init_pos have been tried so far,
    #   therefore we advance in init_pos a step.
    elif(nj[init_pos] == (len(v) - init_pos)):
        # Make the initialization done for v initially, but taking as initial
        #  value the right nearest to init_pos.
        init_pos += 1
        nj = np.ones(len(v), dtype=int)
        lj = np.zeros(len(v), dtype=int)
        pos = init_pos
        set_sum = v[pos]
        save_list = [v[pos]]
        # print("c3: save_list: ", save_list)  # debug print

if (init_pos == (len(v) - 1)):
    print("There is not list found")

# All is done


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 30 15:12:43 2019

@author: belov
"""
import numpy as np
import math

def beam_search_decoder(n, model, encoded_image, word_map, id_to_word_dictionary, max_sentence_length):

    sentence_id = np.zeros((n, max_sentence_length))

    prob_seq_lst = []

    for i in range(n):
      prob_seq_lst.append((0,['<START>']))

    i = 0

    while i < max_sentence_length:

      prob_seq_lst_temp = []

      for prev_seq_ind in range(len(prob_seq_lst)):
        word_id = word_map[prob_seq_lst[prev_seq_ind][1][-1]]
        sentence_id[prev_seq_ind, i] = word_id
        next_word_id = model.predict([np.array([encoded_image]), np.reshape(sentence_id[prev_seq_ind, :], (1, max_sentence_length))])
        next_word_id = next_word_id.flatten()

        mid_seq = prob_seq_lst[prev_seq_ind][1].copy()
        next_word_id_sorted = np.sort(next_word_id)
        p = []
        c = 1
        while len(p) < n:
            prob = next_word_id_sorted[-c]
            if prob not in p:
                p.append(prob)
                ind = np.where(next_word_id == prob)
                word = id_to_word_dictionary[ind[0][0]]

                if c == 1:
                    mid_seq.append(word)
                else:
                    mid_seq[i+1] = word

                prob_seq_lst_temp.append((prob_seq_lst[prev_seq_ind][0] + math.log(prob),mid_seq))
                mid_seq = mid_seq.copy()

            c += 1

      prob_seq_lst = []
      i += 1
      prob_seq_sorted = sorted(prob_seq_lst_temp, reverse=True)
      p = []

      for prob_seq in prob_seq_sorted:
          if prob_seq[0] not in p:
              p.append(prob_seq[0])
              prob_seq_lst.append(prob_seq)
          if len(prob_seq_lst) == n:
            break

    sent_sorted = sorted(prob_seq_lst, reverse=True)
    final_output_sentences = [prob_sentence[1] for prob_sentence in sent_sorted]

    return final_output_sentences


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 23 12:53:10 2016

@author: DanielleT
@author: Diede
"""

from datetime import datetime
import pandas as pd
import numpy as np
import LoadData
import CreateClassification as cc
from Aggregator import Aggregator
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC

class SVM_spreiding:

    ''''Initialize SVM_spreiding. Loads the data and computes/loads aggregated data. '''''
    def __init__(self, loadAggData):
        print 'Initializing SVM: Loading Data....'
        self.loadAggData = loadAggData #boolean if aggregated data should be loaded

        #load general data
        generalData = LoadData.load('input')
        self.Xtrain = generalData['X_TRAIN']
        self.Ytrain = generalData['Y_TRAIN']
        self.Xtest = generalData['X_TEST']

        print 'Loading general data is done. Continue with loading caffe features.'
        print datetime.now()

        #if aggregated data needs to be generated, also load caffe features data
        if not self.loadAggData:
            #load train data
            trainFeatureData = LoadData.load_caffe_features('input', trainSet=True)
            self.trainFeatures = trainFeatureData['FEATURES']
            trainOrder = trainFeatureData['ORDER']

            print 'Loading caffe features is done. Continue with splitting train and validation set.'
            print datetime.now()

            #############################################
            #   SPLIT TRAINSET INTO TRAIN AND VALIDATION PART
            ##############################################

            #load business ids of train and validation set
            trainSetIds = np.load('input/trainSet.npy')
            validSetIds = np.load('input/verifSet.npy')

            #merge trainOrder with trainFeatures
            allData = pd.concat([trainOrder, self.trainFeatures], axis=1)

            #create dataframes of photo indices for train and validation set
            trainPhotoIds = self.Xtrain[self.Xtrain.business_id.isin(trainSetIds)]
            validPhotoIds = self.Xtrain[self.Xtrain.business_id.isin(validSetIds)]

            #make photo_ids strings --> merging is possible
            trainPhotoIds['photo_id'].apply(str)
            validPhotoIds['photo_id'].apply(str)

            #merge allData dataframe with photo indice dataframes --> split allData in train and validation
            self.validationFeatures = pd.merge(validPhotoIds, allData, on='photo_id')
            self.trainFeatures = pd.merge(trainPhotoIds, allData, on='photo_id')

            #delete validation dictionary from memory
            del trainFeatureData
            del trainOrder
            del allData

            #############################################
            #   COMPUTE AGGREGATED DATA
            ##############################################

            #compute validation aggregated data
            self.call_aggregator('validation')

            #delete validation data from memory
            del self.validationFeatures

            print 'VALIDATION SET DONE'
            print datetime.now()

            #compute train aggregated data
            self.call_aggregator('train')

            #delete train data from memory
            del trainFeatureData
            del self.trainFeatures
            del self.trainOrder

            print 'TRAIN SET DONE. HALFWAY THERE.'
            print datetime.now()

            #load test data
            testFeatureData = LoadData.load_caffe_features('input', trainSet=False)
            self.testFeatures = testFeatureData['FEATURES']
            self.testOrder = testFeatureData['ORDER']

            #merge testOrder with testFeatures
            self.testOrder.rename(columns={0:'photo_id'},inplace=True)
            allData = pd.concat([self.testOrder, self.testFeatures], axis=1)

            #merge testFeatures with X_test on photo_ids --> business_ids are available
            self.testFeatures = pd.merge(self.Xtest, allData, on='photo_id')

            #compute test aggregated data
            self.call_aggregator('test')

            #delete test data from memory
            del testFeatureData
            del self.testFeatures
            del self.testOrder
        else:
            self.load_aggregated_data()
        print 'Initializing SVM: Loading data is done.'
        print 'KLAAR'
        print datetime.now()

    ''''Call aggregator to compute aggregated data'''''
    def call_aggregator(self, setString):
        if setString == 'train':
            print 'Initializing SVM: Computing aggregated train data...'
            agg = Aggregator(self.trainFeatures, set='Train')
            self.aggTrainData = agg.get_aggregated_data()
        elif setString == 'validation':
            print 'Initializing SVM: Computing aggregated validation data...'
            agg = Aggregator(self.validationFeatures,  set='Validation')
            self.aggValData = agg.get_aggregated_data()
        elif setString == 'test':
            print 'Initializing SVM: Computing aggregated test data...'
            agg = Aggregator(self.testFeatures,  set='Test')
            self.aggTestData = agg.get_aggregated_data()
        else:
            print 'Error call_aggregator. This type of set is unknown.'
        print 'Initializing SVM: Computating is done.'

    ''''Load aggregated data from location'''''
    def load_aggregated_data(self):
        print 'Initializing SVM: Loading aggregated data from csv files...'
        df = pd.read_csv('./input/aggDataTrain.csv')
        self.aggTrainData = df.values
        df = pd.read_csv('./input/aggDataTest.csv')
        self.aggTestData = df.values
        df = pd.read_cs('./input/aggDataValidation.csv')
        self.aggValData = df.values
        print 'Initializing SVM: Loading is done.'

    def train(self):
        print 'Training SVM...'
        self.SVM = OneVsRestClassifier(LinearSVC(random_state=0)).fit(self.aggTrainData, self.Ytrain)
        print 'Training SVM is done.'

    def predict(self):
        print 'Predicting classes...'
        self.Ypred = self.SVM.predict(self.aggTestData)
        print 'Predicting is done.'

    def predict_prob(self):
        print 'Predicting probabilities of classes...'
        self.Yprob = self.SVM.predict(self.aggTestData)
        print 'Predicting probabilities is done.'

    def createClassification(self, filename):
        cc.create(self.Ypred, self.Xtest, filename)
        print 'Classification file created'

    def createProbabilities(self, filename):
        cc.createProbFile(self.Yprob, filename)
        print 'Probabilities file created'

max_stars_repo_name: TheLaurens/Team-Brains

max_issues_repo_name: TheLaurens/Team-Brains

max_forks_repo_name: TheLaurens/Team-Brains


text:
import os
import utils.aolib_util as ut
import utils.sound as sound
import numpy as np
import scipy.signal
import sys
from utils import data_cleaner

def subbands_from_sound(signal, filts):
    ft_signal = np.fft.rfft(signal)
    ft_subbands = filts * ft_signal[:, np.newaxis]
    subbands = np.fft.irfft(ft_subbands, axis = 0)
    return subbands.real

def sound_from_subbands(subbands, filts):
    ft_subbands = np.fft.rfft(subbands, axis = 0)
    ft = filts * ft_subbands
    ift = np.fft.irfft(ft, axis = 0)
    return np.sum(ift, axis = 1)

def freq2erb(freq_hz):
    return 9.265*np.log(1+freq_hz/(24.7*9.265))

def erb2freq(n_erb):
    return 24.7*9.265*(np.exp(n_erb/9.265)-1)

def rfftfreq(n, d=1.0):
    if not (isinstance(n,int) or isinstance(n, np.integer)):
        raise ValueError("n should be an integer")
    val = 1.0/(n*d)
    N = n//2 + 1
    results = np.arange(0, N, dtype=np.int)
    return results * val

def erb_filters(length, sr, nbands, low_lim, hi_lim):
    length = int(length)
    sr = float(sr)
    max_freq = sr/2.
    #freqs = np.fft.fftfreq(length, 1./sr)
    #freqs = freqs[freqs >= 0]
    freqs = rfftfreq(length, 1./sr)
    nfreqs = len(freqs)

    if hi_lim > sr/2:
        hi_lim = max_freq

    cos_filts = np.zeros((nfreqs, nbands))
    cutoffs = erb2freq(np.linspace(freq2erb(low_lim), freq2erb(hi_lim), 2 + nbands))
    for k in range(nbands):
        # adjacent filters overlap by 50%
        l = cutoffs[k]
        h = cutoffs[k+2]
        l_ind = np.flatnonzero(freqs > l)[0]
        h_ind = np.flatnonzero(freqs < h)[-1]
        avg = 0.5*(freq2erb(l) + freq2erb(h))
        rnge = freq2erb(h) - freq2erb(l)
        # Map cutoffs from -pi/2 to pi/2 interval (then to a kind of circular filter, although the value of each
        # filter component corresponds to its distance in ERB space to the average ERB value, so it's kind of nonlinear)
        cos_filts[l_ind : h_ind+1, k] = np.cos(((freq2erb(freqs[l_ind : h_ind+1]) - avg) / rnge) * np.pi)

    filts = np.zeros((nfreqs, nbands + 2))
    filts[:, 1:nbands+1] = cos_filts
    h_ind = np.flatnonzero(freqs < cutoffs[1])[-1]
    # lowpass filter goes to the peak of first cos filter
    filts[:h_ind+1, 0] = np.sqrt(1 - filts[:h_ind+1, 1]**2)
    l_ind = np.flatnonzero(freqs > cutoffs[nbands])[0]
    filts[l_ind:, nbands+1] = np.sqrt(1 - filts[l_ind:, nbands]**2)

    return filts, cutoffs, freqs

def resample(signal, sc, clip = True, num_samples = None):
    n = int(round(signal.shape[0]/sc)) if num_samples is None else num_samples
    r = scipy.signal.resample(signal, n)

    if clip:
        r = np.clip(r, -1, 1)
    return r

def good_resample_length(n, new_sr, old_sr, factor2 = 100):
    factor = new_sr / float(old_sr)
    new_len = int(np.floor(n / factor) * factor)
    return (new_len - (new_len%factor2))

def cut_signal(signal, new_sr, old_sr):
    new_signal = signal[: good_resample_length(signal.shape[0], new_sr, old_sr)]
    return new_signal

def invert_subband_envs(target_envs, final_sr, env_sr, mid_sr = 20000, niters = 3, low_lim = 20, hi_lim = 10000, comp_exp = 0.3):
    # number of subbands (w/o high/low pass)
    nbands = target_envs.shape[1]-2
    snd_len = int(np.ceil(target_envs.shape[0] * float(mid_sr) / float(env_sr)))
    snd_len -= (snd_len % 100)
    target_envs = np.clip(target_envs, 0, 1)

    #snd_len = good_resample_length(target_envs.shape[0], mid_sr, env_sr, 10)
    filts, _, _ = erb_filters(snd_len, mid_sr, nbands, low_lim, hi_lim)
    #cut_signal(signal, new_sr, old_sr):
    synth_sound = np.random.randn(snd_len)
    for i in range(niters):
        # Forward pass: current sound -> downsampled envelopes and full-res phases
        synth_subbands = subbands_from_sound(synth_sound, filts)
        analytic = scipy.signal.hilbert(synth_subbands, axis = 0)
        synth_envs = np.abs(analytic)
        phases = analytic / synth_envs

        #up_target_envs = resample(target_envs, mid_sr / env_sr)
        up_target_envs = scipy.signal.resample(target_envs, phases.shape[0])
        up_target_envs = np.maximum(up_target_envs, 0.)
        up_target_envs **= (1./comp_exp)

        new_analytic = phases * up_target_envs
        synth_subbands = np.real(new_analytic)
        synth_sound = sound_from_subbands(synth_subbands, filts)

    synth_sound = resample(synth_sound, final_sr / float(mid_sr))
    synth_sound = np.clip(synth_sound, -1., 1.)
    return sound.Sound(None, final_sr, synth_sound)

def subband_envs(snd, nbands, env_sr, num_samples = None, ds_sr = 20000, low_lim = 20, hi_lim = 10000, comp_exp = 0.3, desired_rms = None, clip_samples = True):
    ds_sr = float(ds_sr)
    signal0, sr0 = snd.normalized().to_mono().samples.flatten(), snd.rate
    if desired_rms is not None:
        est_rms = max(1e-5, np.sqrt(np.mean(np.asarray(signal0)**2)))
        signal0 = signal0 * (desired_rms / est_rms)

    sr0 = float(sr0) #=>snd.rate (fs)
    signal0 = cut_signal(signal0, ds_sr, sr0)
    signal = resample(signal0, ds_sr / sr0, clip = clip_samples)

    # number of samples in the signal that correspond to one subband envelope feature
    ds_factor = int(np.floor(ds_sr / env_sr))
    signal = signal[:int((len(signal)//ds_factor) * ds_factor)]
    audio_filts, Hz_cutoffs, freqs = erb_filters(len(signal), sr0, nbands, low_lim, hi_lim) # consistent w/ matlab
    subbands = subbands_from_sound(signal, audio_filts)
    analytic_subbands = scipy.signal.hilbert(subbands, axis = 0)
    subband_envs = np.abs(analytic_subbands)
    subband_envs **= comp_exp
    ds_factor = ds_sr/env_sr
    subband_envs = scipy.signal.resample(subband_envs, int(subband_envs.shape[0]/ds_factor))
    return subband_envs

def mod_filters(length, sr, num_mod_channels = 10, low_lim = 0.5, hi_lim = 200., circular_conv = True, q = 1.):
    """ Modulation filters. These are similar to the cochlear filters
    (make_erb_cos_filters), but they're applied to subband envs, rather
    than to the raw waveform, and so they are tuned to lower
    frequencies."""
    # see make_constQ_cos_filters.m
    max_freq = sr/2.
    freqs = rfftfreq(length, 1./sr)
    nfreqs = len(freqs)
    if hi_lim > sr/2:
        hi_lim = max_freq

  # center frequencies evenly spaced on log scale
    cos_filts = np.zeros((nfreqs, num_mod_channels))
    cfs = 2**np.linspace(np.log2(low_lim), np.log2(hi_lim), num_mod_channels)
    for k in range(num_mod_channels):
        # increase the width of the filter as a function of frequency
        bw = cfs[k]/q
        l = cfs[k] - bw
        h = cfs[k] + bw
        freq_ok = np.flatnonzero((l < freqs) & (freqs < h))
        l_ind, h_ind = freq_ok[0], 1+freq_ok[-1]
        avg = cfs[k]
        rnge = h - l
        # go from theta = -pi/2 to pi/2
        filt = np.cos(((freqs[l_ind : h_ind] - avg)/rnge)*np.pi)
        cos_filts[l_ind : h_ind, k] = filt

    # the filters won't sum to 1 (not sure why this only applies to the middle of the spectrum)
    # this probably doesn't matter as much for us, since we're not synthesizing sound
    energy = np.sum(cos_filts**2, axis = 1)
    energy_middle = energy[(freqs >= cfs[3]) & (freqs <= cfs[-4])]
    filts = cos_filts / np.sqrt(energy_middle.mean())
    return filts

def mod_power(envs, sr):
    filts = mod_filters(envs.shape[0], sr)
    mod_subbands = []
    for j in range(envs.shape[1]):
        mod_subbands.append(subbands_from_sound(envs[:, j], filts))
    return np.array(mod_subbands).transpose((1,0,2))

def texture_stats(envs, sr, normalize_envs = False):
    # marginal subband stats
    # not using a windowing function
    envs = envs.astype('float64')
    if normalize_envs:
        loudness = np.median(np.sqrt(np.sum(envs**2, axis = 1)))
        envs = envs / loudness
        #assert 0
    else:
        loudness = np.array(0.)

    eps = np.finfo(np.float32).eps
    env_mu = envs.mean(0)
    env_var_unnorm = envs.var(0)
    env_var_norm = env_var_unnorm / (eps + env_mu**2)
    env_stdev_norm = np.sqrt(env_var_norm)

    pairs = [1, 2, 3, 5]
    num_bands = envs.shape[1]
    corrs = []

    for i in range(num_bands):
        for j in pairs:
            ii = i + j
            if ii < num_bands:
                corrs.append(np.corrcoef(envs[:, i], envs[:, ii])[0, 1])

    corrs = np.array(corrs)
    # when a band has zero variance, this can happen
    corrs[np.isnan(corrs)] = 0.

    mods = mod_power(envs, sr)
    mod_pow_norm = np.zeros(mods.shape[1:])
    for j in range(num_bands):
        mod_pow_unnorm = np.mean(mods[:, j]**2, axis = 0)
        mod_pow_norm[j, :] = np.sqrt(mod_pow_unnorm / (eps + env_var_unnorm[j]))

    for x in [env_mu, env_stdev_norm, corrs, mod_pow_norm, loudness]:
        assert not np.any(np.isnan(x))
        assert not np.any(np.isinf(x))

    return env_mu, env_stdev_norm, corrs, mod_pow_norm, loudness

def extract_textures(audio_dir):
    #Magic numbers
    # These were set after reading lots and lots and lots of papers
    # and millions of combinations
    fps = 29.97
    samp_sr = 21000.
    frame_dur = 1/fps
    shift_dur = 4.2
    ds_sr = 20000
    env_sr = 400
    result_array = np.empty((0,502))
    for filename in os.listdir(audio_dir):
        if filename.endswith(".wav"):
            print(filename)
            if (os.path.isfile(sys.argv[2]+"/"+filename+".npy")):
                print("skipping "+filename)
            else:
                try:
                    snd = sound.load_sound(os.path.join(audio_dir, filename)).normalized()
                    sub_envs = subband_envs(snd, nbands=30, ds_sr=ds_sr, env_sr=env_sr)
                    a = texture_stats(sub_envs, sr=env_sr)
                    sound_textures = np.concatenate((a[0], a[1], a[2], a[3].ravel(), np.atleast_1d(a[4])), axis=0)
                    # np.save(sys.argv[2]+"/"+filename+'.npy', sound_textures)
                    result_array = np.append(result_array, [sound_textures], axis=0)
                except Exception:
                    print("something wrong with "+filename)
                    pass
    return result_array

results = extract_textures(sys.argv[1])

path_to_feats = sys.argv[2]
number_of_clusters = int(sys.argv[3])

# Run KMeans

kmeans = KMeans(n_clusters=number_of_clusters).fit(results)

# Generate CSV file with labels

myfile = open(sys.argv[2] + '/stat_feats_k'+str(number_of_clusters)+'.csv', 'w')
for i in range(len(audio_names)):
    myfile.write(audio_names[i][:-4]+".jpg,"+str(kmeans.labels_[i])+"\n")
myfile.close()

#Shuffle CSV file

os.system('shuf '+path_to_feats+'/stat_feats_k'+str(number_of_clusters)+'.csv > '+path_to_feats+'/stat_feats_k'+str(number_of_clusters)+'_shuf.csv')

# Split file into train and validation CSV

number_of_valid_samples = 5208 # Change as per your need

os.system('awk \'NR < '+str(number_of_valid_samples)+' { print >> "'+path_to_feats+'/val_stat_feats_k'+str(number_of_clusters)+'.csv"; next } {print >> "'+path_to_feats+'/train_stat_feats_k'+str(number_of_clusters)+'.csv" }\' '+path_to_feats+'/stat_feats_k'+str(number_of_clusters)+'_shuf.csv')

# 3. Create Trainable Dataset

train_df, val_df = data_cleaner.clean_dataset(path_to_feats+"/train_stat_feats_k"+str(number_of_clusters)+".csv", path_to_feats+"/val_stat_feats_k"+str(number_of_clusters)+".csv")

data_cleaner.create_h5(train_df, val_df, path_to_feats+"/train_stat_feats_k"+str(number_of_clusters)+".hdf5", path_to_feats+"/val_stat_feats_k"+str(number_of_clusters)+".hdf5")

reason: import numpy,import scipy


text:
### This script  will only do the pairwise DE for cell types provided as argument
### because pairwise DE is the slowest step, this makes it simpler to parallelize it
### https://github.com/WormBase/wormcells-viz
### please check that the default arguments match your annotations

### USER DEFINED ARGUMENTS
### PLEASE MAKE SURE THESE ARGUMENTS MATCH YOUR DATA

# path to anndata file on which to train the model
anndata_path = 'cengen.h5ad'
# this should be the label on which you'd like to stratify groups
# typically it is cell_type or cell_subtype
stratification_label = 'cell_subtype'

# minimum number of UMIs seen for each gene to kept
min_gene_counts = 100

### the adata.obs key on which scvi should perform batch correction
batch_key = 'sample_batch'

### END OF USER ARGUMENTS SECTION --- YOU SHOULDN'T NEED TO CHANGE ANYTHING BELOW HERE ####
import sys
print('Number of arguments:', len(sys.argv), 'arguments.')
print('Argument List:', str(sys.argv))
print('Starting imports...')
### IMPORTS ###
import anndata
import scvi
import numpy as np
from tqdm import tqdm
import pandas as pd
import os
import scanpy
import warnings

warnings.filterwarnings("ignore")
print('Using scvi-tools version:', scvi.__version__)

def load_or_train_scvi_model(model_name, anndata_path):
    # Try loading model, if it doesn't exist train from scratch
    print('Trying to load or train model...')
    try:
        model = scvi.model.SCVI.load(model_name)
        print('Loaded model:', model_name)
    except:
        ### DEFINE AND TRAIN MODEL
        # these hyperparameters are fine for a small dataset, with a few batches
        # if integration is a problem then you can try increasing the layers to 3
        # and hidden units to 256

        print('Creating and training model:', model_name)

        adata = anndata.read_h5ad(anndata_path)
        print(adata)
        print('Restricting to genes with minimum counts of ', min_gene_counts)
        adata.var['gene_counts'] = np.squeeze(np.asarray(adata.X.sum(0)))
        adata = adata[:, adata.var.gene_counts > min_gene_counts]
        print(adata)

        ## register adata with SCVI, for more information see
        ## https://docs.scvi-tools.org/en/stable/api/reference/scvi.data.setup_anndata.html
        adata.layers["counts"] = adata.X.copy().tocsr()  # converts to CSR format, preserve counts
        scvi.data.setup_anndata(adata,
                                layer="counts",
                                batch_key=batch_key)

        # typically you don't need to go tweak these parameters for training a model
        model = scvi.model.SCVI(adata,
                                n_hidden=256,
                                n_layers=2,
                                gene_likelihood='nb',
                                dispersion='gene-batch'
                                )

        # MODEL TRAINING
        # this model will train quickly even without a GPU, 25 epochs is not quite enough to
        # finish training, but this notebook is meant to run quickly just for showing the entire
        # data generation pipeline

        model.train(check_val_every_n_epoch=1,
                    use_gpu=True,
                    max_epochs=125,
                    plan_kwargs={'lr': 1e-3})

        train_test_results = model.history['elbo_train']
        train_test_results['elbo_validation'] = model.history['elbo_validation']

        ### MAKE SURE THE MODEL FINISHED TRAINING FOR BEST RESULTS
        print(train_test_results)
        model.save(model_name, save_anndata=True)
        # save the training results to a csv for inspection if needed
        train_test_results.to_csv(model_name + '+train_test_results.csv')
    return model

def compute_pairwise_de_one_group(group1_label,
                                  model,
                                  stratification_label,
                                  model_name
                                  ):
    adata = model.adata
    print('Doing pairwise DE for ', stratification_label, group1_label)
    csv_filename = model_name + '+pairwise_de_one_group+' + group1_label + '+.csv'
    if os.path.isfile(csv_filename):
        print('Skipping pairwise DE, csv file already exists: ', csv_filename)
        return None
    else:

        # for a given group1_label (eg `Intestine`) do pairwise DE vs all other labels in that category (eg all other cell types)
        obs_stratification_label_unique_values = adata.obs[stratification_label].unique()
        print('Starting pairwise DE for ', len(obs_stratification_label_unique_values), ' labels:')
        print(list(obs_stratification_label_unique_values))
        pairwise_de_one_group = pd.DataFrame()
        for group2_label in tqdm(obs_stratification_label_unique_values):
            de_df = model.differential_expression(
                groupby=stratification_label,
                group1=group1_label,
                group2=group2_label,
                silent=True,
                n_samples=5000,
                all_stats=False
            )
            de_df['group1'] = group1_label
            de_df['group2'] = group2_label
            pairwise_de_one_group = pairwise_de_one_group.append(de_df)
        # write to disk
        if not os.path.exists('pairwise_de'):
            os.makedirs('pairwise_de')
        pairwise_de_one_group.to_csv('./pairwise_de/'+csv_filename)
        return pairwise_de_one_group

if __name__ == '__main__':

    group1_label=sys.argv[1]
    stratification_label=sys.argv[2]
    model_name=sys.argv[3]

    print('Doing pairwise DE for label:', group1_label)
    print('Using stratification label:', stratification_label)
    print('Using model:', model_name)

    model = load_or_train_scvi_model(model_name=model_name, anndata_path=anndata_path)

    compute_pairwise_de_one_group(stratification_label=stratification_label,
                                  model=model,
                                  model_name=model_name,
                                  group1_label=group1_label)

max_stars_repo_name: WormBase/wormcells-viz

max_forks_repo_name: WormBase/wormcells-viz