text:
"""Translate a dictionary explaining a labeling rule into a function
"""
import re
from wiser.rules import TaggingRule # A TaggingRule instance is defined for a tagging LF.
from snorkel.labeling import LabelingFunction
from synthesizer.gll import *
import numpy as np

def raw_stringify(s):
    """From a regex create a regular expression that finds occurences of the string as entire words

    Args:
        s (string): the string to look for

    Returns:
        string: a regular expession that looks for this string surrounded by non word characters
    """
    return "(?:(?<=\W)|(?<=^))({})(?=\W|$)".format(re.escape(s))

def find_indices(cond_dict: dict, text: str):
    """Find all instances of this condition in the text
    """
    v = cond_dict.get("type")
    k = cond_dict.get("string")
    case_sensitive = True if cond_dict.get("case_sensitive") else False

    if v == KeyType[NER]:
        doc = nlp(text)
        for ent in doc.ents:
            if ent.label_ == k:
                return [(doc[ent.start].idx, doc[ent.end-1].idx + len(doc[ent.end-1].text))]
        return []
    elif v == KeyType[POS] or v == KeyType[DEP] or v == KeyType[ELMO_SIMILAR] or v == KeyType[BERT_SIMILAR]:
        return [(0,0)]
    elif case_sensitive:
        return [(m.start(), m.end()) for m in re.finditer(k, text)]
    else:
        return [(m.start(), m.end()) for m in re.finditer(k, text, re.IGNORECASE)]

def make_lf(instance, concepts, emb_dict, ui_label_dict):
    def apply_instance(self, instance):
        """
        An ``Instance`` is a collection of :class:`~allennlp.data.fields.field.Field` objects,
        specifying the inputs and outputs to
        some model.  We don't make a distinction between inputs and outputs here, though - all
        operations are done on all fields, and when we return arrays, we return them as dictionaries
        keyed by field name.  A model can then decide which fields it wants to use as inputs as which
        as outputs.

        The ``Fields`` in an ``Instance`` can start out either indexed or un-indexed.  During the data
        processing pipeline, all fields will be indexed, after which multiple instances can be combined
        into a ``Batch`` and then converted into padded arrays.

        Parameters
        ----------
        instance : An ``Instance`` is a collection of :class:`~allennlp.data.fields.field.Field` objects.
            instance['fields'] : ``Dict[str, Field]``
        ex:
            instance['fields']['tags'] = ['ABS', 'I-OP', 'ABS', 'ABS']
        """
        label = self.lf_dict.get(LABEL)
        direction = bool(self.lf_dict.get(DIRECTION))
        conn = self.lf_dict.get(CONNECTIVE)
        conds = self.lf_dict.get(CONDS)
        types = [cond.get("TYPE_") for cond in conds]
        strs_ = [cond.get("string") for cond in conds]
        #labels = np.array([np.array(['ABS'] * len(instance['tokens']), dtype=object)] * len(conds))

        type_ = conds[0].get("TYPE_")
        str_ = conds[0].get("string")
        is_pos = conds[0].get("positive")
        labels = np.array(['ABS'] * len(instance['tokens']), dtype=object)
        if type_ == BERT_SIMILAR or type_ == ELMO_SIMILAR:
            emb_type = 'bert' if type_ == BERT_SIMILAR else 'elmo'
            emb_thres = BERT_THRESHOLD if type_ == BERT_SIMILAR else ELMO_THRESHOLD
            sent_id = self.lf_dict.get(SENT_ID)
            tok_id = self.lf_dict.get(TOK_ID)
            if type(tok_id) == list:
                target_emb = self.emb_dict.emb_dict[emb_type][sent_id][tok_id]
                emb = instance[emb_type]
                cos_scores = target_emb @ emb.T
                if is_pos:
                    similar_inds = (cos_scores > emb_thres)
                else:
                    similar_inds = (cos_scores <= emb_thres)
                similar_ind = similar_inds[0][:-len(tok_id)+1]
                for i in range(1,len(tok_id)):
                    similar_ind = (similar_ind) & (similar_inds[i][i:len(similar_inds[i])-len(tok_id)+1+i])
                for i in range(len(tok_id)):
                    labels[i:len(labels)-len(tok_id)+1+i][similar_ind] = ui_label_dict[label]
            else:
                # target_emb is the emb vec corresponding to the similarity target word
                target_emb = self.emb_dict.emb_dict[emb_type][sent_id][tok_id]
                # emb is an N x M matrix containing token embeddings in a sentence
                emb = instance[emb_type]
                cos_scores = np.dot(emb, target_emb)
                if is_pos:
                    labels[cos_scores > emb_thres] = ui_label_dict[label]
                else:
                    labels[cos_scores <= emb_thres] = ui_label_dict[label]
                # try:
                # except:
                #     print('error')
        elif type_ == POS:
            for i, pos in enumerate([token.pos_ for token in instance['tokens']]):
                if (pos in str_ and is_pos) or (pos not in str_ and not is_pos):
                    labels[i] = ui_label_dict[label]
        elif type_ == DEP:
            for i, dep in enumerate([token.dep_ for token in instance['tokens']]):
                if (dep in str_ and is_pos) or (dep not in str_ and not is_pos):
                    labels[i] = ui_label_dict[label]
        elif type_ == NER:
            for i, ner in enumerate([token.ent_type_ for token in instance['tokens']]):
                if (ner in str_ and is_pos) or (ner not in str_ and not is_pos):
                    labels[i] = ui_label_dict[label]
        return list(labels)

    def lf_init(self):
        pass

    LF_class = type(instance['name'], (TaggingRule,), {"__init__":lf_init, "lf_dict":instance, "apply_instance":apply_instance, "emb_dict":emb_dict, "name":instance['name']} )
    return LF_class()

max_stars_repo_name: megagonlabs/tagruler

max_issues_repo_name: megagonlabs/tagruler

max_forks_repo_name: megagonlabs/tagruler

reason: import numpy


text:
#!/usr/bin/env python

import torch
import torch.nn as nn
import numpy as np

class BReLU(nn.Hardtanh):
    r"""Applies the element-wise function:

    .. math::
        \text{BReLU}(x) = \min(\max(0,x), cutoff)

    Args:
        cutoff: can optionally set the maximum accepted output
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    modified from ReLU6

    Examples::

        >>> m = nn.BReLU(cutoff, inpalce)
        >>> input = torch.randn(2)
        >>> output = m(input)
    """

    def __init__(self, cutoff=24., inplace=False):
        super(BReLU, self).__init__(0., cutoff, inplace)

    def extra_repr(self):
        inplace_str = 'inplace=True' if self.inplace else ''
        return inplace_str

class Mask(nn.Module):
    '''
    Args:
        x: input 4d tensor [batch, channel, height, width]. only support batch first
        length: true height of each data. It's length should be the same as batch size
    '''
    def __init__(self):
        super(Mask, self).__init__()

    def forward(self,x, length):
        assert x.shape[0] == len(length)

        mask = torch.zeros_like(x, dtype=torch.float32)
        for index, length in enumerate(length):
            mask[index, :, :length, :] = 1
        return x * mask

class Conv_bn_mask(nn.Module):
    def __init__(self, ichannel, ochannel, kernel_size, padding, stride, bias=False, track_running_stats=False):
        super(Conv_bn_mask, self).__init__()
        self.conv = nn.Conv2d(ichannel, ochannel, kernel_size=kernel_size, padding=padding, stride=stride, bias=bias)
        self.bn = nn.BatchNorm2d(ochannel, track_running_stats=track_running_stats)
        self.activation = BReLU(cutoff=24)
        self.mask = Mask()
    def forward(self, x, length):
        x = self.conv(x)
        x = self.bn(x)
        x = self.activation(x)
        x = self.mask(x, length)
        return x

ext: py


text:
 #####################################################################################
 # MIT License                                                                       #
 #                                                                                #
 # Copyright (C) 2019 Charly Lamothe, Guillaume Ollier, Balthazar Casalé             #
 #                                                                                #
 # This file is part of Joint-Text-Image-Representation.                             #
 #                                                                                #
 #   Permission is hereby granted, free of charge, to any person obtaining a copy    #
 #   of this software and associated documentation files (the "Software"), to deal   #
 #   in the Software without restriction, including without limitation the rights    #
 #   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell       #
 #   copies of the Software, and to permit persons to whom the Software is           #
 #   furnished to do so, subject to the following conditions:                        #
 #                                                                                #
 #   The above copyright notice and this permission notice shall be included in all  #
 #   copies or substantial portions of the Software.                                 #
 #                                                                                #
 #   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR      #
 #   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,        #
 #   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE     #
 #   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER          #
 #   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,   #
 #   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE   #
 #   SOFTWARE.                                                                       #
 #####################################################################################

from representation.imagenet_utils import preprocess_input

import numpy as np
from keras.applications.resnet50 import ResNet50
from keras.preprocessing import image

class ImageRepresentation(object):

    def __init__(self, img_path):
        self._img_path = img_path
        self._resnet_model = ResNet50(weights='imagenet', include_top=False, pooling='avg')

    @staticmethod
    def load_features(path):
        # numpy matrix containing already computed features for the
        # whole set of 10,000 images in dataset
        return np.load(path)

    def extract_features(self):
        # extract features for new images
        img = image.load_img(self._img_path, target_size=(224, 224))
        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x = preprocess_input(x)
        features = self._resnet_model.predict(x)
        return np.expand_dims(features.flatten(), axis=0)


text:
# pylint: disable=too-many-public-methods
'''
This module sets the specifications for certain atmospheric variables. Typically
this is related to a spec that needs some level of computation, i.e. a set of
colors from a color map.
'''

import abc
from itertools import chain
from matplotlib import cm
from matplotlib import colors as mpcolors
import numpy as np
import yaml
from metpy.plots import ctables

class VarSpec(abc.ABC):

    '''
    Loads a yaml config file with spec settings. Also defines methods for
    declaring more complex specifications for variables based on settings within
    the config file.
    '''

    def __init__(self, config):

        with open(config, 'r') as cfg:
            self.yml = yaml.load(cfg, Loader=yaml.Loader)

    def centered_diff(self):

        ''' Returns the colors specified by levels and cmap in default spec, but
        with white center. '''

        clevs = self.vspec.get('clevs')
        nlev = len(clevs) + 1

        colors = cm.get_cmap(self.vspec.get('cmap'), nlev)(range(nlev))
        mid = nlev // 2

        colors[mid] = [1, 1, 1, 1]
        colors[mid-1] = [1, 1, 1, 1]

        return colors

    @property
    def cin_colors(self) -> np.ndarray:

        ''' Default color map for Convective Inhibition '''

        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([120, 100, 90, 85, 80, 70, 60, 50, 25, 20, 18])
        grays = cm.get_cmap('Greys', 2)([0])
        return np.concatenate((ncar, grays))

    @property
    @abc.abstractmethod
    def clevs(self) -> np.ndarray:

        ''' An abstract method responsible for returning the np.ndarray of contour
        levels for a given field. Numpy arange supports non-integer values. '''

    @property
    @abc.abstractproperty
    def vspec(self):

        ''' The variable plotting specification. The level-specific subgroup
        from a config file like default_specs.yml. '''

    @property
    def ceil_colors(self) -> np.ndarray:

        ''' Default color map for Ceiling '''

        grays = cm.get_cmap('Greys', 2)([0])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([15, 18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120])
        return np.concatenate((grays, ncar, grays))

    @property
    def cldcov_colors(self) -> np.ndarray:

        ''' Default color map for Cloud Cover '''

        grays = cm.get_cmap('Greys', 7)([0, 1, 3])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([120, 100, 90, 85, 80, 70, 60, 50, 25, 20])
        return np.concatenate((grays, ncar))

    @property
    def cref_colors(self) -> np.ndarray:

        ''' Default color map for Reflectivity '''

        ncolors = len(self.clevs)-1
        grays = cm.get_cmap('Greys', 5)([0])
        nws = ctables.colortables.get_colortable(self.vspec.get('cmap'))(range(ncolors))
        white = cm.get_cmap('Greys', 5)([0])
        return np.concatenate((grays, nws, white))

    @property
    def dewp_colors(self) -> np.ndarray:

        ''' Default color map for Dew point temperature '''

        ctable = ctables.colortables.get_colortable(self.vspec.get('cmap')) \
                    (range(0, 42, 1)) # Carbone42_r
        return ctable

    @property
    def fire_power_colors(self) -> np.ndarray:

        ''' Default color map for fire power plot. '''

        blues = cm.get_cmap('Blues', 3)(range(3))
        green_orange = cm.get_cmap('RdYlGn_r', 10)([1, 7, 8, 9])
        return np.concatenate((blues, green_orange))

    def flru_colors(self) -> np.ndarray:

        ''' Default color map for Ceiling '''

        ctable = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([50, 15, 90, 120])
        return ctable

    @property
    def frzn_colors(self) -> np.ndarray:

        ''' Default color map for Frozen Precip % '''

        grays = cm.get_cmap('Greys', 7)([0, 2])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([120, 90, 85, 80, 70, 60, 50, 25, 20, 15])
        return np.concatenate((grays, ncar))

    @property
    def goes_colors(self) -> np.ndarray:

        ''' Default color map for simulated GOES IR satellite '''

        grays = cm.get_cmap('Greys_r', 33)(range(33))
        ctable2 = ctables.colortables.get_colortable(self.vspec.get('cmap')) \
                    (range(65, 150))
        return np.concatenate((grays[-1:], grays, ctable2, grays[1:]))

    @property
    def graupel_colors(self) -> np.ndarray:

        ''' Default color map for Max Vertically Integrated Graupel '''

        grays = cm.get_cmap('Greys', 3)([0])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          (range(20, 128, 6))
        return np.concatenate((grays, ncar))

    @property
    def hail_colors(self) -> np.ndarray:

        ''' Default color map for Hail diameter '''

        grays = cm.get_cmap('Greys', 2)([0])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([100, 15, 18, 20, 25, 60, 80, 85, 90])
        return np.concatenate((grays, ncar))

    @property
    def heat_flux_colors(self) -> np.ndarray:

        ''' Default color map for Latent/Sensible Heat Flux '''

        grays = cm.get_cmap('Greys', 8)([6, 5, 4, 3, 2])
        ctable = ctables.colortables.get_colortable(self.vspec.get('cmap')) \
                    (range(0, 33, 2))
        return np.concatenate((grays, ctable))

    @property
    def lcl_colors(self) -> np.ndarray:

        ''' Default color map for Lifted Condensation Level '''

        ctable = ctables.colortables.get_colortable(self.vspec.get('cmap')) \
                    (range(50, 180, 7)) # rainbow
        return ctable

    @property
    def lifted_index_colors(self) -> np.ndarray:

        ''' Default color map for Lifted Index '''

        ctable = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          (range(4, 125, 4))
        ctable[14] = [1, 1, 1, 1]
        ctable[15] = [1, 1, 1, 1]
        return ctable

    @property
    def mdn_colors(self) -> np.ndarray:

        ''' Default color map for Max Downdraft '''

        grays = cm.get_cmap('Greys', 2)([0])
        others = cm.get_cmap(self.vspec.get('cmap'), 18)(range(18, 1, -1), alpha=0.6)
        return np.concatenate((others, grays))

    @property
    def mean_vvel_colors(self) -> np.ndarray:

        ''' Default color map for Mean Vertical Velocity '''

        ctable = cm.get_cmap(self.vspec.get('cmap'), 128)(range(0, 114, 6))
        ctable[9] = [1, 1, 1, 1]
        return ctable

    @property
    def mup_colors(self) -> np.ndarray:

        ''' Default color map for Max Updraft '''

        grays = cm.get_cmap('Greys', 2)([0])
        others = cm.get_cmap(self.vspec.get('cmap'), 18)(range(1, 18, 1), alpha=0.6)
        return np.concatenate((grays, others))

    @property
    def pbl_colors(self) -> np.ndarray:

        ''' Default color map for PBL Height '''

        return ctables.colortables.get_colortable(self.vspec.get('cmap')) \
                          (range(15, 60, 3))

    @property
    def pcp_colors(self) -> np.ndarray:

        ''' Default color map for Hourly Precipitation '''

        grays = cm.get_cmap('Greys', 6)([0, 3])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([25, 50, 60, 70, 80, 85, 90, 115])
        return np.concatenate((grays, ncar))

    @property
    def ps_colors(self) -> np.ndarray:

        ''' Default color map for Surface Pressure '''

        grays = cm.get_cmap('Greys', 13)(range(13))
        segments = [[16, 53], [86, 105], [110, 151, 2], [172, 202, 2]]
        ncar = cm.get_cmap('gist_ncar', 200)(list(chain(*[range(*i) for i in segments])))
        return np.concatenate((grays, ncar))

    @property
    def pw_colors(self) -> np.ndarray:

        ''' Default color map for Precipitable Water '''

        grays = cm.get_cmap('Greys', 5)([1, 3])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([120, 100, 95, 85, 80, 70, 65, 50, 25, 22, 20, 17])
        bupu = cm.get_cmap('BuPu', 15)([13, 14])
        cool = cm.get_cmap('cool', 15)([10, 9, 12, 7, 5])
        return np.concatenate((grays, ncar, bupu, cool))

    @property
    def radiation_colors(self) -> np.ndarray:

        ''' Default color map for Longwave Radiation '''

        grays = cm.get_cmap('Greys', 2)([0])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          (range(0, 126, 5))
        return np.concatenate((grays, ncar))

    @property
    def radiation_bw_colors(self) -> np.ndarray:

        ''' Default grayscale map for Outgoing Shortwave Radiation '''

        return cm.get_cmap(self.vspec.get('cmap'), 128) \
                          (range(30, 110))

    @property
    def radiation_mix_colors(self) -> np.ndarray:

        ''' Default color map for Longwave Radiation '''

        ncar = ctables.colortables.get_colortable(self.vspec.get('cmap')) \
                            (range(0, 40))
        grays = cm.get_cmap('Greys', 100)(range(10, 100))
        return np.concatenate((ncar, grays))

    @property
    def rainbow12_colors(self) -> np.ndarray:

        ''' Default color map for ACPCP, ACSNOD, HLCY, RH, and SNOD '''

        grays = cm.get_cmap('Greys', 2)([0])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([15, 18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120])
        return np.concatenate((grays, ncar))

    @property
    def rainbow12_reverse(self) -> np.ndarray:

        ''' Default color map for min helicity '''

        return np.flip(self.rainbow12_colors, 0)

    @property
    def shear_colors(self) -> np.ndarray:

        ''' Default color map for Vertical Shear '''

        ctable = cm.get_cmap(self.vspec.get('cmap'), 16) \
                          (range(5, 15))
        ctable[9] = [1, 1, 1, 1]
        return ctable

    @property
    def smoke_colors(self) -> np.ndarray:

        ''' Default color map for smoke plots. '''

        blues = cm.get_cmap('Blues', 6)(range(5))
        green_yellow_red = cm.get_cmap('RdYlGn_r', 18)([1, 3, 5, 9, 12, 13, 14, 16, 18])
        purple = np.array([mpcolors.to_rgba('xkcd:vivid purple')])
        return np.concatenate((blues, green_yellow_red, purple))

    @property
    def snow_colors(self) -> np.ndarray:

        ''' Default color map for Snow fields '''

        grays = cm.get_cmap('Greys', 5)([0, 2])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([15, 18, 20, 25, 50, 60, 74, 81, 85, 90, 100])
        return np.concatenate((grays, ncar))

    @property
    def soilm_colors(self) -> np.ndarray:

        ''' Default color map for Soil Moisture Availability '''

        ncar = cm.get_cmap(self.vspec.get('cmap'), 128)(range(0, 122, 11))
        return ncar

    @property
    def soilw_colors(self) -> np.ndarray:

        ''' Default color map for Soil Moisture '''

        grays = cm.get_cmap('Greys', 2)([1])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 110) \
               ([0, 10, 20, 25, 35, 40, 60, 73, 80, 85, 95, 105])
        return np.concatenate((grays, ncar))

    @property
    def t_colors(self) -> np.ndarray:

        ''' Default color map for Upper-Air Temperature '''

        ncolors = len(self.clevs)
        return cm.get_cmap(self.vspec.get('cmap', 'jet'), ncolors)(range(ncolors))

    @property
    def tsfc_colors(self) -> np.ndarray:

        ''' Default color map for Surface Temperature '''  # WeatherBell-inspired scheme

        temp1 = cm.get_cmap('cool_r', 8)(range(0, 8))
        temp2 = cm.get_cmap('BuGn', 6)(range(2, 6))
        temp3 = cm.get_cmap('Greens_r', 4)(range(0, 4))
        temp4 = cm.get_cmap('RdPu_r', 8)(range(0, 8))
        temp5 = cm.get_cmap('BuPu', 5)(range(0, 4))
        temp6 = cm.get_cmap('RdYlBu_r', 10)(range(1, 10))
        temp7 = cm.get_cmap('RdYlGn', 10)(range(0, 10))

        return np.concatenate((temp1, temp2, temp3, temp4, temp5, temp6, temp7))

    @property
    def terrain_colors(self) -> np.ndarray:

        ''' Default color map for Terrain '''

        ctable = ctables.colortables.get_colortable(self.vspec.get('cmap')) \
                    (range(0, 21, 1))
        return ctable

    @property
    def vis_colors(self) -> np.ndarray:

        ''' Default color map for Visibility

        section names are based on Aviation Flight Rule visibility categories
        LIFR (Low Instrument Flight Rules) -- less than 1 mile
        IFR (Instrument Flight Rules) -- 1 mile to less than 3 miles
        MVFR (Marginal Visual Flight Rules) -- 3 to 5 miles
        VFR (Visual Flight Rules) -- greater than 5 miles
        '''

        lifr = cm.get_cmap('RdPu_r', 20)(range(0, 11))
        ifr = cm.get_cmap('autumn', 30)(range(0, 30))
        mvfr = cm.get_cmap('Blues', 20)(range(10, 20))
        vfr1 = cm.get_cmap('YlGn_r', 60)(range(0, 50))
        vfr2 = cm.get_cmap('Reds', 15)(np.full(51, 1))

        return np.concatenate((lifr, ifr, mvfr, vfr1, vfr2))

    @property
    def vvel_colors(self) -> np.ndarray:

        ''' Default color map for Vetical Velocity '''

        ncar1 = cm.get_cmap(self.vspec.get('cmap'), 128)([15, 18, 20, 25])
        grays = cm.get_cmap('Greys', 2)([0])
        ncar2 = cm.get_cmap(self.vspec.get('cmap'), 128)([60, 70, 80, 85, 90, 100, 120])
        return np.concatenate((ncar1, grays, ncar2))

    @property
    def vort_colors(self) -> np.ndarray:

        ''' Default color map for Absolute Vorticity '''

        grays = cm.get_cmap('Greys', 2)([0])
        ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \
                          ([15, 18, 20, 25, 50, 60, 70, 80, 83, 90, 100, 120])
        return np.concatenate((grays, ncar))

    @property
    def wind_colors(self) -> np.ndarray:

        ''' Default color map for Wind Speed '''

        low = cm.get_cmap(self.vspec.get('cmap'), 129)(range(129, 109, -5))
        high1 = cm.get_cmap(self.vspec.get('cmap'), 129)(range(16, 29, 3))
        high2 = cm.get_cmap(self.vspec.get('cmap'), 129)(range(48, 103, 6))
        return np.concatenate((low, high1, high2))

    @property
    def wind_colors_high(self) -> np.ndarray:

        ''' Default color map for High Wind Speed '''

        low = cm.get_cmap(self.vspec.get('cmap'), 129)(range(129, 108, -7))
        high1 = cm.get_cmap(self.vspec.get('cmap'), 129)(range(16, 29, 4))
        high2 = cm.get_cmap(self.vspec.get('cmap'), 129)(range(46, 95, 7))
        return np.concatenate((low, high1, high2))


text:
#!/usr/bin/env python
# coding: utf-8

# Baseado em https://www.pyimagesearch.com/2020/07/13/r-cnn-object-detection-with-keras-tensorflow-and-deep-learning/

# # Importa os pré-requisitos

import os
from bs4 import BeautifulSoup
from imutils import paths

import cv2
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications import MobileNetV2
from tensorflow.keras.layers import AveragePooling2D
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Model
from tensorflow.keras.models import load_model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.utils import to_categorical
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from skimage.transform import resize
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import argparse
import pickle
import os
from datetime import datetime

# # Define variáveis do ambiente

#Limite das probabilidades - se for menor então considera que o objeto não está na imagem
THRESHOLD = 0.9

# Limite acima do qual considera que as caixas sobrepostas são o mesmo objeto
NMS_THRESHOLD = 0.15

#Número de classes
NRCLASSES = 20

#Dimensões da imagem que deve ser passada para a rede
INPUT_DIMS = (224, 224)
# define the minimum probability required for a positive prediction
# (used to filter out false-positive predictions)
MIN_PROBA = 0.99

#Variáveis para o treimanento
#Learning rate, qtdade de épocas e batch size
INIT_LR = 1e-4
EPOCHS = 100
BS = 16

MAX_PROPOSALS_INFER = 20

#Nome da pasta
COLLAB=False
DEBUG=True

# Prefixo do arquivo do modelo
PREFIX = "object_detector.h5"
PREFIX = PREFIX + datetime.today().strftime('%Y-%m-%d')

# Se está usando o collab então monta o drive
if COLLAB:
	#Mount gdrive
	from google.colab import drive
	drive.mount("/content/gdrive", force_remount=True)
	DATA_PATH="objetos-mesa"
	get_ipython().system("ln -s gdrive/'My Drive'/'Object Detection Dataset'/'objetos-mesa' .")
else:
	DATA_PATH="input-data/dataset-treinamento-2020"

#Inicializa os vetores que vão ser usados no treinamento
data = []
labels = []
image_count = 0

print("Carregando imagens")

#Lê o arquivo index.txt que tem os prefixos das imagens e das anotações
f = open(DATA_PATH + os.path.sep + 'index.txt')
for linha in f:
  #Tira o \n no final de cada linha
	linha = linha[:-1]

	if DEBUG:
		print("[INFO] processing image ", linha)

	# extract the filename from the file path and use it to derive
	# the path to the XML annotation file

  #nome do arquivo da imagem
	imgFilename = DATA_PATH + os.path.sep + linha + '.jpg'

  # para fazer testes em um conjunto menor de imagens
  # descomente as linhas a seguir pois isso irá carregar
  # um conjunto menor de imagens (6 somente)
	#image_count = image_count + 1
	#if (image_count > 5):
	#	break

  #nome do arquivo da anotação
	annotFilename = DATA_PATH + os.path.sep + linha + '.xml'

	# load the annotation file, build the soup, and initialize our
	# list of ground-truth bounding boxes
	contents = open(annotFilename).read()
	soup = BeautifulSoup(contents, "html.parser")

	# extract the image dimensions
	w = int(soup.find("width").string)
	h = int(soup.find("height").string)

	#continue

	# loop over all 'object' elements
	for o in soup.find_all("object"):
		# extract the label and bounding box coordinates
		label = o.find("name").string
		xMin = int(o.find("xmin").string)
		yMin = int(o.find("ymin").string)
		xMax = int(o.find("xmax").string)
		yMax = int(o.find("ymax").string)
		# truncate any bounding box coordinates that may fall
		# outside the boundaries of the image
		xMin = max(0, xMin)
		yMin = max(0, yMin)
		xMax = min(w, xMax)
		yMax = min(h, yMax)

  	# load the input image (224x224) and preprocess it
		image = load_img(imgFilename, target_size=INPUT_DIMS)
		image = img_to_array(image)
		image = preprocess_input(image) #

    # a imagem é lida em 224x224 então é preciso calcular as dimensões
    # dos objetos anotados dentro dessa nova proporção de imagem
		ratio_w0 = round(w / INPUT_DIMS[0])
		ratio_h0 = round(h / INPUT_DIMS[1])
		xMin0 = round(xMin / ratio_w0)
		xMax0 = round(xMax / ratio_w0)
		yMin0 = round(yMin / ratio_h0)
		yMax0 = round(yMax / ratio_h0)

    # seleciona a parte da imagem onde o objeto anotado está
		obj_image = image[yMin0:yMax0, xMin0:xMax0]

    # a entrada da rede espera uma imagem 224x224 então redimensiona o objeto
		obj_resized = resize(obj_image, (INPUT_DIMS[0], INPUT_DIMS[1]))

		data.append(obj_resized)
		labels.append(label)

#quit()

# In[ ]:

matplotlib.use('GTK3Agg')

if DEBUG:
	print(labels[0])
	data[0].shape
	plt.imshow(data[0])
	for i in range(1,8):
	  #print(i)
	  print(labels[i])
	  plt.imshow(data[i])

print("Convertendo imagens para NumPy")

# convert the data and labels to NumPy arrays
data_np = np.array(data, dtype="float32")
data_np.shape
labels_np = np.array(labels)
labels_np.shape
# perform one-hot encoding on the labels
lb = LabelBinarizer()
labels_b = lb.fit_transform(labels_np)

if DEBUG:
	data_np.shape
	labels_b.shape

# # Prepara o modelo

print("Preparando os dados de treinamento")

# partition the data into training and testing splits using 75% of
# the data for training and the remaining 25% for testing
(trainX, testX, trainY, testY) = train_test_split(data_np, labels_b,
	test_size=0.20, stratify=labels, random_state=42)
# construct the training image generator for data augmentation
aug = ImageDataGenerator(
	rotation_range=20,
	zoom_range=0.15,
	width_shift_range=0.2,
	height_shift_range=0.2,
	shear_range=0.15,
	horizontal_flip=True,
	fill_mode="nearest")

# In[ ]:

print("Criando modelo")

# load the MobileNetV2 network, ensuring the head FC layer sets are
# left off
baseModel = MobileNetV2(weights="imagenet", include_top=False,
	input_tensor=Input(shape=(224, 224, 3)))
# construct the head of the model that will be placed on top of the
# the base model
headModel = baseModel.output
headModel = AveragePooling2D(pool_size=(7, 7))(headModel)
headModel = Flatten(name="flatten")(headModel)
headModel = Dense(128, activation="relu")(headModel)
headModel = Dropout(0.5)(headModel)
headModel = Dense(NRCLASSES, activation="softmax")(headModel)
# place the head FC model on top of the base model (this will become
# the actual model we will train)
model = Model(inputs=baseModel.input, outputs=headModel)
# loop over all layers in the base model and freeze them so they will
# *not* be updated during the first training process
# TODO Testar com outras camadas não congeladas
for layer in baseModel.layers:
	layer.trainable = False

# # Treina o modelo
print("Treinando o modelo")

#usar variavel de cima
#EPOCHS=100
# compile our model
print("[INFO] compiling model...")
opt = Adam(lr=INIT_LR)
model.compile(loss="categorical_crossentropy", optimizer=opt,
	metrics=["accuracy"])
# train the head of the network
print("[INFO] training head...")
H = model.fit(
	aug.flow(trainX, trainY, batch_size=BS),
	steps_per_epoch=len(trainX) // BS,
	validation_data=(testX, testY),
	validation_steps=len(testX) // BS,
	epochs=EPOCHS)

# # Mostra as informações do treinamento

# make predictions on the testing set
print("[INFO] evaluating network...")
predIdxs = model.predict(testX, batch_size=BS)
# for each image in the testing set we need to find the index of the
# label with corresponding largest predicted probability
predIdxs = np.argmax(predIdxs, axis=1)
# show a nicely formatted classification report
print(classification_report(testY.argmax(axis=1), predIdxs,
	target_names=lb.classes_))

# plot the training loss and accuracy
N = EPOCHS
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, N), H.history["loss"], label="train_loss")
plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, N), H.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, N), H.history["val_accuracy"], label="val_acc")
plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend(loc="lower left")
#plt.savefig(args["plot"])
plt.savefig('saida')

# # Salvo o modelo

# In[ ]:

# serialize the model to disk
print("[INFO] saving mask detector model...")
model.save(PREFIX+"modelo", save_format="h5")
# serialize the label encoder to disk
print("[INFO] saving label encoder...")
f = open(PREFIX+"encoder", "wb")
f.write(pickle.dumps(lb))
f.close()


text:
## Code based on WGAN implementation in https://github.com/eriklindernoren/Keras-GAN

import numpy as np

from sklearn.base import BaseEstimator, TransformerMixin
from keras.layers.merge import _Merge
from keras.layers import Input, Dense, Reshape, Flatten, Dropout, Concatenate
from keras.layers import BatchNormalization, Activation, ZeroPadding2D
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.models import Sequential, Model
from keras.optimizers import RMSprop
from functools import partial
import keras.backend as K

from keras.utils import get_custom_objects, plot_model

__all__ = ['MolGen']

def swish(x):
    return (K.sigmoid(x) * x)

class RandomWeightedAverage(_Merge):

    def __init__(self, batch_size):
        _Merge.__init__(self)
        self.batch_size=batch_size

    """Provides a (random) weighted average between real and generated image samples"""
    def _merge_function(self, inputs):
        alpha = K.random_uniform((self.batch_size, 1))
        return (alpha * inputs[0]) + ((1 - alpha) * inputs[1])

class MolGen(BaseEstimator, TransformerMixin):

    def __init__(self, latent_dim, output_dim, batch_size=1000, noise_dim=50,
                n_epochs=1000,
                hidden_layer_depth=3, hidden_size=200, activation='swish', verbose=True,
                n_discriminator=5):
        self.latent_dim = latent_dim
        self.output_dim = output_dim
        self.hidden_layer_depth = hidden_layer_depth
        self.hidden_size = hidden_size
        self.activation = activation
        self.batch_size=batch_size
        self.n_epochs = n_epochs
        self.noise_dim = noise_dim
        self.verbose = verbose

        self.n_discriminator = n_discriminator
        optimizer = RMSprop(lr=0.00005)

        # Register swish
        get_custom_objects().update({'swish': swish})

        # Build generator and discriminator
        self.generator = self._build_generator()
        self.discriminator = self._build_discriminator()

        #
        # Construct computational graph for discriminator
        #
        self.generator.trainable = False

        # Real molecule
        real_mol = Input(shape=(self.output_dim,))

        # Discriminator input
        z_disc = Input(shape=(self.latent_dim + self.noise_dim,))
        fake_mol = self.generator(z_disc)

        # conditioning input
        z_cond = Input(shape=(self.latent_dim,))
        # Condition both fake and valid.
        fake_mol_cond = Concatenate(axis=1)([fake_mol, z_cond])
        real_mol_cond = Concatenate(axis=1)([real_mol, z_cond])

        # Discriminator does its job
        fake = self.discriminator(fake_mol_cond)
        valid = self.discriminator(real_mol_cond)

        # Interpolated between real and fake molecule.
        interp_mol = RandomWeightedAverage(batch_size=self.batch_size)([real_mol_cond, fake_mol_cond])
        validity_interp = self.discriminator(interp_mol)

        # Use Python partial to provide loss function with additional
        # 'averaged_samples' argument
        partial_gp_loss = partial(self.ـgradient_penalty_loss,
                          averaged_samples=interp_mol)
        partial_gp_loss.__name__ = 'gradient_penalty' # Keras requires function names

        self.discriminator_model = Model(inputs=[real_mol, z_disc, z_cond],
                            outputs=[valid, fake, validity_interp])
        self.discriminator_model.compile(loss=[self._wasserstein_loss,
                                        self._wasserstein_loss,
                                        partial_gp_loss],
                                        optimizer=optimizer,
                                        loss_weights=[1, 1, 10])

        #
        # Construct computational graph for generator
        #
        self.discriminator.trainable = False
        self.generator.trainable = True

        z_gen = Input(shape=(self.latent_dim + self.noise_dim,))
        mol = self.generator(z_gen)
        z_cond = Input(shape=(self.latent_dim,))
        mol_cond = Concatenate(axis=1)([mol, z_cond])
        valid = self.discriminator(mol_cond)
        self.generator_model = Model([z_gen, z_cond], valid)
        self.generator_model.compile(loss=self._wasserstein_loss, optimizer=optimizer)

        self.is_fitted = False

    def fit(self, X, y):
        # Adversarial ground truths
        valid = -np.ones((self.batch_size, 1))
        fake =  np.ones((self.batch_size, 1))
        dummy = np.zeros((self.batch_size, 1)) # Dummy gt for gradient penalty
        for epoch in range(self.n_epochs):
            for _ in range(self.n_discriminator):
                # ---------------------
                #  Train Discriminator
                # ---------------------

                # Select a random batch of images
                idx = np.random.randint(0, y.shape[0], self.batch_size)
                real_mols = y[idx]

                # Sample generator input
                fake_mols = np.concatenate([X[idx], np.random.normal(0, 1, (self.batch_size, self.noise_dim))], axis=1)

                # Train the critic
                d_loss = self.discriminator_model.train_on_batch([real_mols, fake_mols, X[idx]],
                                                            [valid, fake, dummy])

            # ---------------------
            #  Train Generator
            # ---------------------

            g_loss = self.generator_model.train_on_batch([fake_mols, X[idx]], valid)

            # Plot the progress
            if self.verbose:
                print ("%d [D loss: %f] [G loss: %f]" % (epoch, d_loss[0], g_loss))

        self.is_fitted = True
        return self

    def transform(self, X):

        if self.is_fitted:
            out = self.generator.predict(np.concatenate([X, np.random.normal(0, 1, (X.shape[0], self.noise_dim))], axis=1), batch_size=self.batch_size)
            return out

        raise RuntimeError('Model needs to be fit first.')

    def ـgradient_penalty_loss(self, y_true, y_pred, averaged_samples):
        """
        Computes gradient penalty based on prediction and weighted real / fake samples
        """
        gradients = K.gradients(y_pred, averaged_samples)[0]
        # compute the euclidean norm by squaring ...
        gradients_sqr = K.square(gradients)
        #   ... summing over the rows ...
        gradients_sqr_sum = K.sum(gradients_sqr,
                                  axis=np.arange(1, len(gradients_sqr.shape)))
        #   ... and sqrt
        gradient_l2_norm = K.sqrt(gradients_sqr_sum)
        # compute lambda * (1 - ||grad||)^2 still for each single sample
        gradient_penalty = K.square(1 - gradient_l2_norm)
        # return the mean as loss over all the batch samples
        return K.mean(gradient_penalty)

    def _wasserstein_loss(self, y_true, y_pred):
        return K.mean(y_true * y_pred)

    def _build_generator(self):
        model = Sequential()

        model.add(
            Dense(
                self.hidden_size,
                activation=self.activation,
                input_dim=self.latent_dim + self.noise_dim
            )
        )
        for _ in range(self.hidden_layer_depth - 1):
            model.add(Dense(self.hidden_size, activation=self.activation))

        model.add(Dense(self.output_dim, activation='tanh'))

        model.summary()

        input = Input(shape=(self.latent_dim + self.noise_dim,))
        mol = model(input)

        return Model(input, mol)

    def _build_discriminator(self):
        model = Sequential()

        model.add(
            Dense(
                self.hidden_size,
                activation=self.activation,
                input_dim=self.output_dim + self.latent_dim
            )
        )
        for _ in range(self.hidden_layer_depth - 1):
            model.add(Dense(self.hidden_size, activation=self.activation))

        model.add(Dense(1))

        model.summary()

        mol = Input(shape=(self.output_dim + self.latent_dim,))
        validity = model(mol)

        return Model(mol, validity)

max_stars_repo_name: hsidky/srv

max_issues_repo_name: hsidky/hde

max_forks_repo_name: hsidky/srv


text:
import itertools
from palettable.colorbrewer import qualitative, sequential
import numpy as np

# From matplotlib.colours
def rgb2hex(rgb):
    'Given an rgb or rgba sequence of 0-1 floats, return the hex string'
    return '#%02x%02x%02x' % tuple([int(np.round(val * 255)) for val in rgb[:3]])

def graph_to_sankey(G,
                    groups=None,
                    palette=None,
                    sample=None,
                    hue=None,
                    hue_range=None,
                    hue_norm=False,
                    flow_color=None):
    """Convert to display format, set colours, titles etc."""
    if groups is None:
        groups = []

    def get_data(data, key):
        if key == 'value':
            return data[key]
        else:
            return data['measures'][key]

    if sample is None:
        get_value = lambda data, key: float(get_data(data, key))
    elif sample == 'mean':
        get_value = lambda data, key: get_data(data, key).mean()
    elif callable(sample):
        get_value = lambda data, key: sample(get_data(data, key))
    else:
        get_value = lambda data, key: get_data(data, key)[sample]

    if flow_color is None and hue is None:
        # qualitative colours based on material
        if palette is None:
            palette = 'Pastel1_8'
        if isinstance(palette, str):
            try:
                palette = getattr(qualitative, palette).hex_colors
            except AttributeError:
                raise ValueError('No qualitative palette called {}'.format(palette)) from None
        if not isinstance(palette, dict):
            materials = sorted(set([m for v, w, (m, t) in G.edges(keys=True)]))
            palette = {m: v
                       for m, v in zip(materials, itertools.cycle(palette))}
        get_color = lambda m, data: palette[m]

    elif flow_color is None and hue is not None:
        if palette is None:
            palette = 'Reds_9'
        if isinstance(palette, str):
            try:
                palette = getattr(sequential, palette).mpl_colormap
            except AttributeError:
                raise ValueError('No sequential palette called {}'.format(palette)) from None
        if hue_norm:
            get_hue = lambda data: get_value(data, hue) / get_value(data, 'value')
        elif callable(hue):
            get_hue = hue
        else:
            get_hue = lambda data: get_value(data, hue)
        values = np.array([get_hue(data) for _, _, data in G.edges(data=True)])
        if hue_range is None:
            vmin, vmax = values.min(), values.max()
        else:
            vmin, vmax = hue_range
        get_color = lambda m, data: rgb2hex(palette((get_hue(data) - vmin) / (vmax - vmin)))

    else:
        get_color = flow_color

    links = []
    nodes = []

    for v, w, (m, t), data in G.edges(keys=True, data=True):
        links.append({
            'source': v,
            'target': w,
            'type': m,
            'time': t,
            'value': get_value(data, 'value'),
            'bundles': [str(x) for x in data.get('bundles', [])],
            'color': get_color(m, data),
            'title': str(m),
            'opacity': 1.0,
        })

    for u, data in G.nodes(data=True):
        nodes.append({
            'id': u,
            'title': str(data.get('title', u)),
            'style': data.get('type', 'default'),
            'direction': 'l' if data.get('direction', 'R') == 'L' else 'r',
            'visibility': 'hidden' if data.get('title') == '' else 'visible',
        })

    return {
        'nodes': nodes,
        'links': links,
        'order': G.ordering.layers,
        'groups': groups,
    }

max_stars_repo_name: fullflu/floweaver

max_issues_repo_name: LeoPaoli/sankeyview

max_forks_repo_name: LeoPaoli/sankeyview


text:
import numpy as np
import scipy.spatial as sp
import matplotlib.pyplot as plt

#points = np.array([[0, 0], [0, 1], [0, 2], [1, 0], [1, 1], [1, 2], [2, 0], [2, 1], [2, 2]])
points = np.array([[3, 0], [4.2, 1], [0, 8], [1, 5], [1, 1], [1, 2], [3.5, 0], [4, 1], [6, 6]])

vor = sp.Voronoi(points)
print (vor)
tri = sp.Delaunay(points)
print (tri)

fig = sp.voronoi_plot_2d(vor)
fig = sp.delaunay_plot_2d(tri)
plt.show()
plt.close()

max_stars_repo_name: The-Kristina/CellComp

max_stars_repo_licenses: CNRI-Python RSA-MD Xnet Net-SNMP X11

max_issues_repo_name: The-Kristina/CellComp

max_issues_repo_licenses: CNRI-Python RSA-MD Xnet Net-SNMP X11

max_forks_repo_name: The-Kristina/CellComp

max_forks_repo_licenses: CNRI-Python RSA-MD Xnet Net-SNMP X11

reason: import numpy,import scipy


text:
import copy
import logging
import shap

import numpy as np
import pandas as pd

from alibi.api.defaults import DEFAULT_META_SHAP, DEFAULT_DATA_SHAP
from alibi.api.interfaces import Explanation, Explainer, FitMixin
from scipy import sparse
from shap.common import DenseData, DenseDataWithIndex
from typing import Callable, Dict, List, Optional, Sequence, Union, Tuple
from alibi.utils.wrappers import methdispatch

logger = logging.getLogger(__name__)

SHAP_PARAMS = [
    'link',
    'group_names',
    'groups',
    'weights',
    'summarise_background',
    'summarise_result'
    'kwargs',
]

BACKGROUND_WARNING_THRESHOLD = 300

class KernelShap(Explainer, FitMixin):

    def __init__(self,
                 predictor: Callable,
                 link: str = 'identity',
                 feature_names: Union[List, Tuple, None] = None,
                 categorical_names: Optional[Dict] = None,
                 seed: int = None):
        """
        A wrapper around the shap.KernelExplainer class. This extends the current shap library functionality
        by allowing the user to specify variable groups in order to deal with one-hot encoded categorical
        variables. The user can also specify whether to aggregate the shap values estimate for the encoded levels
        of categorical variables during the explain call.

        Parameters
        ----------
        predictor
            A callable that takes as an input a samples x features array and outputs a samples x n_outputs
            outputs. The n_outputs should represent model output in margin space. If the model outputs
            probabilities, then the link should be set to 'logit' to ensure correct force plots.
        link
            Valid values are 'identity' or 'logit'. A generalized linear model link to connect the feature
            importance values to the model output. Since the feature importance values, phi, sum up to the
            model output, it often makes sense to connect them to the ouput with a link function where
            link(output) = sum(phi). If the model output is a probability then the LogitLink link function
            makes the feature importance values have log-odds units. Therefore, for a model which outputs
            probabilities, link='logit' makes the feature effects have log-odds (evidence) units where
            link='identity' means that the feature effects have probability units. Please see
            https://github.com/slundberg/shap/blob/master/notebooks/kernel_explainer/Squashing%20Effect.ipynb
            for an in-depth discussion about the semantics of explaining the model in the probability vs the
            margin space.
        feature_names
            List with feature names.
        categorical_names
            Dictionary where keys are feature columns and values are list of categories for the feature.
        seed
            Fixes the random number stream, which influences which subsets are sampled during shap value estimation
        """

        super().__init__(meta=copy.deepcopy(DEFAULT_META_SHAP))

        self.link = link
        self.predictor = predictor
        self.feature_names = feature_names if feature_names else []
        self.categorical_names = categorical_names if categorical_names else {}
        self.seed = seed

        # if the user specifies groups but no names, the groups are automatically named
        self.use_groups = False
        # changes if feature groups indices are passed but not names
        self.create_group_names = False
        # if sum of groups entries matches first dimension as opposed to second, warn user
        self.transposed = False
        # if weights are not correctly specified, they are ignored
        self.ignore_weights = False
        # sums up shap values for each level of categorical var
        self.summarise_result = False
        # selects a subset of the background data to avoid excessively slow runtimes
        self.summarise_background = False
        # checks if it has been fitted:
        self._fitted = False

    def _check_inputs(self,
                      background_data: Union[shap.common.Data, pd.DataFrame, np.ndarray, sparse.spmatrix],
                      group_names: Union[Tuple, List, None],
                      groups: Optional[List[Union[Tuple[int], List[int]]]],
                      weights: Union[Union[List[float], Tuple[float]], np.ndarray, None]) -> None:
        """
        If user specifies parameter grouping, then we check input is correct or inform
        them if the settings they put might not behave as expected.
        """

        if isinstance(background_data, shap.common.Data):
            # don't provide checks for situations where the user passes
            # the data object directly
            if not self.summarise_background:
                self.use_groups = False
                return
            # if summarisation took place, we do the checks to ensure everything is correct
            else:
                background_data = background_data.data

        if isinstance(background_data, np.ndarray) and background_data.ndim == 1:
            background_data = np.atleast_2d(background_data)

        if background_data.shape[0] > BACKGROUND_WARNING_THRESHOLD:
            msg = "Large datasets can cause slow runtimes for shap. The background dataset " \
                  "provided has {} records. Consider passing a subset or allowing the algorithm " \
                  "to automatically summarize the data by setting the summarise_background=True or" \
                  "setting summarise_background to 'auto' which will default to {} samples!"
            logging.warning(msg.format(background_data.shape[0], BACKGROUND_WARNING_THRESHOLD))

        if group_names and not groups:
            logging.info(
                "Specified group_names but no corresponding sequence 'groups' with indices "
                "for each group was specified. All groups will have len=1."
            )
            if not len(group_names) in background_data.shape:
                msg = "Specified {} group names but data dimension is {}. When grouping " \
                      "indices are not specifies the number of group names should equal " \
                      "one of the data dimensions! Igoring grouping inputs!"
                logging.warning(msg.format(len(group_names), background_data.shape))
                self.use_groups = False

        if groups and not group_names:
            logging.warning(
                "No group names specified but groups specified! Automatically "
                "assigning 'group_' name for every index group specified!")
            if self.feature_names:
                n_groups = len(groups)
                n_features = len(self.feature_names)
                if n_features != n_groups:
                    msg = "Number of feature names specified did not match the number of groups." \
                          "Specified {} groups and {} features names. Creating default names for " \
                          "specified groups"
                    logging.warning(msg.format(n_groups, n_features))
                    self.create_group_names = True
                else:
                    group_names = self.feature_names
            else:
                self.create_group_names = True

        if groups:
            if not (isinstance(groups[0], tuple) or isinstance(groups[0], list)):
                msg = "groups should be specified as List[Union[Tuple[int], List[int]]] where each " \
                      "sublist represents a group and int represent group instance. Specified group " \
                      "elements have type {}. Ignoring grouping inputs!"
                logging.warning(msg.format(type(groups[0])))
                self.use_groups = False

            expected_dim = sum(len(g) for g in groups)
            if background_data.ndim == 1:
                actual_dim = background_data.shape[0]
            else:
                actual_dim = background_data.shape[1]
            if expected_dim != actual_dim:
                if background_data.shape[0] == expected_dim:
                    logging.warning(
                        "The sum of the group indices list did not match the "
                        "data dimension along axis=1 but matched dimension "
                        "along axis=0. Consider transposing the data!"
                    )
                    self.transposed = True
                else:
                    msg = "The sum of the group sizes specified did not match the number of features. " \
                          "Sum of group sizes: {}. Number of features: {}. Ignoring grouping inputs!"
                    logging.warning(msg.format(expected_dim, actual_dim))
                    self.use_groups = False

            if group_names:
                n_groups = len(groups)
                n_group_names = len(group_names)
                if n_group_names != n_groups:
                    msg = "The number of group names specified does not match the number of groups. " \
                          "Received {} groups and {} names! Ignoring grouping inputs!"
                    logging.warning(msg.format(n_groups, n_group_names))
                    self.use_groups = False

        if weights is not None:
            if background_data.ndim == 1 or background_data.shape[0] == 1:
                logging.warning(
                    "Specified weights but the background data has only one record. "
                    "Weights will be ignored!"
                )
                self.ignore_weights = True
            else:
                data_dim = background_data.shape[0]
                feat_dim = background_data.shape[1]
                weights_dim = len(weights)
                if data_dim != weights_dim:
                    if not (feat_dim == weights_dim and self.transposed):
                        msg = "The number of weights specified did not match data dimension. " \
                              "Number of weights: {}. Number of datapoints: {}. Weights will " \
                              "be ignored!"
                        logging.warning(msg.format(weights_dim, data_dim))
                        self.ignore_weights = True

            # NB: we have already summarised the data at this point
            if self.summarise_background:

                weights_dim = len(weights)
                if background_data.ndim == 1:
                    n_background_samples = 1
                else:
                    if not self.transposed:
                        n_background_samples = background_data.shape[0]
                    else:
                        n_background_samples = background_data.shape[1]

                if weights_dim != n_background_samples:
                    msg = "The number of weights vector provided ({}) did not match the number of " \
                          "summary data points ({}). The weights provided will be ignored!"
                    logging.warning(msg.format(weights_dim, n_background_samples))

                    self.ignore_weights = True

    def _summarise_background(self,
                              background_data: Union[shap.common.Data, pd.DataFrame, np.ndarray, sparse.spmatrix],
                              n_background_samples: int) -> \
            Union[shap.common.Data, pd.DataFrame, np.ndarray, sparse.spmatrix]:
        """
        Summarises the background data to n_background_samples in order to reduce the computational cost. If the
        background data is a shap.common.Data object, no summarisation is performed.

        Returns
        -------
            If the user has specified grouping, then the input object is subsampled and an object of the same
            type is returned. Otherwise, a shap.common.Data object containing the result of a kmeans algorithm
            is wrapped in a shap.common.DenseData object and returned. The samples are weighted according to the
            frequency of the occurence of the clusters in the original data.
        """

        if isinstance(background_data, shap.common.Data):
            msg = "Received option to summarise the data but the background_data object " \
                  "was an instance of shap.common.Data. No summarisation will take place!"
            logging.warning(msg)
            return background_data

        if background_data.ndim == 1:
            msg = "Received option to summarise the data but the background_data object only had " \
                  "one record with {} features. No summarisation will take place!"
            logging.warning(msg.format(len(background_data)))
            return background_data

        self.summarise_background = True

        # if the input is sparse, we assume there are categorical variables and use random sampling, not kmeans
        if self.use_groups or self.categorical_names or isinstance(background_data, sparse.spmatrix):
            return shap.sample(background_data, nsamples=n_background_samples)
        else:
            logging.info(
                "When summarising with kmeans, the samples are weighted in proportion to their "
                "cluster occurrence frequency. Please specify a different weighting of the samples "
                "through the by passing a weights of len=n_background_samples to the constructor!"
            )
            return shap.kmeans(background_data, n_background_samples)

    @methdispatch
    def _get_data(self,
                  background_data: Union[shap.common.Data, pd.DataFrame, np.ndarray, sparse.spmatrix],
                  group_names: Sequence,
                  groups: List[Sequence[int]],
                  weights: Sequence[Union[float, int]],
                  **kwargs):
        """
        Groups the data if grouping options are specified, returning a shap.common.Data object in this
        case. Otherwise, the original data is returned and handled internally by the shap library.
        """

        raise TypeError("Type {} is not supported for background data!".format(type(background_data)))

    @_get_data.register(shap.common.Data)
    def _(self, background_data, *args, **kwargs) -> shap.common.Data:
        """
        Initialises background data if user passes a shap.common.Data object as input.
        Input  is returned as this is a native object to shap.

        Note: If self.summarise_background = True, then a shap.common.Data object is
        returned if the user passed a shap.common.data object to fit or didn't specify groups.
        """

        group_names, groups, weights = args
        if weights is not None and self.summarise_background:
            if not self.ignore_weights:
                background_data.weights = weights
            if self.use_groups:
                background_data.groups = groups
                background_data.group_names = group_names
                background_data.group_size = len(groups)

        return background_data

    @_get_data.register(np.ndarray)  # type: ignore
    def _(self, background_data, *args, **kwargs) -> Union[np.ndarray, shap.common.Data]:
        """
        Initialises background data if the user passes a numpy array as input.
        If the user specifies feature grouping then a shap.common.DenseData object
        is returned. Weights are handled separately to avoid triggering assertion
        correct inside shap library. Otherwise, the original data is returned and
        is handled by the shap library internally.
        """

        group_names, groups, weights = args
        new_args = (group_names, groups, weights) if weights is not None else (group_names, groups)
        if self.use_groups:
            return DenseData(background_data, *new_args)
        else:
            return background_data

    @_get_data.register(sparse.spmatrix)  # type: ignore
    def _(self, background_data, *args, **kwargs) -> Union[shap.common.Data, sparse.spmatrix]:
        """
        Initialises background data if user passes a sparse matrix as input. If the
        user specifies feature grouping, then the sparse array is converted to a dense
        array. Otherwise, the original array is returned and handled internally by shap
        library.
        """

        group_names, groups, weights = args
        new_args = (group_names, groups, weights) if weights is not None else (group_names, groups)

        if self.use_groups:
            logging.warning(
                "Grouping is not currently compatible with sparse matrix inputs. "
                "Converting background data sparse array to dense matrix."
            )
            background_data = background_data.toarray()
            return DenseData(
                background_data,
                *new_args,
            )

        return background_data

    @_get_data.register(pd.core.frame.DataFrame)  # type: ignore
    def _(self, background_data, *args, **kwargs) -> Union[shap.common.Data, pd.core.frame.DataFrame]:
        """
        Initialises background data if user passes a pandas.core.frame.DataFrames as input.
        If the user has specified groups and given a data frame, initialise shap.common.DenseData
        explicitly as this is not handled by shap library internally. Otherwise, data initialisation,
        is left to the shap library.
        """

        _, groups, weights = args
        new_args = (groups, weights) if weights is not None else (groups,)
        if self.use_groups:
            logging.info("Group names are specified by column headers, group_names will be ignored!")
            keep_index = kwargs.get("keep_index", False)
            if keep_index:
                return DenseDataWithIndex(
                    background_data.values,
                    list(background_data.columns),
                    background_data.index.values,
                    background_data.index.name,
                    *new_args,
                )
            else:
                return DenseData(
                    background_data.values,
                    list(background_data.columns),
                    *new_args,
                )
        else:
            return background_data

    @_get_data.register(pd.core.frame.Series)  # type: ignore
    def _(self, background_data, *args, **kwargs) -> Union[shap.common.Data, pd.core.frame.Series]:
        """
        Initialises background data if user passes a pandas Series object as input.
        Original object is returned as this is initialised internally by shap is there
        is no group structure specified. Otherwise the a shap.common.DenseData object
        is initialised.
        """

        _, groups, _ = args
        if self.use_groups:
            return DenseData(
                background_data.values.reshape(1, len(background_data)),
                list(background_data.index),
                groups,
            )

        return background_data

    def _update_metadata(self, data_dict: dict, params: bool = False) -> None:
        """
        This function updates the metadata of the explainer using the data from
        the data_dict. If the params option is specified, then each key-value
        pair is added to the metadata 'params' dictionary only if the key is
        included in SHAP_PARAMS.

        Parameters
        ----------
        data_dict
            Dictionary containing the data to be stored in the metadata.
        params
            If True, the method updates the 'params' attribute of the metatadata.
        """

        if params:
            for key in data_dict.keys():
                if key not in SHAP_PARAMS:
                    continue
                else:
                    self.meta['params'].update([(key, data_dict[key])])
        else:
            self.meta.update(data_dict)

    def fit(self,  # type: ignore
            background_data: Union[np.ndarray, sparse.spmatrix, pd.DataFrame, shap.common.Data],
            summarise_background: Union[bool, str] = False,
            n_background_samples: int = BACKGROUND_WARNING_THRESHOLD,
            group_names: Union[Tuple, List, None] = None,
            groups: Optional[List[Union[Tuple[int], List[int]]]] = None,
            weights: Union[Union[List[float], Tuple[float]], np.ndarray, None] = None,
            **kwargs) -> "KernelShap":
        """
        This takes a background dataset (usually a subsample of the training set) as an input along with several
        user specified options and initialises a KernelShap explainer. The runtime of the algorithm depends on the
        number of samples in this dataset and on the number of features in the dataset. To reduce the size of the
        dataset, the summarise_background option and n_background_samples should be used. To reduce the feature
        dimensionality, encoded categorical variables can be treated as one during the feature perturbation process;
        this decreases the effective feature dimensionality, can reduce the variance of the shap values estimation and
        reduces slightly the number of calls to the predictor. Further runtime savings can be achieved by changing the
        nsamples parameter in the call to explain. Runtime reduction comes with an accuracy tradeoff, so it is better
        to experiment with a runtime reduction method and understand results stability before using the system.

        Parameters
        -----------
        background_data
            Data used to estimate feature contributions and baseline values for force plots. The rows of the
            background data should represent samples and the columns features.
        summarise_background
            A large background dataset impacts the runtime and memory footprint of the algorithm. By setting
            this argument to True, only n_background_samples from the provided data are selected. If group_names or
            groups arguments are specified, the algorithm assumes that the data contains categorical variables so
            the records are selected uniformly at random. Otherwise, shap.kmeans (a wrapper around sklearn kmeans
            implementation) is used for selection. If set to 'auto', a default of BACKGROUND_WARNING_THRESHOLD
            samples is selected.
        n_background_samples
            The number of samples to keep in the background dataset if summarise_background=True.
        groups:
            A list containing sublists specifying the indices of features belonging to the same group.
        group_names:
            If specified, this array is used to treat groups of features as one during feature perturbation.
            This feature can be useful, for example, to treat encoded categorical variables as one and can
            result in computational savings (this may require adjusting the nsamples parameter).
        weights:
            A sequence or array of weights. This is used only if grouping is specified and assigns a weight
            to each point in the dataset.
        kwargs:
            Expected keyword arguments include "keep_index" and should be used if a data frame containing an
            index column is passed to the algorithm.
        """

        np.random.seed(self.seed)

        self._fitted = True
        # user has specified variable groups
        use_groups = groups is not None or group_names is not None
        self.use_groups = use_groups

        if summarise_background:
            if isinstance(summarise_background, str):
                if not isinstance(background_data, shap.common.Data):
                    n_samples = background_data.shape[0]
                else:
                    n_samples = background_data.data.shape[0]
                n_background_samples = min(n_samples, BACKGROUND_WARNING_THRESHOLD)
            background_data = self._summarise_background(background_data, n_background_samples)

        # check user inputs to provide warnings if input is incorrect
        self._check_inputs(background_data, group_names, groups, weights)
        if self.create_group_names:
            group_names = ['group_{}'.format(i) for i in range(len(groups))]
        # disable grouping or data weights if inputs are not correct
        if self.ignore_weights:
            weights = None
        if not self.use_groups:
            group_names, groups = None, None
        else:
            self.feature_names = group_names

        # perform grouping if requested by the user
        self.background_data = self._get_data(background_data, group_names, groups, weights, **kwargs)
        self._explainer = shap.KernelExplainer(
            self.predictor,
            self.background_data,
            link=self.link,
        )  # type: shap.KernelExplainer
        self.expected_value = self._explainer.expected_value
        if not self._explainer.vector_out:
            logging.warning(
                "Predictor returned a scalar value. Ensure the output represents a probability or decision score "
                "as opposed to a classification label!"
            )

        # update metadata
        params = {
            'groups': groups,
            'group_names': group_names,
            'weights': weights,
            'kwargs': kwargs,
            'summarise_background': self.summarise_background,
            'grouped': self.use_groups,
            'transpose': self.transposed,
        }
        self._update_metadata(params, params=True)

        return self

    def explain(self,
                X: Union[np.ndarray, pd.DataFrame, sparse.spmatrix],
                summarise_result: bool = False,
                cat_vars_start_idx: List[int] = None,
                cat_vars_enc_dim: List[int] = None,
                **kwargs) -> Explanation:
        """
        Explains the instances in the array X.

        Parameters
        ----------
        X
            Array with instances to be explained.
        summarise_result
            Specifies whether the shap values corresponding to dimensions of
            encoded categorical variables should be summed so that a single
            shap value is returned for each categorical variable. Both
            the start indices of the categorical variables (`cat_vars_start_idx`)
            and the encoding dimensions (`cat_vars_enc_dim`) have to be specified
        cat_vars_start_idx
            A sequence containing the start indices of the categorical variables.
            If specified, cat_vars_enc_dim should also be specified.
        cat_vars_enc_dim
            A sequence containing the length of the encoding dimension for each
            categorical variable.
        kwargs
            Keyword arguments specifying explain behaviour. Valid arguments are:
                *nsamples: controls the number of predictor calls and therefore runtime.
                *l1_reg: controls the explanation sparsity.
            For more details, please see https://shap.readthedocs.io/en/latest/.

        Returns
        -------
        explanation
            An explanation object containing the algorithm results.
        """

        if not self._fitted:
            raise TypeError(
                "Called explain on an unfitted object! Please fit the "
                "explainer using the .fit method first!"
            )

        # convert data to dense format if sparse
        if self.use_groups and isinstance(X, sparse.spmatrix):
            X = X.toarray()

        shap_values = self._explainer.shap_values(X, **kwargs)
        # for scalar model outputs a single numpy array is returned
        if isinstance(shap_values, np.ndarray):
            shap_values = [shap_values]
        if summarise_result:
            self.summarise_result = True
            if not cat_vars_start_idx or not cat_vars_start_idx:
                logging.warning(
                    "Results cannot be summarised as either the"
                    "start indices for categorical variables or"
                    "the encoding dimensions were not passed!"
                )
                self.summarise_result = False
            elif self.use_groups:
                logger.warning(
                    "Specified both groups as well as summarisation for categorical variables. "
                    "By grouping, only one shap value is estimated for each categorical variable. "
                    "Summarisation is thus not necessary!"
                )
                self.summarise_result = False
            else:
                summarised_shap = []
                for shap_array in shap_values:
                    summarised_shap.append(sum_categories(shap_array, cat_vars_start_idx, cat_vars_enc_dim))
                shap_values = summarised_shap

        self._update_metadata({"summarise_result": self.summarise_result}, params=True)

        return self.build_explanation(X, shap_values, self.expected_value)

    def build_explanation(self,
                          X: Union[np.ndarray, pd.DataFrame, sparse.spmatrix],
                          shap_values: List[np.ndarray],
                          expected_value: List) -> Explanation:
        """
        Create an explanation object.

        Parameters
        ----------
        X
            Array of instances to be explained.
        shap_values
            Each entry is a n_instances x n_features array, and the length of the list equals the dimensionality
            of the predictor output. The rows of each array correspond to the shap values for the instances with
            the corresponding row index in X
        expected_value
            A list containing the expected value of the prediction for each class.

        Returns
        -------
            An explanation containing a meta field with basic classifier metadata
            # TODO: Plotting default should be same space as the explanation? How do we figure out what space they
            #  explain in?
        """

        # TODO: DEFINE COMPLETE SCHEMA FOR THE METADATA (ONGOING)

        raw_predictions = self._explainer.linkfv(self.predictor(X))
        argmax_pred = np.argmax(np.atleast_2d(raw_predictions), axis=1)
        importances = self.rank_by_importance(shap_values)

        if isinstance(X, sparse.spmatrix):
            X = X.toarray()
        else:
            X = np.array(X)

        # output explanation dictionary
        data = copy.deepcopy(DEFAULT_DATA_SHAP)
        data.update(
            shap_values=shap_values,
            expected_value=expected_value,
            link=self.link,
            categorical_names=self.categorical_names,
            feature_names=self.feature_names
        )
        data['raw'].update(
            raw_prediction=raw_predictions,
            prediction=argmax_pred,
            instances=X,
            importances=importances
        )

        return Explanation(meta=copy.deepcopy(self.meta), data=data)

    def rank_by_importance(self, shap_values: List[np.ndarray]) -> Dict:
        """
        Given the shap values estimated for a multi-output model, this feature ranks
        features according to their importance. The feature importance is the average
        absolute value for a given feature.

        Parameters
        ----------
        shap_values
            Each element corresponds to a samples x features array of shap values corresponding
            to each model output.

        Returns
        -------
        importances
            A dictionary containing a key for each model output ('0', '1', ...) and a key for
            aggregated model output ('aggregated'). Each value is a dictionary contains a 'ranked_effect' field,
            populated with an array of values representing the average magnitude for each shap value,
            ordered from highest (most important) to the lowest (least important) feature. The 'names'
            field contains the corresponding feature names.
        """

        if len(shap_values[0].shape) == 1:
            shap_values = [np.atleast_2d(arr) for arr in shap_values]

        if not self.feature_names:
            self.feature_names = ['feature_{}'.format(i) for i in range(shap_values[0].shape[1])]
        else:
            if len(self.feature_names) != shap_values[0].shape[1]:
                msg = "The feature names provided do not match the number of shap values estimated. " \
                      "Received {} feature names but estimated {} shap values!"
                logging.warning(msg.format(len(self.feature_names), shap_values[0].shape[1]))
                self.feature_names = ['feature_{}'.format(i) for i in range(shap_values[0].shape[1])]

        importances = {}  # type: Dict[str, Dict[str, np.ndarray]]
        avg_mag = []  # type: List

        # rank the features by average shap value for each class in turn
        for class_idx in range(len(shap_values)):
            avg_mag_shap = np.abs(shap_values[class_idx]).mean(axis=0)
            avg_mag.append(avg_mag_shap)
            feature_order = np.argsort(avg_mag_shap)[::-1]
            most_important = avg_mag_shap[feature_order]
            most_important_names = [self.feature_names[i] for i in feature_order]
            importances[str(class_idx)] = {
                'ranked_effect': most_important,
                'names': most_important_names,
            }

        # rank feature by average shap value for aggregated classes
        combined_shap = np.sum(avg_mag, axis=0)
        feature_order = np.argsort(combined_shap)[::-1]
        most_important_c = combined_shap[feature_order]
        most_important_c_names = [self.feature_names[i] for i in feature_order]
        importances['aggregated'] = {
            'ranked_effect': most_important_c,
            'names': most_important_c_names
        }

        return importances

def sum_categories(values: np.ndarray, start_idx: Sequence[int], enc_feat_dim: Sequence[int]):
    """
    For each entry in start_idx, the function sums the following k columns where k is the
    corresponding entry in the enc_feat_dim sequence. The columns whose indices are not in
    start_idx are left unchanged.

    Parameters
    ----------
    values
        The array whose columns will be summed.
    start_idx
        The start indices of the columns to be summed.
    enc_feat_dim
        The number of columns to be summed, one for each start index.

    Returns
    -------
    new_values
        An array whose columns have been summed according to the entries in start_idx and enc_feat_dim.
    """

    if start_idx is None or enc_feat_dim is None:
        raise ValueError("Both the start indices or the encoding dimension need to be specified!")

    if not len(enc_feat_dim) == len(start_idx):
        raise ValueError("The lengths of the sequences of start indices and encodings must be equal!")

    n_encoded_levels = sum(enc_feat_dim)
    if n_encoded_levels > values.shape[1]:
        raise ValueError("The sum of the encoded features dimensions exceeds data dimension!")

    new_values = np.zeros((values.shape[0], values.shape[1] - n_encoded_levels + len(enc_feat_dim)))

    # find all the other indices of categorical columns other than those specified
    cat_cols = []
    for start, feat_dim in zip(start_idx, enc_feat_dim):
        for i in range(1, feat_dim):
            cat_cols.append(start + i)

    # sum the columns corresponding to a categorical variable
    enc_idx, new_vals_idx = 0, 0
    for idx in range(values.shape[1]):
        if idx in start_idx:
            feat_dim = enc_feat_dim[enc_idx]
            enc_idx += 1
            stop_idx = idx + feat_dim
            new_values[:, new_vals_idx] = np.sum(values[:, idx:stop_idx], axis=1)
        elif idx in cat_cols:
            continue
        else:
            new_values[:, new_vals_idx] = values[:, idx]
        new_vals_idx += 1

    return new_values

max_stars_repo_name: daavoo/alibi

max_issues_repo_name: daavoo/alibi

max_forks_repo_name: daavoo/alibi

reason: import numpy,from scipy


text:
import numpy as np

# L(ŷ,y) = - ∑yi*log(ŷi)

def entropia_custo(d1, d2):
  H=0
  for p, q in zip(d1, d2):
    H+=(p * - np.log2(q))
  print("Custo: {:.3f} bits".format(H))

valor_real = [1,0,0]
#valor_estimado = [0.6,0.15,0.25]
valor_estimado = [0.9,0.05,0.05]

entropia_custo(valor_real, valor_estimado)

from scipy.stats import entropy

def cross_entropy_via_scipy(x, y):
    return  entropy(x) + entropy(x, y)

m = cross_entropy_via_scipy(valor_real, valor_estimado)
print(m)

max_stars_repo_name: maledicente/cursos

max_issues_repo_name: maledicente/cursos

max_forks_repo_name: maledicente/cursos


text:
import pandas as pd
import nltk
#nltk.download('wordnet')
from langdetect import detect
import numpy as np
import re
import stopwordsiso as stopwords
import emoji
from emoji import demojize
from nltk.stem.wordnet import WordNetLemmatizer

df = pd.read_csv("usersInstagramMicroTopic.csv")

#nlp clean text word based
def clean_text(text):
    try:
        language = detect(text)
    except:
        language = "Not available"

    if(stopwords.has_lang(language)):
      stop = stopwords.stopwords(language)
    else:
      stop = stopwords.stopwords("en")

    text = demojize(text) #convert emojis to text
    temp = text.lower()
    temp = re.sub("'", "", temp) # to avoid removing contractions in english
    temp = re.sub("@[A-Za-z0-9_]+","", temp) #remove entire tag (@ and also name tagged)
    temp = re.sub("#","", temp) #remove only hashtag symbol
    temp = re.sub(r'http\S+', '', temp)  #remove links
    temp = re.sub('[()!?.,;:]', ' ', temp) #remove ()!?.,:;
    temp = re.sub('\[.*\].*',' ', temp) #remove []
    temp = re.sub("[^a-z]"," ", temp) #remove numbers
    temp = temp.split()
    temp = [w for w in temp if not w in stop] #remove stopwords
    temp = [WordNetLemmatizer().lemmatize(w, pos='v') for w in temp] #not for sentence based
    return temp

caption = df['caption']
pictures = df['picsText']
all_caps=[]
all_pics=[]
for j in range(0,len(df['id'])):
  capList= []
  for cap in caption[j].split("', "):
      #another split to handle "
      for c in cap.split('", '):
        t = clean_text(c)

        for k in t:
          capList.append(k)
  all_caps.append(capList)
  picList= []
  for pic in pictures[j].split("', "):
      #another split because replies use " instead of '
      for p in pic.split('", '):
        t2 = clean_text(p)

        for k2 in t2:
          picList.append(k2)
  all_pics.append(picList)

df['caption']= all_caps
df['picsText'] = all_pics
df.to_csv('usersInstagramCleanWordBased.csv', encoding='UTF8',index=False)
print("Clean Word Based dataset created")

max_stars_repo_name: paolofiorio/Micro-Influencer-Classifier

max_issues_repo_name: paolofiorio/Micro-Influencer-Classifier

max_forks_repo_name: paolofiorio/Micro-Influencer-Classifier


text:
import torch
import torch.nn as nn
import numpy as np
from .utils import DEVICE, load_model, assign_weights, flatten_params
# from .train_methods import eval_single_epoch

def get_line_loss(start_w, w, loader, config):
    interpolation  = None
    if 'line' in config['lmc_interpolation'] or 'integral' in config['lmc_interpolation']:
        interpolation = 'linear'
    elif 'stochastic' in config['lmc_interpolation']:
        interpolation = 'stochastic'
    else:
        raise Exception("non-implemented interpolation")

    m = load_model('{}/{}.pth'.format(config['exp_dir'], 'init')).to(DEVICE)
    total_loss = 0
    accum_grad = None
    criterion = nn.CrossEntropyLoss().to(DEVICE)

    #print("DEBUG >> LMC interpolation is >> {}".format(interpolation))
    if interpolation == 'linear':
        for t in np.arange(0.0, 1.01, 1.0/float(config['lmc_line_samples'])):
            grads = []
            cur_weight = start_w + (w - start_w) * t
            m = assign_weights(m, cur_weight).to(DEVICE)
            current_loss = get_clf_loss(m, loader)
            current_loss.backward()

            for name, param in m.named_parameters():
                grads.append(param.grad.view(-1))
            grads = torch.cat(grads)
            if accum_grad is None:
                accum_grad = grads
            else:
                accum_grad += grads
        return accum_grad

    elif interpolation == 'stochastic':
        for data, target, task_id in loader:
                grads = []
                t = np.random.uniform()
                cur_weight = start_w + (w - start_w) * t
                m = assign_weights(m, cur_weight).to(DEVICE)
                m.eval()
                data = data.to(DEVICE)#.view(-1, 784)
                target = target.to(DEVICE)
                output = m(data, task_id)
                current_loss = criterion(output, target)
                current_loss.backward()
                for name, param in m.named_parameters():
                    grads.append(param.grad.view(-1))
                grads = torch.cat(grads)
                if accum_grad is None:
                    accum_grad = grads
                else:
                    accum_grad += grads
        return accum_grad

    else:
        return None

def get_clf_loss(net, loader):
    criterion = nn.CrossEntropyLoss().to(DEVICE)
    net.eval()
    test_loss = 0
    count = 0

    for data, target, task_id in loader:
            count += len(target)
            data = data.to(DEVICE)#.view(-1, 784)
            target = target.to(DEVICE)
            output = net(data, task_id)
            test_loss += criterion(output, target)#*len(target)
    test_loss /= count
    return test_loss

def bezier_path_opt(w_1, w_2, theta, loader, config):
    accum_grad = None
    m = load_model('{}/{}.pth'.format(config['exp_dir'], 'init')).to(DEVICE)
    for t in np.arange(0.0, 1.01, 1.0/float(config['lmc_line_samples'])):
            grads = []
            # cur_weight = start_w + (w - start_w) * t
            cur_weight = ((1-t)**2)*w_1 + 2*t*(1-t)*theta + (t**2)*w_2
            m = assign_weights(m, cur_weight).to(DEVICE)
            current_loss = get_clf_loss(m, loader)
            current_loss.backward()

            for name, param in m.named_parameters():
                grads.append(param.grad.view(-1))
            grads = torch.cat(grads)
            if accum_grad is None:
                accum_grad = grads
            else:
                accum_grad += grads
    return accum_grad

max_stars_repo_name: imirzadeh/CL-LinearModeConnectivity

max_issues_repo_name: imirzadeh/CL-LinearModeConnectivity

max_forks_repo_name: imirzadeh/CL-LinearModeConnectivity


text:
#! /usr/bin/env python
# -*- coding: utf-8 -*-

#预测天气情况，且预测天气时，以最近的时间进行预测，即最新的时间对应的预测结果,预测暴雨天
import os
import sys
from svmutil import *
import MySQLdb #连接数据库
import numpy as np
import datetime

###根据事先已经求得的六个特征的最大值和最小值###

def GetLastTime_Data():#获取目前最新的相关数据，并进行预处理,连接数据库（通按时间排序，获取最新结果）

    #####################################################################################################
    #首先获取六大指标的相关参数,
    #对原始数据进行标准化处理，即具体形式如下：x`(i) = [x(i) - min( x(k) )] / [max( x(k) ) - min( x(k) )]
    #即在表weatherinformation中，执行此条类型语句
    Max_Temperature = 37.0
    Min_Temperature = 2.0
    Max_Min_Temperature = Max_Temperature - Min_Temperature
    Max_Humidity = 100.0
    Min_Humidity = 19.0
    Max_Min_Humidity = Max_Humidity - Min_Humidity
    Max_Visibility = 2000.0
    Min_Visibility = 27.0
    Max_Min_Visibility = Max_Visibility - Min_Visibility
    Max_WindSpeed = 15.1
    Min_WindSpeed = 0.0
    Max_Min_WindSpeed = Max_WindSpeed - Min_WindSpeed
    Max_WindDes = 359.9
    Min_WindDes = 0.0
    Max_Min_WindDes = Max_WindDes - Min_WindDes
    Max_Surface_Temperature = 59.4
    Min_Surface_Temperature = 2.9
    Max_Min_Surface_Temperature = Max_Surface_Temperature - Min_Surface_Temperature
    ######################################################################################################
    #对于已经处理后的表weatherinformation_Again中，并不需要处理预处理数据（标准化数据）
    #首先连接数据库
    conn = MySQLdb.connect(user="root", host="localhost", passwd="", db="WarningSystemDatabase")
    cur = conn.cursor()
    key = "max(GetTime)"
    tablename = "weatherinformation_Again"
    #首先进行最大时间值获取
    sql = "select " + key + " from " + tablename #获取最大数据的SQL语句
    #执行语句
    cur.execute(sql)
    #获取此刻时间<只有一条>
    row = cur.fetchone()
    lastGetTime = row[0] #获取此刻时间，且为str类型

    #之后获取相关数据，即六大特征
    key = "Temperature,Humidity,Visibility,WindSpeed,WindDes,Surface_Temperature"
    sql = "select " + key + " from " + tablename #获取相关值语句
    condition = " where GetTime=('%s')" % (lastGetTime.strftime('%Y-%m-%d %H:%M:%S'))#获取最新时间的数据条件
    sql = sql + condition
    #执行语句
    cur.execute(sql)
    #获取此刻时间的六大特征
    row = cur.fetchone()
    TemperatureString = row[0]
    HumidityString = row[1]
    VisibilityString = row[2]
    WindSpeedString = row[3]
    WindDesString = row[4]
    Surface_TemperatureString = row[5]
    #之后对数据进行相关整理，以获得满足条件的结果！！！
    #构建天气参数字典
    weather_Parameter = []
    weather_Parameter.append(TemperatureString)
    weather_Parameter.append(HumidityString)
    weather_Parameter.append(VisibilityString)
    weather_Parameter.append(WindSpeedString)
    weather_Parameter.append(WindDesString)
    weather_Parameter.append(Surface_TemperatureString)
    #PS:weather_Parameter = [row[i] for i in range(6)] #简便方法构建
    key = [i + 1 for i in range(6)] #构建列表[1,2,3,4,5,6]
    #开始构建字典
    weather_Primary = dict(zip(key, weather_Parameter))
    x_predict = []
    x_predict.append(weather_Primary)
    y_predict = [-1] #默认情况下，预测未来时，值全设为-1，即无雾情况
    return y_predict, x_predict #返回相关的y,x值

def Predict_RainStorm(model):#输入预测模板，进行相关预测
    #对于预测模板是否为null进行检测
    if model == "":
        return 0 #作为不存在模板的标志
    m = svm_load_model(model)
    #获取当前的预测值及其相关参数
    #y, x = svm_read_problem('test.txt')
    y, x = GetLastTime_Data() #进行赋值
    #对暴雨天情况进行值预测
    p_label, p_acc, p_val = svm_predict(y[:], x[:], m)
    p_label = int(p_label[0]) #每次只返回一个值
    #输出相关预测值
    print "Predict value --->" + str(p_label) #关于暴雨天情况的预测结果
    return p_label #并且返回相关预测结果（PS：对于是否有暴雨情况预测时，1表示有暴雨，-1表示无暴雨）
                   #对于暴雨等级的判断时候，1表示一级情况
                   #2表示二级情况，3表示三级情况

def Predict_Half_Hour_IS_RainStorm():#预测半个小时之后是否存在暴雨天情况
    #model = 'F:\Python27\libsvm-3.20\libsvm-3.20\python\Again_Again_Half_Hour_all.model' #预测半个小时之后是否存在暴雨天情况的模板
    model = "" #缺少暴雨天预测模板
    IS_RainStorm = Predict_RainStorm(model) #预测是否存在暴雨天情况
    #将值转化为Int类型
    #IS_RainStorm = int(IS_RainStorm)
    if IS_RainStorm == 1:#表示存在暴雨天，则进一步判断暴雨天等级
        RainStorm_Rank = Predict_Half_Hour_RainStorm_Rank()
        return RainStorm_Rank #返回暴雨天等级
    else:#即IS_Fog = -1情况时，即不存在暴雨天
        return IS_RainStorm #返回-1，即不存在暴雨天情况

def Predict_Half_Hour_RainStorm_Rank():#预测半个小时之后存在暴雨天等级
    #model = 'F:\Python27\libsvm-3.20\libsvm-3.20\python\Again_Again_Half_Hour_Value_True_all.model'#预测半个小时之后存在暴雨天等级的模板
    model = "" #缺少暴雨天预测模板
    RainStorm_Rank = Predict_RainStorm(model) #预测暴雨天等级情况
    return RainStorm_Rank #返回相关等级

def Predict_Hour_IS_RainStorm():#预测一个小时之后是否存在暴雨天情况
    #model = 'F:\Python27\libsvm-3.20\libsvm-3.20\python\Again_Again_Hour_All.model' #预测一个小时之后是否存在暴雨天情况的模板
    model = "" #缺少暴雨天预测模板
    IS_RainStorm = Predict_RainStorm(model) #预测是否存在暴雨天情况
    if IS_RainStorm == 1:#表示存在暴雨天，则进一步判断暴雨天等级
        RainStorm_Rank = Predict_Hour_RainStorm_Rank()
        return RainStorm_Rank #返回暴雨天等级
    else:#即IS_RainStorm = -1情况时，即不存在暴雨天
        return IS_RainStorm #返回-1，即不存在暴雨天情况

def Predict_Hour_RainStorm_Rank():#预测一个小时之后存在暴雨天等级
    #model = 'F:\Python27\libsvm-3.20\libsvm-3.20\python\Again_Again_Hour_Value_True_All.model' #预测一个小时之后存在暴雨天等级的模板
    model = "" #缺少暴雨天预测模板
    RainStorm_Rank = Predict_RainStorm(model) #预测暴雨天等级情况
    return RainStorm_Rank #返回相关等级

def test():

    #test读取数据情况
    y, x = GetLastTime_Data() #进行赋值
    print y
    print type(y)
    print x
    print type(x)
    print Predict_Half_Hour_IS_RainStorm()
    print Predict_Half_Hour_RainStorm_Rank()

#执行测试
test()

max_stars_repo_name: qichaotang/Source-Code-s-Reading

max_issues_repo_name: qichaotang/Source-Code-s-Reading

max_forks_repo_name: qichaotang/Source-Code-s-Reading


text:
# Copyright 2021 IBM Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import numpy as np
from bayes_opt import BayesianOptimization

class ImbalanceOptimizer:
    def __init__(self, config, logger, objective):
        # Initialize Core Attributes
        self.config = config
        self.logger = logger
        self.objective = objective
        self.io_cfg = self.config.imbopt

        # Initialize Weights
        # TODO: Define methods for how to initialize weights.
        self.weights = np.ones((len(self.objective.y_train), ))
        self.objective.init_weights(self.weights)

        # Initialize Optimizer
        self.bo = BayesianOptimization(self.objective.get_objective,
                    {'gridValue': (self.io_cfg.grid_value.lower,
                                   self.io_cfg.grid_value.upper)})

    def optimize(self):
        # Pre-Search Iterpolation Over Search Space
        probe_points = [0.001, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.999]
        for pt in probe_points: self.bo.probe(params={'gridValue': pt})
        self.bo.maximize(init_points=self.io_cfg.init_points,
                         n_iter=self.io_cfg.n_iterations,
                         acq='ucb')

        # Obtain Optimal Weight Values from Optimization
        value = self.bo.max['params']['gridValue']
        self.weights = np.array([self.weights[i] * (value if (self.objective.y_train[i] == 1)
                        else (1 - value)) for i in range(len(self.weights))])

        return self.weights