text:
# -*- coding: utf-8 -*-
##########################################################################
# NSAp - Copyright (C) CEA, 2020
# Distributed under the terms of the CeCILL-B license, as published by
# the CEA-CNRS-INRIA. Refer to the LICENSE file or to
# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
# for details.
##########################################################################

"""
The Variational U-Net auto-encoder.
"""

# Imports
import logging
import collections
import torch
import torch.nn as nn
import torch.nn.functional as func
from pynet.interfaces import DeepLearningDecorator
from pynet.utils import Networks
import numpy as np
from .base import BaseVAE
from ..unet import Down, Up, Conv1x1x1

# Global parameters
logger = logging.getLogger("pynet")

@Networks.register
@DeepLearningDecorator(family=("encoder", "vae"))
class VUNet(BaseVAE):
    """ VUNet.

    The Variational U-Net is a convolutional encoder-decoder neural network.
    The convolutional encoding/decoding parts are the same as the UNet.

    The model is composed of two sub-networks:

    1. Given x (image), encode it into a distribution over the latent space -
       referred to as Q(z|x).
    2. Given z in latent space (code representation of an image), decode it
       into the image it represents - referred to as f(z).
    """

    def __init__(self, latent_dim, in_channels=1, depth=5,
                 start_filts=64, up_mode="transpose",
                 batchnorm=True, dim="3d", input_shape=None,
                 num_classes=None):
        """ Init class.

        Parameters
        ----------
        latent_dim: int
            the latent dimension.
        in_channels: int, default 1
            number of channels in the input tensor.
        depth: int, default 5
            number of layers in the U-Net.
        start_filts: int, default 64
            number of convolutional filters for the first conv.
        up_mode: string, default 'transpose'
            type of upconvolution. Choices: 'transpose' for transpose
            convolution, 'upsample' for nearest neighbour upsampling.
        batchnorm: bool, default False
            normalize the inputs of the activation function.
        dim: str, default '3d'
            '3d' or '2d' input data.
        input_shape: uplet
            the tensor data shape (X, Y, Z) used during upsample (by default
            use a scale factor of 2).
        num_classes: int, default None
            the number of classes for the conditioning.
        """
        # Inheritance
        nn.Module.__init__(self)

        # Check inputs
        if dim in ("2d", "3d"):
            self.dim = dim
        else:
            raise ValueError(
                "'{}' is not a valid mode for merging up and down paths. Only "
                "'3d' and '2d' are allowed.".format(dim))
        if up_mode in ("transpose", "upsample"):
            self.up_mode = up_mode
        else:
            raise ValueError(
                "'{}' is not a valid mode for upsampling. Only 'transpose' "
                "and 'upsample' are allowed.".format(up_mode))

        # Declare class parameters
        self.latent_dim = latent_dim
        self.num_classes = num_classes
        self.in_channels = in_channels
        self.start_filts = start_filts
        self.depth = depth
        self.down = []
        self.up = []
        self.shapes = None
        if input_shape is not None:
            self.shapes = self.downsample_shape(
                input_shape, nb_iterations=(depth - 1))
            self.shapes = self.shapes[::-1]

        # Create the encoder pathway
        self.hidden_dims = []
        for cnt in range(depth):
            in_channels = self.in_channels if cnt == 0 else out_channels
            out_channels = self.start_filts * (2**cnt)
            self.hidden_dims.append(out_channels)
            pooling = False if cnt == 0 else True
            self.down.append(
                Down(in_channels, out_channels, self.dim, pooling=pooling,
                     batchnorm=batchnorm))

        # Create the decoder pathway
        # - careful! decoding only requires depth-1 blocks
        for cnt in range(depth - 1):
            in_channels = out_channels
            out_channels = in_channels // 2
            shape = None
            if self.shapes is not None:
                shape = self.shapes[cnt + 1]
            self.up.append(
                Up(in_channels, out_channels, up_mode=up_mode, dim=self.dim,
                   merge_mode="none", batchnorm=batchnorm, shape=shape))

        # Add the list of modules to current module
        self.down = nn.Sequential(*self.down)
        hidden_dim = self.hidden_dims[-1] * np.prod(self.shapes[0])
        self.mu = nn.Linear(hidden_dim, latent_dim)
        self.var = nn.Linear(hidden_dim, latent_dim)
        self.latent_to_hidden = nn.Linear(latent_dim, hidden_dim)
        self.up = nn.Sequential(*self.up)
        self.conv_final = Conv1x1x1(out_channels, self.in_channels, self.dim)
        self.logit = nn.Tanh()

        # Kernel initializer
        self.kernel_initializer()

    def encode(self, x):
        """ Encodes the input by passing through the encoder network
        and returns the latent codes.

        Parameters
        ----------
        x: Tensor, (N, C, X, Y, Z)
            input tensor to encode.

        Returns
        -------
        mu: Tensor (N, D)
            mean of the latent Gaussian.
        logvar: Tensor (N, D)
            standard deviation of the latent Gaussian.
        """
        logger.debug("Encode...")
        self.debug("input", x)
        x = self.down(x)
        self.debug("down", x)
        x = torch.flatten(x, start_dim=1)
        self.debug("flatten", x)
        # Split x into mu and var components of the latent Gaussian
        # distribution
        z_mu = self.mu(x)
        z_logvar = self.var(x)
        self.debug("z_mu", z_mu)
        self.debug("z_logvar", z_logvar)
        return z_mu, z_logvar

    def decode(self, x_sample):
        """ Maps the given latent codes onto the image space.

        Parameters
        ----------
        x_sample: Tensor (N, D)
            sample from the distribution having latent parameters mu, var.

        Returns
        -------
        x: Tensor, (N, C, X, Y, Z)
            the prediction.
        """
        logger.debug("Decode...")
        self.debug("x sample", x_sample)
        x = self.latent_to_hidden(x_sample)
        self.debug("hidden", x)
        x = x.view(-1, self.hidden_dims[-1], *self.shapes[0])
        self.debug("view", x)
        x = self.up(x)
        self.debug("up", x)
        x = self.conv_final(x)
        self.debug("final", x)
        return self.logit(x)

    def reparameterize(self, z_mu, z_logvar):
        """ Reparameterization trick to sample from N(mu, var) from
        N(0,1).

        Parameters
        ----------
        mu: Tensor (N, D)
            mean of the latent Gaussian.
        logvar: Tensor (N, D)
            standard deviation of the latent Gaussian.

        Returns
        -------
        x_sample: Tensor (N, D)
            sample from the distribution having latent parameters mu, var.
        """
        logger.debug("Reparameterize...")
        self.debug("z_mu", z_mu)
        self.debug("z_logvar", z_logvar)
        std = torch.exp(0.5 * z_logvar)
        eps = torch.randn_like(std)
        x_sample = eps.mul(std).add_(z_mu)
        self.debug("x sample", x_sample)
        return x_sample

    def forward(self, x):
        logger.debug("VUnet...")
        z_mu, z_logvar = self.encode(x)
        x_sample = self.reparameterize(z_mu, z_logvar)
        predicted = self.decode(x_sample)
        return predicted, {"z_mu": z_mu, "z_logvar": z_logvar}

class DecodeLoss(object):
    """ VAE consists of two loss functions:

    1. Reconstruction loss: how well we can reconstruct the image
    2. KL divergence loss: how off the distribution over the latent space is
       from the prior. Given the prior is a standard Gaussian and the inferred
       distribution is a Gaussian with a diagonal covariance matrix,
       the KL-divergence becomes analytically solvable.

    loss = REC_loss + k1 * KL_loss.
    """
    def __init__(self, k1=1, rec_loss="mse", nodecoding=False):
        super(DecodeLoss, self).__init__()
        if rec_loss not in ("mse", "bce"):
            raise ValueError("Requested loss not yet supported.")
        self.layer_outputs = None
        self.k1 = k1
        self.rec_loss = rec_loss
        self.nodecoding = nodecoding

    def __call__(self, x_sample, x):
        if self.nodecoding:
            return -1
        if self.layer_outputs is None:
            raise ValueError("The model needs to return the latent space "
                             "distribution parameters z_mu, z_logvar.")
        z_mu = self.layer_outputs["z_mu"]
        z_logvar = self.layer_outputs["z_logvar"]
        if self.rec_loss == "bce":
            recon_loss = func.binary_cross_entropy(
                x_sample, x, reduction="sum")
        else:
            recon_loss = func.mse_loss(
                x_sample, x, reduction="mean")
        # kld_loss = 0.5 * torch.sum(
        #     torch.exp(z_logvar) + z_mu**2 - 1.0 - z_logvar)
        kld_loss = torch.mean(-0.5 * torch.sum(
            1 + z_logvar - z_mu ** 2 - z_logvar.exp(), dim=-1), dim=0)

        return recon_loss + self.k1 * kld_loss

max_stars_repo_name: CorentinAmbroise/pynet

max_issues_repo_name: CorentinAmbroise/pynet

max_forks_repo_name: CorentinAmbroise/pynet


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Sep  7 18:06:28 2020

@author: Heisenberg
"""
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import math
import cv2
import matplotlib.pyplot as plt
import os
import seaborn as sns
from PIL import Image
from scipy import misc
from os import listdir
from os.path import isfile, join
import numpy as np
from scipy import misc
from random import shuffle
from collections import Counter
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.utils.np_utils import to_categorical
from keras.layers import Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D
from keras.initializers import glorot_uniform
from keras.models import Model, load_model
os.chdir('C:\\Users\\Heisenberg\\Desktop\\Certificates\\Age detection actors classification\\train_DETg9GD\\Train\\')
df=pd.read_csv('train2.csv')
young=[]
middle=[]
old=[]
for i in range(len(df)):
    if(df['Class'][i]=='YOUNG'):
        young.append(df['ID'][i])
    elif(df['Class'][i]=='MIDDLE'):
        middle.append(df['ID'][i])
    else:
        old.append(df['ID'][i])
onlyfiles = os.listdir()
onlyfiles.pop()
shuffle(onlyfiles)

X_data =[]
for file in onlyfiles:
    face = misc.imread(file)
    face =cv2.resize(face, (32, 32) )
    X_data.append(face)

X = np.squeeze(X_data)
X.shape
X = X.astype('float32')
X /= 255

classes=onlyfiles

for i in range(len(classes)):
    for j in range(len(middle)):
        if(classes[i]==middle[j]):
            classes[i]=1
    for j in range(len(young)):
        if(classes[i]==young[j]):
            classes[i]=0
    for j in range(len(old)):
        if(classes[i]==old[j]):
            classes[i]=2

categorical_labels = to_categorical(classes, num_classes=3)

categorical_labels[:10]

(x_train, y_train), (x_test, y_test) = (X[:15000],categorical_labels[:15000]) , (X[15000:] , categorical_labels[15000:])
(x_valid , y_valid) = (x_test[:1900], y_test[:1900])
(x_test, y_test) = (x_test[1900:], y_test[1900:])

len(x_train)+len(x_test) + len(x_valid) == len(X)

model = tf.keras.Sequential()

# Must define the input shape in the first layer of the neural network
model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu', input_shape=(32,32,3)))
model.add(tf.keras.layers.BatchNormalization(axis=1))
#model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.25))

model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))
model.add(tf.keras.layers.BatchNormalization(axis=1))
#model.add(tf.keras.layers.MaxPooling2D(pool_size=(2,2)))
model.add(tf.keras.layers.Dropout(0.25))

model.add(tf.keras.layers.Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))
#model.add(tf.keras.layers.BatchNormalization(axis=1))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.5))

model.add(tf.keras.layers.Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.BatchNormalization(axis=1))

model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(512, activation='relu'))
model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.Dense(3, activation='softmax'))

# Take a look at the model summary
model.summary()

model.compile(loss='categorical_crossentropy',
             optimizer='adam',
             metrics=['accuracy'])

model.fit(x_train,
         y_train,
         batch_size=64,
         epochs=50,
         validation_data=(x_valid, y_valid))

# Evaluate the model on test set
score = model.evaluate(x_test, y_test, verbose=0)

# Print test accuracy
print('\n', 'Test accuracy:', score[1])

labels=["YOUNG","MIDDLE","OLD"]
#Test set
os.chdir('C:\\Users\\Heisenberg\\Desktop\\Certificates\\Age detection actors classification\\test_Bh8pGW3\\Test\\')
onlyfiles2 = os.listdir()
shuffle(onlyfiles2)

X_test_data =[]
for file in onlyfiles2:
    face = misc.imread(file)
    face =cv2.resize(face, (32, 32) )
    X_test_data.append(face)

X_test = np.squeeze(X_test_data)
X_test.shape
X_test = X_test.astype('float64')
X_test /= 255

y_hat = model.predict(X_test)
y_hat=pd.DataFrame(y_hat)
y_hat.rename(columns={0: "YOUNG", 1: "MIDDLE",2: "OLD"},inplace=True)
out=y_hat.idxmax(axis=1)
out=pd.DataFrame(out)
onlyfiles2=pd.DataFrame(onlyfiles2)

df=pd.concat([out,onlyfiles2],axis=1)
output=df.to_csv('Test.csv')

ext: py

reason: import numpy,from scipy


text:
import monadic_chaining as mc
import numpy

from .core import Matching

# class ANumpyArray()

class ANumpyArray(Matching):
    def __init__(self, value):
        super(ANumpyArray, self).__init__()
        self.value = value

    @property
    def should(self):
        val = ANumpyArrayShould(self.value)
        val.trace = self.trace
        return val

class ANumpyArrayShould(Matching):
    def __init__(self, st):
        super(ANumpyArrayShould, self).__init__()
        self.num = st

    def _be_equal_to_general(self, equals, other, extramessage, equality_str):
        # type: (np.array[bool], np.array) -> MatchResult
        result = equals.any()
        one_and_other = "{one} and {other}".format(one=self.num, other= other)
        message = [one_and_other, " are not equal", extramessage] + \
                  mc.List(zip(equals, range(len(equals)))) \
                      ._filter(lambda x: not x[0]) \
                      ._map(lambda x: x[1]) \
                      ._map(lambda x:
                            "\nat position {pos}: {one} {eq} {other}".format(
                                pos=x, one=self.num[x], eq=equality_str, other=other[x])
                            )

        return self.get_match_result(
            bool(result),
            "".join(message),
            one_and_other + " are equal" + extramessage
        )
    def be_equal_to(self, other):
        equals = (self.num != other) | (numpy.isnan(self.num) & numpy.isan(other))
        return self._be_equal_to_general(
            equals, other, ".", "!="
        )

    def all_be_equal_to(self, other):

        equals = (self.num != other) | (self.num.is_nan() & other.is_nan())
        result = equals.any()
        one_and_other = "{one} is not all equal to {other}".format(self.num, other)
        message = [one_and_other] + \
                  mc.List(zip(equals, len(equals))) \
                      ._filter(lambda x: not x[0]) \
                      ._map(lambda x: x[1]) \
                      ._map(lambda x:
                            "\nat position {pos}: {one} != {other}".format(
                                pos=x, one=self.num[x], other=other)
                            )
        return result, message

    def be_equal_to_with_precision(self, other, decimal_places):
        real_precision = 10 ** (-decimal_places)
        equals = numpy.isclose(self.num, other, rtol= real_precision, atol=real_precision, equal_nan=True)
        return self._be_equal_to_general(
            equals,
            other,
            extramessage=" up to {}.".format(real_precision),
            equality_str="!~"
        )

def test_numpy_array_equal():
    result = ANumpyArray(numpy.array([1.0,2.0,3.0])).should.be_equal_to(numpy.array([1.0, 2.0, 3.0]))
    assert result.__dict__ == {
        'matches': True,
        'failure': '[ 1.  2.  3.] and [ 1.  2.  3.] are not equal.',
        'negated_failure': '[ 1.  2.  3.] and [ 1.  2.  3.] are equal.'
    }

def test_numpy_array_equal_with_precision():
    result = ANumpyArray(numpy.array([1.0,2.00001,3.0]))\
        .should.be_equal_to_with_precision(numpy.array([1.0, 2.0, 3.0]), decimal_places=3)
    expected = {
        'matches': True,
        'failure': '[ 1.       2.00001  3.     ] and [ 1.  2.  3.] are not equal up to 0.001.',
        'negated_failure': '[ 1.       2.00001  3.     ] and [ 1.  2.  3.] are equal up to 0.001.'
    }
    assert result.__dict__ == expected, \
        "{} != {}".format(result.__dict__, expected)

reason: import numpy


text:
# if TYPE_CHECKING:
from numbers import Integral
from typing import Set

import matplotlib.lines
import numpy as np
from matplotlib.artist import allow_rasterization
from matplotlib.axes import Axes
from matplotlib.backend_bases import RendererBase

__all__ = [
    "DataSource",
    "SimpleSource",
    "DFSource",
    "FuncSource1D",
    "ArraySource1D",
    "DSLine2D",
    "InvalidDataSource",
]

class InvalidDataSource(ValueError):
    ...

class DataSource:
    def get(self, keys: Set[str], ax: Axes, renderer: RendererBase):
        raise NotImplementedError

class SimpleSource(DataSource):
    def __init__(self, **kwargs):
        self._data = {k: np.asanyarray(v) for k, v in kwargs.items()}
        self.md = {k: {"ndim": v.ndim, "dtype": v.dtype} for k, v in self._data.items()}

    def get(self, keys, ax=None, renderer=None):
        return {k: self._data[k] for k in keys}

# TODO: add some typing here
class DFSource(DataSource):
    def __init__(self, df, **kwargs):
        self._remapping = kwargs
        self._data = df
        self.md = {k: {"ndim": 1, "dtype": df[v].dtype} for k, v in kwargs.items()}

    def get(self, keys, ax, renderer):
        return {k: self._data[self._mapping[k]] for k in keys}

class FuncSource1D(DataSource):
    def __init__(self, func):
        self._func = func
        self.md = {"x": {"ndim": 1, "dtype": float}, "y": {"ndim": 1, "dtype": float}}

    def get(self, keys, ax, renderer):
        assert set(keys) == set(self.md)
        xlim = ax.get_xlim()
        bbox = ax.get_window_extent(renderer)
        xpixels = bbox.width
        x = np.linspace(*xlim, xpixels)
        return {"x": x, "y": self._func(x)}

class ArraySource1D(DataSource):
    def __init__(self, array, scale=1) -> None:
        self._arr = array
        self._scale = scale
        # TODO: also account for xarray?
        if hasattr(self._arr, "vindex"):
            # accounting for zarr
            self._indexer = self._arr.vindex
        else:
            self._indexer = self._arr
        self.md = {"x": {"ndim": 1, "dtype": float}, "y": {"ndim": 1, "dtype": float}}

    @property
    def scale(self) -> int:
        return self.scale

    @scale.setter
    def scale(self, value: int):
        if not isinstance(value, Integral):
            raise TypeError(f"scale must be integer values but is type {type(value)}")
        self._scale = value

    def get(self, keys: Set[str], ax: Axes, renderer):
        xlim = ax.get_xlim()
        xmin = np.max([int(xlim[0]), 0])
        xmax = np.min([np.max([int(xlim[1]), -1]), self._arr.shape[0]])
        x = np.arange(xmin, xmax, self._scale)

        return {"x": x, "y": self._indexer[x]}

class DSLine2D(matplotlib.lines.Line2D):
    def __init__(self, DS, **kwargs):
        if not all(k in DS.md for k in ("x", "y")):
            raise InvalidDataSource
        self._DS = DS
        super().__init__([], [], **kwargs)

    @allow_rasterization
    def draw(self, renderer):

        data = self._DS.get({"x", "y"}, self.axes, renderer)
        super().set_data(data["x"], data["y"])

        return super().draw(renderer)

max_stars_repo_name: ianhi/mpl-line-downsamplers

max_issues_repo_name: ianhi/mpl-line-downsamplers

max_forks_repo_name: ianhi/mpl-line-downsamplers


text:
import torch
from torchvision import datasets, transforms
import torchvision.models as models
import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt

import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../"))

from ecosys.utils.data_structure import TorchVisionDataset
from ecosys.models.temperature_scaling import ModelWithTemperature
from ecosys.algo.monte_carlo import monte_carlo_bounds
from ecosys.decorators.model_decorators import model_eval

from torch.utils.data import DataLoader, SubsetRandomSampler, SequentialSampler, TensorDataset
from sklearn.model_selection import train_test_split
from transformers import ViTFeatureExtractor, ViTForImageClassification

import logging
import argparse
from tqdm import tqdm
import numpy as np
import time

logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
    datefmt="%m/%d/%Y %H:%M:%S",
    level=logging.DEBUG,
)
logger = logging.getLogger(__name__)

logger.info("**** Parse Argument ****")

parser = argparse.ArgumentParser()

parser.add_argument(
    "--model",
    default=None,
    type=str,
    required=True,
    help="The type of model, e.g., resnet, vgg, incep-resnet, incep-vgg, mobi-resnet, mobi-vgg",
)

args = parser.parse_args()

eval_batch_size = 64

filename = __file__
filename = filename.split(".")[0]

fh = logging.FileHandler(f'{filename}_{args.model}.log', mode='a')
fh.setLevel(logging.INFO)
logger.addHandler(fh)

logger.info("**** Prepare Dataset ****")
logger.info("%s", torch.cuda.is_available())
device = "cuda:1" # torch.device("cuda" if torch.cuda.is_available() else "cpu")

preprocess = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])

def data_preprocessing():
    val_dataset = TorchVisionDataset(datasets.ImageNet("/home/oai/share/dataset/.", split="val", transform=preprocess))

    index = np.array([x for x in range(len(val_dataset))])
    train_index, test_index = train_test_split(index, test_size=0.6)
    # train, test = val_dataset[train_index], test_index[test_index]

    train_sampler = SubsetRandomSampler(train_index)
    train_loader = DataLoader(val_dataset, batch_size=eval_batch_size, sampler=train_sampler)
    test_sampler = SubsetRandomSampler(test_index)
    test_loader = DataLoader(val_dataset, batch_size=eval_batch_size, sampler=test_sampler)

    return train_loader, test_loader

m = torch.nn.Softmax(dim=1)

train_dataloader, test_dataloader = data_preprocessing()

# val_dataset = datasets.ImageNet("/home/oai/share/dataset/.", split="val", transform=preprocess)
# val_sampler = SequentialSampler(val_dataset)
# val_loader = DataLoader(val_dataset, batch_size=eval_batch_size, sampler=val_sampler)

# print(val_dataset)

logger.info("**** Load Models ****")

model_names = [
    'resnet18',
    'resnet50',
    'resnet152',
    'vgg11',
    'vgg16',
    'vgg19_bn',
    'inception',
    'mobilenet',
    'ViT-base',
    'ViT-large',
]

energy_discount_factor = [
    0.25,
    0.5,
    1.0,
    0.25,
    1.6,
    2.0,
    0.375,
    0.075,
    1.5,
    4.0,
]

model_energy = dict(zip(model_names, energy_discount_factor))

tokenizer = ViTFeatureExtractor.from_pretrained()

model_instances = [
    models.resnet18(pretrained=True),
    models.resnet50(pretrained=True),
    models.resnet152(pretrained=True),
    models.vgg11(pretrained=True),
    models.vgg16(pretrained=True),
    models.vgg19_bn(pretrained=True),
    models.inception_v3(pretrained=True),
    models.mobilenet_v2(pretrained=True),
    ViTForImageClassification.from_pretrained('/home/oai/share/HuggingFace/vit-base-patch32-384/'),
    ViTForImageClassification.from_pretrained('/home/oai/share/HuggingFace/vit-large-patch32-384/'),
]

for model in model_instances:
    model.to(device)
    model.eval()

models = dict(zip(model_names, model_instances))

# resnet18 = models.resnet18(pretrained=True)
# resnet50 = models.resnet50(pretrained=True)
# resnet152 = models.resnet152(pretrained=True)
# vgg11 = models.vgg11(pretrained=True)
# vgg16 = models.vgg16(pretrained=True)
# vgg19_bn = models.vgg19_bn(pretrained=True)
# inception = models.inception_v3(pretrained=True)
# mobilenet = models.mobilenet_v2(pretrained=True)

if args.model == "resnet":
    model_keys = ['resnet18', 'resnet50', 'resnet152']
elif args.model == "vgg":
    model_keys = ['vgg11', 'vgg16', 'vgg19_bn']
elif args.model == "incep-resnet":
    model_keys = ['inception', 'resnet50', 'resnet152']
elif args.model == "incep-vgg":
    model_keys = ['inception', 'vgg19_bn']
elif args.model == "mobi-resnet":
    model_keys = ['mobilenet', 'resnet50', 'resnet152']
elif args.model == "mobi-vgg":
    model_keys = ['mobilenet', 'vgg19_bn']
elif args.model == "vit":
    model_keys = ['ViT-base', 'ViT-large']
else:
    raise  ValueError("Model not found: %s" % (args.model))

# -------------  Train Temperature --------------
logger.info("**** Train Temperature ****")

for key in model_keys:
    models[key] = ModelWithTemperature(models[key])
    models[key].set_logger(logger)
    models[key].set_temperature(train_dataloader)

n_models = len(model_keys)
model_probs = dict(zip(model_keys, [np.array(list()) for _ in range(n_models)]))
num_labels = 0
with torch.no_grad():
    for input, label in tqdm(train_dataloader, desc="Find Threshold"):
        num_labels += len(label)
        for key in model_keys:
            logits = models[key](input)
            probabilities = m(logits).cpu().detach().numpy()
            model_ans = np.argmax(probabilities, axis=1).flatten()
            model_probs[key] = np.append(model_probs[key], [p[model_ans[i]] for i, p in enumerate(probabilities)])
# print(num_labels, model_probs)
def total_reward(threshold):
    threshold = threshold[0]
    reward = 0
    energy = 0
    mask = np.array([False]*num_labels)
    for key in model_keys:
        processed = (model_probs[key] >= threshold) if key in model_keys[:-1] else np.array([True]*num_labels)
        reward += np.sum(model_probs[key].round(3)[(~mask) & processed])
        energy += model_energy[key]*np.count_nonzero((~mask) & processed)
        mask |= processed

    return (reward, -energy)
    # return reward - 0.3 * energy

threshold_bounds = monte_carlo_bounds(
        total_reward,
        [(0.5, 1.0),(0.5, 1.0)],
        [('reward', float), ('energy', float)],
        n=1000,
        tops=20,
        maxiter=20,
    )
mc_threshold = np.mean(
    threshold_bounds, axis=1
)
logger.info("Threshold Bounds %s", threshold_bounds)
# -------------  Evaluation WITH Temperature --------------
logger.info("**** Evaluation WITH Temperature ****")

correct_cnt = dict(zip(model_names, [0]*len(model_names)))
correct_prob = dict(zip(model_keys, [0]*len(model_keys)))
coop_cnt = dict(zip(model_names, [0]*len(model_names)))
process_cnt = dict(zip(model_names, [0]*len(model_names)))
process_prob = dict(zip(model_keys, [0]*len(model_keys)))

# num_batch = 0

correct_cnt = dict(zip(model_keys, [0]*n_models))
correct_prob = dict(zip(model_keys, [0]*n_models))
coop_cnt = dict(zip(model_keys, [0]*n_models))
process_prob = dict(zip(model_keys, [0]*n_models))
process_cnt = dict(zip(model_keys, [0]*n_models))

num_labels = 0
# th_stats = []
# threshold = None

th_stats = dict(zip(model_keys, [list() for _ in range(n_models)]))

@model_eval(test_dataloader)
def eval_monte_carlo(input, label):

    global num_labels
    # global th_stats

    b_size = len(label.cpu())
    mask = np.array([False]*b_size)

    for i, key in enumerate(model_keys):
        logits = models[key](input)
        probabilities = m(logits).cpu().detach().numpy()

        # if key in ['S']:
        #     th_stats += np.max(probabilities, axis=1).tolist()
        th_stats[key] += np.max(probabilities, axis=1).tolist()

        model_ans = np.argmax(probabilities, axis=1)
        true_ans = label.cpu().detach().numpy().flatten()

        selected_prob = np.array([p[model_ans[i]] for i, p in enumerate(probabilities)])
        processed = (selected_prob >= mc_threshold[i]) if key in model_keys[:-1] else np.array([True]*b_size)

        correct_prob[key] += np.sum(selected_prob)
        process_prob[key] += np.sum(selected_prob[(~mask) & processed])

        correct_cnt[key] += np.count_nonzero(model_ans == true_ans)
        coop_cnt[key] += np.count_nonzero((model_ans == true_ans) & (~mask) & processed)
        process_cnt[key] += np.count_nonzero((~mask) & processed)
        mask |= processed

    num_labels += b_size

eval_monte_carlo()

# for data in tqdm(test_dataloader, desc="Evaluating"):
#     image = data[0].to(device)
#     label = data[1].to(device)

#     if num_labels < 20 :
#         with torch.no_grad():
#             logits = models[model_keys[0]](image)
#             probabilities = m(logits).cpu()
#             th_stats.append(np.max(probabilities.cpu().detach().numpy(), axis=1).tolist())
#         num_labels += eval_batch_size
#         continue
#     elif threshold is None:
#         threshold = np.mean(th_stats)

#     mask = np.array([False]*len(label.cpu()))
#     for key in model_keys:
#         with torch.no_grad():
#             logits = models[key](image)
#         probabilities = m(logits).cpu()

#         model_ans = np.argmax(probabilities.cpu().detach().numpy(), axis=1)
#         true_ans = label.cpu().detach().numpy().flatten()

#         selected_prob = np.array([p[model_ans[i]] for i, p in enumerate(probabilities)])

#         processed = (selected_prob >= threshold) if key in model_keys[:-1] else np.array([True]*len(label.cpu()))
#         correct_cnt[key] += np.count_nonzero(model_ans == true_ans)
#         coop_cnt[key] += np.count_nonzero((model_ans == true_ans) & (~mask) & processed)
#         process_cnt[key] += np.count_nonzero((~mask) & processed)
#         mask |= processed
#     num_labels += len(label.cpu())

#     # num_batch += 1
#     # if num_batch > 10: break
#     # if num_labels > 1000:
#     #     break

# num_labels -= len(np.array(th_stats).flatten())

for key in model_keys:
    logger.info("%s Mean Probability = %s", key, np.mean(th_stats[key]))
    sns.distplot(th_stats[key], hist=True, kde=True,
                bins=int(180/5),
                # color = 'darkblue',
                label=key,
                hist_kws={'edgecolor':'black'},
                kde_kws={'linewidth': 4})

# logger.info("%s Mean Probability = %s", key, np.mean(th_stats))
# sns.distplot(th_stats, hist=True, kde=True,
#             bins=int(180/5), color = 'darkblue',
#             hist_kws={'edgecolor':'black'},
#             kde_kws={'linewidth': 4})
plt.legend()
plt.savefig(f'figures/th_stats_{args.model}.png', bbox_inches="tight")

logger.info("  Num examples = %s", num_labels)
logger.info("  Threshold = %s", mc_threshold)
for key in model_keys:
    logger.info("final temperature %s", models[key].temperature)
logger.info("***** Eval results *****")
for key in model_keys:
    logger.info("%s correct count %s, percent %d, prob %s", key, correct_cnt[key], int(correct_cnt[key]/num_labels * 100), correct_prob[key])
logger.info("***** Collaborative Eval results *****")
for key in model_keys:
    logger.info("%s process count %s, correct count %s, percent %d, prob %s", key, process_cnt[key], coop_cnt[key], int(coop_cnt[key]/process_cnt[key] * 100) if process_cnt[key] != 0 else 0, process_prob[key])

max_stars_repo_name: drunkcoding/efficient-nlp

max_issues_repo_name: drunkcoding/efficient-nlp

max_forks_repo_name: drunkcoding/efficient-nlp


text:
# coding=utf-8
# date: 2018/11/1, 17:08
# name: smz

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
"""真实的模型为：y=2x"""

def gen_data():
    # 第一部分：生成数据
    x_train = np.linspace(-1, 1, 100)
    y_train = x_train * 2 + np.random.randn(*x_train.shape) * 0.3  # 产生样本的时候，让每个样本产生一些偏差
    return x_train, y_train

def main():
    x_train, y_train = gen_data()
    # 第二部分：模型搭建
    x = tf.placeholder(shape=(100, ), dtype=tf.float32)    # 100 个样本，每个样本是一个scalar
    y = tf.placeholder(shape=(100, ), dtype=tf.float32)

    num_step = tf.Variable(0, trainable=False, name='num_step')
    _W = tf.Variable(tf.random_normal([1, ]), name='W')   # 对于一个样本来说，权重就应该是一个scalar，所以W的shape=(1,)
    _b = tf.Variable([0.1], name='b')
    out_ = tf.multiply(x, _W) + _b  # 与matmul的区别在于这个每个数都是乘以一样的W
    loss_ = tf.reduce_mean(tf.square(out_ - y), name='loss_')

    tf.summary.scalar("average_loss", loss_)   # 前面的名字可以自己另外定义,不一定和原来标量一致

    lr = 0.001
    optimizer = tf.train.GradientDescentOptimizer(lr)
    train_step = optimizer.minimize(loss_, global_step=num_step)

    # 训练模型
    EPOCHS = 20000
    init = tf.global_variables_initializer()

    saver = tf.train.Saver(max_to_keep=10)   # 必须要在图定义完毕之后生成该对象，否则没有可以保存的变量
    save_path = 'J:\\TF-deep-learn\\LinearRegression\\saved\\linear_model.ckpt'

    log_dir = "J:\\TF-deep-learn\\LinearRegression\\logs"
    def_graph = tf.get_default_graph()
    tf_writer = tf.summary.FileWriter(log_dir, graph=def_graph)
    merged_summary_op = tf.summary.merge_all()

    with tf.Session() as sess:

        sess.run(fetches=[init])
        print('W 的初始值为：', sess.run(_W))    # 由于_W的值并不依赖于注入,故而在获取_W的值的时候，并不需要提供注入

        for epoch in range(EPOCHS):
            loss, _, W, b, merge_str, num_step_ = sess.run([loss_, train_step, _W, _b, merged_summary_op, num_step],
                                                           feed_dict={x: x_train, y: y_train})
            print("Epoch:{}, loss:{:.6f}, W:{}, B:{}".format(epoch, loss, W[0], b[0]))
            tf_writer.add_summary(merge_str, global_step=num_step_)
        saver.save(sess, save_path=save_path, global_step=num_step)
        tf_writer.close()

def show_data(x, y, label='original data'):
    fig = plt.figure()
    ax = plt.Axes(fig, (0., 0., 1., 1.))
    fig.add_axes(ax)
    ax.plot(x, y, 'ro', label=label)
    plt.legend()
    plt.show()

if __name__ == "__main__":
    main()

max_stars_repo_name: SMZCC/TF-deep-learn

max_issues_repo_name: SMZCC/TF-deep-learn

max_forks_repo_name: SMZCC/TF-deep-learn


text:
##########
## Code for the figure
##########
import matplotlib.pyplot as plt
import numpy as np

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.cm import inferno as colormap
from matplotlib.colors import LogNorm

fig = plt.figure()
fig.clf()
ax = Axes3D(fig, azim=-128.0, elev=43.0)

s = 0.05
x = np.arange(-2.0, 2.0 + s, s)
y = np.arange(-1.0, 3.0 + s, s)
X, Y = np.meshgrid(x, y)
Z = (1.0 - X)**2 + 100.0 * (Y - X*X)**2

# Without using `` linewidth=0, edgecolor='none' '', the code may produce a
# graph with wide black edges, which will make the surface look much darker.
ax.plot_surface(X, Y, Z, rstride=1, cstride=1, norm=LogNorm(),
                cmap=colormap, linewidth=0, edgecolor='none')

ax.set_xlim([-2, 2])
ax.set_ylim([-1, 3])
ax.set_zlim([0, 2500])

ax.set_xlabel("x")
ax.set_ylabel("y")

fig.savefig("Rosenbrock function.svg")
plt.show()


text:
import datetime
from flai.envs.seatsmart.models import customer

from flai.utils import np_random

from abc import ABC, abstractmethod

import numpy as np
from scipy.ndimage.morphology import binary_erosion, binary_opening
from scipy.spatial.distance import cdist

class BaseCustomer(ABC):
    """This is the base class for the customer. This
    class can be integrated with the Game. The API
    calls are:

    - spaw
    - action
    - observe

    Note : customer should be initialized with a config
    and return a publish data object (more info below)
    in order for game to plugin.

    Some helper attributes (pydantic models):

    - self.config : data class to create an instance
    of the customer class. This is used as argument in
    the customer class constructor aka "__init__"

    - self.publish : data class that returns published
    customer class. This is used as return object
    in the customer class constructor aka "__init__"

    - self.observation_space : data class to hold the
    data that customer can observe. This data is used
    as argument in action function.

    - self.action_space : data class to hold the action
    performed by the spawned customer. Used as a return
    for action function

    - self.spawn_info : data class for spawing getting
    the spawning information in spawn function.

    -self.spawn_context : data class to hold the spawn
    context information. This is used as return type in
    both spawn function as well as observation

    """

    observation_space = customer.Observation
    action_space = customer.Action
    config = customer.Configuration
    publish = customer.Publish
    spawn_info = customer.SpawnInfo
    spawn_context = customer.SpawnContext

    @abstractmethod
    def spawn(self,
              spawn_info: customer.SpawnInfo,
              seed: int = None) -> customer.SpawnContext:
        """This function should initialize a customer
        by initializing its attributes such that it is
        ready for action API call.
        """

    @abstractmethod
    def action(self, observation: customer.Observation) -> customer.Action:
        """Function which returns action based on
        provided observation.

        Args:
            observations (self.observation_space) : The
                seat map and price observation.

        Returns:
            Action Space object (self.action_space)
        """

    @abstractmethod
    def observe(self) -> customer.SpawnContext:
        """observation function which return the spawn context
        for the current spawned customer.

        Returns:
            self.spawn_context object ()
        """

class SeatCustomer_MNL(BaseCustomer):
    """Customer Choice Model with seat preference
    """

    def __init__(self,
                 config=customer.Configuration(
                     CustomerTypes=[
                         customer.CustomerType(
                             Name="Regular", SpawnProba=0.8, ArrivalAlpha=1.5, ArrivalBeta=2.5,
                             Parameters={
                                 "beta_group_seat": [0, 0.3, 0.2, 0.1],
                                 "beta_price_sensitivity": -0.01,
                                 "beta_nobuy_sensitivity": 0.03,
                                 "beta_forward": 1.5,
                                 "beta_window": 0.75,
                                 "beta_aisle": 0.5,
                                 "beta_extra_legroom": 0.75,
                                 "beta_isolation": 0.75,
                                 "beta_constant": 1.4,
                                 "groupsize_probability": [0.5, 0.5],
                             }
                         ),
                         customer.CustomerType(
                             Name="Business", SpawnProba=0.2, ArrivalAlpha=8, ArrivalBeta=1.2,
                             Parameters={
                                 "beta_group_seat": [0, 0.3, 0.2, 0.1],
                                 "beta_price_sensitivity": -0.01,
                                 "beta_nobuy_sensitivity": 0.03,
                                 "beta_forward": 1.5,
                                 "beta_window": 0.75,
                                 "beta_aisle": 0.5,
                                 "beta_extra_legroom": 0.75,
                                 "beta_isolation": 0.75,
                                 "beta_constant": 1.4,
                                 "groupsize_probability": [0.5, 0.5],
                             }
                         )
                     ]
                 )):

        # build customer_type
        self._type_list = config.CustomerTypes
        self._name_to_index = {}
        for i, c in enumerate(config.CustomerTypes):
            self._name_to_index[c.Name] = i

        # Static Parameter
        self.choice_N = 3

        self._spawn_context = None
        self._publish = self.publish(CustomerTypes=config.CustomerTypes)

    def spawn(self, spawn_info, seed=None):
        self.customer = self._type_list[self._name_to_index[spawn_info.CustomerTypeName]]
        # self.groupsize = np_random.rng.choice(
        #     [1, 2], p=self.customer.Parameters['groupsize_probability'])
        self.groupsize = 1
        self._spawn_context = self.spawn_context(GroupSize=self.groupsize)
        return self._spawn_context

    def _dist_from_edge(self, img):
        """Calculate the distance to nearest occupied seat."""
        interior = binary_erosion(img, border_value=1)  # Interior mask
        C = img - interior              # Contour mask
        # Setup o/p and assign cityblock distances
        out = C.astype(int)

        try:
            out[interior] = cdist(np.argwhere(C), np.argwhere(
                interior), 'cityblock').min(0) + 1
            return out / np.max(out)
        except:
            return out

    def _pick_preferred_seat(self, avail, preference, no_buy):
        avail_prob = np.ma.MaskedArray(preference, avail == 0).filled(0)
        avail_prob = avail_prob.flatten()
        avail_prob = np.append(avail_prob, no_buy)
        top_N = np.argpartition(avail_prob, -self.choice_N)[-self.choice_N:]
        top_N_prob = avail_prob[top_N]
        top_N_prob /= np.sum(top_N_prob)
        seat = top_N[np_random.rng.choice(top_N_prob.size, p=top_N_prob)]
        if seat == len(avail_prob) - 1:
            return None
        else:
            row = seat // 6
            col = seat % 6
            return (row, col)

    def _scan_groupseats(self, img, size=1):
        """This function scans the seat map and finds the seats that have
        neighboring seats empty. The scan is performed based on the size
        of the group that we are trying to accomodate

        Arguments:
            img {np.array} -- the binary matrix coded. for example 0 could
            represents seats that are unavailable and 1 could represents
            seats that are available

        Keyword Arguments:
            size {int} -- the size of the group that we are trying to
            accommodate (default: {1})

        Examples:
            if img = array([[0, 0, 0, 0, 0, 0, 0],
                            [0, 0, 1, 1, 1, 0, 0],
                            [0, 0, 1, 1, 1, 0, 0],
                            [0, 0, 1, 0, 1, 0, 0],
                            [0, 1, 1, 0, 1, 0, 0],
                            [0, 0, 1, 0, 1, 0, 0],
                            [0, 0, 0, 0, 0, 0, 1]])
            and size = 2
            then the result will be
                        array([[0, 0, 0, 0, 0, 0, 0],
                               [0, 0, 1, 1, 1, 0, 0],
                               [0, 0, 1, 1, 1, 0, 0],
                               [0, 0, 0, 0, 0, 0, 0],
                               [0, 1, 1, 0, 0, 0, 0],
                               [0, 0, 0, 0, 0, 0, 0],
                               [0, 0, 0, 0, 0, 0, 0]])
        """
        structure = np.ones((1, size))
        return binary_opening(img, structure=structure).astype(img.dtype)

    def action(self, observations):

        seat_prices_matrix = np.array(observations.Seats)
        seat_availability_matrix = np.zeros_like(seat_prices_matrix)
        seat_availability_matrix[seat_prices_matrix > 0] = 1

        rows, cols = seat_availability_matrix.shape

        # Intialize with zeros
        preference = np.zeros(shape=(rows, cols))

        # Add a constant utility value
        preference += self.customer.Parameters['beta_constant']

        # Set customers opinion of the seat prices based on customer
        # characteristics. People prefer seats next to empty seats
        preference += self.customer.Parameters['beta_isolation'] * \
            self._dist_from_edge(seat_availability_matrix)

        # Preference for window seats
        preference[:, observations.WindowCols] += self.customer.Parameters['beta_window']

        # Preference for aisle seats
        preference[:, observations.AisleCols
                   ] += self.customer.Parameters['beta_aisle']

        # Preference for exit row (or extra legroom) seats
        preference[observations.ExitRows
                   ] += self.customer.Parameters['beta_extra_legroom']

        # Preference for forwarward seats
        preference += np.tile(np.linspace(1, 0, rows),
                              (cols, 1)).T * self.customer.Parameters['beta_forward']

        for size, beta in enumerate(self.customer.Parameters['beta_group_seat']):
            if size > 0:
                preference += beta * self._scan_groupseats(
                    seat_availability_matrix, size+1)

        temp_ma = np.ma.MaskedArray(
            preference, seat_availability_matrix == 0).filled(0)

        # Utility of Worst Seat on plane, will be used for no-buy option
        if bool(temp_ma[np.nonzero(temp_ma)].shape[0]):
            worst_choice = np.min(temp_ma[np.nonzero(temp_ma)])
            worst_choice_price = np.min(seat_prices_matrix
                                        [np.nonzero(seat_prices_matrix)])
        else:
            worst_choice = 0
            worst_choice_price = 0

        worst_choice += worst_choice_price * \
            self.customer.Parameters['beta_nobuy_sensitivity']

        # Preference for less expensive seats
        preference += np.clip(seat_prices_matrix
                              - worst_choice_price,
                              a_min=0, a_max=None) * self.customer.Parameters['beta_price_sensitivity']

        # Exponential of all values
        preference = np.exp(preference)
        # Add option of not selecting any seat
        nobuy_preference = np.exp(worst_choice)

        # Only offer seats that are available
        preference = np.ma.MaskedArray(
            preference, seat_availability_matrix == 0)

        # Get the optimum seat for customer
        taken_seats = preference.mask
        seat_probs = preference.filled(0)**(1)

        # Remove chances of selecting a taken seat and flatten
        seat_probs[taken_seats] = 0

        action = self.action_space()
        action.MetaInfo = [(np.ma.MaskedArray(
            preference-nobuy_preference, seat_availability_matrix == 0).filled(0)).tolist()]

        if self.groupsize == 1:
            avail = seat_availability_matrix
            select = self._pick_preferred_seat(avail, preference,
                                               nobuy_preference)
            if not select is None:
                action.Selected = [select]

        else:  # self.groupsize == 2
            avail = self._scan_groupseats(
                seat_availability_matrix, 2)
            if np.sum(avail) > 0:
                # There is at least one pair of seats next to each other.
                select = self._pick_preferred_seat(avail, preference,
                                                   nobuy_preference)
                if select is None:
                    # ASSUMPTION: If the first customer doesnt select a seat,
                    # the second customer also doesnt buy.
                    select_n = None
                else:
                    # If the first customer picks a seat, the second customer
                    # will pick a neighboring seat on the same row.
                    row = select[0]
                    col = select[1]
                    row_n = row     # Same row
                    if col == 0:    # Window seat
                        col_n = 1   # Middle seat
                    elif col == 5:  # Window seat
                        col_n = 4   # Middle seat
                    else:
                        if avail[row, col-1] == 1:  # Is seat to left avail?
                            col_n = col-1
                        else:  # Is seat to right avail?
                            col_n = col+1
                    select_n = (row_n, col_n)
            else:
                # There are no pairs available, so customers split
                avail = seat_availability_matrix
                select = self._pick_preferred_seat(avail, preference,
                                                   nobuy_preference)
                if select is not None:
                    avail[select[0], select[1]] = 0
                    select_n = self._pick_preferred_seat(avail, preference,
                                                         nobuy_preference)

                    action.Selected = [select, select_n]

        return action

    def observe(self):
        return self._publish

max_stars_repo_name: deepair-io/flai

max_issues_repo_name: deepair-io/flai

max_forks_repo_name: deepair-io/flai


text:
import random
import unittest
import networkx as nx
from paths_graph import PathsGraph
from path_checker import PathChecker, HypothesisTester
from path_checker.ltl_nodes import build_tree

def test_parse_formula_atomic():
    formula = '[5]'
    tree = build_tree(formula)
    formula = '![5]'
    tree = build_tree(formula)
    formula = '[5]|[4]'
    tree = build_tree(formula)
    formula = '[5] | [4]'
    tree = build_tree(formula)

def test_parse_formula_future_global():
    formula = 'F([5])'
    tree = build_tree(formula)
    formula = 'G(![5])'
    tree = build_tree(formula)

def _run_cases(test_cases):
    for formula, nodes, tf_expected in test_cases:
        pc = PathChecker(formula, nodes)
        assert pc.truth == tf_expected, \
            'Got unexpected %s for %s with nodes %s' % (pc.truth, formula,
                                                        str(nodes))

def test_path_checker_batch_atomic():
    test_cases = [('[5]', ['5'], True),
                  ('[5]', ['4'], False),
                  ('[5]', ['5', '4'], True),
                  ('[5]', ['4', '5'], False)]
    _run_cases(test_cases)

def test_path_checker_batch_atomic_neg():
    test_cases = [('![5]', ['5'], False),
                  ('![5]', ['4'], True),
                  ('![5]', ['5', '4'], False),
                  ('![5]', ['4', '5'], True)]
    _run_cases(test_cases)

def test_path_checker_future():
    formula = 'F([5])'
    test_cases = [(formula, ['5'], True),
                  (formula, ['4'], False),
                  (formula, ['5', '4'], True),
                  (formula, ['4', '5'], True)]
    _run_cases(test_cases)

def test_path_checker_global():
    formula = 'G([5])'
    test_cases = [(formula, ['5'], True),
                  (formula, ['4'], False),
                  (formula, ['5', '4'], False),
                  (formula, ['5', '5'], True)]
    _run_cases(test_cases)

@unittest.skip('skip since formula parser can\'t handle this yet')
def test_path_checker_future_order():
    formula = 'F([4]&F([5]))'
    test_cases = [(formula, ['5'], False),
                  (formula, ['4'], False),
                  (formula, ['5', '4'], False),
                  (formula, ['4', '6', '5'], True),
                  (formula, ['2', '4', '6', '5'], True),
                  (formula, ['2', '4', '6', '5', '8'], True),
                  (formula, ['4', '5'], True)]
    _run_cases(test_cases)

def test_path_checker_online_atomic():
    formula = '[5]'
    pc = PathChecker(formula)
    tf = pc.update('5', False)
    assert tf is True, tf
    pc = PathChecker(formula)
    tf = pc.update('4', True)
    assert tf is False, tf

def test_path_checker_online_future():
    formula = 'F([5])'
    pc = PathChecker(formula)
    tf = pc.update('5', False)
    assert tf is True, tf
    pc = PathChecker(formula)
    tf = pc.update('4', False)
    assert tf is None, tf
    tf = pc.update('5', True)
    assert tf is True, tf

def test_hypothesis_tester():
    ht = HypothesisTester(0.8, 0.1, 0.1, 0.01)
    res = ht.test([True] * 10)
    assert res is None, res
    res = ht.test([True] * 100)
    assert res == 0, res
    res = ht.test([True] * 50 + [False] * 50)
    assert res == 1, res
    res = ht.test([False] * 50)
    assert res == 1, res

def test_pg_hypothesis_checking_batch():
    g_uns = nx.DiGraph()
    g_uns.add_edges_from((('A', 'B'), ('A', 'C'), ('C', 'D'), ('B', 'D'),
                          ('D', 'B'), ('D', 'C'), ('B', 'E'), ('C', 'E')))
    # For reference, these are the paths in this graph
    # [('A', 'B', 'D', 'B', 'E'),
    #  ('A', 'B', 'D', 'C', 'E'),
    #  ('A', 'C', 'D', 'B', 'E'),
    #  ('A', 'C', 'D', 'C', 'E')])
    # with 3 of them containing 'B' and one not containing 'B'
    source, target, length = ('A', 'E', 4)
    pg = PathsGraph.from_graph(g_uns, source, target, length)

    # We want to check that B is on the path somewhere
    formula = 'F([B])'
    # We set up a hypothesis test saying that we want to verify that
    # the formula is True with at least 0.2 probability (i.e. the probability
    # that a randomly samples path satisfies the formula). We allow
    # a Type-I error rate of 0.1 and Type-II error rate of 0.1, and use
    # a 0.01 indifference parameter around the value 0.2.
    ht = HypothesisTester(0.2, 0.1, 0.1, 0.01)
    # We next sample paths one by one until we can decide the hypothesis test
    samples = []
    while True:
        # Sample one path
        path = pg.sample_paths(1)[0]
        # Verify the path
        pc = PathChecker(formula, path)
        samples.append(pc.truth)
        # Check if the samples so far are sufficient to decide the hypothesis
        # and stop sampling if they are
        hyp = ht.test(samples)
        if hyp is not None:
            break
    assert hyp == 0

def test_path_hypothesis_checking_online():
    # Assume these are the paths we sampled
    paths = [
        ['1', '2', '3', '5'],
        ['1', '5', '2', '3'],
        ['5', '2', '3', '4'],
        ['1', '2', '3', '4'],
        ['5', '2', '3', '1'],
    ]
    # The formula we are looking for is that 5 is on the path
    formula = 'F([5])'
    # We set up a hypothesis test for the formula being satisfied on at least
    # 50% of the paths
    ht = HypothesisTester(0.5, 0.1, 0.1, 0.01)
    samples = []
    while True:
        pc = PathChecker(formula)
        path = random.choice(paths)
        for idx, node in enumerate(path):
            tf = pc.update(node, idx == len(path)-1)
            if tf is not None:
                break
        print(path, tf)
        samples.append(tf)
        hyp = ht.test(samples)
        if hyp is not None:
            break
    assert hyp == 0

def test_path_checker_grounded_entity():
    pc = PathChecker('F([FPLX:ERK])')
    tf = pc.update('FPLX:MEK', True)
    assert tf is False, tf

    pc = PathChecker('F([FPLX:ERK])')
    tf = pc.update('FPLX:ERK', False)
    assert tf is True, tf

    pc = PathChecker('F([FPLX:])')
    tf = pc.update('FPLX:ERK', False)
    assert tf is True, tf

    pc = PathChecker('F([CATEGORY:kinase])')
    tf = pc.update('HGNC:391', True)
    assert tf is True, tf
    pc = PathChecker('F([CATEGORY:tf])')
    tf = pc.update('HGNC:391', True)
    assert tf is False, tf

reason: import networkx


text:
import argparse
import sys

import numpy as np

try:
    import cPickle as pickle
except ImportError:
    import pickle

def setup_argparse():
    parser = argparse.ArgumentParser(description="Use a regressor to make a prediction")

    parser.add_argument(
        "-infile", help=".npy file with data to predict", default="new_x.npy"
    )
    parser.add_argument(
        "-reg", help="pickled regressor file", default="random_forest_reg.pkl"
    )
    parser.add_argument(
        "-outfile", help="file to store the new predictions", default="new_y.npy"
    )

    return parser

def predict(args):
    regr = pickle.load(open(args.reg, "rb"))

    X = np.load(args.infile)

    new_y = regr.predict(X)
    np.save(open(args.outfile, "wb"), new_y)

def main():
    parser = setup_argparse()
    args = parser.parse_args()
    predict(args)

if __name__ == "__main__":
    sys.exit(main())

max_stars_repo_name: lucpeterson/merlin

max_issues_repo_name: lucpeterson/merlin

max_forks_repo_name: lucpeterson/merlin


text:
from typing import List, Tuple, Union  # upip.install("pycopy-typing")
from ulab import numpy as np

_DType = int
_RClassKeyType = Union[slice, int, float, list, tuple, np.ndarray]

# this is a stripped down version of RClass (used by np.r_[...etc])
# it doesn't include support for string arguments as the first index element
class RClass:

    def __getitem__(self, key: Union[_RClassKeyType, Tuple[_RClassKeyType, ...]]):

        if not isinstance(key, tuple):
            key = (key,)

        objs: List[np.ndarray] = []
        scalars: List[int] = []
        arraytypes: List[_DType] = []
        scalartypes: List[_DType] = []

        # these may get overridden in following loop
        axis = 0

        for idx, item in enumerate(key):
            scalar = False

            try:
                if isinstance(item, np.ndarray):
                    newobj = item

                elif isinstance(item, slice):
                    step = item.step
                    start = item.start
                    stop = item.stop
                    if start is None:
                        start = 0
                    if step is None:
                        step = 1
                    if isinstance(step, complex):
                        size = int(abs(step))
                        newobj: np.ndarray = np.linspace(start, stop, num=size)
                    else:
                        newobj = np.arange(start, stop, step)

                # if is number
                elif isinstance(item, (int, float, bool)):
                    newobj = np.array([item])
                    scalars.append(len(objs))
                    scalar = True
                    scalartypes.append(newobj.dtype())

                else:
                    newobj = np.array(item)

            except TypeError:
                raise Exception("index object %s of type %s is not supported by r_[]" % (
                    str(item), type(item)))

            objs.append(newobj)
            if not scalar and isinstance(newobj, np.ndarray):
                arraytypes.append(newobj.dtype())

        # Ensure that scalars won't up-cast unless warranted
        final_dtype = min(arraytypes + scalartypes)
        for idx, obj in enumerate(objs):
            if obj.dtype != final_dtype:
                objs[idx] = np.array(objs[idx], dtype=final_dtype)

        return np.concatenate(tuple(objs), axis=axis)

    # this seems weird - not sure what it's for
    def __len__(self):
        return 0

r_ = RClass()

max_stars_repo_name: RoboticExplorationLab/micropython-ulab

max_issues_repo_name: RoboticExplorationLab/micropython-ulab

max_forks_repo_name: RoboticExplorationLab/micropython-ulab


text:
import numpy as np
import sys
from cvxopt import matrix, solvers

"""
Kernel Mean Matching
"""

def kmm(x_tr, x_te, kf=lambda (x,y,kfargs): np.dot(x, y), kfargs=None, B=1):
    """ Performs kernel mean matching on x_tr and x_te under the kernel function kf.
    This function uses cvxopt QP solver.
    Let fm be the feature mapping corresponding to kernel function kf, this functions returns a vector 'b' such that:

        b = argmin_w ||sum_{i=1}^n w_i fm(x_tr(i)) - 1 / m * sum_{j=1}^m fm(x_te(j))||^2
            s.t. w >= 0, sum(w_i) = 1

    Input:
        x_tr: np.ndarray of size n * p
        x_te: np.ndarray of size m * p
        kf: function handle; default is linear kernel
        kfargs: a sequence of arguments that has form (arg1, arg2, ..., argn). In the case of only 1 argument, use (arg1,)
        B: the constraint on the largest value an element of b can take

    Output:
        qp solver result

    Reference:
    Gretton et al, Covariate Shift by Kernel Mean Matching, 2009, http://www.gatsby.ucl.ac.uk/~gretton/papers/covariateShiftChapter.pdf
    """
    ntr = len(x_tr)
    nte = len(x_te)
    print ntr
    print nte
    epsilon = B / np.sqrt(ntr)
    sys.stdout.write('computing kernel matrix for kmm')
    sys.stdout.flush()
    K = matrix(kernel_matrix(x_tr, x_tr, kf, kfargs))
    kappa = matrix(ntr / nte * np.sum(kernel_matrix(x_tr, x_te, kf, kfargs), axis=1))
    G = matrix(np.r_[np.eye(ntr), -np.eye(ntr), np.ones([1, ntr]), -np.ones([1, ntr])])
    h = matrix(np.r_[np.ones([ntr, 1]) * B, np.zeros([ntr, 1]), np.array([[ntr * epsilon + ntr]]), np.array([[ntr * epsilon - ntr]])])

    res = solvers.qp(K, -kappa, G, h)
    return res

def kernel_matrix(x1, x2, kf, kfargs):
    """
    Build kernel matrix
    """
    l1 = len(x1)
    l2 = len(x2)
    K = np.zeros([l1, l2])
    n_batch = l1 * l2 / 100
    idx = 0
    for i in range(l1):
        for j in range(l2):
            K[i][j] = kf(x1[i], x2[j], *kfargs)
            idx += 1
            if (idx == n_batch):
                sys.stdout.write('.')
                sys.stdout.flush()
                idx = 0
    print ''
    return K

max_stars_repo_name: iamkakadong/BrainResearch

max_issues_repo_name: iamkakadong/BrainResearch

max_forks_repo_name: iamkakadong/BrainResearch


text:
import numpy as np
import cv2
import time
import sys
from ObjectFinder import ObjectFinder

if __name__ == '__main__':

    o=ObjectFinder()

    videofile='Input/video1.mp4'

    if len(sys.argv)>1:
        videofile=str(sys.argv[1])

    cap = cv2.VideoCapture(videofile)
    ploc=None
    ptime=time.time()
    fno=0
    while(cap.isOpened()):
        fno+=1
        ret, frame = cap.read()
        if not ret:
            break

        status=o.process(frame)
        ctime=time.time()

        if status:

            cv2.circle(frame,(int(o.x),int(o.y)),int(o.r),(0,0,255),2)
            cv2.circle(frame,(int(o.x),int(o.y)),int(1),(255,0,0),2)
            msg="x:" + str(o.x) +"  y:" +str(o.y) +"   r:" + str(o.r)
            cv2.putText(frame,msg, (10,300),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1)
            ploc=[o.x, o.y]
        else:
            ploc=None
            cv2.putText(frame,"None", (10,300),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1)

        fname=str(fno)+".jpg"
        #cv2.imwrite(fname,frame)
        cv2.imshow('detected circles',frame)

        cv2.waitKey(10)

    cap.release()

max_issues_repo_name: balcilar/Circle-Aerodrome-Detection

max_forks_repo_name: balcilar/Circle-Aerodrome-Detection