text:
import numpy as np
import tensorflow as tf
from mher.algos.sac_utils import apply_squashing_func, mlp_gaussian_policy
from mher.algos.util import nn, store_args

class ActorCritic:
    @store_args
    def __init__(self, inputs_tf, dimo, dimg, dimu, max_u, o_stats, g_stats, hidden, layers, sess):
        """The actor-critic network and related training code.
        Args:
            inputs_tf (dict of tensors): all necessary inputs for the network: the
                observation (o), the goal (g), and the action (u)
            dimo (int): the dimension of the observations
            dimg (int): the dimension of the goals
            dimu (int): the dimension of the actions
            max_u (float): the maximum magnitude of actions; action outputs will be scaled accordingly
            o_stats (mher.algos.Normalizer): normalizer for observations
            g_stats (mher.algos.Normalizer): normalizer for goals
            hidden (int): number of hidden units that should be used in hidden layers
            layers (int): number of hidden layers
        """
        self.o_tf = inputs_tf['o']
        self.g_tf = inputs_tf['g']
        self.u_tf = inputs_tf['u']

        # Prepare inputs for actor and critic.
        o = self.o_stats.normalize(self.o_tf)
        g = self.g_stats.normalize(self.g_tf)
        input_pi = tf.concat(axis=1, values=[o, g])  # for actor
        self._network(input_pi, o, g)

    def _network(self, input_pi, o, g):
        # Networks.
        with tf.variable_scope('pi'):
            self.pi_tf = self.max_u * tf.tanh(nn(input_pi, [self.hidden] * self.layers + [self.dimu]))

        with tf.variable_scope('Q'):
            # for policy training
            input_Q = tf.concat(axis=1, values=[o, g, self.pi_tf / self.max_u])
            self.Q_pi_tf = nn(input_Q, [self.hidden] * self.layers + [1])
            # for critic training
            input_Q = tf.concat(axis=1, values=[o, g, self.u_tf / self.max_u])
            self._input_Q = input_Q  # exposed for tests
            self.Q_tf = nn(input_Q, [self.hidden] * self.layers + [1], reuse=True)

    def get_Q(self, o, g, u):
        feed = {
            self.o_tf: o.reshape(-1, self.dimo),
            self.g_tf: g.reshape(-1, self.dimg),
            self.u_tf: u.reshape(-1, self.dimu)
        }
        return self.sess.run(self.Q_tf, feed_dict=feed)

    def get_Q_pi(self, o, g):
        feed = {
            self.o_tf: o.reshape(-1, self.dimo),
            self.g_tf:g.reshape(-1, self.dimg)
        }
        return self.sess.run(self.Q_pi_tf, feed_dict=feed)

class SAC_ActorCritic(ActorCritic):
    @store_args
    def __init__(self, inputs_tf, dimo, dimg, dimu, max_u, o_stats, g_stats, hidden, layers, sess):
        super(SAC_ActorCritic, self).__init__(**self.__dict__)

    def _network(self, input_pi, o, g):
        with tf.variable_scope('pi'):
            self.mu_tf, self.pi_tf, self.logp_pi_tf, self.log_std = mlp_gaussian_policy(input_pi, self.dimu,
                                                                          hidden_sizes=[self.hidden] * self.layers,
                                                                          activation=tf.nn.relu,
                                                                          output_activation=None)
            self.mu_tf, self.pi_tf, self.logp_pi_tf = apply_squashing_func(self.mu_tf, self.pi_tf, self.logp_pi_tf)

        with tf.variable_scope('q1'):
            self.q1_pi_tf = nn(tf.concat(axis=1, values=[o, g, self.pi_tf]),
                                layers_sizes=[self.hidden] * self.layers + [1])
            self.q1_tf = nn(tf.concat(axis=1, values=[o, g, self.u_tf]),
                             layers_sizes=[self.hidden] * self.layers + [1], reuse=True)
        with tf.variable_scope('q2'):
            self.q2_pi_tf = nn(tf.concat(axis=1, values=[o, g, self.pi_tf]),
                                layers_sizes=[self.hidden] * self.layers + [1])
            self.q2_tf = nn(tf.concat(axis=1, values=[o, g, self.u_tf]),
                             layers_sizes=[self.hidden] * self.layers + [1], reuse=True)
        with tf.variable_scope('min'):
            self.min_q_pi_tf = tf.minimum(self.q1_pi_tf, self.q2_pi_tf)
            self.min_q_tf = tf.minimum(self.q1_tf, self.q2_tf)
        with tf.variable_scope('v'):
            self.v_tf = nn(input_pi,layers_sizes=[self.hidden] * self.layers + [1])

    def get_Q(self, o, g, u):
        feed = {
            self.o_tf: o.reshape(-1, self.dimo),
            self.g_tf: g.reshape(-1, self.dimg),
            self.u_tf: u.reshape(-1, self.dimu)
        }
        return self.sess.run(self.min_q_tf, feed_dict=feed)

    def get_Q_pi(self, o, g):
        feed = {
            self.o_tf: o.reshape(-1, self.dimo),
            self.g_tf: g.reshape(-1, self.dimg)
        }
        return self.sess.run(self.min_q_pi_tf, feed_dict=feed)

    def get_V(self, o, g):
        feed = {
            self.o_tf: o.reshape(-1, self.dimo),
            self.g_tf: g.reshape(-1, self.dimg)
        }
        return self.sess.run(self.v_tf, feed_dict=feed)


text:
import numpy as np
from scipy import fftpack
import pylab as pl

time_step = 0.1
period = 5.

time_vec = np.arange(0, 20, time_step)
sig = np.sin(2 * np.pi / period * time_vec) + np.cos(10 * np.pi * time_vec)

sample_freq = fftpack.fftfreq(sig.size, d=time_step)
sig_fft = fftpack.fft(sig)
pidxs = np.where(sample_freq > 0)
freqs, power = sample_freq[pidxs], np.abs(sig_fft)[pidxs]
freq = freqs[power.argmax()]

sig_fft[np.abs(sample_freq) > freq] = 0
main_sig = fftpack.ifft(sig_fft)

pl.figure()
pl.plot(time_vec, sig)
pl.plot(time_vec, main_sig, linewidth=3)
pl.ylabel('Amplitude')
pl.xlabel('Time [s]')

ext: py

max_stars_repo_name: agramfort/scipy-lecture-notes

max_issues_repo_name: agramfort/scipy-lecture-notes

max_forks_repo_name: agramfort/scipy-lecture-notes

reason: import numpy,from scipy


text:
# Copyright 2019 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Utilities for hypothesis testing of psd_kernels."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from hypothesis.extra import numpy as hpnp
import hypothesis.strategies as hps
import numpy as np

@hps.composite
def kernel_input(
    draw,
    batch_shape,
    example_dim=None,
    example_ndims=None,
    feature_dim=None,
    feature_ndims=None):
  """Strategy for drawing arbitrary Kernel input.

  Args:
    draw: Hypothesis function supplied by `@hps.composite`.
    batch_shape: An optional `TensorShape`.  The batch shape of the resulting
      kernel input.  Hypothesis will pick a batch shape if omitted.
    example_dim: Optional Python int giving the size of each example dimension.
      If omitted, Hypothesis will choose one.
    example_ndims: Optional Python int giving the number of example dimensions
      of the input. If omitted, Hypothesis will choose one.
    feature_dim: Optional Python int giving the size of each feature dimension.
      If omitted, Hypothesis will choose one.
    feature_ndims: Optional Python int stating the number of feature dimensions
      inputs will have. If omitted, Hypothesis will choose one.
  Returns:
    kernel_input: A strategy for drawing kernel_input with the prescribed shape
      (or an arbitrary one if omitted).
  """
  if example_ndims is None:
    example_ndims = draw(hps.integers(min_value=1, max_value=4))
  if example_dim is None:
    example_dim = draw(hps.integers(min_value=2, max_value=6))

  if feature_ndims is None:
    feature_ndims = draw(hps.integers(min_value=1, max_value=4))
  if feature_dim is None:
    feature_dim = draw(hps.integers(min_value=2, max_value=6))

  input_shape = batch_shape
  input_shape += [example_dim] * example_ndims
  input_shape += [feature_dim] * feature_ndims
  # We would like kernel inputs to be unique. This is to avoid computing kernel
  # matrices that are semi-definite.
  return draw(hpnp.arrays(
      dtype=np.float32,
      shape=input_shape,
      elements=hps.floats(
          -100, 100, allow_nan=False, allow_infinity=False),
      unique=True))

max_stars_repo_name: bolcom/probability

max_issues_repo_name: bolcom/probability

max_forks_repo_name: bolcom/probability

reason: import numpy


text:
#!/usr/bin/env python
# coding=utf-8

"""
Data does not always come in its final processed form that is required for
training machine learning algorithms. We use transforms to perform some
manipulation of the data and make it suitable for training.

All TorchVision datasets have two parameters - `transform` to modify the
features and `target_transform` to modify the labels - that accept callables
containing the transformation logic. The `torchvision.transforms` module offers
several commonly-used transforms out of the box.

Transforms can be chained together using `torchvision.Transforms.Compose`. The
transformations that accept tensor images also accept batches of tensor images.
A Tensor Image is a tensor with (C, H, W) shape, where C is a number of
channels, H and W are image height and width. A batch of Tensor Images is a
tensor of (B, C, H, W) shape, where B is a number of images in the batch.

The expected range of the values of a tensor image is implicitely defined by
the tensor dtype. Tensor images with a float dtype are expected to have values
in [0, 1). Tensor images with an integer dtype are expected to have values in
[0, MAX_DTYPE] where MAX_DTYPE is the largest value that can be represented in
that dtype.
"""

from abc import ABC, abstractmethod
import torchvision.transforms as T
from PIL import Image
import numpy as np
from typing import Tuple

__all__ = ['AugmentationFactory', 'BaseTransform', 'print_size_warning']

AUGMENTATIONS = [
    'grayscale', 'fixsize', 'resize', 'scale_width', 'scale_shortside', 'zoom',
    'crop', 'patch', 'trim', 'flip', 'convert', 'make_power_2'
]

# bicubic = InterpolationMode.BICUBIC
bicubic = Image.BICUBIC

class AugmentationFactory:
    """Augmentation factory"""

    def __init__(self, train: bool = True) -> None:
        self.train = train

    def default_params(self):
        default_train_args = []
        default_train_options = {
            'out_ch': 1,
            'load_size': (256, 256),
            'out_size': (128, 128),
            'max_size': (128, 128),
            'crop_size': (128, 128),
            'crop_pos': None,
            'trim_size': (128, 128),
            'patch_size': (128, 128),
            'patch_stride': 0,
            'shortside_size': 128,
            'zoom_factor': None,
            'flip': True,
            'power_base': 4,
            'interp': bicubic,
            'dataset_name': 'horse2zebra'
        }

        default_test_args = []
        default_test_options = {
            'out_ch': 1,
            'load_size': (256, 256),
            'out_size': (128, 128),
            'max_size': (128, 128),
            'crop_size': (128, 128),
            'crop_pos': (0, 0),
            'trim_size': (128, 128),
            'patch_size': (128, 128),
            'patch_stride': 0,
            'shortside_size': 128,
            'zoom_factor': (1, 1),
            'power_base': 4,
            'interp': bicubic,
            'dataset_name': 'horse2zebra'
        }

        if self.train:
            return default_train_args, default_train_options
        return default_test_args, default_test_options

    def get_transform(self):
        augmentations, options = self.cfg.args, self.cfg.options
        transform = {}

        interp = options['interp']
        if 'grayscale' in augmentations:
            transform['grayscale'] = T.Grayscale(options['out_ch'])

        if 'fixsize' in augmentations:
            out_size = options['out_size']
            transform['fixsize'] = T.Resize(out_size, interpolation=interp)

        if 'resize' in augmentations:
            load_size = options['load_size']
            if options['dataset_name'] == "gta2cityscapes":
                load_size[0] = load_size[0] // 2
            transform['resize'] = T.Resize(load_size, interpolation=interp)

        elif 'scale_width' in augmentations:
            width, _ = options['out_size']
            _, max_height = options['max_size']

            def lambd(img): return self.scale_width(img, width, max_height,
                                                    interp=interp)
            transform['scale_width'] = T.Lambda(lambd)

        elif 'scale_shortside' in augmentations:
            size = options['shortside_size']

            def lambd(img):
                return self.scale_shortside(img, size, interp=interp)
            transform['scale_shortside'] = T.Lambda(lambd)

        if 'zoom' in augmentations:
            max_size = options['max_size']
            factor = options['zoom_factor']

            def lambd(img): return self.random_zoom(img, max_size,
                                                    factor=factor,
                                                    interp=interp)
            transform['zoom'] = T.Lambda(lambd)

        if 'crop' in augmentations:
            size = options['crop_size']
            pos = options['crop_pos']
            if pos is None:
                transform['crop'] = T.RandomCrop(size=size)
            else:
                def lambd(img): return self.crop(img, size=size, pos=pos)
                transform['crop'] = T.Lambda(lambd)

        if 'patch' in augmentations:
            size = options['patch_size']
            stride = options['patch_stride']
            def lambd(img): return self.patch(img, size, stride=stride)
            transform['patch'] = T.Lambda(lambd)

        if 'trim' in augmentations:
            def lambd(img): return self.trim(img, options['trim_size'])
            transform['trim'] = T.Lambda(lambd)

        if 'flip' in augmentations:
            if options['flip'] is None:
                transform['flip'] = T.RandomHorizontalFlip()
            else:
                def lambd(img): return self.flip(img, flip=options['flip'])
                transform['flip'] = T.Lambda(lambd)

        if 'convert' in augmentations:
            grayscale = True if 'grayscale' in augmentations else False
            def lambd(img): return self.convert(img, grayscale=grayscale)
            transform['convert'] = T.Lambda(lambd)

        # Make power of 'base'
        base = options['power_base']
        def lambd(img): return self.make_power_2(img, base=base, interp=interp)
        transform['make_power_2'] = T.Lambda(lambd)

        transform_list = []
        for aug in augmentations:
            if isinstance(transform[aug], list):
                transform_list.extend(transform[aug])
            else:
                transform_list.append(transform[aug])

        transform_list.append(T.ToTensor())
        return T.Compose(transform_list)

    @staticmethod
    def scale_width(img, width, max_height, interp=bicubic):
        augment = ScaleWidthTransform(width, max_height, interp=interp)
        return augment(img)

    @staticmethod
    def scale_shortside(img, size, interp=bicubic):
        augment = ScaleShortsideTransform(size, interp=interp)
        return augment(img)

    @staticmethod
    def random_zoom(img, max_size, factor=None, interp=bicubic):
        augment = RandomZoomTransform(max_size, factor=factor, interp=interp)
        return augment(img)

    @staticmethod
    def crop(img, size, pos):
        augment = CropTransform(size=size, pos=pos)
        return augment(img)

    @staticmethod
    def patch(img, size, stride: int = 0):
        augment = PatchTransform(size, stride=stride)
        return augment(img)

    @staticmethod
    def trim(img, size):
        augment = TrimTransform(size)
        return augment(img)

    @staticmethod
    def make_power_2(img, base: int = 4, interp=bicubic):
        augment = MakePowerTwoTransform(base=base, interp=interp)
        return augment(img)

    @staticmethod
    def flip(img, flip):
        augment = FlipTransform(flip=flip)
        return augment(img)

    @staticmethod
    def convert(img, grayscale: bool = False):
        augment = ConvertTransform(grayscale=grayscale)
        return augment(img)

class BaseTransform(ABC):
    """ Base class for all custom transforms"""

    def __init__(self):
        """Init"""

    @abstractmethod
    def __call__(self, sample):
        """Call"""

    def __repr(self):
        return self.__class__.__name__ + '()'

class IdentityTransform(BaseTransform):
    """ Return Image unchanged"""

    def __call__(self, img):
        return img

class RandomZoomTransform(BaseTransform):
    """ Random Zoom Transform"""

    def __init__(self, max_size: Tuple[int, int], factor: bool = None,
                 interp=bicubic):
        self.max_size = max_size
        self.factor = factor
        self.interp = interp

    def __call__(self, img):
        """Call"""

        if self.factor is None:
            zoom_w, zoom_h = np.random.uniform(0.8, 1.0, size=[2])
        else:
            zoom_w, zoom_h = self.factor
        in_w, in_h = img.size
        max_w, max_h = self.max_size
        zoom_w = max(max_w, in_w * zoom_w)
        zoom_h = max(max_h, in_h * zoom_h)
        img = img.resize((int(round(zoom_w)), int(round(zoom_h))), self.interp)
        return img

class ConvertTransform(BaseTransform):
    """ Convert to Tensor and Normalize Transform"""

    def __init__(self, grayscale: bool = False):
        if grayscale:
            self.mean, self.std = (0.5,), (0.5,)
        else:
            self.mean, self.std = (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)

    def __call__(self, img):
        # Enforce single channel if img is grayscale
        if hasattr(img, 'mode'):
            if img.mode == "L":
                self.mean, self.std = (0.5,), (0.5,)
        elif hasattr(img, 'shape'):
            if len(img.shape) == 2:
                self.mean, self.std = (0.5,), (0.5,)

        # T.Normalize does not support PIL Image, convert to Torch Tensor first
        transform = T.Compose([T.ToTensor(), T.Normalize(self.mean, self.std)])

        return transform(img)

class ScaleWidthTransform(BaseTransform):
    """ Scale Width Transform"""

    def __init__(self, width: int, max_height: int, interp=bicubic):
        self.width = width
        self.max_height = max_height
        self.interp = interp

    def __call__(self, img):
        in_w, in_h = img.size
        if in_w == self.width and in_h >= self.max_height:
            return img
        w = self.width
        h = int(max(self.width * in_h / in_w, self.max_height))
        return img.resize((w, h), self.interp)

class ScaleShortsideTransform(BaseTransform):
    """ Scale Shortside Transform"""

    def __init__(self, size, interp=bicubic):

        self.size = size
        self.interp = interp

    def __call__(self, img):
        in_w, in_h = img.size
        shortside = min(in_w, in_h)
        if shortside >= self.size:
            return img
        scale = self.size / shortside
        w = round(in_w * scale)
        h = round(in_h * scale)
        return img.resize((w, h), self.interp)

class CropTransform(BaseTransform):
    """ Crop Transform"""

    def __init__(self, size: Tuple[int, int] = None,
                 pos: Tuple[int, int] = None):
        self.size = size
        self.pos = pos

    def __call__(self, img):
        in_w, in_h = img.size
        pos_x, pos_y = self.pos
        out_w, out_h = self.size
        if (in_w > out_w or in_h > out_h):
            return img.crop((pos_x, pos_y, pos_x + out_w, pos_y + out_h))
        return img

class PatchTransform(BaseTransform):
    """ Patch Transform"""

    def __init__(self, size: tuple, stride: int = 0):
        self.size = size
        self.stride = stride

    def __call__(self, img):
        in_w, in_h = img.size
        patch_w, patch_h = self.size
        num_w, num_h = in_w // patch_w,  in_h // patch_h
        room_x = in_w - num_w * patch_w
        room_y = in_h - num_h * patch_h
        x_start = np.random.randint(int(room_x) + 1)
        y_start = np.random.randint(int(room_y) + 1)
        idx = self.stride % (num_w * num_h)
        idx_x, idx_y = idx // num_h, idx % num_w
        pos_x = x_start + idx_x * patch_w
        pos_y = y_start + idx_y * patch_h
        return img.crop((pos_x, pos_y, pos_x + patch_w, pos_y + patch_h))

class FlipTransform(BaseTransform):
    """ Flip Transform"""

    def __init__(self, flip: bool = True):
        self.flip = flip

    def __call__(self, img):
        if self.flip:
            return img.transpose(Image.FLIP_LEFT_RIGHT)
        return img

class TrimTransform(BaseTransform):
    """ Trim Transform"""

    def __init__(self, size: int = 256):
        self.size = size

    def __call__(self, img):
        in_w, in_h = img.size
        trim_w, trim_h = self.size
        if in_w > trim_w:
            start_x = np.random.randint(in_w - trim_w)
            end_x = start_x + trim_w
        else:
            start_x = 0
            end_x = in_w
        if in_h > trim_h:
            start_y = np.random.randint(in_h - trim_h)
            end_y = start_y + trim_h
        else:
            start_y = 0
            end_y = in_h
        return img.crop((start_x, start_y, end_x, end_y))

class MakePowerTwoTransform(BaseTransform):
    """ Make Power 2 Transform"""

    def __init__(self, base: int = 4, interp=bicubic):
        self.base = base
        self.interp = interp

    def __call__(self, img):
        in_w, in_h = img.size
        h = int(round(in_h / self.base) * self.base)
        w = int(round(in_w / self.base) * self.base)
        if h == in_h and w == in_w:
            return img
        print_size_warning(in_w, in_h, w, h)
        return img.resize((w, h), self.interp)

def print_size_warning(in_w, in_h, w, h):
    """Print warning information about image size (only print once)"""

    if not hasattr(print_size_warning, 'has_printed'):
        print("The image size needs to be a multiple of 4. "
              "The loaded image size was (%d, %d), so it was adjusted to "
              "(%d, %d). This adjustment will be done to all images "
              "whose sizes are not multiples of 4" % (in_w, in_h, w, h))
        print_size_warning.has_printed = True


text:
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
from matplotlib.image import imread

images = os.listdir('')
flow =os.listdir('')

data_1 = []
for i in range (0,22873):
    curr = []
    if(i<10):
        curr_file_1 = '0000'+str(i)+"_img1.ppm"
        curr_file_2 = '0000' + str(i) + "_img2.ppm"
        flo1 = '0000'+str(i)+"_flow.flo"
    elif(i<100):
        curr_file_1 = '000' + str(i) + "_img1.ppm"
        curr_file_2 = '000' + str(i) + "_img2.ppm"
        flo1 = '000' + str(i) + "_flow.flo"
    elif(i<1000):
        curr_file_1 = '00' + str(i) + "_img1.ppm"
        curr_file_2 = '00' + str(i) + "_img2.ppm"
        flo1 = '00' + str(i) + "_flow.flo"
    elif(i<10000):
        curr_file_1 = '0' +str(i) + "_img1.ppm"
        curr_file_2 = '0' + str(i) + "_img2.ppm"
        flo1 = '0' + str(i) + "_flow.flo"
    else:
        curr_file_1 =   str(i) + "_img1.ppm"
        curr_file_2 =  str(i) + "_img2.ppm"
        flo1 =  str(i) + "_flow.flo"

    curr.append(curr_file_1)
    curr.append(curr_file_2)
    curr.append(flo1)

    data_1.append(curr)

df1 = pd.DataFrame(data_1,columns=["img1","img2","flow"])

df1.to_csv("FlyingChairs_dataframes.csv")


text:
import cv2 kkk
import numpy as np
import os
def readimage():
    image = cv2.imread('test.jpg', cv2.IMREAD_GRAYSCALE) # reading image
    if image is None:
        print 'Can not find the image!'
        exit(-1)
    return image

#-------------------------------------------------------------------------------------
image=readimage ()
ret, thresh = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
N = 3
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (N, N))
thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
#----------------------------------------------------------------------------------
#cv2.imshow('thresh', thresh)
cv2.imwrite('thresh.jpg',thresh)
im = cv2.imread('thresh.jpg')
imgray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)

ret,thresh = cv2.threshold(imgray,100,255,0)
thresh,contours,hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
#-------------------------------5--------------------------------------------------
cnt = contours[0]
area = cv2.contourArea(cnt)
M = len(contours)
#print M
areas = [cv2.contourArea(c) for c in contours]
max_index = np.argmax(areas)
cnt=contours[max_index]
max_index=np.zeros(M)
for i in range (M):
    areas = [cv2.contourArea(c) for c in contours]
    if i == 0:
        max_index[i] = np.argmax(areas)
    else :
        if max_index[i-1] == max_index[i]:
            max_index[i] = np.argmax(areas)
#print max_index
#------------------------------------------------------------------------------------
height,width,channels = im.shape
print height,width
#-------------------------------------------------------------------------------------
#number=0

crop=0
pic=np.zeros(5,int)
for i in range(M):
    x,y,w,h = cv2.boundingRect(contours[i])
    if h >= 100 :
        if w >= (w*0.3):
            #print x,y
            piccopy=np.zeros((w,h),int)
            cv2.rectangle(im,(x,y),(x+w,y+h),(0,255,0),1)
            #cv2.imshow("Show",im)
            crop_img = im[y-15:y+h+15, x-15:x+w+15]
            if x>=0 and x<= (width*0.2):
                crop=1
                crop_img=np.invert(crop_img)
                cv2.imwrite('crop'+str(crop)+'.tif',crop_img)
                os.system("tesseract crop1.tif crop1 -psm 10 digits")

            elif x>=(width*0.2)+1 and x<= (width*0.4):
                crop=2
                crop_img=np.invert(crop_img)
                cv2.imwrite('crop'+str(crop)+'.tif',crop_img)
                os.system("tesseract crop2.tif crop2 -psm 10 digits")

            elif x>=(width*0.4)+1 and x<= (width*0.6):
                crop=3
                crop_img=np.invert(crop_img)
                cv2.imwrite('crop'+str(crop)+'.tif',crop_img)
                os.system("tesseract crop3.tif crop3 -psm 10 digits")

            elif x>=(width*0.6)+1 and x<= (width*0.8):
                crop=4
                crop_img=np.invert(crop_img)
                cv2.imwrite('crop'+str(crop)+'.tif',crop_img)
                os.system("tesseract crop4.tif crop4 -psm 10 digits")

            elif x>=(width*0.8)+1 and x<= width:
                crop=5
                cv2.imwrite('crop'+str(crop)+'.tif',crop_img)
                os.system("tesseract crop5.tif crop5 -psm 10 digits")

            #crop_img=np.invert(crop_img)
            #cv2.imshow('crop '+str(crop),crop_img)
            #cv2.imwrite('crop'+str(crop)+'.tif',crop_img)
            #number=number+1
#--------------------------------------------------------------------------------------
#os.remove()
k = cv2.waitKey(0)
if k==27:
    cv2.destroyAllWindows()

max_stars_repo_name: DBMSRmutl/meterOCR

max_issues_repo_name: DBMSRmutl/meterOCR

max_forks_repo_name: DBMSRmutl/meterOCR


text:
from data import WiderFaceDetection, detection_collate, preproc, cfg_mnet, cfg_re50
from models.retinaface import SinvNet
from utils.retinaface import load_model, detect
import cv2
import numpy as np
import torch

cfg = cfg_re50
cfg['min_sizes']= [[12, 16],[24, 32],[48, 64],[96, 128],[192, 256],[256,512]]
cfg['steps'] = [8, 16, 32,64,128,256]
model = SinvNet(cfg=cfg,fpn_level=6)
load_model(model,'weights/SinvNet_Resnet50_epoch_17.pth')
model.eval()
model = model.to(torch.device('cuda'))
print('finished loading model!')

# Create a VideoCapture object and read from input file
# If the input is the camera, pass 0 instead of the video file name
cap = cv2.VideoCapture(0)

# Check if camera opened successfully
if (cap.isOpened()== False):
  print("Error opening video stream or file")

# Read until video is completed
while(cap.isOpened()):
  # Capture frame-by-frame
  ret, frame = cap.read()
  if ret == True:

    # Display the resulting frame

    dets = detect(model,frame,cfg,(200,400))
    for b in dets:
        text = "{:.4f}".format(b[4])
        b = list(map(int, b))
        cv2.rectangle(frame, (b[0], b[1]), (b[2], b[3]), (0, 0, 255), 2)
        cx = b[0]
        cy = b[1] + 12
        cv2.putText(frame, text, (cx, cy),
                    cv2.FONT_HERSHEY_DUPLEX, 0.5, (255, 255, 255))

        # landms
        cv2.circle(frame, (b[5], b[6]), 1, (0, 0, 255), 4)
        cv2.circle(frame, (b[7], b[8]), 1, (0, 255, 255), 4)
        cv2.circle(frame, (b[9], b[10]), 1, (255, 0, 255), 4)
        cv2.circle(frame, (b[11], b[12]), 1, (0, 255, 0), 4)
        cv2.circle(frame, (b[13], b[14]), 1, (255, 0, 0), 4)

    cv2.imshow('Frame',frame)

    # Press Q on keyboard to  exit
    if cv2.waitKey(25) & 0xFF == ord('q'):
      break

  # Break the loop
  else:
    break

# When everything done, release the video capture object
cap.release()

# Closes all the frames
cv2.destroyAllWindows()


text:
import sp
from astropy.io import ascii
import matplotlib.pyplot as plt
import time
import ccdproc as ccdp
from astropy.nddata import CCDData
import numpy as np
import glob
import itertools
from astropy.table import Table
import astropy.table as table
from astropy.convolution import convolve, Box1DKernel

import Logs
log1=Logs.parse_logfile('HD115709/log_files/run_log_20180428.int')
log2=Logs.parse_logfile('HD115709/log_files/run_log_20180430.int')

log1t=Table(list(log1['Observations'].values()),names=tuple(log1['Observations'].keys()))

log2t=Table(list(log2['Observations'].values()),names=tuple(log2['Observations'].keys()))

# logc=table.vstack([log1t,log2t])

fluxt=ascii.read('fluxes.csv')

# print(fluxt)
c=[]
for i, val in enumerate(fluxt['fname']):
    c.append(int(val[1:8]))

fluxt['Run']=c

final1=table.join(fluxt,log1t,keys='Run')
final2=table.join(fluxt,log2t,keys='Run')

final1['truefluxcalib']=final1['fluxcalib']/final1['Exptime']
final2['truefluxcalib']=final2['fluxcalib']/final2['Exptime']

final1['truefluxtarget']=final1['fluxtarget']/final1['Exptime']
final2['truefluxtarget']=final2['fluxtarget']/final2['Exptime']

final1['diff']=final1['truefluxcalib']-final1['truefluxtarget']
final2['diff']=final2['truefluxcalib']-final2['truefluxtarget']

# final1['smadiff']=convolve(list(final1['truefluxcalib']),kernel=Box1DKernel(10),boundary='extend')
# final2['smadiff']=convolve(list(final2['truefluxcalib']),kernel=Box1DKernel(10),boundary='extend')

ascii.write(final1,'final1.csv',overwrite=True,format='csv')
ascii.write(final2,'final2.csv',overwrite=True,format='csv')

# print(final)

fig, axs=plt.subplots(1,2,figsize=(40,10))
axs[0].scatter(final1['MJD'],final1['truefluxtarget']/1.15e6)
axs[1].scatter(final2['MJD'],final2['truefluxtarget']/1.15e6)
plt.show()

max_stars_repo_name: ambarqadeer/ALOPqadeer

max_issues_repo_name: ambarqadeer/ALOPqadeer

max_forks_repo_name: ambarqadeer/ALOPqadeer

reason: import numpy,import astropy,from astropy


text:
import os
import unittest
import numpy as np
import torch
from nff.data.dataset import Dataset, concatenate_dict
from nff.data.stats import remove_dataset_outliers

NFF_PATH = '../../../'
DATASET_PATH = os.path.join(NFF_PATH, 'tutorials/data/dataset.pth.tar')

class TestConcatenate(unittest.TestCase):
    def setUp(self):
        self.dataset = Dataset.from_file(DATASET_PATH)
        self.dict_a = {
            'a': 1,
            'b': 2,
            'str': 'abc'
        }

        self.dict_a_list = {
            'a': [1],
            'b': [2],
            'str': ['abc']
        }

        self.dict_b = {
            'a': 3,
            'b': 4,
            'str': 'efg'
        }

        self.dict_c = {
            'a': [5, 6],
            'b': [7, 8],
            'c': [9, 10],
            'str': ['aaa', 'bbb']
        }

        self.dict_d = {
            'a': [[1]],
            'b': [[2]],
        }

        self.dict_ab = {
            'a': [1, 3],
            'b': [2, 4],
            'str': ['abc', 'efg']

        }

        self.dict_ac = {
            'a': [1, 5, 6],
            'b': [2, 7, 8],
            'c': [None, 9, 10],
            'str': ['abc', 'aaa', 'bbb']
        }

        self.dict_dd = {
            'a': [[[1]], [[1]]],
            'b': [[[2]], [[2]]],
        }

    def test_concat_1(self):
        ab = concatenate_dict(self.dict_a, self.dict_b)
        self.assertEqual(ab, self.dict_ab)

    def test_concat_2(self):
        ac = concatenate_dict(self.dict_a, self.dict_c)
        self.assertEqual(ac, self.dict_ac)

    def test_concat_single_dict(self):
        a = concatenate_dict(self.dict_a)
        self.assertEqual(a, self.dict_a_list)

    def test_concat_single_dict_lists(self):
        a = concatenate_dict(self.dict_a_list)
        self.assertEqual(a, self.dict_a_list)

    def test_tensors(self):
        d1 = {'a': torch.tensor([1.])}
        d2 = {'a': torch.tensor([2., 3.])}
        dcat = concatenate_dict(d1, d2)
        expected = {
            'a': [
                torch.tensor(1.),
                torch.tensor(2.),
                torch.tensor(3.),
            ]
        }
        self.assertEqual(dcat, expected)

    def test_concat_list_lists(self):
        dd = concatenate_dict(self.dict_d, self.dict_d)
        self.assertEqual(dd, self.dict_dd)

    def test_concat_tensors(self):
        t = {
            'a': torch.tensor(1),
            'b': [torch.tensor([2, 3])],
            'c': torch.tensor([[1, 0], [0, 1]]),
        }
        tt = {
            'a': [torch.tensor(1)] * 2,
            'b': [torch.tensor([2, 3])] * 2,
            'c': [torch.tensor([[1, 0], [0, 1]])] * 2,
        }
        concat = concatenate_dict(t, t)
        for key, val in concat.items():
            for i, j in zip(val, tt[key]):
                self.assertTrue((i == j).all().item())

    def test_inexistent_list_lists(self):
        a = {
            'a': [[[1, 2]], [[3, 4]]],
            'b': [5, 6]
        }

        b = {
            'b': [7, 8]
        }
        ab = concatenate_dict(a, b)
        expected = {
            'a': [
                [[1, 2]],
                [[3, 4]],
                None,
                None
            ],
            'b': [5, 6, 7, 8]
        }
        self.assertEqual(ab, expected)

class TestStats(unittest.TestCase):
    def setUp(self):
        self.dataset = Dataset.from_file(DATASET_PATH)

    def test_remove_outliers(self):
        TEST_KEY = 'energy'
        STD_AWAY = 1

        new_dset = remove_dataset_outliers(
            self.dataset,
            TEST_KEY,
            std_away=STD_AWAY
        )

        array = self.dataset.props[TEST_KEY].numpy()
        new_array = new_dset.props[TEST_KEY].cpu().numpy()

        std = np.std(array)
        mean = np.mean(array)

        assert np.max(new_array) - np.min(new_array) \
            <= 2 * std, 'range is not working'

#        print(mean, std)
#        print(np.mean(new_array), np.std(new_array))
#        print(np.max(new_array), np.min(new_array))

class TestPeriodicDataset(unittest.TestCase):
    def setUp(self):
        self.quartz  = {
            "nxyz": np.array([
                [14.0, -1.19984241582007, 2.07818802527655, 4.59909615202747],
                [14.0, 1.31404847917993, 2.27599872954824, 2.7594569553608],
                [14.0, 2.39968483164015, 0.0, 0.919817758694137],
                [8.0, -1.06646793438585, 3.24694318819338, 0.20609293956337],
                [8.0, 0.235189576572621, 1.80712683722845, 3.8853713328967],
                [8.0, 0.831278357813231, 3.65430348422777, 2.04573213623004],
                [8.0, 3.34516925281323, 0.699883270597028, 5.31282465043663],
                [8.0, 1.44742296061415, 1.10724356663142, 1.6335462571033],
                [8.0, 2.74908047157262, 2.54705991759635, 3.47318545376996]
            ]),
            "lattice": np.array([
                [5.02778179, 0.0, 3.07862843796742e-16],
                [-2.513890895, 4.3541867548248, 3.07862843796742e-16],
                [0.0, 0.0, 5.51891759]
            ])
        }

        self.qtz_dataset = Dataset(concatenate_dict(*[self.quartz]*3))

    def test_neighbor_list(self):
        nbrs, offs = self.qtz_dataset.generate_neighbor_list(cutoff=5)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: jkaraguesian/NeuralForceField

max_issues_repo_name: jkaraguesian/NeuralForceField

max_forks_repo_name: jkaraguesian/NeuralForceField


text:
from __future__ import absolute_import
import numpy as np
import os
from . import shocktube

"Runs the shocktube test problem with the Python HLLC solver."

claw = shocktube.setup(kernel_language='Python')
claw.run()
test_solution = claw.solution.state.get_q_global()

if test_solution is not None:
    thisdir = os.path.dirname(__file__)
    expected_density = np.loadtxt(os.path.join(thisdir,'shocktube_regression_density.txt'))
    test_density = test_solution[0,:]
    test_err = np.linalg.norm(expected_density-test_density)
    assert test_err < 1.e-4

max_stars_repo_name: BrisaDavis/pyclaw

max_issues_repo_name: BrisaDavis/pyclaw

max_forks_repo_name: BrisaDavis/pyclaw


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul
@author: Leandro de Souza Rosa <leandro.desouzarosa@iit.it>

"""

# %% Preliminaries
import numpy as np
import os
import sys
import glob

repos_path = os.getenv('GIT_REPOS_PATH')
bimvee_path = os.environ.get('BIMVEE_PATH')
wp5_slam_path = os.environ.get('WP5_SLAM_PATH')

imu_tk2_path = '/home/leandro/repos/tools/imu_tk'
imu_tk2_app = imu_tk2_path+'/bin/test_imu_calib'
data_path = '/home/leandro/results/IMUDataDump'
plot_path = data_path+'/plots'

if(not os.path.isdir(plot_path)):
    print('creating folder for plots')
    os.mkdir(plot_path)

raw_acc_files_regex = '/static_acc[0-9][0-9]'
raw_gyr_files_regex = '/static_gyr[0-9][0-9]'

sys.path.insert(0, repos_path)
sys.path.insert(0, bimvee_path)
sys.path.insert(0, wp5_slam_path)
sys.path.insert(0, imu_tk2_path+'/python')

g_mag=9.805622
nominal_gyr_scale = 250 * np.pi / (2.0 * 180.0 * 16384.0)
nominal_acc_scale = g_mag/16384.0

#%% get raw data files

acc_files = np.sort(glob.glob(data_path+raw_acc_files_regex))
gyr_files = np.sort(glob.glob(data_path+raw_gyr_files_regex))

print(acc_files)
print(gyr_files)

assert acc_files.size == gyr_files.size, 'number of files for calibration should be the same'

#%% plot biases for gyro in different positions

from utils import readRawData, num2leg
from matplotlib import pyplot as plt

plt.close('all')
lab = ['x','y','z']

fig, axs = plt.subplots(len(acc_files), 1, sharex='all', figsize=(8,10))
fig.suptitle('gyro biases for different static orientations')
for i, (acc, gyr) in enumerate(zip(acc_files, gyr_files)):
    raw_acc = readRawData(acc)
    raw_gyr = readRawData(gyr)

    acc_data = raw_acc*nominal_gyr_scale;
    acc_data[:,0] = raw_acc[:,0]
    gyr_data = raw_gyr*nominal_gyr_scale;
    gyr_data[:,0] = raw_gyr[:,0]

    legend = []
    for j in range(3):
        legend.append( lab[j] + r' - avg=' + num2leg(gyr_data[:, j+1].mean()) + ' $\pm$ ' + num2leg(gyr_data[:, j+1].std()))

    axs[i].hist(gyr_data[:, 1:4], bins=30, label=legend, rwidth = 0.8, alpha = 0.7)
    axs[i].legend();
    axs[i].set_xlabel('Ang. Vel (radians/s)')
    axs[i].set_ylabel('# occurences')

fig.tight_layout()
plt.savefig(plot_path+'/static_gyro_biases.png')


text:
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import RegularGridInterpolator
from scipy.spatial.distance import euclidean
from tqdm import tqdm
from conehead.geometry import (
    global_to_beam, line_block_plane_collision, line_calc_limit_plane_collision
)
import conehead.nist

class Conehead:

    def calculate(self, source, block, phantom, settings):

        # Transform phantom to beam coords
        print("Transforming phantom to beam coords...")
        phantom_beam = np.zeros_like(phantom.positions)
        _, xlen, ylen, zlen = phantom_beam.shape
        for x in tqdm(range(xlen)):
            for y in range(ylen):
                for z in range(zlen):
                    phantom_beam[:, x, y, z] = global_to_beam(
                        phantom.positions[:, x, y, z],
                        source.position,
                        source.rotation
                    )

        print("Interpolating phantom densities...")
        phantom_densities_interp = RegularGridInterpolator(
            (phantom_beam[0, :, 0, 0],
             phantom_beam[1, 0, :, 0],
             phantom_beam[2, 0, 0, :]),
            phantom.densities,
            method='nearest',
            bounds_error=False,
            fill_value=0
        )

        # Create dose grid (just the same size as the phantom for now)
        self.dose_grid_positions = np.copy(phantom_beam)

        # Perform hit testing to find which dose grid voxels are in the beam
        print("Performing hit-testing of dose grid voxels...")
        _, xlen, ylen, zlen = self.dose_grid_positions.shape
        dose_grid_blocked = np.zeros((xlen, ylen, zlen))
        dose_grid_OAD = np.zeros((xlen, ylen, zlen))
        for x in tqdm(range(xlen)):
            for y in range(ylen):
                for z in range(zlen):
                    voxel = self.dose_grid_positions[:, x, y, z]
                    psi = line_block_plane_collision(voxel)
                    dose_grid_blocked[x, y, z] = (
                        block.block_values_interp([psi[0], psi[1]])
                    )
                    # Save off-axis distance (at iso plane) for later
                    dose_grid_OAD[x, y, z] = (
                        euclidean(np.array([0, 0, source.SAD]), psi)
                    )

        # Calculate effective depths of dose grid voxels
        print("Calculating effective depths of dose grid voxels...")
        dose_grid_d_eff = np.zeros_like(dose_grid_blocked)
        xlen, ylen, zlen = dose_grid_d_eff.shape
        for x in tqdm(range(xlen)):
            for y in range(ylen):
                for z in range(zlen):
                    voxel = self.dose_grid_positions[:, x, y, z]
                    psi = line_calc_limit_plane_collision(voxel)
                    dist = np.sqrt(np.sum(np.power(voxel - psi, 2)))
                    num_steps = np.floor(dist / settings['stepSize'])
                    xcoords = np.linspace(voxel[0], psi[0], num_steps)
                    ycoords = np.linspace(voxel[1], psi[1], num_steps)
                    zcoords = np.linspace(voxel[2], psi[2], num_steps)
                    dose_grid_d_eff[x, y, z] = np.sum(
                        phantom_densities_interp(
                            np.dstack((xcoords, ycoords, zcoords))
                        ) * settings['stepSize']
                    )

        # Calculate photon fluence at dose grid voxels
        print("Calculating fluence...")
        self.dose_grid_fluence = np.zeros_like(dose_grid_blocked)
        xlen, ylen, zlen = self.dose_grid_fluence.shape
        self.dose_grid_fluence = (
            settings['sPri'] * -source.SAD /
            self.dose_grid_positions[2, :, :, :] *
            dose_grid_blocked
        )

        # Calculate beam softening factor for dose grid voxels
        print("Calculating beam softening factor...")
        f_soften = np.ones_like(dose_grid_OAD)
        f_soften[dose_grid_OAD < settings['softLimit']] = 1 / (
            1 - settings['softRatio'] *
            dose_grid_OAD[dose_grid_OAD < settings['softLimit']]
        )

        # Calculate TERMA of dose grid voxels
        print("Calculating TERMA...")
        E = np.linspace(settings['eLow'], settings['eHigh'], settings['eNum'])
        spectrum_weights = source.weights(E)
        mu_water = conehead.nist.mu_water(E)
        self.dose_grid_terma = np.zeros_like(dose_grid_blocked)
        xlen, ylen, zlen = self.dose_grid_terma.shape
        for x in tqdm(range(xlen)):
            for y in range(ylen):
                for z in range(zlen):
                    self.dose_grid_terma[x, y, z] = (
                        np.sum(
                            spectrum_weights *
                            self.dose_grid_fluence[x, y, z] *
                            np.exp(
                                -mu_water * f_soften[x, y, z] *
                                dose_grid_d_eff[x, y, z]
                            ) * E * mu_water
                        )
                    )

    def plot(self):
        # Plotting for debug purposes
        print("Calculation complete. Now plotting...")
        f1 = plt.figure()  # noqa: F841
        ax = plt.gca()
        im = ax.imshow(
            np.rot90(self.dose_grid_terma[:, 20, :]),
            extent=[-20.5, 20.5, -40.5, .5],
            aspect='equal'
        )
        # Minor ticks
        ax.set_xticks(np.arange(-19.5, 20.0, 1.0), minor=True)
        ax.set_yticks(np.arange(-39.5, 0, 1.0), minor=True)
        ax.grid(which="minor", color="black", linestyle='-', linewidth=1)
        plt.colorbar(im)

        f2 = plt.figure()  # noqa: F841
        ax2 = plt.gca()
        ax2.plot(
            -self.dose_grid_positions[2, 20, 20, :] - 100,
            (self.dose_grid_fluence[20, 20, :] /
             np.max(self.dose_grid_fluence[20, 20, :]) * 100),
            label='Fluence'
        )
        ax2.plot(
            -self.dose_grid_positions[2, 20, 20, :] - 100,
            (self.dose_grid_terma[20, 20, :] /
             np.max(self.dose_grid_terma[20, 20, :]) * 100),
            label='TERMA'
        )
        ax2.set_xlim([0, 40.0])
        ax2.set_ylim([0, 100])
        ax2.set_title("Central Axis Quantites")
        ax2.set_xlabel("Depth [cm]")
        ax2.set_ylabel("Relative Value [%]")
        ax2.legend()
        plt.show()

max_stars_repo_name: SimonBiggs/conehead

max_issues_repo_name: SimonBiggs/conehead

max_forks_repo_name: SimonBiggs/conehead


text:
# -*- coding: utf-8 -*-
# Copyright 2020 The PsiZ Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Module for simulating agent behavior.

Classes:
    RankAgent: An object that can be initialized using a psychological
        embedding and used to simulate Rank similarity judgments.

"""
import numpy as np
import tensorflow as tf
import tensorflow_probability as tfp

from psiz.agents.base import Agent
from psiz.trials import RankObservations

class RankAgent(Agent):  # pylint: disable=too-few-public-methods
    """Agent that simulates Rank similarity judgments.

    Attributes:
        model: A PsychologicalEmbedding object that supplies a
            similarity function and embedding points.
        groups: Array of integers indicating group membership
            information.

    Methods:
        simulate: Stochastically simulate similarity judgments.

    """

    def __init__(self, model, groups=None):
        """Initialize.

        Arguments:
            model: A concrete instance of a PsychologicalEmedding
                object.
            groups (optional): Array-like integers indicating group
                membership information. For example, `[4, 3]` indicates
                that the first optional column has the value 4 and the
                second optional column has the value 3.

        """
        Agent.__init__(self)
        self.model = model
        if groups is None:
            groups = []
        self.groups = groups

    def simulate(self, docket, batch_size=None):
        """Stochastically simulate similarity judgments.

        Arguments:
            docket: A RankDocket object representing the
                to-be-judged trials. The order of the stimuli in the
                stimulus set is ignored for the simulations.
            batch_size (optional): If None, `batch_size` is equal to
                the total number of trials.

        Returns:
            RankObservations object representing the judged trials. The
                order of the stimuli is now informative.

        """
        if batch_size is None:
            batch_size = docket.n_trial

        if len(self.groups) == 0:
            group_matrix = None
        else:
            group_matrix = np.expand_dims(self.groups, axis=0)
            group_matrix = np.repeat(group_matrix, docket.n_trial, axis=0)

        # Create TF dataset.
        ds_docket = docket.as_dataset(group_matrix).batch(
            batch_size, drop_remainder=False
        )

        # Call model with TensorFlow formatted docket and
        # stochastically sample an outcome.
        stimulus_set = None
        for data in ds_docket:
            dict_x, _, _ = tf.keras.utils.unpack_x_y_sample_weight(data)

            batch_stimulus_set = _rank_sample(
                dict_x['stimulus_set'],
                tf.reduce_mean(self.model(dict_x, training=False), axis=1)
            )
            if stimulus_set is None:
                stimulus_set = [batch_stimulus_set]
            else:
                stimulus_set.append(batch_stimulus_set)
        stimulus_set = tf.concat(stimulus_set, 0).numpy() - 1

        obs = RankObservations(
            stimulus_set,
            n_select=docket.n_select,
            is_ranked=docket.is_ranked,
            groups=group_matrix
        )
        return obs

def _rank_sample(stimulus_set, probs):
    """Stochasatically select outcome.

    Arguments:
        stimulus_set:
            shape=(batch_size, n_reference + 1, n_outcome)
        probs:
            shape=(batch_size, n_outcome)

    Returns:
        stimulus_set_selected:
            shape=(batch_size, n_reference + 1)

    """
    outcome_distribution = tfp.distributions.Categorical(
        probs=probs
    )
    idx_sample = outcome_distribution.sample()
    idx_batch = tf.range(tf.shape(idx_sample)[0])
    idx_batch_sample = tf.stack([idx_batch, idx_sample], axis=1)

    # Retrieve stimulus set associated with particular outcome.
    stimulus_set = tf.transpose(stimulus_set, perm=[0, 2, 1])
    stimulus_set_selected = tf.gather_nd(stimulus_set, idx_batch_sample)
    return stimulus_set_selected

max_stars_repo_name: greenfieldvision/psiz

max_issues_repo_name: greenfieldvision/psiz

max_forks_repo_name: greenfieldvision/psiz


text:
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.ensemble import IsolationForest

column_name_list = ["class", 'cap-shape',
                    'cap-surface',
                    'cap-color',
                    'bruises',
                    'odor',
                    'gill-attachment',
                    'gill-spacing',
                    'gill-size',
                    'gill-color',
                    'stalk-shape',
                    'stalk-surface-above-ring',
                    'stalk-surface-below-ring',
                    'stalk-color-above-ring',
                    'stalk-color-below-ring',
                    'veil-type',
                    'veil-color',
                    'ring-number',
                    'ring-type',
                    'spore-print-color',
                    'population',
                    'habitat']

def readCsv(csvFile):
    return pd.read_csv(csvFile, names=column_name_list)

def preprocessData(df):
    X = df.drop(['odor'], axis=1)
    y = pd.DataFrame(df['odor'])
    converted_X = pd.get_dummies(X)
    converted_y = pd.get_dummies(y)
    # converted_y.hist()
    # plt.show()
    X_train, X_test, y_train, y_test = train_test_split(
        converted_X, converted_y, train_size=0.66, test_size=0.33, random_state=42)
    # y_train.hist()
    # plt.show()
    return [X_train, X_test, y_train, y_test, converted_X, converted_y]

def preprocessReducedData(reduced_X, originalData):
    # print(reduced_X)
    # print(list(reduced_X.columns))
    y = pd.DataFrame(originalData['odor'])
    converted_y = pd.get_dummies(y)
    # print(converted_y)
    X_train, X_test, y_train, y_test = train_test_split(
        reduced_X, converted_y, train_size=0.66, test_size=0.33, random_state=42)
    return [X_train, X_test, y_train, y_test, reduced_X, converted_y]

def preprocessWithAD(converted_X):
    ad_model = IsolationForest(random_state=0)
    converted_X['outliers'] = pd.Series(ad_model.fit_predict(converted_X))
    converted_X['outliers'] = converted_X['outliers'].map({1: 0, -1: 1})
    print(converted_X['outliers'].value_counts())
    # for i in range(len(converted_X['outliers'])):
    #     if converted_X.loc[i].at['outliers']:
    #         converted_X = converted_X.drop(i)
    # # print(converted_X)
    # converted_X = converted_X.drop(columns=['outliers'], axis=1)
    # # print(converted_X)
    # return converted_X

def preprocessDeletedData(df):
    X = df.drop(['odor'], axis=1)
    y = pd.DataFrame(df['odor'])
    converted_X = pd.get_dummies(X)
    converted_y = pd.get_dummies(y)
    for column in converted_X.columns:
        if "-" in column:
            converted_X = converted_X.drop([column], axis=1)
    converted_y = converted_y.drop(['odor_-'], axis=1)
    X_train, X_test, y_train, y_test = train_test_split(
        converted_X, converted_y, train_size=0.66, test_size=0.33, random_state=42)
    return [X_train, X_test, y_train, y_test, converted_X, converted_y]

# if __name__ == '__main__':
#     df = readCsv(r"C:\Users\shalev\Desktop\Introduction_to_AI\Introduction-to-AI\Data\mushrooms_data_missing.csv")
#     preprocessDeletedData(df)
#     # data = preprocessData(df)[5]
#     # print(data.shape)
#
#     # print(data.shape[1])
#     # preprocessWithAD(data)

max_stars_repo_name: shalev-habany/Introduction-to-AI

max_issues_repo_name: shalev-habany/Introduction-to-AI

max_forks_repo_name: shalev-habany/Introduction-to-AI


text:
import yaml
import argparse
import numpy as np
import torch
from bitstring import BitArray
from torch.autograd import Variable
from models import *
from experiment import VAEXperiment
import torch.backends.cudnn as cudnn
from pytorch_lightning import Trainer
from pytorch_lightning.loggers import TestTubeLogger

from pytorch_lightning.callbacks import ModelCheckpoint
#trainer = Trainer()

def quantize(data,pred,error_bound):
    radius=32768
    diff = data - pred
    quant_index = (int) (abs(diff)/ error_bound) + 1
    #print(quant_index)
    if (quant_index < radius * 2) :
        quant_index =quant_index>> 1
        half_index = quant_index
        quant_index =quant_index<< 1
        #print(quant_index)
        quant_index_shifted=0
        if (diff < 0) :
            quant_index = -quant_index
            quant_index_shifted = radius - half_index
        else :
            quant_index_shifted = radius + half_index

        decompressed_data = pred + quant_index * error_bound
        #print(decompressed_data)
        if abs(decompressed_data - data) > error_bound :
            #print("b")
            return 0,data
        else:
            #print("c")
            data = decompressed_data
            return quant_index_shifted,data

    else:
        #print("a")
        return 0,data

parser = argparse.ArgumentParser(description='Generic runner for VAE models')
parser.add_argument('--config',  '-c',
                    dest="filename",
                    metavar='FILE',
                    help =  'path to the config file',
                    default='configs/vae.yaml')
parser.add_argument('--ckpt',  '-k',type=str
                   )
parser.add_argument('--mode',  '-m',type=str,default="c"
                   )
parser.add_argument('--lsize','-ls',type=int,default=0)
parser.add_argument('--input',  '-i',type=str
                   )
parser.add_argument('--error',  '-e',type=float,default=1e-2
                   )
parser.add_argument('--latents',  '-l',type=str,default=None
                   )
parser.add_argument('--quant',  '-q',type=str,default=None
                   )
parser.add_argument('--unpred',  '-u',type=str,default=None
                   )
parser.add_argument('-recon',  '-r',type=str,default=None
                   )
parser.add_argument('--decomp',  '-d',type=str,default=None
                   )
parser.add_argument('--bits',  '-b',type=int,
                   default=32)
parser.add_argument('--height',  '-hgt',type=int,
                   default=1800)
parser.add_argument('--width',  '-w',type=int,
                   default=3600)
parser.add_argument('--size','-s',type=int,
                   default=64)
args = parser.parse_args()

with open(args.filename, 'r') as file:
    try:
        config = yaml.safe_load(file)
    except yaml.YAMLError as exc:
        print(exc)

model = vae_models[config['model_params']['name']](**config['model_params'])
test = VAEXperiment(model,config['exp_params'])
checkpoint = torch.load(args.ckpt, map_location=lambda storage, loc: storage)
test.load_state_dict(checkpoint['state_dict'])

height=args.height
width=args.width
size=args.size

array=np.fromfile(args.input,dtype=np.float32).reshape((height,width))
picts=[]
for x in range(0,height,size):
    for y in range(0,width,size):
        endx=min(x+size,height)
        endy=min(y+size,width)
        pict=array[x:endx,y:endy]
        padx=size-pict.shape[0]
        pady=size-pict.shape[1]
        pict=np.pad(pict,((0,padx),(0,pady)))
        pict=np.expand_dims(pict,0)
                    #print(array[x:x+size,y:y+size])
        picts.append(pict)
picts=np.array(picts)
minimum=np.min(picts)
maximum=np.max(picts)
rng=maximum-minimum
outputs=test(torch.from_numpy(picts))

if args.mode=="c":
    zs=outputs[2].detach().numpy()
    predict=outputs[0].detach().numpy()

else:
    zs=np.fromfile(args.latents,dtype=np.float32).reshape((-1,args.lsize))
    predict=test.model.decode(torch.from_numpy(zs)).detach().numpy()

#predict=outputs[0].numpy()
print(zs.size)
qs=[]
us=[]
recon=np.zeros((height,width),dtype=np.float32)
eb=args.error*rng

if args.bits==32:

    temp_latents=zs.flatten()
    '''
    latents=[]
    for element in list(temp_latents):
        bar=BitArray(float=element,length=32)
        bs="".join([str(int(x)) for x in bar])

        latents.append(eval(r'0b'+ bs))
    '''
    latents=temp_latents

    idx=0
    for x in range(0,height,size):
        for y in range(0,width,size):
            endx=min(x+size,height)
            endy=min(y+size,width)

            for a in range(x,endx):
                for b in range(y,endy):
                    orig=picts[idx][0][a-x][b-y]
                    pred=predict[idx][0][a-x][b-y]
                    recon[a][b]=pred
                    quant,decomp=quantize(orig,pred,eb)
                    qs.append(quant)
                    if quant==0:
                        us.append(decomp)
                    array[a][b]=decomp
            idx=idx+1

else:
    radius=2**args.bits
    rs=outputs[0].detach().numpy()
    zmin=np.min(zs)
    zmax=np.max(zs)
    latents=[]
    for i in range(zs.shape[0]):
        for j in range(zs.shape[1]):
            tmp=int((zs[i][j]-zmin)*radius/(zmax-zmin))
            latents.append(tmp)
            zs[i][j]=(tmp/radius)*(zmax-zmin)+zmin
    predict=test.model.decode(torch.from_numpy(zs)).detach().numpy()
    idx=0
    for x in range(0,height,size):
        for y in range(0,width,size):
            endx=min(x+size,height)
            endy=min(y+size,width)

            for a in range(x,endx):
                for b in range(y,endy):
                    orig=picts[idx][0][a-x][b-y]
                    pred=predict[idx][0][a-x][b-y]
                    recon[a][b]=rs[idx][0][a-x][b-y]
                    quant,decomp=quantize(orig,pred,eb)
                    qs.append(quant)
                    if quant==0:
                        us.append(decomp)
                    array[a][b]=decomp
            idx=idx+1

latents=np.array(latents,dtype=np.float32)
quants=np.array(qs,dtype=np.int32)
unpreds=np.array(us,dtype=np.float32)
if args.latents!=None and args.mode=="c":
    latents.tofile(args.latents)
if args.quant!=None:
    quants.tofile(args.quant)
if args.unpred!=None:
    unpreds.tofile(args.unpred)
if args.recon!=None:
    recon.tofile(args.recon)
if args.decomp!=None:
    array.tofile(args.decomp)


text:
# -*- coding: utf-8 -*-
import functools
import sys
from argparse import ArgumentParser
from contextlib import contextmanager
import tensorflow as tf
from pprint import pformat

from matplotlib import pyplot
from tensorflow.contrib.framework import arg_scope, add_arg_scope

import tfsnippet as spt
from tfsnippet import DiscretizedLogistic
from tfsnippet.examples.utils import (MLResults,
                                      save_images_collection,
                                      bernoulli_as_pixel,
                                      bernoulli_flow,
                                      bernoulli_flow,
                                      print_with_title)
import numpy as np
from tfsnippet.layers import pixelcnn_2d_output

from tfsnippet.preprocessing import UniformNoiseSampler

from ood_regularizer.experiment.datasets.overall import load_overall, load_complexity
from ood_regularizer.experiment.datasets.svhn import load_svhn
from ood_regularizer.experiment.models.utils import get_mixed_array
from ood_regularizer.experiment.utils import make_diagram, get_ele, plot_fig
import os
from imgaug import augmenters as iaa

class ExpConfig(spt.Config):
    # model parameters
    z_dim = 256
    act_norm = False
    weight_norm = False
    batch_norm = False
    l2_reg = 0.0002
    kernel_size = 3
    shortcut_kernel_size = 1
    nf_layers = 20
    pixelcnn_level = 5

    # training parameters
    result_dir = None
    write_summary = True
    max_epoch = 40
    warm_up_start = 300
    initial_beta = -3.0
    uniform_scale = False
    use_transductive = True
    mixed_train = False
    mixed_train_epoch = 64
    mixed_train_skip = 64
    mixed_times = 64
    mixed_replace = 64
    mixed_replace_ratio = 1.0
    dynamic_epochs = False
    retrain_for_batch = True

    in_dataset_test_ratio = 1.0
    pretrain = True
    distill_ratio = 1.0
    stand_weight = 0.1

    in_dataset = 'cifar10'
    out_dataset = 'svhn'

    max_step = None
    batch_size = 64
    smallest_step = 5e-5
    initial_lr = 0.0002
    lr_anneal_factor = 0.5
    lr_anneal_epoch_freq = []
    lr_anneal_step_freq = None

    n_critical = 5
    # evaluation parameters
    train_n_qz = 1
    test_n_qz = 10
    test_batch_size = 64
    test_epoch_freq = 100
    plot_epoch_freq = 20

    epsilon = -20.0
    min_logstd_of_q = -3.0

    sample_n_z = 100

    x_shape = (32, 32, 3)
    x_shape_multiple = 3072
    extra_stride = 2
    count_experiment = False

config = ExpConfig()

@add_arg_scope
def batch_norm(inputs, training=False, scope=None):
    return tf.layers.batch_normalization(inputs, training=training, name=scope)

@add_arg_scope
def dropout(inputs, training=False, scope=None):
    print(inputs, training)
    return spt.layers.dropout(inputs, rate=0.2, training=training, name=scope)

@add_arg_scope
@spt.global_reuse
def p_net(input):
    input = tf.to_float(input)
    # prepare for the convolution stack
    output = spt.layers.pixelcnn_2d_input(input)

    # apply the PixelCNN 2D layers.
    for i in range(config.pixelcnn_level):
        output = spt.layers.pixelcnn_conv2d_resnet(
            output,
            out_channels=64,
            vertical_kernel_size=(2, 3),
            horizontal_kernel_size=(2, 2),
            activation_fn=tf.nn.leaky_relu,
            normalizer_fn=None,
            dropout_fn=dropout
        )
    output_list = [spt.layers.pixelcnn_conv2d_resnet(
        output,
        out_channels=256,
        vertical_kernel_size=(2, 3),
        horizontal_kernel_size=(2, 2),
        activation_fn=tf.nn.leaky_relu,
        normalizer_fn=None,
        dropout_fn=dropout
    ) for i in range(config.x_shape[-1])]
    # get the final output of the PixelCNN 2D network.
    output_list = [pixelcnn_2d_output(output) for output in output_list]
    output = tf.stack(output_list, axis=-2)
    print(output)
    output = tf.reshape(output, (-1,) + config.x_shape + (256,))  # [batch, height, weight, channel, 256]
    return output

class MyIterator(object):
    def __init__(self, iterator):
        self._iterator = iter(iterator)
        self._next = None
        self._has_next = True
        self.next()

    @property
    def has_next(self):
        return self._has_next

    def next(self):
        if not self._has_next:
            raise StopIteration()

        ret = self._next
        try:
            self._next = next(self._iterator)
        except StopIteration:
            self._next = None
            self._has_next = False
        else:
            self._has_next = True
        return ret

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

def limited(iterator, n):
    i = 0
    try:
        while i < n:
            yield next(iterator)
            i += 1
    except StopIteration:
        pass

def get_var(name):
    pfx = name.rsplit('/', 1)
    if len(pfx) == 2:
        vars = tf.global_variables(pfx[0] + '/')
    else:
        vars = tf.global_variables()
    for var in vars:
        if var.name.split(':', 1)[0] == name:
            return var
    raise NameError('Variable {} not exist.'.format(name))

def main():
    # parse the arguments
    arg_parser = ArgumentParser()
    spt.register_config_arguments(config, arg_parser, title='Model options')
    spt.register_config_arguments(spt.settings, arg_parser, prefix='tfsnippet',
                                  title='TFSnippet options')
    arg_parser.parse_args(sys.argv[1:])

    # print the config
    print_with_title('Configurations', pformat(config.to_dict()), after='\n')

    # open the result object and prepare for result directories
    results = MLResults(config.result_dir)
    results.save_config(config)  # save experiment settings for review
    while True:
        try:
            results.make_dirs('plotting/sample', exist_ok=True)
            results.make_dirs('plotting/z_plot', exist_ok=True)
            results.make_dirs('plotting/train.reconstruct', exist_ok=True)
            results.make_dirs('plotting/test.reconstruct', exist_ok=True)
            results.make_dirs('train_summary', exist_ok=True)
            results.make_dirs('checkpoint/checkpoint', exist_ok=True)
            break
        except Exception:
            pass

    if config.count_experiment:
        with open('/home/cwx17/research/ml-workspace/projects/wasserstein-ood-regularizer/count_experiments', 'a') as f:
            f.write(results.system_path("") + '\n')
            f.close()

    # prepare for training and testing data
    # It is important: the `x_shape` must have channel dimension, even it is 1! (i.e. (28, 28, 1) for MNIST)
    # And the value of images should not be normalized, ranged from 0 to 255.
    # prepare for training and testing data
    (x_train, y_train, x_test, y_test) = load_overall(config.in_dataset)
    (svhn_train, svhn_train_y, svhn_test, svhn_test_y) = load_overall(config.out_dataset)
    config.x_shape = x_train.shape[1:]
    config.x_shape_multiple = 1
    for x in config.x_shape:
        config.x_shape_multiple *= x
    if config.x_shape == (28, 28, 1):
        config.extra_stride = 1
    if x_train.shape[-1] == 3:
        config.stand_weight = 0.2

    # input placeholders
    input_x = tf.placeholder(
        dtype=tf.int32, shape=(None,) + config.x_shape, name='input_x')
    learning_rate = spt.AnnealingVariable(
        'learning_rate', config.initial_lr, config.lr_anneal_factor)

    # derive the loss and lower-bound for training
    with tf.name_scope('training'), \
         arg_scope([batch_norm, dropout], training=True):
        train_p_net = p_net(input_x)
        theta_loss = tf.reduce_sum(
            tf.nn.sparse_softmax_cross_entropy_with_logits(labels=input_x, logits=train_p_net),
            axis=np.arange(-len(config.x_shape), 0)
        )
        theta_loss = tf.reduce_mean(theta_loss)

    # derive the nll and logits output for testing
    with tf.name_scope('testing'):
        test_p_net = p_net(input_x)
        ele_test_ll = -tf.reduce_sum(
            tf.nn.sparse_softmax_cross_entropy_with_logits(labels=input_x, logits=test_p_net),
            axis=np.arange(-len(config.x_shape), 0)
        ) / config.x_shape_multiple / np.log(2)

    # derive the optimizer
    with tf.name_scope('optimizing'):
        theta_params = tf.trainable_variables('p_net')
        with tf.variable_scope('theta_optimizer'):
            theta_optimizer = tf.train.AdamOptimizer(learning_rate)
            theta_grads = theta_optimizer.compute_gradients(theta_loss, theta_params)
        with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
            theta_train_op = theta_optimizer.apply_gradients(theta_grads)

    train_flow = spt.DataFlow.arrays([x_train], config.batch_size, shuffle=True, skip_incomplete=True)

    mixed_array = np.concatenate([x_test, svhn_test])
    print(mixed_array.shape)
    index = np.arange(len(mixed_array))
    np.random.shuffle(index)
    index = index[:len(index) // config.mixed_times]
    config.mixed_train_skip = config.mixed_train_skip // config.mixed_times
    config.mixed_train_epoch = config.mixed_train_epoch * config.mixed_times
    index = index[:200]
    mixed_array = mixed_array[index]

    reconstruct_test_flow = spt.DataFlow.arrays([x_test], 100, shuffle=True, skip_incomplete=True)
    reconstruct_train_flow = spt.DataFlow.arrays([x_train], 100, shuffle=True, skip_incomplete=True)
    reconstruct_omega_test_flow = spt.DataFlow.arrays([svhn_test], 100, shuffle=True, skip_incomplete=True)
    reconstruct_omega_train_flow = spt.DataFlow.arrays([svhn_train], 100, shuffle=True, skip_incomplete=True)

    with spt.utils.create_session().as_default() as session, \
            train_flow.threaded(5) as train_flow:
        spt.utils.ensure_variables_initialized()

        experiment_dict = {
            'tinyimagenet': '/mnt/mfs/mlstorage-experiments/cwx17/cd/d5/02279d802d3a67e701f5',
            'svhn': '/mnt/mfs/mlstorage-experiments/cwx17/1a/d5/02732c28dc8d842701f5',
            'celeba': '/mnt/mfs/mlstorage-experiments/cwx17/bd/d5/02279d802d3aec4401f5',
            'cifar10': '/mnt/mfs/mlstorage-experiments/cwx17/6d/d5/02c52d867e43834401f5',
            'cifar100': '/mnt/mfs/mlstorage-experiments/cwx17/5d/d5/02c52d867e43d6b101f5',
            'fashion_mnist': '/mnt/mfs/mlstorage-experiments/cwx17/36/e5/02279d802d3a8bf522f5',
            'constant': '/mnt/mfs/mlstorage-experiments/cwx17/d5/e5/02c52d867e43c3c622f5',
            'omniglot': '/mnt/mfs/mlstorage-experiments/cwx17/e5/e5/02c52d867e43e4c622f5',
            'mnist': '/mnt/mfs/mlstorage-experiments/cwx17/4c/d5/02812baa4f7012d622f5',
            'noise': '/mnt/mfs/mlstorage-experiments/cwx17/7c/d5/02812baa4f70a83722f5',
            'kmnist': '/mnt/mfs/mlstorage-experiments/cwx17/f5/e5/02c52d867e43382822f5',
            'not_mnist': '/mnt/mfs/mlstorage-experiments/cwx17/66/e5/02279d802d3a7c8822f5',
            'not_mnist28': '/mnt/mfs/mlstorage-experiments/cwx17/50/e5/02812baa4f700cc9d2f5',
            'fashion_mnist28': '/mnt/mfs/mlstorage-experiments/cwx17/ef/e5/02c52d867e430cc9d2f5',
            'omniglot28': '/mnt/mfs/mlstorage-experiments/cwx17/c4/e5/02732c28dc8d0cc9d2f5',
            'kmnist28': '/mnt/mfs/mlstorage-experiments/cwx17/d4/e5/02732c28dc8d0cc9d2f5',
            'mnist28': '/mnt/mfs/mlstorage-experiments/cwx17/ff/e5/02c52d867e43906cd2f5',
            'noise28': '/mnt/mfs/mlstorage-experiments/cwx17/f4/e5/02732c28dc8deb5cd2f5',
            'constant28': '/mnt/mfs/mlstorage-experiments/cwx17/f0/f5/02279d802d3a7b5cd2f5'
        }
        print(experiment_dict)
        if config.in_dataset in experiment_dict:
            restore_dir = experiment_dict[config.in_dataset] + '/checkpoint'
            restore_checkpoint = os.path.join(
                restore_dir, 'checkpoint',
                'checkpoint.dat-{}'.format(config.max_epoch))
        else:
            restore_dir = results.system_path('checkpoint')
            restore_checkpoint = None

        # train the network
        with spt.TrainLoop(tf.trainable_variables(),
                           var_groups=['q_net', 'p_net', 'posterior_flow', 'G_theta', 'D_psi', 'G_omega', 'D_kappa'],
                           max_epoch=config.max_epoch + 1,
                           max_step=config.max_step,
                           summary_dir=(results.system_path('train_summary')
                                        if config.write_summary else None),
                           summary_graph=tf.get_default_graph(),
                           early_stopping=False,
                           checkpoint_dir=results.system_path('checkpoint'),
                           restore_checkpoint=restore_checkpoint
                           ) as loop:

            loop.print_training_summary()
            spt.utils.ensure_variables_initialized()

            epoch_iterator = loop.iter_epochs()
            # print(loop.epoch)
            # adversarial training
            for epoch in epoch_iterator:
                if epoch > config.max_epoch:
                    mixed_ll = get_ele(ele_test_ll, spt.DataFlow.arrays([mixed_array], config.test_batch_size), input_x)

                    def stand(base, another_arrays=None):
                        mean, std = np.mean(base), np.std(base)
                        return_arrays = []
                        for array in another_arrays:
                            return_arrays.append(-np.abs((array - mean) / std) * config.stand_weight)
                        return return_arrays

                    cifar_train_nll = get_ele(ele_test_ll,
                                              spt.DataFlow.arrays([x_train], config.test_batch_size),
                                              input_x)
                    [mixed_stand] = stand(cifar_train_nll, [mixed_ll])

                    loop.collect_metrics(stand_histogram=plot_fig(
                        [mixed_stand[index < len(x_test)], mixed_stand[index >= len(x_test)]],
                        ['red', 'green'],
                        [config.in_dataset + ' Test',
                         config.out_dataset + ' Test'], 'log(bit/dims)',
                        'stand_histogram'))

                    mixed_kl = []

                    def get_ll(x, print_log=True):
                        return get_ele(ele_test_ll,
                                       spt.DataFlow.arrays([np.expand_dims(x, 0)], config.test_batch_size), input_x,
                                       print_log=print_log)

                    if not config.pretrain:
                        session.run(tf.global_variables_initializer())
                    loop.make_checkpoint()
                    learning_rate.anneal()
                    learning_rate.anneal()
                    print('Starting testing')
                    for i in range(0, len(mixed_array), config.mixed_train_skip):
                        if config.retrain_for_batch:
                            loop._checkpoint_saver.restore_latest()
                        if config.dynamic_epochs:
                            repeat_epoch = int(
                                config.mixed_train_epoch * len(mixed_array) / (9 * i + len(mixed_array)))
                            repeat_epoch = max(1, repeat_epoch)
                        else:
                            repeat_epoch = config.mixed_train_epoch

                        repeat_epoch = repeat_epoch * config.mixed_train_skip // config.batch_size
                        # data generator generate data for each batch
                        # repeat_epoch will determine how much time it generates
                        for pse_epoch in range(repeat_epoch):
                            mixed_index = np.random.randint(i if config.retrain_for_batch else 0,
                                                            min(len(mixed_array), i + config.mixed_train_skip),
                                                            config.batch_size)
                            batch_x = mixed_array[mixed_index]
                            # print(batch_x.shape)

                            aug = iaa.Affine(
                                translate_percent={'x': (-0.1, 0.1), 'y': (-0.1, 0.1)},
                                # order=3,  # turn on this if not just translation
                                rotate=(-60, 60),
                                mode='edge',
                                backend='cv2'
                            )
                            batch_x = aug(images=batch_x)
                            # print(batch_x.shape)
                            for step, [x] in loop.iter_steps(train_flow):
                                break
                            if np.random.rand() < config.mixed_replace_ratio:
                                if config.mixed_replace > 0:
                                    batch_x[:-config.mixed_replace] = x[:-config.mixed_replace]
                            else:
                                batch_x = x
                            # print(batch_x.shape)
                            if pse_epoch == 0:
                                try:
                                    save_images_collection(
                                        images=batch_x,
                                        filename='aug_example.png',
                                        grid_size=(8, batch_x.shape[0] // 8),
                                        results=results,
                                    )
                                except Exception as e:
                                    print(e)

                            if config.distill_ratio != 1.0:
                                ll = mixed_ll[mixed_index]
                                ll_omega = session.run(ele_test_ll, feed_dict={
                                    input_x: batch_x
                                })
                                batch_index = np.argsort(ll - ll_omega)
                                batch_index = batch_index[:int(len(batch_index) * config.distill_ratio)]
                                batch_x = batch_x[batch_index]

                            _, batch_VAE_loss = session.run([theta_train_op, theta_loss], feed_dict={
                                input_x: batch_x
                            })
                            loop.collect_metrics(theta_loss=batch_VAE_loss)
                        mixed_kl.append(get_ele(ele_test_ll,
                                                spt.DataFlow.arrays([mixed_array[i: i + config.mixed_train_skip]],
                                                                    config.test_batch_size), input_x))
                        print(repeat_epoch, len(mixed_kl))
                        print(mixed_kl[i] - mixed_ll[i])
                        print(index[i] < len(x_test))
                        loop.print_logs()

                    mixed_kl = np.concatenate(mixed_kl)
                    mixed_kl = mixed_kl - mixed_ll
                    cifar_kl = mixed_kl[index < len(x_test)]
                    svhn_kl = mixed_kl[index >= len(x_test)]

                    loop.collect_metrics(kl_histogram=plot_fig([-cifar_kl, -svhn_kl],
                                                               ['red', 'green'],
                                                               [config.in_dataset + ' Test',
                                                                config.out_dataset + ' Test'], 'log(bit/dims)',
                                                               'kl_histogram'))
                    mixed_kl = mixed_kl - mixed_stand
                    cifar_kl = mixed_kl[index < len(x_test)]
                    svhn_kl = mixed_kl[index >= len(x_test)]
                    loop.collect_metrics(kl_with_stand_histogram=plot_fig([-cifar_kl, -svhn_kl],
                                                                          ['red', 'green'],
                                                                          [config.in_dataset + ' Test',
                                                                           config.out_dataset + ' Test'],
                                                                          'log(bit/dims)',
                                                                          'kl_with_stand_histogram'))
                    loop.print_logs()
                    break

                for step, [x] in loop.iter_steps(train_flow):
                    _, batch_VAE_loss = session.run([theta_train_op, theta_loss], feed_dict={
                        input_x: x
                    })
                    loop.collect_metrics(theta_loss=batch_VAE_loss)

                if epoch in config.lr_anneal_epoch_freq:
                    learning_rate.anneal()

                if epoch == config.max_epoch:
                    loop._checkpoint_saver.save(epoch)

                loop.collect_metrics(lr=learning_rate.get())
                loop.print_logs()

    # print the final metrics and close the results object
    print_with_title('Results', results.format_metrics(), before='\n')
    results.close()

if __name__ == '__main__':
    main()

max_stars_repo_name: chenwenxiao/DOI

max_issues_repo_name: chenwenxiao/DOI

max_forks_repo_name: chenwenxiao/DOI