text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2022 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import math
import re

import numpy as np

from tensorflow.core.framework import attr_value_pb2
from tensorflow.core.framework import graph_pb2
from tensorflow.core.framework import node_def_pb2
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import graph_util
from tensorflow.python.framework import tensor_util
from tensorflow.python.platform import flags as flags_lib
from tensorflow.python.platform import tf_logging
from tensorflow.python.tools import strip_unused_lib

from neural_compressor.utils.utility import dump_elapsed_time

class FuseDecomposedBNOptimizer():
    def __init__(self, input_graph_def):
        self.input_graph_def = input_graph_def

    @dump_elapsed_time("Pass FuseDecomposedBNOptimizer")
    def do_transformation(self):
        """Fuse individual ops in batch normalization to FusedBatchNorm.
        In some models, the batch normalizatin is performed via a group of individual
        ops instead of using single FusedBatchNorm op. This function identifies a
        pattern of batch normalization subgraph which is made of multiple ops and
        transforms the graph by replacing those individual ops with FusedBatchNorm op.
        This will provide the opportunity to further fold the FusedBatchNorm with
        convolution ops to reduce the computation steps during inference.
        This function currently recognizes batch normalization patterns described
        below, this could be extended if newer patterns are seen. Also, the fusion
        is only attempted if the input graph is in NHWC format or has no format set.
        Computation function:
            (X * multiplier) + (Beta - Mean * multiplier)
            where multiplier = rsqrt (Variance + Epsilon) * Gamma
                            OR = rsqrt (Variance + Epsilon) when Gamma is 1
        Subgraph:
        {"Add"
            {{"Mul"  // mul_0
                {{"*"},  // input to apply batchnorm
                {"Mul"  // mul_1, same op is used inside the Sub block
                    {{"Rsqrt"
                        {"Add"
                            {{"Const" | "Reshape(Const)"},  // Variance
                            {"Const"}  // Epsilon
                            }
                        }
                        },  // end - Rsqrt
                        {"Const" | "Reshape(Const)"}  // Gamma
                    }
                    }  // end - mul_1
                }
            },  // end - mul_0
            {"Sub"
                {{"Const" | "Reshape(Const)"},  // Beta
                {"Mul"  // mul_3
                    {{"Const" | "Reshape(Const)"},  // Mean
                    {"Mul"  // same mul_1 op as in previous block
                        {{"Rsqrt"
                            {"Add"
                                {{"Const" | "Reshape(Const)"},  // Variance
                                {"Const"}  // Epsilon
                                }
                            }
                        },  // end - Rsqrt
                        {"Const" | "Reshape(Const)"}  // Gamma
                        }
                        }  // end - mul_1
                    }
                    }  // end - mul_3
                }
                }  // end - Sub
            }
        }  // end - Add
        Subgraph pattern when gamma value is 1 and the gamma scaling Mul is skipped
        {"Add"
            {{"Mul"  // mul_0
                {{"*"},  // input to apply batchnorma
                {"Rsqrt"  // same Rsqrt op used in Sub block
                    {"Add"
                        {{"Const" | "Reshape(Const)"},  // Variance
                        {"Const"}  // Epsilon
                        }
                    }
                    }  // end - Rsqrt
                }
                },  // end - mul_0
                {"Sub"
                {{"Const" | "Reshape(Const)"},  // Beta
                {"Mul"  // mul_1
                    {{"Const" | "Reshape(Const)"},  // Mean
                    {"Rsqrt"  // same Rsqrt op as in previous mul_0 block
                        {"Add"
                            {{"Const" | "Reshape(Const)"},  // Variance
                            {"Const"}  // Epsilon
                            }
                        }
                        }  // end - Rsqrt
                    }
                }  // end - mul_1
                }
                }  // end - Sub
            }
        }  // end - Add
        Args:
            input_graph_def: A GraphDef containing a model.
        Returns:
            Modified graph with individual ops that made up of batch normalization
            fused to FusedBatchNorm.
        Raises:
            ValueError: If the graph is badly formed with duplicate node names.
        """
        input_node_map = {}
        for node in self.input_graph_def.node:
            if node.name not in input_node_map:
                input_node_map[node.name] = node
            else:
                raise ValueError("Duplicate node names detected for ", node.name)

        # Check format and only proceed if graph is in NHWC or has no format set.
        data_format = None
        for node in self.input_graph_def.node:
            if "data_format" in node.attr.keys():
                data_format = node.attr["data_format"]
            if data_format is not None and data_format.s != b"NHWC":
                tf_logging.warn("%s in %s format, not candidate for batchnorm fusion."
                            % (node.name, data_format.s))
                return self.input_graph_def
            else:
                continue

        nodes_to_skip = {}
        new_ops = []
        for node in self.input_graph_def.node:
            if node.op != "Add":
                continue

            # Add (Mul, Sub) or Add (Sub, Mul)
            input0_op = node_from_map(input_node_map, node.input[0])
            input1_op = node_from_map(input_node_map, node.input[1])

            if input0_op.op == "Mul" and input1_op.op == "Sub":
                data_scale_mul_op = input0_op
                bias_mean_sub_op = input1_op
            elif input0_op.op == "Sub" and input1_op.op == "Mul":
                bias_mean_sub_op = input0_op
                data_scale_mul_op = input1_op
            else:
                continue

            # Mul (input, Mul)
            input_data_op = node_from_map(input_node_map, data_scale_mul_op.input[0])
            scale_op = node_from_map(input_node_map, data_scale_mul_op.input[1])

            if scale_op.op == "Rsqrt":
                gamma_op = None
                gamma_reshape_op = None
                rsqrt_op = scale_op
            elif scale_op.op == "Mul":
                # Mul (Rsqrt, Constant_gamma)
                rsqrt_op = node_from_map(input_node_map, scale_op.input[0])
                gamma_op, gamma_reshape_op = bypass_reshape(input_node_map,
                                                            scale_op.input[1])
                if rsqrt_op.op != "Rsqrt":
                    continue
                if gamma_op.op != "Const" or get_const_dim_count(gamma_op) != 1:
                    continue
            else:
                continue

            # Sub (Constant_beta, Mul)
            beta_op, beta_reshape_op = bypass_reshape(input_node_map,
                                                    bias_mean_sub_op.input[0])
            mean_scale_mul_op = node_from_map(input_node_map, bias_mean_sub_op.input[1])
            if mean_scale_mul_op.op != "Mul":
                continue
            if beta_op.op != "Const" or get_const_dim_count(beta_op) != 1:
                continue

        # Common scale applies to both input and running mean
            if scale_op != node_from_map(input_node_map,
                                        mean_scale_mul_op.input[1]):
                continue

            mean_op, mean_reshape_op = bypass_reshape(input_node_map,
                                                    mean_scale_mul_op.input[0])
            if mean_op.op != "Const" or get_const_dim_count(mean_op) != 1:
                continue

            # Add (Constant_variance, Constant_epsilon)
            variance_epsilon_add_op = node_from_map(input_node_map, rsqrt_op.input[0])
            if variance_epsilon_add_op.op != "Add":
                continue

            variance_op, variance_reshape_op = bypass_reshape(
                input_node_map, variance_epsilon_add_op.input[0])
            epsilon_op = node_from_map(input_node_map,
                                    variance_epsilon_add_op.input[1])
            if epsilon_op.op != "Const" or get_const_dim_count(epsilon_op) != 0:
                continue
            if variance_op.op != "Const" or get_const_dim_count(variance_op) != 1:
                continue

            epsilon = values_from_const(epsilon_op)

            nodes_to_skip[node.name] = True
            nodes_to_skip[data_scale_mul_op.name] = True
            nodes_to_skip[bias_mean_sub_op.name] = True
            nodes_to_skip[mean_scale_mul_op.name] = True
            nodes_to_skip[scale_op.name] = True
            if scale_op.op != "Rsqrt":
                nodes_to_skip[rsqrt_op.name] = True
            nodes_to_skip[variance_epsilon_add_op.name] = True
            if gamma_reshape_op is not None:
                nodes_to_skip[gamma_reshape_op.name] = True
            if beta_reshape_op is not None:
                nodes_to_skip[beta_reshape_op.name] = True
            if mean_reshape_op is not None:
                nodes_to_skip[mean_reshape_op.name] = True
            if variance_reshape_op is not None:
                nodes_to_skip[variance_reshape_op.name] = True

            if gamma_op is None:
                gamma_op = node_def_pb2.NodeDef()
                gamma_op.op = "Const"
                # Assign name with same root of Rsqrt op's name plus "gamma"
                m = re.search(r"(.*)/(.*)", scale_op.name)
                if m:
                    gamma_op.name = m.group(1) + "/gamma"
                else:
                    gamma_op.name = scale_op.name + "/gamma"
                gamma_op.attr["dtype"].CopyFrom(beta_op.attr["dtype"])
                beta_value = values_from_const(beta_op)
                gamma_op.attr["value"].CopyFrom(
                    attr_value_pb2.AttrValue(tensor=tensor_util.make_tensor_proto(
                    1, beta_value.dtype.type, beta_value.shape,
                    allow_broadcast=True)))
                new_ops.append(gamma_op)

            new_fused_batchnorm_op = node_def_pb2.NodeDef()
            new_fused_batchnorm_op.op = "FusedBatchNorm"
            new_fused_batchnorm_op.name = node.name
            new_fused_batchnorm_op.attr["T"].CopyFrom(node.attr["T"])
            new_fused_batchnorm_op.attr["is_training"].CopyFrom(
                attr_value_pb2.AttrValue(b=False))
            new_fused_batchnorm_op.attr["epsilon"].CopyFrom(
                attr_value_pb2.AttrValue(f=epsilon.tolist()))
            if data_format is not None:
                new_fused_batchnorm_op.attr["data_format"].CopyFrom(data_format)
            new_fused_batchnorm_op.input.extend([input_data_op.name, gamma_op.name,
                                                beta_op.name, mean_op.name,
                                                variance_op.name])

            new_ops.append(new_fused_batchnorm_op)

        result_graph_def = graph_pb2.GraphDef()
        for node in self.input_graph_def.node:
            if node.name in nodes_to_skip:
                continue
            new_node = node_def_pb2.NodeDef()
            new_node.CopyFrom(node)
            retained_input = []
            for input_node in new_node.input:
                if not input_node.startswith("^") or input_node[1:] not in nodes_to_skip:
                    retained_input.append(input_node)
            new_node.input[:] = retained_input
            result_graph_def.node.append(new_node)

        result_graph_def.node.extend(new_ops)
        result_graph_def.versions.CopyFrom(self.input_graph_def.versions)
        return result_graph_def

def node_name_from_input(node_name):
    """Strips off ports and other decorations to get the underlying node name."""
    if node_name.startswith("^"):
        node_name = node_name[1:]
    m = re.search(r"(.*):\d+$", node_name)
    if m:
        node_name = m.group(1)
    return node_name

def node_from_map(node_map, name):
    """Pulls a node def from a dictionary for a given name.
    Args:
        node_map: Dictionary containing an entry indexed by name for every node.
        name: Identifies the node we want to find.
    Returns:
        NodeDef of the node with the given name.
    Raises:
        ValueError: If the node isn't present in the dictionary.
    """
    stripped_name = node_name_from_input(name)
    if stripped_name not in node_map:
        raise ValueError("No node named '%s' found in map." % name)
    return node_map[stripped_name]

def values_from_const(node_def):
    """Extracts the values from a const NodeDef as a numpy ndarray.
    Args:
        node_def: Const NodeDef that has the values we want to access.
    Returns:
        Numpy ndarray containing the values.
    Raises:
        ValueError: If the node isn't a Const.
    """
    if node_def.op != "Const":
        raise ValueError(
            "Can not extract constant value from a node that is not Const. Got:\n"
            f"{node_def}")
    input_tensor = node_def.attr["value"].tensor
    tensor_value = tensor_util.MakeNdarray(input_tensor)
    return tensor_value

def valid_reshape_inputs(reshape_in0_ndef, reshape_in1_ndef):
    if reshape_in0_ndef.op != "Const" or reshape_in1_ndef.op != "Const" \
    or get_const_dim_count(reshape_in0_ndef) != 1:
        return False
    input0_vec_size = values_from_const(reshape_in0_ndef).shape[0]
    const_value = values_from_const(reshape_in1_ndef)
    shape_ndims = const_value.ndim
    if shape_ndims != 1:
        raise ValueError("Num of dims of the shape must be 1, got {}.".format(
            shape_ndims))
    for value in const_value.tolist()[:-1]:
        if value != 1:
            return False
    if (const_value.tolist()[-1] != input0_vec_size):
        return False
    return True

def bypass_reshape(input_node_map, input_name):
    reshape_ndef = None
    maybe_reshape_ndef = node_from_map(input_node_map, input_name)
    input_ndef = maybe_reshape_ndef
    if maybe_reshape_ndef.op == "Reshape":
        reshpae_input0_ndef = node_from_map(input_node_map,
                                            maybe_reshape_ndef.input[0])
        reshpae_input1_ndef = node_from_map(input_node_map,
                                            maybe_reshape_ndef.input[1])
        if reshpae_input0_ndef.op == "Const" and reshpae_input1_ndef.op == "Const" \
            and valid_reshape_inputs(reshpae_input0_ndef, reshpae_input1_ndef):
            input_ndef = reshpae_input0_ndef
            reshape_ndef = maybe_reshape_ndef
    return input_ndef, reshape_ndef

def get_const_dim_count(node_def):
    """Get the number of dimensions for a Const node.
    Args:
        node_def: Const NodeDef.
    Returns:
        Number of dimensions for the Const node.
    """
    const_value = values_from_const(node_def)
    return const_value.ndim

max_stars_repo_name: intel/neural-compressor

max_issues_repo_name: intel/lp-opt-tool

max_forks_repo_name: intel/neural-compressor


text:
import torch
import numpy as np

from agent import DDPGAgent
from utils import MultiAgentReplayBuffer

class MADDPG:

    def __init__(self, env, buffer_maxlen):
        self.env = env
        self.num_agents = env.n
        self.replay_buffer = MultiAgentReplayBuffer(self.num_agents, buffer_maxlen)
        self.agents = [DDPGAgent(self.env, i) for i in range(self.num_agents)]

    def get_actions(self, states):
        actions = []
        for i in range(self.num_agents):
            action = self.agents[i].get_action(states[i])
            actions.append(action)
        return actions

    def update(self, batch_size):
        obs_batch, indiv_action_batch, indiv_reward_batch, next_obs_batch, \
            global_state_batch, global_actions_batch, global_next_state_batch, done_batch = self.replay_buffer.sample(batch_size)

        for i in range(self.num_agents):
            obs_batch_i = obs_batch[i]
            indiv_action_batch_i = indiv_action_batch[i]
            indiv_reward_batch_i = indiv_reward_batch[i]
            next_obs_batch_i = next_obs_batch[i]

            next_global_actions = []
            for agent in self.agents:
                next_obs_batch_i = torch.FloatTensor(next_obs_batch_i)
                indiv_next_action = agent.actor.forward(next_obs_batch_i)
                indiv_next_action = [agent.onehot_from_logits(indiv_next_action_j) for indiv_next_action_j in indiv_next_action]
                indiv_next_action = torch.stack(indiv_next_action)
                next_global_actions.append(indiv_next_action)
            next_global_actions = torch.cat([next_actions_i for next_actions_i in next_global_actions], 1)

            self.agents[i].update(indiv_reward_batch_i, obs_batch_i, global_state_batch, global_actions_batch, global_next_state_batch, next_global_actions)
            self.agents[i].target_update()

    def run(self, max_episode, max_steps, batch_size):
        episode_rewards = []
        for episode in range(max_episode):
            states = self.env.reset()
            episode_reward = 0
            for step in range(max_steps):
                actions = self.get_actions(states)
                #print(actions)
                next_states, rewards, dones, _ = self.env.step(actions)
                episode_reward += np.mean(rewards)
                #print(step)

                if all(dones) or step == max_steps - 1:
                    dones = [1 for _ in range(self.num_agents)]
                    self.replay_buffer.push(states, actions, rewards, next_states, dones)
                    episode_rewards.append(episode_reward)
                    print("episode: {}  |  reward: {}  \n".format(episode, np.round(episode_reward, decimals=4)))
                    break
                else:
                    dones = [0 for _ in range(self.num_agents)]
                    self.replay_buffer.push(states, actions, rewards, next_states, dones)
                    states = next_states

                    if len(self.replay_buffer) > batch_size:
                        self.update(batch_size)

ext: py

reason: import numpy


text:
import PIL
import imageio
import numpy as np
import os

from collections import defaultdict
from torch.utils.data import Dataset

from tqdm.autonotebook import tqdm

dir_structure_help = r"""
TinyImageNetPath
├── test
│   └── images
│       ├── test_0.JPEG
│       ├── t...
│       └── ...
├── train
│   ├── n01443537
│   │   ├── images
│   │   │   ├── n01443537_0.JPEG
│   │   │   ├── n...
│   │   │   └── ...
│   │   └── n01443537_boxes.txt
│   ├── n01629819
│   │   ├── images
│   │   │   ├── n01629819_0.JPEG
│   │   │   ├── n...
│   │   │   └── ...
│   │   └── n01629819_boxes.txt
│   ├── n...
│   │   ├── images
│   │   │   ├── ...
│   │   │   └── ...
├── val
│   ├── images
│   │   ├── val_0.JPEG
│   │   ├── v...
│   │   └── ...
│   └── val_annotations.txt
├── wnids.txt
└── words.txt
"""

def download_and_unzip(URL, root_dir):
  error_message = "Download is not yet implemented. Please, go to {URL} urself."
  raise NotImplementedError(error_message.format(URL))

def _add_channels(img, total_channels=3):
  while len(img.shape) < 3:  # third axis is the channels
    img = np.expand_dims(img, axis=-1)
  while(img.shape[-1]) < 3:
    img = np.concatenate([img, img[:, :, -1:]], axis=-1)
  return img

"""Creates a paths datastructure for the tiny imagenet.

DO NOT INSTANTIATE THIS!
This class is protected in order to enable the singleton.

Args:
  root_dir: Where the data is located
  download: Download if the data is not there

Members:
  label_id:
  ids:
  nit_to_words:
  data_dict:

"""
class _TinyImageNetPaths(object):
  def __init__(self, root_dir, download=False):
    if download:
      download_and_unzip('http://cs231n.stanford.edu/tiny-imagenet-200.zip',
                         root_dir)
    train_path = os.path.join(root_dir, 'train')
    val_path = os.path.join(root_dir, 'val')
    test_path = os.path.join(root_dir, 'test')

    wnids_path = os.path.join(root_dir, 'wnids.txt')
    words_path = os.path.join(root_dir, 'words.txt')

    self._make_paths(train_path, val_path, test_path,
                     wnids_path, words_path)

  def _make_paths(self, train_path, val_path, test_path,
                  wnids_path, words_path):
    self.ids = []
    with open(wnids_path, 'r') as idf:
      for nid in idf:
        nid = nid.strip()
        self.ids.append(nid)
    self.nid_to_words = defaultdict(list)
    with open(words_path, 'r') as wf:
      for line in wf:
        nid, labels = line.split('\t')
        labels = list(map(lambda x: x.strip(), labels.split(',')))
        self.nid_to_words[nid].extend(labels)

    self.paths = {
      'train': [],  # [img_path, id, nid, box]
      'val': [],  # [img_path, id, nid, box]
      'test': []  # img_path
    }

    # Get the test paths
    self.paths['test'] = list(map(lambda x: os.path.join(test_path, x),
                                      os.listdir(test_path)))
    # Get the validation paths and labels
    with open(os.path.join(val_path, 'val_annotations.txt')) as valf:
      for line in valf:
        fname, nid, x0, y0, x1, y1 = line.split()
        fname = os.path.join(val_path, 'images', fname)
        bbox = int(x0), int(y0), int(x1), int(y1)
        label_id = self.ids.index(nid)
        self.paths['val'].append((fname, label_id, nid, bbox))

    # Get the training paths
    train_nids = os.listdir(train_path)
    for nid in train_nids:
      anno_path = os.path.join(train_path, nid, nid+'_boxes.txt')
      imgs_path = os.path.join(train_path, nid, 'images')
      label_id = self.ids.index(nid)
      with open(anno_path, 'r') as annof:
        for line in annof:
          fname, x0, y0, x1, y1 = line.split()
          fname = os.path.join(imgs_path, fname)
          bbox = int(x0), int(y0), int(x1), int(y1)
          self.paths['train'].append((fname, label_id, nid, bbox))

r"""Data structure to hold the data paths to the images.
Only used if the images are loaded manually, and not from the pickle.

TODO(z-a-f): Make sure this is not pickled.

Args:
  root_dir: Where the data is located
  download: Download if the data is not there

Members:
  label_id:
  ids:
  nit_to_words:
  data_dict:
"""
class TinyImageNetPaths(object):
  instance = None
  def __new__(cls, *args, **kwargs):
    if not TinyImageNetPaths.instance:
      TinyImageNetPaths.instance = _TinyImageNetPaths(*args, **kwargs)
    return TinyImageNetPaths.instance
  def __getattr__(self, name):
    return getattr(self.instance, name, value)
  def __setattr__(self, name):
    return setattr(self.instance, name, value)

r"""Basic data loader."""
class _Loader:
  def __init__(self, tin_dataset, batch_size):
    self.tin = tin_dataset
    if not self.tin.preload:
      raise NotImplementedError("Cannot use batch loader without preloading")
    self.batch_size = batch_size
  def __len__(self):
    batch_num = len(self.tin.img_data) // self.batch_size
    if self.batch_size * batch_num < len(self.tin.img_data):
      batch_num += 1
    return batch_num
  def __iter__(self):
    idx = 0
    while idx < len(self.tin):
      if self.batch_size == 1:
        yield self.tin[idx]
      else:
        imgs = self.tin.img_data[idx:idx+self.batch_size]
        lbls = None if self.tin.mode == 'test' else self.tin.label_data[idx:idx+self.batch_size]
        sample = {'image': imgs, 'label': lbls}
        if self.tin.transform and self.tin._train:
          sample = self.tin.transform(sample)
        yield self.tin.final_transform(sample)
      idx += self.batch_size

"""Datastructure for the tiny image dataset.

Args:
  root_dir: Root directory for the data
  mode: One of "train", "test", or "val"
  preload: Preload into memory
  load_transform: Transformation to use at the preload time
  transform: Transformation to use at the retrieval time
  download: Download the dataset
  max_samples: Maximum number of samples to get

Members:
  tinp: Instance of the TinyImageNetPaths
  img_data: Image data
  label_data: Label data
"""
class TinyImageNetDataset(Dataset):
  def __init__(self, root_dir, mode='train', preload=True, load_transform=None,
               transform=None, download=False, max_samples=None,
               randomize=False, final_transform=None):
    tinp = TinyImageNetPaths(root_dir, download)
    self.mode = mode
    self.label_idx = 1  # from [image, id, nid, box]
    self.preload = preload
    self.transform = transform
    self.final_transform = final_transform
    if self.final_transform is None:
      self.final_transform = lambda x: x

    self.IMAGE_SHAPE = (64, 64, 3)

    self.img_data = []
    self.label_data = []

    self.max_samples = max_samples
    self.samples = tinp.paths[mode]
    self.samples_num = len(self.samples)

    self._train = (self.mode != 'test')

    if randomize:
      self.samples = np.random.permutation(self.samples)

    if self.max_samples is not None:
      self.samples_num = min(self.max_samples, self.samples_num)
      self.samples = self.samples[:self.samples_num]

    if self.preload:
      load_desc = "Preloading {} data...".format(self.mode)
      self.img_data = np.zeros((self.samples_num,) + self.IMAGE_SHAPE,
                               dtype=np.float32)
      self.label_data = np.zeros((self.samples_num,), dtype=np.int)
      for idx in tqdm(range(self.samples_num), desc=load_desc):
        s = self.samples[idx]
        img = imageio.imread(s[0])
        img = _add_channels(img)
        self.img_data[idx] = img
        if self.mode != 'test':
          self.label_data[idx] = s[self.label_idx]

      if load_transform:
        result = load_transform({'image': self.img_data,
                                 'label': self.label_data})
        self.img_data = result['image']
        self.label_data = result['label']

  def __len__(self):
    return self.samples_num

  def __getitem__(self, idx):
    if self.preload:
      img = self.img_data[idx]
      lbl = None if self.mode == 'test' else self.label_data[idx]
    else:
      s = self.samples[idx]
      img = imageio.imread(s[0])
      lbl = None if self.mode == 'test' else s[self.label_idx]
    img = img.reshape((1, *img.shape))
    lbl = lbl.reshape((1, *lbl.shape))
    sample = {'image': img, 'label': lbl}

    if self.transform and self._train:
      sample = self.transform(sample)
    return self.final_transform(sample)

  r"""Returns an instance of the data loader class that loads batches."""
  def batch_loader(self, batch_size):
    return _Loader(self, batch_size)

  r"""Sets the dataset to the eval mode -- no transformation will be done."""
  def eval(self):
    self._train = False

  r"""Sets the dataset to the train mode -- all transformations will be done."""
  def train(self):
    if self.mode == 'test':
      raise RuntimeError("Cannot train on test data")
    self._train = True

  r"""Sets the data to CUDA mode."""
  def cuda(self):
    self.img_data = self.img_data.cuda()
    self.label_data = self.label_data.cuda()
    return self

  r"""Sets the data to CPU mode."""
  def cpu(self):
    self.img_data = self.img_data.cpu()
    self.label_data = self.label_data.cpu()
    return self


text:
from typing import Union
import numpy as np
from astropy import units as u
from .IPSF import IPSF
from ..sensor.PixelMask import PixelMask
from scipy.optimize import newton, fmin, bisect
from scipy.special import j0, j1
from scipy.signal import fftconvolve
from scipy.integrate import quad
from scipy.interpolate import interp1d

class Airy(IPSF):
    """
    A class for modelling the PSF using an airy disk.
    """

    @u.quantity_input(wl="length", d_aperture="length")
    def __init__(self, f_number: float, wl: u.Quantity, d_aperture: u.Quantity, osf: float, pixel_size: u.Quantity):
        """
        Initialize a new PSF from a airy disk.

        Parameters
        ----------
        f_number : float
            The working focal number of the optical system
        wl : Quantity
            The central wavelength which is used for calculating the PSF
        d_aperture : Quantity
            The diameter of the telescope's aperture.
        osf : float
            The oversampling factor to be used for oversampling the PSF with regards to the pixel size.
        pixel_size : Quantity
            The size of a pixel as length-quantity.
        """
        self.__f_number = f_number
        self.__wl = wl
        self.__d_aperture = d_aperture
        self.__osf = osf
        self.__pixel_size = pixel_size
        self.__psf_jitter = None

    def calcReducedObservationAngle(self, contained_energy: Union[str, int, float],
                                    jitter_sigma: u.Quantity = None, obstruction: float = 0.0) -> u.Quantity:
        """
        Calculate the reduced observation angle in lambda / d_ap for the given contained energy.

        Parameters
        ----------
        contained_energy : Union[str, int, float]
            The percentage of energy to be contained within a circle with the diameter reduced observation angle.
        jitter_sigma : Quantity
            Sigma of the telescope's jitter in arcsec
        obstruction : float
            The central obstruction as ratio A_ob / A_ap

        Returns
        -------
        reduced_observation_angle: Quantity
            The reduced observation angle in lambda / d_ap
        """
        # Calculate the reduced observation angle in lambda / D for the given encircled energy
        reduced_observation_angle = 0 * u.dimensionless_unscaled
        if type(contained_energy) == str:
            # Encircled energy is of type string
            if contained_energy.lower() == "peak":
                # For the peak value of the PSF, the observation angle becomes zero which leads to one exposed
                # pixel later in the code
                reduced_observation_angle = 0 * u.dimensionless_unscaled
            elif contained_energy.lower() == "fwhm":
                # Width of the FWHM of the airy disk
                reduced_observation_angle = 1.028
                if not np.isclose(obstruction, 0.0):
                    # Use obstructed airy disk
                    reduced_observation_angle = newton(lambda y: self.__airy(np.pi * y, np.sqrt(obstruction)) - 0.5,
                                                       reduced_observation_angle / 2) * 2
                    contained_energy = "fwhm"
            elif contained_energy.lower() == "min":
                # Width of the first minimum of the airy disk
                reduced_observation_angle = 1.22 * 2
                contained_energy = 0.8377 * u.dimensionless_unscaled
                if not np.isclose(obstruction, 0.0):
                    # Use obstructed airy disk
                    reduced_observation_angle = fmin(lambda y: self.__airy(np.pi * y, np.sqrt(obstruction)),
                                                     reduced_observation_angle / 2, disp=False)[0] * 2
                    contained_energy = self.__airy_int(np.pi * reduced_observation_angle / 2,
                                                       np.sqrt(obstruction)) * u.dimensionless_unscaled
        else:
            # Calculate the width numerically from the integral of the airy disk
            contained_energy = contained_energy / 100 * u.dimensionless_unscaled
            reduced_observation_angle = 2 * bisect(
                lambda y: self.__airy_int(np.pi * y, np.sqrt(obstruction)) - contained_energy.value, 0, 100)
        if jitter_sigma is not None and (isinstance(contained_energy, u.Quantity) or isinstance(contained_energy, str)
                                         and contained_energy.lower() == "fwhm"):
            # Convert jitter to reduced observation angle in lambda / d_ap
            jitter_sigma = jitter_sigma.to(u.rad).value * self.__d_aperture / self.__wl.to(u.m)
            # Calculate necessary grid length to accommodate the psf and 3-sigma of the gaussian
            grid_width = (reduced_observation_angle / 2 + 3 * jitter_sigma.value)
            # Calculate the reduced observation angle of a single detector pixel
            reduced_observation_angle_pixel = (self.__pixel_size / (
                    self.__f_number * self.__d_aperture) * self.__d_aperture / self.__wl).decompose()
            # Calculate the width of each grid element
            dx = reduced_observation_angle_pixel.value / self.__osf
            # Calculate the necessary number of points on the grid
            n_points = np.ceil(grid_width / dx)
            # Calculate the corresponding x-coordinates of each grid element
            x = np.arange(1, n_points + 1) * dx
            # Calculate the psf from an airy disk for each element on the grid
            psf = self.__airy(np.pi * x, np.sqrt(obstruction))
            # Calculate the integral of the undisturbed airy disk in order to scale the result of the convolution
            # This corresponds to an integration in polar coordinates: I(r)*r dr dphi
            total = np.sum(psf * x) * dx * 2 * np.pi
            # Mirror the PSF to the negative x-domain
            psf = np.concatenate((np.flip(psf), np.array([1]), psf))
            # Calculate a gaussian kernel
            kernel = 1 / (2 * np.pi * jitter_sigma.value ** 2) * np.exp(- x ** 2 / (2 * jitter_sigma.value ** 2))
            # Mirror the kernel to the negative x-domain
            kernel = np.concatenate((np.flip(kernel), np.array([1 / (2 * np.pi * jitter_sigma.value ** 2)]), kernel))
            # Normalize the kernel
            kernel = kernel / np.sum(kernel)
            # Convolve the PSF with gaussian kernel
            psf = fftconvolve(psf, kernel, mode="full")
            # Reduce the PSF to the positive x-domain
            psf = psf[int((psf.shape[0] - 1) / 2):]
            # Scale the integral of the disturbed PSF equal to the undisturbed PSF
            # This corresponds to an integration in polar coordinates: I(r)*r dr dphi
            psf = psf / (np.sum(psf * np.arange(psf.shape[0]) * dx) * dx * 2 * np.pi) * total

            self.__psf_jitter = psf
            if isinstance(contained_energy, str) and contained_energy.lower() == "fwhm":
                reduced_observation_angle = np.argmax(psf < psf[0] / 2) * reduced_observation_angle_pixel.value / \
                                            self.__osf * 2
            else:
                # Calculate the rolling integral of the PSF
                # This corresponds to an integration in polar coordinates: I(r)*r dr dphi
                psf_int = np.cumsum(psf * np.arange(psf.shape[0]) * dx) * dx * 2 * np.pi
                # Calculate the percentage of encircled energy by dividing by the infinite integral of the undisturbed
                # airy disk
                psf_int = psf_int / (4 / np.pi * 1 / (1 - obstruction))
                # Calculate the reduced observation angle
                reduced_observation_angle = np.argmax(psf_int > contained_energy) * dx * 2

        return reduced_observation_angle * u.dimensionless_unscaled

    @staticmethod
    def __airy(x: Union[float, np.ndarray], obstruction: float = None) -> Union[float, np.ndarray]:
        """
        Calculate function values of the airy disk

        Parameters
        ----------
        x : Union[float, np.ndarray]
            radial coordinate to calculate the function value for.
        obstruction : float
            The linear central obstruction ratio of the aperture.

        Returns
        -------
        res : Union[float, np.ndarray]
            The function values of the airy disk at the given coordinates
        """
        # Standardize input values
        if not isinstance(x, np.ndarray):
            x = np.array([x])
        # Initialize return values and assign values for the singularity at x=0
        res = np.zeros(x.shape)
        res[np.isclose(x, 0.0)] = 1.0
        x_temp = x[np.invert(np.isclose(x, 0.0))]
        if obstruction and not np.isclose(obstruction, 0.0):
            # Use obstructed airy disk
            # See also https://en.wikipedia.org/wiki/Airy_disk#Obscured_Airy_pattern
            res[np.invert(np.isclose(x, 0.0))] = 1 / (1 - obstruction ** 2) ** 2 * (
                    2 * (j1(x_temp) - obstruction * j1(obstruction * x_temp)) / x_temp) ** 2
        else:
            # Use unobstructed airy disk
            # See also https://en.wikipedia.org/wiki/Airy_disk#Mathematical_formulation
            res[np.invert(np.isclose(x, 0.0))] = (2 * j1(x_temp) / x_temp) ** 2
        # Unbox arrays of length 1
        if len(res.shape) == 1 and len(res) == 1:
            res = res[0]
        return res

    @staticmethod
    def __airy_int(x: float, obstruction: float = None) -> float:
        """
        Calculate the integral of the airy disk from 0 to x.

        Parameters
        ----------
        x : float
            The upper limit for the integration.
        obstruction : float
            The linear central obstruction ratio of the aperture.

        Returns
        -------
        res : float
            The integral of the airy disk.
        """
        if np.isclose(x, 0.0):
            # Short circuit for an integration-range of length 0
            return 0.
        else:
            if obstruction and not np.isclose(obstruction, 0.0):
                # Use integral of obstructed airy disk
                # See also https://en.wikipedia.org/wiki/Airy_disk#Obscured_Airy_pattern
                return 1 / (1 - obstruction ** 2) * (1 - j0(x) ** 2 - j1(x) ** 2 + obstruction ** 2 * (
                        1 - j0(obstruction * x) ** 2 - j1(obstruction * x) ** 2) - 4 * obstruction * quad(
                    lambda y: j1(y) * j1(obstruction * y) / y, 0, x, limit=100, epsrel=1e-6)[0])
            else:
                # Use unobstructed airy disk
                # See also https://en.wikipedia.org/wiki/Airy_disk#Mathematical_formulation
                return 1 - j0(x) ** 2 - j1(x) ** 2

    def mapToPixelMask(self, mask: PixelMask, jitter_sigma: u.Quantity = None, obstruction: float = 0.0) -> PixelMask:
        """
        Map the integrated PSF values to a sensor grid.

        Parameters
        ----------
        obstruction
        mask : PixelMask
            The pixel mask to map the values to. The values will only be mapped onto entries with the value 1.
        jitter_sigma : Quantity
            Sigma of the telescope's jitter in arcsec
        obstruction : float
            The central obstruction as ratio A_ob / A_ap

        Returns
        -------
        mask : PixelMask
            The pixel mask with the integrated PSF values mapped onto each pixel.
        """
        # Calculate the indices of all non-zero elements of the mask
        y_ind, x_ind = np.nonzero(mask)
        # Extract a rectangle containing all non-zero values of the mask
        mask_red = mask[y_ind.min():(y_ind.max() + 1), x_ind.min():(x_ind.max() + 1)]
        # Calculate the new PSF-center indices of the reduced mask
        psf_center_ind = [mask.psf_center_ind[0] - y_ind.min(), mask.psf_center_ind[1] - x_ind.min()]
        # Oversample the reduced mask
        mask_red_os = self._rebin(mask_red, self.__osf).view(PixelMask)
        # Calculate the new PSF-center indices of the reduced mask
        psf_center_ind = [(x + 0.5) * self.__osf - 0.5 for x in psf_center_ind]

        reduced_observation_angle_pixel = (mask.pixel_size / (
                self.__f_number * self.__d_aperture) * self.__d_aperture / self.__wl).decompose()
        x_mesh, y_mesh = np.meshgrid(
            (np.arange(mask_red_os.shape[1]) - psf_center_ind[
                1]) * reduced_observation_angle_pixel.value / self.__osf,
            (np.arange(mask_red_os.shape[0]) - psf_center_ind[
                0]) * reduced_observation_angle_pixel.value / self.__osf)
        dist = np.sqrt(x_mesh ** 2 + y_mesh ** 2)
        if jitter_sigma is None:
            res = self.__airy(dist * np.pi, np.sqrt(obstruction))
        else:
            if self.__psf_jitter is None:
                self.calcReducedObservationAngle("fwhm", jitter_sigma, obstruction)
            psf_jitter = self.__psf_jitter
            x = np.arange(psf_jitter.shape[0]) * reduced_observation_angle_pixel.value / self.__osf
            psf_interp = interp1d(x=x, y=psf_jitter, kind='cubic', copy=False, bounds_error=False,
                                  fill_value="extrapolate")
            res = psf_interp(dist)

        res = self._rebin(res, 1 / self.__osf)
        res = (mask_red * res).view(np.ndarray)
        # Integrate the reduced mask and divide by the indefinite integral to get relative intensities
        res = res * (reduced_observation_angle_pixel.value / self.__osf) ** 2 / (4 / np.pi) * (1 - obstruction)
        # reintegrate the reduced mask into the complete mask
        mask[y_ind.min():(y_ind.max() + 1), x_ind.min():(x_ind.max() + 1)] = res
        return mask

reason: import numpy,from scipy,from astropy


text:
# PsykoDev
import os
import platform
import numpy as np
from time import sleep
from PIL import ImageGrab
from game_control import *
from predict import predict
from game_control import *
from keras.models import model_from_json

def main():
    # Get Model:
    model_file = open('Data/Model/model.json', 'r')
    model = model_file.read()
    model_file.close()
    model = model_from_json(model)
    model.load_weights("Data/Model/weights.h5")

    print('AI has started!')

    while 1:
        # Get screenshot:
        screen = ImageGrab.grab()
        # Image to numpy array:
        screen = np.array(screen)
        # 4 channel(PNG) to 3 channel(JPG)
        Y = predict(model, screen)
        if Y == [0,0,0,0]:
            # Not action
            continue
        elif Y[0] == -1 and Y[1] == -1:
            # Only keyboard action.
            key = get_key(Y[3])
            if Y[2] == 1:
                # Press:
                press(key)
            else:
                # Release:
                release(key)
        elif Y[2] == 0 and Y[3] == 0:
            # Only mouse action.
            click(Y[0], Y[1])
        else:
            # Mouse and keyboard action.
            # Mouse:
            click(Y[0], Y[1])
            # Keyboard:
            key = get_key(Y[3])
            if Y[2] == 1:
                # Press:
                press(key)
            else:
                # Release:
                release(key)

if __name__ == '__main__':
    main()

max_stars_repo_path: AI/Python/AI Play Game/ai.py

max_stars_repo_name: XanTiuM-Dev/kiwi-dev

max_issues_repo_path: AI/Python/AI Play Game/ai.py

max_issues_repo_name: XanTiuM-Dev/kiwi-dev

max_forks_repo_path: AI/Python/AI Play Game/ai.py

max_forks_repo_name: XanTiuM-Dev/kiwi-dev


text:
import boto3
from tensorflow.python.keras.preprocessing import image
from tensorflow.python.keras.applications.resnet50 import preprocess_input, decode_predictions
import numpy as np
import uuid
from time import time

from squeezenet import SqueezeNet

s3_client = boto3.client('s3')

tmp = "/data/"

def predict(img_local_path):
    start = time()
    model = SqueezeNet(weights='imagenet')
    img = image.load_img(img_local_path, target_size=(227, 227))
    x = image.img_to_array(img)
    x = np.expand_dims(x, axis=0)
    x = preprocess_input(x)
    preds = model.predict(x)
    res = decode_predictions(preds)
    latency = time() - start
    return latency, res

def handler(event):
    #input_bucket = event['input_bucket']
    #object_key = event['object_key']

    #model_object_key = event['model_object_key']  # example : squeezenet_weights_tf_dim_ordering_tf_kernels.h5
    #model_bucket = event['model_bucket']

    #download_path = tmp + '{}{}'.format(uuid.uuid4(), object_key)
    #s3_client.download_file(input_bucket, object_key, download_path)

    #model_path = tmp + '{}{}'.format(uuid.uuid4(), model_object_key)
    #s3_client.download_file(model_bucket, model_object_key, model_path)

    download_path = event['image_path']
    latency, result = predict(download_path)

    _tmp_dic = {x[1]: {'N': str(x[2])} for x in result[0]}

    return latency

def invokeHandler():
    startTime = int(round(time() * 1000))
    ret = handler({'image_path':'/data/animal-dog.jpg'})
    retTime = int(round(time() * 1000))

    output = {'results': ret,
        'startTime': startTime,
        'retTime' : retTime,
        'invokeTime': startTime
        }
    logf = open("log.txt", "w")
    logf.write(str(output))

    #print(output)

if __name__ == "__main__":
    invokeHandler()


text:
from av import AudioFrame, VideoFrame, AudioLayout, AudioFormat
import numpy

_aformat_dtypes = {
    'dbl': '<f8',
    'dblp': '<f8',
    'flt': '<f4',
    'fltp': '<f4',
    's16': '<i2',
    's16p': '<i2',
    's32': '<i4',
    's32p': '<i4',
    'u8': 'u1',
    'u8p': 'u1',
}

_aformats = {
    '<f8': 'dblp',
    '<f4': 'fltp',
    '<i2': 's16p',
    '<i4': 's32p',
    'u1': 'u8p',
}

def _AFrameToNDArray(frame):
    """
    A reimplementation of frame.to_ndarray that overcomes the segmentation
    fault that occurs with 8-channel audio frames, and returns an array with
    shape (frame.samples, nb_channels).
    """

    try:
        dtype = numpy.dtype(_aformat_dtypes[frame.format.name])
    except KeyError:
        raise ValueError(
            ("Conversion from {!r} format to numpy array is not"
                " supported.").format(frame.format.name))

    nb_channels = len(frame.layout.channels)

    if frame.format.is_planar:
        count = frame.samples
    else:
        count = frame.samples * nb_channels

    # convert and return data
    arrays = []

    for i, x in enumerate(frame.planes):
        if i >= nb_channels:
            break

        arrays.append(numpy.frombuffer(x, dtype=dtype, count=count))

    if frame.format.is_planar:
        return numpy.vstack(arrays).transpose()

    array = numpy.vstack(arrays)
    return array.reshape(frame.samples, nb_channels)

def _VFrameToNDArray(frame):
    A = frame.to_ndarray()
    H = frame.height
    W = frame.width

    if frame.format.name in ('yuv420p', 'yuvj420p'):
        Y = A[:H]
        UV = A[H:].reshape(H*W//2)
        U = UV[:H*W//4].reshape(H//2, W//2)
        V = UV[H*W//4:].reshape(H//2, W//2)
        return (Y, U, V)

    elif frame.format.name == "yuyv422":
        Y = A[:, :, 0]
        U = A[:, ::2, 1]
        V = A[:, 1::2, 1]
        return (Y, U, V)

    return A

def toNDArray(frame):
    if isinstance(frame, AudioFrame):
        return _AFrameToNDArray(frame)

    elif isinstance(frame, VideoFrame):
        return _VFrameToNDArray(frame)

    else:
        raise TypeError(
            "Expected AudioFrame or VideoFrame, got"
            f" {frame.__class__.__name__} instead.")

def toAFrame(array, layout=None):
    """
    Construct a frame from a numpy array.
    """

    assert array.ndim == 1 or (
        array.ndim == 2 and 1 <= array.shape[1] <= 8), "Arrays with"\
        " nb_channels > 8 unsupported"

    if layout is None:
        if array.ndim <= 1 or array.shape[1] == 1:
            layout = "mono"

        elif array.shape[1] == 2:
            layout = "stereo"

        elif array.shape[1] == 3:
            layout = "2.1"

        elif array.shape[1] == 4:
            layout = "4.0"

        elif array.shape[1] == 5:
            layout = "4.1"

        elif array.shape[1] == 6:
            layout = "5.1(side)"

        elif array.shape[1] == 7:
            layout = "6.1"

        elif array.shape[1] == 8:
            layout = "7.1"

    nb_channels = len(AudioLayout(layout).channels)
    samples = array.shape[0]
    assert array.shape[1] == nb_channels, f"Layout {layout} requires"\
        f" {nb_channels}, not {array.shape[1]}."

    # map numpy type to avcodec type

    try:
        format = _aformats[array.dtype.str]
    except KeyError:
        raise ValueError(
            f'Conversion from numpy array with dtype `{array.dtype.str}`"\
                " is not yet supported')

    if AudioFormat(format).is_planar:
        array = array.transpose().copy("c")
    else:
        array = array.reshape(1, samples*nb_channels).copy("c")

    frame = AudioFrame(format=format, layout=layout, samples=samples)

    for i, plane in enumerate(frame.planes):
        if i >= nb_channels:
            break

        plane.update(array[i, :])

    return frame

def _YUVtoVFrame(Y, U, V, format=None):
    assert Y.ndim == 2, "Y.ndim must be 2."
    assert U.ndim == 2, "U.ndim must be 2."
    assert V.ndim == 2, "V.ndim must be 2."
    assert Y.shape[0] % 2 == 0, "Y.shape[0] must be even."
    assert Y.shape[1] % 2 == 0, "Y.shape[1] must be even."
    assert U.shape == V.shape, "V.shape must be equal to U.shape."

    H, W = Y.shape
    h, w = U.shape

    if 2*h == H and 2*w == W:
        if format is None:
            format = "yuv420p"

        A = numpy.concatenate(
            (Y.reshape(H*W), U.reshape(h*w),
             V.reshape(h*w))).reshape(3*H//2, W)
        return VideoFrame.from_ndarray(A, format=format)

    elif h == H and 2*w == W:
        if format is None:
            format = "yuyv422"

        UV = numpy.concatenate(
            (U, V), axis=1).reshape(
                -1, 2, w).swapaxes(1, 2).reshape(-1, W)
        A = numpy.moveaxis((Y, UV), 0, 2)
        return VideoFrame.from_ndarray(A, format=format)

def toVFrame(array, format=None):
    if isinstance(array, tuple) and len(array) == 3:
        return _YUVtoVFrame(*array, format=format)

    if format is not None:
        if array.ndim >= 3 and array.shape[2] == 4:
            format = "rgba"

        elif array.ndim >= 3 and array.shape[2] == 3:
            format = "rgb24"

        elif array.ndim >= 3 and array.shape[2] == 1:
            format = "gray8"

    return VideoFrame.from_ndarray(array)

def aconvert(frame, format):
    if frame.format.name == format:
        return frame

    A = toNDArray(frame)
    dtype = numpy.dtype(_aformat_dtypes[format])

    if format in ("fltp", "dblp", "flt", "dbl"):
        if A.dtype in (numpy.int8, numpy.uint8):
            A = A/2**7

        elif A.dtype in (numpy.int16, numpy.uint16):
            A = A/2**15

        elif A.dtype in (numpy.int32, numpy.uint32):
            A = A/2**31

        elif A.dtype in (numpy.int64, numpy.uint64):
            A = A/2**63

        A = numpy.array(A, dtype=dtype)

    elif format in ("s16", "s16p"):
        if A.dtype in (numpy.float32, numpy.float64):
            A = A*2**15

        elif A.dtype in (numpy.int32, numpy.uint32):
            A = A/2**16

        elif A.dtype in (numpy.int8, numpy.uint8):
            A = A*2**8

        A = numpy.array(A.clip(min=-2**15, max=2**15 - 1), dtype=dtype)

    elif format in ("s32", "s32p"):
        if A.dtype in (numpy.float32, numpy.float64):
            A = A*2**31

        elif A.dtype in (numpy.int16, numpy.uint16):
            A = A*2**16

        elif A.dtype in (numpy.int8, numpy.uint8):
            A = A*2**24

        A = numpy.array(A.clip(min=-2**31, max=2**31 - 1), dtype=dtype)

    elif format in ("u8", "u8p"):
        if A.dtype in (numpy.float32, numpy.float64):
            A = A*2**7

        elif A.dtype in (numpy.int16, numpy.uint16):
            A = A/2**8

        elif A.dtype in (numpy.int32, numpy.uint32):
            A = A/2**24

        A = numpy.array(A.clip(min=0, max=2**8 - 1), dtype=dtype)

    newframe = toAFrame(A, layout=frame.layout.name)
    newframe.pts = frame.pts
    newframe.rate = frame.rate
    newframe.time_base = frame.time_base
    return newframe

max_stars_repo_name: shersonb/python-transcode

max_issues_repo_name: shersonb/python-transcode

max_forks_repo_name: shersonb/python-transcode

reason: import numpy,from numpy


text:
# 딥러닝을 구동하는 데 필요한 케라스 함수를 불러옴
from keras.models import Sequential, load_model
from keras.layers.core import Dense
from sklearn.preprocessing import LabelEncoder

# 필요한 라이브러리를 불러옴
import pandas as pd
import numpy as np
import tensorflow as tf

#######################################################################
# seed 값 생성
seed = 0
np.random.seed(seed)
tf.set_random_seed(seed)

# 데이터 불러오기
raw_data = pd.read_csv('dataset1/wine.csv', header = None)
data = raw_data.copy()
data = data.sample(frac = 1)

# 데이터 내용 확인하기
data.head()
data.info()
data.describe()

# 데이터 나누기
dataset = data.values
X = dataset[:,0:12]
Y = dataset[:,12]

# # 학습셋과 테스트셋을 나눔
# from sklearn.model_selection import train_test_split
#
# X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.3, random_state = seed)

# 모델 설계하기
model = Sequential()
model.add(Dense(30, input_dim = 12, activation = 'relu'))
model.add(Dense(12, activation = 'relu'))
model.add(Dense(8, activation = 'relu'))
model.add(Dense(1, activation = 'sigmoid'))

# 모델 컴파일하기
model.compile(loss = 'binary_crossentropy',
              optimizer = 'adam',
              metrics = ['accuracy'])

# 모델 업데이트하기
import os

MODEL_DIR = 'Study/model1/'
if not os.path.exists(MODEL_DIR):
    os.mkdir(MODEL_DIR)

modelpath = "Study/model1/{epoch:02d}-{val_loss:.4f}.hdf5"  # 에폭횟수-오차.hdf5

# 모델 업데이트 및 저장
from keras.callbacks import ModelCheckpoint

checkpointer = ModelCheckpoint(filepath = modelpath,
                               monitor = 'val_loss',     # 테스트셋의 오차
                               verbose = 1,              # 해당함수의 진행 사항 출력
                               save_best_only = True)    # 오차가 줄어들었을때만 저장

# 학습 자동 중단 설정
from keras.callbacks import EarlyStopping

early_stopping_callback = EarlyStopping(monitor = 'val_loss',
                                        patience = 100)

# 모델 실행
history = model.fit(X, Y,
                    validation_split = 0.2,
                    epochs = 3500,
                    batch_size = 500,
                    verbose = 0,
                    callbacks = [early_stopping_callback, checkpointer])

# # 테스트를 위해 메모리 내의 모델을 삭제
# del model
#
# # 모델을 새로 불러옴
# model = load_model('Study/model1/1015-0.0393.hdf5')

# 결과 출력
# print('\ Accuracy: %.4f' % (model.evaluate(X_test, Y_test)[1]))

# y_vloss에 테스트셋으로 실험 결과의 오차 값을 저장
y_vloss = history.history['val_loss']

# y_acc에 학습셋으로 측정한 정확도의 값을 저장
y_acc = history.history['accuracy']

# x 값을 지정하고 정확도를 파란색으로, 오차를 빨간색으로 표시
import matplotlib.pyplot as plt

x_len = np.arange(len(y_acc))
plt.plot(x_len, y_vloss, 'o', c = 'red', markersize = 3)
plt.plot(x_len, y_acc, 'o', c = 'blue', markersize = 3)


text:
import pandas as pd
import numpy as np

np.random.seed(101)

df = pd.DataFrame(np.random.randn(5, 4), index='A B C D E'.split(), columns='W X Y Z'.split())

df
print(df)
print('\n')
print(df['W'])
print('\n')
print(df[['W', 'Z']])

max_stars_repo_path: Pequenos programas/tg/dataScience2.py

max_issues_repo_path: Pequenos programas/tg/dataScience2.py

max_forks_repo_path: Pequenos programas/tg/dataScience2.py


text:
import json
from typing import Union
import numpy as np

# Modified from
# https://gist.github.com/jannismain/e96666ca4f059c3e5bc28abb711b5c92#file-compactjsonencoder-py
# to handle more classes

class CompactJSONEncoder(json.JSONEncoder):
    """A JSON Encoder that puts small containers on single lines."""

    CONTAINER_TYPES = (list, tuple, dict)
    """Container datatypes include primitives or other containers."""

    MAX_ITEMS = 6
    """Maximum number of items in container that might be put on single line."""

    INDENTATION_CHAR = " "

    def __init__(self, max_width = 80, precise = False, *args, **kwargs):
        self.max_width = max_width
        self.precise = precise
        super().__init__(*args, **kwargs)
        self.indentation_level = 0

    def encode(self, o):
        """Encode JSON object *o* with respect to single line lists."""
        if isinstance(o, (list, tuple)):
            if self._put_on_single_line(o):
                return "[" + ", ".join(self.encode(el) for el in o) + "]"
            else:
                self.indentation_level += 1
                output = [self.indent_str + self.encode(el) for el in o]
                self.indentation_level -= 1
                return "[\n" + ",\n".join(output) + "\n" + self.indent_str + "]"
        elif isinstance(o, dict):
            if o:
                if self._put_on_single_line(o):
                    return "{ " + ", ".join(f"{self.encode(k)}: {self.encode(el)}" for k, el in o.items()) + " }"
                else:
                    self.indentation_level += 1
                    output = [self.indent_str + f"{json.dumps(k)}: {self.encode(v)}" for k, v in o.items()]
                    self.indentation_level -= 1
                    return "{\n" + ",\n".join(output) + "\n" + self.indent_str + "}"
            else:
                return "{}"
        elif isinstance(o, float):  # Use scientific notation for floats, where appropiate
            if self.precise:
                return format(o, ".12g")
            else:
                return format(o, "g")
        elif isinstance(o, str):  # escape newlines
            o = o.replace("\n", "\\n")
            return f'"{o}"'
        elif isinstance(o, np.int32):
            return json.dumps(int(o))
        elif isinstance(o, np.bool_):
            return json.dumps(bool(o))
        elif isinstance(o, np.ndarray):
            return self.encode(list(o))
        else:
            return json.dumps(o)

    def _put_on_single_line(self, o):
        return self._primitives_only(o) and len(o) <= self.MAX_ITEMS and len(str(o)) - 2 <= self.max_width

    def _primitives_only(self, o: Union[list, tuple, dict]):
        if isinstance(o, (list, tuple)):
            return not any(isinstance(el, self.CONTAINER_TYPES) for el in o)
        elif isinstance(o, dict):
            return not any(isinstance(el, self.CONTAINER_TYPES) for el in o.values())

    @property
    def indent_str(self) -> str:
        return self.INDENTATION_CHAR*(self.indentation_level*self.indent)


text:
import numpy as np
import math
import networkx as nx
import scipy.spatial
import random

def generate_square(num_nodes, radius, side_len):
    """Generate a network into a square shaped area using random placement and unit disk graph communication model.

    :param num_nodes: number of sensors
    :param radius: communication radius of the sensors
    :param side_len: length of the sides of the square
    :return: A list of generated node positions and the network connectivity graph
    """

    # try to generate a random graph with these parameters 1000 times
    # break if the random graph is connected
    for X in range(0, 1000):
        # make graph, add nodes and random positions
        connectivity_graph = nx.Graph()
        connectivity_graph.add_nodes_from(range(0, num_nodes))
        positions = dict()
        positions_ = list()
        for i in range(0, num_nodes):
            positions[i] = [np.random.rand()*side_len, np.random.rand()*side_len]
            positions_.append(positions[i])

        # calculate pairwise distances
        y = scipy.spatial.distance.pdist(np.asarray(positions_), 'euclidean')
        dist_matrix = scipy.spatial.distance.squareform(y)

        # add edges based on radius
        for i in range(0, num_nodes-1):
            for j in range(i, num_nodes):
                if dist_matrix[i, j] <= radius and i != j:
                    connectivity_graph.add_edge(i, j)
        # if the generated network is not connected, generate a new one
        if nx.is_connected(connectivity_graph):
            return positions, connectivity_graph

    assert False, 'Error: Unable to create connected graph using the provided parameters!\n'

def generate_square_quasi(num_nodes, radius, side_len, p=0.5):
    """Generate a network into a square shaped area using random placement and QUD graph communication model.

    :param num_nodes: number of sensors
    :param radius: communication radius of the sensors
    :param side_len: length of the sides of the square
    :param p: parameter for the quasi communication model
    :return: A list of generated node positions and the network connectivity graph
    """
    radius2 = radius
    radius1 = radius * p

    # try to generate a random graph with these parameters 1000 times
    # break if the random graph is connected
    for X in range(0, 1000):
        # make graph, add nodes and random positions
        connectivity_graph = nx.Graph()
        connectivity_graph.add_nodes_from(range(0, num_nodes))
        positions = dict()
        positions_ = list()
        for i in range(0, num_nodes):
            positions[i] = [np.random.rand()*side_len, np.random.rand()*side_len]
            positions_.append(positions[i])

        # calculate pairwise distances
        y = scipy.spatial.distance.pdist(np.asarray(positions_), 'euclidean')
        dist_matrix = scipy.spatial.distance.squareform(y)

        # add edges based on radius and param p
        for i in range(0, num_nodes-1):
            for j in range(i, num_nodes):
                if dist_matrix[i, j] <= radius1 and i != j:
                    connectivity_graph.add_edge(i, j)
                if radius1 < dist_matrix[i, j] <= radius2:
                    asd = np.random.rand()
                    alpha = radius1 / radius2
                    prob = (alpha/(1 - alpha)) * ((radius2/dist_matrix[i, j]) - 1)

                    if asd < prob:
                        connectivity_graph.add_edge(i, j)

        # if the generated network is not connected, generate a new one
        if nx.is_connected(connectivity_graph):
            return positions, connectivity_graph

    assert False, 'Error: Unable to create connected graph using the provided parameters!\n'

def generate_grid(num_nodes, radius, side_len, noise=0.75):
    """Generate a network into a square shaped area using perturbed grid placement and UDG communication model.

    :param num_nodes: number of sensors
    :param radius: communication radius of the sensors
    :param side_len: length of the sides of the square
    :param noise: noise used for perturbation
    :return: A list of generated node positions and the network connectivity graph
    """
    assert (math.sqrt(num_nodes).is_integer()), 'the number of sensors must be a square number'

    # try to generate a random graph with these parameters 1000 times
    # break if the random graph is connected
    for X in range(0, 1000):
        h = int(math.sqrt(num_nodes))
        node_dist = side_len / h
        max_noise = node_dist * noise

        connectivity_graph = nx.Graph()
        connectivity_graph.add_nodes_from(range(0, num_nodes))
        positions_ = list()
        positions = dict()

        # deploy network on grid
        for i in range(0, h):
            for j in range(0, h):
                rx = random.uniform(-1*max_noise, max_noise)
                ry = random.uniform(-1*max_noise, max_noise)
                positions[i + j * h] = [i * node_dist + rx, j * node_dist + ry]
                positions_.append(positions[i + j * h])

        y = scipy.spatial.distance.pdist(np.asarray(positions_), 'euclidean')
        dist_matrix = scipy.spatial.distance.squareform(y)

        # add edges based on radius
        for i in range(0, num_nodes-1):
            for j in range(i, num_nodes):
                if dist_matrix[i, j] <= radius and i != j:
                    connectivity_graph.add_edge(i, j)

        if nx.is_connected(connectivity_graph):
            return positions, connectivity_graph

    assert False, 'Error: the graph is not connected with these parameters!\n'

def generate_grid_quasi(num_nodes, radius, side_len, p=0.5, noise=0.75):
    """Generate a network into a square shaped area using perturbed grid placement and QUDG communication model.

    :param num_nodes: number of sensors
    :param radius: communication radius of the sensors
    :param side_len: length of the sides of the square
    :param p: parameter for the quasi communication model
    :param noise: noise used for perturbation
    :return: A list of generated node positions and the network connectivity graph
    """
    assert (math.sqrt(num_nodes).is_integer()), 'Error: the number of sensors must be a square number\n'

    radius2 = radius
    radius1 = radius * p

    # try to generate a random graph with these parameters 1000 times
    # break if the random graph is connected
    for X in range(0, 1000):
        h = int(math.sqrt(num_nodes))
        node_dist = side_len / h
        max_noise = node_dist*noise

        connectivity_graph = nx.Graph()
        connectivity_graph.add_nodes_from(range(0, num_nodes))
        positions_ = list()
        positions = dict()

        # deploy network on grid
        for i in range(0, h):
            for j in range(0, h):
                rx = random.uniform(-1*max_noise, max_noise)  # max_noise*np.random.rand() - max_noise
                ry = random.uniform(-1*max_noise, max_noise)  # max_noise*np.random.rand() - max_noise
                positions[i + j * h] = [i * node_dist + rx, j * node_dist + ry]
                positions_.append(positions[i + j * h])

        y = scipy.spatial.distance.pdist(np.asarray(positions_), 'euclidean')
        dist_matrix = scipy.spatial.distance.squareform(y)

        # add edges based on radius
        for i in range(0, num_nodes-1):
            for j in range(i, num_nodes):
                if dist_matrix[i, j] <= radius1 and i != j:
                    connectivity_graph.add_edge(i, j)
                if radius1 < dist_matrix[i, j] <= radius2:
                    asd = np.random.rand()
                    alpha = radius1 / radius2
                    prob = (alpha/(1 - alpha)) * ((radius2/dist_matrix[i, j]) - 1)
                    if asd < prob:
                        connectivity_graph.add_edge(i, j)

        if nx.is_connected(connectivity_graph):
            return positions, connectivity_graph

    assert False, 'Error: Unable to create connected graph using the provided parameters!\n'

reason: import numpy,import scipy,import networkx


text:
# Import
import vegapy
import os
import astropy.units as u
import numpy as np

# Settings
verbose = 0
visual = 0

# Initialize
PATH, FILE = os.path.split(__file__)
SOURCE_PATH = os.path.join(PATH, '../source/')
print('>>> TEST: {}'.format(FILE))

# Initilization tests
det = vegapy.Detector((64, 64), pixel_size=0.01*u.arcsec, randomkeywortwithoutmeaning=3)

det_RON = vegapy.Detector((64, 64), pixel_size=0.01*u.arcsec, readout_noise=35*u.electron/u.pix)

exposure_array = np.ones((64, 64)) * u.ph / u.s
RO = det_RON(exposure_array, integration_time=200*u.ms, target_FoV=det_RON.FoV)
if verbose:
	print(RO)

try:
	det.__dict__
	print(">>> TEST DETECTOR: Successful")
except:
	print(">>> TEST DETECTOR: NOT successful")

max_stars_repo_name: felixbosco/vegapy

max_issues_repo_name: scivision/vegapy

max_forks_repo_name: scivision/vegapy

reason: import numpy,import astropy


text:
import torch
from torch.utils.data import TensorDataset
import pytest
import numpy as np

from knodle.trainer.wscrossweigh.data_splitting_by_rules import get_rules_sample_ids, get_samples_labels_idx_by_rule_id

@pytest.fixture(scope='session')
def get_test_data():

    rule_assignments_t = np.array([[1, 0, 0],
                                   [1, 0, 0],
                                   [0, 1, 0],
                                   [0, 1, 0],
                                   [0, 0, 1],
                                   [0, 0, 1],
                                   [0, 0, 1]])

    inputs_x = TensorDataset(torch.Tensor(np.array([[0, 0, 0, 0, 0, 0, 0, 0],
                                                    [1, 1, 1, 1, 1, 1, 1, 1],
                                                    [2, 2, 2, 2, 2, 2, 2, 2],
                                                    [3, 3, 3, 3, 3, 3, 3, 3],
                                                    [4, 4, 4, 4, 4, 4, 4, 4],
                                                    [5, 5, 5, 5, 5, 5, 5, 5],
                                                    [6, 6, 6, 6, 6, 6, 6, 6],
                                                    [7, 7, 7, 7, 7, 7, 7, 7]])))

    rule_matches_z = np.array([[0, 1, 1, 0, 0, 0, 0],
                               [1, 0, 1, 1, 0, 0, 0],
                               [0, 0, 0, 0, 1, 0, 0],
                               [0, 0, 0, 0, 0, 0, 1],
                               [0, 0, 0, 0, 0, 1, 0],
                               [1, 0, 1, 0, 0, 0, 0],
                               [1, 1, 1, 1, 0, 0, 0],
                               [0, 0, 0, 1, 0, 0, 0]])

    rule2sample_id = {0: {1, 5, 6},
                              1: {0, 6},
                              2: {0, 1, 5, 6},
                              3: {1, 6, 7},
                              4: {2},  # no rel sample
                              5: {4},  # no rel sample
                              6: {3}}  # no rel sample

    return inputs_x, rule_matches_z, rule_assignments_t, rule2sample_id

@pytest.fixture(scope='session')
def get_cw_data_test(get_test_data):
    # no filtering, no no_match samples
    test_rules_idx = [1, 2]
    labels = np.array([[0.5, 0.5, 0], [0.3, 0.7, 0], [0, 0, 1], [0, 0, 1],
                       [0, 0, 1], [0.5, 0.5, 0], [0.5, 0.5, 0], [0, 1, 0]])

    return [[get_test_data[0],
             labels,
             test_rules_idx,
             get_test_data[3],
             torch.Tensor(np.array([[0, 0, 0, 0, 0, 0, 0, 0],
                                    [1, 1, 1, 1, 1, 1, 1, 1],
                                    [5, 5, 5, 5, 5, 5, 5, 5],
                                    [6, 6, 6, 6, 6, 6, 6, 6]])),
             np.array([[0.5, 0.5, 0], [0.3, 0.7, 0], [0.5, 0.5, 0], [0.5, 0.5, 0]]),
             np.array([0, 1, 5, 6])
             ]]

@pytest.fixture(scope='session')
def get_cw_data_train(get_test_data):
    # rules to be filtered: 1, 2, no_match samples should be added

    test_samples_idx = [0, 6, 1, 5]
    train_rules_idx = [0, 3, 4, 5, 6]  # 1, 5, 6, 7 --> delete inters --> 7 --> + no rel --> 2, 3, 4, 7
    labels = np.array([[0.5, 0.5, 0], [0.3, 0.7, 0], [0, 0, 1], [0, 0, 1],
                       [0, 0, 1], [0.5, 0.5, 0], [0.5, 0.5, 0], [0, 1, 0]])

    return [[get_test_data[0],
             labels,
             train_rules_idx,
             get_test_data[3],
             test_samples_idx,
             torch.Tensor(np.array([[2, 2, 2, 2, 2, 2, 2, 2],
                                    [3, 3, 3, 3, 3, 3, 3, 3],
                                    [4, 4, 4, 4, 4, 4, 4, 4],
                                    [7, 7, 7, 7, 7, 7, 7, 7]])),
             np.array([[0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0]]),
             np.array([2, 3, 4, 7])
             ]]

def test_sample_ids_matched_rules_correspondence(get_test_data):
    assert get_rules_sample_ids(get_test_data[1]) == get_test_data[3]

def test_get_cw_data_test(get_cw_data_test):
    for data in get_cw_data_test:
        samples, labels, ids = get_samples_labels_idx_by_rule_id(
            data[0], data[1], data[2], data[3]
        )

        assert torch.equal(samples.tensors[0], data[4])
        np.testing.assert_array_equal(labels, data[5])
        np.testing.assert_array_equal(ids, data[6])

def test_get_cw_data_train(get_cw_data_train):
    for data in get_cw_data_train:
        samples, labels, ids = get_samples_labels_idx_by_rule_id(
            data[0], data[1], data[2], data[3], data[4]
        )

        assert torch.equal(samples.tensors[0], data[5])
        np.testing.assert_array_equal(labels, data[6])
        np.testing.assert_array_equal(ids, data[7])

def test_random_check(get_cw_data_train):
    for data in get_cw_data_train:
        samples, labels, ids = get_samples_labels_idx_by_rule_id(
            data[0], data[1], data[2], data[3], data[4]
        )
        rnd_tst = np.random.randint(0, samples.tensors[0].shape[0])  # take some random index
        tst_sample = samples.tensors[0][rnd_tst, :]
        tst_idx = ids[rnd_tst]
        tst_label = labels[rnd_tst, :] if len(labels.shape) > 1 else labels[rnd_tst]

        tst_sample_true = data[5][rnd_tst, :]
        tst_label_true = data[6][rnd_tst, :] if len(data[6].shape) > 1 else data[6][rnd_tst]
        tst_idx_true = data[7][rnd_tst]

        assert torch.equal(tst_sample, tst_sample_true)
        np.testing.assert_array_equal(tst_label, tst_label_true)
        np.testing.assert_array_equal(tst_idx, tst_idx_true)

max_stars_repo_name: michael-aloys/knodle

max_issues_repo_name: michael-aloys/knodle

max_forks_repo_name: michael-aloys/knodle


text:
# get Max and Min from only lesion

import os
import csv
import fnmatch
import SimpleITK
import numpy

from mahotas.features.texture import haralick_labels

from GLCM import GLCMFeatures
from Gabor import ExtendGaborFeatures
from LBP import ExtendLBPFeatures

# Image Root Dir
rootDir ='/Users/yanzhexu/Desktop/Research/KRAS/Feb. 2017'

outputDir = '/Users/yanzhexu/Desktop/Research/KRAS/Final_results'

featuresOutFn = 'KRAS_features_normalbyLesion_LBP3.csv'

recfile = 'largest_rec.csv'
dcmfile = 'image.dcm'

def GrayScaleNormalization(imgArray, imgMax,imgMin):

    imgRange = imgMax - imgMin
    imgArray = (imgArray - imgMin) * (255.0 / imgRange)

    # transfer to closest int
    imgArray = numpy.rint(imgArray).astype(numpy.int16)

    return imgArray

def Read2DImage(fileName, rotateAngle=0):
    rawImage = SimpleITK.ReadImage(fileName)
    imgArray = SimpleITK.GetArrayFromImage(rawImage)

    # Convert 3D Image to 2D
    if len(imgArray.shape) == 3:
        imgArray = imgArray[0, :, :]

    return imgArray

def genFeatures():

    GLCMAngleList = ['Avg']

    LBPRadius = 3
    LBPnPoints = 8 * LBPRadius
    LBPMethod = 'uniform'
    # LBPnBins = 12

    LBPFeatureList = []
    for x in xrange(0, LBPnPoints + 1):
        LBPFeatureList.append('LBP_%02d' % x)
    LBPFeatureList.append('LBP_Other')

    Gaborsigma_range = (1.0, 3.0)
    Gaborfreq_range = (0.1, 0.3, 0.5)

    kernel_bank = []

    Gaborkernel_bank = ExtendGaborFeatures.genKernelBank(Gaborsigma_range, Gaborfreq_range, kernel_bank)

    GaborFeatureList = ['Gabor_Mean', 'Gabor_Std']

    # Generate full list of features combined with parameters
    featureTitle = ['PatientID', 'Phase', 'ROI_X', 'ROI_Y', 'Width', 'Height']
    for GLCMAngle in GLCMAngleList:
        for featureName in haralick_labels[:-1]:
            featureTitle.append(featureName + '_' + GLCMAngle)

    featureTitle = featureTitle + LBPFeatureList

    for GaborSigma in Gaborsigma_range:
        for GaborFreq in Gaborfreq_range:
            for featureName in GaborFeatureList:
                featureTitle.append(featureName + '_' + str(GaborSigma) + '_' + str(GaborFreq))

    MeanStdLBfeaturelist = ['LargestBox_Raw_Mean', 'LargestBox_Raw_Std']
    featureTitle = featureTitle + MeanStdLBfeaturelist

    featuresCSVFn = os.path.join(outputDir, featuresOutFn)

    with open(featuresCSVFn, 'wb') as featureCSVFile:
        featureWriter = csv.writer(featureCSVFile, dialect='excel')
        featureWriter.writerow(featureTitle)

        for casefile in os.listdir(rootDir):
            if casefile.startswith('.'):
                continue
            if casefile.startswith('..'):
                continue
            if fnmatch.fnmatch(casefile, '*Icon*'):
                continue

            if fnmatch.fnmatch(casefile,'*xlsx*'):
                continue

            print casefile

            patientID = casefile.split('_ ')[0]
            print patientID

            patientfilepath = os.path.join(rootDir,casefile)

            phaseDir = dict()
            for patientPhaseDir in os.listdir(patientfilepath):
                if patientPhaseDir.startswith('.'):
                    continue
                if patientPhaseDir.startswith('..'):
                    continue
                if fnmatch.fnmatch(patientPhaseDir, '*Icon*'):
                    continue

                if fnmatch.fnmatch(patientPhaseDir, '*Lesion*'):
                    phaseDir['lesion'] = patientPhaseDir
                    phasename = patientPhaseDir

                    print patientPhaseDir

                if fnmatch.fnmatch(patientPhaseDir, '*Control*'):
                    phaseDir['control'] = patientPhaseDir

                    print patientPhaseDir

            if 'lesion' not in phaseDir:
                continue

            lesionfile = os.path.join(patientfilepath, phaseDir['lesion'])
            lesionlargestrecpath = os.path.join(lesionfile, recfile)
            lesionDicom = os.path.join(lesionfile, dcmfile)

            # normalfile = os.path.join(patientfilepath, phaseDir['control'])
            # normallargestrecpath = os.path.join(normalfile, recfile)
            # normalDicom = os.path.join(normalfile, dcmfile)

            with open(lesionlargestrecpath, 'r') as roiFile:
                roiList = csv.DictReader(roiFile, dialect='excel')

                for aROI in roiList:

                    xcoord = int(aROI['X'])
                    ycoord = int(aROI['Y'])
                    width = int(aROI['W'])
                    height = int(aROI['H'])

                lesiondicomImage = Read2DImage(lesionDicom)
                # normaldicomImage = Read2DImage(normalDicom)

                lesionsubImage = lesiondicomImage[ycoord:(ycoord + height), xcoord:(xcoord + width)]
                # normalsubImage = normaldicomImage[ycoord:(ycoord + height), xcoord:(xcoord + width)]

                mean_LargBox = numpy.mean(lesionsubImage)
                std_LargBox = numpy.std(lesionsubImage)

                # get max gray scale and min grayscale from both lesion and normal dicom
                subImageMax = lesionsubImage.max()
                subImageMin = lesionsubImage.min()

                subImageLBP = lesiondicomImage[ycoord - LBPRadius:(ycoord + height) + LBPRadius,
                              xcoord - LBPRadius:(xcoord + width) + LBPRadius]

                subImageGLCM = GrayScaleNormalization(lesionsubImage, subImageMax, subImageMin)

                aFeature = [patientID, phasename, aROI['X'], aROI['Y'], aROI['W'], aROI['H']]

                # GLCM
                glcmFeatures = GLCMFeatures.calcFeatures(subImageGLCM)

                for GLCMAngle in GLCMAngleList:
                    for featureName in haralick_labels[:-1]:
                        aFeature.append(glcmFeatures[GLCMAngle][featureName])

                # LBP
                extendsubImageLBP = GrayScaleNormalization(subImageLBP, subImageMax, subImageMin)

                lbpFeatures = ExtendLBPFeatures.calcFeatures(extendsubImageLBP, LBPnPoints, LBPRadius, LBPMethod)
                aFeature = aFeature + lbpFeatures.tolist()

                # Gabor
                GaborFeatures = ExtendGaborFeatures.calcFeatures(lesiondicomImage, xcoord, ycoord, width, height,
                                                                 Gaborkernel_bank, subImageMax, subImageMin)

                for gaborfeature in GaborFeatures:
                    aFeature = aFeature + gaborfeature.tolist()

                aFeature = aFeature + [mean_LargBox, std_LargBox]

                featureWriter.writerow(aFeature)

genFeatures()

max_stars_repo_name: joshlyman/TextureAnalysis


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 13 22:32:55 2018

@author: vipul tushar pradyumna
"""

import cv2
import numpy as np
faceDetect = cv2.CascadeClassifier('/home/tushar/Downloads/Projects_ml/Face_recognition/recognition/haarcascades/haarcascade_frontalface_default.xml');
cam = cv2.VideoCapture(0);
rec = cv2.face.LBPHFaceRecognizer_create();
rec.read("recognizer/trainingData.yml");
id = 0
fontface = cv2.FONT_HERSHEY_SIMPLEX
fontsize = 1
fontcolor = (0,511,1)
while(True):
    ret,img = cam.read();
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    faces = faceDetect.detectMultiScale(gray,1.3,5);
    for(x,y,w,h) in faces:
        cv2.rectangle(img,(x,y),(x+w,y+h),(0,0,180),2)
        id,conf = rec.predict(gray[y:y+h,x:x+w])
        if(id==1):
               id="Tushar"
        elif(id==2):
               id="Vipul"
        elif(id==3):
               id="Hemang"
        cv2.putText(img,str(id),(x,y+h+25),fontface,fontsize,fontcolor,2);
    cv2.imshow("Face",img);
    if(cv2.waitKey(1)==ord('q')):
        break;
cam.release()
cv2.destroyAllWindows()


text:
#! /bin/python3

import numpy as np
from agent import *
from mtools import *

# Create the environment
env, brain_name, action_size, state_size = create_env()

# Create the agent
agent = DQNAgent(action_size, state_size,
	alpha=1,
	gamma=0.99,
	eps=1,
	eps_decay=0.99995,
	eps_min=0.05,
	memory_size=10000,
	batch_size=64,
	train_time=4,
	exploration_boost=0.00, # give exploration boost to penalize spinning around
	reset_eps_every=300*700, # 300 iterations per episode * n episodes
	# DQN parameters
	hidden_layers=[1024, 256, 128, 64],
	activation=['relu', 'relu', 'relu', 'relu', 'linear'],
	lr=0.0005,
	tau=0.001)

# Simulate the agent
scores = simulate(env=env, brain_name=brain_name, agent=agent, n_episodes=4000, verbose=3, filename='mbot.bin')

# Save the agent
save_agent(agent, filename='agent.bin')

# Plot the results
plot_score(scores, filename='agent_score.png', title='Agent Score')

# Close the environment
env.close()


text:
from PIL import Image
from io import BytesIO
from torchvision import transforms

import cv2
import numpy as np
import random

CROP_SIZE = 224
MANIP_PROBABILITY = 0.5

class Augmentator(object):
    def __init__(self, in_train_mode=True):
        self._in_train_mode = in_train_mode

        self._manip_list = []

        self._manip_list.append(lambda img: Augmentator._jpg_manip(img, 70))
        self._manip_list.append(lambda img: Augmentator._jpg_manip(img, 90))

        self._manip_list.append(lambda img: Augmentator._gamma_manip(img, 0.8))
        self._manip_list.append(lambda img: Augmentator._gamma_manip(img, 1.2))

        self._manip_list.append(lambda img: Augmentator._bicubic_manip(img, 0.5))
        self._manip_list.append(lambda img: Augmentator._bicubic_manip(img, 0.8))
        self._manip_list.append(lambda img: Augmentator._bicubic_manip(img, 1.5))
        self._manip_list.append(lambda img: Augmentator._bicubic_manip(img, 2.0))

        self._transform = Augmentator._get_transforms()

    @staticmethod
    def _get_transforms():
        return transforms.Compose([
            transforms.ToPILImage(),
            transforms.RandomCrop(CROP_SIZE),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
        ])

    @staticmethod
    def _jpg_manip(image, quality):
        buffer = BytesIO()
        image = Image.fromarray(image)
        image.save(buffer, format='jpeg', quality=quality)
        buffer.seek(0)
        result = Image.open(buffer)
        result = np.array(result)
        return result

    @staticmethod
    def _gamma_manip(image, gamma):
        result = np.uint8(cv2.pow(image / 255., gamma) * 255.)
        return result

    @staticmethod
    def _bicubic_manip(image, scale):
        result = cv2.resize(image, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
        return result

    def __call__(self, image, aug_type):
        if 0 <= aug_type <= 7:
            manip_func = self._manip_list[aug_type]
            image = manip_func(image)
        elif aug_type == 8:
            image = np.rot90(image)
        elif aug_type == 9:
            image = np.rot90(image, 2)
        elif aug_type == 10:
            image = np.rot90(image, -1)

        if self._in_train_mode and random.random() < MANIP_PROBABILITY:
            manip_func = random.choice(self._manip_list)
            image = manip_func(image)

        image = self._transform(image)
        return image


text:
import pandas as pd
import numpy as np
from sklearn import linear_model
from sklearn.pipeline import Pipeline
from sklearn import preprocessing
from sklearn.externals import joblib

class TacPredictor:

  """
    For our tic tac toe board:
    X : shape=[n,9]
        9 features having values {-1,0,1} where
                    -1 := "Empty"
                     0 := "O"
                     1 := "X"
        representing each position on the board
    Y :
        9 classes representing each position on the board.
        The column vector representing representing Y is assumed
        to be the last column of the input dataset.

  """

  dataset = pd.DataFrame([])
  dataN = len(dataset)

  model = Pipeline(steps=[('logreg',linear_model.LogisticRegression())])

  def __init__(self,features,datafile):
    self.datafile = datafile
    dataset = pd.DataFrame([],columns=features)
    self.featureN = len(features)

  def addPoint(row):
    self.dataset.append(row)

  def loadDataset():
    self.dataset = pd.read_csv(datafile)

  def saveDataset():
    self.dataset.to_csv(datafile)

  def fit():
    # Shuffle the dataset
    dataset = dataset.sample(frac=1).reset_index(drop=True)

    """
      Extract Feature Vectors X and Output Vector y from
      Raw Dateset;
      Assumption : Output Vector y is always the last column of
      the dataset
    """
    dataset_DF_X = self.dataset[features[:-1]]
    dataset_DF_Y = self.dataset[features[-1:]]

    """
      Split dataset into
      80% Training Set
      and
      20% Test Set
    """
    train_DF_X = dataset_DF_X[:-0.2*dataN]
    train_DF_Y = dataset_DF_Y[:-0.2*dataN]

    test_DF_X = dataset_DF_X[-0.2*dataN:]
    test_DF_Y = dataset_DF_Y[-0.2*dataN:]

    # Train the model
    model.fit(train_DF_X,train_DF_Y)

    metrics = { 'accuracyscore' : model.score(test_DF_X,test_DF_Y) }

    return metrics

  def predict(board):
    return model.predict(board)

  def saveModel(filename):
    joblib.dump(model,"{}.pkl".format(filename))

  def loadModel(filename):
    model = joblib.load("{}.pkl".format(filename))