text:
from typing import Tuple
from .common import Vector3D
import numpy as np
from .rotation import euler_rotation

class Camera:
    def __init__(self, resolution: Tuple[int, int], position: Vector3D, euler: Vector3D, fov: float = 45.):
        self.resolution = resolution
        self._position = position
        self.rotation = euler_rotation(euler[0], euler[1], euler[2])
        self.field_of_view = fov

    @property
    def position(self):
        return np.array(self._position)

    def generate_pixel_locations(self):
        """ Generate the pixel locations on the camera plane. """

        width, height = self.resolution
        AR = int(width / height)
        yy, zz = np.mgrid[AR:-AR:complex(0, width), 1:-1:complex(0, height)]
        xx = np.ones(shape=yy.shape) * (1 / np.tan(np.radians(self.field_of_view) / 2))  # Distance of grid from origin

        return np.array([xx, yy, zz])


text:
from .agent import Agent
from .directions import DIRECTIONS

import numpy as np

class Game(object):
    world: [[]] = None
    score = 0
    agent: Agent = None
    size = 4
    game_over = False

    def __init__(self, size, agent, map):
        self.agent = agent
        self.agent.pos = (0, 0)
        self.size = size
        self.score = 1000
        self.world = map
        print(" init game")

    def get_score(self):
        return self.score

    def move_agent(self):

        agent = self.agent
        world = self.world

        old_pos = (agent.pos[0], agent.pos[1])
        n_pos = (agent.pos[0] + agent.direction[0],
                 agent.pos[1] + agent.direction[1])
        # or world[n_pos[0]][n_pos[1]] == 'W'
        if world[n_pos[0]][n_pos[1]] == 'P':
            self.score -= 1000
            self.agent.died = True
            self.game_over = True
            print("End game now")

        self.agent.pos = n_pos

        if 'B' in world[n_pos[0]][n_pos[1]]:
            self.agent.sensor['breeze'] = True
        else:
            self.agent.sensor['breeze'] = False

        if 'S' in world[n_pos[0]][n_pos[1]]:
            self.agent.sensor['stench'] = True
        else:
            self.agent.sensor['stench'] = False

        if 'G' in world[n_pos[0]][n_pos[1]]:
            self.agent.sensor['glimmer'] = True
        else:
            self.agent.sensor['glimmer'] = False

        self.score -= 10
        return True

    def turn_agent(self, direction: int):
        if direction == self.agent.LEFT:
            print("Agent [TURN LEFT]")
            if (self.agent.direction == DIRECTIONS.NORTH):
                self.agent.direction = DIRECTIONS.WEST
            elif (self.agent.direction == DIRECTIONS.EAST):
                self.agent.direction = DIRECTIONS.NORTH
            elif (self.agent.direction == DIRECTIONS.SOUTH):
                self.agent.direction = DIRECTIONS.EAST
            elif (self.agent.direction == DIRECTIONS.WEST):
                self.agent.direction = DIRECTIONS.SOUTH
            else:
                print("Error, cannot [tURN LEFT]")
                return
        elif direction == self.agent.RIGHT:
            print("Agent [TURN RIGHT]")
            if (self.agent.direction == DIRECTIONS.NORTH):
                self.agent.direction = DIRECTIONS.EAST
            elif (self.agent.direction == DIRECTIONS.EAST):
                self.agent.direction = DIRECTIONS.SOUTH
            elif (self.agent.direction == DIRECTIONS.SOUTH):
                self.agent.direction = DIRECTIONS.WEST
            elif (self.agent.direction == DIRECTIONS.WEST):
                self.agent.direction = DIRECTIONS.NORTH
            else:
                print("Error, cannot [tURN RIGHT]")
                return
        else:
            print("Error,cannot reach0")
            return
        # score not change

    def process_shot(self):
        if(self.world[self.agent.pos[0]][self.agent.pos[1]] == 'W'):
            self.world[self.agent.pos[0]][self.agent.pos[1]] = '-'
        else:
            print("No wumpus")

    def agent_grab_gold(self):
        if self.world[self.agent.pos[0]][self.agent.pos[1]] == 'G':
            print("Collected at  {0} {1} ".format(
                self.agent.pos[0], self.agent.pos[1]))
            self.world[self.agent.pos[0]][self.agent.pos[1]] = '-'
            self.game_over = True
            self.score += 1000
            print("End game")

        else:
            print("No gold here")

# 1 2 3
# 4 5 6  4 => (1,0)
# 7 8 9

ext: py

reason: import numpy


text:
from typing import Optional

import numpy as np
from scipy.sparse import csr_matrix

class RatingScaleMixin:
    """Provide functionality to scale ratings."""

    def __init__(self, rating_normalization: Optional[str] = 'mean') -> None:
        self.rating_normalization = rating_normalization
        self._scaler = None

    def _fit_scaler(self, interactions: csr_matrix) -> None:
        """Calculate statistics to scale ratings.

        :param interactions: interactions between users and items.
        """
        bias_ = np.zeros(interactions.shape[0])
        scale_ = np.ones(interactions.shape[0])
        count = interactions.getnnz(axis=1)
        if self.rating_normalization == 'mean':
            bias_ = np.divide(interactions.sum(axis=1).A1, count,
                              out=np.zeros(len(count)), where=count != 0)
        elif self.rating_normalization == 'z-score':
            bias_ = np.divide(interactions.sum(axis=1).A1, count,
                              out=np.zeros(len(count)), where=count != 0)
            interactions_ = interactions.copy()
            interactions_.data **= 2
            bias_sq = np.divide(interactions_.sum(axis=1).A1, count,
                                out=np.zeros(len(count)), where=count != 0)
            scale_ = np.sqrt(bias_sq - bias_ ** 2)
            scale_[scale_ == 0] = 1
        elif self.rating_normalization is not None:
            raise ValueError(f'Unknown normalization strategy: '
                             f'{self.rating_normalization}')
        self._scaler = np.column_stack([bias_, scale_])

    def _unscale_ratings(self, ratings: np.ndarray,
                         indices: np.ndarray) -> np.ndarray:
        """Convert ratings to the original scale.

        :param indices: users' or items' indices.
        :param ratings: rating values.
        :return: unscaled ratings.
        """
        return ratings * self._scaler[indices, 1] + self._scaler[indices, 0]

    def _scale_ratings(self, interactions: csr_matrix) -> None:
        """Scale rating values in the interaction matrix.

        Scaling is performed inplace.

        :param interactions: ratings to scale.
        """
        row_indices, col_indices = interactions.nonzero()
        interactions[row_indices, col_indices] -= \
            self._scaler[row_indices, 0]
        interactions[row_indices, col_indices] /= \
            self._scaler[row_indices, 1]

reason: import numpy,from scipy


text:
import nibabel as nib
import numpy as np

def random_crop_3d(img_arr_list, patch_size):
    '''
    random crop for all the image arrays in the list.
    image will be cropped from a random generated left top corner
    the images must have the same shape
    '''
    img_shape = img_arr_list[0].shape
    for i in img_arr_list:
        assert i.shape == img_shape, "image shapes not consistent in the list"

    ix, iy, iz = img_shape
    px, py, pz = patch_size

    gx, gy, gz = np.random.randint(low=[ix-px, iy-py, iz-pz], size=3)

    return [i[gx:gx+px, gy:gy+py, gz:gz+pz] for i in img_arr_list]

def center_crop():
    pass

def resample():
    pass

def normalize(arr, method='01'):
    return (arr - np.mean(arr)) / np.std(arr)

max_stars_repo_name: QianyeYang/mpmrireg

max_issues_repo_name: QianyeYang/mpmrireg

max_forks_repo_name: QianyeYang/mpmrireg


text:
import os
import math
import json
import random
import argparse
import datetime
import itertools
from pathlib import Path
from functools import partial
import multiprocessing as mp
import multiprocessing.pool
from contextlib import contextmanager
from typing import Tuple, List, Dict, Callable

import cv2
import torch
import rasterio
import numpy as np
from shapely import geometry

from config import cfg

def jread(path: str) -> Dict:
    with open(str(path), 'r') as f:
        data = json.load(f)
    return data

def jdump(data: Dict, path: str) -> None:
    with open(str(path), 'w') as f:
        json.dump(data, f, indent=4)

def filter_ban_str_in_name(s: str, bans: List[str]): return any([(b in str(s)) for b in bans])

def get_filenames(path: str, pattern: str, filter_out_func: Callable) -> str:
    """
    pattern : "*.json"
    filter_out : function that return True if file name is acceptable
    """

    filenames = list(Path(path).glob(pattern))
    print("******************************",path, filenames)
    assert (filenames), f'There is no matching filenames for {path}, {pattern}'
    filenames = [fn for fn in filenames if not filter_out_func(fn)]
    # assert (filenames), f'There is no matching filenames for {filter_out_func}'
    return filenames

def polyg_to_mask(polyg: np.ndarray, wh: Tuple[int, int], fill_value: int) -> np.ndarray:
    """Convert polygon to binary mask.
    """

    polyg = np.int32([polyg])
    mask = np.zeros([wh[0], wh[1]], dtype=np.uint8)
    cv2.fillPoly(mask, polyg, fill_value)
    return mask

def json_record_to_poly(record: Dict) -> List[geometry.Polygon]:
    """Get list of polygons from record.
    """

    num_polygons = len(record['geometry']['coordinates'])
    if num_polygons == 1:     # Polygon
        list_coords = [record['geometry']['coordinates'][0]]
    elif num_polygons > 1:    # MultiPolygon
        list_coords = [record['geometry']['coordinates'][i][0] for i in range(num_polygons)]
    else:
        raise Exception("No polygons are found")

    try:
        polygons = [geometry.Polygon(coords) for coords in list_coords]
    except Exception as e:
        print(e, list_coords)
    return polygons

def make_folders(cfg):
    cfg_postfix = 'PAMBUH'
    timestamp = '{:%Y_%b_%d_%H_%M_%S}'.format(datetime.datetime.now())
    name = Path(timestamp + '_' + cfg_postfix)
    fname = cfg.OUTPUTS / name
    fname.mkdir(exist_ok=True)
    return fname

def save_models(d, postfix, output_folder):
    for k, v in d.items():
        torch.save(v.state_dict(), output_folder/os.path.join("models",f"{k}_{postfix}.pkl"))

def dump_params(cfg, output_path):
    with open(os.path.join(output_path, 'cfg.yaml'), 'w') as f:
        f.write(cfg.dump())

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--local_rank", default=0, type=int)
    args = parser.parse_args()
    return args

def get_basics_rasterio(name):
    file = rasterio.open(str(name))
    return file, file.shape, file.count

def get_tiff_block(ds, x, y, w, h=None, bands=3):
    if h is None: h = w
    return ds.read(list(range(1, bands+1)), window=rasterio.windows.Window(x, y, w, h))

def save_tiff_uint8_single_band(img, path, bits=1):
    assert img.dtype == np.uint8
    if img.max() <= 1. : print(f"Warning: saving tiff with max value is <= 1, {path}")
    h, w = img.shape
    dst = rasterio.open(path, 'w', driver='GTiff', height=h, width=w, count=1, nbits=bits, dtype=np.uint8)
    dst.write(img, 1)
    dst.close()
    del dst

def cfg_frz(func):
    def frz(*args, **kwargs):
        cfg.defrost()
        r = func(*args, **kwargs)
        cfg.freeze()
        return r
    return frz

@contextmanager
def poolcontext(*args, **kwargs):
    pool = mp.Pool(*args, **kwargs)
    yield pool
    pool.terminate()

def mp_func(foo, args, n):
    args_chunks = [args[i:i + n] for i in range(0, len(args), n)]
    with poolcontext(processes=n) as pool:
        res = pool.map(foo, args_chunks)
    return [ri for r in res for ri in r]

def mp_func_gen(foo, args, n, progress=None):
    args_chunks = [args[i:i + n] for i in range(0, len(args), n)]
    results = []
    with poolcontext(processes=n) as pool:
        gen = pool.imap(foo, args_chunks)
        if progress is not None: gen = progress(gen, total=len(args_chunks))
        for r in gen:
            results.extend(r)
    return results

def get_cortex_polygons(anot_structs_json: Dict) -> List[geometry.Polygon]:
    return get_polygons_by_type(anot_structs_json, 'Cortex')

def get_polygons_by_type(anot_structs_json: Dict, name: str) -> List[geometry.Polygon]:
    polygons = []
    for record in anot_structs_json:
        if record['properties']['classification']['name'] == name:
            polygons += json_record_to_poly(record)
    return polygons

def flatten_2dlist(list2d: List) -> List:
    """Converts 2d list into 1d list.
    """

    list1d = list(itertools.chain(*list2d))
    return list1d

def tiff_merge_mask(path_tiff, path_mask, path_dst, path_mask2=None):
    # will use shitload of mem
    img = rasterio.open(path_tiff).read()
    mask = rasterio.open(path_mask).read()
    #assert mask.max() <= 1 + 1e-6

    if img.shape[0] == 1:
        img = np.repeat(img, 3, 0)

    red = mask * 200 if mask.max() <= 1 + 1e-6 else mask
    img[1,...] = img.mean(0)
    img[0,...] = red

    if path_mask2 is not None:
        mask2 = rasterio.open(path_mask2).read()
        blue = mask2 * 200 if mask2.max() <= 1 + 1e-6 else mask2
        #assert mask2.max() <= 1 + 1e-6
        img[2,...] = blue

    _, h, w = img.shape
    dst = rasterio.open(path_dst, 'w', driver='GTiff', height=h, width=w, count=3, dtype=np.uint8)
    dst.write(img, [1,2,3]) # 3 bands
    dst.close()
    del dst

def gen_pt_in_poly(polygon: geometry.Polygon,
                   max_num_attempts=50) -> geometry.Point:
    """Generates randomly point within given polygon. If after max_num_attempts point has been not
    found, then returns centroid of polygon.
    """

    min_x, min_y, max_x, max_y = polygon.bounds

    num_attempts = 0
    while num_attempts < max_num_attempts:
        random_point = geometry.Point([random.uniform(min_x, max_x), random.uniform(min_y, max_y)])
        if random_point.within(polygon): return random_point
        num_attempts += 1
    return polygon.centroid

def rgb2gray(rgb: np.ndarray) -> np.ndarray:
    """Gets np.ndarray (3, ...) or (..., 3) and returns gray scale np.ndarray (...)."""

    first_channel = rgb.shape[0]
    if first_channel == 3:
        rgb = np.swapaxes(np.swapaxes(rgb, 0, 2), 0, 1) # (3, ...) -> (..., 3)
    return np.dot(rgb[..., :3], [0.299, 0.587, 0.144])

def save_arr_as_tiff(arr: np.ndarray, path: str, nbits: int = 8) -> None:
    """Gets np.ndarray (num_bands, h, w) and returns gray scale np.ndarray (h, w) in uint8."""

    num_bands, h, w = arr.shape

    dst = rasterio.open(path, 'w', driver='GTiff',
                        height=h, width=w, count=num_bands,
                        nbits=nbits, dtype=np.uint8)
    dst.write(arr)
    dst.close()
    del dst

class NoDaemonProcess(mp.Process):
    # make 'daemon' attribute always return False
    def _get_daemon(self):
        return False
    def _set_daemon(self, value):
        pass
    daemon = property(_get_daemon, _set_daemon)

class NoDaemonPool(mp.pool.Pool):
    Process = NoDaemonProcess

def sigmoid(x): return 1 / (1 + np.exp(-x))

max_stars_repo_name: cns-iu/HuBMAP---Hacking-the-Kidney

max_issues_repo_name: cns-iu/HuBMAP---Hacking-the-Kidney

max_forks_repo_name: cns-iu/HuBMAP---Hacking-the-Kidney


text:
'''
Copyright 2015 Serendio Inc.
Author - Satish Palaniappan

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and limitations under the License.
'''
__author__ = "Satish Palaniappan"

import numpy as np
from sklearn import cluster, datasets, preprocessing
import pickle
import gensim
import time
import re
import tokenize
from scipy import spatial

def save_obj(obj, name ):
    with open( name + '.pkl', 'wb') as f:
        pickle.dump(obj, f,  protocol=2)

def load_obj(name ):
    with open( name + '.pkl', 'rb') as f:
        return pickle.load(f)

def combine(v1,v2):
    A = np.add(v1,v2)
    M = np.multiply(A,A)
    lent=0
    for i in M:
        lent+=i
    return np.divide(A,lent)

# 3M word google dataset of pretrained 300D vectors
model = gensim.models.Word2Vec.load_word2vec_format('vectors.bin', binary=True)
model.init_sims(replace=True)

#### getting all vecs from w2v using the inbuilt syn0 list see code
# X_Scaled_Feature_Vecs = []
# for w in model.vocab:
#     X_Scaled_Feature_Vecs.append(model.syn0[model.vocab[w].index])
# model.syn0 = X_Scaled_Feature_Vecs
# X_Scaled_Feature_Vecs = None
# X_Scaled_Feature_Vecs = model.syn0
# ### scaling feature vecs
# min_max_scaler = preprocessing.MinMaxScaler()
# X_Scaled_Feature_Vecs = min_max_scaler.fit_transform(X)

# X_Scaled_Feature_Vecs = X

# W2V = dict(zip(model.vocab, X_Scaled_Feature_Vecs))

#Cosine Distance
# from scipy import spatial

# dataSetI = model["travel"]
# dataSetII = model["travelling"]
# result = 1 - spatial.distance.cosine(dataSetI, dataSetII)
# print(result)

X_Scaled_Feature_Vecs=[]
for word in model.vocab:
    X_Scaled_Feature_Vecs.append(model[word])

# ######## Interested Categories
cat = ["advertising","beauty","business","celebrity","diy craft","entertainment","family","fashion","food","general","health","lifestyle","music","news","pop","culture","social","media","sports","technology","travel","video games"]
nums = range(0,22)
num2cat = dict(zip(nums, cat))

# new Categories Seeds (787 seeds) DICT [seed: cat]
Word2CatMap = load_obj("baseWord2CatMap")

baseWords = Word2CatMap.keys()

catVec=[]
newBaseWords =[]
# load from C file output
for bw in baseWords:
    try:
        catVec.append(np.array(model[bw]))
        newBaseWords.append(bw)
    except:
        words = bw.split()
        try:
            vec = np.array(model[words[0]])
            for word in words[1:]:
                try:
                    vec = combine(vec,np.array(model[word]))
                except:
                    #print(word + " Skipped!")
                    continue
            catVec.append(vec)
            newBaseWords.append(bw)
        except:
            #print(words)
            continue
# print(len(catVec))
# print(len(newBaseWords))
#cluster Size
# newBaseWords has the list of new base words that are in word2vec vocab
k = len(catVec)

# form a num(k) to cat(22) mapping
numK2CatMap = dict()
for w in newBaseWords:
    numK2CatMap[newBaseWords.index(w)] = Word2CatMap[w]
# kmeans
##### better code
t0 = time.time()
# Assign Max_Iter to 1 (ONE) if u just want to fit vectors around seeds
kmeans = cluster.KMeans(n_clusters=k, init=np.array(catVec), max_iter=1).fit(X_Scaled_Feature_Vecs)
#kmeans = cluster.KMeans(n_clusters=22, init=np.array(catVec), max_iter=900).fit(X_Scaled_Feature_Vecs)
print(str(time.time()-t0))
print(kmeans.inertia_)

###### After Fiting the Cluster Centers are recomputed : update catVec (Order Preserved)
catVec = kmeans.cluster_centers_

# #test
# for c in catVec:
#     print(num2cat[kmeans.predict(c)[0]])

##### save best for future use
save_obj(kmeans,"clusterLarge")
KM = kmeans
# Cluster_lookUP = dict(zip(model.vocab, KM.labels_))
Cluster_lookUP = dict()
Cluster_KlookUP = dict()
for word in model.vocab:
    kmap = KM.predict(model[word])[0]
    Cluster_lookUP[word] = numK2CatMap[kmap]
    Cluster_KlookUP[word] = kmap

## Precomputing the cosine similarities

Cosine_Similarity = dict()
for k in Cluster_lookUP.keys():
    # if len(Cluster_lookUP[k]) == 1:
    Cosine_Similarity[k] = 1 - spatial.distance.cosine(model[k], catVec[Cluster_KlookUP[k]])
    # else:
    #     Cosine_Similarity[k] = [1 - spatial.distance.cosine(model[k], catVec[wk]) for wk in Cluster_KlookUP[k]]

#check
print(num2cat[Cluster_lookUP["flight"][0]] + "   "+str(Cosine_Similarity["flight"]))
print(num2cat[Cluster_lookUP["gamecube"][0]] +"   "+str(Cosine_Similarity["gamecube"]))

#Saving Models
# for 22 topics
save_obj(Cluster_lookUP,"Cluster_lookUP")
save_obj(Cosine_Similarity,"Cosine_Similarity")
save_obj(num2cat,"num2cat")
save_obj(catVec,"catVec")
save_obj(numK2CatMap,"numK2CatMap")

max_stars_repo_name: serendio-labs/diskoveror-ta

max_issues_repo_name: serendio-labs/diskoveror-ta

max_forks_repo_name: serendio-labs/diskoveror-ta


text:
import numpy as np
import torch
import torch.nn as nn
import torch.optim
from torch.autograd import variable
from helpers import constants, torch_utils, utils
from dnn_units.lstm_attention import LSTMAttentionDot, SoftDotAttention

class Encoder(nn.Module):
    def __init__(self, config, use_features=True):
        super(Encoder, self).__init__()
        self.config = config
        if use_features:
            input_size = config['embedding_size'] + 1
        else:
            input_size = config['embedding_size']
        hidden_size = int(config['hidden_size'] / 2)
        self.rnn = nn.LSTM(input_size=input_size, hidden_size=hidden_size,
                        num_layers=config['num_layers'], dropout=config['dropout'],
                        bidirectional=True, batch_first=False)

    def forward(self, inputs):
        batch_size = inputs.size()[1]
        state_shape = self.config['num_layers'] * 2, batch_size, int(self.config['hidden_size'] / 2)
        h0 = variable.Variable(inputs.data.new(*state_shape).zero_(), requires_grad=False).cuda()
        c0 = variable.Variable(inputs.data.new(*state_shape).zero_(), requires_grad=False).cuda()

        outputs, (ht, ct) = self.rnn(inputs, (h0, c0))
        return outputs, ht[-2:].transpose(0, 1).contiguous().view(batch_size, -1)

class LanguageDiscriminator(nn.Module):
    def __init__(self, config,
        embeddings,
        text_field_predictor,
        base_lstm):
        super(LanguageDiscriminator, self).__init__()
        self.config = config
        self.embeddings = embeddings
        self.text_field_predictor = text_field_predictor
        self.base_lstm = base_lstm
        self.predictor = self.get_predictor()

    def get_predictor(self):
        fc = nn.Sequential(
                nn.Linear(self.config['hidden_size'], 1),
                nn.Sigmoid())
        return fc

    def forward(self, inputs, contexts, answer_features):
        context_embeddings = self.text_field_predictor.forward_prepro(contexts, input_masks=None, answer_features=answer_features)
        input_embeddings = self.embeddings(inputs)
        batch_size = inputs.size(1)
        state_shape = (batch_size, self.config['hidden_size'])
        h0 = c0 = variable.Variable(input_embeddings.data.new(*state_shape).zero_()).cuda()
        cur_states = (h0, c0)

        out, hidden = self.base_lstm.forward(input_embeddings, cur_states, context_embeddings)

        h = hidden[0]
        pred = torch.squeeze(self.predictor(h))
        return pred

class TextFieldPredictor(nn.Module):
    def __init__(self, config, embeddings):
        super(TextFieldPredictor, self).__init__()
        self.config = config
        self.embeddings = embeddings
        self.encoder = Encoder(config)
        self.attention = SoftDotAttention(config['hidden_size'])

    def forward_prepro(self, input, input_masks, answer_features=None):
        self.input = input
        self.input_masks = input_masks
        input_embeddings = self.embeddings(input)

        if answer_features is not None:
            unsqueezed_answer_features = torch.unsqueeze(answer_features, 2)
            self.input_embeddings = torch.cat((input_embeddings, unsqueezed_answer_features), 2)
        else:
            self.input_embeddings = input_embeddings

        self.lstm_embeddings, _ = self.encoder(self.input_embeddings) # Assumes NOT batch first

        # Get batch first lstm embeddings
        self.batch_first_lstm_embeddings = self.lstm_embeddings.transpose(0, 1)
        return self.lstm_embeddings

    def forward_similarity(self, hidden_state):
        # print(self.batch_first_lstm_embeddings.size())
        # print(hidden_state.size())
        h_tilde, attentions = self.attention(hidden_state,
            self.batch_first_lstm_embeddings, self.input_masks)
        return h_tilde, torch.log(attentions), self.input

class SoftmaxPredictor(nn.Module):
    def __init__(self, config):
        super(SoftmaxPredictor, self).__init__()
        self.config = config
        self.projection = nn.Linear(config['hidden_size'], config['vocab_size'])
        self.log_softmax = nn.LogSoftmax()

    def forward(self, hidden_state):
        return self.log_softmax(self.projection(hidden_state))

class LanguageModel(nn.Module):
    def __init__(self, config):
        super(LanguageModel, self).__init__()
        self.config = config
        self.model_type = constants.MODEL_TYPE_LANGUAGE_MODEL
        self.init_predictors()
        self.init_base_lstm()

    def get_type(self):
        return self.model_type

    def init_predictors(self):
        self.embedder = self.get_embedder()
        self.combiner = self.get_combiner()
        self.text_field_predictor = TextFieldPredictor(self.config, self.embedder)
        self.softmax_predictor = SoftmaxPredictor(self.config)

    def get_embedder(self):
        embedder = nn.Embedding(self.config['vocab_size'], self.config['embedding_size'])
        if self.config['use_pretrained_embeddings']:
            embeddings = utils.load_matrix(self.config['pretrained_embeddings_path'])
            embedder.weight.data.copy_(torch.from_numpy(embeddings))
        return embedder

    def get_combiner(self):
        combiner = nn.Sequential(
            nn.Linear(self.config['hidden_size'], self.config['hidden_size']),
            nn.Tanh(),
            nn.Linear(self.config['hidden_size'], 2),
            nn.LogSoftmax())
        return combiner

    def init_base_lstm(self):
        self.base_lstm = LSTMAttentionDot(input_size=self.config['embedding_size'],
            hidden_size=self.config['hidden_size'],
            batch_first=self.config['batch_first'])

    def predict(self, input_token,
        context_tokens,
        end_token,
        answer_features,
        max_length=20,
        min_length=3):
        """
        input_token: Input token to start with
        context_tokens: Context tokens to use
        Do greedy decoding using input token and context tokens
        """

        predicted_tokens = []
        total_loss = 0.0

        batch_first_context_tokens = context_tokens.transpose(0, 1)
        context_embeddings = self.text_field_predictor.forward_prepro(context_tokens, input_masks=None,
            answer_features=answer_features)

        state_shape = (1, self.config['hidden_size'])
        h0 = c0 = variable.Variable(context_embeddings.data.new(*state_shape).zero_())
        cur_states = (h0, c0)

        def step(input_token, states):
            cur_input_embedding = self.embedder(input_token)
            hidden_states, new_states = self.base_lstm.forward(cur_input_embedding, \
                states, context_embeddings)

            reshaped_hidden_states = hidden_states.view(-1, hidden_states.size(-1))
            predictor_probs = self.combiner(reshaped_hidden_states)

            language_probs = self.softmax_predictor(reshaped_hidden_states)
            reshaped_language_probs = language_probs.view(-1, language_probs.size(-1))

            _, attentions, inputs = self.text_field_predictor.forward_similarity(hidden_states)

            combined_predictions = self.combine_predictions_single(\
                context_tokens=batch_first_context_tokens,
                predictor_probs=predictor_probs,
                attentions=attentions,
                language_probs=reshaped_language_probs)

            loss, token = torch.max(combined_predictions, 1)
            return loss, token, new_states

        loss, new_token, new_states = step(input_token, cur_states)

        while (not torch_utils.to_bool(new_token.data == end_token) or
            len(predicted_tokens) < min_length) and len(predicted_tokens) < max_length:
            predicted_tokens.append(new_token)
            loss, new_token, new_states = step(new_token, new_states)
        return predicted_tokens

    def forward(self, input_tokens, context_tokens, context_masks, answer_features):
        self.batch_first_context_tokens = context_tokens.transpose(0, 1)
        self.context_embeddings = self.text_field_predictor.forward_prepro(context_tokens, context_masks, answer_features)
        self.input_embeddings = self.embedder(input_tokens)

        batch_size = input_tokens.size(1)
        token_length = input_tokens.size(0)

        state_shape = (batch_size, self.config['hidden_size'])
        h0 = c0 = variable.Variable(self.input_embeddings.data.new(*state_shape).zero_(), requires_grad=False)

        hidden_states, res = self.base_lstm.forward(self.input_embeddings, \
            (h0, c0), \
            self.context_embeddings)

        reshaped_hidden_states = hidden_states.view(batch_size * token_length, -1)
        predictor_probs = self.combiner(reshaped_hidden_states)
        reshaped_predictor_probs = predictor_probs.view(token_length, batch_size, predictor_probs.size(-1))

        language_probs = self.softmax_predictor(reshaped_hidden_states)
        reshaped_language_probs = language_probs.view(token_length, batch_size, language_probs.size(-1))

        attentions_list = []
        for i in range(0, token_length):
            _, attentions, inputs = self.text_field_predictor.forward_similarity(hidden_states[i, :, :])
            attentions_list.append(attentions)
        attentions_sequence = torch.stack(attentions_list, 0)

        combined_predictions = self.combine_predictions(context_tokens=self.batch_first_context_tokens,
            predictor_probs=reshaped_predictor_probs,
            attentions=attentions_sequence,
            language_probs=reshaped_language_probs)

        #return reshaped_language_probs
        return combined_predictions

    def combine_predictions_single(self, context_tokens,
        predictor_probs,
        attentions,
        language_probs):

        max_attention_length = attentions.size(1)
        pad_size = self.config['vocab_size'] - max_attention_length
        batch_size = attentions.size(0)

        context_tokens_padding = variable.Variable(torch.LongTensor(batch_size, pad_size).zero_()).cuda()
        attentions_padding = variable.Variable(torch.zeros(batch_size, pad_size)).cuda() + -1e10
        stacked_context_tokens = torch.cat((context_tokens, context_tokens_padding), 1)

        softmax_probs = predictor_probs[:, 0]
        text_field_probs = predictor_probs[:, 1]

        replicated_softmax_probs = softmax_probs.unsqueeze(1)
        replicated_text_field_probs = text_field_probs.unsqueeze(1)

        dims = replicated_softmax_probs.size()
        dims1 = replicated_text_field_probs.size()

        expanded_softmax_probs = replicated_softmax_probs.expand(dims[0],  self.config['vocab_size'])
        expanded_text_field_probs = replicated_text_field_probs.expand(dims[0], max_attention_length)

        stacked_attentions = torch.cat((attentions, attentions_padding), 1)
        attention_results = variable.Variable(torch.zeros(batch_size, self.config['vocab_size'])).cuda() + -1e10

        attention_results.scatter_(1, stacked_context_tokens, stacked_attentions)
        use_softmax_predictor = softmax_probs > text_field_probs
        if torch_utils.to_bool(use_softmax_predictor.data):
            return language_probs
        else:
            return attention_results

    def combine_predictions(self, context_tokens,
        predictor_probs,
        attentions,
        language_probs):

        max_attention_length = attentions.size(2)
        pad_size = self.config['vocab_size'] - max_attention_length
        batch_size = attentions.size(1)
        seq_size = attentions.size(0)

        context_tokens_padding = variable.Variable(torch.LongTensor(batch_size, pad_size).zero_(), requires_grad=False).cuda()
        attentions_padding = variable.Variable(torch.zeros(batch_size, pad_size) + -1e10, requires_grad=False).cuda()
        stacked_context_tokens = torch.cat((context_tokens, context_tokens_padding), 1)

        total_attention_results = []
        softmax_probs = predictor_probs[:, :, 0]
        text_field_probs = predictor_probs[:, :, 1]

        replicated_softmax_probs = softmax_probs.unsqueeze(2)
        replicated_text_field_probs = text_field_probs.unsqueeze(2)

        dims = replicated_softmax_probs.size()
        dims1 = replicated_text_field_probs.size()

        expanded_softmax_probs = replicated_softmax_probs.expand(dims[0], dims[1], self.config['vocab_size'])
        expanded_text_field_probs = replicated_text_field_probs.expand(dims[0], dims[1], max_attention_length)

        for i in range(0, seq_size):
            selected_text_field_probs = expanded_text_field_probs[i, :, :]
            selected_attention = attentions[i, :, :] + selected_text_field_probs
            stacked_attentions = torch.cat((selected_attention, attentions_padding), 1)

            attention_results = variable.Variable(torch.zeros(batch_size, self.config['vocab_size']) + -1e10).cuda()
            attention_results.scatter_(1, stacked_context_tokens, stacked_attentions)
            total_attention_results.append(attention_results)

        concated_attention_results = torch.stack(total_attention_results, 0)
        final_probs = torch.log(torch.exp(concated_attention_results) + torch.exp(language_probs + expanded_softmax_probs))

        return final_probs

max_stars_repo_name: davidgolub/QuestionGeneration

max_issues_repo_name: davidgolub/QuestionGeneration

max_forks_repo_name: davidgolub/QuestionGeneration


text:
import numpy as np
from impyute.ops import matrix

def _is_gt_5(x):
    return x > 5

def test_map_nd_2d():
    arr = np.arange(10).reshape([5, 2])
    expected = np.array([
        [False, False],
        [False, False],
        [False, False],
        [True, True],
        [True, True],
    ])
    actual = matrix.map_nd(_is_gt_5, arr)
    assert matrix.every_nd(bool, expected == actual)

max_stars_repo_name: eltonlaw/impy

max_issues_repo_name: eltonlaw/impy

max_forks_repo_name: eltonlaw/impy


text:
import os
import json
import yaml
import math
import logging
import socket
import time

import numpy as np
import cv2
from scipy.stats import multivariate_normal

from cyclops.type_hints import color_type, coordinate, pixel_coord, Callable
from cyclops.communication import PoseServer

class ParticleFilter(object):

    def __init__(self, parameters_file: str, camera_parameters_file: str,
                 camera_scale: float, front_color: color_type,
                 rear_color: color_type):
        self.parameters_file = parameters_file
        self.camera_parameters_file = camera_parameters_file
        self.camera_scale = camera_scale
        self.front_color = front_color
        self.rear_color = rear_color
        self.is_in_progress = False

        self.loader(self.check_parameters_file, self.load_parameters)
        self.loader(self.check_camera_parameters_file, self.load_camera_parameters)

    @staticmethod
    def loader(condition_to_load: Callable, load_function: Callable,
               exception: Exception = FileNotFoundError):
        if condition_to_load():
            load_function()
        else:
            raise exception()

    def check_parameters_file(self):
        return os.path.exists(self.parameters_file)

    def load_parameters(self):
        with open(self.parameters_file, 'r') as handle:
            parameters = json.load(handle)
            self.camera_source = parameters['camera_source']
            self.image_width = parameters['image_width']
            self.image_height = parameters['image_height']
            self.number_of_particles = parameters['number_of_particles']
            self.process_covariance = np.diag(parameters['process_covariance'])
            self.measurement_covariance = np.diag(parameters['measurement_covariance'])
            self.inital_xy_covariance = np.diag(parameters['inital_xy_covariance'])
            self.reference_distance = parameters['reference_distance']
            self.server_port = parameters['server_port']

            self.frame_translation = [self.image_width // 2, self.image_height // 2]
            self.configuration_space_x_min, self.configuration_space_y_min = \
                self.convert_pixel_space_to_physical_space((0, 0))
            self.configuration_space_x_max, self.configuration_space_y_max = \
                self.convert_pixel_space_to_physical_space((self.image_width, self.image_height))

    def check_camera_parameters_file(self):
        return os.path.exists(self.camera_parameters_file)

    def load_camera_parameters(self):
        with open(self.camera_parameters_file,'r') as handle:
            camera_info = yaml.load(handle)
            camera_matrix = camera_info['camera_matrix']['data']
            self.camera_matrix = np.reshape(np.array(camera_matrix),(3,3))
            distortion = camera_info['distortion_coefficients']['data']
            self.distortion = np.array(distortion)

    def convert_pixel_space_to_physical_space(self, location: coordinate):
        x = (location[0] - self.frame_translation[0]) / self.camera_scale
        y = (-1 * location[1] + self.frame_translation[1]) / self.camera_scale
        return x, y

    def initialize_particles(self, start_location: pixel_coord = None):
        logging.debug('Particle Initialization')
        self.reset_filter()
        self.distribute_particle_angles_uniformly()

        if start_location:
            self.distribute_particle_positions_with_gaussian(start_location)
        else:
            self.distribute_particle_positions_uniformly()

    def reset_filter(self):
        self.particles = np.zeros((3, self.number_of_particles))
        self.weights = np.ones(self.number_of_particles)

    def distribute_particle_positions_uniformly(self):
        self.particles[0, :] = np.random.uniform(self.configuration_space_x_min,
                                                 self.configuration_space_x_max,
                                                 self.number_of_particles)
        self.particles[1, :] = np.random.uniform(self.configuration_space_y_min,
                                                 self.configuration_space_y_max,
                                                 self.number_of_particles)

    def distribute_particle_positions_with_gaussian(self, start_location: pixel_coord):
        start_location = self.convert_pixel_space_to_physical_space(start_location)
        self.particles[:2, :] = np.transpose(np.random.multivariate_normal(
            list(start_location), self.inital_xy_covariance, self.number_of_particles))

    def distribute_particle_angles_uniformly(self):
        self.particles[2, :] = np.random.uniform(0.0, math.pi * 2, self.number_of_particles)

    def apply_mode_to_particle_thetas(self):
        self.particles[2, :] = np.apply_along_axis(
            lambda x: x % (math.pi * 2), axis=0, arr=self.particles[2, :])

    def run(self):
        self.pose_server = PoseServer(port=self.server_port)
        self.create_video_capture()
        self.create_window()
        self.create_density_functions()
        self.is_in_progress = True

        while self.is_in_progress:
            self.get_undistorted_image()
            self.convert_particles()
            self.run_process_model()
            self.run_measurement_update()
            self.resample_particles()
            self.compute_belief_from_particles()
            self.visualize()
            self.update_pose_server()
            self.wait_key()

    def create_video_capture(self):
        self.video_capture = cv2.VideoCapture(self.camera_source)

    def create_window(self):
        self.window = 'PF Localization'
        cv2.namedWindow(self.window, cv2.WINDOW_AUTOSIZE)

    def create_density_functions(self):
        self.measurement_probability_for_xy = multivariate_normal(
            mean=self.front_color, cov=self.measurement_covariance)

        self.measurement_probability_for_theta = multivariate_normal(
            mean=self.rear_color, cov=self.measurement_covariance)

    def create_socket(self):
        self.socket = socket.socket()
        self.server_address = (socket.gethostname(), 12349)
        self.socket.connect(self.server_address)

    def get_undistorted_image(self):
        self.capture_image()
        self.undistort_image()

    def capture_image(self):
        _ , self.captured_image = self.video_capture.read()

    def undistort_image(self):
        self.undistorted_image = cv2.undistort(self.captured_image, self.camera_matrix, self.distortion)

    def convert_particles(self):
        self.particles_in_pixel_space = self.convert_array_from_physical_to_pixel_space(self.particles)

    def run_process_model(self):
        logging.debug('Process')
        self.process_noise = np.transpose(np.random.multivariate_normal(
            mean=(0., 0., 0.), cov=self.process_covariance, size=self.number_of_particles))
        self.particles += self.process_noise

    def run_measurement_update(self):
        logging.debug('Measurement Update')
        self.convert_particles()
        measurements_for_xy = self.get_measurements(self.particles_in_pixel_space)
        weights_xy = self.get_probabilities_for_xy_measurements(measurements_for_xy)

        locations_for_angle_measurements = self.get_locations_for_angle_measurements()
        pixel_locations_for_angle_measurement = self.convert_array_from_physical_to_pixel_space(
            locations_for_angle_measurements)
        measurements_for_angle = self.get_measurements(pixel_locations_for_angle_measurement)
        weights_theta = self.get_probabilities_for_angle_measurements(measurements_for_angle)

        # self.weights = np.mean(np.array([weights_xy, weights_theta]), axis=0)
        self.weights = np.multiply(weights_xy, weights_theta)
        self.normalize_weights()

    def convert_array_from_physical_to_pixel_space(self, array: np.array):
        converted_array = np.copy(array)

        converted_array[0, :] = np.apply_along_axis(
            lambda x: x * self.camera_scale + self.frame_translation[0],
            axis=0, arr=array[0, :])

        converted_array[1, :] = np.apply_along_axis(
            lambda y: (-1 * y * self.camera_scale) + self.frame_translation[1],
            axis=0, arr=array[1, :])
        return converted_array.astype(int)

    def get_image_value_at(self, coord: coordinate):
        (x, y) = coord
        if self.check_pixel_coordinates(x, y):
            return self.undistorted_image[int(y), int(x)]
        else:
            return (-1, -1, -1)

    def check_pixel_coordinates(self, x, y):
        return (0 <= x < self.image_width) and (0 <= y < self.image_height)

    def get_measurements(self, array_of_pixel_coordinates: np.array):
        return np.apply_along_axis(self.get_image_value_at, axis=0, arr=array_of_pixel_coordinates[:2, :])

    def get_probabilities_for_xy_measurements(self, measurements: np.array):
        def _probability(measurement):
            if (measurement[0], measurement[1], measurement[2]) == (-1, -1, -1):
                return 0.
            else:
                return self.measurement_probability_for_xy.pdf((measurement[0], measurement[1], measurement[2]))
        return np.apply_along_axis(_probability, axis=0, arr=measurements)

    def get_locations_for_angle_measurements(self):
        def _get_location(particle):
            if (particle[2] % math.pi) == 0:
                delta_x = self.reference_distance
                delta_y = 0
            elif (particle[2] % math.pi) == math.pi / 2.:
                delta_x = 0
                delta_y = self.reference_distance
            else:
                delta_x = self.reference_distance * math.cos(particle[2])
                delta_y = self.reference_distance * math.sin(particle[2])

            return np.array([particle[0] - delta_x, particle[1] - delta_y])

        return np.apply_along_axis(_get_location, axis=0, arr=self.particles)

    def get_probabilities_for_angle_measurements(self, measurements: np.array):
        return np.apply_along_axis(
            lambda m: self.measurement_probability_for_theta.pdf((m[0], m[1], m[2])), axis=0, arr=measurements)

    def normalize_weights(self):
        normalization_factor = np.sum(self.weights)
        self.weights = self.weights / normalization_factor

    def resample_particles(self):
        logging.debug('Resample')
        weights_cdf = np.cumsum(self.weights)
        random_number = np.random.uniform(0., 1/self.number_of_particles)
        resampled_particle_indeces = []
        for i in range(self.number_of_particles):
            idx = np.min(np.argwhere(weights_cdf >= random_number + (i - 1) / self.number_of_particles))
            resampled_particle_indeces.append(idx)
        self.particles = self.particles[:,resampled_particle_indeces]
        self.apply_mode_to_particle_thetas()

    def compute_belief_from_particles(self):
        self.median_x = int(np.median(self.particles_in_pixel_space[0, :]))
        self.median_y = int(np.median(self.particles_in_pixel_space[1, :]))
        self.median_theta = -np.median(self.particles[2, :])
        self.median_x_physical, self.median_y_physical = \
            self.convert_pixel_space_to_physical_space((self.median_x, self.median_y))
        self.median_theta_physical = -self.median_theta

    def visualize(self):
        _image = self.undistorted_image.copy()
        _belief_line_length_pixels = 40
        _particle_line_length_pixels = 20

        for i in range(self.number_of_particles):
            x, y, theta = self.particles_in_pixel_space[0, i], \
                          self.particles_in_pixel_space[1, i], \
                          -self.particles_in_pixel_space[2, i]
            if self.check_pixel_coordinates(x, y):
                cv2.arrowedLine(_image, pt1=(x, y),
                    pt2=(int(x + _particle_line_length_pixels * math.cos(theta)),
                         int(y + _particle_line_length_pixels * math.sin(theta))),
                    color=(100, 250, 0), thickness=1)

        cv2.arrowedLine(_image,
            pt1=(self.median_x, self.median_y),
            pt2=(int(self.median_x + _belief_line_length_pixels * math.cos(self.median_theta)),
                 int(self.median_y + _belief_line_length_pixels * math.sin(self.median_theta))),
            color=(200, 0, 0), thickness=2)

        cv2.putText(_image, '{:.2f}, {:.2f}, {:.1f}'.format(
            self.median_x_physical, self.median_y_physical,
            math.degrees(self.median_theta_physical)), (self.median_x, self.median_y),
            cv2.FONT_HERSHEY_SIMPLEX, 0.5, thickness=2, color=(200, 25, 200))

        cv2.imshow(self.window, _image)

    def update_pose_server(self):
        self.socket = socket.socket()
        self.server_address = (socket.gethostname(), self.server_port)

        try:
            self.socket.connect(self.server_address)
            _current_belief_in_bytes = str.encode(str({'purpose': 'update',
                                                    'id': '0',
                                                    'timestamp': time.time(),
                                                    'x': self.median_x_physical,
                                                    'y': self.median_y_physical,
                                                    'theta': self.median_theta_physical}))
            self.socket.send(_current_belief_in_bytes)
        except Exception as ex:
            print(ex)

        self.socket.close()

    def wait_key(self):
        return cv2.waitKey(50)

    @property
    def is_in_progress(self):
        return self._is_in_progress

    @is_in_progress.setter
    def is_in_progress(self, value: bool):
        self._is_in_progress = value

max_stars_repo_name: emreay-/cyclops

max_issues_repo_name: emreay-/cyclops

max_forks_repo_name: emreay-/cyclops


text:
# SPDX-FileCopyrightText: BioPAL <biopal@esa.int>
# SPDX-License-Identifier: MIT

import numpy as np
from scipy.interpolate import interp1d
from biopal.io.xml_io import raster_info

def apply_calibration_screens(beta0_calibrated, beta0_raster_info, cal_screens, screens_raster_info, master_id):

    for idx, pf_name in enumerate(beta0_calibrated.keys()):

        if idx == 0:
            rg_axis_index = 0
            az_axis_index = 1

            # input original axis
            max_val_in_rg = screens_raster_info.num_samples * screens_raster_info.pixel_spacing_slant_rg
            rg_ax_in = np.arange(0, max_val_in_rg, raster_info.pixel_spacing_slant_rg)

            max_val_in_az = screens_raster_info.num_lines * screens_raster_info.pixel_spacing_az
            az_ax_in = np.arange(0, max_val_in_az, screens_raster_info.pixel_spacing_az)

            # output interpolated axis
            max_val_out_rg = beta0_raster_info.num_samples * beta0_raster_info.pixel_spacing_slant_rg
            rg_ax_out = np.arange(0, max_val_out_rg, beta0_raster_info.pixel_spacing_slant_rg)

            max_val_out_az = beta0_raster_info.num_lines * beta0_raster_info.pixel_spacing_az
            az_ax_out = np.arange(0, max_val_out_az, beta0_raster_info.pixel_spacing_az)

        # interpolate current screen over the data definiton axes:
        cal_scree_interp = cal_screens[pf_name]

        # interpolation of data along axis_index
        interp_fun = interp1d(rg_ax_in, cal_scree_interp, rg_axis_index, bounds_error=0)
        cal_scree_interp = interp_fun(rg_ax_out)
        interp_fun = interp1d(az_ax_in, cal_scree_interp, az_axis_index, bounds_error=0)
        cal_scree_interp = interp_fun(az_ax_out)

        for pol_id in beta0_calibrated[pf_name]:
            beta0_calibrated[pf_name][pol_id] = np.multiply(
                beta0_calibrated[pf_name][pol_id], np.exp(-1j * cal_scree_interp)
            )

    return beta0_calibrated

max_stars_repo_name: rpitonak/BioPAL

max_issues_repo_name: rpitonak/BioPAL

max_forks_repo_name: rpitonak/BioPAL


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""A sketch-based semi-Markov learner with approximate duration estimation.

Testing module for predicure.
"""

# Copyright © 2017 Carson J. Q. Farmer <carson@set.gl>
# All rights reserved. BSD-2 Licensed.

from __future__ import print_function, division, absolute_import

import random
import pytest
from scipy.stats import poisson
from addc import AddC, Centroid, KernelCentroid
from math import isinf, isnan

from predicure import MarkovSketch

import numpy as np

# Only works on *nix systems
word_file = "/usr/share/dict/words"
WORDS = open(word_file).read().splitlines()

def contains_same(s, t):
    s, t = set(s), set(t)
    return s >= t and s <= t

def all_close(s, t, tol=1e-8):
    # Ignores inf and nan values...
    return all(abs(a - b) < tol for a, b in zip(s, t)
               if not isinf(a) and not isinf(b) and
                  not isnan(a) and not isnan(b))

@pytest.fixture(scope="module")
def Events(state_count=10):
    '''Return sequence of random 'states' with random 'times'.

    This is really lazy/messy, but it works for now.
    '''
    states = [random.choice(WORDS) for _ in range(state_count)]
    # Random holding times between 12 hours and 1 hour
    holds = [random.randint(1*60, 5*60) for _ in range(state_count)]
    total_time = 0
    events = []
    while total_time < 60*72: # 72 hours for now
        i = random.randint(0, len(states)-1)
        dwell = poisson.rvs(holds[i])
        state = states[i]
        time = 0
        while time < dwell:
            events.append((state, total_time))
            step = random.gauss(0.5, .05)  # Basicaly one event every 30 seconds
            time += step
            total_time += step
    return events

class TestMarkovSketch:
    """Main test class."""

    def test_init(self):
        ac = MarkovSketch()
        assert len(ac.states) < 1
        # More asserts need to be added here

    def test_add(self):
        ps = Events()
        ac = MarkovSketch()
        for p in ps:
            ac += p
        assert len(ac) == len(set(i[0] for i in ps))
        assert ac.npoints == len(ps)
        # More asserts need to be added here

    def test_batch(self):
        ps = Events()
        ac = MarkovSketch()
        assert len(ac) == 0
        ac.batch(ps)
        assert ac.npoints == len(ps)
        assert len(ac) == len(set(i[0] for i in ps))

    def test_compare_batch_add(self):
        ps = Events()
        ac1 = MarkovSketch()
        ac2 = MarkovSketch()
        for p in ps:
            ac1 += p
        ac2.batch(ps)
        for a, b in zip(ac1.states, ac2.states):
            assert contains_same(a, b)
        for a, b in zip(ac1, ac2):
            assert a[0] == b[0]
            assert a[1].keys() == b[1].keys()

    def test_len(self):
        ps = Events()
        ac = MarkovSketch()
        assert len(ac) == 0
        ac.batch(ps)
        print(ac.states)
        assert len(ac) == len(set(i[0] for i in ps))

    def test_contains(self):
        ps = Events()
        ac = MarkovSketch()
        assert ps[0][0] not in ac
        ac.batch(ps)
        assert ps[0][0] in ac

    def test_call_and_states(self):
        ps = Events()
        ac = MarkovSketch()
        # assert len(ac()) == len(ac.states) == 0
        ac.batch(ps)
        # assert len(ac()) == len(ac.states) == len(set(i[0] for i in ps))

    def test_iter(self):
        ps = Events()
        ac = MarkovSketch().batch(ps)
        assert hasattr(ac, '__iter__')
        assert ac.npoints == len(ps)

    def test_npoints(self):
        ps = Events()
        ac = MarkovSketch().batch(ps)
        assert ac.npoints == len(ps)
        assert ac.npoints > len(ac)
        assert isinstance(ac.npoints, int)

    def test_durations(self):
        pass

max_stars_repo_name: carsonfarmer/predicure

max_issues_repo_name: carsonfarmer/predicure

max_forks_repo_name: carsonfarmer/predicure


text:
"""
==========================================================================
Using the free water elimination model to remove free water contamination
==========================================================================

As shown previously (see :ref:`example_reconst_dti`), the diffusion tensor
model is a simple way to characterize diffusion anisotropy. However, in regions
near the cerebral ventricle and parenchyma can be underestimated by partial
volume effects of the cerebral spinal fluid (CSF). This free water
contamination can particularly corrupt diffusion tensor imaging analysis of
microstructural changes when different groups of subject show different brain
morphology (e.g. brain ventricle enlargement associated with brain tissue
atrophy that occurs in several brain pathologies and ageing).

A way to remove this free water influences is to expand the DTI model to take
into account an extra compartment representing the contributions of free water
diffusion. The expression of the expanded DTI model is shown below:

.. math::

    S(\mathbf{g}, b) = S_0(1-f)e^{-b\mathbf{g}^T \mathbf{D}
    \mathbf{g}}+S_0fe^{-b D_{iso}}

where $\mathbf{g}$ and $b$ are diffusion gradient direction and weighted
(more information see :ref:`example_reconst_dti`), $S(\mathbf{g}, b)$ is the
diffusion-weighted signal measured, $S_0$ is the signal in a measurement with
no diffusion weighting, $\mathbf{D}$ is the diffusion tensor, $f$ the volume
fraction of the free water component, and $D_iso$ is the isotropic value of the
free water diffusion (normally set to $3.0 \times 10^{-3} mm^{2}s^{-1}$).

In this example, we show how to process a diffusion weighting dataset using the
free water elimination.

Let's start by importing the relevant modules:
"""

import numpy as np
import dipy.reconst.fwdti as fwdti
import dipy.reconst.dti as dti
import matplotlib.pyplot as plt
from dipy.data import fetch_cenir_multib
from dipy.data import read_cenir_multib
from dipy.segment.mask import median_otsu

"""
Without spatial constrains the free water elimination model cannot be solved
in data acquired from one non-zero b-value _[Hoy2014]. Therefore, here we
download a dataset that was required from multiple b-values.
"""

fetch_cenir_multib(with_raw=False)

"""
From the downloaded data, we read only the data acquired with b-values up to
2000 $s.mm^{-2} to decrease the influence of non-Gaussian diffusion effects
of the tisse which are not taken into account by the free water elimination
model _[Hoy2014].
"""

bvals = [200, 400, 1000, 2000]

img, gtab = read_cenir_multib(bvals)

data = img.get_data()

affine = img.affine

"""
The free water DTI model can take some minutes to process the full data set.
Thus, we remove the background of the image to avoid unnecessary calculations.
"""

maskdata, mask = median_otsu(data, 4, 2, False, vol_idx=[0, 1], dilate=1)

"""
Moreover, for illustration purposes we process only an axial slice of the
data.
"""

axial_slice = 40

mask_roi = np.zeros(data.shape[:-1], dtype=bool)
mask_roi[:, :, axial_slice] = mask[:, :, axial_slice]

"""
The free water elimination model fit can then be initialized by instantiating
a FreeWaterTensorModel class object:
"""

fwdtimodel = fwdti.FreeWaterTensorModel(gtab)

"""
The data can then be fitted using the ``fit`` function of the defined model
object:
"""

fwdtifit = fwdtimodel.fit(data, mask=mask_roi)

"""
This 2-steps procedure will create a FreeWaterTensorFit object which contains
all the diffusion tensor statistics free for free water contaminations. Below
we extract the fractional anisotropy (FA) and the mean diffusivity (MD) of the
free water diffusion tensor."""

FA = fwdtifit.fa
MD = fwdtifit.md

"""
For comparison we also compute the same standard measures processed by the
standard DTI model
"""

dtimodel = dti.TensorModel(gtab)

dtifit = dtimodel.fit(data, mask=mask_roi)

dti_FA = dtifit.fa
dti_MD = dtifit.md

"""
Below the FA values for both free water elimnantion DTI model and standard DTI
model are plotted in panels A and B, while the repective MD values are ploted
in panels D and E. For a better visualization of the effect of the free water
correction, the differences between these two metrics are shown in panels C and
E. In addition to the standard diffusion statistics, the estimated volume
fraction of the free water contamination is shown on panel G.
"""

fig1, ax = plt.subplots(2, 4, figsize=(12, 6),
                        subplot_kw={'xticks': [], 'yticks': []})

fig1.subplots_adjust(hspace=0.3, wspace=0.05)
ax.flat[0].imshow(FA[:, :, axial_slice].T, origin='lower',
                  cmap='gray', vmin=0, vmax=1)
ax.flat[0].set_title('A) fwDTI FA')
ax.flat[1].imshow(dti_FA[:, :, axial_slice].T, origin='lower',
                  cmap='gray', vmin=0, vmax=1)
ax.flat[1].set_title('B) standard DTI FA')

FAdiff = abs(FA[:, :, axial_slice] - dti_FA[:, :, axial_slice])
ax.flat[2].imshow(FAdiff.T, cmap='gray', origin='lower', vmin=0, vmax=1)
ax.flat[2].set_title('C) FA difference')

ax.flat[3].axis('off')

ax.flat[4].imshow(MD[:, :, axial_slice].T, origin='lower',
                  cmap='gray', vmin=0, vmax=2.5e-3)
ax.flat[4].set_title('D) fwDTI MD')
ax.flat[5].imshow(dti_MD[:, :, axial_slice].T, origin='lower',
                  cmap='gray', vmin=0, vmax=2.5e-3)
ax.flat[5].set_title('E) standard DTI MD')

MDdiff = abs(MD[:, :, axial_slice] - dti_MD[:, :, axial_slice])
ax.flat[6].imshow(MDdiff.T, origin='lower', cmap='gray', vmin=0, vmax=2.5e-3)
ax.flat[6].set_title('F) MD difference')

F = fwdtifit.f

ax.flat[7].imshow(F[:, :, axial_slice].T, origin='lower',
                  cmap='gray', vmin=0, vmax=1)
ax.flat[7].set_title('G) free water volume')

plt.show()
fig1.savefig('In_vivo_free_water_DTI_and_standard_DTI_measures.png')

"""
.. figure:: In_vivo_free_water_DTI_and_standard_DTI_measures.png
   :align: center
   ** In vivo diffusion measures obtain from the free water DTI and standard
   DTI. The values of Fractional Anisotropy for the free water DTI model and
   standard DTI model and their difference are shown in the upper panels (A-C),
   while respective MD values are shown in the lower panels (D-F). In addition
   the free water volume fraction estimated from the fwDTI model is shown in
   panel G**.

From the figure, one can observe that the free water elimination model
produces in general higher values of FA and lower values of MD than the
standard DTI model. These differences in FA and MD estimation are expected
due to the suppression of the free water isotropic diffusion components.
Unexpected high amplitudes of FA are however observed in the periventricular
gray mater. This is a known artefact of regions associated to voxels with high
water volume fraction (i.e. voxels containing basically CSF). We are able to
remove this problematic voxels by excluding all FA values associated with
measured volume fractions above a reasonable threshold of 0.7:
"""

FA[F > 0.7] = 0
dti_FA[F > 0.7] = 0

"""
Above we reproduce the plots of the in vivo FA from the two DTI fits and where
we can see that the inflated FA values were practically removed:
"""

fig1, ax = plt.subplots(1, 3, figsize=(9, 3),
                        subplot_kw={'xticks': [], 'yticks': []})

fig1.subplots_adjust(hspace=0.3, wspace=0.05)
ax.flat[0].imshow(FA[:, :, axial_slice].T, origin='lower',
                  cmap='gray', vmin=0, vmax=1)
ax.flat[0].set_title('A) fwDTI FA')
ax.flat[1].imshow(dti_FA[:, :, axial_slice].T, origin='lower',
                  cmap='gray', vmin=0, vmax=1)
ax.flat[1].set_title('B) standard DTI FA')

FAdiff = abs(FA[:, :, axial_slice] - dti_FA[:, :, axial_slice])
ax.flat[2].imshow(FAdiff.T, cmap='gray', origin='lower', vmin=0, vmax=1)
ax.flat[2].set_title('C) FA difference')

plt.show()
fig1.savefig('In_vivo_free_water_DTI_and_standard_DTI_corrected.png')

"""
.. figure:: In_vivo_free_water_DTI_and_standard_DTI_corrected.png
   :align: center
   ** In vivo FA measures obtain from the free water DTI (A) and standard
   DTI (B) and their difference (C). Problematic inflated FA values of the
   images were removed by dismissing voxels above a volume fraction threshold
   of 0.7 **.

References:

.. [Hoy2014] Hoy, A.R., Koay, C.G., Kecskemeti, S.R., Alexander, A.L., 2014.
             Optimization of a free water elimination two-compartmental model
             for diffusion tensor imaging. NeuroImage 103, 323-333.
             doi: 10.1016/j.neuroimage.2014.09.053
"""

max_stars_repo_name: omarocegueda/dipy


text:
from argparse import Namespace

import numpy
import pytest
import torch

from espnet.nets.asr_interface import dynamic_import_asr
from espnet.nets.beam_search import BeamSearch
from espnet.nets.lm_interface import dynamic_import_lm
from espnet.nets.scorers.length_bonus import LengthBonus

rnn_args = Namespace(
    elayers=1,
    subsample="1_2_2_1_1",
    etype="vggblstm",
    eunits=16,
    eprojs=8,
    dtype="lstm",
    dlayers=1,
    dunits=16,
    atype="location",
    aheads=2,
    awin=5,
    aconv_chans=4,
    aconv_filts=10,
    lsm_type="",
    lsm_weight=0.0,
    sampling_probability=0.0,
    adim=16,
    dropout_rate=0.0,
    dropout_rate_decoder=0.0,
    nbest=5,
    beam_size=2,
    penalty=0.5,
    maxlenratio=1.0,
    minlenratio=0.0,
    ctc_weight=0.2,
    lm_weight=0.0,
    rnnlm=None,
    streaming_min_blank_dur=10,
    streaming_onset_margin=2,
    streaming_offset_margin=2,
    verbose=2,
    outdir=None,
    ctc_type="warpctc",
    report_cer=False,
    report_wer=False,
    sym_space="<space>",
    sym_blank="<blank>",
    sortagrad=0,
    grad_noise=False,
    context_residual=False,
    use_frontend=False,
    replace_sos=False,
    tgt_lang=False
)

transformer_args = Namespace(
    adim=16,
    aheads=2,
    dropout_rate=0.0,
    transformer_attn_dropout_rate=None,
    elayers=2,
    eunits=16,
    dlayers=2,
    dunits=16,
    sym_space="<space>",
    sym_blank="<blank>",
    transformer_init="pytorch",
    transformer_input_layer="conv2d",
    transformer_length_normalized_loss=True,
    report_cer=False,
    report_wer=False,
    ctc_type="warpctc",
    lsm_weight=0.001,
)

# from test.test_e2e_asr_transformer import prepare
def prepare(E2E, args, mtlalpha=0.0):
    args.mtlalpha = mtlalpha
    args.char_list = ['a', 'e', 'i', 'o', 'u']
    idim = 40
    odim = 5
    model = dynamic_import_asr(E2E, "pytorch")(idim, odim, args)
    batchsize = 5
    x = torch.randn(batchsize, 40, idim)
    ilens = [40, 30, 20, 15, 10]
    n_token = odim - 1
    # avoid 0 for eps in ctc
    y = (torch.rand(batchsize, 10) * n_token % (n_token - 1)).long() + 1
    olens = [3, 9, 10, 2, 3]
    for i in range(batchsize):
        x[i, ilens[i]:] = -1
        y[i, olens[i]:] = -1

    data = []
    for i in range(batchsize):
        data.append(("utt%d" % i, {
            "input": [{"shape": [ilens[i], idim]}],
            "output": [{"shape": [olens[i]]}]
        }))
    return model, x, torch.tensor(ilens), y, data, args

@pytest.mark.parametrize(
    "model_class, args, ctc_weight, lm_weight, bonus, device, dtype",
    [(nn, args, ctc, lm, bonus, device, dtype)
     for device in ("cpu", "cuda")
     for nn, args in (("transformer", transformer_args), ("rnn", rnn_args))
     for ctc in (0.0, 0.5, 1.0)
     for lm in (0.0, 0.5)
     for bonus in (0.0, 0.1)
     for dtype in ("float16", "float32", "float64")]
)
def test_beam_search_equal(model_class, args, ctc_weight, lm_weight, bonus, device, dtype):
    if device == "cuda" and not torch.cuda.is_available():
        pytest.skip("no cuda device is available")
    if device == "cpu" and dtype == "float16":
        pytest.skip("cpu float16 implementation is not available in pytorch yet")

    # seed setting
    torch.manual_seed(123)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False  # https://github.com/pytorch/pytorch/issues/6351

    dtype = getattr(torch, dtype)
    model, x, ilens, y, data, train_args = prepare(model_class, args, mtlalpha=ctc_weight)
    model.eval()
    char_list = train_args.char_list
    lm_args = Namespace(type="lstm", layer=1, unit=2, embed_unit=2, dropout_rate=0.0)
    lm = dynamic_import_lm("default", backend="pytorch")(len(char_list), lm_args)
    lm.eval()

    # test previous beam search
    args = Namespace(
        beam_size=3,
        penalty=bonus,
        ctc_weight=ctc_weight,
        maxlenratio=0,
        lm_weight=lm_weight,
        minlenratio=0,
        nbest=5
    )

    feat = x[0, :ilens[0]].numpy()
    # legacy beam search
    with torch.no_grad():
        nbest = model.recognize(feat, args, char_list, lm.model)

    # new beam search
    scorers = model.scorers()
    if lm_weight != 0:
        scorers["lm"] = lm
    scorers["length_bonus"] = LengthBonus(len(char_list))
    weights = dict(decoder=1.0 - ctc_weight, ctc=ctc_weight, lm=args.lm_weight, length_bonus=args.penalty)
    model.to(device, dtype=dtype)
    model.eval()
    beam = BeamSearch(
        beam_size=args.beam_size,
        vocab_size=len(char_list),
        weights=weights,
        scorers=scorers,
        token_list=train_args.char_list,
        sos=model.sos,
        eos=model.eos,
    )
    beam.to(device, dtype=dtype)
    beam.eval()
    with torch.no_grad():
        enc = model.encode(torch.as_tensor(feat).to(device, dtype=dtype))
        nbest_bs = beam(x=enc, maxlenratio=args.maxlenratio, minlenratio=args.minlenratio)
    if dtype == torch.float16:
        # skip because results are different. just checking it is decodable
        return

    for i, (expected, actual) in enumerate(zip(nbest, nbest_bs)):
        actual = actual.asdict()
        assert expected["yseq"] == actual["yseq"]
        numpy.testing.assert_allclose(expected["score"], actual["score"], rtol=1e-6)

max_stars_repo_name: LuoTianqi/espnet

max_issues_repo_name: actuy/espnet

max_forks_repo_name: actuy/espnet


text:
import random
import os
import numpy as np
import torch
import pandas as pd
from mean_average_precision import MetricBuilder

from dataset import classes

def seed_everything(seed=123):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)

def refine_dataframe(in_df):
    out_df = []
    for studyid, grp in in_df.groupby('studyid'):
        if len(grp) > 1:
            check = False
            for _, row in grp.iterrows():
                if row['hasbox']:
                    check = True
            if check:
                tmp_df = grp.loc[grp['hasbox']==True]
                out_df.append(tmp_df)
            else:
                out_df.append(grp)
        else:
            out_df.append(grp)

    out_df = pd.concat(out_df, ignore_index=True)
    return out_df

def get_study_map(df, pred_dict, num_classes=6, stride=0.1):
    assert num_classes in [4,6]
    metric_fn = MetricBuilder.build_evaluation_metric("map_2d", async_mode=True, num_classes=num_classes)

    ### Study level ###
    for studyid, grp in df.groupby('studyid'):
        gts = []
        for clsidx, clsname in enumerate(classes):
            assert len(np.unique(grp[clsname].values)) == 1
            if grp[clsname].values[0] == 1:
                gts.append([0, 0, 1, 1, clsidx, 0, 0])
        gts = np.array(gts)

        study_preds = []
        for _, row in grp.iterrows():
            study_preds.append(pred_dict[row['imageid']])
        study_preds = np.array(study_preds)
        study_preds = np.mean(study_preds, axis=0)

        preds = []
        for clsidx in range(len(classes)):
            preds.append([0, 0, 1, 1, clsidx, study_preds[clsidx]])
        preds = np.array(preds)

        metric_fn.add(preds, gts)

    ### Image level ###
    if num_classes == 6:
        for _, row in df.iterrows():
            gts = []
            arr = row['label'].split(' ')
            nums = len(arr) // 6
            for i in range(nums):
                class_name = arr[6*i]
                conf = int(arr[6*i+1])
                if class_name == 'opacity':
                    clsid = 5
                else:
                    clsid = 4
                x1 = int(float(arr[6*i+2]))
                y1 = int(float(arr[6*i+3]))
                x2 = int(float(arr[6*i+4]))
                y2= int(float(arr[6*i+5]))
                gts.append([x1, y1, x2, y2, clsid, 0, 0])
            gts = np.array(gts)

            preds = np.array([[0, 0, 1, 1, 4, 1]])

            metric_fn.add(preds, gts)

    result = metric_fn.value(iou_thresholds=0.5, recall_thresholds=np.arange(0., 1.0+stride, stride), mpolicy='soft')
    average_precision = {}
    for clsid in range(num_classes):
        average_precision[clsid] = []
    for k, v in result.items():
        if k=='mAP':
            continue
        for clsid in range(num_classes):
            average_precision[clsid].append(v[clsid]['ap'])

    output = {
        'mAP': result['mAP'],
    }
    for clsid in range(num_classes):
        average_precision[clsid] = np.mean(average_precision[clsid])
        if clsid < len(classes):
            output[classes[clsid]] = average_precision[clsid]
        elif clsid == 4:
            output['none'] = average_precision[clsid]
        else:
            output['opacity'] = average_precision[clsid]
    return output


text:
import arboretum
import numpy as np
from sklearn.datasets import load_boston
import json
from sklearn.model_selection import train_test_split

def rmse(y, y_hat):
    diff = np.power(y - y_hat, 2)
    return np.sqrt(np.mean(diff))

if __name__ == "__main__":
    boston = load_boston()

    data_train, data_test, y_train,  y_test = train_test_split(
        boston.data, boston.target, test_size=0.2, random_state=42)

    data = arboretum.DMatrix(data_train, y=y_train)

    config = {'objective': 0,
              'internals':
              {
                  'double_precision': True,
                  'compute_overlap': 1,
              },
              'verbose':
              {
                  'gpu': True
              },
              'tree':
              {
                  'eta': .1,
                  'max_depth': 5,
                  'gamma': 0.0,
                  'min_child_weight': 2,
                  'min_leaf_size': 2,
                  'colsample_bytree': 0.8,
                  'colsample_bylevel': 0.8,
                  'lambda': 0.0,
                  'alpha': 0.0
              }}

    # init model
    model = arboretum.Garden(config, data)

    # grow trees
    for i in range(50):
        model.grow_tree()

    # predict on train data set
    pred_train = model.predict(data)

    pred_test = model.predict(arboretum.DMatrix(data_test))

    # print first n records from train set
    print(pred_train[0:10], y_train[0:10])

    # print first n records from test set
    print(pred_test[0:10], y_test[0:10])

    # RMSE
    print('train:', rmse(pred_train, y_train),
          'test:', rmse(pred_test, y_test))


text:
'''

Multi Layer Perceptron

Programmed partially or completely by Gautam Sharma and partial code taken from official sklearn documentation

We start by importing appropriate libraries

'''

import pandas as pd
import numpy as np
import sklearn
from sklearn.neural_network import MLPRegressor
from sklearn import linear_model
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as pyplot

'''
Importing data from project2data.csv file provided in .zip file
----------------------------------------------------------------------
'''
data=pd.read_csv("final.csv",sep=",")
data=data[["Temperature AVG","Relative Humidity AVG","Wind Speed Daily AVG","Success Percentage"]]

print (data.head())  # prints first five elements

predict="Success Percentage"
year="Date"

X= np.array(data.drop([predict],1))
y= np.array(data[predict])

x_train, x_test, y_train, y_test= sklearn.model_selection.train_test_split(X,y,test_size=0.2)

"""
Starting MLP
----------------------------------------------------------------------
"""

MLP= MLPRegressor(hidden_layer_sizes=(8,), activation='relu', solver='adam',  alpha=0.05, batch_size='auto',
                  learning_rate='constant',
                  learning_rate_init=0.01, power_t=0.5, max_iter=2000, shuffle=True, random_state=None, tol=0.0001,
                  verbose=False,
                  warm_start=False, momentum=0.5, nesterovs_momentum=True, early_stopping=False, validation_fraction=0.1,
                  beta_1=0.9, beta_2=0.999, epsilon=1e-08, n_iter_no_change=10, max_fun=15000)

"""
Training and Testing Phase
----------------------------------------------------------------------
"""

MLP.fit(x_train,y_train)
y_pred= MLP.predict(x_test)
y_pred1= MLP.predict(x_train)
MLP.fit(x_train,y_train)
"""
Calculating RMSE for all 6 cases
----------------------------------------------------------------------
"""

RMSE=mean_squared_error(y_test, y_pred, squared=False)
print("RMSE: ", RMSE)
RMSE1=mean_squared_error(y_train, y_pred1 ,squared=False)
print("RMSE: ", RMSE1)

score= MLP.score( x_test, y_test, sample_weight=None)
print(score)

"""
Plotting loss functions as a function of iterations
----------------------------------------------------------------------
"""
fig, ax = pyplot.subplots()

ax.set(xlabel='Iterations', ylabel='Loss',
       title='Loss function convergence with increasing features')
ax.grid()

pyplot.plot(MLP.loss_curve_)

ax.legend([RMSE])

pyplot.show()


text:
## import camera GPIO pin signals

### IMPORT VIDEOS
## Define DIRECTORY of video(s) to use and IMPORT videos (as read objects) into openCV
## Be careful to follow input the directories properly below. Input directory
## and file name (or file name prefix) in either the m

import os
import cv2
import numpy as np

def getVideoDirectoryProperties(multiple_files_pref , dir_vid , fileName_vid_prefix , fileName_vid_suffix , fileName_vid , slash_type , print_fileNames_pref):
    """
    gets video directory properties
    Args:
        multiple_files_pref (bool): whether or not you are looking for multiple files
        dir_vid (string): path to video(s)
    Returns:
    """

    # # This option imports all of the videos with a defined file name prefix in a folder
    # # OR just imports a single defined file
    # multiple_files_pref = 1

    # dir_vid = r'/media/rich/bigSSD RH/res2p/Camera data/round 4 experiments/mouse 6.28/20201102/cam1'
    # # dir_vid = r'/media/rich/bigSSD RH/res2p/Camera data/round 4 experiments/mouse 6.28/20201102/cam1'

    # # Used only if 'multiple_files_pref'==1
    # fileName_vid_prefix = 'cam1_2020-11-02-185734-'
    # fileName_vid_suffix = '.avi'

    # # Used only if 'multiple_files_pref'==0
    # fileName_vid = 'gmou06_082720_faceTrack_session1_DeInter100.avi'

    ### == IMPORT videos ==
    print_fileNames_pref = 1

    if multiple_files_pref:
        ## first find all the files in the directory with the file name prefix
        fileNames_allInPathWithPrefix = []
        for ii in os.listdir(dir_vid):
            if os.path.isfile(os.path.join(dir_vid,ii)) and fileName_vid_prefix in ii and fileName_vid_suffix in ii:
                fileNames_allInPathWithPrefix.append(ii)
        numVids = len(fileNames_allInPathWithPrefix)

        ## make a variable containing all of the file paths
        path_vid_allFiles = list()
        for ii in range(numVids):
            path_vid_allFiles.append(f'{dir_vid}{slash_type}{fileNames_allInPathWithPrefix[ii]}')

    else: ## Single file import
        path_vid = f'{dir_vid}{slash_type}{fileName_vid}'
        path_vid_allFiles = list()
        path_vid_allFiles.append(path_vid)
        numVids = 1
    path_vid_allFiles = sorted(path_vid_allFiles)

    ## get info on the imported video(s): num of frames, video height and width, framerate
    if multiple_files_pref:
        path_vid = path_vid_allFiles[0]
        video = cv2.VideoCapture(path_vid_allFiles[0])
        numFrames_firstVid = int(video.get(cv2.CAP_PROP_FRAME_COUNT))

        numFrames_allFiles = np.ones(numVids) * np.nan # preallocation
        for ii in range(numVids):
            video = cv2.VideoCapture(path_vid_allFiles[ii])
            numFrames_allFiles[ii] = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
        numFrames_total_rough = np.uint64(sum(numFrames_allFiles))

        print(f'number of videos: {numVids}')
        print(f'number of frames in FIRST video (roughly):  {numFrames_firstVid}')
        print(f'number of frames in ALL videos (roughly):   {numFrames_total_rough}')
    else:
        video = cv2.VideoCapture(path_vid_allFiles[0])
        numFrames_onlyVid = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
        numFrames_total_rough = numFrames_onlyVid
        numFrames_allFiles = numFrames_total_rough
        print(f'number of frames in ONLY video:   {numFrames_onlyVid}')

    Fs = video.get(cv2.CAP_PROP_FPS) ## Sampling rate (FPS). Manually change here if necessary
    print(f'Sampling rate pulled from video file metadata:   {round(Fs,3)} frames per second')

    if print_fileNames_pref:
        print(f'\n {np.array(path_vid_allFiles).transpose()}')

    video.set(1,1)
    ok, frame = video.read()
    vid_height = frame.shape[0]
    vid_width = frame.shape[1]

    return path_vid_allFiles , vid_height , vid_width , numFrames_total_rough , Fs


text:
import contextlib
import itertools
import string
import unittest

from pudb.var_view import (
    BasicValueWalker,
    FrameVarInfo,
    InspectInfo,
    PudbCollection,
    PudbMapping,
    PudbSequence,
    STRINGIFIERS,
    ValueWalker,
    get_stringifier,
    ui_log,
)

class A:
    pass

class A2:
    pass

def test_get_stringifier():
    try:
        import numpy as np
    except ImportError:
        numpy_values = []
    else:
        numpy_values = [np.float32(5), np.zeros(5)]

    for value in [
            A, A2, A(), A2(), "lól".encode(), "lól",
            1233123, ["lól".encode(), "lól"],
            ] + numpy_values:
        for display_type in STRINGIFIERS:
            iinfo = InspectInfo()
            iinfo.display_type = display_type

            strifier = get_stringifier(iinfo)

            s = strifier(value)
            assert isinstance(s, str)

class FrameVarInfoForTesting(FrameVarInfo):
    def __init__(self, paths_to_expand=None):
        super().__init__()
        if paths_to_expand is None:
            paths_to_expand = set()
        self.paths_to_expand = paths_to_expand

    def get_inspect_info(self, id_path, read_only):
        iinfo = super().get_inspect_info(
            id_path, read_only)
        iinfo.access_level = "all"
        iinfo.display_type = "repr"
        iinfo.show_methods = True

        if id_path in self.paths_to_expand:
            iinfo.show_detail = True
        return iinfo

class Reasonable:
    def __init__(self):
        self.x = 42

    def bar(self):
        return True

    @property
    def red(self):
        return "red"

    @classmethod
    def blue(cls):
        return "blue"

    @staticmethod
    def green():
        return "green"

    def _private(self):
        return "shh"

    def __magicsomething__(self):
        return "amazing"

def method_factory(method_name):
    def method(self, *args, **kwargs):
        func = getattr(self.__internal_dict__, method_name)
        try:
            return func(*args, **kwargs)
        except KeyError:
            # Classes without __iter__ are expected to raise IndexError in this
            # sort of case. Frustrating, I know.
            if (method_name == "__getitem__"
                    and args and isinstance(args[0], int)):
                raise IndexError
            raise
    return method

def generate_containerlike_class():
    methods = {
        "__contains__",
        "__getitem__",
        "__iter__",
        "__len__",
        "__reversed__",
        "count",
        "get",
        "index",
        "items",
        "keys",
        "values",
    }

    # Deliberately starting from 0
    for r in range(0, len(methods) + 1):
        for selected_methods in sorted(
                map(sorted, itertools.combinations(methods, r))):

            class ContainerlikeClass:
                def __init__(self, iterable):
                    self.__internal_dict__ = dict(iterable)

                @classmethod
                def name(cls):
                    return "ContainerlikeClass:{}".format(
                        ":".join(selected_methods))

            for method in selected_methods:
                func = method_factory(method)
                setattr(ContainerlikeClass, method, func)

            yield ContainerlikeClass

class BaseValueWalkerTestCase(unittest.TestCase):
    """
    There are no actual tests defined in this class, it provides utitlities
    useful for testing the variable view in variaous ways.
    """
    EMPTY_ITEM = (ValueWalker.EMPTY_LABEL, None)
    MOD_STR = " [all+()]"

    def setUp(self):
        self.values_to_expand = []
        self.class_counts = {
            "mappings": 0,
            "sequences": 0,
            "collections": 0,
            "other": 0,
        }

    def value_string(self, obj, expand=True):
        if expand and obj in self.values_to_expand:
            return repr(obj) + self.MOD_STR
        return repr(obj)

    def walked_values(self):
        return [(w.var_label, w.value_str)
                for w in self.walker.widget_list]

    def expected_attrs(self, obj):
        """
        `dir()` the object and return (label, value string) pairs for each
        attribute. Should match the order that these attributes would appear in
        the var_view.
        """
        return [("." + str(label),
                 self.value_string(getattr(obj, label), expand=False))
                for label in sorted(dir(obj))]

    @contextlib.contextmanager
    def patched_logging(self):
        """
        Context manager that patches ui_log.exception such that the test will
        fail if it is called.
        """
        def fake_exception_log(*args, **kwargs):
            self.fail("ui_log.exception was unexpectedly called")

        old_logger = ui_log.exception
        ui_log.exception = fake_exception_log
        try:
            yield
        finally:
            ui_log.exception = old_logger

    def assert_walks_contents(self, container, label="xs"):
        expand_paths = {label}
        self.values_to_expand = [container]
        self.walker = BasicValueWalker(FrameVarInfoForTesting(expand_paths))

        # Build out list of extected view contents according to container type.
        expected = [(label, self.value_string(container))]
        if isinstance(container, PudbMapping):
            expected.extend([(f"[{repr(key)}]", repr(container[key]))
                             for key in container.keys()]
                            or [self.EMPTY_ITEM])
            self.class_counts["mappings"] += 1
        elif isinstance(container, PudbSequence):
            expected.extend([(f"[{repr(index)}]", repr(entry))
                             for index, entry in enumerate(container)]
                            or [self.EMPTY_ITEM])
            self.class_counts["sequences"] += 1
        elif isinstance(container, PudbCollection):
            expected.extend([("[]", repr(entry))
                             for entry in container]
                            or [self.EMPTY_ITEM])
            self.class_counts["collections"] += 1
        else:
            self.class_counts["other"] += 1
        expected.extend(self.expected_attrs(container))

        with self.patched_logging():
            self.walker.walk_value(parent=None, label=label, value=container)

        received = self.walked_values()
        self.assertListEqual(expected, received)

    def assert_class_counts_equal(self, seen=None):
        """
        This is kinda weird since at first it looks like its testing the test
        code, but really it's testing the `isinstance` checks. But it is also
        true that it tests the test code, kind of like a sanity check.
        """
        expected = {
            "mappings": 0,
            "sequences": 0,
            "collections": 0,
            "other": 0,
        }
        if seen is not None:
            expected.update(seen)
        self.assertDictEqual(expected, self.class_counts)

class ValueWalkerTest(BaseValueWalkerTestCase):
    def test_simple_values(self):
        self.walker = BasicValueWalker(FrameVarInfoForTesting())
        values = [
            # numbers
            0,
            1,
            -1234567890412345243,
            float(4.2),
            float("inf"),
            complex(1.3, -1),

            # strings
            "",
            "a",
            "foo bar",
            "  lots\tof\nspaces\r ",
            "♫",

            # other
            False,
            True,
            None,
        ]
        for label, value in enumerate(values):
            with self.patched_logging():
                self.walker.walk_value(parent=None,
                                       label=str(label),
                                       value=value)

        expected = [(str(_label), repr(x)) for _label, x in enumerate(values)]
        received = self.walked_values()
        self.assertListEqual(expected, received)

    def test_simple_values_expandable(self):
        """
        Simple values like numbers and strings are now expandable so we can
        peak under the hood and take a look at their attributes. Make sure
        that's working properly.
        """
        values = [
            # numbers
            0,
            1,
            -1234567890412345243,
            float(4.2),
            float("inf"),
            complex(1.3, -1),

            # strings
            "",
            "a",
            "foo bar",
            # "  lots\tof\nspaces\r ",  # long, hits continuation item
            "♫",

            # other
            False,
            True,
            None,
        ]
        for value in values:
            self.assert_walks_contents(value)

    def test_set(self):
        self.assert_walks_contents({
            42, "foo", None, False, (), ("a", "tuple")
        })
        self.assert_class_counts_equal({"collections": 1})

    def test_frozenset(self):
        self.assert_walks_contents(frozenset([
            42, "foo", None, False, (), ("a", "tuple")
        ]))
        self.assert_class_counts_equal({"collections": 1})

    def test_dict(self):
        self.assert_walks_contents({
            0:                   42,
            "a":                 "foo",
            "":                  None,
            True:                False,
            frozenset(range(3)): "abc",
            ():                  "empty tuple",
            (1, 2, "c", ()):     "tuple",
        })
        self.assert_class_counts_equal({"mappings": 1})

    def test_list(self):
        self.assert_walks_contents([
            42, "foo", None, False, (), ("a", "tuple")
        ])
        self.assert_class_counts_equal({"sequences": 1})

    def test_tuple(self):
        self.assert_walks_contents((
            42, "foo", None, False, (), ("a", "tuple")
        ))
        self.assert_class_counts_equal({"sequences": 1})

    def test_containerlike_classes(self):
        class_count = 0
        for cls_idx, containerlike_class in enumerate(
                generate_containerlike_class()):
            label = containerlike_class.name()
            value = containerlike_class(zip(string.ascii_lowercase,
                                            range(3, 10)))
            self.assert_walks_contents(container=value, label=label)
            class_count = cls_idx + 1

        self.assert_class_counts_equal({
            "mappings": 256,
            "sequences": 256,
            "collections": 256,
            "other": 1280,
        })

        walked_total = (self.class_counts["mappings"]
                        + self.class_counts["sequences"]
                        + self.class_counts["collections"]
                        + self.class_counts["other"])

        # +1 here because enumerate starts from 0, not 1
        self.assertEqual(class_count, walked_total)

    def test_empty_frozenset(self):
        self.assert_walks_contents(frozenset())

    def test_empty_set(self):
        self.assert_walks_contents(set())

    def test_empty_dict(self):
        self.assert_walks_contents(dict())

    def test_empty_list(self):
        self.assert_walks_contents(list())

    def test_reasonable_class(self):
        """
        Are the class objects themselves expandable?
        """
        self.assert_walks_contents(Reasonable, label="Reasonable")
        self.assert_class_counts_equal({"other": 1})

    def test_maybe_unreasonable_classes(self):
        """
        Are class objects, that might look like containers if we're not
        careful, reasonably expandable?
        """
        for containerlike_class in generate_containerlike_class():
            self.assert_walks_contents(
                container=containerlike_class,
                label=containerlike_class.name()
            )

        # This effectively makes sure that class definitions aren't considered
        # containers.
        self.assert_class_counts_equal({"other": 2048})


text:
import numpy as np
import skimage.transform

def reshape_normalized_box(box_normalized,
                           image_size):
  x_min, y_min, x_max, y_max = box_normalized
  new_height, new_width = image_size
  x_min = x_min * new_width
  y_min = y_min * new_height
  x_max = x_max * new_width
  y_max = y_max * new_height

  return [x_min, y_min, x_max, y_max]

def resize(mask,
           size):
  return skimage.transform.resize(mask, size,
                           order=1, mode='constant', cval=0,
                           clip=True, preserve_range=False)

def reshape_single_mask(box_normalized,
                        mask,
                        class_id,
                        image_size,
                        threshold):

  [x_min, y_min, x_max, y_max] = reshape_normalized_box(box_normalized,
                                                      image_size)
  mask = mask[class_id]
  mask = resize(mask, (int(y_max) - int(y_min), int(x_max) - int(x_min)))
  mask = np.where(mask >= threshold, 1, 0).astype(np.uint8)

  full_mask = np.zeros(image_size, dtype=np.uint8)
  full_mask[int(y_min):int(y_max), int(x_min):int(x_max)] = mask

  return full_mask

def reshape_masks_to_image_size(boxes_normalized,
                                num_boxes,
                                classes,
                                masks,
                                image_size,
                                threshold=0.5):
  """

  Args:
    boxes_normalized: [N, 4]
    num_boxes: []
    masks: [N, Hm, Wm] (Usually [N, 33, 33])
    image_size: [2] = (H, W)
    threshold: []

  Returns: [N, H, W]

  """

  full_masks = list()
  for i in range(num_boxes):
    full_mask = reshape_single_mask(boxes_normalized[i],
                                    masks[i],
                                    classes[i],
                                    image_size,
                                    threshold)
    full_masks.append(full_mask)
  if num_boxes > 0:
    full_masks = np.stack(full_masks, axis=0)
  else:
    full_masks = np.zeros(image_size)

  return full_masks