text:
#参考：https://www.cnblogs.com/54hys/p/12325182.html#title1
#https://blog.csdn.net/stupid_3/article/details/83184691
#https://github.com/graykode/nlp-tutorial/blob/master/5-2.BERT/BERT.py
#https://github.com/jadore801120/attention-is-all-you-need-pytorch/blob/master/
# https://github.com/jadore801120/attention-is-all-you-need-pytorch/blob/master/transformer/Models.py
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

def get_pad_mask(seq, pad_idx):
    return (seq != pad_idx).unsqueeze(-2)

def get_subsequent_mask(seq):
    ''' For masking out the subsequent info. '''
    sz_b, len_s = seq.size()
    subsequent_mask = (1 - torch.triu(
        torch.ones((1, len_s, len_s), device=seq.device), diagonal=1)).bool()
    return subsequent_mask

class PositionalEncoding(nn.Module):
    def __init__(self, embed_size, dropout, max_len=1000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.P = np.zeros((1, max_len, embed_size))
        X = np.arange(0, max_len).reshape(-1, 1) / np.power(10000, np.arange(0, embed_size, 2) / embed_size)
        self.P[:, :, 0::2] = np.sin(X)
        self.P[:, :, 1::2] = np.cos(X)
        self.P = torch.FloatTensor(self.P)

    def forward(self, X):
        if X.is_cuda and not self.P.is_cuda:
            self.P = self.P.cuda()
        X = X + self.P[:, :X.shape[1], :]
        return self.dropout(X)

class ScaledDotProductAttention(nn.Module):
    ''' Scaled Dot-Product Attention '''

    def __init__(self, temperature, attn_dropout=0.1):
        super().__init__()
        self.temperature = temperature
        self.dropout = nn.Dropout(attn_dropout)

    def forward(self, q, k, v, mask=None):

        attn = torch.matmul(q / self.temperature, k.transpose(2, 3))

        if mask is not None:
            attn = attn.masked_fill(mask == 0, -1e9)

        attn = self.dropout(F.softmax(attn, dim=-1))
        output = torch.matmul(attn, v)

        return output, attn

class MultiHeadAttention(nn.Module):
    ''' Multi-Head Attention module '''

    def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1):
        super().__init__()

        self.n_head = n_head
        self.d_k = d_k
        self.d_v = d_v

        self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False)
        self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False)
        self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False)
        self.fc = nn.Linear(n_head * d_v, d_model, bias=False)

        self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5)

        self.dropout = nn.Dropout(dropout)
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)

    def forward(self, q, k, v, mask=None):

        d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
        sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1)

        residual = q

        # Pass through the pre-attention projection: b x lq x (n*dv)
        # Separate different heads: b x lq x n x dv
        q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)
        k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)
        v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)

        # Transpose for attention dot product: b x n x lq x dv
        q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)

        if mask is not None:
            mask = mask.unsqueeze(1)   # For head axis broadcasting.

        q, attn = self.attention(q, k, v, mask=mask)

        # Transpose to move the head dimension back: b x lq x n x dv
        # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv)
        q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1)
        q = self.dropout(self.fc(q))
        q += residual

        q = self.layer_norm(q)

        return q, attn

class PositionwiseFeedForward(nn.Module):
    ''' A two-feed-forward-layer module '''

    def __init__(self, d_in, d_hid, dropout=0.1):
        super().__init__()
        self.w_1 = nn.Linear(d_in, d_hid) # position-wise
        self.w_2 = nn.Linear(d_hid, d_in) # position-wise
        self.layer_norm = nn.LayerNorm(d_in, eps=1e-6)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):

        residual = x

        x = self.w_2(F.relu(self.w_1(x)))
        x = self.dropout(x)
        x += residual

        x = self.layer_norm(x)

        return x

class EncoderLayer(nn.Module):
    ''' Compose with two layers '''

    def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1):
        super(EncoderLayer, self).__init__()
        self.slf_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
        self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout)

    def forward(self, enc_input, slf_attn_mask=None):

        enc_output, enc_slf_attn = self.slf_attn(enc_input, enc_input, enc_input, mask=slf_attn_mask)
        enc_output = self.pos_ffn(enc_output)
        return enc_output, enc_slf_attn

class DecoderLayer(nn.Module):
    ''' Compose with three layers '''

    def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1):
        super(DecoderLayer, self).__init__()
        self.slf_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
        self.enc_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
        self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout)

    def forward(self, dec_input, enc_output,slf_attn_mask=None, dec_enc_attn_mask=None):

        dec_output, dec_slf_attn = self.slf_attn(dec_input, dec_input, dec_input, mask=slf_attn_mask)
        dec_output, dec_enc_attn = self.enc_attn(dec_output, enc_output, enc_output, mask=dec_enc_attn_mask)
        dec_output = self.pos_ffn(dec_output)
        return dec_output, dec_slf_attn, dec_enc_attn

class Encoder(nn.Module):
    ''' A encoder model with self attention mechanism. '''

    def __init__(
            self, n_src_vocab, d_word_vec, n_layers, n_head, d_k, d_v,
            d_model, d_inner, pad_idx, dropout=0.1, n_position=200):

        super().__init__()

        self.src_word_emb = nn.Embedding(n_src_vocab, d_word_vec, padding_idx=pad_idx)
        self.position_enc = PositionalEncoding(d_word_vec, n_position=n_position)
        self.dropout = nn.Dropout(p=dropout)
        self.layer_stack = nn.ModuleList([
            EncoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=dropout)
            for _ in range(n_layers)])
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)

    def forward(self, src_seq, src_mask, return_attns=False):

        enc_slf_attn_list = []

        # -- Forward

        enc_output = self.dropout(self.position_enc(self.src_word_emb(src_seq)))
        enc_output = self.layer_norm(enc_output)

        for enc_layer in self.layer_stack:
            enc_output, enc_slf_attn = enc_layer(enc_output, slf_attn_mask=src_mask)
            enc_slf_attn_list += [enc_slf_attn] if return_attns else []

        if return_attns:
            return enc_output, enc_slf_attn_list
        return enc_output,

class Decoder(nn.Module):
    ''' A decoder model with self attention mechanism. '''

    def __init__(
            self, n_trg_vocab, d_word_vec, n_layers, n_head, d_k, d_v,
            d_model, d_inner, pad_idx, n_position=200, dropout=0.1):

        super().__init__()

        self.trg_word_emb = nn.Embedding(n_trg_vocab, d_word_vec, padding_idx=pad_idx)
        self.position_enc = PositionalEncoding(d_word_vec, n_position=n_position)
        self.dropout = nn.Dropout(p=dropout)
        self.layer_stack = nn.ModuleList([
            DecoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=dropout)
            for _ in range(n_layers)])
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)

    def forward(self, trg_seq, trg_mask, enc_output, src_mask, return_attns=False):

        dec_slf_attn_list, dec_enc_attn_list = [], []

        # -- Forward
        dec_output = self.dropout(self.position_enc(self.trg_word_emb(trg_seq)))
        dec_output = self.layer_norm(dec_output)

        for dec_layer in self.layer_stack:
            dec_output, dec_slf_attn, dec_enc_attn = dec_layer(
                dec_output, enc_output, slf_attn_mask=trg_mask, dec_enc_attn_mask=src_mask)
            dec_slf_attn_list += [dec_slf_attn] if return_attns else []
            dec_enc_attn_list += [dec_enc_attn] if return_attns else []

        if return_attns:
            return dec_output, dec_slf_attn_list, dec_enc_attn_list
        return dec_output,

class Transformer(nn.Module):
    ''' A sequence to sequence model with attention mechanism. '''

    def __init__(
            self, n_src_vocab, n_trg_vocab, src_pad_idx, trg_pad_idx,
            d_word_vec=512, d_model=512, d_inner=2048,
            n_layers=6, n_head=8, d_k=64, d_v=64, dropout=0.1, n_position=200,
            trg_emb_prj_weight_sharing=True, emb_src_trg_weight_sharing=True):

        super().__init__()

        self.src_pad_idx, self.trg_pad_idx = src_pad_idx, trg_pad_idx

        self.encoder = Encoder(
            n_src_vocab=n_src_vocab, n_position=n_position,
            d_word_vec=d_word_vec, d_model=d_model, d_inner=d_inner,
            n_layers=n_layers, n_head=n_head, d_k=d_k, d_v=d_v,
            pad_idx=src_pad_idx, dropout=dropout)

        self.decoder = Decoder(
            n_trg_vocab=n_trg_vocab, n_position=n_position,
            d_word_vec=d_word_vec, d_model=d_model, d_inner=d_inner,
            n_layers=n_layers, n_head=n_head, d_k=d_k, d_v=d_v,
            pad_idx=trg_pad_idx, dropout=dropout)

        self.trg_word_prj = nn.Linear(d_model, n_trg_vocab, bias=False)

        for p in self.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform_(p)

        assert d_model == d_word_vec, \
        'To facilitate the residual connections, \
         the dimensions of all module outputs shall be the same.'

        self.x_logit_scale = 1.
        if trg_emb_prj_weight_sharing:
            # Share the weight between target word embedding & last dense layer
            self.trg_word_prj.weight = self.decoder.trg_word_emb.weight
            self.x_logit_scale = (d_model ** -0.5)

        if emb_src_trg_weight_sharing:
            self.encoder.src_word_emb.weight = self.decoder.trg_word_emb.weight

    def forward(self, src_seq, trg_seq):
        #self.src_pad_idx
        src_mask = get_pad_mask(src_seq, self.src_pad_idx)
        trg_mask = get_pad_mask(trg_seq, self.trg_pad_idx) & get_subsequent_mask(trg_seq)

        enc_output, *_ = self.encoder(src_seq, src_mask)
        dec_output, *_ = self.decoder(trg_seq, trg_mask, enc_output, src_mask)
        seq_logit = self.trg_word_prj(dec_output) * self.x_logit_scale

        return seq_logit.view(-1, seq_logit.size(2))

max_stars_repo_name: Dou-Yu-xuan/deep-learning-visal


text:
import data
import numpy as np

from PIL import Image
from . import ARR_PATH

def fullsize_sequence():
    return data.fullsize_sequence(ARR_PATH, scale=2, subset='train', downgrade='bicubic', image_ids=range(1, 5))

def cropped_sequence(batch_size=2):
    return data.cropped_sequence(ARR_PATH, scale=2, subset='train', downgrade='bicubic', image_ids=range(1, 5), batch_size=batch_size)

def test_batch_size(conversion):
    sequence = cropped_sequence(batch_size=3)
    assert len(sequence) == 2

    lr_batch, hr_batch = sequence[0]
    assert lr_batch.shape == (3, 48, 48, 3)
    assert hr_batch.shape == (3, 96, 96, 3)

    lr_batch, hr_batch = sequence[1]
    assert lr_batch.shape == (1, 48, 48, 3)
    assert hr_batch.shape == (1, 96, 96, 3)

def test_full_image_size(conversion):
    sequence = fullsize_sequence()
    lr_batch, hr_batch = sequence[0]
    assert lr_batch.shape == (1, 702, 1020, 3)
    assert hr_batch.shape == (1, 1404, 2040, 3)

def test_cropped_image_size(conversion):
    sequence = cropped_sequence()
    lr_batch, hr_batch = sequence[0]
    assert lr_batch.shape == (2, 48, 48, 3)
    assert hr_batch.shape == (2, 96, 96, 3)

def test_bicubic_downscale_fullsize(conversion):
    assert_bicubic_downscale(fullsize_sequence())

def test_bicubic_downscale_cropped(conversion):
    assert_bicubic_downscale(cropped_sequence())

def assert_bicubic_downscale(sequence, bound=20):
    for i in range(len(sequence)):
        lr_batch, hr_batch = sequence[i]
        for lr, hr in zip(lr_batch, hr_batch):
            assert_bicubic_downscale_1(lr, hr, bound)

def assert_bicubic_downscale_1(lr, hr, max_diff=2, min_frac=0.97):
    """
    Asserts that the pixel-wise value difference between an lr image and a bicubic downscaled hr
    image is less than or equal `max_diff` for at least the given fraction (`min_frac`) of pixels.
    """

    ds = Image.fromarray(hr).resize((lr.shape[1], lr.shape[0]), resample=Image.BICUBIC)
    ds = np.array(ds, dtype='int16')

    lr = lr.astype('int16')
    diff = np.abs(lr - ds)

    assert np.sum(diff < 3) / np.size(lr) >= min_frac

ext: py

max_stars_repo_name: htleeab/super-resolution

reason: import numpy


text:
import h5py
import numpy as np
import seaborn as sns
import matplotlib as mpl
import matplotlib.pyplot as plt
import glob
import os
from tqdm import tqdm
from tqdm import trange
import pickle

def official():
    origin_dir = '/media/kadn/DATA2/AgentHuman/SeqTrain'
    h5files = glob.glob(os.path.join(origin_dir,'*.h5'))
    h5files = h5files[:]
    steer=[[],[],[],[]]

    count0 = 0

    for h5file in tqdm(h5files):
        try:
            data = h5py.File(h5file,'r')
            for i in range(data['targets'].shape[0]):
                target = data['targets'][i]
                if target[24] == 0:
                    count0 += 1
                    print('count %d' % count0, h5file)
                    continue
                steer[np.uint8(target[24]-2)].append(target[0])
            data.close()
        except Exception as exp:
            print(exp)
            print('filename: {}'.format(h5file))

    with open('/home/kadn/official.pk', 'wb') as f:
        pickle.dump(steer, f)

def officialxy():
    origin_dir = '/media/kadn/DATA2/AgentHuman/SeqTrain'
    h5files = glob.glob(os.path.join(origin_dir,'*.h5'))
    h5files = h5files[:]
    x,y = [],[]

    for h5file in tqdm(h5files):
        try:
            data = h5py.File(h5file,'r')
            for i in range(data['targets'].shape[0]):
                target = data['targets'][i]
                if target[24] == 0:
                    continue
                x.append(target[8]/100.0), y.append(target[9]/100.0)
            data.close()
        except Exception as exp:
            print(exp)
            print('filename: {}'.format(h5file))

    with open('/home/kadn/officialxy.pk', 'wb') as f:
        pickle.dump([x,y], f)

def ourdata_steer():
    data_dir = '/home/kadn/AUTODRIVING/data/*/epi*'

    h5files = glob.glob(os.path.join(data_dir,'*.h5'))
    h5files = h5files[:]
    steer = [[], [], [], []]

    for h5file in tqdm(h5files):
        data = h5py.File(h5file,'r')
        for i in range(data['targets'].shape[0]):
            target = data['targets'][i]
            if target[24] == 0:
                continue
            steer[np.uint8(target[24] - 2)].append(target[0])
    with open('/home/kadn/data8_14.pk', 'wb') as f:
        pickle.dump(steer, f)

ourdata_steer()

max_stars_repo_name: kadn/carla

max_issues_repo_name: kadn/carla

max_forks_repo_name: kadn/carla


text:
import numpy as np
import json, pickle, os, argparse

def parse_args():
    parser = argparse.ArgumentParser(description='Train the Frequenct Prior For RelDN.')
    parser.add_argument('--overlap', action='store_true',
                        help="Only count overlap boxes.")
    parser.add_argument('--json-path', type=str, default='~/.mxnet/datasets/visualgenome',
                        help="Only count overlap boxes.")
    args = parser.parse_args()
    return args

args = parse_args()
use_overlap = args.overlap
PATH_TO_DATASETS = os.path.expanduser(args.json_path)
path_to_json = os.path.join(PATH_TO_DATASETS, 'rel_annotations_train.json')

# format in y1y2x1x2
def with_overlap(boxA, boxB):
    xA = max(boxA[2], boxB[2])
    xB = min(boxA[3], boxB[3])

    if xB > xA:
        yA = max(boxA[0], boxB[0])
        yB = min(boxA[1], boxB[1])

        if yB > yA:
            return 1

    return 0

def box_ious(boxes):
    n = len(boxes)
    res = np.zeros((n, n))
    for i in range(n-1):
        for j in range(i+1, n):
            iou_val = with_overlap(boxes[i], boxes[j])
            res[i, j] = iou_val
            res[j, i] = iou_val
    return res

with open(path_to_json, 'r') as f:
    tmp = f.read()
    train_data = json.loads(tmp)

fg_matrix = np.zeros((150, 150, 51), dtype=np.int64)
bg_matrix = np.zeros((150, 150), dtype=np.int64)

for _, item in train_data.items():
    gt_box_to_label = {}
    for rel in item:
        sub_bbox = rel['subject']['bbox']
        ob_bbox = rel['object']['bbox']
        sub_class = rel['subject']['category']
        ob_class = rel['object']['category']
        rel_class = rel['predicate']

        sub_node = tuple(sub_bbox)
        ob_node = tuple(ob_bbox)
        if sub_node not in gt_box_to_label:
            gt_box_to_label[sub_node] = sub_class
        if ob_node not in gt_box_to_label:
            gt_box_to_label[ob_node] = ob_class

        fg_matrix[sub_class, ob_class, rel_class + 1] += 1

    if use_overlap:
        gt_boxes = [*gt_box_to_label]
        gt_classes = np.array([*gt_box_to_label.values()])
        iou_mat = box_ious(gt_boxes)
        cols, rows = np.where(iou_mat)
        if len(cols) and len(rows):
            for col, row in zip(cols, rows):
                bg_matrix[gt_classes[col], gt_classes[row]] += 1
        else:
            all_possib = np.ones_like(iou_mat, dtype=np.bool)
            np.fill_diagonal(all_possib, 0)
            cols, rows = np.where(all_possib)
            for col, row in zip(cols, rows):
                bg_matrix[gt_classes[col], gt_classes[row]] += 1
    else:
        for b1, l1 in gt_box_to_label.items():
            for b2, l2 in gt_box_to_label.items():
                if b1 == b2:
                    continue
                bg_matrix[l1, l2] += 1

eps = 1e-3
bg_matrix += 1
fg_matrix[:, :, 0] = bg_matrix
pred_dist = np.log(fg_matrix / (fg_matrix.sum(2)[:, :, None] + eps) + eps)

if use_overlap:
    with open('freq_prior_overlap.pkl', 'wb') as f:
        pickle.dump(pred_dist, f)
else:
    with open('freq_prior.pkl', 'wb') as f:
        pickle.dump(pred_dist, f)

max_stars_repo_name: ketyi/dgl

max_issues_repo_name: ketyi/dgl

max_forks_repo_name: ketyi/dgl


text:
# This file is part of the P3IV Simulator (https://github.com/fzi-forschungszentrum-informatik/P3IV),
# copyright by FZI Forschungszentrum Informatik, licensed under the BSD-3 license (see LICENSE file in main directory)

import numpy as np
import warnings

class Mixture(object):
    def __init__(self, *args, **kwargs):
        """
        A mixture of probability distributions
        :param weights: weights of the components of the mixture
        :param args: [truncation_upper_array, truncation_lower_array]
        :param kwargs: "mean", "covariance", "mixture_data_type", "components"
        """

        self.components = []  # Contains the GaussianData components of the mixture
        self.data_length = None
        self.nr_of_components = 0
        self.weights = []
        self.__mixture_data_type = None
        self._mean = None  # one row represents a one gaussian. Will be an array of arrays (2D).
        self._covariance = None  # one row represents a one gaussian.

        if all(i in list(kwargs.keys()) for i in ["mixture_data_type", "weights", "mean", "covariance"]):
            """
            if len(kwargs["mean"].shape) == 1:
                kwargs["mean"] = np.expand_dims(kwargs["mean"], axis=0)
            if len(kwargs["covariance"].shape) == 1:
                kwargs["covariance"] = np.expand_dims(kwargs["covariance"], axis=0)
            assert len(kwargs["mean"].shape) == 2
            assert kwargs["mean"].shape == kwargs["covariance"].shape
            assert len(kwargs["weights"]) == kwargs["mean"].shape[0] >= 1
            """
            self._set_properties_of_distribution(*args, **kwargs)
        elif "components" in list(kwargs.keys()):
            assert len(kwargs["weights"]) == len(kwargs["components"]) >= 1
            self._set_distribution_components(kwargs["weights"], kwargs["components"])
        elif len(kwargs["weights"]) < 1:
            raise Exception("Weights not specified!")
        else:
            raise Exception("Mixture data cannot be created")

        if np.sum(kwargs["weights"]) != 1.0:
            text = "Weights of the mixture does not sum to 1, but to : " + str(np.sum(kwargs["weights"]))
            raise Exception(text)
        elif len(kwargs["weights"]) == 1:
            warnings.warn("This is not a MixtureData; it has a single component")

    def _set_properties_of_distribution(self, *args, **kwargs):
        self.__mixture_data_type = kwargs["mixture_data_type"]
        self.weights = kwargs["weights"]  # weights of the mixture components
        self._mean = np.asarray(kwargs["mean"])  # one row represents a one gaussian.
        self._covariance = np.asarray(kwargs["covariance"])  # one row represents a one gaussian.
        self.data_length = self._mean.shape[-1]
        self.nr_of_components = len(self.weights)

        self._create_components(*args)

    def _set_distribution_components(self, weights, components):

        self.__mixture_data_type = type(components[0])
        self.weights = weights
        self.components = components

        self.data_length = components[0].mean.shape[-1]
        self.nr_of_components = len(self.weights)
        self._mean = np.empty([len(components), self.data_length])
        self._covariance = np.empty([len(components), self.data_length])
        for i in range(len(components)):
            self._mean[i, :] = components[i].mean
            self._covariance[i, :] = components[i].covariance

    def range(self, sigma):
        """
        Returns the bounds of all components
        """
        bounds = []
        for c in self.components:
            bounds.append(c.range(sigma))
        return np.asarray(bounds)

    def _create_components(self, *args):
        """
        Set the components that build up the mixture. Store them in self.components
        :param args: a list which comprises upper and lower truncation array for each comp.
        :return:
        """

        self.components = []
        for i in range(len(self.weights)):

            if args:
                new_args = [a[i] for a in args]
                self.components.append(
                    self.__mixture_data_type(*new_args, mean=self._mean[i], covariance=self._covariance[i])
                )
                continue

            self.components.append(self.__mixture_data_type(*args, mean=self._mean[i], covariance=self._covariance[i]))

    def cdf(self, idx, x):
        cdf_comp = np.array([comp.cdf(idx, x) for comp in self.components])

        return np.sum(self.weights * cdf_comp)

    def __str__(self):
        l1 = "%s\n" % (str(type(self).__name__))
        l2 = "Has %s components\n" % (str(self.nr_of_components))
        l3 = "With weights : %s" % (str(self.weights))
        return l1 + l2 + l3

    @property
    def mean(self):
        return self._mean

    @mean.setter
    def mean(self, mean):
        self._mean = mean
        self._create_components()

    @property
    def covariance(self):
        return self._covariance

    @covariance.setter
    def covariance(self, covariance):
        self._covariance = covariance
        self._create_components()


text:
import os
import pickle

import numpy as np
from tqdm import tqdm

import phns

DATA_PATH = os.path.dirname(os.path.realpath(__file__)) + "/../data/"

print(f"Loading data from {DATA_PATH}")
with open(DATA_PATH + "timit_bench.pkl", "rb") as f:
    data = pickle.load(f)

# missing words:
#    exceptions = [
#        "motorists'",
#        "somebody'll",
#        "andrei's",
#
#        "morphophonemic",
#        "nihilistic",
#        "radiosterilization",
#        "exhusband",
#        "smolderingly",
#        "geocentricism",
#        "unmagnified",
#        "stirrin",
#        "utopianism",
#        "infuriation",
#        "preprepared",
#        "understandingly",
#        "eventualities",
#        "micrometeorites",
#        "herdin'",
#        "responsively",
#        "demineralization",
#        "unwaveringly",
#        "cap'n",
#        "mournfully",
#        "autofluorescence",
#        "fasciculations",
#        "weatherstrip",
#        "nonsystematic",
#        "traditionalism",
#        "chorused",
#        "micrometeorite",
#        "reupholstering",
#        "castorbeans"
#    ]

cers = []
cers_no_contractions = []
skipped = 0
missing = set()
for item in tqdm(data):
    # Preprocessing data
    _phns = phns.utils.timit_to_cmu(item["phns"])
    _phns = [phns.Phn(phn) for phn in _phns if phn != "sil"]

    graph = phns.from_text(
        item["text"],
        missing_handler=lambda word: missing.add(word),
        apply_confusion=True,
    )
    if graph:
        result = phns.closest(_phns, graph)
        cers.append(result["cmu_cer"])
        if result["cmu_cer"] > 0.3 and False:
            print(item["text"])
            print("phns", item["phns"])
            print("_phns", [phn.val for phn in _phns])
            print("targt", [phn.val for phn in result["target"]])
            print("match", [graph.nodes[m].value.val for m in result["match"]])
            print(result)
            import ipdb

            ipdb.set_trace()
    else:
        skipped += 1

print("skipped: ", skipped)
print("missing: ", len(missing))
print(missing)
print(
    {
        "25%": np.percentile(cers, 25),
        "50%": np.percentile(cers, 50),
        "75%": np.percentile(cers, 75),
        "95%": np.percentile(cers, 95),
        "mean": np.mean(cers),
    }
)

max_stars_repo_name: DeepLenin/phns

max_issues_repo_name: DeepLenin/phns

max_forks_repo_name: DeepLenin/phns


text:
import numpy as np
import math
from scipy import linalg

d = 2

def calc_exp(X, h):
    rx, Ux = linalg.eigh(X)
    Ux_inv = linalg.inv(Ux)
    tx = np.diag(np.array([math.exp(h * i) for i in rx]))
    eX = Ux.dot(tx.dot(Ux_inv))
    return eX

def trotter(X,Y,Z,h,n):
    eZ = calc_exp(Z,h)
    eX = calc_exp(X,h/n)
    eY = calc_exp(Y,h/n)
    S = np.diag(np.ones(d))
    eXeY = eX.dot(eY)
    for i in range(n):
        S = S.dot(eXeY)
    return linalg.norm(eZ - S)/linalg.norm(eZ)

def trotter2nd(X,Y,Z,h,n):
    eZ = calc_exp(Z,h)
    eX = calc_exp(X,h/n*0.5)
    eY = calc_exp(Y,h/n)
    S = np.diag(np.ones(d))
    eXeYeX = eX.dot(eY.dot(eX))
    for i in range(n):
        S = S.dot(eXeYeX)
    return linalg.norm(eZ - S)/linalg.norm(eZ)

def n_dependence(X,Y,Z,h,order):
    filename = "n_%x.dat" % order
    t = trotter if order == 1 else trotter2nd
    print filename
    f = open(filename,'w')
    for i in range(100):
        n = i+1
        f.write(str(n) + " " +  str(t(X,Y,Z,h,n)))
        f.write("\n")
    f.close()

def h_dependence(X,Y,Z,n, order):
    t = trotter if order == 1 else trotter2nd
    filename = "h%x_%x.dat" %(n, order)
    print filename
    f = open(filename,'w')
    for i in range(100):
        h = (i+1)/100.0
        f.write(str(h) + " " +  str(t(X,Y,Z,h,n)))
        f.write("\n")
    f.close()

np.random.seed(1)
x1 = np.random.rand(d,d)
x2 = np.random.rand(d,d)
X = x1.dot(x1.T)
Y = x2.dot(x2.T)
Z = X + Y

n_dependence(X,Y,Z,1.0,1)
n_dependence(X,Y,Z,1.0,2)

h_dependence(X,Y,Z,1,1)
h_dependence(X,Y,Z,2,1)
h_dependence(X,Y,Z,4,1)

h_dependence(X,Y,Z,1,2)
h_dependence(X,Y,Z,2,2)
h_dependence(X,Y,Z,4,2)

max_stars_repo_name: TejasAvinashShetty/lie-trotter-sample

max_issues_repo_name: TejasAvinashShetty/lie-trotter-sample

reason: import numpy,from scipy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Dec 13 15:34:25 2018

@author: mimbres
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.optim.lr_scheduler import StepLR
import numpy as np
import glob, os
import argparse
from tqdm import trange, tqdm
from spotify_data_loader import SpotifyDataloader
import time

parser = argparse.ArgumentParser(description="Sequence Skip Prediction")
parser.add_argument("-c","--config",type = str, default = "./config_init_dataset.json")
parser.add_argument("-s","--save_path",type = str, default = "./save/exp1/")
parser.add_argument("-l","--load_continue_latest",type = str, default = None)
parser.add_argument("-f","--feature_dim",type = int, default = 64)
parser.add_argument("-r","--relation_dim",type = int, default = 8)
parser.add_argument("-w","--class_num",type = int, default = 2)
parser.add_argument("-e","--epochs",type = int, default= 1000)
parser.add_argument("-t","--test_episode", type = int, default = 1000)
parser.add_argument("-lr","--learning_rate", type = float, default = 0.001)
parser.add_argument("-g","--gpu",type=int, default=0)
#parser.add_argument("-e","--embed_hidden_unit",type=int, default=2)
args = parser.parse_args()

# Hyper Parameters
FEATURE_DIM = args.feature_dim
RELATION_DIM = args.relation_dim
CLASS_NUM = args.class_num
EPOCHS = args.epochs
TEST_EPISODE = args.test_episode
LEARNING_RATE = args.learning_rate
GPU = args.gpu

mtrain_loader  = SpotifyDataloader(config_fpath=args.config,
                                  mtrain_mode=True, # True, because we use part of trainset as testset
                                  data_sel=(0, 99965071),#(99965071, 124950714), # 20%를 테스트
                                  batch_size=1,
                                  shuffle=False)

#%%
train_sessions_iter = iter(mtrain_loader)
for train_session in trange(len(train_sessions_iter), desc='varify', position=0):
    feats, labels, num_items, index = train_sessions_iter.next()
    if train_session==0:
        a0f = feats.detach().numpy()
        a0l = labels.detach().numpy()
        #tqdm.write(np.array2string(a0f))
        tqdm.write(np.array2string(a0l))

    if train_session==1:
        a1f = feats.numpy()
        a1l = labels.numpy()
        #tqdm.write(a1f)
        #tqdm.write(a1l)

    if train_session==2:
        a2f = feats.numpy()
        a2l = labels.numpy()
        #tqdm.write(a2f)
        #tqdm.write(a2l)

    if train_session==3:
        a3f = feats.numpy()
        a3l = labels.numpy()
        #tqdm.write(a3f)
        #tqdm.write(a2l)

    if train_session > 4:
        cur_feat = feats.numpy()
        cur_labels = labels.numpy()
        if sum(sum(sum(a0f-cur_feat)))==0:
            tqdm.write("found same feature matching a0 in {}th data, label diff={}".format(train_session, sum(sum(sum(a0l-cur_labels)))))

        if sum(sum(sum(a1f-cur_feat)))==0:
            tqdm.write("found same feature matching a1 in {}th data, label diff={}".format(train_session, sum(sum(sum(a1l-cur_labels)))))

        if sum(sum(sum(a2f-cur_feat)))==0:
            tqdm.write("found same feature matching a2 in {}th data, label diff={}".format(train_session, sum(sum(sum(a2l-cur_labels)))))

        if sum(sum(sum(a3f-cur_feat)))==0:
            tqdm.write("found same feature matching a3 in {}th data, label diff={}".format(train_session, sum(sum(sum(a3l-cur_labels)))))

    #time.sleep(0.001)

max_stars_repo_name: mimbres/SeqSkip

max_issues_repo_name: mimbres/SeqSkip

max_forks_repo_name: mimbres/SeqSkip


text:
import numpy as np
import matplotlib.pyplot as plt
from scipy import interpolate
from multiprocessing import Pool
from functools import partial
from xml.dom import minidom
import re
from svgpathtools import parse_path

def point_distance(p1, p2):
    return np.sqrt((p2[0] - p1[0]) ** 2 + (p2[1] - p1[1]) ** 2)

def check_sink_proximity(p, sinks, maxdist=0.1):
    f = partial(point_distance, p2=p)
    dist = list(map(f, sinks))
    return any(x < maxdist for x in dist)

def compute_trace(p, maxiter, max_sink_dist, grid, u_i, v_i, dt, sinks, limit=1500, max_iter_sink_proximity=0):
    ps = []
    n = 0
    sink_soak = 0
    while (
        point_in_canvas(p, grid)
        and n <= maxiter
        and sink_soak <= max_iter_sink_proximity  # allow only 20 more steps if getting close to sink
    ):
        ps.append(p.copy())
        dx = u_i(p[0], p[1])[0] * dt
        dy = v_i(p[0], p[1])[0] * dt
        p[0] = p[0] + dx
        p[1] = p[1] + dy
        n += 1
        if check_sink_proximity(p, sinks, maxdist=max_sink_dist):
            sink_soak += 1
    # limit length of points returned
    skips = max(1, int(len(ps) / limit))
    return ps[::skips]

def point_in_canvas(point, grid):
    if (
        point[0] < grid["x_start"]
        or point[0] > grid["x_end"]
        or point[1] < grid["y_start"]
        or point[1] > grid["y_end"]
    ):
        return False
    else:
        return True

class streamLines:
    def __init__(self, grid, colors=['#000000']):
        self.x = []
        self.y = []
        self.dx = []
        self.dy = []
        self.X = []
        self.Y = []
        self.psis = []
        self.colors = colors
        self.streamtraces = []
        self.streamtraces_color = []
        self.source_sink = []
        self.vortices=[]
        self.seeds = []
        self.sl_config = {}
        self.grid = grid
        grid_shape = (grid["no_points_x"], grid["no_points_y"])
        self.psi = np.zeros(grid_shape)
        self.u = np.zeros(grid_shape)
        self.v = np.zeros(grid_shape)
        self.generate_mesh()

    def init_from_svg(
        self,
        file,
        config={
            "source_color": "000000",  # black
            "sink_color": "ff0000",  # red
            "seeds_color": "00ff00",  # green
            "vortex_color_cw": "0000ff",  # blue
            "vortex_color_ccw": "000080",  # navy
            "source_strength_default": 0.1,
            "sink_strength_default": -0.1,
            "vortex_default_strength": 0.1,
            "points_per_unit_length": 1,
            "scale": 5,
        },
        plot=False,
    ):

        sources = []
        sinks = []
        seeds = []
        vortices = []
        vortices_strength = []
        sinks_strength = []
        sources_strength = []

        svg_xml = minidom.parse(file)
        paths = svg_xml.getElementsByTagName("path")
        circles = svg_xml.getElementsByTagName("circle")
        for path in paths:
            path = path.toxml()
            color = re.findall(r"#([a-f,0-9]{6})", path)[0]
            d = re.findall(r'\sd="([A-Z,a-z,0-9,\s,\.,\-]*)"', path)[0]
            path = parse_path(d)
            num_samples = int(path.length() * config["points_per_unit_length"])
            if color == config["sink_color"]:
                for j in range(num_samples):
                    sinks.append(path.point(j / num_samples))
                    sinks_strength.append(config["sink_strength_default"])
            elif color == config["source_color"]:
                for j in range(num_samples):
                    sources.append(path.point(j / num_samples))
                    sources_strength.append(config["source_strength_default"])
            elif color == config["seeds_color"]:
                for j in range(num_samples):
                    seeds.append(path.point(j / num_samples))
            elif color == config["vortex_color"]:
                for j in range(num_samples):
                    vortices.append(path.point(j / num_samples))
                    vortices_strength.append(config["vortex_default_strength"])
            else:
                raise Exception("Please make sure features are definded for colors")

        for circle in circles:
            circle = circle.toxml()
            cx = float(re.findall(r'\scx="([0-9,\.,\-]*)"', circle)[0])
            cy = float(re.findall(r'\scy="([0-9,\.,\-]*)"', circle)[0])
            r = float(re.findall(r'\sr="([0-9,\.]*)"', circle)[0])
            color = re.findall(r"#([a-f,0-9]{6})", circle)[0]
            c = complex(cx, cy)

            if color == config["sink_color"]:
                sinks.append(c)
                sinks_strength.append(-r * config["scale"])
            elif color == config["source_color"]:
                sources.append(c)
                sources_strength.append(r * config["scale"])
            elif color == config["seeds_color"]:
                seeds.append(c)
            elif color == config["vortex_color_cw"]:
                vortices.append(c)
                vortices_strength.append(r * config["scale"])
            elif color == config["vortex_color_ccw"]:
                vortices.append(c)
                vortices_strength.append(-r * config["scale"])

        if plot:
            _, ax1 = plt.subplots()
            ax1.scatter(
                [x.real for x in sinks],
                [x.imag for x in sinks],
                color="#" + config["sink_color"],
            )
            ax1.scatter(
                [x.real for x in sources],
                [x.imag for x in sources],
                color="#" + config["source_color"],
            )
            ax1.scatter(
                [x.real for x in seeds],
                [x.imag for x in seeds],
                color="#" + config["seeds_color"],
            )
            ax1.scatter(
                [x.real for x in vortices],
                [x.imag for x in vortices],
                color="#" + config["vortex_color_cw"],
            )

            ax1.invert_yaxis()

        self.seeds = [[x.real, x.imag] for x in seeds]
        for i in range(len(sources)):
            self.add_source_sink(sources_strength[i], sources[i].real, sources[i].imag)

        for i in range(len(sinks)):
            self.add_source_sink(sinks_strength[i], sinks[i].real, sinks[i].imag)

        for i in range(len(vortices)):
            self.add_vortex(vortices_strength[i], vortices[i].real, vortices[i].imag)

    def generate_mesh(self):
        self.x = np.linspace(
            self.grid["x_start"], self.grid["x_end"], self.grid["no_points_x"]
        )
        self.y = np.linspace(
            self.grid["y_start"], self.grid["y_end"], self.grid["no_points_y"]
        )
        self.dx = (self.grid["x_end"] - self.grid["x_start"]) / self.grid["no_points_x"]
        self.dy = (self.grid["y_end"] - self.grid["y_start"]) / self.grid["no_points_y"]
        self.X, self.Y = np.meshgrid(self.x, self.y)

    def add_source_sink(self, strength, x, y):
        """
        Returns the stream-function generated by a source/sink.

        Parameters
        ----------
        strength: float
            Strength of the source/sink.
        x: float
            x-coordinate of the source (or sink).
        y: float
            y-coordinate of the source (or sink).

        Returns
        -------
        psi: 2D Numpy array of floats
            The stream-function.
        """
        self.source_sink.append([x, y, strength])
        psi = strength / (2 * np.pi) * np.arctan2((self.Y - y), (self.X - x))
        self.psis.append(psi)
        self.psi = np.add(self.psi, psi)
        self.u = self.u + (
            strength
            / (2 * np.pi)
            * (self.X - x)
            / ((self.X - x) ** 2 + (self.Y - y) ** 2)
        )
        self.v = self.v + (
            strength
            / (2 * np.pi)
            * (self.Y - y)
            / ((self.X - x) ** 2 + (self.Y - y) ** 2)
        )

    def add_free_stream(self, u_inf=0, v_inf=0):
        psi = u_inf * self.Y - v_inf * self.X
        self.psis.append(psi)
        self.psi = np.add(self.psi, psi)
        self.u = self.u + u_inf
        self.v = self.v + v_inf

    def add_doublet(self, strength, x, y):
        """
        Returns the stream-function generated by a doublet.

        Parameters
        ----------
        strength: float
            Strength of the doublet.
        x: float
            x-coordinate of the doublet.
        y: float
            y-coordinate of the doublet.

        Returns
        -------
        psi: 2D Numpy array of floats
            The stream-function.
        """
        psi = (
            -strength
            / (2 * np.pi)
            * (self.Y - y)
            / ((self.X - x) ** 2 + (self.Y - y) ** 2)
        )
        self.psis.append(psi)
        self.psi = np.add(self.psi, psi)
        self.u = self.u - (
            strength
            / (2 * np.pi)
            * ((self.X - x) ** 2 - (self.Y - y) ** 2)
            / ((self.X - x) ** 2 + (self.Y - y) ** 2) ** 2
        )
        self.v = self.v - (
            strength
            / (2 * np.pi)
            * 2
            * (self.X - x)
            * (self.Y - y)
            / ((self.X - x) ** 2 + (self.Y - y) ** 2) ** 2
        )

    def add_vortex(self, strength, x, y):
        """
        Returns the velocity field generated by a vortex.

        Parameters
        ----------
        strength: float
            Strength of the vortex.
        xv: float
            x-coordinate of the vortex.
        yv: float
            y-coordinate of the vortex.
        X: 2D Numpy array of floats
            x-coordinate of the mesh points.
        Y: 2D Numpy array of floats
            y-coordinate of the mesh points.
        """
        self.vortices.append([x, y, strength])
        psi = strength / (4 * np.pi) * np.log((self.X - x) ** 2 + (self.Y - y) ** 2)
        self.psis.append(psi)
        self.psi = np.add(self.psi, psi)
        self.u = self.u + strength / (2 * np.pi) * (self.Y - y) / (
            (self.X - x) ** 2 + (self.Y - y) ** 2
        )
        self.v = self.v - strength / (2 * np.pi) * (self.X - x) / (
            (self.X - x) ** 2 + (self.Y - y) ** 2
        )

    def reset_psi(self):
        self.psi = self.psis[0]
        if len(self.psis > 0):
            for i in range(1, len(self.psis)):
                self.psi = np.add(self.psi, self.psis[i])

    def _circle_points(self, p, r, n_points):
        points = []
        for alpha in np.linspace(0, 2 * np.pi, n_points + 1):
            points.append([p[0] + np.cos(alpha) * r, p[1] + np.sin(alpha) * r])
        return points

    def calc_streamtraces(
        self,
        n_streamtraces=10,
        dt=0.005,
        maxiter=500,
        radius=0.1,
        max_sink_dist=0.1,
        max_iter_sink_proximity=0,
        seeds=["random"],
        n_cpu=1,
    ):

        self.sl_config = {
            "n_streamtraces": n_streamtraces,
            "dt": dt,
            "maxiter": maxiter,
            "radius": radius,
            "max_sink_dist": max_sink_dist,
            "seeds": seeds,
        }

        u_i = interpolate.interp2d(self.x, self.y, self.u)
        v_i = interpolate.interp2d(self.x, self.y, self.v)

        if not isinstance(seeds, list):
            raise Exception("seeds argument must be list")

        for item in seeds:
            if item == "random":
                xs = np.random.uniform(
                    self.grid["x_start"], self.grid["x_end"], size=n_streamtraces
                )
                ys = np.random.uniform(
                    self.grid["y_start"], self.grid["y_end"], size=n_streamtraces
                )
                self.seeds.extend(np.dstack((xs, ys))[0].tolist())
                self.streamtraces_color.append(np.random.choice(self.colors))

            elif item == "sources":
                seeds_center = [[x[0], x[1]] for x in self.source_sink if x[2] > 0]
                n_streamstraces_per_source = int(n_streamtraces / len(seeds_center))
                for p in seeds_center:
                    self.seeds.extend(
                        self._circle_points(p, radius, n_streamstraces_per_source)
                    )
                    self.streamtraces_color.extend((n_streamstraces_per_source + 1) * [np.random.choice(self.colors)])

            elif item == "grid":
                grid_points = np.dstack((self.X.flatten(), self.Y.flatten()))[0]
                gridskip = int(
                    self.grid["no_points_x"] * self.grid["no_points_y"] / n_streamtraces
                )
                self.seeds.extend(grid_points[::gridskip].tolist())
                self.streamtraces_color.extend(np.random.choice(self.colors, n_streamtraces))

            sinks = [[x[0], x[1]] for x in self.source_sink if x[2] < 0]
            if n_cpu > 1:
                with Pool(n_cpu) as pool:
                    self.streamtraces = pool.map(
                        partial(
                            compute_trace,
                            maxiter=maxiter,
                            max_sink_dist=max_sink_dist,
                            max_iter_sink_proximity=max_iter_sink_proximity,
                            sinks=sinks,
                            grid=self.grid,
                            u_i=u_i,
                            v_i=v_i,
                            dt=dt,
                        ),
                        self.seeds,
                    )
            else:
                for seed in self.seeds:
                    self.streamtraces.append(
                        compute_trace(
                            seed,
                            maxiter=maxiter,
                            max_sink_dist=max_sink_dist,
                            max_iter_sink_proximity=max_iter_sink_proximity,
                            sinks=sinks,
                            grid=self.grid,
                            u_i=u_i,
                            v_i=v_i,
                            dt=dt,
                        )
                    )

    def write_svg_init(
        self,
        file,
        config={
            "source_color": "000000",  # black
            "sink_color": "ff0000",  # red
            "seeds_color": "00ff00",  # green
            "vortex_color_cw": "0000ff",  # blue
            "vortex_color_ccw": "000080",  # navy)
            "scale": 5,
        },
    ):
        sources = [x for x in self.source_sink if x[2] >=0]
        sinks = [x for x in self.source_sink if x[2] <0]

        f = open(file, "w")
        f.write(
            f"""<svg height="{self.grid['y_end'] - self.grid['y_start']}" width="{self.grid['x_end'] - self.grid['x_start']}">"""
        )

        for source in sources:
            f.write(
                f"""<circle cx="{source[0]}" cy="{source[1]}" r="{source[2] / config['scale']}" fill="none" stroke="#{config['source_color']}" />\n"""
            )
        for sink in sinks:
            f.write(
                f"""<circle cx="{sink[0]}" cy="{sink[1]}" r="{abs(sink[2])/ config['scale']}" fill="none" stroke="#{config['sink_color']}" />\n"""
            )
        for vortex in self.vortices:
            if vortex[2] >= 0:
                color = config["vortex_color_cw"]
            else:
                color = config["vortex_color_ccw"]
            f.write(
                f"""<circle cx="{vortex[0]}" cy="{vortex[1]}" r="{abs(vortex[2])/ config['scale']}" fill="none" stroke="#{color}" />\n"""
            )
        for seed in self.seeds:
            f.write(
                f"""<circle cx="{seed[0]}" cy="{seed[1]}" r="1" fill="none" stroke="#{config['seeds_color']}" />\n"""
            )

        y = -100
        for key in self.sl_config.keys():
            f.write(f"""<text x="0" y="{y}">{key}:{self.sl_config[key]}</text>""")
            y += 10

        f.write("</svg>")
        f.close()

    def write_svg(
        self, file, height=297, width=420, offset_x=420 / 2, offset_y=297 / 2
    ):
        print("writing to svg ...", end="")
        dx = self.grid["x_end"] - self.grid["x_start"]
        fx = width / dx
        dy = self.grid["y_end"] - self.grid["y_start"]
        fy = height / dy

        f = open(file, "w")
        f.write(f'<svg height="{height}" width="{width}">')

        d = ""

        for streamline in self.streamtraces:
            # d += f'<circle cx="{seed[1]}" cy="{seed[0]}" r="5" fill="none" stroke="black" />\n'
            if len(streamline) > 2:
                d += f'<path d="M {streamline[0][0]*fx + offset_x} {streamline[0][1]*fy + offset_y} '
                for i in range(len(streamline) - 1):
                    d += f"L {streamline[i+1][0]*fx + offset_x} {streamline[i+1][1]*fy + offset_y} "
                d += '" stroke="black" fill="none" />\n'
        f.write(d)
        f.write("</svg>")
        f.close()

    def plot(self, num_level=25, legend=True):
        fig, ax = plt.subplots(figsize=(12, 10))
        c = ax.contour(self.x, self.y, self.psi, num_level)
        if legend:
            fig.colorbar(c, ax=ax, shrink=0.9)

max_stars_repo_name: marcbiester/plotart

max_issues_repo_name: marcbiester/plotart

max_forks_repo_name: marcbiester/plotart


text:
import numpy as np
import pandas as pd

ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]
bins = [18, 25, 35, 60, 100]
cat = pd.cut(ages, bins)
cats
cat
cat.codes
cat.categories
pd.value_counts(cat)
cat = pd.cut(ages, bins, right=False)
cat
cat.categories
group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']
pd.cut(ages, bins, labels=group_names)
data = np.random.rand(20)
data
pd.cut(data, 4, precision=2)

max_stars_repo_name: eroicaleo/LearningPython

max_issues_repo_name: eroicaleo/LearningPython

max_forks_repo_name: eroicaleo/LearningPython


text:
import numpy as np
from message import Message
from thespian.actors import *

class Agent(Actor):
    def __init__(self):
        self.wealth = np.random.randint(1, 10)

    def get_wealth(self):
        return self.wealth

    def receiveMessage(self, message, sender):
        if isinstance(message, Message):
            if message.recipients == "Agents":
                if message.directive == "get_wealth":
                    self.send(sender, self.wealth)

max_stars_repo_name: messiest/gamelab

max_issues_repo_name: messiest/gamelab

max_forks_repo_name: messiest/gamelab


text:
# test on word2vec package

import os
import sys

reload(sys)

import random
import multiprocessing
import gensim, logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

import numpy as np
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

class dir_sentences(object):
	def __init__(self, dirname):
		self.dirname = dirname

	def __iter__(self):
		for filename in os.listdir(self.dirname):
			for line in open(os.path.join(self.dirname, filename)):
				yield line.split()

class w2v_model(object):
    """ word2vec, doc2vec """
    def __init__(self):
    	self.model = Word2Vec()

    def train(self, sentences):
	# ignore words less then 5 times, parallel threads to be 8, 100 neuron size (50 to 100)
	bigram_transformer = gensim.models.Phrases(sentences)
	self.model = Word2Vec(bigram_transformer[sentences], min_count=10, size=300 ,window=5, workers=8)
	self.model.save('models/mymodel')
	#self.model.save_word2vec_format('models/vector')
	#self.model.init_sims(replace=True)
	#self.model.build_vocab()

    def ctrain(self, sentences):
		self.model = Word2Vec.load('models/mymodel')
		self.model.train(sentences)
		self.model.save_word2vec_format('models/vector')

    def accuracy(self):
		self.model.accuracy('data/questions-words.txt')

# merge wiki data with text8
wiki = dir_sentences('data/wikipedia_sci')
text8 = LineSentence('data/text8')

pre_model = w2v_model()
pre_model.train(text8)
#pre_model.accuracy()

pre_model.ctrain(wiki)
pre_model.accuracy()

#pre_model.save(output)

# load pre-train model from GloVe


text:
import tensorflow as tf
print('Using Tensorflow '+tf.__version__)
import matplotlib.pyplot as plt
import sys
# sys.path.append('../')
import os
import csv
import cv2
import numpy as np
from PIL import Image
import time
import math

import src.siamese as siam
from src.visualization import show_frame, show_crops, show_scores

# gpu_device = 2
# os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(gpu_device)

# read default parameters and override with custom ones
def tracker(graph, scfg, hp, run, design, frame_name_list, pos_x, pos_y, target_w, target_h, final_score_sz, filename, image, templates_z, scores, label, ownerProb, color, sFrame, eFrame, finalImages, sourceAngle, ):
    num_frames = np.size(frame_name_list)
    if(num_frames < eFrame):
        eFrame = num_frames
    if(len(finalImages) == 0):
        newVector = True
    else:
        newVector = False
    # stores tracker's output for evaluation
    bboxes = np.zeros((num_frames,4))
    val = eFrame-sFrame
    print(val)
    scale_factors = hp.scale_step**np.linspace(-np.ceil(hp.scale_num/2), np.ceil(hp.scale_num/2), hp.scale_num)
    # cosine window to penalize large displacements
    hann_1d = np.expand_dims(np.hanning(final_score_sz), axis=0)
    penalty = np.transpose(hann_1d) * hann_1d
    penalty = penalty / np.sum(penalty)

    context = design.context*(target_w+target_h)
    z_sz = np.sqrt(np.prod((target_w+context)*(target_h+context)))
    x_sz = float(design.search_sz) / design.exemplar_sz * z_sz

    # thresholds to saturate patches shrinking/growing
    min_z = hp.scale_min * z_sz
    max_z = hp.scale_max * z_sz
    min_x = hp.scale_min * x_sz
    max_x = hp.scale_max * x_sz

    # run_metadata = tf.RunMetadata()
    # run_opts = {
    #     'options': tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE),
    #     'run_metadata': run_metadata,
    # }

    run_opts = {}

    # with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
    with tf.Session(graph=graph) as sess:
        tf.global_variables_initializer().run()
        # Coordinate the loading of image files.
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)

        # save first frame position (from ground-truth)
        bboxes[0,:] = pos_x-target_w/2, pos_y-target_h/2, target_w, target_h

        image_, templates_z_ = sess.run([image, templates_z], feed_dict={
                                                                        scfg.pos_x_ph: pos_x,
                                                                        scfg.pos_y_ph: pos_y,
                                                                        scfg.z_sz_ph: z_sz,
                                                                        scfg.x_sz0_ph: 0,
                                                                        scfg.x_sz1_ph: 0,
                                                                        scfg.x_sz2_ph: 0,
                                                                        filename: frame_name_list[0]})
        new_templates_z_ = templates_z_

        t_start = time.time()

        # Get an image from the queue
        counter = 0.0
        toolbar_width = 40
        for i in range(sFrame, eFrame):
            sys.stdout.write("[")
            sys.stdout.write("*"*int(float(counter/(eFrame-sFrame))*toolbar_width))
            sys.stdout.write(" "*int((1-counter/(eFrame-sFrame))*toolbar_width))
            sys.stdout.write("]")
            sys.stdout.write("%d/%d"%(counter, (eFrame-sFrame)))
            counter = counter + 1.0
            scaled_exemplar = z_sz * scale_factors
            scaled_search_area = x_sz * scale_factors
            scaled_target_w = target_w * scale_factors
            scaled_target_h = target_h * scale_factors
            image_, scores_ = sess.run(
                [image, scores],
                feed_dict={
                    scfg.pos_x_ph: pos_x,
                    scfg.pos_y_ph: pos_y,
                    scfg.x_sz0_ph: scaled_search_area[0],
                    scfg.x_sz1_ph: scaled_search_area[1],
                    scfg.x_sz2_ph: scaled_search_area[2],
                    templates_z: np.squeeze(templates_z_),
                    filename: frame_name_list[i],
                }, **run_opts)
            scores_ = np.squeeze(scores_)
            # penalize change of scale
            scores_[0,:,:] = hp.scale_penalty*scores_[0,:,:]
            scores_[2,:,:] = hp.scale_penalty*scores_[2,:,:]
            # find scale with highest peak (after penalty)
            new_scale_id = np.argmax(np.amax(scores_, axis=(1,2)))
            # update scaled sizes
            x_sz = (1-hp.scale_lr)*x_sz + hp.scale_lr*scaled_search_area[new_scale_id]
            target_w = (1-hp.scale_lr)*target_w + hp.scale_lr*scaled_target_w[new_scale_id]
            target_h = (1-hp.scale_lr)*target_h + hp.scale_lr*scaled_target_h[new_scale_id]
            # select response with new_scale_id
            score_ = scores_[new_scale_id,:,:]
            score_ = score_ - np.min(score_)
            score_ = score_/np.sum(score_)
            # apply displacement penalty
            score_ = (1-hp.window_influence)*score_ + hp.window_influence*penalty
            pos_x, pos_y = _update_target_position(pos_x, pos_y, score_, final_score_sz, design.tot_stride, design.search_sz, hp.response_up, x_sz)
            # convert <cx,cy,w,h> to <x,y,w,h> and save output
            bboxes[i,:] = pos_x-target_w/2, pos_y-target_h/2, target_w, target_h
            # update the target representation with a rolling average
            if hp.z_lr>0:
                new_templates_z_ = sess.run([templates_z], feed_dict={
                                                                scfg.pos_x_ph: pos_x,
                                                                scfg.pos_y_ph: pos_y,
                                                                scfg.z_sz_ph: z_sz,
                                                                image: image_
                                                                })

                templates_z_=(1-hp.z_lr)*np.asarray(templates_z_) + hp.z_lr*np.asarray(new_templates_z_)

            # update template patch size
            z_sz = (1-hp.scale_lr)*z_sz + hp.scale_lr*scaled_exemplar[new_scale_id]

            if run.visualization:
                if(newVector):
                    singleFrame = cv2.imread(frame_name_list[i],1)
                else:
                    singleFrame = finalImages[i-sFrame]
                cv2.rectangle(singleFrame,
                    (int(bboxes[i,0]),int(bboxes[i,1])),
                    (int(bboxes[i,0]+bboxes[i,2]),int(bboxes[i,1]+bboxes[i,3])),
                    color,2)
                #if(color == (0,255,0)):
                #    ownerProb =  .68 + .32 / 2
                #elif(color == (0,0,255)):
                #    ownerProb = .32 / 2
                #cv2.putText(singleFrame,'P('+label+')='+str(ownerProb),(int(bboxes[i,0]),int(bboxes[i,1])),cv2.FONT_HERSHEY_SIMPLEX,0.5, color,2)
                if(newVector):
                    finalImages.append(singleFrame)
                #video.write(singleFrame)
                #show_frame(image_, bboxes[i,:], 1)
            sys.stdout.write("\b" * int(toolbar_width+5+min(1,counter/10)+min(1,counter/100)+min(1,counter/1000)+min(1,(eFrame-sFrame)/10)+min(1,(eFrame-sFrame)/100)+min(1,(eFrame-sFrame)/1000)))

        sys.stdout.write("\n")
        t_elapsed = time.time() - t_start
        speed = num_frames/t_elapsed

        # Finish off the filename queue coordinator.
        coord.request_stop()
        coord.join(threads)

        # from tensorflow.python.client import timeline
        # trace = timeline.Timeline(step_stats=run_metadata.step_stats)
        # trace_file = open('timeline-search.ctf.json', 'w')
        # trace_file.write(trace.generate_chrome_trace_format())

    plt.close('all')
    cv2.destroyAllWindows()
    #video.release()
    return bboxes, speed, finalImages

def _update_target_position(pos_x, pos_y, score, final_score_sz, tot_stride, search_sz, response_up, x_sz):
    # find location of score maximizer
    p = np.asarray(np.unravel_index(np.argmax(score), np.shape(score)))
    # displacement from the center in search area final representation ...
    center = float(final_score_sz - 1) / 2
    disp_in_area = p - center
    # displacement from the center in instance crop
    disp_in_xcrop = disp_in_area * float(tot_stride) / response_up
    # displacement from the center in instance crop (in frame coordinates)
    disp_in_frame = disp_in_xcrop *  x_sz / search_sz
    # *position* within frame in frame coordinates
    pos_y, pos_x = pos_y + disp_in_frame[0], pos_x + disp_in_frame[1]
    return pos_x, pos_y


text:
# -*- coding: utf-8 -*-
"""
===============================================================================
CubicDual: Generate a cubic lattice with an interpentrating dual network
===============================================================================

"""
import scipy as sp
from OpenPNM.Network.tools import stitch
from OpenPNM.Network import GenericNetwork, Cubic
from OpenPNM.Base import logging
logger = logging.getLogger(__name__)

class CubicDual(GenericNetwork):
    r"""
    Generates a cubic network of the specified size and shape, then inserts
    as second cubic network to the corners of its lattice cells.  These two
    networks are further connected by throats enabling material to be
    exchanged between them.

    Parameters
    ----------
    name : string
        A unique name for the network

    shape : list of ints
        The size and shape of the primary cubic network in terms of the
        number of pores in each direction.  Secondary nodes will be added at
        corners of each unit cell so the dual network will generaly have a
        size of ''shape'' + 1.

    spacing : list of floats
        The distance between pores of the primary network in each of the
        principal directions

    label_1 and label_2 : strings
        The labels to apply to the primary and secondary cubic lattices, which
        defaults to 'primary' and 'secondary' respectively.

    Examples
    --------
    >>>
    """
    def __init__(self, shape=None, spacing=[1, 1, 1], label_1='primary',
                 label_2='secondary', **kwargs):
        super().__init__(**kwargs)
        spacing = sp.array(spacing)
        shape = sp.array(shape)
        # Deal with non-3D shape arguments
        shape = sp.pad(shape, [0, 3-shape.size], mode='constant', constant_values=1)
        net = Cubic(shape=shape, spacing=[1, 1, 1])
        net['throat.'+label_1] = True
        net['pore.'+label_1] = True
        single_dim = shape == 1
        shape[single_dim] = 2
        dual = Cubic(shape=shape-1, spacing=[1, 1, 1])
        faces = [['front', 'back'], ['left', 'right'], ['top', 'bottom']]
        faces = [faces[i] for i in sp.where(~single_dim)[0]]
        faces = sp.array(faces).flatten().tolist()
        dual.add_boundaries(faces)
        # Add secondary network name as a label
        dual['pore.'+label_2] = True
        dual['throat.'+label_2] = True
        # Shift coordinates prior to stitching
        dual['pore.coords'] += 0.5*(~single_dim)
        stitch(net, dual, P_network=net.Ps, P_donor=dual.Ps, len_max=1)
        net['throat.interconnect'] = net['throat.stitched']
        del net['throat.stitched']
        net['pore.coords'] *= spacing
        # Clean-up labels
        net['pore.surface'] = False
        net['throat.surface'] = False
        for face in faces:
            # Remove face label from secondary network since it's internal now
            Ps = net.pores(labels=[face, label_2], mode='intersection')
            net['pore.'+face][Ps] = False
            Ps = net.pores(labels=[face+'_boundary'])
            net['pore.'+face][Ps] = True
            Ps = net.pores(face)
            net['pore.surface'][Ps] = True
            Ts = net.find_neighbor_throats(pores=Ps, mode='intersection')
            net['throat.surface'][Ts] = True
            net['throat.'+face] = net.tomask(throats=Ts)
        [net.pop(item) for item in net.labels() if 'boundary' in item]
        # Label non-surface pores and throats as internal
        net['pore.internal'] = ~net['pore.surface']
        Ts = net.find_neighbor_throats(pores=net['pore.internal'])
        net['throat.internal'] = False
        net['throat.internal'][Ts] = True
        # Transfer all dictionary items from 'net' to 'self'
        [self.update({item: net[item]}) for item in net]
        del self.workspace[net.name]

max_stars_repo_name: thirtywang/OpenPNM

max_issues_repo_name: thirtywang/OpenPNM

max_forks_repo_name: thirtywang/OpenPNM

reason: import scipy


text:
#!/usr/bin/env/ python

import os
import pandas as pd
import numpy as np
from bisect import bisect_left

def get_file_list(path=os.getcwd()):
    """
    Gets a listing of well files in a specified directory

    Parameters:

    path (string): The directory path for the well files

    Returns:

    list: list of files in specified directory
    """
    return os.listdir(path)

def pair_well_files(file_list):
    """This function reads in the contents of a directory
    And pairs unique well LAS file with well deviation survey
    Assumes file_list a list
    Returns a dictionary
    """
    pass

def read_well_las(well_file):
    """This function reads in a LAS file
    Assumes well_file a string
    Returns a pandas dataframe
    """
    pass

def read_dev_surv(dev_surv_file):
    """This function reads in a CSV file
    Assumes dev_surv_file a string
    Returns a pandas dataframe
    """
    pass

def dog_leg_severity(md, inc, az, units):
    """Calculates the dog leg severity as input to minimum curvature for TVD correction

    Parameters:

    md (ndarray-like): Measured Depth

    inc (ndarray-like): Wellbore inclination

    az (ndarray-like): Wellbore azimuth

    units (string): Depth measurement units, 'm' or 'ft'

    Returns:

    ndarray, ndarray: Dog Leg, Dog Leg Severity
    """
    # initialize an empty numpy array for Dog Leg Severity
    dls = np.zeros(len(md))
    dl = np.zeros(len(md))
    # hard code the dog leg severity to be zero at the first sample
    dl[0] = 0
    dls[0]= 0
    # calculate measured depth increment, skipping the first sample
    mdStep = np.diff(md, n=1)
    # create inc1, inc2 arrays offset by 1 index
    inc1 = np.deg2rad(inc[0:-1])
    inc2 = np.deg2rad(inc[1:])
    inc2.reset_index(drop=True, inplace=True)
    # create az1, az2 arrays offset by 1 index
    az1 = np.deg2rad(az[0:-1])
    az2 = np.deg2rad(az[1:])
    az2.reset_index(drop=True, inplace=True)
    a = np.multiply(np.sin(inc1), np.sin(inc2))
    b = np.cos(az2 - az1)
    c = np.multiply(a, b)
    d = np.multiply(np.cos(inc1), np.cos(inc2))
    e = np.arccos(c + d)
    dl[1:] = np.rad2deg(e)
    try:
        if units.lower() == 'm':
            f = np.multiply(dl, 30.)
        elif units.lower() == 'ft':
            f = np.multiply(dl, 100.)
        else:
            raise ValueError
    except ValueError:
        raise ValueError('Invalid units')
    g = np.divide(f[1:], mdStep)
    dls[1:] = g
    return dl, dls

def minimum_curvature(md, inc, dl, tvd0):
    """This function corrects wellbore deviation using the minimum curvature method

    Parameters:

    md (ndarray-like): Measured Depth

    inc (ndarray-like): Wellbore Inclination

    dl (ndarray-like): Dog Leg

    tvd0 (float): True Vertical Depth of initial depth sample (usually 0.0)

    Returns:

    ndarry: True Vertical Depth
    """
    # initialize an empty numpy array for TVD
    tvd = np.zeros(len(md))
    # hard code the TVD of the first sample to be zero
    tvd[0] = tvd0
    # create md1, md2 arrays offset by 1 index
    md1 = md[0:-1]
    md2 = md[1:]
    md2.reset_index(drop=True, inplace=True)
    # create inc1, inc2 arrays offset by 1 index
    inc1 = np.deg2rad(inc[0:-1])
    inc2 = np.deg2rad(inc[1:])
    inc2.reset_index(drop=True, inplace=True)
    # calculate ratio factor
    rF = np.zeros(len(md))
    a_rF = np.tan(np.divide(np.deg2rad(dl), 2.))
    b_rF = np.zeros(len(a_rF))
    b_rF[dl > 0] = np.divide(2., np.deg2rad(dl[dl > 0]))
    b_rF[dl == 0] = 1
    rF = np.multiply(a_rF, b_rF)
    # calculate TVD
    a = np.cos(inc1) + np.cos(inc2)
    b = np.divide((md2 - md1), 2.)
    c = np.multiply(rF[1:], b)
    d = np.multiply(a, c)
    tvd[1:] = np.cumsum(d) + tvd0
    return tvd

def getClosestValues(myList, myNumber):
    """
    Assumes myList is sorted. Returns closest value to myNumber.

    If two numbers are equally close, return the smallest number.
    """
    pos = bisect_left(myList, myNumber)
    if pos == 0:
        return myList[0]
    if pos == len(myList):
        return myList[-1]
    before = myList[pos - 1]
    after = myList[pos]
    if after - myNumber < myNumber - before:
        return after
    else:
        return before

def uniform_md(start_depth=0, kb=32, stop_depth=10000, inc=0.5, units='ft'):
    """This function builds uniformly spaced depth reference curves
    Defaults start_depth, kb, stop_depth, inc, and units
    Returns a pandas series
    """
    # calcuate how many samples are needed for max TD
    size = int(round(stop_depth, 1)) + 1
    md = list()
    for i in range(0, size * 10000, int(inc * 10000)):
        md.append(float(i) / 10000)

    # create a dataframe for the uniform md
    return pd.DataFrame({'MD':md})

def reindex_curves(original_curves, reference_curves):
    """This function merges the original curves with the reference curves
    Assumes original_curves, reference_curves a pandas dataframe
    Returns a pandas dataframe
    """
    pass


text:
#
# Copyright (c) 2009, Georgia Tech Research Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the Georgia Tech Research Corporation nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY GEORGIA TECH RESEARCH CORPORATION ''AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL GEORGIA TECH BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

#  \author Hai Nguyen (Healthcare Robotics Lab, Georgia Tech.)

import rospy
import std_srvs.srv as srv
from hrl_msgs.msg import FloatArray
from rospy.impl.tcpros import DEFAULT_BUFF_SIZE

import time
import numpy as np
import geometry_msgs.msg as gm
import sensor_msgs.msg as sm
import threading

##
# Converts any ros message class into a dictionary
# (currently used to make PoseStamped messages picklable)
# @param msg
# @return a nested dict with rospy.Time turned into normal unix time float
def ros_to_dict(msg):
    d = {}
    for f in msg.__slots__:
        val = eval('msg.%s' % f)
        methods = dir(val)
        if 'to_time' in methods:
            val = eval('val.to_time()')
        elif '__slots__' in methods:
            val = ros_to_dict(val)
        d[f] = val
    return d

###
## Converts a list of pr2_msgs/PressureState into a matrix
##
## @return left mat, right mat, array
#def pressure_state_to_mat(contact_msgs):
#    times = np.array([c.header.stamp.to_time() for c in contact_msgs])
#    left, right = zip(*[[list(c.l_finger_tip), list(c.r_finger_tip)] for c in contact_msgs])
#
#    left = np.matrix(left).T
#    right = np.matrix(right).T
#    return left, right, times

def np_to_pointcloud(points_mat, frame):
    pc = sm.PointCloud()
    pc.header.stamp = rospy.get_rostime()
    pc.header.frame_id = frame
    for i in range(points_mat.shape[1]):
        p32 = gm.Point32()
        p32.x = points_mat[0,i]
        p32.y = points_mat[1,i]
        p32.z = points_mat[2,i]
        pc.points.append(p32)
    return pc

def np_to_colored_pointcloud(points_mat, intensity, frame):
    pc = np_to_pointcloud(points_mat, frame)
    pc.channels.append(sm.ChannelFloat32())
    pc.channels[0].name = 'intensity'
    pc.channels[0].values = intensity.A1.tolist()
    return pc

##
# @param points_mat 2xn
# @param intensities 3xn
# @param frame
def np_to_rgb_pointcloud(points_mat, intensities, frame):
    pc = np_to_pointcloud(points_mat, frame)
    pc.channels.append(sm.ChannelFloat32())
    pc.channels[0].name = 'r'
    pc.channels[0].values = intensities[2,:].A1.tolist()

    pc.channels.append(sm.ChannelFloat32())
    pc.channels[1].name = 'g'
    pc.channels[1].values = intensities[1,:].A1.tolist()

    pc.channels.append(sm.ChannelFloat32())
    pc.channels[2].name = 'b'
    pc.channels[2].values = intensities[0,:].A1.tolist()
    return pc

def pointcloud_to_np(pc):
    plist = []
    for p in pc.points:
        plist.append([p.x, p.y, p.z])
    return np.matrix(plist).T

#class LaserScanner:
#    def __init__(self, service):
#        srv_name = '/%s/single_sweep_cloud' % service
#        self.sp = rospy.ServiceProxy(srv_name, snp.BuildCloudAngle)
#
#    def scan_np(self, start, end, duration):
#        resp = self.sp(start, end, duration)
#        return pointcloud_to_np(resp.cloud)
#
#    def scan(self, start, end, duration):
#        resp = self.sp(start, end, duration)
#        return resp.cloud

##
# Iterator function for simplified filtered bag reading. Works with large bags/messages.
#
# @param file_name bag file name
# @param topics list of topics that you care about (leave blank to get everything)
# @return list of (topic_name, message, rostime)
def bag_iter(file_name, topics=[]):
    f = open(file_name)
    tdict = {}
    for t in topics:
        tdict[t] = True

    for r in rosrecord.logplayer(f):
        if tdict.has_key(r[0]):
            yield r

    f.close()

##
# Select topics from a given bag.  Use this only if you're dealing with small
# messages as not everything will fit into RAM.
#
# @param file_name bag file name
# @param topics list of topics that you care about (leave blank to get everything)
# @return dict with each entry containing a list of [(topic_name, message, rostime), (...), ...]
def bag_sel_(file_name, topics=[]):
    print 'deprecated'
    d = {}
    for topic, msg, t in bag_iter(file_name, topics):
        if not d.has_key(topic):
            d[topic] = []
        d[topic].append((topic, msg, t.to_time()))
    return d

def bag_sel(file_name, topics=[]):
    d = {}
    for topic, msg, t in bag_iter(file_name, topics):
        if not d.has_key(topic):
            d[topic] = {'topic':[], 'msg':[], 't':[]}
        d[topic]['topic'].append(topic)
        d[topic]['msg'].append(msg)
        d[topic]['t'].append(t.to_time())
        #d[topic].append((topic, msg, t.to_time()))
    return d

##
# Used on ROS server (service provider) side to conveniently declare
# a function that returns nothing as a service.
#
# ex.  obj = YourObject()
#      rospy.Service('your_service', YourSrv,
#                    wrap(obj.some_function,
#                         # What that function should be given as input from the request
#                         ['request_data_field_name1',  'request_data_field_name2'],
#                         # Class to use when construction ROS response
#                         response=SomeClass
#                         ))
def wrap(f, inputs=[], response=srv.EmptyResponse, verbose=False):
    def _f(request):
        arguments = [eval('request.' + arg) for arg in inputs]
        if verbose:
            print 'Function', f, 'called with args:', arguments
        try:
            returns = f(*arguments)
            if returns == None:
                return response()
            else:
                return response(*returns)
        except Exception, e:
            print e
    return _f

def ignore_return(f):
    def _f(*args):
        f(*args)
    return _f

class UnresponsiveServerError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

##
# Listens over ros network
class FloatArrayListener:
    ##
    # Constructor
    #
    # @param node_name name of node if current process has not initialized ROS
    # @param listen_channel name of channel to listen to
    # @param frequency frequency this node should expect to get messages,
    #        this value is used for determinining when messages are stale
    def __init__(self, node_name, listen_channel, frequency):
        try:
            print node_name, ': inited node.'
            rospy.init_node(node_name, anonymous=True)
        except rospy.ROSException, e:
            #print e
            pass

        self.reading             = None
        self.last_message_number = None
        self.last_msg_time       = None
        self.last_call_back      = None

        self.delay_tolerance     = 300.0 / 1000.0 #Because of Nagel's
        self.delay_time          = None
        self.time_out            = 1.2 #Because of Nagel's algorithm!!! FIX THIS WITH NEW ROS!
        self.count = 0

        self.error = False

        def callback(msg):
            msg_time      = msg.header.stamp.to_time()
            msg_number    = msg.header.seq
            self.reading  = np.matrix(msg.data, 'f').T, msg_number

            #Check for delayed messages
            if self.last_msg_time == None:
                self.last_msg_time = msg_time

            time_diff = msg_time - self.last_msg_time
            if time_diff > self.delay_tolerance:
                self.delay_time = msg_time - self.last_msg_time
            #print 1000*time_diff
            self.last_msg_time  = msg_time

            ctime = time.time()
            #if self.last_call_back != None:
            #    print 1000.0*(ctime - self.last_call_back)
            #self.last_call_back = time.time()
            #if self.last_call_back != None:
            #    print 'last called back at %.2f ms ago, %d'% (1000*(ctime-self.last_call_back), self.count)
            self.last_call_back = ctime
            self.count = self.count + 1

        rospy.Subscriber(listen_channel, FloatArray, callback)
        self.node_name = node_name
        print node_name,': subscribed to', listen_channel

    def _check_timeout(self):
        if self.last_call_back != None:
            #ctime = time.time()
            time_diff = time.time() - self.last_call_back
            if time_diff > self.delay_tolerance:
                print self.node_name, ': have not heard back from publisher in', 1000*time_diff, 'ms'
                time.sleep(5/1000.0)

            if time_diff > 1.0:
                self.error = True

            if time_diff > self.time_out:
                #print 'raising hell!', ctime, self.last_call_back
                #if self.times > 0:
                    #print 'Times', self.times
                print "FloatArrayListener: Server have not responded for %.2f ms" % (1000 * time_diff)
                #exit()
                #raise UnresponsiveServerError("Server have not responded for %.2f ms" % (1000 * time_diff))
                #else:
                #    self.times = self.times + 1

    def read(self, fresh=True):
        if not fresh and self.reading == None:
            return None
        else:
            t = time.time()
            while self.reading  == None:
                time.sleep(1.0)
                print self.node_name, ': no readings for %.2f s' % (time.time() - t)

        reading = self.reading
        if fresh:
            while reading[1] == self.last_message_number:
                #self._check_timeout()
                time.sleep(1/1000.0)
                if self.delay_time != None:
                    delay_time = self.delay_time
                    self.delay_time = None #prevent multiple Exceptions from being thrown
                    print 'WARNING: delayed by', delay_time * 1000.0
                reading = self.reading
        else:
            self._check_timeout()

        self.last_message_number = reading[1]
        return reading[0]

##
# Takes a normal ROS callback channel and gives it an on demand query style
# interface.
class GenericListener:
    ##
    # Message has to have a header
    # @param node_name name of node (if haven't been inited)
    # @param message_type type of message to listen for
    # @param listen_channel ROS channel to listen
    # @param frequency the frequency to expect messages (used to print warning statements to console)
    # @param message_extractor function to preprocess the message into a desired format
    # @param queue_size ROS subscriber queue (None = infinite)
    def __init__(self, node_name, message_type, listen_channel,
                 frequency, message_extractor=None, queue_size=None):
        try:
            print node_name, ': inited node.'
            rospy.init_node(node_name, anonymous=True)
        except rospy.ROSException, e:
            pass
        self.last_msg_returned   = None   #Last message returned to callers from this class
        self.last_call_back      = None   #Local time of last received message
        self.delay_tolerance     = 1/frequency #in seconds
        self.reading             = {'message':None, 'msg_id':-1}
        self.curid               = 0
        self.message_extractor = message_extractor

        def callback(*msg):
            #If this is a tuple (using message filter)
            if 'header' in dir(msg):
                if msg.__class__ == ().__class__:
                    msg_number = msg[0].header.seq
                else:
                    msg_number = msg.header.seq
            else:
                msg_number = self.curid
                self.curid += 1

            #*msg makes everything a tuple.  If length is one, msg = (msg, )
            if len(msg) == 1:
                msg = msg[0]

            self.reading  = {'message':msg, 'msg_id':msg_number}

            #Check for delayed messages
            self.last_call_back = time.time() #record when we have been called back last

        if message_type.__class__ == [].__class__:
            import message_filters
            subscribers = [message_filters.Subscriber(channel, mtype) for channel, mtype in zip(listen_channel, message_type)]
            queue_size = 10
            ts = message_filters.TimeSynchronizer(subscribers, queue_size)
            ts.registerCallback(callback)
        else:
            rospy.Subscriber(listen_channel, message_type, callback,
                             queue_size = queue_size)

        self.node_name = node_name
        #print node_name,': subscribed to', listen_channel
        rospy.loginfo('%s: subscribed to %s' % (node_name, listen_channel))

    def _check_for_delivery_hiccups(self):
        #If have received a message in the past
        if self.last_call_back != None:
            #Calculate how it has been
            time_diff = time.time() - self.last_call_back
            #If it has been longer than expected hz, complain
            if time_diff > self.delay_tolerance:
                print self.node_name, ': have not heard back from publisher in', time_diff, 's'

    def _wait_for_first_read(self, quiet=False):
        if not quiet:
            rospy.loginfo('%s: waiting for reading ...' % self.node_name)
        while self.reading['message'] == None and not rospy.is_shutdown():
            time.sleep(0.1)
            #if not quiet:
            #    print self.node_name, ': waiting for reading ...'

    ##
    # Supported use cases
    # rfid   - want to get a reading, can be stale, no duplication allowed (allow None),        query speed important
    # hokuyo - want to get a reading, can be stale, no duplication allowed (don't want a None), willing to wait for new data (default)
    # ft     - want to get a reading, can be stale, duplication allowed    (don't want a None), query speed important
    # NOT ALLOWED                                   duplication allowed,                        willing to wait for new data
    def read(self, allow_duplication=False, willing_to_wait=True, warn=False, quiet=True):
        if allow_duplication:
            if willing_to_wait:
                raise RuntimeError('Invalid settings for read.')
            else:
                # ft - want to get a reading, can be stale, duplication allowed (but don't want a None), query speed important
                #self._wait_for_first_read(quiet)
                reading                = self.reading
                self.last_msg_returned = reading['msg_id']
                if self.message_extractor is not None:
                    return self.message_extractor(reading['message'])
                else:
                    return reading['message']
        else:
            if willing_to_wait:
                # hokuyo - want to get a reading, can be stale, no duplication allowed (don't want a None), willing to wait for new data (default)
                self._wait_for_first_read(quiet)
                while self.reading['msg_id'] == self.last_msg_returned and not rospy.is_shutdown():
                    if warn:
                        self._check_for_delivery_hiccups()
                    time.sleep(1/1000.0)
                reading = self.reading
                self.last_msg_returned = reading['msg_id']
                if self.message_extractor is not None:
                    return self.message_extractor(reading['message'])
                else:
                    return reading['message']
            else:
                # rfid   - want to get a reading, can be stale, no duplication allowed (allow None),        query speed important
                if self.last_msg_returned == self.reading['msg_id']:
                    return None
                else:
                    reading = self.reading
                    self.last_msg_returned = reading['msg_id']
                    if self.message_extractor is not None:
                        return self.message_extractor(reading['message'])
                    else:
                        return reading['message']

##
#Class for registering as a subscriber to a specified topic, where
#the messages are of a given type. Only calls the callback at a
#prescribed rate.
class RateListener():
    #Constructor.

    # Wraps the Subscriber class to only call the callback at
    # no quicker than the specified rate.

    #@param rate: time in seconds for subscriber to call callback
    #@type rate: float
    #@param name: graph resource name of topic, e.g. 'laser'.
    #@type  name: str
    #@param data_class: data type class to use for messages,
    #  e.g. std_msgs.msg.String
    #@type  data_class: L{Message} class
    #@param callback: function to call ( fn(data)) when data is
    #  received. If callback_args is set, the function must accept
    #  the callback_args as a second argument, i.e. fn(data,
    #  callback_args).  NOTE: Additional callbacks can be added using
    #  add_callback().
    #@type  callback: str
    #@param callback_args: additional arguments to pass to the
    #  callback. This is useful when you wish to reuse the same
    #  callback for multiple subscriptions.
    #@type  callback_args: any
    #@param queue_size: maximum number of messages to receive at
    #  a time. This will generally be 1 or None (infinite,
    #  default). buff_size should be increased if this parameter
    #  is set as incoming data still needs to sit in the incoming
    #  buffer before being discarded. Setting queue_size
    #  buff_size to a non-default value affects all subscribers to
    #  this topic in this process.
    #@type  queue_size: int
    #@param buff_size: incoming message buffer size in bytes. If
    #  queue_size is set, this should be set to a number greater
    #  than the queue_size times the average message size. Setting
    #  buff_size to a non-default value affects all subscribers to
    #  this topic in this process.
    #@type  buff_size: int
    #@param tcp_nodelay: if True, request TCP_NODELAY from
    #  publisher.  Use of this option is not generally recommended
    #  in most cases as it is better to rely on timestamps in
    #  message data. Setting tcp_nodelay to True enables TCP_NODELAY
    #  for all subscribers in the same python process.
    #@type  tcp_nodelay: bool
    #@raise ROSException: if parameters are invalid
    def __init__(self, rate, name, data_class, callback=None, callback_args=None,
                 queue_size=None, buff_size=DEFAULT_BUFF_SIZE, tcp_nodelay=False):
        self.rate = rate
        self.callback = callback
        self.lasttime = 0.0
        self.sub = rospy.Subscriber(name, data_class, self._process_msg, callback_args,
                              queue_size, buff_size, tcp_nodelay)

    def _process_msg(self, data, callback_args=None):
        delay = rospy.Time.now().to_sec() - self.lasttime
        if delay >= self.rate or self.lasttime == 0.0:
            if callback_args == None:
                self.callback(data)
            else:
                self.callback(data, callback_args)
            self.lasttime = rospy.Time.now().to_sec()

    def unregister(self):
        self.sub.unregister()

    def get_num_connections():
        return self.sub.get_num_connections()

class RateCaller():

    def __init__(self, func, rate, args=None):
        self.func = func
        self.rate = rate
        self.args = args
        self.thread = None
        self.is_running = False
        self.beg_time = None

    def run(self):
        if not self.is_running:
            self.beg_time = time.time()
            self.num_time = 0
            self.thread = threading.Timer(self.rate, self._run)
            self.is_running = True
            self.thread.start()

    def _run(self):
        if self.is_running:
            if self.args is None:
                self.func()
            else:
                self.func(*self.args)
            self.num_time += 1

            diff_time = time.time() - self.beg_time
            timer_len = self.rate * (self.num_time + 1) - diff_time
            if timer_len <= 0.:
                timer_len = 0.
            self.thread = threading.Timer(timer_len, self._run)
            self.thread.start()

    def stop(self):
        if self.thread is not None and self.is_running:
            self.thread.cancel()
            self.is_running = False

#class TransformBroadcaster:
#
#    def __init__(self):
#        self.pub_tf = rospy.Publisher("/tf", tf.msg.tfMessage)
#
#    ## send transform as a tfmessage.
#    # @param tf_stamped - object of class TransformStamped (rosmsg show TransformStamped)
#    def send_transform(self,tf_stamped):
#        tfm = tf.msg.tfMessage([tf_stamped])
#        self.pub_tf.publish(tfm)

##
# Calls the service with the given name and given parameters and returns
# the output of the service.
# @param service_name the full name of the service to be called
# @param params the list of parameters to be passed into the service
# @filename name of a pickle object to be created if not None
def call_save_service(service_name, service_def, params, filename=None):
    srv = rospy.ServiceProxy(service_name, service_def)
    try:
        resp = srv(*params)
        if not filename is None:
            save_pickle(resp, filename)
        srv.close()
        return resp
    except rospy.ServiceException, e:
        print "Service error"
    srv.close()
    return None

#class ROSPoll:
#    def __init__(self, topic, type):
#        self.data       = None
#        self.t          = time.time()
#        self.old        = self.t
#        self.subscriber = rospy.TopicSub(topic, type, self.callback)
#
#    def callback(self, data):
#        self.data = data
#        self.t    = time.time()
#
#    ##
#    # Returns only fresh data!
#    def read(self):
#        t = time.time()
#        while self.old == self.t:
#            time.sleep(.1)
#            if (time.time() - t) > 3.0:
#                print 'ROSPoll: failed to read. Did you do a rospy.init_node(name)?'
#                return None
#        self.old = self.t
#        return self.data
#
##
# Sample Code
#if __name__ == '__main__':
#    from pkg import *
#    from urg_driver.msg import urg as UrgMessage
#    rospy.init_node('test') #Important before doing anything ROS related!
#
#    poller = ROSPoll('urg', UrgMessage)
#    msg    = poller.read()
#    print msg.__class__

    #def _check_freshness(self, current_time):
    #    if self.last_received_time != None:
    #        time_diff = current_time - self.last_received_time
    #        #print '%.2f' %( time_diff * 1000.0)
    #        if time_diff > self.stale_tolerance:
    #            self.throw_exception = time_diff
    #def _stale_alert(self):
    #    if self.throw_exception != None:
    #        self.throw_exception = None
    #        raise RuntimeError('Have not heard from publisher for %.2f ms' % (1000.0 * self.throw_exception))

###
## Wrapper helper function
#def empty(f):
#    def _f(request):
#        print 'Request', request, ' for function', f, 'received.'
#        try:
#            f()
#        except Exception, e:
#            print e
#        return srv.EmptyResponse()
#    return _f

    #def read(self, fresh=True, no_wait, warn=True):
    #    #if reading
    #    reading = self.reading
    #    if reading != None:
    #        if fresh:
    #            #While the current message is equal the the last message we returned caller
    #            while reading['msg_id'] == self.last_msg_returned:
    #                if warn:
    #                    self._check_for_delivery_hiccups()
    #                reading = self.reading
    #                time.sleep(1/1000.0)
    #            self.last_msg_returned = reading['msg_id']
    #            return reading['message']

    #        elif allow_duplicates:
    #            self.last_msg_returned = reading['msg_id']
    #            return reading['message']

    #        else:
    #            #not fresh & don't allow duplicates
    #            if reading['msg_id'] == self.last_msg_returned:
    #                return None
    #            else:
    #                self.last_msg_returned = reading['msg_id']
    #                return reading['message']
    #    else:
    #        fresh and allow_duplicates
    #        fresh and not allow_duplicates
    #        not fresh and not allow_duplicates
    #        not fresh and allow_duplicates

   #     #Is this the first time we have been called?
   #     if not fresh and self.reading == None:
   #         return None
   #     else:
   #         while self.reading  == None:
   #             time.sleep(.3)
   #             print self.node_name, ': waiting for reading ...'

   #     reading = self.reading
   #     if fresh:
   #         #While the current message is equal the the last message we returned caller
   #         while reading[1] == self.last_msg_returned:
   #             if warn:
   #                 self._check_for_delivery_hiccups()
   #             #if self.delay_time != None:
   #             #    delay_time = self.delay_time
   #             #    self.delay_time = None #prevent multiple Exceptions from being thrown
   #             #    if warn:
   #             #        print 'WARNING: delayed by', delay_time * 1000.0
   #             reading = self.reading
   #             time.sleep(1/1000.0)
   #     else:
   #         self._check_timeout()

   #     self.last_msg_returned = reading[1]
   #     return reading[0]

max_stars_repo_name: gt-ros-pkg/hrl-lib

max_issues_repo_name: gt-ros-pkg/hrl-lib

max_forks_repo_name: gt-ros-pkg/hrl-lib


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import math
import time
import ctypes
import socket
from threading import Event, Thread
from socketserver import ThreadingMixIn
from http.server import BaseHTTPRequestHandler, HTTPServer
try:
	import mss
except:
	print('Install mss with "pip3 install mss"')
	quit()
try:
	import win32api
	import win32gui
	from win32con import MONITOR_DEFAULTTONEAREST
except:
	print('Install pywin32 with "pip3 install pywin32"')
	quit()
try:
	import cv2
	import numpy as np
except:
	print('Install OpenCV with "pip3 install opencv-python"')
	quit()

ip = socket.gethostbyname(socket.gethostname())
port = 9090
x, y, w, h = 0, 0, 0, 0

class CamHandler(BaseHTTPRequestHandler):
	def do_GET(self):
		global ip, port
		jpegQuality = [int(cv2.IMWRITE_JPEG_QUALITY), 90]
		if self.path.endswith('/rgb.mjpg'):
			self.send_response(200)
			self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
			self.send_header('Access-Control-Allow-Origin', '*')
			self.end_headers()
			while True:
				if(rgbframe.any() != None):
					pass
				r, buf = cv2.imencode('.jpg', rgbframe, jpegQuality)
				try:
					self.wfile.write('--jpgboundary\r\n'.encode())
					self.end_headers()
					self.wfile.write(bytearray(buf))
				except:
					pass
			return

		if self.path.endswith('/depth.mjpg'):
			self.send_response(200)
			self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
			self.send_header('Access-Control-Allow-Origin', '*')
			self.end_headers()
			while True:
				if(depthframe.any() != None):
					pass
				r, buf = cv2.imencode('.jpg', depthframe, jpegQuality)
				try:
					self.wfile.write('--jpgboundary\r\n'.encode())
					self.end_headers()
					self.wfile.write(bytearray(buf))
				except:
					pass
			return

		if self.path == '/':
			self.send_response(200)
			self.send_header('Content-type', 'text/html')
			self.end_headers()
			self.wfile.write(''.encode())
			self.wfile.write(('').encode())
			self.wfile.write(('').encode())
			self.wfile.write(''.encode())
			return

class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
	"""Handle requests in a separate thread."""

def callback(hwnd, extra):
	global x, y, w, h
	if win32gui.GetWindowText(hwnd)[:9] == 'Minecraft':
		shcore = ctypes.windll.shcore
		dpi_aware = shcore.SetProcessDpiAwareness(2)
		rect = win32gui.GetWindowRect(hwnd)
		x = rect[0] + 8
		y = rect[1] + 31
		w = rect[2] - x - 8
		h = rect[3] - y - 8
		print('Found "%s" ' % win32gui.GetWindowText(hwnd) + 'window of size (%d,%d) ' % (w, h) + 'in location (%d,%d).' % (x, y))

def main():
	global rgbframe, depthframe
	server = ThreadedHTTPServer(('0.0.0.0', port), CamHandler)
	print('Starting server on address ' + ip + ':' + str(port))
	target = Thread(target = server.serve_forever, args = ())
	target.start()
	mon = {'top': y, 'left': x, 'width': w, 'height': h}
	sct = mss.mss()

	while True:
		start = time.time()
		img = np.asarray(sct.grab(mon))
		if((img[0,0][0] == img[0,0][1] == img[0,0][2]) and (img[50,50][0] == img[50,50][1] == img[50,50][2])):
			depthframe = img
		else:
			rgbframe = img

		#cv2.imshow('Depthmap', depthframe)
		#cv2.imshow('RGB', rgbframe)
		#if cv2.waitKey(25) & 0xFF == ord("q"):
		#	cv2.destroyAllWindows()
		#	break

win32gui.EnumWindows(callback, None)
if(x ==0 and y == 0 and w == 0 and h == 0):
	print('No Minecraft window found.')
	quit()
rgbframe = np.zeros([w, h, 3], dtype = np.uint8)
rgbframe.fill(0)
depthframe = np.zeros([w, h, 3], dtype = np.uint8)
depthframe.fill(0)
main()