text:
# -*- coding: utf-8 -*-
"""Implementation of Gaussian DyBMs whose parameters are learned in batch manner.

"""

__author__ = "Rudy Raymond"
__copyright__ = "(C) Copyright IBM Corp. 2017"

import numpy as np
from copy import deepcopy
from six.moves import xrange

from .. import arraymath as amath
from ..time_series.rnn_gaussian_dybm import GaussianDyBM
from ..time_series.batch_dybm import BatchLinearDyBM
from ..base.generator import SequenceGenerator
from ..base.sgd import AdaGrad

class BatchGaussianDyBM(GaussianDyBM):

    """
    GaussianDyBM with batch learning over past input sequence

    Parameters
    ----------
    in_dim : int
        dimension of input time-series
    out_dim : int, optional
        dimension of target time-series
    delay : int, optional
        length of fifo queue plus one
    decay_rates : list, optional
        decay rates of eligibility traces
    SGD : object of SGD.SGD, optional
        object of a stochastic gradient method
    L1 : float, optional
        strength of L1 regularization
    L2 : float, optional
        strength of L2 regularization
    insert_to_etrace : str, {"w_delay", "wo_delay"},  optional
        "w_delay" : insert pattern observed d-1 time steps ago into
        eligibility traces
        "wo_delay" : insert the latest pattern into eligibility traces
    batch_method: string, optional
        Choose from "Ridge", "Lasso", or "MultiTaskLasso". Default is Ridge
    positive: boolean, optional
        If true and batch_method="Lasso", the coefficients are all positive
        values

    Attributes
    ----------
    in_dim : int
        in_dim
    out_dim : int
        out_dim
    n_etrace : int
        the number of eligibility traces
    len_fifo : int
        the length of FIFO queues (delay - 1)
    decay_rates : array, shape (n_etrace,1)
        decay rates of eligibility traces
    variables : dict
        dictionary of model parameters
    variables["W"] : array, shape (len_fifo, in_dim, out_dim)
        variables["W"][l] corresponds to the weight from the input observed at
        time step t - l - 1 to the mean at time step t (current time).
    variables["b"] : array, shape (out_dim,)
        variables["b"] corresponds to the bias to out_pattern.
    variables["s"] : array, shape (out_dim,)
        variables["s"][n] corresponds to the standard deviation of
        out_pattern[n] (or scale parameter in other words).
    variables["V"] : array, shape (n_etrace, in_dim, out_dim)
        variables["V"][k] corresponds to the weight from the k-th eligibility
        trace to the mean.
    fifo : deque
        FIFO queue storing len_fifo in_patterns, each in_pattern has shape (in_dim,).
    e_trace : array, shape (n_etrace, in_dim)
        e_trace[k, :] corresponds to the k-th eligibility trace.
    """

    def __init__(self, in_dim, out_dim=None, delay=2, decay_rates=[],
                 SGD=None, L1=0., L2=0., insert_to_etrace="wo_delay",
                 batch_method="Ridge", positive=False):
        self.batch_method = batch_method
        self._positive = positive
        GaussianDyBM.__init__(self, in_dim, out_dim=out_dim, delay=delay,
                              decay_rates=decay_rates, SGD=SGD, L1=L1, L2=L2,
                              insert_to_etrace=insert_to_etrace)

    def init_state(self, in_seq=None, out_seq=None):
        """
        Initializing FIFO queue and eligibility traces.
        Weight matrices of GaussianDyBM are initialized by batch-learning on
        in_seq and out_seq, if in_seq is not None.
        Otherwise, if in_seq is None it is the same as GaussianDyBM.

        Parameters
        ----------
        in_seq : optional, list of numpy or arraymath array each of shape (in_dim, )
            input sequence
        out_seq : optional, list of numpy or arraymath array each of shape (out_dim, )
            output sequence

        """

        GaussianDyBM.init_state(self)
        if in_seq is not None:
            if out_seq is None:
                out_seq = in_seq
            self._learn_batch(in_seq, out_seq)

    def fit(self, in_seq, out_seq=None, createDyBM=True):
        """
        Fit GaussianDyBM with in_seq and out_seq and set FIFO and
        the eligibility trace

        Parameters
        ----------
        in_seq: list of arraymath array each of shape(in_dim, )
            input sequence
        out_seq: optional, list of numpy or arraymath array each of shape
            (out_dim, ) output sequence
        createDyBM: optional, boolean
            if True, return GaussianDyBM fitted to in_seq and out_seq

        Returns
        -------
        GaussianDyBM fitted to in_seq and out_seq if createDyBM is True

        """

        self.learn_batch(in_seq, out_seq)
        for in_pattern in in_seq:
            self._update_state(in_pattern)

        if createDyBM:
            return super(BatchGaussianDyBM, self)

    def predict(self, in_seq):
        """
        Predict using GaussianDyBM without updating its parameters.
        Use learn_one_step() to learn with parameter updates

        Parameters
        ----------
        in_seq: list of arraymath array each of shape(in_dim, )
            input sequence

        Returns
        -------
        out_seq : sequence of output
            generator of output sequence

        """

        answer = []
        for in_pattern in in_seq:
            self._update_state(in_pattern)
            answer.append(self.predict_next())
        return SequenceGenerator(answer)

    def learn_batch(self, in_seq, out_seq=None):
        """
        Initialize weight matrices of GaussianDyBM by batch-learning

        Parameters
        ----------
        in_seq : list of numpy or arraymath array each of shape (in_dim, )
            input sequence
        out_seq : optional, list of numpy or arraymath array each of shape
            (out_dim, ) output sequence

        """

        if out_seq is None:
            out_seq = in_seq
        self._learn_batch(in_seq, out_seq)

    def _learn_batch(self, in_seq, out_seq):
        """
        private method to perform batch learning

        """

        in_seq = [amath.array(i) for i in in_seq]
        out_seq = [amath.to_numpy(o) for o in out_seq]

        #  get mean and variances of out_seq
        means = np.zeros(self.out_dim)
        stdevs = np.zeros(self.out_dim)
        for i, each in enumerate(zip(*out_seq)):
            means[i] = np.mean(each)
            stdevs[i] = np.std(each)
        #  normalize out_seq
        normalized_out_seq = deepcopy(out_seq)
        for j in xrange(len(normalized_out_seq)):
            normalized_out_seq[j] = (normalized_out_seq[j] - means)/stdevs

        #  perform BatchLinearDyBM on normalized out_seq
        linDyBM = BatchLinearDyBM(self.in_dim, self.out_dim, delay=self.len_fifo+1,
                                  decay_rates=self.decay_rates, SGD=AdaGrad(),
                                  L1=self.L1["V"], L2=self.L2["V"],
                                  use_bias=True, sigma=0,
                                  insert_to_etrace=self.insert_to_etrace,
                                  esn=None, batch_method=self.batch_method,
                                  positive=self._positive)
        linDyBM.init_state(in_seq, normalized_out_seq)  # initialize parameters
        normalized_pred = linDyBM.predict(in_seq).to_list()  # get parameters

        #  get mean squared errors
        sqerrs = np.zeros(self.out_dim)
        for t in xrange(len(normalized_pred)):
            y1, y2 = normalized_pred[t], normalized_out_seq[t]
            for i in xrange(self.out_dim):
                sqerrs[i] += (y1[i] - y2[i])**2
        T = len(normalized_pred)
        for i in xrange(self.out_dim):
            sqerrs[i] /= T

        #  assign standard deviations
        for i in xrange(self.out_dim):
            self.variables["s"][0][i] = np.sqrt(sqerrs[i]) * stdevs[i]

        #  assign parameters of GaussianDyBM from LinearDyBM
        Sigma = amath.diag(amath.array(stdevs))
        self.variables["b"] = \
            amath.dot(Sigma, linDyBM.variables["b"]) + amath.array(means)
        for l in range(self.len_fifo):
            self.variables["W"][l] = amath.dot(linDyBM.variables["W"][l], Sigma)

        for k in range(self.n_etrace):
            self.variables["V"][k] = amath.dot(linDyBM.variables["V"][k], Sigma)

max_stars_repo_name: rraymondhp/dybm

max_issues_repo_name: rraymondhp/dybm

max_forks_repo_name: rraymondhp/dybm


text:
"""Google Football environment."""

import collections
import copy

import gym
import numpy as np

from alpacka.envs import base
from alpacka.third_party import gfootball as gfootball_utils

try:
    import gfootball.env as football_env
except ImportError:
    football_env = None

class GoogleFootball(base.ModelEnv):
    """Google Research Football conforming to the ModelEnv interface."""

    state_size = 480000

    class Renderer(base.EnvRenderer):
        """Renderer for GFootball.

        Uses the state visualization from GFootball's dumped videos.
        """

        def render_state(self, state_info):
            return gfootball_utils.get_frame(state_info)

        def render_action(self, action):
            action_set = football_env.football_action_set.full_action_set
            return str(action_set[action])

    def __init__(self,
                 env_name='academy_empty_goal_close',
                 representation='simple115',
                 rewards='scoring,checkpoints',
                 stacked=False,
                 dump_path=None,
                 solved_at=1,
                 **kwargs):
        if football_env is None:
            raise ImportError('Could not import gfootball! '
                              'HINT: Perform the setup instructions here: '
                              'https://github.com/google-research/football')

        self._solved_at = solved_at
        self._env = football_env.create_environment(
            env_name=env_name,
            representation=representation,
            rewards=rewards,
            stacked=stacked,
            write_full_episode_dumps=dump_path is not None,
            write_goal_dumps=False,
            logdir=dump_path or '',
            **kwargs
        )

        self.action_space = self._env.action_space
        self.observation_space = copy.copy(self._env.observation_space)
        self.observation_space.dtype = np.float32
        assert len(self.observation_space.shape) in (1, 3), (
            'Unsupported observation shape: {}.'.format(
                self.observation_space.shape
            )
        )
        self._has_pixel_input = len(self.observation_space.shape) == 3

    def _scale_obs(self, obs):
        if self._has_pixel_input:
            # Pixel observations are given as uint8. Cast to float and scale
            # to the [0, 1) interval.
            return (obs.astype(np.float32) - 128) / 128
        else:
            # Float observations don't need scaling.
            return obs

    def reset(self):
        # pylint: disable=protected-access
        obs = self._env.reset()
        env = self._env.unwrapped
        env._env._trace._trace = collections.deque([], 4)

        return self._scale_obs(obs)

    def step(self, action):
        obs, reward, done, info = self._env.step(action)
        if done:
            info['solved'] = info['score_reward'] >= self._solved_at
        return self._scale_obs(obs), reward, done, info

    def render(self, mode='human'):
        return self._env.render(mode)

    def close(self):
        self._env.close()

    def seed(self, seed=None):
        raise NotImplementedError

    def clone_state(self):
        # pylint: disable=protected-access
        raw_state = self._env.get_state()
        size_encoded = len(raw_state).to_bytes(3, byteorder='big')
        # Byte suffix to enforce self.state_size of state.
        suffix = bytes(self.state_size - len(size_encoded) - len(raw_state))
        resized_state = size_encoded + raw_state + suffix
        return np.frombuffer(resized_state, dtype=np.uint8)

    def restore_state(self, state):
        assert state.size == self.state_size, (
            f'State size does not match: {state.size} != {self.state_size}')

        # First 3 bytes encodes size of state.
        size_decoded = int.from_bytes(list(state[:3]), byteorder='big')
        raw_state = state[3:(size_decoded + 3)]
        assert (state[(size_decoded + 3):] == 0).all()

        self._env.set_state(bytes(raw_state))

        # Temporary fix for a bug in Football related to restoring state after
        # done. Proper fix is on the way:
        # https://github.com/google-research/football/pull/135
        # pylint: disable=protected-access,import-outside-toplevel,no-name-in-module,import-error
        from gfootball.env import observation_processor
        env = self._env.unwrapped._env
        if env._trace is None:
            env._trace = observation_processor.ObservationProcessor(env._config)

        return self._scale_obs(self._observation)

    @property
    def _observation(self):
        # TODO(xxx): Hacky, clean it when implementation of football allow it
        # pylint: disable=protected-access
        observation = self._env.unwrapped._env.observation()
        observation = self._env.unwrapped._convert_observations(
            observation, self._env.unwrapped._agent,
            self._env.unwrapped._agent_left_position,
            self._env.unwrapped._agent_right_position
        )
        # pylint: enable=protected-access

        # Lets apply observation transformations from wrappers.
        # WARNING: This assumes that only ObservationWrapper(s) in the wrappers
        # stack transform observation.
        env = self._env
        observation_wrappers = []
        while True:
            if isinstance(env, gym.ObservationWrapper):
                observation_wrappers.append(env)
            if isinstance(env, football_env.wrappers.FrameStack):
                # TODO(xxx): Black magic! We know that FrameStack keeps already
                # processed observations and we can return it here. Loose this
                # assumption.
                return env._get_observation()  # pylint: disable=protected-access
            if isinstance(env, gym.Wrapper):
                env = env.env
            else:
                break

        for wrapper in reversed(observation_wrappers):
            observation = wrapper.observation(observation)

        return observation

    @property
    def state_info(self):
        observations = self._env.unwrapped.observation()
        print(observations)
        assert len(observations) == 1
        return {
            key: value
            for (key, value) in observations[0].items()
            if key in {
                'left_team', 'left_team_direction',
                'right_team', 'right_team_direction',
                'ball', 'ball_direction', 'ball_owned_team',
                'active', 'game_mode',
            }
        }

ext: py

max_stars_repo_name: shoot-tree-search/sts

max_issues_repo_name: shoot-tree-search/sts

max_forks_repo_name: shoot-tree-search/sts

reason: import numpy


text:
import os
import sys
import argparse
from glob import glob

import json
import numpy as np
from PIL import Image
from multiprocessing import Pool
from scipy.interpolate import interp2d
import openslide

from utils import ensure_dir

def select_preds(pred, x_left, x_right, y_top, y_bottom):
    if x_left <= pred[0] <= x_right and y_top <= pred[1] <= y_bottom:
        return True
    else:
        return False

def ensure_cors(x_cors, y_cors, Z, x_min, x_max, y_min, y_max, x, y, pw, ph):
    if x_min > x:
        x_cors = np.insert(x_cors, 0, x)
        Z = np.insert(Z, 0, Z[:, 0], axis=1)
        x_min = x

    if x_max < x + pw:
        x_cors = np.insert(x_cors, x_cors.shape[0], x + pw)
        Z = np.insert(Z, Z.shape[1], Z[:, -1], axis=1)
        x_max = x + pw

    if y_min > y:
        y_cors = np.insert(y_cors, 0, y)
        Z = np.insert(Z, 0, Z[0, :], axis=0)
        y_min = y

    if y_max < y + ph:
        y_cors = np.insert(y_cors, y_cors.shape[0], y + ph)
        Z = np.insert(Z, Z.shape[0], Z[-1, :], axis=0)
        y_max = y + ph

    return x_cors, y_cors, x_min, x_max, y_min, y_max, Z

def interp(arguments):
    WSI_path, pred_path, output_dir, patch_size = arguments
    # some code is adopted from Le's code
    done_fn = '{}/done.txt'.format(output_dir)
    if os.path.isfile(done_fn):
        return None

    margin = 5
    margin_sp = 10
    try:
        oslide = openslide.OpenSlide(WSI_path)
        if openslide.PROPERTY_NAME_MPP_X in oslide.properties:
            mpp = float(oslide.properties[openslide.PROPERTY_NAME_MPP_X])
        elif "XResolution" in oslide.properties:
            mpp = float(oslide.properties["XResolution"]);
        elif "tiff.XResolution" in oslide.properties:
            mpp = float(oslide.properties["tiff.XResolution"]);
        else:
            mpp = 0.250

        width = oslide.dimensions[0]
        height = oslide.dimensions[1]
    except:
        print('Error in {}: exception caught exiting'.format(WSI_path))
        raise Exception('{}: exception caught exiting'.format(WSI_path))

    pw, ph = patch_size

    # Load prediction results
    cor_prob = []

    with open(pred_path) as file:
        for line in file:
            x, y, prob, _ = line.split(" ")
            x, y, prob = int(x), int(y), float(prob)
            cor_prob.append((x, y, prob))

    # cor_prob = sorted(cor_prob, key=lambda item: (item[0], item[1]))
    cor_prob_np = np.array(cor_prob)
    x_cors_all = np.unique(cor_prob_np[:, 0].astype(np.uint32))
    y_cors_all = np.unique(cor_prob_np[:, 1].astype(np.uint32))
    x_3_res = np.argpartition(x_cors_all, 3)
    y_3_res = np.argpartition(y_cors_all, 3)
    x_3_smallest = np.sort(x_cors_all[x_3_res[:3]])
    y_3_smallest = np.sort(y_cors_all[y_3_res[:3]])

    n_cors = len(cor_prob)
    spw = x_3_smallest[2] - x_3_smallest[1]
    sph = y_3_smallest[2] - y_3_smallest[1]

    for x in range(1, width, pw):
        for y in range(1, height, ph):

             # the segmentation results do not have sizes that are < patch_sizes
            if x + pw > width - margin:
                continue
            if y + ph > height - margin:
                continue

            if pw <= 3 or ph <= 3:
                continue

            x_left = max(x - spw - margin_sp, 1)
            x_right = min(x + pw + spw + margin_sp, width)
            y_top = max(y - sph - margin_sp, 1)
            y_bottom = min(y + ph + sph + margin_sp, height)
            selected_cors = filter(lambda item: select_preds(item, x_left, x_right, y_top, y_bottom), cor_prob)
            selected_cors = list(selected_cors)
            selected_cors = sorted(selected_cors, key=lambda item: (item[0], item[1]))
            selected_cors = np.array(selected_cors)

            if selected_cors.shape[0] > 0:
                x_cors = np.sort(np.unique(selected_cors[:, 0].astype(np.uint32)))
                y_cors = np.sort(np.unique(selected_cors[:, 1].astype(np.uint32)))
                nx, ny = x_cors.shape[0], y_cors.shape[0]
                if ny * nx == selected_cors.shape[0]:
                    Z = np.reshape(selected_cors[:, 2], (ny, nx), order='F')
                else:
                    Z = np.zeros((ny, nx)).astype(np.float16)
                    d_x = dict(zip(x_cors, range(nx)))
                    d_y = dict(zip(y_cors, range(ny)))
                    for i in range(selected_cors.shape[0]):
                        row = d_y[int(selected_cors[i, 1])]
                        col = d_x[int(selected_cors[i, 0])]
                        Z[row, col] = selected_cors[i, 2]

                x_min, x_max = np.min(x_cors), np.max(x_cors)
                y_min, y_max = np.min(y_cors), np.max(y_cors)
                x_cors, y_cors, x_min, x_max, y_min, y_max, Z = ensure_cors(x_cors, y_cors, Z, x_min, x_max, y_min, y_max, x, y, pw, ph)

                if x_cors.shape[0] < 16 or y_cors.shape[0] < 16:
                    f = interp2d(x_cors, y_cors, Z, kind='linear')
                else:
                    f = interp2d(x_cors, y_cors, Z, kind='cubic')
                x2 = np.linspace(x_min, x_max, x_max - x_min + 1)
                y2 = np.linspace(y_min, y_max, y_max - y_min + 1)
                Z2 = f(x2, y2)
                Z2 = np.clip(Z2, 0, 1)  # clip to [0, 1]
                ind_x_start = int(x - x_min)
                ind_y_start = int(y - y_min)
                Z2_cropped = Z2[ind_y_start : ind_y_start+ph, ind_x_start : ind_x_start+pw].astype(np.float16)
                Z.astype(np.float16)
            else:
                Z = np.zeros((1,1)).astype(np.float16)
                Z2_cropped = np.zeros((ph, pw)).astype(np.float16)

            """
            fn_pre = "{}/{}_{}_{}_{}_{}_1_PREINTP.npy".format(output_dir, x, y, pw, ph, mpp)
            fn = "{}/{}_{}_{}_{}_{}_1_INTP.npy".format(output_dir, x, y, pw, ph, mpp)

            np.save(fn_pre, Z)
            np.save(fn, Z2_cropped)
            """
            fn_pre = "{}/{}_{}_{}_{}_{}_1_PREINTP.png".format(output_dir, x, y, pw, ph, mpp)
            fn = "{}/{}_{}_{}_{}_{}_1_INTP.png".format(output_dir, x, y, pw, ph, mpp)

            Image.fromarray((Z * 255).astype(np.uint8)).save(fn_pre)
            Image.fromarray((Z2_cropped * 255).astype(np.uint8)).save(fn)

            # print("{}_{}_{}_{}_{}_1_INTP done!".format(x, y, pw, ph, mpp))

    with open(done_fn, 'a') as f:
        f.write(WSI_path)
        f.write('\n width: {}, height: {}\n'.format(width, height))
    # print('{} done!'.format(WSI_path))

def query_fn(wsi_id, path_list):
    for path in path_list:
        basename = path.split('/')[-1]
        if basename.find(wsi_id) != -1:
            return path
    return None

def interp_main(config, start_idx, end_idx, ncores):
    wsi_root = config['WSIs']['root_path']
    nuclei_seg_root = config['Nuclei_segs']['root_path']
    tumor_preds_root = config['Tumor_preds']['root_path']
    til_preds_root = config['TIL_preds']['root_path']

    # wsi_output_path = config['WSIs']['output_path']
    # nuclei_segs_output_path = config['Nuclei_segs']['output_path']
    tumor_preds_output_path = config['Tumor_preds']['output_path']
    til_preds_output_path = config['TIL_preds']['output_path']

    wsi_path_list = glob('{}/*.svs'.format(wsi_root))
    nuclei_segs_path_list = glob('{}/*.svs'.format(nuclei_seg_root))
    tumor_preds_path_list = glob('{}/prediction-*[!.low_res]'.format(tumor_preds_root))
    til_preds_path_list = glob('{}/prediction-*[!.low_res]'.format(til_preds_root))

    tasks = []

    for wsi_path in wsi_path_list:
        wsi_fn = wsi_path.split('/')[-1]
        wsi_id = wsi_fn.split('.')[0]
        # not diagnoised
        if wsi_id[-3:-1].lower() != 'dx':
            continue

        nuclei_segs_path = query_fn(wsi_id, nuclei_segs_path_list)
        if nuclei_segs_path is None:
            continue

        tumor_path = query_fn(wsi_id, tumor_preds_path_list)
        if tumor_path is None:
            continue

        til_path = query_fn(wsi_id, til_preds_path_list)
        if til_path is None:
            continue

        tumor_preds_output_path_cur = '{}/{}'.format(tumor_preds_output_path, wsi_id)
        ensure_dir(tumor_preds_output_path_cur)
        tasks.append((wsi_path, tumor_path, tumor_preds_output_path_cur, (4000,4000)))

        til_preds_output_path_cur = '{}/{}'.format(til_preds_output_path, wsi_id)
        ensure_dir(til_preds_output_path_cur)
        tasks.append((wsi_path, til_path, til_preds_output_path_cur, (4000,4000)))

    total_num = int(len(tasks) / 2)
    print('Total number of WSIs is {}'.format(total_num))

    end_idx = min(end_idx, total_num)
    tasks = tasks[start_idx*2:end_idx*2]

    if ncores > 1:
        # multiprocessing
        p = Pool(ncores)
        print('Using {} cores'.format(ncores))
        print('Doing interpolation ...')
        for i, _ in enumerate(p.imap_unordered(interp, tasks), 1):
            sys.stderr.write('\rInterpolation {0:%} done! '.format(i / len(tasks)))
    else:
        for task in tasks:
            interp(task)
    print('\nAll done! \n')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Interpolation')
    parser.add_argument('-c', '--config', default=None, type=str, help='config file path (default: None)')
    parser.add_argument('--start_idx', type=int, default=0, help='start index')
    parser.add_argument('--end_idx', type=int, default=50000, help='end index, (not included)')
    parser.add_argument('--ncores', type=int, default=10, help='number of cores')

    args = parser.parse_args()

    if args.config:
        config = json.load(open(args.config))
    else:
        raise AssertionError("Configuration file need to be specified. Add '-c config.json', for example.")

    print(config)

    start_idx = int(args.start_idx)
    end_idx = int(args.end_idx)
    ncores = int(args.ncores)

    ensure_dir(config['WSIs']['output_path'])
    ensure_dir(config['Nuclei_segs']['output_path'])
    ensure_dir(config['Tumor_preds']['output_path'])
    ensure_dir(config['TIL_preds']['output_path'])

    interp_main(config, start_idx, end_idx, ncores)

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

# Importing needed libraries
import datetime
import pandas as pd

# Fetching the data from official site of Ministry of Health and Family Welfare | Government of India
try:
    url = "https://www.mohfw.gov.in/"
    dfs = pd.read_html(url)

    for i in range(len(dfs)):
        df = dfs[i]
        if (len(df.columns) == 6):
            cols_match = sum(df.columns==['S. No.', 'Name of State / UT', 'Active Cases*',
                               'Cured/Discharged/Migrated*', 'Deaths**', 'Total Confirmed cases*'])
            if (cols_match == 6):
                now = datetime.datetime.now()
                dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
                df.to_csv("/home/caesar/covid_19.csv")
                break
except:
    df = pd.read_csv("/home/caesar/covid_19.csv")
    df = df.drop(columns=["Unnamed: 0"])
    now = datetime.datetime.fromtimestamp(os.path.getmtime("/home/caesar/covid_19.csv"))
    dt_string = now.strftime("%d/%m/%Y %H:%M:%S")

df

# In[2]:

# Preprocessing the column data to remove any special characters inbetween because we cannot rely on the source completely.
for col in ['Active Cases*', 'Cured/Discharged/Migrated*', 'Deaths**', 'Total Confirmed cases*' ]:
    if df[col].dtypes=='O':
        df[col] = df[col].str.replace('\W', '')

# In[3]:

# Fetching out the values from the DataFrame
m = 35
states = df["Name of State / UT"][:m]
active_cases = df["Active Cases*"][:m].astype(int)
confirmed_cases = df["Total Confirmed cases*"][:m].astype(int)
casualties = df["Deaths**"][:m].astype(int)

# In[4]:

# Plotting the bar graph!
get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.pyplot as plt
import numpy as np

max_cases = max(active_cases)
total_active_cases = int(df["Active Cases*"][36])
total_death_casualties = df["Deaths**"][36]
total_cured_cases = df["Cured/Discharged/Migrated*"][36]
total_confirmed_cases = df["Total Confirmed cases*"][36]
barWidth = 0.4
r1 = np.arange(len(active_cases))
r2 = [x + barWidth for x in r1]

plt.figure(figsize=(16, 8))
plt.bar(r1, active_cases, color="royalblue", width=barWidth, edgecolor="white", label="Active Cases")
plt.bar(r2, casualties, color="orchid", width=barWidth, edgecolor="white", label="Death Casualties")
plt.xlabel("States / UT", fontweight="bold", fontsize=16)
plt.xticks([r + barWidth - 0.19 for r in range(len(active_cases))], states, rotation="78", fontsize=14)
plt.ylabel("Number of COVID-19 cases", fontweight="bold", fontsize=16)
plt.legend(fontsize=16, loc="upper right")
plt.text(-1, max_cases, "Total active cases: " + str(total_active_cases), fontsize=16)
plt.text(-1, max_cases - 2000, "Total death casualties: " + str(total_death_casualties), fontsize=16)
plt.text(-1, max_cases - 4000, "Total cured cases: " + str(total_cured_cases), fontsize=16)
plt.title("Data updated at: " + dt_string, loc="center")
plt.show()

# In[5]:

# A more visualistic comparison!

mortality_rate = str(round(total_active_cases / int(total_confirmed_cases), 2))
sizes=[total_confirmed_cases, total_active_cases]
names=["Total Confirmed Cases", "Total Active Cases"]
plt.pie(sizes, explode=(0, 0.1), labels=names, autopct='%1.1f%%', shadow=True, startangle=90)
plt.text(-1, 1.2, "Mortality Rate: " + mortality_rate, fontsize=16)
plt.show()

# In[6]:

# In case you need a more fancy donut-like graph!
from palettable.colorbrewer.qualitative import Pastel1_7
sizes=[total_confirmed_cases, total_active_cases]
names=["Total Confirmed Cases", "Total Active Cases"]
my_circle=plt.Circle((0,0), 0.7, color='white')
plt.pie(sizes, labels=names, colors=Pastel1_7.hex_colors, autopct='%1.1f%%', explode=(0, 0.1))
p=plt.gcf()
p.gca().add_artist(my_circle)
plt.text(-1, 1.2, "Mortality Rate: " + mortality_rate, fontsize=16)
plt.show()


text:
import logging

import numpy as np
from Bio import SeqIO

logger = logging.getLogger(__name__)

import os
import re
import sys
import click
import pandas as pd
import typing as t

sys.path.append("..")
from utils.rna_struct_utils import RNAStructUtils

df = pd.DataFrame({"id": [1, 2, 3, 4]})

def get_secondary_struct(
    sequence_data_path: str, workdir: str, significance_score_cutoff: float = 0.9
) -> t.Tuple[
    t.List[str],
    t.List[str],
    t.List[str],
    t.List[int],
    t.List[int],
    t.List[float],
    t.List[bool],
    t.List[float],
    t.List[float],
    t.List[float],
    t.List[float],
    t.List[str]
]:
    """
    this pipeline follows the one of RNASIV, which can be found in: https://www.mdpi.com/1999-4915/11/5/401/htm#B30-viruses-11-00401
    :param sequence_data_path: alignment data path to provide as input to the rna secondary structures prediction
    :param workdir: directory to write the pipeline output files in
    :param significance_score_cutoff: threshold between 0 and 1 determining the cutoff of secondary structure RNAz
    probability based on which the structure will be determined as significant or not
    :return: a dataframe corresponding to the secondary structures inferred for the respective cluster id
    """

    (
        struct_representation,
        struct_sequence,
        struct_src_aln_path,
        struct_start_position,
        struct_end_position,
        struct_prob,
        struct_significance,
        struct_mfe,
        struct_zscore,
        struct_entropy,
        struct_conservation_index,
        struct_pred_src
    ) = ([], [], [], [], [], [], [], [], [], [], [], [])

    if not os.path.exists(sequence_data_path):
        logger.error(
            f"no MSA is available at {sequence_data_path} and thus no secondary structures will be computed"
        )
        return (
            struct_representation,
            struct_sequence,
            struct_src_aln_path,
            struct_start_position,
            struct_end_position,
            struct_prob,
            struct_significance,
            struct_mfe,
            struct_zscore,
            struct_entropy,
            struct_conservation_index,
            struct_pred_src,
        )

    num_sequences = len(list(SeqIO.parse(sequence_data_path, format="fasta")))
    secondary_structures = []
    os.makedirs(workdir, exist_ok=True)
    if num_sequences > 1:
        logger.info(f"computing rnaz reliable windows for prediction")
        rnaz_window_output_path = f"{workdir}/rnaz_window.out"
        RNAStructUtils.exec_rnaz_window(input_path=sequence_data_path, output_path=rnaz_window_output_path)
        if os.stat(rnaz_window_output_path).st_size > 0:
            logger.info(f"executing RNAz predictor on initial windows")
            rnaz_output_path = f"{workdir}/rnaz_initial.out"
            res = RNAStructUtils.exec_rnaz(input_path=rnaz_window_output_path, output_path=rnaz_output_path)
            logger.info(f"clustering RNAz hits of overlapping windows")
            rnaz_cluster_output_path = f"{workdir}/rnaz_cluster.dat"
            res = RNAStructUtils.exec_rnaz_cluster(input_path=rnaz_output_path, output_path=rnaz_cluster_output_path)
            if res == 0:
                logger.info(f"extracting sequence data per selected window for mlocarna refinement")
                rnaz_candidates_output_dir = f"{workdir}/rnaz_candidates_sequence_data/"
                RNAStructUtils.parse_candidates(candidates_info_path=rnaz_cluster_output_path, sequence_data_path=rnaz_window_output_path, output_dir=rnaz_candidates_output_dir)
                logger.info(f"creating refined alignments of candidates with mlocarna")
                mlocarna_output_dir = f"{workdir}/rnaz_candidates_mlocarna_aligned/"
                os.makedirs(mlocarna_output_dir, exist_ok=True)
                for path in os.listdir(rnaz_candidates_output_dir):
                    input_path = f"{rnaz_candidates_output_dir}{path}"
                    output_path = f"{mlocarna_output_dir}{path.replace('.fasta', '.clustal')}"
                    res = RNAStructUtils.exec_mlocarna(input_path=input_path, output_path=output_path)
                logger.info(f"executing prediction on aligned windows with rnaz to be able to classify the selected structures")
                rnaz_refined_output_dir = f"{workdir}/rnaz_final_output/"
                os.makedirs(rnaz_refined_output_dir, exist_ok=True)
                for path in os.listdir(mlocarna_output_dir):
                    if ".clustal" in path:
                        input_path=f"{mlocarna_output_dir}{path}"
                        output_path = f"{rnaz_refined_output_dir}{path.replace('.clustal', '_rnaz.out')}"
                        res = RNAStructUtils.exec_rnaz(input_path=input_path, output_path=output_path)
                logger.info(f"parsing the obtained rna structures")
                for path in os.listdir(rnaz_refined_output_dir):
                    if ".out" in path:
                        struct = RNAStructUtils.parse_rnaz_output(rnaz_output_path=f"{rnaz_refined_output_dir}{path}", significance_score_cutoff=significance_score_cutoff)
                        secondary_structures.append(struct)
    else:
        logger.info(f"executing RNALfold on the single sequence obtained for the species")
        rnalfold_output_path = f"{workdir}/rnalfold.out"
        res = RNAStructUtils.exec_rnalfold(input_path=sequence_data_path, output_path=rnalfold_output_path)
        if res == 0:
            secondary_structures = RNAStructUtils.parse_rnalfold_result(rnalfold_path=rnalfold_output_path, sequence_data_path=sequence_data_path)

    functional_structures = [struct for struct in secondary_structures if bool(struct.is_significant) and bool(struct.is_functional_structure)]
    logger.info(f"out of {len(secondary_structures)}, {len(functional_structures)} are significant and functional")
    if len(functional_structures) > 1:
        logger.info(f"the mean z-score for the predicted structures is {np.mean([struct.mean_zscore for struct in functional_structures])} and standard deviation of {np.std([struct.mean_zscore for struct in functional_structures])}")
    for struct in secondary_structures: # here, I will save all the structures and filter out weight them by svm_rna_probability (= prb > 0.5 means it is a functional RNA, prob larger than 0.9 is more stringent and what was used in RNASIV)
        struct_representation.append(struct.consensus_representation)
        struct_sequence.append(struct.consensus_sequence)
        struct_start_position.append(struct.start_position)
        struct_end_position.append(struct.end_position)
        struct_src_aln_path.append(struct.alignment_path)
        struct_prob.append(struct.svm_rna_probability)
        struct_significance.append(struct.is_significant)
        struct_mfe.append(struct.mean_single_sequence_mfe)
        struct_zscore.append(struct.mean_zscore)
        struct_entropy.append(struct.shannon_entropy)
        struct_conservation_index.append(struct.structure_conservation_index)
        struct_pred_src.append(struct.structure_prediction_tool)

    return (
        struct_representation,
        struct_sequence,
        struct_src_aln_path,
        struct_start_position,
        struct_end_position,
        struct_prob,
        struct_significance,
        struct_mfe,
        struct_zscore,
        struct_entropy,
        struct_conservation_index,
        struct_pred_src
    )

def compute_rna_secondary_structures(
    input_df: pd.DataFrame,
    sequence_data_dir: str,
    workdir: str,
    output_path: str,
    significance_score_cutoff: float = 0.9,
):
    """
    :param input_df: dataframe with viral species of interest
    :param sequence_data_dir: directory holding sequence data of the viral species of interest
    :param workdir: directory to
    :param output_path: path of output dataframe
    :param significance_score_cutoff: significance_score_cutoff: threshold between 0 and 1 determining the cutoff of secondary structure RNAz
    probability based on which the structure will be determined as significant or not
    :return:
    """
    secondary_structures_df = pd.DataFrame(
        {"virus_species_name": input_df["virus_species_name"].unique()}
    )
    secondary_struct_fields = [
        "struct_representation",
        "struct_sequence",
        "struct_src_aln_path",
        "struct_start_pos",
        "struct_end_pos",
        "struct_prob",
        "struct_significance",
        "struct_mfe",
        "struct_zscore",
        "struct_entropy",
        "struct_conservation_index",
        "struct_prediction_tool"
    ]
    secondary_structures_df[secondary_struct_fields] = secondary_structures_df[["virus_species_name"]].apply(
        func=lambda sp_name: get_secondary_struct(
            sequence_data_path=f"{sequence_data_dir}{re.sub('[^0-9a-zA-Z]+', '_', sp_name.values[0])}_aligned.fasta",
            workdir=f"{workdir}/{re.sub('[^0-9a-zA-Z]+', '_', sp_name.values[0])}/",
            significance_score_cutoff=significance_score_cutoff),
        axis=1,
        result_type="expand")
    secondary_structures_df = secondary_structures_df.set_index(['virus_species_name']).apply(pd.Series.explode, axis=0).reset_index()
    secondary_structures_df.to_csv(output_path, index=False)

@click.command()
@click.option(
    "--associations_data_path",
    type=click.Path(exists=True, file_okay=True, readable=True),
    help="input path of associations grouped viral species and host species",
)
@click.option(
    "--sequence_data_dir",
    type=click.Path(exists=False, file_okay=True, readable=True),
    help="directory holding sequence data files per species with their collected sequences",
)
@click.option(
    "--workdir",
    type=click.Path(exists=False, file_okay=True, readable=True),
    help="directory to hold the RNA prediction pipeline files in",
    required=False,
    default=None
)
@click.option(
    "--log_path",
    type=click.Path(exists=False, file_okay=True, readable=True),
    help="path holding the logging of the script",
)
@click.option(
    "--df_output_path",
    type=click.Path(exists=False, file_okay=True, readable=True),
    help="path holding the output dataframe to write",
)
@click.option(
    "--significance_score_cutoff",
    type=click.FloatRange(min=0, max=1),
    help="significance_score_cutoff: threshold between 0 and 1 determining the cutoff of secondary structure RNAz probability based on which the structure will be determined as significant or not",
    required=False,
    default = 0.9,
)
@click.option(
    "--limit_to_species_with_multiple_sequences",
    type=bool,
    help="significance_score_cutoff: threshold between 0 and 1 determining the cutoff of secondary structure RNAz probability based on which the structure will be determined as significant or not",
    required=False,
    default = True,
)
def predict_secondary_structures(
    associations_data_path: click.Path,
    sequence_data_dir: click.Path,
    workdir: t.Optional[click.Path],
    log_path: click.Path,
    df_output_path: click.Path,
    significance_score_cutoff: float,
    limit_to_species_with_multiple_sequences: bool,
):
    # initialize the logger
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s module: %(module)s function: %(funcName)s line: %(lineno)d %(message)s",
        handlers=[
            logging.StreamHandler(sys.stdout),
            logging.FileHandler(str(log_path)),
        ],
        force=True,  # run over root logger settings to enable simultaneous writing to both stdout and file handler
    )

    if not workdir:
        workdir = f"{os.path.dirname(str(associations_data_path))}/rna_pred_aux/"
        logger.info(f"creating working directory {workdir}")
        os.makedirs(workdir, exist_ok=True)

    associations_data = pd.read_csv(associations_data_path)
    if limit_to_species_with_multiple_sequences:
        associations_data = associations_data.loc[associations_data['#sequences'] > 1]

    compute_rna_secondary_structures(
        input_df=associations_data,
        sequence_data_dir=str(sequence_data_dir),
        workdir=str(workdir),
        output_path=str(df_output_path),
        significance_score_cutoff=significance_score_cutoff
    )

if __name__ == '__main__':
    predict_secondary_structures()


text:
import unittest

import numpy
import six

from chainer import functions
from chainer import testing
from chainer.testing import attr
from chainer.utils import type_check

@testing.parameterize(*testing.product_dict(
    [
        {'shape': (2, 3, 4), 'y_shape': (2, 3, 8), 'xs_length': 2},
        {'shape': (3, 4), 'y_shape': (3, 4, 2), 'xs_length': 2},
        {'shape': (3), 'y_shape': (1, 3, 2), 'xs_length': 2},
        {'shape': (), 'y_shape': (1, 1, 2), 'xs_length': 2},
        {'shape': (2, 3, 4), 'y_shape': (2, 3, 4), 'xs_length': 1},
        {'shape': (3, 4), 'y_shape': (3, 4, 1), 'xs_length': 1},
        {'shape': (3), 'y_shape': (1, 3, 1), 'xs_length': 1},
        {'shape': (), 'y_shape': (1, 1, 1), 'xs_length': 1},
    ],
    [
        {'dtype': numpy.float16},
        {'dtype': numpy.float32},
        {'dtype': numpy.float64},
    ]
))
@testing.inject_backend_tests(
    None,
    # CPU tests
    [
        {},
    ]
    # GPU tests
    + testing.product({
        'use_cuda': [True],
        'use_cudnn': ['never', 'always'],
        'cuda_device': [0, 1],
    })
    # ChainerX tests
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0', 'cuda:1'],
    })
)
class TestDstack(testing.FunctionTestCase):

    def setUp(self):
        self.check_double_backward_options = {}
        if self.dtype == numpy.float16:
            self.check_double_backward_options = {'atol': 5e-4, 'rtol': 5e-3}

    def generate_inputs(self):
        xs = tuple([
            numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
            for i in six.moves.range(self.xs_length)
        ])
        return xs

    def forward_expected(self, inputs):
        xs = inputs
        y_expected = numpy.dstack(xs)
        return y_expected,

    def forward(self, inputs, device):
        xs = inputs
        y = functions.dstack(xs)
        return y,

@testing.parameterize(
    {'a_shape': (2, 3, 4, 6), 'b_shape': (2, 3, 4, 5), 'valid': False},
    {'a_shape': (2, 3, 5, 6), 'b_shape': (2, 3, 4, 6), 'valid': True},
    {'a_shape': (2, 4, 5), 'b_shape': (3, 4, 5), 'valid': False},
    {'a_shape': (3, 4, 6), 'b_shape': (3, 4, 5), 'valid': True},
    {'a_shape': (3, 6, 5), 'b_shape': (3, 4, 5), 'valid': False},
    {'a_shape': (3, 4), 'b_shape': (4, 4), 'valid': False},
    {'a_shape': (3, 4), 'b_shape': (3, 3), 'valid': False},
    {'a_shape': (3,), 'b_shape': (4,), 'valid': False},
    {'a_shape': (3), 'b_shape': (3, 3), 'valid': False},
    {'a_shape': (), 'b_shape': (1), 'valid': False},
)
class TestDstackTypeCheck(unittest.TestCase):

    def setUp(self):
        self.xs = [
            numpy.random.uniform(-1, 1, self.a_shape).astype(numpy.float32),
            numpy.random.uniform(-1, 1, self.b_shape).astype(numpy.float32),
        ]

    def check_value_check(self):
        if self.valid:
            # Check if it throws nothing
            functions.dstack(self.xs)
        else:
            with self.assertRaises(type_check.InvalidType):
                functions.dstack(self.xs)

    def test_value_check_cpu(self):
        self.check_value_check()

    @attr.gpu
    def test_value_check_gpu(self):
        self.check_value_check()

testing.run_module(__name__, __file__)

max_stars_repo_name: zaltoprofen/chainer

max_issues_repo_name: zaltoprofen/chainer

max_forks_repo_name: zaltoprofen/chainer


text:
import cv2
import numpy as np

def sharp(image):
    kernel = np.array([[0, -1, 0], [-1, 5, -1], [0, -1, 0]], np.floqat32)
    dst = cv2.filter2D(image, -1, kernel=kernel)
    return (dst)

vid = cv2.VideoCapture(0)
while (True):
    ret, frame = vid.read()
    sharpen = sharp(frame)
    numpy_horizontal = np.hstack((frame, sharpen))
    cv2.imshow('Numpy Horizontal', numpy_horizontal)

    # the 'q' button is set as the
    # quitting button you may use any
    # desired button of your choice
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# After the loop release the cap object
vid.release()
# Destroy all the windows
cv2.destroyAllWindows()


text:
try:
    pass

except:
    raise Exception('Impossible to import spline library. You need to compile it with cython first')

from splines_cython import USpline, eval_UBspline

import numpy
import numpy as np

def mlinspace(smin,smax,orders):
    if len(orders) == 1:
        res = np.atleast_2d( np.linspace(np.array(smin),np.array(smax),np.array(orders)) )
        return res.copy() ## workaround for strange bug
    else:
        meshes = np.meshgrid( *[numpy.linspace(smin[i],smax[i],orders[i]) for i in range(len(orders))], indexing='ij' )
        table = np.row_stack( [l.flatten() for l in meshes])
        return np.ascontiguousarray(table)
#class MultivariateSplines(MultivariateSplinesCython):
#
#    def __init__(self, smin,smax,orders):

#        MultivariateSplinesCython.__init__(self,smin,smax,orders)

#        from dolo.numeric.misc import cartesian

#        grid = cartesian( [numpy.linspace(self.smin[i], self.smax[i], self.orders[i]) for i in range(self.d)] ).T
#        self.grid = numpy.ascontiguousarray(grid)
#        print(grid.shape)

class MultivariateSplines:

    __grid__ = None

    def __init__(self, smin, smax, orders, dtype=np.float64):

        self.d = len(smin)
        assert(len(smax) == self.d)
        assert(len(orders) == self.d)
        self.smin = np.array(smin, dtype=dtype)
        self.smax = np.array(smax, dtype=dtype)
        self.orders = np.array(orders, dtype=np.int)
        self.dtype= dtype
        self.__splines__ = None

    def set_values(self, values):

        if not np.all( np.isfinite(values)):
            raise Exception('Trying to interpolate non-finite values')

        values = np.ascontiguousarray(values, dtype=self.dtype) # we don't need that since USpline already checks for contiguity

        n_v = values.shape[0]
        self.__splines__ = []
        for i in range(n_v):
            sp = USpline(self.smin,self.smax,self.orders,values[i,:].reshape(self.orders) , dtype=self.dtype )
            sp.coefs = np.ascontiguousarray( sp.coefs, dtype=self.dtype)
            self.__splines__.append( sp )

    def interpolate(self, points, with_derivatives=False):

        import time

        points = np.ascontiguousarray(points, dtype=self.dtype)

        if points.ndim == 1:
            raise Exception('Expected 2d array. Received {}d array'.format(points.ndim))
        if points.shape[0] != self.d:
            raise Exception('First dimension should be {}. Received : {}.'.format(self.d, points.shape[0]))
        if not np.all( np.isfinite(points)):
            raise Exception('Spline interpolator evaluated at non-finite points.')

        n_v = len(self.__splines__)
        N = points.shape[1]
        n_s = points.shape[0]

        if not with_derivatives:
            values = np.empty((n_v,N), dtype=self.dtype)
            for i in range(n_v):
                sp =  self.__splines__[i]
                values[i,:] = eval_UBspline(self.smin, self.smax, self.orders, sp.coefs, points, diff=False )
            return values
        else:
            values = np.empty((n_v,N), dtype=self.dtype)
            dvalues = np.empty((n_v,n_s,N), dtype=self.dtype)
            for i in range(n_v):
                sp =  self.__splines__[i]

                [value, d_value] =  eval_UBspline(self.smin, self.smax, self.orders, sp.coefs, points, diff=True )
                values[i,:] = value
                dvalues[i,:,:] = d_value

            return [values,dvalues]

    @property
    def grid(self):
        if self.__grid__ is None:
            self.__grid__ = mlinspace(self.smin, self.smax, self.orders)
        return self.__grid__

    def __call__(self, s):
        return self.interpolate(s)

max_stars_repo_name: christophe-gouel/dolo

max_issues_repo_name: christophe-gouel/dolo

max_forks_repo_name: christophe-gouel/dolo


text:
# Copyright (c) 2019, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: MIT
# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/MIT

import time
import torch
import logging
import numpy as np

def create_worker_logger(worker_id, exp_dir, group_id=None):
    # create logger
    logger = logging.getLogger('{:02d}'.format(worker_id))
    logger.setLevel(logging.DEBUG)

    # # create file formatter
    # log_name = '{}.log'.format(worker_id) if group_id is None else '{}.{}.log'.format(worker_id, group_id)
    # fh = logging.FileHandler(filename=os.path.join(exp_dir, log_name))
    # fh.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(message)s'))
    # fh.setLevel(logging.DEBUG)
    # logger.addHandler(fh)

    # create console handler
    ch = logging.StreamHandler()
    ch.setFormatter(logging.Formatter('%(name)s - %(message)s'))
    ch.setLevel(logging.INFO)
    logger.addHandler(ch)

    return logger

class ReplayBuffer:
    def __init__(self, model, config):
        self.model = model

        self.capacity = config['buffer_capacity']
        self.batch_size = config['batch_size']
        self.min_size = max(config['min_buffer_size'], config['batch_size'])

        self.ep_buffer = [{}] * self.capacity
        self.opt_count = [0] * self.capacity

        self._pointer = 0
        self._looped = False

        self.profiler = {
            'st': time.time(),
            'last_idle': time.time(),
            'time_idle': 0,
            'time_sync': 0,
            'time_batch': 0,
            'size': [],
            'opts': [],
        }

    def reset_profiler(self):
        self.profiler = {
            'st': time.time(),
            'last_idle': time.time(),
            'time_idle': 0,
            'time_sync': 0,
            'time_batch': 0,
            'size': [],
            'opts': [],
        }

    @property
    def size(self):
        return int(self.capacity) if self._looped else int(self._pointer)

    def add_episode(self, transition_dicts):
        """Integrate new episodes from the workers into the buffers"""
        self.profiler['time_idle'] += time.time() - self.profiler['last_idle']
        st = time.time()
        self.ep_buffer[self._pointer] = {}
        self.opt_count[self._pointer] = 0
        for t in transition_dicts:
            self.ep_buffer[self._pointer] = {k: v.detach() for k, v in t.items()}
            self._pointer += 1
            if self._pointer >= self.capacity:
                self._looped = True
                self._pointer = 0

        self.profiler['size'].append(self.size)
        self.profiler['time_sync'] += time.time() - st
        self.profiler['last_idle'] = time.time()

    def make_batch(self, normalize=True):
        """Convert the buffer into inputs for DataParallel training"""
        self.profiler['time_idle'] += time.time() - self.profiler['last_idle']
        st = time.time()
        keys = self.ep_buffer[0].keys()
        batch = dict()
        ep_idxs = np.random.permutation(np.arange(self.size))[:self.batch_size]
        for k in keys:
            batch[k] = torch.stack([self.ep_buffer[i][k] for i in ep_idxs]).detach()
        for i in ep_idxs:
            self.opt_count[i] += 1

        if normalize:
            # Apply normalization to the batch
            batch = self.model.normalize_batch(batch)

        # Profile time statistics
        self.profiler['time_batch'] += time.time() - st
        self.profiler['last_idle'] = time.time()

        return batch

    def profile(self, time_window=900.):
        time_window = float(time_window)
        dur = time.time() - self.profiler['st']
        if dur < time_window:
            return

        print('\nTrainer time expenditure:\n  Syncing: {:5.2f}%,  Batching: {:5.2f}%  Training: {:5.2f}%'.format(
            100*self.profiler['time_sync']/dur,
            100*self.profiler['time_batch']/dur,
            100*self.profiler['time_idle']/dur,
        ), flush=True)
        # print('Average buffer size: {:7.1f}'.format(np.mean(self.profiler['size'])), flush=True)
        # print('Average episode use: {:7.1f}'.format(np.mean(self.profiler['opts'])), flush=True)
        # print(' ', flush=True)
        print('Current buffer size: {:7.1f}\n'.format(self.size), flush=True)

        self.reset_profiler()

max_stars_repo_name: salesforce/sibling-rivalry

max_issues_repo_name: salesforce/sibling-rivalry

max_forks_repo_name: salesforce/sibling-rivalry


text:
import cv2
import numpy as np

if __name__ == '__main__':
    image = cv2.imread('input/i3.jpg')

    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    thresh, bw = cv2.threshold(gray, 50, 255, cv2.THRESH_BINARY)
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
    bw = cv2.erode(bw, kernel)
    bw = cv2.dilate(bw, kernel)

    bw, contours, hierarchy = cv2.findContours(bw, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    moments = [cv2.moments(c) for c in contours]
    centroids = [(m['m10'] / m['m00'], m['m01'] / m['m00']) for m in moments]

    out_size = 500
    output = np.zeros((out_size, out_size, 3))
    output_points = np.float32([[0, out_size], [0, 0], [out_size, 0]])

    # Pixel half size, manually estimated from input photo
    pixel_half = 8
    delta = 0.7

    for x, y in centroids:
        source_points = np.float32([[x - pixel_half + delta, y + pixel_half],
                                    [x - pixel_half + delta, y - pixel_half],
                                    [x + pixel_half + delta * 2, y - pixel_half]])
        transform = cv2.getAffineTransform(source_points, output_points)
        output += cv2.warpAffine(image, transform, (out_size, out_size))

    output /= len(centroids)
    output = cv2.normalize(output, None, 0, 255, cv2.NORM_MINMAX)

    cv2.imwrite('output/o3.png', output)

max_stars_repo_name: radu-b/pixel

max_issues_repo_name: radu-b/pixel

max_forks_repo_name: radu-b/pixel


text:
from numpy import zeros, where, array, unique, concatenate

#from pyNastran.dev.bdf_vectorized.cards.materials.mat1 import MAT1
#from pyNastran.dev.bdf_vectorized.cards.materials.mats1 import MATS1
#from pyNastran.dev.bdf_vectorized.cards.materials.mat4 import MAT4
#from pyNastran.dev.bdf_vectorized.cards.materials.mat10 import MAT10

from pyNastran.utils.numpy_utils import integer_types
from pyNastran.dev.bdf_vectorized.utils import slice_to_iter

#from pyNastran.bdf.cards.materials import (MAT1, MAT2, MAT4, MAT5, MAT8,
    #MAT10, MAT11) #, MATS1)

class Materials:
    def __init__(self, model):
        self.model = model
        self.n = 0
        self.is_built = False

        self.mat1 = model.mat1
        self.mats1 = model.mats1
        #self.mat2 = model.mat2
        #self.mat4 = model.mat4
        #self.mat5 = model.mat5
        self.mat8 = model.mat8
        #self.mat10 = model.mat10
        #self.mat11 = model.mat11
        self.mathp = model.mathp

    #def add_mat1(self, card, comment):
        #self.model.log.debug('adding mat1')
        #self.mat1.add(card=card, comment=comment)

    #def add_mats1(self, card, comment):
        #self.mats1.add(card, comment)

    #def add_mat2(self, card, comment):
        #self.mat2.add(card, comment)

    #def add_mat4(self, card, comment):
        #self.mat4.add(card, comment)

    #def add_mat5(self, card, comment):
        #self.mat5.add(card, comment)

    #def add_mat8(self, card, comment):
        #self.mat8.add(card, comment)

    #def add_mat10(self, card, comment):
        #self.mat10.add(card, comment)

    #def add_mat11(self, card, comment):
        #self.mat11.add(card, comment)

    #def add_mathp(self, card, comment):
        #self.mathp.add(card, comment)

    def allocate(self, card_count):
        self.model.log.info('allocate Materials')
        n = 0
        types = self._get_types()
        names = self._get_type_names()
        for name, mat in zip(names, types):
            if name in card_count:
                self.model.log.info('    allocate %s->%s' % (name, card_count[name]))
                n = card_count[name]
                mat.allocate(n)
                self.n += n
        self.n = n
        #aaa

    def build(self):
        self.is_built = True
        n = 0
        types = self._get_types()
        names = self._get_type_names()
        self.model.log.debug('building Materials')
        for name, mat in zip(names, types):
            ni = len(mat)
            n += ni
            mat.build()
            self.model.log.debug('    building %s; n=%s' % (mat.__class__.__name__, ni))
        self.n = n

    def get_stats(self):
        msg = []
        types = self._get_types()
        names = self._get_type_names()
        for mat, name in zip(types, names):
            msg.append('  %-8s: %i' % (name, len(mat)))
            #nmat = mat.n
            #if nmat:
                #msg.append('  %-8s: %i' % (mat.type, nmat))
        return msg

    def get_material_id_by_property_id(self, property_id):
        n = len(property_id)
        types = self._get_types(nlimit=True)
        _material_id = concatenate([ptype.material_id for ptype in types])
        self.model.log.debug(_material_id)
        assert _material_id.shape == (n, ), _material_id.shape
        return _material_id

    #def get_density_by_material_id(self, material_id):
        #rho = zeros(len(material_id), dtype='float64')
        #for i, mid in enumerate(material_id):
            #mat = self.get_structural_material(mid)
            #rho[i] = mat.rho
        #return rho

    def get_density_by_material_id(self, material_id):
        int_flag = True if isinstance(material_id, integer_types) else False
        if isinstance(material_id, integer_types):
            material_id = array([material_id], dtype='int32')

        n = len(material_id)
        self.model.log.debug('material_id=%s' % material_id)
        umids = unique(material_id)
        umids_found = zeros(len(umids), dtype='int32')

        density = zeros(n, dtype='float64')
        mat_types = [
            self.mat1,
            #self.mat2,
            self.mat8,
            self.mathp,
        ]
        for mat_type in mat_types:
            for iumid, mid in enumerate(umids):
                if mid in mat_type.material_id:
                    rho = mat_type.get_density_by_material_id([mid])
                    i = where(mid == material_id)[0]
                    density[i] = rho
                    umids_found[iumid] = 1
                #else:
                    #print('mid=%s not found in Type=%s materials=%s' % (mid, mat_type.type, mat_type.material_id))
        assert density.shape == (n, ), density.shape

        if umids_found.min() == 0:
            izero = where(umids_found == 0)[0]
            msg = 'mids=%s not found' % umids[izero]
            raise RuntimeError(msg)
        return density[0] if int_flag else density

    def get_density_E_by_material_id(self, material_id):
        assert self.is_built, self.is_built
        int_flag = True if isinstance(material_id, integer_types) else False
        n = len(material_id)
        density = zeros(n, dtype='float64')
        E = zeros(n, dtype='float64')

        for i, mid in enumerate(material_id):
            mat = self.get_structural_material(mid)
            density[i] = mat.rho
            E[i] = mat.E()
        assert density.shape == (n, ), density.shape
        assert E.shape == (n, ), E.shape
        return density[0], E[0] if int_flag else density, E

    #def _get_param_by_material_id(self, material_id, name):
        #n = len(material_id)
        #nsm = zeros(n)
        #for i, mid in enumerate(material_id):
            #mat = self.get_structural_material(mid)
            #nsm[i] = getattr(mat, name)
        #assert nsm.shape == (n, ), nsm.shape
        #return nsm

    def get_nonstructural_mass_by_material_id(self, material_id):
        #return _get_param_by_material_id(material_id, 'nsm')
        n = len(material_id)
        nsm = zeros(n)
        for i, mid in enumerate(material_id):
            mat = self.get_structural_material(mid)
            nsm[i] = mat.nsm
        assert nsm.shape == (n, ), nsm.shape
        return nsm

    def get_E_by_material_id(self, material_id):
        #return _get_param_by_material_id(material_id, 'E')
        n = len(material_id)
        E = zeros(n)
        for i, mid in enumerate(material_id):
            mat = self.get_shell_material(mid)
            E[i] = mat.get_E_by_material_id(mid)
        assert E.shape == (n, ), E.shape
        return E

    def get_G_by_material_id(self, material_id):
        n = len(material_id)
        G = zeros(n)
        for i, mid in enumerate(material_id):
            mat = self.get_shell_material(mid)
            G[i] = mat.get_G_by_material_id(mid)
        assert G.shape == (n, ), G.shape
        return G

    #def get_material_index_by_material_id(self, property_id):
        #if property_id is None:
            #return arange(self.n)
        #return searchsorted(property_id, self.property_id)

    def get_structural_material(self, material_id):
        if material_id in self.mat1:
            return self.mat1.slice_by_material_id(material_id)
        elif material_id in self.mat8:
            return self.mat2.slice_by_material_id(material_id)
        raise RuntimeError('Could not find material_id=%r' % material_id)

    def get_shell_material(self, material_id):
        #if material_id in self.mats1.material_id:
            #return self.mats1.slice_by_material_id(material_id)
        if material_id in self.mat1.material_id:
            return self.mat1.slice_by_material_id(material_id)
        #elif material_id in self.mat2.material_id:
            #return self.mat2.slice_by_material_id(material_id)
        elif material_id in self.mat8.material_id:
            return self.mat8.slice_by_material_id(material_id)
        raise RuntimeError('Could not find material_id=%r' % material_id)

    def get_solid_material(self, material_id):
        #if material_id in self.mats1:
            #return self.mats1.slice_by_material_id(material_id)
        #if material_id in self.mat11:
            #return self.mat11.slice_by_material_id(material_id)
        if material_id in self.mat1.material_id:
            return self.mat1.slice_by_material_id(material_id)
        elif material_id in self.mat9.material_id:
            return self.mat9.slice_by_material_id(material_id)
        elif material_id in self.mat10.material_id:
            return self.mat10.slice_by_material_id(material_id)
        raise RuntimeError('Could not find material_id=%r' % material_id)

    def get_axial_material(self, material_id):
        if material_id in self.mat3.material_id:
            return self.mat3.slice_by_material_id(material_id)
        raise RuntimeError('Could not find material_id=%r' % material_id)

    def get_thermal_material(self, material_id):
        if material_id in self.mat4.material_id:
            return self.mat4.slice_by_material_id(material_id)
        elif material_id in self.mat5.material_id:
            return self.mat5.slice_by_material_id(material_id)
        raise RuntimeError('Could not find material_id=%r' % material_id)

    def __getitem__(self, material_id):
        TypeMap = {
            'MAT1'  : (self.mat1, self.mat1.material_id),
            #'MAT2'  : (self.mat2, self.mat2.material_id),
            #'MAT4'  : (self.mat4, self.mat4.material_id),
            #'MAT5'  : (self.mat5, self.mat5.material_id),
            'MAT8'  : (self.mat8, self.mat8.material_id),
            #'MAT10' : (self.mat10, self.mat10.material_id),

            #'MATS1' : (self.mats1, self.mats1.material_id),
        }
        material_id, int_flag = slice_to_iter(material_id)
        out = []
        self.model.log.debug('material_ids = %s' % material_id)
        for mid in material_id:
            obj = None
            for Type, (mdict, mids) in TypeMap.items():
                if mdict.n:
                    self.model.log.debug('Type=%s mid=%s mids=%s' % (Type, mid, mids))
                    if mid in mids:
                        self.model.log.debug(' *mid=%i Type=%s was found' % (mid, Type))
                        i = where(mid == mids)[0]
                        obj = mdict.slice_by_material_id([mid])
                        break
                    else:
                        self.model.log.debug('  mid=%i Type=%s was not found' % (mid, Type))
            out.append(obj)
        return out[0] if int_flag else out

    #def __getitem2__(self, material_id):
        #assert isinstance(material_id, integer_types), 'material_id=%r' % material_id
        #types = self._get_types()
        #for mat in types:
            #if material_id in mat.material_id:
                #return mat
        #raise RuntimeError('Could not find material_id=%r' % material_id)

    def __len__(self):
        types = self._get_types()
        n = 0
        for mat in types:
            #n += mat.n
            n += len(mat)
        return n

    def __iter__(self):
        types = self._get_types()
        for materials in types:
            if materials.n:
                for mid, mat in materials.items():
                    yield mat

    def _get_types(self):
        return [self.mat1,
                #self.mat2, self.mat4,
                self.mat8,
                #self.mat10,
                self.mats1,
                self.mathp,
                ]

    def _get_type_names(self):
        return ['MAT1',
                #'MAT2', 'MAT4',
                'MAT8',
                #'MAT10',
                'MATS1',
                'MATHP',
                ]

    def _verify(self, xref=True):
        for mid, material in sorted(self.mat1.items()):
            material._verify(xref)

    def write_card(self, bdf_file, size=8, is_double=False, material_id=None):
        self.model.log.debug('Materials.write_card.n=%s' % self.n)
        if self.n:
            bdf_file.write('$MATERIALS\n')
            types = [
                self.mat1,
                #self.mat2, self.mat4,
                self.mat8,
                #self.mat10,
                self.mats1,
                self.mathp,
            ]
            for materials in types:
                if materials.n:
                    self.model.log.debug('    writing %s' % materials.type)
                    materials.write_card(bdf_file, size=size, material_id=material_id)

reason: from numpy


text:
__author__ = "\n".join(['Aric Hagberg (hagberg@lanl.gov)',
						'Pieter Swart (swart@lanl.gov)',
						'Dan Schult(dschult@colgate.edu)'])
#    Copyright (C) 2004-2009 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
#
__docformat__ = "restructuredtext en"

from networkx.exception import NetworkXException, NetworkXError
from copy import deepcopy

class Graph(object):
    def __init__(self, data=None, name='', **attr):
        self.graph = {}   # dictionary for graph attributes
        self.node = {}    # empty node dict (created before convert)
        self.adj = {}     # empty adjacency dict
        # load graph attributes (must be after convert)
        self.graph.update(attr)
        self.name = name
        self.edge = self.adj

    def __str__(self):
        return self.name

    def __iter__(self):
        return self.adj.iterkeys()

    def __contains__(self,n):
        try:
            return n in self.adj
        except TypeError:
            return False

    def __len__(self):
        return len(self.adj)

    def __getitem__(self, n):
        return self.adj[n]

    def add_node(self, n, attr_dict=None, **attr):
        # set up attribute dict
        if attr_dict is None:
            attr_dict=attr
        else:
            try:
                attr_dict.update(attr)
            except AttributeError:
                raise NetworkXError(\
                    "The attr_dict argument must be a dictionary.")
        if n not in self.adj:
            self.adj[n] = {}
            self.node[n] = attr_dict
        else: # update attr even if node already exists
            self.node[n].update(attr_dict)

    def add_nodes_from(self, nodes, **attr):
        for n in nodes:
            if n not in self.adj:
                self.adj[n] = {}
                self.node[n] = attr.copy()
            else:
                self.node[n].update(attr)

    def remove_node(self,n):

        adj = self.adj
        try:
            nbrs = adj[n].keys() # keys handles self-loops (allow mutation later)
            del self.node[n]
        except KeyError: # NetworkXError if n not in self
            raise NetworkXError("The node %s is not in the graph."%(n,))
        for u in nbrs:
            del adj[u][n]   # remove all edges n-u in graph
        del adj[n]          # now remove node

    def remove_nodes_from(self, nodes):

        adj = self.adj
        for n in nodes:
            try:
                del self.node[n]
                for u in adj[n].keys():   # keys() handles self-loops
                    del adj[u][n]         #(allows mutation of dict in loop)
                del adj[n]
            except KeyError:
                pass

    def nodes_iter(self, data=False):

        if data:
            return self.node.iteritems()
        return self.adj.iterkeys()

    def nodes(self, data=False):

        return list(self.nodes_iter(data=data))

    def number_of_nodes(self):

        return len(self.adj)

    def order(self):

        return len(self.adj)

    def has_node(self, n):

        try:
            return n in self.adj
        except TypeError:
            return False

    def add_edge(self, u, v, attr_dict=None, **attr):

        # set up attribute dictionary
        if attr_dict is None:
            attr_dict=attr
        else:
            try:
                attr_dict.update(attr)
            except AttributeError:
                raise NetworkXError(\
                    "The attr_dict argument must be a dictionary.")
        # add nodes
        if u not in self.adj:
            self.adj[u] = {}
            self.node[u] = {}
        if v not in self.adj:
            self.adj[v] = {}
            self.node[v] = {}
        # add the edge
        datadict=self.adj[u].get(v,{})
        datadict.update(attr_dict)
        self.adj[u][v] = datadict
        self.adj[v][u] = datadict

    def add_edges_from(self, ebunch, attr_dict=None, **attr):

        # set up attribute dict
        if attr_dict is None:
            attr_dict=attr
        else:
            try:
                attr_dict.update(attr)
            except AttributeError:
                raise NetworkXError(\
                    "The attr_dict argument must be a dictionary.")
        # process ebunch
        for e in ebunch:
            ne=len(e)
            if ne==3:
                u,v,dd = e
            elif ne==2:
                u,v = e
                dd = {}
            else:
                raise NetworkXError(\
                    "Edge tuple %s must be a 2-tuple or 3-tuple."%(e,))
            if u not in self.adj:
                self.adj[u] = {}
                self.node[u] = {}
            if v not in self.adj:
                self.adj[v] = {}
                self.node[v] = {}
            datadict=self.adj[u].get(v,{})
            datadict.update(attr_dict)
            datadict.update(dd)
            self.adj[u][v] = datadict
            self.adj[v][u] = datadict

    def add_weighted_edges_from(self, ebunch, **attr):

        self.add_edges_from(((u,v,{'weight':d}) for u,v,d in ebunch),**attr)

    def remove_edge(self, u, v):

        try:
            del self.adj[u][v]
            if u != v:  # self-loop needs only one entry removed
                del self.adj[v][u]
        except KeyError:
            raise NetworkXError("The edge %s-%s is not in the graph"%(u,v))

    def remove_edges_from(self, ebunch):

        for e in ebunch:
            u,v = e[:2]  # ignore edge data if present
            if u in self.adj and v in self.adj[u]:
                del self.adj[u][v]
                if u != v:  # self loop needs only one entry removed
                    del self.adj[v][u]

    def has_edge(self, u, v):

        try:
            return v in self.adj[u]
        except KeyError:
            return False

    def neighbors(self, n):

        try:
            return self.adj[n].keys()
        except KeyError:
            raise NetworkXError("The node %s is not in the graph."%(n,))

    def neighbors_iter(self, n):

        try:
            return self.adj[n].iterkeys()
        except KeyError:
            raise NetworkXError("The node %s is not in the graph."%(n,))

    def edges(self, nbunch=None, data=False):

        return list(self.edges_iter(nbunch, data))

    def edges_iter(self, nbunch=None, data=False):

        seen={}     # helper dict to keep track of multiply stored edges
        if nbunch is None:
            nodes_nbrs = self.adj.iteritems()
        else:
            nodes_nbrs=((n,self.adj[n]) for n in self.nbunch_iter(nbunch))
        if data:
            for n,nbrs in nodes_nbrs:
                for nbr,data in nbrs.iteritems():
                    if nbr not in seen:
                        yield (n,nbr,data)
                seen[n]=1
        else:
            for n,nbrs in nodes_nbrs:
                for nbr in nbrs:
                    if nbr not in seen:
                        yield (n,nbr)
                seen[n] = 1
        del seen

    def get_edge_data(self, u, v, default=None):

        try:
            return self.adj[u][v]
        except KeyError:
            return default

    def adjacency_list(self):

        return map(list,self.adj.itervalues())

    def adjacency_iter(self):

        return self.adj.iteritems()

    def degree(self, nbunch=None, with_labels=False, weighted=False):

        if with_labels:           # return a dict
            return dict(self.degree_iter(nbunch,weighted=weighted))
        elif nbunch in self:      # return a single node
            return self.degree_iter(nbunch,weighted=weighted).next()[1]
        else:                     # return a list
            return [d for (n,d) in self.degree_iter(nbunch,weighted=weighted)]

    def degree_iter(self, nbunch=None, weighted=False):

        if nbunch is None:
            nodes_nbrs = self.adj.iteritems()
        else:
            nodes_nbrs=((n,self.adj[n]) for n in self.nbunch_iter(nbunch))

        if weighted:
        # edge weighted graph - degree is sum of nbr edge weights
            for n,nbrs in nodes_nbrs:
                yield (n, sum((nbrs[nbr].get('weight',1) for nbr in nbrs)) +
                              (n in nbrs and nbrs[n].get('weight',1)))
        else:
            for n,nbrs in nodes_nbrs:
                yield (n,len(nbrs)+(n in nbrs)) # return tuple (n,degree)

    def clear(self):

        self.name = ''
        self.adj.clear()
        self.node.clear()
        self.graph.clear()

    def copy(self):

        return deepcopy(self)

    def is_multigraph(self):

        return False

    def is_directed(self):

        return False

    def to_directed(self):

        from networkx import DiGraph
        G=DiGraph()
        G.name=self.name
        G.add_nodes_from(self)
        G.add_edges_from( ((u,v,deepcopy(data))
                           for u,nbrs in self.adjacency_iter()
                           for v,data in nbrs.iteritems()) )
        G.graph=deepcopy(self.graph)
        G.node=deepcopy(self.node)
        return G

    def to_undirected(self):

        return deepcopy(self)

    def subgraph(self, nbunch):

        bunch =self.nbunch_iter(nbunch)
        # create new graph and copy subgraph into it
        H = self.__class__()
        H.name = "Subgraph of (%s)"%(self.name)
        # namespace shortcuts for speed
        H_adj=H.adj
        self_adj=self.adj
        # add nodes and edges (undirected method)
        for n in bunch:
            Hnbrs={}
            H_adj[n]=Hnbrs
            for nbr,d in self_adj[n].iteritems():
                if nbr in H_adj:
                    # add both representations of edge: n-nbr and nbr-n
                    Hnbrs[nbr]=d
                    H_adj[nbr][n]=d
        # copy node and attribute dictionaries
        H.node=self.node.copy()
        H.graph=self.graph.copy()
        return H

    def nodes_with_selfloops(self):

        return [ n for n,nbrs in self.adj.iteritems() if n in nbrs ]

    def selfloop_edges(self, data=False):

        if data:
            return [ (n,n,nbrs[n])
                     for n,nbrs in self.adj.iteritems() if n in nbrs ]
        else:
            return [ (n,n)
                     for n,nbrs in self.adj.iteritems() if n in nbrs ]

    def number_of_selfloops(self):

        return len(self.selfloop_edges())

    def size(self, weighted=False):

        return sum(self.degree(weighted=weighted))/2

    def number_of_edges(self, u=None, v=None):

        if u is None: return self.size()
        if v in self.adj[u]:
            return 1
        else:
            return 0

    def add_star(self, nlist, **attr):

        v=nlist[0]
        edges=((v,n) for n in nlist[1:])
        self.add_edges_from(edges, **attr)

    def add_path(self, nlist, **attr):

        edges=zip(nlist[:-1],nlist[1:])
        self.add_edges_from(edges, **attr)

    def add_cycle(self, nlist, **attr):

        edges=zip(nlist,nlist[1:]+[nlist[0]])
        self.add_edges_from(edges, **attr)

    def nbunch_iter(self, nbunch=None):

        if nbunch is None:   # include all nodes via iterator
            bunch=self.adj.iterkeys()
        elif nbunch in self: # if nbunch is a single node
            bunch=[nbunch].__iter__()
        else:                # if nbunch is a sequence of nodes
            def bunch_iter(nlist,adj):
                try:
                    for n in nlist:
                        if n in adj:
                            yield n
                except TypeError,e:
                    message=e.args[0]
                    print message
                    # capture error for non-sequence/iterator nbunch.
                    if 'iterable' in message:
                        raise NetworkXError(
                            "nbunch is not a node or a sequence of nodes.")
                    # capture error for unhashable node.
                    elif 'hashable' in message:
                        raise NetworkXError(
                            "Node %s in the sequence nbunch is not a valid node."%n)
                    else: raise
            bunch=bunch_iter(nbunch,self.adj)
        return bunch

max_stars_repo_name: mrv-developers/mrv-networkx

max_issues_repo_name: mrv-developers/mrv-networkx

max_forks_repo_name: mrv-developers/mrv-networkx

reason: from networkx


text:
from nltk.cluster.util import cosine_distance
import numpy as np
import nltk.corpus
from nltk.stem import WordNetLemmatizer
from nltk import word_tokenize
from nltk.corpus import stopwords
from nltk import ne_chunk
from nltk import RegexpParser

# ------------------------------------------
#          TEXT BEAN HELPER METHODS
# ------------------------------------------

def __sentence_similarity(sent1, sent2, stopwords=None):
    if stopwords is None:
        stopwords = []

    sent1 = [w.lower() for w in sent1]
    sent2 = [w.lower() for w in sent2]

    all_words = list(set(sent1 + sent2))

    vector1 = [0] * len(all_words)
    vector2 = [0] * len(all_words)

    for w in sent1:
        if w in stopwords:
            continue
        vector1[all_words.index(w)] += 1

    for w in sent2:
        if w in stopwords:
            continue
        vector2[all_words.index(w)] += 1

    return 1 - cosine_distance(vector1, vector2)

def build_similarity_matrix(sentences, stop_words):
    similarity_matrix = np.zeros((len(sentences), len(sentences)))

    for idx1 in range(len(sentences)):
        for idx2 in range(len(sentences)):
            if idx1 == idx2:
                continue
            similarity_matrix[idx1][idx2] = __sentence_similarity(sentences[idx1], sentences[idx2], stop_words)

    return similarity_matrix

def read_text(text):
    article = text.split(". ")
    sentences = []

    for sentence in article:
        sentences.append(sentence.replace("[^a-zA-Z]", " ").split(" "))
    sentences.pop()

    return sentences

def post_text_process(text):
    """
      Performs a post process with the text, preparing it for text deep analysis
      or other purposes.
    """
    # Splits words to tokens
    tokenized_word = word_tokenize(text.lower())

    # Normalize words to normal form
    # aka: playing -> play
    lemmatizer = WordNetLemmatizer()
    lemmatized_words = []
    for word in tokenized_word:
        lemmatized_words.append(lemmatizer.lemmatize(word))

    # Remove stopwords from the set
    stopwords_set = set(stopwords.words('english'))
    stopwords_set.add('.')
    stopwords_text = [word for word in lemmatized_words if word not in stopwords_set]

    # Named entity recognition
    tags = nltk.pos_tag(stopwords_text)
    chunk = ne_chunk(tags)

    chunking_rule = "NP: {<DT>?<JJ>*<NN>}"
    chunking_text_parsed = RegexpParser(chunking_rule)
    chunking_result = chunking_text_parsed.parse(tags)

    # Returns important information created by the post processing
    return chunking_result, chunk

def syllable_count(word):
    word = word.lower()
    count = 0
    vowels = "aeiouy"
    if word[0] in vowels:
        count += 1
    for index in range(1, len(word)):
        if word[index] in vowels and word[index - 1] not in vowels:
            count += 1
    if word.endswith("e"):
        count -= 1
    if count == 0:
        count += 1
    return count

def levenshtein_distance(sent1, sent2):
    sent1 = [w.lower() for w in sent1]
    sent2 = [w.lower() for w in sent2]

    all_words = list(set(sent1 + sent2))
    vector1 = [0] * len(all_words)
    vector2 = [0] * len(all_words)

    for w in sent1:
        vector1[all_words.index(w)] += 1

    for w in sent2:
        vector2[all_words.index(w)] += 1

    return 1 - cosine_distance(vector1, vector2)

max_stars_repo_name: vicotrbb/Pylexitext

max_issues_repo_name: vicotrbb/Pylexitext

max_forks_repo_name: vicotrbb/Pylexitext


text:
import random

import numpy as np
import astropy.units as u
from shapely import ops
import shapely.geometry as geom
import shapely.prepared as prepgeom

__all__ = ["Field"]

class Field:
    '''Representation of an observational photometric field

    Constructed based on imaged field coordinate boundaries, in RA and DEC,
    this class exists to facilitate the analysis of all mass function data.

    Polygons representing the observed fields are constructed, corrected,
    merged and prepped in order to best handle the extraction of radial slices
    and the monte carlo integration of any function within it's bounds.

    Parameters
    ----------
    coords :  numpy.ndarray or shapely.geometry.Polygon
        Input polygon descriptor, either in the form of a already created
        polygon (or multi-polygon) or as an array of ordered coordinates

    cen : 2-tuple of float, optional
        The central coordinate of the relevant globular cluster, in RA and
        DEC. If `cen` does not have any associated units, one must be supplied
        to the `unit` parameter. This is only used for initialization, all
        coordinates will be centred around (0, 0). Defaults to assuming this
        centring has already happened

    unit : str, optional
        Unit associated with all coordinates. Must be provided if `cen` has
        no units. Note that all coordinates will be converted to arcmin during
        processing, this parameter only defines the input values.

    preprep : bool, optional
        Whether to run `Field.prep` upon initialization. Defaults to False

    Attributes
    ----------
    polygon : shapely.geometry.Polygon
        The `shapely` `Polygon` or `MultiPolygon` object representing the field

    area : astropy.Quantity
        The total area contained by the fields, in arcmin^2

    Notes
    -----
    `preprep` will construct the prepared polygons immediately, however this
    geometry cannot be pickled, and thus the `Field` will not be passable using
    something like mpi. If that is desired, `prep` will have to be called
    manually at a later time, before any calls to `__contains__`

    See Also
    --------
    shapely : Source for all geometry operations
    '''

    def __contains__(self, other):
        # TODO might not be fair to *require* _prepped. Also will error badly
        if self._multi:
            return any(p.contains(other) for p in self._prepped)
        else:
            return self._prepped.contains(other)

    def _correct(self, crd, cen=(0, 0), unit=None):
        '''Centre, convert and RA correct the given array of `crd`s'''
        RA, DEC = crd[:, 0], crd[:, 1]

        # Add and convert units to arcmin
        if not hasattr(crd, 'unit'):
            RA, DEC = RA << u.Unit(unit), DEC << u.Unit(unit)

        cen <<= RA.unit

        # Centre and correct RA
        RA, DEC = (RA - cen[0]) * np.cos(DEC), DEC - cen[1]

        RA, DEC = RA.to_value(u.arcmin), DEC.to_value(u.arcmin)

        return np.c_[RA, DEC]

    def prep(self):
        '''Prepare the polygons for quicker containment searches'''
        if self._multi:
            self._prepped = [prepgeom.prep(p) for p in self.polygon]
        else:
            self._prepped = prepgeom.prep(self.polygon)

        return self._prepped

    def __init__(self, coords, cen=(0, 0), unit=None, preprep=False):

        # ------------------------------------------------------------------
        # Parse the coords argument
        # ------------------------------------------------------------------

        # if already a polygon, assume its already been corrected
        if isinstance(coords, geom.Polygon):
            self._multi = False
        elif isinstance(coords, geom.MultiPolygon):
            self._multi = True

        # is a single polygon of coordinates
        elif isinstance(coords, np.ndarray) and coords.ndim == 2:
            self._multi = False
            coords = self._correct(coords, cen, unit)

        # assume it's iterable of coords or polygons
        else:
            self._multi = True
            coords = [self._correct(c, cen, unit) for c in coords]

        # ------------------------------------------------------------------
        # Set up the polygons
        # ------------------------------------------------------------------

        # Combine and smooth all polygons
        if self._multi:

            self.polygon = ops.unary_union([geom.Polygon(c).buffer(0)
                                            for c in coords])

        else:
            self.polygon = geom.Polygon(coords).buffer(0)

        # Explicitly check the polygons again, as they sometimes change above
        if isinstance(self.polygon, geom.Polygon):
            self._multi = False
        elif isinstance(self.polygon, geom.MultiPolygon):
            self._multi = True

        # If desired, prep the polygons
        if preprep:
            self.prep()

        # Compute polygon area, in correct units
        self.area = self.polygon.area << u.arcmin**2

    @classmethod
    def from_dataset(cls, dataset, cen):
        '''Create this field from a corresponding `fitter.core.data.Dataset`'''
        import string

        unit = dataset.mdata['field_unit']

        coords = []
        for ch in string.ascii_letters:
            try:
                coords.append(dataset['fields'].mdata[f'{ch}'])
            except KeyError:
                break

        if len(coords) == 1:
            coords = coords[0]

        return cls(coords, cen=cen, unit=unit)

    def slice_radially(self, r1, r2):
        '''Return a new field representing a radial "slice" of this field'''

        # make sure that r1,r2 are in arcmin
        r1, r2 = r1.to_value('arcmin'), r2.to_value('arcmin')

        origin = geom.Point((0, 0))

        shell = origin.buffer(r2) - origin.buffer(r1)

        return Field(self.polygon & shell)

    def MC_sample(self, M, return_points=False):
        '''Randomly sample `M` points from this field

        Random points are generated (using `random.uniform`) between the
        minimum and maximum bounds of each polygon, and simple rejection
        sampling is used to determine the points within each polygon.
        The number of points per polygon (in fields with multiple seperate
        polygons) is in proportion to their respective areas.

        Parameters
        ----------
        M : int
            The number of points to sample

        return_points : bool, optional
            If `True`, returns a `shapely.geometry.MultiPoint` object containing
            all the points, otherwise simply returns the radial position of
            each (in arcmin). Defaults to `False`

        Returns
        -------
        sample : astropy.Quantity or shapely.geometry.MultiPoint
            Array of `M` sampled points, either in radial positions or
            coordinates, based on `return_points`

        '''
        # TODO we should accept a seed or something for this randomness

        def rejection_sample(poly, prep_poly, M_i):
            '''Rejection-sample `Mj` points from this given sub-`poly`'''

            if poly.is_empty:
                return []

            minx, miny, maxx, maxy = poly.bounds

            points = []

            # TODO this randomness could probably be done better?
            while len(points) < M_i:
                rand_x = random.uniform(minx, maxx)
                rand_y = random.uniform(miny, maxy)

                test_pnt = geom.Point(rand_x, rand_y)

                if prep_poly.contains(test_pnt):
                    points.append(test_pnt)

            return points

        # If this field hasn't been prepped yet, do that now, once
        if not hasattr(self, '_prepped'):
            self.prep()

        # Do each poly of a multipoly seperate so the bounds aren't huge
        if self._multi:
            points = []
            tot_area = self.polygon.area

            for poly, prep_poly in zip(self.polygon, self._prepped):
                M_i = round(M * poly.area / tot_area)
                points += rejection_sample(poly, prep_poly, M_i)

        else:
            points = rejection_sample(self.polygon, self._prepped, M)

        # return the sample of points
        if return_points:
            self._prev_sample = geom.MultiPoint(points)
        else:
            origin = geom.Point((0, 0))
            self._prev_sample = [p.distance(origin) for p in points] << u.arcmin

        return self._prev_sample

    def MC_integrate(self, func, sample=None, M=None):
        '''Monte Carlo integration of `func` over this field

        Using a random sample of points within this field, either generated
        beforehand or using `Field.MC_sample`, computes the integral of the
        given `func` over the entire field by simply evaluating and summing
        the function over all samples, and normalizing to the field area.

        Will support units, if supported by the given function.

        Parameters
        ----------
        func : callable
            The (vectorized) function to integrate over. Must accept an array
            of points represented by their radial distances from the origin.

        sample : numpy.ndarray, optional
            The sample of points to be used in the integration, represented by
            their radial distances from the origin. If None (default), a new
            sample will be generated using `Field.MC_sample` and `M`

        M : int, optional
            The number of points to be sampled from `Field.MC_sample`, if
            `sample` is None.
        '''

        if sample is None:

            if M is None:
                mssg = "must supply one of `sample` or `M`"
                raise TypeError(mssg)

            sample = self.MC_sample(M)

        M = sample.size

        res = np.sum(func(sample))

        # Only use area units if the integrand has units as well
        V = self.area

        # TODO have to do this cause I cant get composite equivalencies to work
        if hasattr(func, '_xunit'):
            V = V.to(func._xunit**2)

        if not hasattr(res, 'unit'):
            V = V.value

        return (V / M) * res

    # ----------------------------------------------------------------------
    # Plotting functionality
    # ----------------------------------------------------------------------

    def _patch(self, *args, **kwargs):
        '''Create a `PathPatch` based on the polygon boundaries'''
        from matplotlib.path import Path
        from matplotlib.patches import PathPatch

        if self.polygon.is_empty:
            raise ValueError("This Field is empty, cannot create patch")

        coords, codes = [], []

        for poly in (self.polygon if self._multi else [self.polygon]):

            for line in [poly.exterior, *poly.interiors]:
                coords += line.coords

                codes += [Path.MOVETO]
                codes += ([Path.LINETO] * (len(line.coords) - 2))
                codes += [Path.CLOSEPOLY]

        path = Path(coords, codes)
        return PathPatch(path, *args, **kwargs)

    def plot(self, ax, prev_sample=False, adjust_view=True, **kwargs):
        '''Plot this field onto a given ax as a polygonal patch

        Given an already-initialized matplotlib axes, add a patch representing
        all polygons in this field, using a `PathPatch` constructed from
        each polygons boundary lines.

        This method only adds a patch to existing plots, all other plotting
        logic must be handled seperately

        Parameters
        ----------
        ax : matplotlib.axes.Axes
            The matplotlib axes object to add this plot to

        prev_sample : bool, optional
            If True, also attempt to add a scatterplot of the last sampled
            points. `Field.MC_sample` must have been recently run, with
            `return_points=True` for this to function. Defaults to False

        adjust_view : bool, optional
            If True (default), also calls the `autoscale_view` method of `ax`.
            Simply adding a patch to a plot will not move the view to match,
            and patches could end up outside of the window limits. This
            parameter corrects that.
        '''

        pt = ax.add_patch(self._patch(**kwargs))

        if prev_sample:
            try:
                smpl = self._prev_sample

                if hasattr(smpl, 'geom_type'):
                    smpl_xy = zip(*[p.xy for p in smpl])
                    sc = ax.scatter(*smpl_xy, marker='.')
                    sc.set_zorder(pt.zorder + 1)

                else:
                    mssg = ("Must sample with `return_points=True` in order to"
                            "plot sampled points")
                    raise RuntimeError(mssg)

            except AttributeError:
                mssg = "No previous sample, call `Field.MC_sample` first"
                raise RuntimeError(mssg)

        if adjust_view:
            ax.autoscale_view()

        if prev_sample:
            return pt, sc
        else:
            return pt

reason: import numpy,import astropy