text:
import re
import os
import numpy as np
import matplotlib.pyplot as plt

version_add_evaluate_train_data = True
log_dir = '/media/sda1/sleep-data/gysj/log-reid/train-hyg'
# checkpoint_suffix = '_sf52lr1ef4'

log_param_list = [
                  'vggm1024v3pre-nolrn-gstev2-did512-bh60-lr1ef2',
                  'vggm1024v3pre-xtlv2-bbs-bh60-lr1ef2',
                  'vggm1024v3pre-xtl-bbs-bh60-lr1ef2',
                  'vggm1024v3pre-xent-cclbs-bh60-lr1ef2',
                  # 'vggm1024v2pre-xent-ccl-veri776-bh60-lr1ef2-mxe90',
                  # 'vggm1024v2pre-gstev2-no-intra-bh60-lr1ef2',
                  # 'vggm1024v2pre-gstev2-bh60-lr1ef2',
                  # 'vggm1024v2pre-gste-small-batch-lr1ef2',
                  # 'vggm1024v2pre-gste-veri776-bh60-lr1ef2',
                  # 'vggm1024v2pre-anglem1-veri776-bh60-lr1ef2',
                  # 'vggm1024v2pre-anglem4-veri776-bh60-lr1ef2',
                  # 'vggm1024v2pre-xent-veri776-bh56-lr1ef2',
                  # 'vggm1024v2pre-xent-ccl-veri776-bh60-lr1ef2-maxepoch90',
                  # 'vggm1024v2pre-xent-ccl-veri776-bh60-lr1ef2',
                  # 'vggm1024v2pre-xent-cclv2-veri776-bh60-lr1ef2',
                  # 'vggm1024v2pre-xent-trtrnt-veri776-bh60-lr1ef2',
                 ]
suffix_model_list = [
                     # 'vggm1024-tripletrnt-veri776-bh64-lr1ef4',
                     # 'vggm1024-tripletv2rnt-veri776-bh64-lr1ef3',
                     # 'hacnn-xent-market1501-bh64-lr1ef2',
                    ]

pattern_eval_step = re.compile('eval_step=(\d+)')
pattern_print_freq = re.compile('print_freq=(\d+)')
pattern_batch_count = re.compile('\[\d+\]\[\d+/(\d+)\]')
pattern_best_rank1 = re.compile('Best\sRank-1\s(\d+.\d+)')
pattern_btrk1_epoch = re.compile('achieved\sat\sepoch\s(\d+)')
pattern_loss = re.compile('Loss\s(\d+.\d+)\s.(\d+.\d+).')
pattern_map = re.compile('temAP:\s(\d+.\d+)')
pattern_cmc1 = re.compile('teRank-1\s\s:\s(\d+.\d+)')
pattern_cmc5 = re.compile('teRank-5\s\s:\s(\d+.\d+)')
pattern_cmc10 = re.compile('teRank-10\s:\s(\d+.\d+)')
pattern_cmc20 = re.compile('teRank-20\s:\s(\d+.\d+)')
pattern_tmap = re.compile('trmAP:\s(\d+.\d+)')
pattern_tcmc1 = re.compile('trRank-1\s\s:\s(\d+.\d+)')
pattern_tcmc5 = re.compile('trRank-5\s\s:\s(\d+.\d+)')
pattern_tcmc10 = re.compile('trRank-10\s:\s(\d+.\d+)')
pattern_tcmc20 = re.compile('trRank-20\s:\s(\d+.\d+)')

def get_train_info():
    train_info_dict = {}
    for param in log_param_list:
        if param in suffix_model_list:
            log_file = '/media/sda1/sleep-data/gysj/log-reid/train-hyg/' + param + '/log_train' \
                       + checkpoint_suffix + '.txt'
        else:
            log_file = '/media/sda1/sleep-data/gysj/log-reid/train-hyg/' + param + '/log_train.txt'
        f = open(log_file)
        lines = f.readlines()
        f.close()
        eval_step = 0
        print_freq = 0
        batch_count = 0
        best_rank1 = 0
        btrk1_epoch = 0
        loss_list = []
        map_list = []
        cmc1_list = []
        cmc5_list = []
        cmc10_list = []
        cmc20_list = []
        tmap_list = []
        tcmc1_list = []
        tcmc5_list = []
        tcmc10_list = []
        tcmc20_list = []
        for line in lines:
            match = pattern_eval_step.search(line.strip())
            if match is not None:
                match_group_eval_step = match.groups()
                eval_step = int(match_group_eval_step[0])
                break
        for line in lines:
            match = pattern_print_freq.search(line.strip())
            if match is not None:
                match_group_print_freq = match.groups()
                print_freq = int(match_group_print_freq[0])
                break

        for line in lines:
            match = pattern_batch_count.search(line.strip())
            if match is not None:
                match_group_batch_count = match.groups()
                batch_count = int(match_group_batch_count[0])
                break

        for line in lines:
            match = pattern_best_rank1.search(line.strip())
            if match is not None:
                match_group_best_rank1 = match.groups()
                best_rank1 = float(match_group_best_rank1[0])
                break

        for line in lines:
            match = pattern_btrk1_epoch.search(line.strip())
            if match is not None:
                match_group_btrk1_epoch = match.groups()
                btrk1_epoch = int(match_group_btrk1_epoch[0])
                break

        for line in lines:
            match_loss = pattern_loss.search(line.strip())
            match_map = pattern_map.search(line.strip())
            match_cmc1 = pattern_cmc1.search(line.strip())
            match_cmc5 = pattern_cmc5.search(line.strip())
            match_cmc10 = pattern_cmc10.search(line.strip())
            match_cmc20 = pattern_cmc20.search(line.strip())
            if match_loss is not None:
                match_group_loss = match_loss.groups()
                loss_v = float(match_group_loss[0])
                loss_ave = float(match_group_loss[1])
                loss_list.append([loss_v, loss_ave])
            if match_map is not None:
                map_list += [float(match_map.groups()[0])] * eval_step
            if match_cmc1 is not None:
                cmc1_list += [float(match_cmc1.groups()[0])] * eval_step
            if match_cmc5 is not None:
                cmc5_list += [float(match_cmc5.groups()[0])] * eval_step
            if match_cmc10 is not None:
                cmc10_list += [float(match_cmc10.groups()[0])] * eval_step
            if match_cmc20 is not None:
                cmc20_list += [float(match_cmc20.groups()[0])] * eval_step
        assert len(map_list) == len(cmc1_list) == len(cmc5_list) == len(cmc10_list) == len(cmc20_list)
        # assert best_rank1 == cmc1_list[btrk1_epoch-1]

        if version_add_evaluate_train_data:
            for line in lines:
                match_tmap = pattern_tmap.search(line.strip())
                match_tcmc1 = pattern_tcmc1.search(line.strip())
                match_tcmc5 = pattern_tcmc5.search(line.strip())
                match_tcmc10 = pattern_tcmc10.search(line.strip())
                match_tcmc20 = pattern_tcmc20.search(line.strip())
                if match_tmap is not None:
                    tmap_list += [float(match_tmap.groups()[0])] * eval_step
                if match_tcmc1 is not None:
                    tcmc1_list += [float(match_tcmc1.groups()[0])] * eval_step
                if match_tcmc5 is not None:
                    tcmc5_list += [float(match_tcmc5.groups()[0])] * eval_step
                if match_tcmc10 is not None:
                    tcmc10_list += [float(match_tcmc10.groups()[0])] * eval_step
                if match_tcmc20 is not None:
                    tcmc20_list += [float(match_tcmc20.groups()[0])] * eval_step
            assert len(tmap_list) == len(tcmc1_list) == len(tcmc5_list) == len(tcmc10_list) == len(tcmc20_list)

        loss_list = np.array(loss_list)
        step = batch_count // print_freq
        loss_list_step = loss_list[step-1::step, :]
        map_list = np.array(map_list)
        cmc1_list = np.array(cmc1_list)
        cmc5_list = np.array(cmc5_list)
        cmc10_list = np.array(cmc10_list)
        cmc20_list = np.array(cmc20_list)
        train_info_dict[param] = [loss_list_step, loss_list, map_list, cmc1_list,
                                  cmc5_list, cmc10_list, cmc20_list, [best_rank1, btrk1_epoch]]

        if version_add_evaluate_train_data:
            tmap_list = np.array(tmap_list)
            tcmc1_list = np.array(tcmc1_list)
            tcmc5_list = np.array(tcmc5_list)
            tcmc10_list = np.array(tcmc10_list)
            tcmc20_list = np.array(tcmc20_list)
            train_info_dict[param] = [loss_list_step, loss_list,
                                      map_list, cmc1_list, cmc5_list, cmc10_list, cmc20_list,
                                      tmap_list, tcmc1_list, tcmc5_list, tcmc10_list, tcmc20_list,
                                      [best_rank1, btrk1_epoch]]

    return train_info_dict

train_info_dict = get_train_info()
type_list = ['-lssp', '-ls', '-mp', '-c1', '-c5', '-c10', '-c20', '-tmp', '-tc1', '-tc5', '-tc10', '-tc20']
type_plot_index = [0, 2, 3, 7, 8]
# type_plot_index = [1]
# type_plot_index = [0]
color = ['r', 'g', 'b', 'y', 'c', 'k', 'tab:purple', 'm']
# color = ['C0', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9']
type_index2 = 0
title = "vggm1024v2pre"

fig = plt.figure()
ax0 = fig.add_subplot(111, title=title)
for i, param in enumerate(log_param_list):
    for type_index1 in type_plot_index:
        x_range = range(len(train_info_dict[param][type_index1]))
        if type_index1 == 0 or type_index1 == 1:
            ax0.plot(x_range, train_info_dict[param][type_index1][:, type_index2],
                     color[i], label=param + type_list[type_index1])
            continue
        if type_index1 == 2:
            last_value_str = '-{:.2f}-e{:d}-{:.2f}'.format(train_info_dict[param][type_index1][-1],
                                                           train_info_dict[param][-1][1],
                                                           train_info_dict[param][-1][0])
            ax0.plot(x_range, train_info_dict[param][type_index1] / 100 * 6, color[i],
                     label=param + type_list[type_index1] + last_value_str)
            continue
        last_value_str = '-{:.2f}'.format(train_info_dict[param][type_index1][-1])
        ax0.plot(x_range, train_info_dict[param][type_index1] / 100 * 6, color[i],
                 label=param + type_list[type_index1] + last_value_str)
# ax0.legend(loc=0)  # best
# ax0.legend(loc=1)  # upper right
# ax0.legend(loc=2)  # upper left
# ax0.legend(loc=3)  # lower left
# ax0.legend(loc=4)  # lower right
fig.savefig(os.path.join(log_dir, 'vggm1024v3pre_xent_xtl_xtlv2_gstev2-2'))
plt.show()

max_stars_repo_name: xzgz/vehicle-reid

max_issues_repo_name: xzgz/vehicle-reid

max_forks_repo_name: xzgz/vehicle-reid


text:
"""Implementation of an exact MDP solver.

The full specification of the process includes the states, the actions
possible at each state, the transition probabilities induced by each
action, and the conditional expectation of rewards given the current state,
action, and new state.

An MDP is encoded as a list of states. Each state comprises a list of actions.
Each action is a list of Outcome instances. A terminal state has only one
possible action, which is a self-loop with probability 1 and reward 0.

It might be better to have states be dictionaries of actions instead of lists.
This would allow for actions like "forward" to be consistent across different
states, instead of being encoded by multiple potentially distinct integer
indices. This wouldn't create a particularly large code change.

The functions value_update and value_iteration_iter can be composed to build
many MDP solvers. Value iteration exclusively runs the latter. It may be more
efficient to run multiple iterations of value_update before running
value_iteration_iter (Sutton 83). The function update_policy is for updating
only the policy, without refining value estimates online.
"""

from dataclasses import dataclass
from typing import List, Sequence

import matplotlib.pyplot as plt
import numpy as np

@dataclass(frozen=True)
class Outcome:
    prob: float
    state: int
    reward: float

Action = Sequence[Outcome]
State = Sequence[Action]
MDP = Sequence[State]
Policy = List[int]

def value_action(
    outcomes: Sequence[Outcome], values: Sequence[float], gamma: float
) -> float:
    return sum(o.prob * (o.reward + gamma * values[o.state]) for o in outcomes)

def value_update(policy: Policy, mdp: MDP, values: List[float], gamma: float) -> float:
    """Update value estimates of MDP states according to a fixed policy"""
    delta = 0
    for state, actions in enumerate(mdp):
        # evaluate state_i
        selected_action = policy[state]
        outcome_dist = actions[selected_action]
        new_value = value_action(outcome_dist, values, gamma)
        update = np.abs(values[state] - new_value)
        values[state] = new_value
        if update > delta:
            delta = update
    return delta

def value_iteration_iter(mdp: MDP, values: List[float], gamma: float) -> float:
    """Update value estimates of MDP states while taking greedy actions"""
    delta = 0
    for state, actions in enumerate(mdp):
        new_value = max(value_action(a, values, gamma) for a in actions)
        update = np.abs(values[state] - new_value)
        values[state] = new_value
        if update > delta:
            delta = update
    return delta

def evaluate_policy(
    policy: Policy,
    mdp: MDP,
    values: List[float],
    gamma: float,
    convergence_eps: float = 0.001,
) -> List[float]:
    delta = convergence_eps + 1
    while delta >= convergence_eps:
        delta = value_update(policy, mdp, values, gamma)
    return values

def update_policy(
    policy: Policy, mdp: MDP, values: Sequence[float], gamma: float
) -> bool:
    """Update a policy greedily based on the MDP and values given.

    Returns True if the policy has converged, False if changes were made.
    Note: changes policy in-place.
    """
    # compute the greedy policy based on the mdp
    converged = True
    for state, actions in enumerate(mdp):

        def value_func(a: int) -> float:
            return value_action(actions[a], values, gamma)

        # compute argmax_{a \in actions} value(s, a)
        old_policy = policy[state]
        new_policy = max(range(len(actions)), key=value_func)
        if old_policy != new_policy:
            converged = False
        policy[state] = new_policy
    return converged

def compute_policy(mdp: MDP, gamma: float) -> Policy:
    nstates = len(mdp)
    policy: Policy = np.zeros(nstates, dtype=np.int)
    values: List[float] = np.zeros(nstates)
    converged = False
    while not converged:
        values = evaluate_policy(policy, mdp, values, gamma)
        # sns.heatmap(np.reshape(values, (4, 4)), annot=True)
        # plt.show()
        converged = update_policy(policy, mdp, values, gamma)
        # print(np.reshape(policy, (4, 4)))
        # print(np.reshape(values, (4, 4)))
    return policy

def value_iteration(mdp: MDP, gamma: float, epsilon: float = 0.01) -> Policy:
    nstates = len(mdp)
    values = np.zeros(nstates)
    converged = False
    while not converged:
        delta = value_iteration_iter(mdp, values, gamma)
        # plt.plot(values)
        # plt.show()
        converged = delta < epsilon
    policy: List[int] = np.zeros(nstates, dtype=np.int)
    update_policy(policy, mdp, values, gamma)
    return policy

def mdp_example() -> None:
    gamma = 0.9
    # simple gridworld policy - cells (0,0) and (3,3) are goal states

    def make_state(cell_num: int) -> State:
        if cell_num == 0 or cell_num == 15:
            return [[Outcome(1, cell_num, 0)]]  # terminal state
        x = cell_num % 4
        y = cell_num // 4
        left = (x - 1, y, [Outcome(1, cell_num - 1, -1)])
        right = (x + 1, y, [Outcome(1, cell_num + 1, -1)])
        up = (x, y - 1, [Outcome(1, cell_num - 4, -1)])
        down = (x, y + 1, [Outcome(1, cell_num + 4, -1)])
        return [
            move[2]
            for move in [left, right, up, down]
            if (move[0] >= 0 and move[0] < 4 and move[1] >= 0 and move[1] < 4)
        ]

    mdp = [make_state(num) for num in range(16)]
    # opt = compute_policy(mdp, gamma)
    opt = value_iteration(mdp, gamma)
    print(np.reshape(opt, (4, 4)))

def gambler_example() -> None:
    gamma = 0.9

    def make_state(money: int) -> State:
        if money > 100 or money < 0:
            raise ValueError("Money must be in [0, 100]")
        if money == 0:
            return [[Outcome(1, 0, 0)]]
        if money == 100:
            return [[Outcome(1, 100, 0)]]
        actions = []
        for bet in range(0, money + 1):
            positive = min(money + bet, 100)
            pos_reward = 1 if positive == 100 else 0
            negative = money - bet
            neg_reward = 0
            actions.append(
                [Outcome(0.4, positive, pos_reward), Outcome(0.6, negative, neg_reward)]
            )
        return actions

    mdp = [make_state(num) for num in range(101)]
    opt = value_iteration(mdp, gamma)
    plt.plot(opt)
    plt.show()

if __name__ == "__main__":
    # mdp_example()
    gambler_example()

ext: py

reason: import numpy


text:
from scipy.io import loadmat

data = loadmat('../../pyamg/gallery/example_data/recirc_flow.mat')

A = data['A'].tocsr()

from pyamg import *
import numpy
b = numpy.ones((A.shape[0],))
#ml = smoothed_aggregation_solver(A, symmetry='symmetric',max_coarse=5)
ml = smoothed_aggregation_solver(A, symmetry='nonsymmetric',max_coarse=5)

res=[]
x = ml.solve(b,residuals=res)

from pylab import *
semilogy(res)
xlabel('iteration')
ylabel('residual norm')
show()

max_stars_repo_name: pombreda/pyamg

max_issues_repo_name: pombreda/pyamg

max_forks_repo_name: pombreda/pyamg

reason: import numpy,from scipy


text:
#Neural network utils
#This code was not implemented by me

import numpy as np
from functools import reduce

flatten_list_of_arrays = lambda list_of_arrays: reduce(
    lambda acc, v: np.array([*acc.flatten(), *v.flatten()]),
    list_of_arrays
)

def inflate_matrixes(flat_thetas, shapes):
    layers = len(shapes) + 1
    sizes = [shape[0] * shape[1] for shape in shapes]
    steps = np.zeros(layers, dtype=int)

    for i in range(layers - 1):
        steps[i + 1] = steps[i] + sizes[i]

    return [
        flat_thetas[steps[i]: steps[i + 1]].reshape(*shapes[i])
        for i in range(layers - 1)
    ]

def sigmoid(z):
    a = [(1 / (1 + np.exp(-x))) for x in z]
    return np.asarray(a).reshape(z.shape)

def back_propagation(flat_thetas, shapes, X, Y):
    m, layers = len(X), len(shapes) + 1
    thetas = inflate_matrixes(flat_thetas, shapes)

    a = feed_forward(thetas, X)

    deltas = [*range(layers - 1), a[-1] - Y]
    for i in range(layers - 2, 0, -1):
        deltas[i] = (deltas[i + 1] @ np.delete(thetas[i], 0, 1)) * (a[i] * (1 - a[i]))

    deltas_l = []
    for i in range(layers - 1):
        deltas_l.append(
            (deltas[i + 1].T
            @
            np.hstack((
                np.ones(len(a[i])).reshape(len(a[i]), 1),
                a[i]
            ))) / m
        )
    deltas_l = np.asarray(deltas_l)

    return flatten_list_of_arrays(
        deltas_l
    )

def cost_function(flat_thetas, shapes, X, Y):
    a = feed_forward(
        inflate_matrixes(flat_thetas, shapes),
        X
    )
    return -(Y * np.log(a[-1]) + (1 - Y) * np.log(1 - a[-1])).sum() / len(X)

def feed_forward(thetas, X):
    mt_a = [np.asarray(X)]

    for i in range(len(thetas)):
        mt_a.append(
            sigmoid(
                np.matmul(
                    np.hstack((
                        np.ones(len(X)).reshape(len(X), 1),
                        mt_a[i]
                    )), thetas[i].T
                )
            )
        )
    return mt_a


text:
# Author: Tao Hu <taohu620@gmail.com>
from tensorpack.utils import logger
from tensorpack.dataflow.imgaug.base import ImageAugmentor
import numpy as np
import cv2

def RandomResizeSlight(ds,xrange=(0.9, 1.1), yrange=(0.9, 1.1),aspect_ratio_thres=0.1,minimum=(0,0)):
    image = ds[0]
    label = ds[1]

    assert aspect_ratio_thres >= 0
    if aspect_ratio_thres == 0:
        assert xrange == yrange

    def is_float(tp):
        return isinstance(tp[0], float) or isinstance(tp[1], float)

    assert is_float(xrange) == is_float(yrange), "xrange and yrange has different type!"
    _is_scale = is_float(xrange)

    def _get_augment_params(img):
        cnt = 0
        h, w = img.shape[:2]

        def get_dest_size():
            if _is_scale:
                sx = np.random.uniform(xrange[0],xrange[1],size=[])
                if aspect_ratio_thres == 0:
                    sy = sx
                else:
                    sy = np.random.uniform(yrange[0],yrange[1],size=[])
                destX = max(sx * w, minimum[0])
                destY = max(sy * h, minimum[1])
            else:
                sx = np.random.uniform(xrange[0],xrange[1],size=[])
                if aspect_ratio_thres == 0:
                    sy = sx * 1.0 / w * h
                else:
                    sy = np.random.uniform(yrange[0],yrange[1],size=[])
                destX = max(sx, minimum[0])
                destY = max(sy, minimum[1])
            return (int(destX + 0.5), int(destY + 0.5))

        while True:
            destX, destY = get_dest_size()
            if aspect_ratio_thres > 0:  # don't check when thres == 0
                oldr = w * 1.0 / h
                newr = destX * 1.0 / destY
                diff = abs(newr - oldr) / oldr
                if diff >= aspect_ratio_thres + 1e-5:
                    cnt += 1
                    if cnt > 50:
                        logger.warn("RandomResize failed to augment an image")
                        return h, w, h, w
                        break
                    continue
                return h, w, destY, destX

    h, w, destY, destX = _get_augment_params(image)
    image = cv2.resize(image, (destY, destX),interpolation=cv2.INTER_LINEAR)
    label = cv2.resize(label, (destY,destX),interpolation=cv2.INTER_NEAREST)
    return [image,label]

def RandomResize(ds,xrange=(0.7, 1.5), yrange=(0.7, 1.5),aspect_ratio_thres=0.1,minimum=(0,0)):
    image = ds[0]
    label = ds[1]

    assert aspect_ratio_thres >= 0
    if aspect_ratio_thres == 0:
        assert xrange == yrange

    def is_float(tp):
        return isinstance(tp[0], float) or isinstance(tp[1], float)

    assert is_float(xrange) == is_float(yrange), "xrange and yrange has different type!"
    _is_scale = is_float(xrange)

    def _get_augment_params(img):
        cnt = 0
        h, w = img.shape[:2]

        def get_dest_size():
            if _is_scale:
                sx = np.random.uniform(xrange[0],xrange[1],size=[])
                if aspect_ratio_thres == 0:
                    sy = sx
                else:
                    sy = np.random.uniform(yrange[0],yrange[1],size=[])
                destX = max(sx * w, minimum[0])
                destY = max(sy * h, minimum[1])
            else:
                sx = np.random.uniform(xrange[0],xrange[1],size=[])
                if aspect_ratio_thres == 0:
                    sy = sx * 1.0 / w * h
                else:
                    sy = np.random.uniform(yrange[0],yrange[1],size=[])
                destX = max(sx, minimum[0])
                destY = max(sy, minimum[1])
            return (int(destX + 0.5), int(destY + 0.5))

        while True:
            destX, destY = get_dest_size()
            if aspect_ratio_thres > 0:  # don't check when thres == 0
                oldr = w * 1.0 / h
                newr = destX * 1.0 / destY
                diff = abs(newr - oldr) / oldr
                if diff >= aspect_ratio_thres + 1e-5:
                    cnt += 1
                    if cnt > 50:
                        logger.warn("RandomResize failed to augment an image")
                        return h, w, h, w
                        break
                    continue
                return h, w, destY, destX

    h, w, destY, destX = _get_augment_params(image)
    image = cv2.resize(image, (destY, destX),interpolation=cv2.INTER_LINEAR)
    label = cv2.resize(label, (destY,destX),interpolation=cv2.INTER_NEAREST)
    return [image,label]

class RandomCropWithPadding(ImageAugmentor):
    def __init__(self, crop_size, ignore_label=255):
        super(RandomCropWithPadding, self).__init__()
        self.crop_size = crop_size
        if isinstance(crop_size,int):
            self.crop_size = (crop_size,crop_size)

        self.ignore_label = ignore_label
        self._init()

    def _get_augment_params(self, img):
        self.h0 = img.shape[0]
        self.w0 = img.shape[1]

        if self.crop_size[0] > self.h0:
            top = (self.crop_size[0] - self.h0) / 2
            bottom = (self.crop_size[0] - self.h0) - top
        else:
            top = 0
            bottom = 0

        if self.crop_size[1] > self.w0:
            left = (self.crop_size[1] - self.w0) / 2
            right = (self.crop_size[1] - self.w0) - left
        else:
            left = 0
            right = 0
        new_shape = (top + bottom + self.h0, left + right + self.w0)
        diffh = new_shape[0] - self.crop_size[0]
        assert diffh >= 0
        crop_start_h = 0 if diffh == 0 else self.rng.randint(diffh)
        diffw = new_shape[1] - self.crop_size[1]
        assert diffw >= 0
        crop_start_w = 0 if diffw == 0 else self.rng.randint(diffw)
        return (top, bottom, left, right, crop_start_h, crop_start_w)

    def _augment(self, img, param, id = 0):
        top, bottom, left, right, crop_start_h, crop_start_w = param
        if id <= 1:
            il = self.ignore_label
        else:
            il = 0

        img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=il)

        assert crop_start_h + self.crop_size[0] <= img.shape[0], crop_start_w + self.crop_size[1] <= img.shape[1]
        return img[crop_start_h:crop_start_h + self.crop_size[0], crop_start_w:crop_start_w + self.crop_size[1]]

max_stars_repo_name: MarcWong/tensorpack

max_issues_repo_name: MarcWong/tensorpack

max_forks_repo_name: MarcWong/tensorpack


text:
#!/usr/bin/python3

# Functions to handle Input
#############################################################################################
def read_csv():
  # simple function to read data from a file
  data = pd.read_csv('out.csv', sep=';')
  return data

def read_sacct():
  # function to read the data directly from sacct
  print('not implemented yet')

# Functions to handle string/value conversion
#############################################################################################
# function converts format (DD-)HH:MM:SS to seconds
def ave2sec(x):
  if ( '-' in x ):
    vals = x.split('-')
    times = vals[1].split(':')
    sec = 24*3600*int(vals[0])+3600*int(times[0])+60*int(times[1])+int(times[2])
  else:
    times = x.split(':')
    sec = 3600*int(times[0])+60*int(times[1])+int(times[2])
  return (sec)

def scalingref(x):
  # returns reference scaling factor for MPI jobs based on 1.5 factor:
  # doubling cores should make parformance x1.5 (or better)
  if int(x) == 1:
    ref = 1
  else:
    ref = np.power(1/1.5,np.log2(int(x)))

  return ref

def rss2g(x):
  return int(float(x[:-1]))/1024

def reqmem2g(x):
  return int(float(x[:-2]))/1024

# Functions to handle DataFrames
#############################################################################################
def parse_df(data):
  # convert multi-line DataFrame to more compact form for analysis
  from datetime import datetime

  data[['id','subid']] = data.JobID.str.split('_',1, expand=True)
  data.drop(['subid'],axis=1, inplace=True)

  df=pd.DataFrame()
  df=data[~data['JobID'].str.contains("\.")]
  df.rename(columns={'State': 'Parentstate'}, inplace=True)

  data2=data.shift(-1).dropna(subset=['JobID'])
  df2=data2[data2['JobID'].str.contains("\.batch")]

  data2=data.shift(-2).dropna(subset=['JobID'])
  df3=data2[data2['JobID'].str.contains("\.0")]

  df.update(df2.MaxRSS)
  df.update(df3.MaxRSS)
  df.update(df2.AveCPU)
  df.update(df3.AveCPU)
  df=df.join(df2[['State']])
  df.update(df3.State)

  # drop columns that all all nan
  df.dropna(axis=1, inplace=True, how='all')

  # drop rows where any element is nan (errors in the data)
  df.dropna(axis=0, inplace=True, how='any')
  df.reset_index(inplace=True)

  df.loc[:,'State']=df.State.apply(str)

  # add extra columns to df for analysis purposes
  df.insert(len(df.columns),'runtime',(pd.to_datetime(df['End'])-pd.to_datetime(df['Start']))/np.timedelta64(1,'s'))
  df.insert(len(df.columns),'waittime',(pd.to_datetime(df['Start'])-pd.to_datetime(df['Submit']))/np.timedelta64(1,'s'))

  # NOTE: this value ok only if State=Completed, for State=timeout values close to RawCPUTime
  df.insert(len(df.columns),'cpuavesec',df['AveCPU'].apply(ave2sec) )

  # scaling reference number for MPI jobs
  df.insert(len(df.columns),'cpufactor',df['cpuavesec']/df['runtime'] )

  df.loc[:,'MaxRSS']=df['MaxRSS'].apply(rss2g)
  df.insert(len(df.columns),'MemReqTotal', df['ReqMem'].apply(reqmem2g)*df['NCPUS'] )
  df.insert(len(df.columns),'MemRatio', df['MaxRSS']/df['MemReqTotal'] )
  df.insert(len(df.columns),'TimeRatio', df['runtime']/df['Timelimit'].apply(ave2sec)  )

  df.drop(['Elapsed','Start','End','Submit','AveCPU','CPUTimeRAW','JobID'], axis=1, inplace=True)

  return df

def analyze_df(df):
  # real work happens here. Based on proper conditions mark flags for jobs that need furher investigation
  # - cpu-load below treshold
  # - memory usage below treshold or very close to 100% (nearly killed due to memory exceeded)
  # - huge percentage of jobs killed with TIMEOUT
  # - runtimes very close to TIMEOUT value OR runtimes very short
  usrdf = pd.DataFrame(columns=['User','NJobs','UniqueJobs','%failed','%timeout','%poor_cpu_util','%low_mem_util_warn','%high_mem_util_warn','%close2timeout','%jobs_below_15min'])

  df.insert(len(df.columns),'warning', '0')
  for user in df.User.unique():
    df2 = df[ df['User'] == user]
    account = df2.Account.iloc[0]
    jobs = len(df2)
    unique_jobs = len(df2.id.unique())
    okjobs = len(df2 [ df2['State'] == 'COMPLETED'] )

    # Failed jobs
    failed = len (df2[ df2['State'] == 'FAILED'] )
    pfailed = 0
    if failed > 0:
       pfailed = failed/jobs

    # timeout jobs
    timeout = len(df2[ (df2['State'] == 'FAILED' ) & (df2['Parentstate'] == 'TIMEOUT') ])
    ptimeout = 0
    if timeout > 0:
        ptimeout = timeout/failed

    # runtime check, % of jobs where runtime is <15min (=900s)
    short_runtime = len(df2[ (df2['State'] == 'COMPLETED' ) & (df2['runtime'] < 900 ) ])
    pshort_runtime = 0
    if short_runtime > 0:
      pshort_runtime = short_runtime/okjobs

    # poor cpu utlization means that cpu-load is below 50% of the peak for whole run.
    # for multi-cpu, use scaling factor (assume 1.5 MPI scaling to be sufficient).
    poor_utilization = len(df2[ (df2['State'] == 'COMPLETED' ) & (df2['cpufactor'] < 0.5 ) & (~df2['Partition'].str.contains("gpu") ) ])
    putilization = 0
    if poor_utilization > 0:
      putilization = poor_utilization/okjobs

    # memory usage
    low_memory = len(df2[ (df2['State'] == 'COMPLETED' ) & (df2['MemRatio'] < 0.15 ) ])
    high_memory = len(df2[ (df2['State'] == 'COMPLETED' ) & (df2['MemRatio'] > 0.9 ) ])
    plowmem = 0
    phighmem = 0
    if low_memory > 0:
      plowmem = low_memory/okjobs
    if high_memory > 0:
      phighmem = high_memory/okjobs

    # runtime check
    rtime = len(df2[ (df2['State'] == 'COMPLETED' ) & (df2['TimeRatio'] > 0.9 ) ])
    prtime = 0
    if rtime > 0:
      prtime = rtime/okjobs

    user2=user+'('+account+')'
    usrdf.loc[len(usrdf)]=[user2,jobs,unique_jobs,pfailed*100,ptimeout*100,putilization*100,plowmem*100,phighmem*100,prtime*100,pshort_runtime]

  return usrdf

# Functions to handle Output summary
#############################################################################################
def print_data(data):
  print('============================')
  print(data)

def writecsv(data):
  data.to_csv('result.csv', sep=';', index=False)

def help():
  # print useage information
  msg = """
  Usage:
  -h           : this help message
  -csv <file>  : read the data from a file instead of calling sacct directly
  """
  print(msg)

if __name__ == '__main__':
    from argparse import ArgumentParser
    import sys
    import pandas as pd
    import numpy as np
    import time

    times=[]
    comment=[]

    times.append([time.process_time(),'start'])

    data=read_csv()
    times.append([time.process_time(),'read csv'])
    data=parse_df(data)
    times.append([time.process_time(),'parse_df'])
    usrdf=analyze_df(data)
    #times.append([time.process_time(),'analyze_df'])

    #print_data(usrdf)
    writecsv(usrdf)
    #times.append([time.process_time(),'print data'])

    prev=times[0][0]
    #print(times)
    #for i in times:
    #    print("%s : %.2f seconds" % (i[1], i[0]-prev) )
    #    prev=i[0]


text:
from math import log
from scipy.stats import wilcoxon

def dcg(judgments, indices):
    """
    Compute DCG given an iterable of result positions and judgments for a
    result at each position.

    Note that we use 0-based indexing (hence the i + 2 in the discount term).

    Args:
        judgments: An iterable of numeric relevance judgments
        indices: An optional iterable of result positions for each judgment

    Returns: The discounted cumulative gain as a float.
    """
    return sum([(2**j - 1) / log(i + 2, 2) for (i, j) in
                zip(indices, judgments)])

def ndcg(judgments, indices=None, ideal_judgments=None):
    """
    Compute NDCG given an iterable of judgments for a result at each position
    in a result set, along with an optional iterable of corresponding result
    position indices, and an iterable of maximum possible judgments at each
    position.

    Args:
        judgments: An iterable of numeric relevance judgments
        indices: An optional iterable of result positions for each judgment
        ideal_judgments: An optional iterable of numeric judgments representing
            the maximum attainable judgment at each result position

    Returns: The normalized discounted cumulative gain as a float.
    """
    indices = indices if indices is not None else range(len(judgments))

    ideal_judgments = ideal_judgments if ideal_judgments is not None \
        else sorted(judgments, reverse=True)

    return dcg(judgments, indices) / dcg(ideal_judgments, indices)

def is_statistically_significant(g1_query_dcgs, g2_query_dcgs, alpha=.05):
    """
    Run the paired Wilcoxon signed rank test to determine if the difference
    between the groups is statistically significant.

    Args:
        g1_query_dcgs: An iterable of query DCG scores
        g2_query_dcgs: An iterable of query DCG scores

    Returns: A pair containing a bool indicating whether the difference is
        statistically significant, and the associated p-value.

    References:
        [Wilcoxon signed-rank test](https://en.wikipedia.org/wiki/Wilcoxon_signed-rank_test)
    """
    T, p = wilcoxon(g1_query_dcgs, g2_query_dcgs, zero_method="pratt")
    return (bool(p <= alpha), p)  # converting numpy bool to bool

max_stars_repo_name: EdsonGermano/arcs

max_issues_repo_name: EdsonGermano/arcs

max_forks_repo_name: EdsonGermano/arcs

reason: from scipy


text:
import pandas as pd
import numpy as np
import lightgbm as lgb
from feature_engineering import *
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVR
from sklearn.linear_model import LinearRegression, Ridge

if __name__ == '__main__':

    folder = 'data_task1/'

    ss = pd.read_csv(folder + 'sample_submission.csv')
    tables = ['sip', 'chugun', 'produv', 'lom', 'plavki', 'chronom', 'target']

    # читаем тренировочные таблицы
    sip = pd.read_csv(folder + 'sip_train.csv')
    chugun = pd.read_csv(folder + 'chugun_train.csv')
    produv = pd.read_csv(folder + 'produv_train.csv')
    lom = pd.read_csv(folder + 'lom_train.csv')
    plavki = pd.read_csv(folder + 'plavki_train.csv')
    chronom = pd.read_csv(folder + 'chronom_train.csv')
    gas = pd.read_csv(folder + 'gas_train.csv')
    target = pd.read_csv(folder + 'target_train.csv')

    # Готовим датасет
    chronom_features = prepare_chronom_features(chronom)
    sip_features = prepare_sip_features(sip)
    chugun_features = prepare_chugun_features(chugun)
    produv_features = prepare_produv_features(produv, chronom)
    lom_features = prepare_lom_features(lom)
    gas_features = prepare_gas_features(gas, chronom)

    X = pd.concat([chronom_features, chugun_features, produv_features, lom_features, gas_features], axis=1)
    y = target.set_index('NPLV')

    f_columns = X.columns
    X.columns = [f'feature_{i}' for i in range(X.shape[1])]

    scaler = StandardScaler()
    X_ = scaler.fit_transform(X.loc[:, X.isna().sum() == 0].clip(0, 10000))

    # Обучение
    regT = lgb.LGBMRegressor(num_leaves=7, n_estimators=100)
    #regT = LinearRegression()
    regT.fit(X, np.log(y['TST'].clip(1550, 1700)))

    regC = Ridge(50)
    regC.fit(X_,  np.log(1 + y['C'].fillna(0.04).clip(0.02, 0.08)))

    # inference

    ss = pd.read_csv(folder + 'sample_submission.csv')

    # читаем тестовые таблицы

    sip = pd.read_csv(folder + 'sip_test.csv')
    chugun = pd.read_csv(folder + 'chugun_test.csv')
    produv = pd.read_csv(folder + 'produv_test.csv')
    lom = pd.read_csv(folder + 'lom_test.csv')
    plavki = pd.read_csv(folder + 'plavki_test.csv')
    chronom = pd.read_csv(folder + 'chronom_test.csv')
    gas = pd.read_csv(folder + 'gas_test.csv')

    chronom_features = prepare_chronom_features(chronom)
    sip_features = prepare_sip_features(sip)
    chugun_features = prepare_chugun_features(chugun)
    produv_features = prepare_produv_features(produv, chronom)
    lom_features = prepare_lom_features(lom)
    gas_features = prepare_gas_features(gas, chronom)

    X_test = pd.concat([chronom_features, chugun_features, produv_features, lom_features, gas_features], axis=1)
    X_test.columns = [f'feature_{i}' for i in range(X_test.shape[1])]

    X_test = X_test[X.columns]

    ss['TST'] = np.exp(regT.predict(X_test))
    ss['C'] = regC.predict(scaler.transform(X_test.clip(0, 10000).loc[:, X.isna().sum() == 0]))

    ss.to_csv('submission.csv', index=False)

max_stars_repo_name: AV-Borisov/Hackathon-Evraz-AI

max_issues_repo_name: AV-Borisov/Hackathon-Evraz-AI

max_forks_repo_name: AV-Borisov/Hackathon-Evraz-AI


text:
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import math
import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from paddle import ParamAttr
from paddle.nn.initializer import KaimingUniform, Uniform
from paddlex.ppdet.core.workspace import register
from paddlex.ppdet.modeling.heads.centernet_head import ConvLayer

__all__ = ['FairMOTEmbeddingHead']

@register
class FairMOTEmbeddingHead(nn.Layer):
    """
    Args:
        in_channels (int): the channel number of input to FairMOTEmbeddingHead.
        ch_head (int): the channel of features before fed into embedding, 256 by default.
        ch_emb (int): the channel of the embedding feature, 128 by default.
        num_identifiers (int): the number of identifiers, 14455 by default.

    """

    def __init__(self,
                 in_channels,
                 ch_head=256,
                 ch_emb=128,
                 num_identifiers=14455):
        super(FairMOTEmbeddingHead, self).__init__()
        self.reid = nn.Sequential(
            ConvLayer(
                in_channels, ch_head, kernel_size=3, padding=1, bias=True),
            nn.ReLU(),
            ConvLayer(
                ch_head, ch_emb, kernel_size=1, stride=1, padding=0, bias=True))
        param_attr = paddle.ParamAttr(initializer=KaimingUniform())
        bound = 1 / math.sqrt(ch_emb)
        bias_attr = paddle.ParamAttr(initializer=Uniform(-bound, bound))
        self.classifier = nn.Linear(
            ch_emb,
            num_identifiers,
            weight_attr=param_attr,
            bias_attr=bias_attr)
        self.reid_loss = nn.CrossEntropyLoss(ignore_index=-1, reduction='sum')
        # When num_identifiers is 1, emb_scale is set as 1
        self.emb_scale = math.sqrt(2) * math.log(
            num_identifiers - 1) if num_identifiers > 1 else 1

    @classmethod
    def from_config(cls, cfg, input_shape):
        if isinstance(input_shape, (list, tuple)):
            input_shape = input_shape[0]
        return {'in_channels': input_shape.channels}

    def forward(self, feat, inputs):
        reid_feat = self.reid(feat)
        if self.training:
            loss = self.get_loss(reid_feat, inputs)
            return loss
        else:
            reid_feat = F.normalize(reid_feat)
            return reid_feat

    def get_loss(self, feat, inputs):
        index = inputs['index']
        mask = inputs['index_mask']
        target = inputs['reid']
        target = paddle.masked_select(target, mask > 0)
        target = paddle.unsqueeze(target, 1)

        feat = paddle.transpose(feat, perm=[0, 2, 3, 1])
        feat_n, feat_h, feat_w, feat_c = feat.shape
        feat = paddle.reshape(feat, shape=[feat_n, -1, feat_c])
        index = paddle.unsqueeze(index, 2)
        batch_inds = list()
        for i in range(feat_n):
            batch_ind = paddle.full(
                shape=[1, index.shape[1], 1], fill_value=i, dtype='int64')
            batch_inds.append(batch_ind)
        batch_inds = paddle.concat(batch_inds, axis=0)
        index = paddle.concat(x=[batch_inds, index], axis=2)
        feat = paddle.gather_nd(feat, index=index)

        mask = paddle.unsqueeze(mask, axis=2)
        mask = paddle.expand_as(mask, feat)
        mask.stop_gradient = True
        feat = paddle.masked_select(feat, mask > 0)
        feat = paddle.reshape(feat, shape=[-1, feat_c])
        feat = F.normalize(feat)
        feat = self.emb_scale * feat
        logit = self.classifier(feat)
        target.stop_gradient = True
        loss = self.reid_loss(logit, target)
        valid = (target != self.reid_loss.ignore_index)
        valid.stop_gradient = True
        count = paddle.sum((paddle.cast(valid, dtype=np.int32)))
        count.stop_gradient = True
        if count > 0:
            loss = loss / count

        return loss

max_stars_repo_name: follower/PaddleX

max_issues_repo_name: liyhc/PaddleX

max_forks_repo_name: liyhc/PaddleX


text:
import h5py
import numpy as np
from tensorflow.keras.utils import Sequence

from .wb_attack_data_generator import WBAttackGenerator

class WBAttackLoader(Sequence):
    """
    Generator class for tensorflow fit_Generator function.
    This is needed because the Training data is too large to fit into RAM
    The Generator class just ready a batch from the disk incrementally
    """

    def __init__(self, train_data_file, test_data_file, train_indices, test_indices, batch_size,
                 load_in_memory=False, shuffle=True):
        """
        Initialize Loader. Compared to Steffens code much has been removed,
        simply due to the fact that we know certainly that only the last layer with all its components are used
        """
        self.train_data_file = train_data_file
        self.test_data_file = test_data_file
        self.train_indices = train_indices
        self.test_indices = test_indices
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.load_in_memory = load_in_memory
        if self.load_in_memory:
            with h5py.File(self.train_data_file, 'r') as train_file, h5py.File(self.test_data_file, 'r') as test_file:
                self.train_data = self._get_data(None, file=train_file)
                self.test_data = self._get_data(None, file=test_file)
        assert self.get_train_amount() >= max(train_indices), f"train indices contain index out of range {max(train_indices)} > {self.get_train_amount()}"
        assert self.get_test_amount() >= max(test_indices), f"test indices contain index out of range {max(test_indices)} > {self.get_test_amount()}"

    def __len__(self):
        return int(np.floor(len(self.train_indices) * 2 / self.batch_size))

    def _get_data(self, ind, file):
        """
        Read the Data from the File.
        Subsequent index reads must be consecutive (hdfs restriction)
        """
        if ind is None:  # is no index is given, load all data
            return np.array(file[WBAttackGenerator.DS_NAME][:])
        return np.array(file[WBAttackGenerator.DS_NAME][ind])

    def __getitem__(self, idx):
        """
        Get a batch
        """
        start_i = idx * self.batch_size // 2
        end_i = (idx + 1) * self.batch_size // 2
        train_indices = self.train_indices[start_i: end_i]
        test_indices = self.test_indices[start_i: end_i]
        if self.load_in_memory:
            batch_x = []
            batch_y = []
            for train_i, test_i in zip(train_indices, test_indices):
                batch_x.append(self.train_data[train_i])
                batch_y.append(1)
                batch_x.append(self.test_data[test_i])
                batch_y.append(0)
        else:
            with h5py.File(self.train_data_file, 'r') as train_file, h5py.File(self.test_data_file, 'r') as test_file:
                batch_x = []
                batch_y = []
                for train_i, test_i in zip(train_indices, test_indices):
                    batch_x.append(self._get_data(train_i, train_file))
                    batch_y.append(1)
                    batch_x.append(self._get_data(test_i, test_file))
                    batch_y.append(0)
        return np.array(batch_x), np.array(batch_y)

    def on_epoch_end(self):
        """
        Shuffle when epoch ends
        """
        if self.shuffle:
            # Updates indexes after each epoch
            np.random.shuffle(self.train_indices)
            np.random.shuffle(self.test_indices)

    def get_input_size(self):
        """
        Input size of the training data.
        Steffen calculated that by: weights*layers+num_classes+1
        """
        train_file = h5py.File(self.train_data_file, 'r')
        print(train_file['attack_data'].shape)
        return train_file['attack_data'][0].shape[1]

    def get_train_amount(self):
        """
        get amount of training data
        """
        train_file = h5py.File(self.train_data_file, 'r')
        return train_file['attack_data'].shape[0]

    def get_test_amount(self):
        """
        get amount of test data
        """
        test_file = h5py.File(self.test_data_file, 'r')
        return test_file['attack_data'].shape[0]

    def get_indices(self):
        indices = []
        for train_i, test_i in list(zip(self.train_indices, self.test_indices)):
            indices.append([1, train_i])
            indices.append([0, test_i])

        return indices


text:
#Metrics
import torch
import numpy as np
import pandas as pd
from torch.nn import functional as F
from src import data

def site_confusion(y_true, y_pred, site_lists):
    """What proportion of misidentified species come from the same site?
    Args:
        y_true: string values of true labels
        y_pred: string values or predicted labels
        site_lists: list of site labels for each string label taxonID -> sites
    Returns:
        Within site confusion score
    """
    within_site = 0
    cross_site = 0
    for index, value in enumerate(y_pred):
        #If not correctly predicted
        if not value == y_true[index]:
            correct_sites = site_lists[y_true[index]]
            incorrect_site = site_lists[y_pred[index]]

            #Do they co-occur?
            site_overlap = any([site in incorrect_site for site in correct_sites])
            if site_overlap:
                within_site +=1
            else:
                cross_site +=1
        else:
            pass

    #don't divide by zero
    if within_site + cross_site == 0:
        return 0

    #Get proportion of within site error
    proportion_within = within_site/(within_site + cross_site)

    return proportion_within

def genus_confusion(y_true, y_pred, scientific_dict):
    """What proportion of misidentified species come from the same genus?
    Args:
        y_true: taxonID of true labels
        y_pred: taxonID of predicted labels
        scientific_dict: a dict of taxonID -> scientific name
    Returns:
        Within site confusion score
    """
    within_genus = 0
    cross_genus = 0
    for index, value in enumerate(y_pred):
        #If not correctly predicted
        if not value == y_true[index]:
            true_genus = scientific_dict[y_true[index]][0].split()[0]
            pred_genus = scientific_dict[y_pred[index]][0].split()[0]

            if true_genus == pred_genus:
                within_genus +=1
            else:
                cross_genus +=1

    #don't divide by zero
    if within_genus + cross_genus == 0:
        return 0

    #Get proportion of within site error
    proportion_within = within_genus/(within_genus + cross_genus)

    return proportion_within

def novel_prediction(model, csv_file, config):
    """Predict a dataset of species not included in the dataset and get the final activation score before/after softmax"""
    novel_ds = data.TreeDataset(csv_file, image_size=config["image_size"], config=config)

    data_loader = torch.utils.data.DataLoader(
        novel_ds,
        batch_size=config["batch_size"],
        num_workers=config["workers"])

    model.eval()
    top_scores = []
    softmax_scores = []
    individuals = []
    for batch in data_loader:
        individual, inputs, targets = batch
        with torch.no_grad():
            pred = model(inputs["HSI"])
            top_score = pred[np.arange(len(pred)), np.argmax(pred, 1)]
            softmax_layer = F.softmax(pred, dim=1)
            softmax_score = softmax_layer[np.arange(len(softmax_layer)), np.argmax(softmax_layer, 1)]

        individuals.append(individual)
        top_scores.append(top_score)
        softmax_scores.append(softmax_score)

    top_scores = np.concatenate(top_scores)
    individuals = np.concatenate(individuals)
    softmax_scores = np.concatenate(softmax_scores)
    features = pd.DataFrame({"individualID":individuals, "top_score": top_scores,"softmax_score":softmax_scores})

    original = pd.read_csv(csv_file)
    mergeddf = features.merge(original)

    return mergeddf

max_stars_repo_name: henrykironde/DeepTreeAttention

max_issues_repo_name: henrykironde/DeepTreeAttention

max_forks_repo_name: henrykironde/DeepTreeAttention


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import random
import math
import json
import numpy as np
import tensorflow as tf
import h5py

import util

class BridgingModel(object):
  def __init__(self, config):
    self.config = config
    self.context_embeddings = util.EmbeddingDictionary(config["context_embeddings"])
    self.head_embeddings = util.EmbeddingDictionary(config["head_embeddings"], maybe_cache=self.context_embeddings)
    self.char_embedding_size = config["char_embedding_size"]
    self.char_dict = util.load_char_dict(config["char_vocab_path"])
    self.max_span_width = config["max_span_width"]
    self.genres = {g: i for i, g in enumerate(config["genres"])}
    if config["lm_path"]:
      self.lm_file = h5py.File(self.config["lm_path"], "r")
    else:
      self.lm_file = None
    self.lm_layers = self.config["lm_layers"]
    self.lm_size = self.config["lm_size"]
    self.eval_data = None  # Load eval data lazily.
    self.undersampling_probability = self.config["undersampling_probability"]
    self.second_undersampling_probability = self.config["second_undersampling_probability"]
    self.cross_validation_fold = self.config["cross_validation_fold"]
    self.skip_comparative_bridging = 'skip_comparative_bridging' in self.config and self.config['skip_comparative_bridging']

    input_props = []
    input_props.append((tf.float32, [None, None, self.context_embeddings.size]))  # Context embeddings.
    input_props.append((tf.float32, [None, None, self.head_embeddings.size]))  # Head embeddings.
    input_props.append((tf.float32, [None, None, self.lm_size, self.lm_layers]))  # LM embeddings.
    input_props.append((tf.int32, [None, None, None]))  # Character indices.
    input_props.append((tf.int32, [None]))  # Text lengths.
    input_props.append((tf.int32, [None]))  # Speaker IDs.
    input_props.append((tf.int32, []))  # Genre.
    input_props.append((tf.bool, []))  # Is training.
    input_props.append((tf.int32, [None]))  # Gold starts.
    input_props.append((tf.int32, [None]))  # Gold ends.
    input_props.append((tf.int32, [None]))  # Cluster ids.
    input_props.append((tf.int32, [None]))  # Bridging antecedent cluster ids
    input_props.append((tf.int32, [None]))  # IS status 0-DN 1-DO 2-Bridging
    input_props.append((tf.bool, [None]))  # undersampling mask

    self.input_tensors = [tf.placeholder(dtype, shape) for dtype, shape in input_props]

    self.global_step = tf.Variable(0, name="global_step", trainable=False)
    self.reset_global_step = tf.assign(self.global_step, 0)

    self.predictions, self.loss = self.get_predictions_and_loss(*self.input_tensors)
    learning_rate = tf.train.exponential_decay(self.config["learning_rate"], self.global_step,
                                               self.config["decay_frequency"], self.config["decay_rate"],
                                               staircase=True)
    trainable_params = tf.trainable_variables()
    gradients = tf.gradients(self.loss, trainable_params)
    gradients, _ = tf.clip_by_global_norm(gradients, self.config["max_gradient_norm"])
    optimizers = {
      "adam": tf.train.AdamOptimizer,
      "sgd": tf.train.GradientDescentOptimizer
    }
    optimizer = optimizers[self.config["optimizer"]](learning_rate)
    self.train_op = optimizer.apply_gradients(zip(gradients, trainable_params), global_step=self.global_step)

  def restore(self, session):
    # Don't try to restore unused variables from the TF-Hub ELMo module.
    vars_to_restore = [v for v in tf.global_variables() if "module/" not in v.name]
    saver = tf.train.Saver(vars_to_restore)
    checkpoint_path = os.path.join(self.config["log_dir"], "model.max.ckpt")
    print("Restoring from {}".format(checkpoint_path))
    session.run(tf.global_variables_initializer())
    saver.restore(session, checkpoint_path)

  def load_lm_embeddings(self, doc_key):
    if self.lm_file is None:
      return np.zeros([0, 0, self.lm_size, self.lm_layers])
    file_key = doc_key.replace("/", ":")
    group = self.lm_file[file_key]
    num_sentences = len(list(group.keys()))
    sentences = [group[str(i)][...] for i in range(num_sentences)]
    lm_emb = np.zeros([num_sentences, max(s.shape[0] for s in sentences), self.lm_size, self.lm_layers])
    for i, s in enumerate(sentences):
      lm_emb[i, :s.shape[0], :, :] = s
    return lm_emb

  def tensorize_mentions(self, mentions):
    starts, ends = [], []
    for m in mentions:
      starts.append(m[0])
      ends.append(m[1])

    return np.array(starts), np.array(ends)

  def tensorize_example(self, example, is_training):
    if is_training:
      undersampling_probability = self.undersampling_probability if example[
        "main_train"] else self.second_undersampling_probability
    clusters = example["clusters"]
    gold_mentions = sorted(tuple(m) for m in util.flatten(clusters))
    gold_mention_map = {m: i for i, m in enumerate(gold_mentions)}
    cluster_ids = np.zeros(len(gold_mentions))
    ismap = {}
    for cluster_id, cluster in enumerate(clusters):
      for mid, mention in enumerate(cluster):
        cluster_ids[gold_mention_map[tuple(mention)]] = cluster_id + 1
        ismap[(mention[0], mention[1])] = 0 if mid == 0 else 1 #0-DN 1-DO 2-Bridging

    sentences = example["sentences"]
    num_words = sum(len(s) for s in sentences)

    max_sentence_length = max(len(s) for s in sentences)
    max_word_length = max(max(max(len(w) for w in s) for s in sentences), max(self.config["filter_widths"]))
    text_len = np.array([len(s) for s in sentences])
    tokens = [[""] * max_sentence_length for _ in sentences]
    context_word_emb = np.zeros([len(sentences), max_sentence_length, self.context_embeddings.size])
    head_word_emb = np.zeros([len(sentences), max_sentence_length, self.head_embeddings.size])
    char_index = np.zeros([len(sentences), max_sentence_length, max_word_length])
    for i, sentence in enumerate(sentences):
      for j, word in enumerate(sentence):
        tokens[i][j] = word
        context_word_emb[i, j] = self.context_embeddings[word]
        head_word_emb[i, j] = self.head_embeddings[word]
        char_index[i, j, :len(word)] = [self.char_dict[c] for c in word]

    if "speakers" in example:
      speakers = util.flatten(example["speakers"])
      assert num_words == len(speakers)
      speaker_dict = {s: i for i, s in enumerate(set(speakers))}
      speaker_ids = np.array([speaker_dict[s] for s in speakers])
    else:
      speaker_ids = np.zeros(num_words,dtype=np.int32)

    doc_key = example["doc_key"]
    genre = self.genres[doc_key[:2]]
    gold_starts, gold_ends = self.tensorize_mentions(gold_mentions)
    gold_mentions_to_clusters = {(s, e): cid for s, e, cid in zip(gold_starts, gold_ends, cluster_ids)}

    if self.skip_comparative_bridging and 'bridging_pairs_no_comparative' in example:
      #for training ISNOTE and using BASHI as second train corpus, see Hou et al 2020 ACL for more detail
      bridging_map = {(p[0][0], p[0][1]): (p[1][0], p[1][1]) for p in example["bridging_pairs_no_comparative"]}
    else:
      bridging_map = {(p[0][0], p[0][1]): (p[1][0], p[1][1]) for p in example["bridging_pairs"]}
    bridging_ante_cids = np.zeros(len(gold_mentions), np.int32)
    is_status = np.zeros(len(gold_mentions), np.int32)
    us_mask = np.ones(len(gold_mentions), dtype=np.bool)
    mask_cap = np.ones(len(gold_mentions))
    for i, (s, e) in enumerate(zip(gold_starts, gold_ends)):
      if (s, e) in bridging_map:
        bridging_ante_cids[i] = gold_mentions_to_clusters[bridging_map[(s, e)]]
        is_status[i] = 2  # 2-bridging
        mask_cap[i] = 1.0
      else:
        is_status[i] = ismap[(s, e)]  # 0-DN 1-DO
        if is_training:
          mask_cap[i] = undersampling_probability[is_status[i]] * self.config["ntimes_negative_examples"]

    lm_emb = self.load_lm_embeddings(doc_key)
    if is_training:
      return (context_word_emb, head_word_emb, lm_emb, char_index, text_len, speaker_ids, genre, is_training, gold_starts,
    gold_ends, cluster_ids, bridging_ante_cids, is_status,mask_cap)

    return (context_word_emb, head_word_emb, lm_emb, char_index, text_len, speaker_ids, genre, is_training, gold_starts,
    gold_ends, cluster_ids, bridging_ante_cids, is_status, us_mask)

  def truncate_example(self, context_word_emb, head_word_emb, lm_emb, char_index, text_len, speaker_ids, genre,
                       is_training, gold_starts, gold_ends, cluster_ids, bridging_ante_cids, is_status,mask_cap):
    num_words = sum(text_len)
    us_mask = np.random.random(gold_starts.shape[0]) < mask_cap

    if num_words > self.config["max_training_words"]:
      num_sentences = context_word_emb.shape[0]
      max_training_sentences = num_sentences

      while num_words > self.config["max_training_words"]:
        max_training_sentences -= 1
        sentence_offset = random.randint(0, num_sentences - max_training_sentences)
        word_offset = text_len[:sentence_offset].sum()
        num_words = text_len[sentence_offset:sentence_offset + max_training_sentences].sum()

      context_word_emb = context_word_emb[sentence_offset:sentence_offset + max_training_sentences, :, :]
      head_word_emb = head_word_emb[sentence_offset:sentence_offset + max_training_sentences, :, :]
      lm_emb = lm_emb[sentence_offset:sentence_offset + max_training_sentences, :, :, :]
      char_index = char_index[sentence_offset:sentence_offset + max_training_sentences, :, :]
      text_len = text_len[sentence_offset:sentence_offset + max_training_sentences]

      speaker_ids = speaker_ids[word_offset: word_offset + num_words]
      gold_spans = np.logical_and(gold_starts >= word_offset, gold_ends < word_offset + num_words)
      gold_starts = gold_starts[gold_spans] - word_offset
      gold_ends = gold_ends[gold_spans] - word_offset
      cluster_ids = cluster_ids[gold_spans]
      bridging_ante_cids = bridging_ante_cids[gold_spans]
      is_status = is_status[gold_spans]
      us_mask = us_mask[gold_spans]

    return (context_word_emb, head_word_emb, lm_emb, char_index, text_len, speaker_ids, genre, is_training, gold_starts, gold_ends, cluster_ids, bridging_ante_cids, is_status, us_mask)

  def get_candidate_labels(self, candidate_starts, candidate_ends, labeled_starts, labeled_ends, labels,
                           bridging_ante_cids):
    same_start = tf.equal(tf.expand_dims(labeled_starts, 1),
                          tf.expand_dims(candidate_starts, 0))  # [num_labeled, num_candidates]
    same_end = tf.equal(tf.expand_dims(labeled_ends, 1),
                        tf.expand_dims(candidate_ends, 0))  # [num_labeled, num_candidates]
    same_span = tf.logical_and(same_start, same_end)  # [num_labeled, num_candidates]
    candidate_labels = tf.matmul(tf.expand_dims(labels, 0), tf.to_int32(same_span))  # [1, num_candidates]
    candidate_labels = tf.squeeze(candidate_labels, 0)  # [num_candidates]
    candidate_bridging_ante_cids = tf.matmul(tf.expand_dims(bridging_ante_cids, 0),
                                             tf.to_int32(same_span))  # [1, num_candidates]
    candidate_bridging_ante_cids = tf.squeeze(candidate_bridging_ante_cids, 0)  # [num_candidates]
    return candidate_labels, candidate_bridging_ante_cids

  def get_dropout(self, dropout_rate, is_training):
    return 1 - (tf.to_float(is_training) * dropout_rate)

  def distance_pruning(self, top_span_emb, top_span_mention_scores, c):
    k = util.shape(top_span_emb, 0)
    top_antecedent_offsets = tf.tile(tf.expand_dims(tf.range(c) + 1, 0), [k, 1])  # [k, c]
    raw_top_antecedents = tf.expand_dims(tf.range(k), 1) - top_antecedent_offsets  # [k, c]
    top_antecedents_mask = raw_top_antecedents >= 0  # [k, c]
    top_antecedents = tf.maximum(raw_top_antecedents, 0)  # [k, c]

    top_fast_antecedent_scores = tf.expand_dims(top_span_mention_scores, 1) + tf.gather(top_span_mention_scores,
                                                                                top_antecedents)  # [k, c]
    top_fast_antecedent_scores += tf.log(tf.to_float(top_antecedents_mask))  # [k, c]
    return top_antecedents, top_antecedents_mask, top_fast_antecedent_scores, top_antecedent_offsets

  def get_predictions_and_loss(self, context_word_emb, head_word_emb, lm_emb, char_index, text_len, speaker_ids,
                               genre, is_training, gold_starts, gold_ends, cluster_ids, bridging_ante_cids, is_status,
                               us_mask):
    self.dropout = self.get_dropout(self.config["dropout_rate"], is_training)
    self.lexical_dropout = self.get_dropout(self.config["lexical_dropout_rate"], is_training)
    self.lstm_dropout = self.get_dropout(self.config["lstm_dropout_rate"], is_training)

    num_sentences = tf.shape(context_word_emb)[0]
    max_sentence_length = tf.shape(context_word_emb)[1]

    context_emb_list = [context_word_emb]
    head_emb_list = [head_word_emb]

    if self.config["char_embedding_size"] > 0:
      char_emb = tf.gather(
        tf.get_variable("char_embeddings", [len(self.char_dict), self.config["char_embedding_size"]]),
        char_index)  # [num_sentences, max_sentence_length, max_word_length, emb]
      flattened_char_emb = tf.reshape(char_emb, [num_sentences * max_sentence_length, util.shape(char_emb, 2),
                                                 util.shape(char_emb,
                                                            3)])  # [num_sentences * max_sentence_length, max_word_length, emb]
      flattened_aggregated_char_emb = util.cnn(flattened_char_emb, self.config["filter_widths"],
                                               self.config["filter_size"])  # [num_sentences * max_sentence_length, emb]
      aggregated_char_emb = tf.reshape(flattened_aggregated_char_emb, [num_sentences, max_sentence_length,
                                                                       util.shape(flattened_aggregated_char_emb,
                                                                                1)])  # [num_sentences, max_sentence_length, emb]
      context_emb_list.append(aggregated_char_emb)
      head_emb_list.append(aggregated_char_emb)

    if self.lm_file:
      lm_emb_size = util.shape(lm_emb, 2)
      lm_num_layers = util.shape(lm_emb, 3)
      with tf.variable_scope("lm_aggregation"):
        self.lm_weights = tf.nn.softmax(
          tf.get_variable("lm_scores", [lm_num_layers], initializer=tf.constant_initializer(0.0)))
        self.lm_scaling = tf.get_variable("lm_scaling", [], initializer=tf.constant_initializer(1.0))
      flattened_lm_emb = tf.reshape(lm_emb, [num_sentences * max_sentence_length * lm_emb_size, lm_num_layers])
      flattened_aggregated_lm_emb = tf.matmul(flattened_lm_emb, tf.expand_dims(self.lm_weights,
                                                                               1))  # [num_sentences * max_sentence_length * emb, 1]
      aggregated_lm_emb = tf.reshape(flattened_aggregated_lm_emb, [num_sentences, max_sentence_length, lm_emb_size])
      aggregated_lm_emb *= self.lm_scaling
      context_emb_list.append(aggregated_lm_emb)

    context_emb = tf.concat(context_emb_list, 2)  # [num_sentences, max_sentence_length, emb]
    head_emb = tf.concat(head_emb_list, 2)  # [num_sentences, max_sentence_length, emb]
    context_emb = tf.nn.dropout(context_emb, self.lexical_dropout)  # [num_sentences, max_sentence_length, emb]
    head_emb = tf.nn.dropout(head_emb, self.lexical_dropout)  # [num_sentences, max_sentence_length, emb]

    text_len_mask = tf.sequence_mask(text_len, maxlen=max_sentence_length)  # [num_sentence, max_sentence_length]

    context_outputs = self.lstm_contextualize(context_emb, text_len, text_len_mask)  # [num_words, emb]

    genre_emb = tf.gather(tf.get_variable("genre_embeddings", [len(self.genres), self.config["feature_size"]]),
                          genre)  # [emb]

    flattened_head_emb = self.flatten_emb_by_sentence(head_emb, text_len_mask)  # [num_words]

    top_span_starts = gold_starts
    top_span_ends = gold_ends
    top_span_cluster_ids = cluster_ids

    top_span_emb = self.get_span_emb(flattened_head_emb, context_outputs, top_span_starts, top_span_ends)
    top_span_mention_scores = tf.zeros_like(gold_starts, dtype=tf.float32)  # [k]

    top_span_speaker_ids = tf.gather(speaker_ids, top_span_starts)
    top_span_bridging_ante_cids = bridging_ante_cids
    top_us_mask = us_mask
    top_is_status = is_status

    k = util.shape(top_span_starts, 0)

    c = tf.minimum(self.config["max_top_antecedents"], k)

    top_antecedents, top_antecedents_mask, top_fast_antecedent_scores, top_antecedent_offsets = self.distance_pruning(
      top_span_emb, top_span_mention_scores, c)

    top_antecedent_emb = tf.gather(top_span_emb, top_antecedents)  # [k, c, emb]

    pair_emb = self.get_pair_embeddings(top_span_emb, top_antecedents, top_antecedent_emb, top_antecedent_offsets,
                                        top_span_speaker_ids, genre_emb)  # [k, c,emb]

    top_antecedent_cluster_ids = tf.gather(top_span_cluster_ids, top_antecedents)  # [k, c]
    top_antecedent_cluster_ids += tf.to_int32(tf.log(tf.to_float(top_antecedents_mask)))  # [k, c]

    shared_depth = 0
    if self.config["shared_depth"] > 0:
      flattened_pair_emb = tf.reshape(pair_emb, [k * c, util.shape(pair_emb, 2)])
      shared_depth = min(self.config["shared_depth"], self.config["ffnn_depth"])
      for i in range(shared_depth):
        hidden_weights = tf.get_variable("shared_hidden_weights_{}".format(i),
                                         [util.shape(flattened_pair_emb, 1), self.config["ffnn_size"]])
        hidden_bias = tf.get_variable("shared_hidden_bias_{}".format(i), [self.config["ffnn_size"]])
        flattened_pair_emb = tf.nn.relu(tf.nn.xw_plus_b(flattened_pair_emb, hidden_weights, hidden_bias))
        flattened_pair_emb = tf.nn.dropout(flattened_pair_emb, self.dropout)
      pair_emb = tf.reshape(flattened_pair_emb, [k, c, self.config["ffnn_size"]])

    ante_score_list = []
    pairwise_label_list = []
    dummy_scores = tf.zeros([k, 1])  # [k, 1]
    ante_score_list.append(dummy_scores)

    with tf.variable_scope("slow_bridging_scores"):
      slow_bridging_scores = util.ffnn(pair_emb, self.config["ffnn_depth"] - shared_depth, self.config["ffnn_size"], 1,
                                       self.dropout)  # [k, c, 1]
      slow_bridging_scores = tf.squeeze(slow_bridging_scores, 2)  # [k, c]
      top_bridging_scores = slow_bridging_scores + top_fast_antecedent_scores
      ante_score_list.append(top_bridging_scores)

    bridging_cluster_indicator = tf.equal(top_antecedent_cluster_ids,
                                          tf.expand_dims(top_span_bridging_ante_cids, 1))  # [k, c]
    non_dummy_bridging_indicator = tf.expand_dims(top_span_bridging_ante_cids > 0, 1)  # [k, 1]

    bridging_pairwise_labels = tf.logical_and(bridging_cluster_indicator, non_dummy_bridging_indicator)  # [k, c]
    pairwise_label_list.append(bridging_pairwise_labels)

    if self.config["train_with_coref"]:
      with tf.variable_scope("slow_coreference_scores"):
        slow_coref_scores = util.ffnn(pair_emb, self.config["ffnn_depth"] - shared_depth, self.config["ffnn_size"], 1,
                                      self.dropout)  # [k, c, 1]
        slow_coref_scores = tf.squeeze(slow_coref_scores, 2)  # [k, c]
        top_coref_scores = slow_coref_scores + top_fast_antecedent_scores
        ante_score_list.append(top_coref_scores)

      coref_cluster_indicator = tf.equal(top_antecedent_cluster_ids, tf.expand_dims(top_span_cluster_ids, 1))  # [k,c]

      non_dummy_coref_indicator = tf.expand_dims(top_span_cluster_ids > 0, 1)  # [k,1]

      coref_pairwise_labels = tf.logical_and(coref_cluster_indicator, non_dummy_coref_indicator)  # [k,c]
      pairwise_label_list.append(coref_pairwise_labels)

    top_antecedent_scores = tf.concat(ante_score_list, 1)  # [k, c + 1] or [k, 2*c+1]
    pairwise_labels = tf.concat(pairwise_label_list, 1)  # [k,c] or [k,2*c]

    top_antecedent_scores = tf.boolean_mask(top_antecedent_scores, top_us_mask)
    pairwise_labels = tf.boolean_mask(pairwise_labels, top_us_mask)

    dummy_labels = tf.logical_not(tf.reduce_any(pairwise_labels, 1, keepdims=True))  # [k, 1]
    pairwise_labels = tf.concat([dummy_labels, pairwise_labels], 1)  # [k, c + 1] or [k,2*c+1]

    loss = self.softmax_loss(top_antecedent_scores, pairwise_labels)
    loss = tf.reduce_sum(loss)

    if self.config["use_gold_bridging_anaphora"]:
      bridging_mask = tf.equal(top_is_status, 2)  # bridging
      top_span_starts = tf.boolean_mask(top_span_starts, bridging_mask)
      top_span_ends = tf.boolean_mask(top_span_ends, bridging_mask)
      top_antecedents = tf.boolean_mask(top_antecedents, bridging_mask)
      top_antecedent_scores_output = tf.boolean_mask(top_bridging_scores, bridging_mask)
    elif self.config["remove_coref_anaphora"]:
      bridging_mask = tf.not_equal(top_is_status, 1)  # DO
      top_span_starts = tf.boolean_mask(top_span_starts, bridging_mask)
      top_span_ends = tf.boolean_mask(top_span_ends, bridging_mask)
      top_antecedents = tf.boolean_mask(top_antecedents, bridging_mask)
      top_antecedent_scores_output = tf.boolean_mask(tf.concat([dummy_scores, top_bridging_scores], 1), bridging_mask)
    else:
      top_antecedent_scores_output = top_antecedent_scores

    return [top_span_starts, top_span_ends, top_span_cluster_ids, top_antecedents, top_antecedent_scores_output], loss

  def get_span_emb(self, head_emb, context_outputs, span_starts, span_ends):
    span_emb_list = []

    span_start_emb = tf.gather(context_outputs, span_starts)  # [k, emb]
    span_emb_list.append(span_start_emb)

    span_end_emb = tf.gather(context_outputs, span_ends)  # [k, emb]
    span_emb_list.append(span_end_emb)

    span_width = tf.minimum(1 + span_ends - span_starts, self.config["max_span_width"])  # [k]

    if self.config["use_features"]:
      span_width_index = span_width - 1  # [k]
      span_width_emb = tf.gather(
        tf.get_variable("span_width_embeddings", [self.config["max_span_width"], self.config["feature_size"]]),
        span_width_index)  # [k, emb]
      span_width_emb = tf.nn.dropout(span_width_emb, self.dropout)
      span_emb_list.append(span_width_emb)

    if self.config["model_heads"]:
      span_indices = tf.expand_dims(tf.range(self.config["max_span_width"]), 0) + tf.expand_dims(span_starts,
                                                                                1)  # [k, max_span_width]
      span_indices = tf.minimum(util.shape(context_outputs, 0) - 1, span_indices)  # [k, max_span_width]
      span_text_emb = tf.gather(head_emb, span_indices)  # [k, max_span_width, emb]
      with tf.variable_scope("head_scores"):
        self.head_scores = util.projection(context_outputs, 1)  # [num_words, 1]
      span_head_scores = tf.gather(self.head_scores, span_indices)  # [k, max_span_width, 1]
      span_mask = tf.expand_dims(tf.sequence_mask(span_width, self.config["max_span_width"], dtype=tf.float32),
                                 2)  # [k, max_span_width, 1]
      span_head_scores += tf.log(span_mask)  # [k, max_span_width, 1]
      span_attention = tf.nn.softmax(span_head_scores, 1)  # [k, max_span_width, 1]
      span_head_emb = tf.reduce_sum(span_attention * span_text_emb, 1)  # [k, emb]
      span_emb_list.append(span_head_emb)

    span_emb = tf.concat(span_emb_list, 1)  # [k, emb]
    return span_emb  # [k, emb]

  def softmax_loss(self, antecedent_scores, antecedent_labels):
    gold_scores = antecedent_scores + tf.log(tf.to_float(antecedent_labels))  # [k, max_ant + 1]
    marginalized_gold_scores = tf.reduce_logsumexp(gold_scores, [1])  # [k]
    log_norm = tf.reduce_logsumexp(antecedent_scores, [1])  # [k]
    return log_norm - marginalized_gold_scores  # [k]

  def bucket_distance(self, distances):
    """
    Places the given values (designed for distances) into 10 semi-logscale buckets:
    [0, 1, 2, 3, 4, 5-7, 8-15, 16-31, 32-63, 64+].
    """
    logspace_idx = tf.to_int32(tf.floor(tf.log(tf.to_float(distances)) / math.log(2))) + 3
    use_identity = tf.to_int32(distances <= 4)
    combined_idx = use_identity * distances + (1 - use_identity) * logspace_idx
    return tf.clip_by_value(combined_idx, 0, 9)

  def get_pair_embeddings(self, top_span_emb, top_antecedents, top_antecedent_emb, top_antecedent_offsets,
                          top_span_speaker_ids, genre_emb):
    k = util.shape(top_span_emb, 0)
    c = util.shape(top_antecedents, 1)

    feature_emb_list = []

    if self.config["use_metadata"]:
      top_antecedent_speaker_ids = tf.gather(top_span_speaker_ids, top_antecedents)  # [k, c]
      same_speaker = tf.equal(tf.expand_dims(top_span_speaker_ids, 1), top_antecedent_speaker_ids)  # [k, c]
      speaker_pair_emb = tf.gather(tf.get_variable("same_speaker_emb", [2, self.config["feature_size"]]),
                                   tf.to_int32(same_speaker))  # [k, c, emb]
      feature_emb_list.append(speaker_pair_emb)

      tiled_genre_emb = tf.tile(tf.expand_dims(tf.expand_dims(genre_emb, 0), 0), [k, c, 1])  # [k, c, emb]
      feature_emb_list.append(tiled_genre_emb)

    if self.config["use_features"]:
      antecedent_distance_buckets = self.bucket_distance(top_antecedent_offsets)  # [k, c]
      antecedent_distance_emb = tf.gather(tf.get_variable("antecedent_distance_emb", [10, self.config["feature_size"]]),
                                          antecedent_distance_buckets)  # [k, c]
      feature_emb_list.append(antecedent_distance_emb)

    feature_emb = tf.concat(feature_emb_list, 2)  # [k, c, emb]
    feature_emb = tf.nn.dropout(feature_emb, self.dropout)  # [k, c, emb]

    target_emb = tf.expand_dims(top_span_emb, 1)  # [k, 1, emb]
    similarity_emb = top_antecedent_emb * target_emb  # [k, c, emb]
    target_emb = tf.tile(target_emb, [1, c, 1])  # [k, c, emb]

    pair_emb = tf.concat([target_emb, top_antecedent_emb, similarity_emb, feature_emb], 2)  # [k, c, emb]

    return pair_emb

  def flatten_emb_by_sentence(self, emb, text_len_mask):
    num_sentences = tf.shape(emb)[0]
    max_sentence_length = tf.shape(emb)[1]

    emb_rank = len(emb.get_shape())
    if emb_rank == 2:
      flattened_emb = tf.reshape(emb, [num_sentences * max_sentence_length])
    elif emb_rank == 3:
      flattened_emb = tf.reshape(emb, [num_sentences * max_sentence_length, util.shape(emb, 2)])
    else:
      raise ValueError("Unsupported rank: {}".format(emb_rank))
    return tf.boolean_mask(flattened_emb, tf.reshape(text_len_mask, [num_sentences * max_sentence_length]))

  def lstm_contextualize(self, text_emb, text_len, text_len_mask):
    num_sentences = tf.shape(text_emb)[0]

    current_inputs = text_emb  # [num_sentences, max_sentence_length, emb]

    for layer in range(self.config["contextualization_layers"]):
      with tf.variable_scope("layer_{}".format(layer)):
        with tf.variable_scope("fw_cell"):
          cell_fw = util.CustomLSTMCell(self.config["contextualization_size"], num_sentences, self.lstm_dropout)
        with tf.variable_scope("bw_cell"):
          cell_bw = util.CustomLSTMCell(self.config["contextualization_size"], num_sentences, self.lstm_dropout)
        state_fw = tf.contrib.rnn.LSTMStateTuple(tf.tile(cell_fw.initial_state.c, [num_sentences, 1]),
                                                 tf.tile(cell_fw.initial_state.h, [num_sentences, 1]))
        state_bw = tf.contrib.rnn.LSTMStateTuple(tf.tile(cell_bw.initial_state.c, [num_sentences, 1]),
                                                 tf.tile(cell_bw.initial_state.h, [num_sentences, 1]))

        (fw_outputs, bw_outputs), _ = tf.nn.bidirectional_dynamic_rnn(
          cell_fw=cell_fw,
          cell_bw=cell_bw,
          inputs=current_inputs,
          sequence_length=text_len,
          initial_state_fw=state_fw,
          initial_state_bw=state_bw)

        text_outputs = tf.concat([fw_outputs, bw_outputs], 2)  # [num_sentences, max_sentence_length, emb]
        text_outputs = tf.nn.dropout(text_outputs, self.lstm_dropout)
        if layer > 0:
          highway_gates = tf.sigmoid(
            util.projection(text_outputs, util.shape(text_outputs, 2)))  # [num_sentences, max_sentence_length, emb]
          text_outputs = highway_gates * text_outputs + (1 - highway_gates) * current_inputs
        current_inputs = text_outputs

    return self.flatten_emb_by_sentence(text_outputs, text_len_mask)

  def get_predicted_bridging_pairs(self, predictions):
    top_span_starts, top_span_ends, top_span_cluster_ids, top_antecedents, top_antecedent_scores = predictions
    pred_bridging_pairs = set()
    pred_bridging_anaphora = set()
    c = np.shape(top_antecedents)[1]  # c
    pred_ante_indices = np.argmax(top_antecedent_scores, axis=1) - (
      0 if self.config["use_gold_bridging_anaphora"] else 1)

    for i, index in enumerate(pred_ante_indices):
      if self.config["use_gold_bridging_anaphora"] or (index < c and index >= 0):
        pred_bridging_pairs.add((top_span_starts[i], top_span_ends[i], top_span_cluster_ids[top_antecedents[i, index]]))
        pred_bridging_anaphora.add((top_span_starts[i], top_span_ends[i]))

    return pred_bridging_pairs, pred_bridging_anaphora

  def get_gold_bridging_pairs(self, gold_starts, gold_ends, cluster_ids, bridgings,pred_bridging_pairs):
    bridging_map = {}
    mention2cid = {(s, e): cid for s, e, cid in zip(gold_starts, gold_ends, cluster_ids)}
    gold_bridging_pairs = set()
    gold_bridging_anaphora = set()
    for p in bridgings:
      ana = (p[0][0], p[0][1])
      ant_cid = mention2cid[(p[1][0], p[1][1])]
      if not ana in bridging_map:
        bridging_map[ana] = set()
      bridging_map[ana].add(ant_cid)

    for ana in bridging_map:
      gold_bridging_anaphora.add(ana)
      s, e = ana
      ant_cid = list(bridging_map[ana])
      if len(ant_cid) > 1:
        cid = ant_cid[0]
        for c in ant_cid:
          if (s, e, c) in pred_bridging_pairs:
            # we follow hou et al to count bridgings with multi-antecedent as correct
            # as long as any gold bridging ant is recovered (only for BASHI) (only 15 multi-ant)
            # The impact on results are very small e.g. 0 ~ 0.2
            cid = c
        gold_bridging_pairs.add((s, e, cid))
      else:
        gold_bridging_pairs.add((s, e, ant_cid[0]))
    return gold_bridging_pairs, gold_bridging_anaphora

  def load_eval_data(self, eval_fold=-1, num_fold=-1):
    if self.eval_data is None:
      def load_line(line):
        example = json.loads(line)
        return self.tensorize_example(example, is_training=False), example
      if num_fold > 1: #cross validation
        with open(self.config["train_path"]) as f:
          self.eval_data = [load_line(l) for i, l in enumerate(f.readlines()) if i % num_fold == eval_fold]
      else:
        with open(self.config["eval_path"]) as f:
          self.eval_data = [load_line(l) for l in f.readlines()]
      print("Loaded {} eval examples.".format(len(self.eval_data)))

  def evaluate(self, session, eval_fold=-1, num_fold=-1, is_final_test=False):
    self.load_eval_data(eval_fold, num_fold)

    if "eval_on_test_part_only" in self.config and self.config["eval_on_test_part_only"]:
      eval_on_test_part_only = True
      print("Evaluate on the test part only!!!!")
    else:
      eval_on_test_part_only = False

    if num_fold > 1 and is_final_test:
      print("Evaluating %d/%d fold." % (eval_fold+1, num_fold))
    tp, fn, fp = 0, 0, 0
    tpa, fna, fpa = 0, 0, 0

    for example_num, (tensorized_example, example) in enumerate(self.eval_data):
      _, _, _, _, _, _, _, _, gold_starts, gold_ends, cluster_ids, bridging_ante_cids, _, _,  = tensorized_example
      feed_dict = {i: t for i, t in zip(self.input_tensors, tensorized_example)}
      predictions = session.run(self.predictions, feed_dict=feed_dict)

      pred_bridging_pairs, pred_bridging_anaphora = self.get_predicted_bridging_pairs(predictions)
      if self.config["has_multi_bridging_ant"]:
        #we follow hou et al to count birdings with multi-antecedent as correct
        # as long as any gold bridging ant is recovered (only for BASHI)
        gold_bridging_pairs, gold_bridging_anaphora = self.get_gold_bridging_pairs(gold_starts, gold_ends, cluster_ids,
                                                                                example["bridging_pairs"],
                                                                                pred_bridging_pairs)
      else:
        gold_bridging_pairs = set(
          [(s, e, cid) for s, e, cid in zip(gold_starts, gold_ends, bridging_ante_cids) if cid > 0])
        gold_bridging_anaphora = set(
          [(s, e) for s, e, cid in zip(gold_starts, gold_ends, bridging_ante_cids) if cid > 0])

      add2eval = True
      if eval_on_test_part_only and not example["doc_key"].endswith('_test'):
        add2eval = False
      if add2eval:
        tp += len(gold_bridging_pairs & pred_bridging_pairs)
        fn += len(gold_bridging_pairs - pred_bridging_pairs)
        fp += len(pred_bridging_pairs - gold_bridging_pairs)

        tpa += len(gold_bridging_anaphora & pred_bridging_anaphora)
        fna += len(gold_bridging_anaphora - pred_bridging_anaphora)
        fpa += len(pred_bridging_anaphora - gold_bridging_anaphora)

      if example_num % 10 == 0:
        print("Evaluated {}/{} examples.".format(example_num + 1, len(self.eval_data)))

    bridging_recall = 0.0 if tp == 0 else float(tp) / (tp + fn)
    bridging_precision = 0.0 if tp == 0 else float(tp) / (tp + fp)
    bridging_f1 = 0.0 if bridging_precision == 0.0 else 2.0 * bridging_recall * bridging_precision / (
          bridging_recall + bridging_precision)

    bridging_anaphora_recall = 0.0 if tpa == 0 else float(tpa) / (tpa + fna)
    bridging_anaphora_precision = 0.0 if tpa == 0 else float(tpa) / (tpa + fpa)
    bridging_anaphora_f1 = 0.0 if bridging_anaphora_precision == 0.0 else 2.0 * bridging_anaphora_recall * bridging_anaphora_precision / (
        bridging_anaphora_recall + bridging_anaphora_precision)

    print("Bridging anaphora detection F1: {:.2f}%".format(bridging_anaphora_f1 * 100))
    print("Bridging anaphora detection recall: {:.2f}%".format(bridging_anaphora_recall * 100))
    print("Bridging anaphora detection precision: {:.2f}%".format(bridging_anaphora_precision * 100))

    print("Bridging F1: {:.2f}%".format(bridging_f1 * 100))
    print("Bridging recall: {:.2f}%".format(bridging_recall * 100))
    print("Bridging precision: {:.2f}%".format(bridging_precision * 100))

    summary_dict = {}
    summary_dict["Bridging anaphora detection F1"] = bridging_anaphora_f1
    summary_dict["Bridging anaphora detection recall"] = bridging_anaphora_recall
    summary_dict["Bridging anaphora detection precision"] = bridging_anaphora_precision

    summary_dict["Bridging F1"] = bridging_f1
    summary_dict["Bridging recall"] = bridging_recall
    summary_dict["Bridging precision"] = bridging_precision
    f1 = bridging_f1

    if is_final_test:
      return tp, fn, fp, tpa, fna, fpa
    return util.make_summary(summary_dict), f1 * 100

max_stars_repo_name: juntaoy/dali-bridging

max_issues_repo_name: juntaoy/dali-bridging

max_forks_repo_name: juntaoy/dali-bridging


text:
import pickle

import numpy as np

from elfi.client import ClientBase
from elfi.examples import ma2
from elfi.executor import Executor
from elfi.model.elfi_model import ComputationContext

def test_pickle_ma2():
    m = ma2.get_model()
    d = m.get_reference('d')

    np.random.seed(0)
    res1 = d.generate(10)

    serialized = pickle.dumps(m)
    m = pickle.loads(serialized)
    d = m.get_reference('d')

    np.random.seed(0)
    res2 = d.generate(10)

    assert np.array_equal(res1, res2)

def test_pickle_ma2_compiled_and_loaded(ma2):
    compiled = ClientBase.compile(ma2.source_net, ['d'])
    loaded = ClientBase.load_data(compiled, ComputationContext(), 0)

    np.random.seed(0)
    result = Executor.execute(loaded)
    res1 = result['d']

    serialized = pickle.dumps(loaded)
    loaded = pickle.loads(serialized)

    np.random.seed(0)
    result = Executor.execute(loaded)
    res2 = result['d']

    assert np.array_equal(res1, res2)

max_stars_repo_name: diadochos/elfi

max_issues_repo_name: diadochos/elfi

max_forks_repo_name: diadochos/elfi


text:
# Data management Libraries
import pandas as pd
import numpy as np

# Import spotipy library
import spotipy
from spotipy.oauth2 import SpotifyClientCredentials, SpotifyOAuth

# Utilities
from time import sleep
import ast

def see_all():
    # Alongate the view on DataFrames

    pd.set_option('display.max_rows', 1000)
    pd.set_option('display.max_columns', 1000)
    pd.set_option('display.width', 1000)

class User:

    def __init__(self, client_ID, client_secret):

        self.id = client_ID
        self.secret = client_secret

        self.renew_sp()

    def renew_sp(self):

        scope = 'playlist-read-private'

        self.sp = spotipy.Spotify(auth_manager=SpotifyOAuth(client_id=self.id,
                                                            client_secret=self.secret,
                                                            redirect_uri="http://localhost:8888/callback",
                                                            scope=scope))

    def read_playlist(self, playlist_df, results):

        # Loops through all playlists saved by the user
        for i, item in enumerate(results):
            # Permission to access user playlists
            response = self.sp.playlist_items(item['uri'],
                                              offset=0,
                                              fields='items.track.id,total',
                                              additional_types=['track'])

            # Converts tracks of playlist dictionary to dataframe
            playlist_tracks = pd.DataFrame.from_dict(response["items"])

            # track_id dictionary to string
            playlist_tracks["track_id"] = playlist_tracks["track"].apply(lambda x: f"spotify:track:{x['id']}")
            playlist_tracks = playlist_tracks.drop(["track"], axis=1)

            # Adds playlist name and creator to the dataframe
            playlist_tracks["playlist_name"] = item['name']
            playlist_tracks["playlist_creator"] = item['owner']['display_name']

            # Concats current loop playlist dataframe with main playlists dataframe
            playlist_df = pd.concat([playlist_df, playlist_tracks]).reset_index(drop=True)

        return playlist_df

    def get_playlist(self):

        results = self.sp.current_user_playlists(limit=50)

        # Creates empty dataframe to be populated by user tracks in playlist
        playlist_df = pd.DataFrame()

        playlist_df = self.read_playlist(playlist_df, results['items'])

        self.playlist_df = playlist_df

    def target_playlist(self, playlist_id):

        # Creates empty dataframe to be populated by user tracks in playlist
        self.target_playlist_df = pd.DataFrame()

        for plyst in playlist_id:

            results = self.sp.playlist(plyst, fields=None, market=None, additional_types=('track', ))

            self.target_playlist_df = self.read_playlist(self.target_playlist_df, [results])

    def level_2_info(self, track_urn):
        # Performs a more indepth analysis of a song

        self.counter_error = 0

        try:

            analysis = self.sp.audio_analysis(track_urn)
            track = self.sp.track(track_urn)
            track_features = self.sp.audio_features(track_urn)

            duration_s = analysis["track"]["duration"]
            loudness_db = analysis["track"]["loudness"]
            tempo_bpm = analysis["track"]["tempo"]
            time_signature = analysis["track"]["time_signature"]
            key = analysis["track"]["key"]
            mode = analysis["track"]["mode"]

            danceability = track_features[0]["danceability"]
            energy = track_features[0]["energy"]
            speechiness = track_features[0]["speechiness"]
            acousticness = track_features[0]["acousticness"]
            instrumentalness = track_features[0]["instrumentalness"]
            liveness = track_features[0]["liveness"]
            valence = track_features[0]["valence"]

            explicit = track["explicit"]
            popularity = track["popularity"]

            album_id = track['album']['id']
            album = self.sp.album(album_id)
            album_genres = album["genres"]

            artist_id = [artist['id'] for artist in track['artists']]
            artist = self.sp.artist(artist_id[0])

            artist_genres = artist['genres']
            artist_followers = artist['followers']['total']
            artist_popularity = artist['popularity']

        except:

            self.renew_sp()

            duration_s = " "
            loudness_db = " "
            tempo_bpm = " "
            time_signature = " "
            key = " "
            mode = " "

            danceability = " "
            energy = " "
            speechiness = " "
            acousticness = " "
            instrumentalness = " "
            liveness = " "
            valence = " "

            explicit = " "
            popularity = " "

            artist_genres = " "
            artist_followers = " "
            artist_popularity = " "

            if self.counter_error == (self.counter - 1):

                sleep(10)
                print("Error occured, I'll take a 10 second nap")

            else:
                print("Error occured, I'll renew the permissions")

            self.counter_error = self.counter

        self.counter += 1

        return f"{duration_s}//{loudness_db}//{tempo_bpm}//{time_signature}//{key}//{mode}//{danceability}//{energy}//{speechiness}//{acousticness}//{instrumentalness}//{liveness}//{valence}//{explicit}//{popularity}//{artist_popularity}//{artist_genres}//{artist_followers}"

    def analysis(self, playlist_df):
        # Analysis of every song from a given playlist

        self.counter = 0
        while_counter = 0

        # Search for the missing tracks of the playlist and deletes them
        playlist_df = playlist_df.loc[playlist_df["track_id"] != "spotify:track:None"].reset_index(drop=True)

        playlist_df["analysis"] = playlist_df["track_id"].apply(lambda x: self.level_2_info(x))

        playlist_df[["duration_s", "loudness_db", "tempo_bpm",
                     "time_signature", "key", "mode",
                     "danceability", "energy", "speechiness",
                     "acousticness", "instrumentalness",
                     "liveness", "valence", "explicit",
                     "popularity", "artist_popularity",
                     "artist_genres", "artist_followers"]] = playlist_df['analysis'].str.split('//', expand=True)

        playlist_df = playlist_df.drop(["analysis"], axis=1)

        reloop_data = playlist_df.loc[playlist_df["duration_s"] == " "].reset_index(drop=True)

        if len(reloop_data) > 0:

            while len(reloop_data) > 0:

                level_2_data = playlist_df.loc[playlist_df["duration_s"] != " "].reset_index(drop=True)
                reloop_data = playlist_df.loc[playlist_df["duration_s"] == " "].reset_index(drop=True)

                reloop_data = reloop_data[['track_id', 'playlist_name', 'playlist_creator']]

                reloop_data["analysis"] = reloop_data["track_id"].apply(lambda x: self.level_2_info(x))

                reloop_data[["duration_s", "loudness_db", "tempo_bpm",
                             "time_signature", "key", "mode",
                             "danceability", "energy", "speechiness",
                             "acousticness", "instrumentalness",
                             "liveness", "valence", "explicit",
                             "popularity", "artist_popularity",
                             "artist_genres", "artist_followers"]] = reloop_data['analysis'].str.split('//',expand=True)

                reloop_data = reloop_data.drop(["analysis"], axis=1)

                playlist_df = pd.concat([level_2_data, reloop_data]).reset_index(drop=True)

                while_counter += 1
                print(while_counter)
                print(reloop_data)

                if while_counter > 10:
                    break

        playlist_df = playlist_df.loc[playlist_df["duration_s"] != " "].reset_index(drop=True)

        return playlist_df


text:
import os

from pynwb.behavior import Position, SpatialSeries
from pynwb.ophys import Fluorescence, RoiResponseSeries, OpticalChannel, \
    TwoPhotonSeries, ImageSegmentation
from pynwb.image import ImageSeries
from pynwb import NWBHDF5IO, NWBFile
from h5py import File
import numpy as np
from datetime import datetime

base_dir = '/Users/bendichter/Desktop/Schnitzer/data/Example Data'

nwbfile = NWBFile(session_start_time=datetime(1900, 1, 1),
                  session_description=' ', identifier='m655_D11_S1')

nwbfile.add_acquisition(ImageSeries(name='video',
                                    format='external',
                                    external_file=['m655_D11_S1.avi'],
                                    starting_time=0.0, rate=np.nan,
                                    starting_frame=[0],
                                    dimension=[250, 250]))

centroid_fname = 'm655_D11_S1_centroids.mat'
pos_path = os.path.join(base_dir, centroid_fname)

with File(pos_path, 'r') as file:
    pos_data = np.array(file['c']).T
spatial_series = SpatialSeries(name='position', data=pos_data, starting_time=0.0,
                               rate=5.0, units='unknown', reference_frame='unknown')

pos = Position(name='position', spatial_series=spatial_series, starting_time=0.0,
               rate=np.nan)

module_pos = nwbfile.create_processing_module(name='position', description='position')

module_pos.add_container(pos)

data_names = ['cellImages', 'cellTraces', 'centroids', 'eventTimes']
mat_data = {}
mat_output_fpath = os.path.join(base_dir, 'm655_D11_S1_output.mat')
with File(mat_output_fpath, 'r') as file:
    for name in data_names:
        mat_data[name] = file['output'][name][:]

optical_channel = OpticalChannel('my channel', 'description', np.nan)

imaging_plane = nwbfile.create_imaging_plane(
    'my_imgpln', optical_channel,
    'description', 'imaging_device_1',
    600., '2.718', 'GFP', 'my favorite brain location',
    [], 4.0, 'manifold unit', 'A frame to refer to')

images_path = os.path.join(base_dir, 'm655_D11_S1.hdf5')
with File(images_path, 'r') as file:
    image_series = TwoPhotonSeries(name='test_iS', dimension=[2], data=file['Data']['Images'][:],
                                   imaging_plane=imaging_plane, starting_frame=[0], starting_time=0.0, rate=5.0,
                                   scan_line_rate=np.nan, pmt_gain=np.nan)
nwbfile.add_acquisition(image_series)

mod = nwbfile.create_processing_module('rois', 'example data module')
img_seg = ImageSegmentation()
ps = img_seg.create_plane_segmentation('Ca2+ imaging example', imaging_plane, 'my_planeseg', image_series)
mod.add_data_interface(img_seg)

for i, img_mask in enumerate(zip(mat_data['cellImages'])):
    pixel_mask = np.array(np.where(img_mask)).T
    ps.add_roi(str(i), pixel_mask, img_mask)

region = ps.create_roi_table_region('all', region=list(range(len(mat_data['cellImages']))))

roi_response = RoiResponseSeries('RoiResponseSeries', mat_data['cellTraces'],
                                 'lumens?', region, rate=5.0, starting_time=0.0)
fl = Fluorescence(roi_response_series=roi_response)
mod.add_data_interface(fl)

fname_out = 'm655_D11_S1.nwb'
with NWBHDF5IO(fname_out, 'w') as io:
    io.write(nwbfile)

with NWBHDF5IO(fname_out, 'r') as io:
    io.read()