text:
import torch.nn as nn
import numpy as np
import numpy.random as npr
from .generate_anchors import generate_anchors
from ..utils.cython_bbox import bbox_overlaps
from ..fastrcnn.bbox_transform import bbox_transform
from ..config import cfg
from torch import Tensor
import torch
from ..network import np_to_tensor
import logging

logger = logging.getLogger("root")
logger.setLevel(logging.DEBUG)

class AnchorTargerLayer(nn.Module):

    """Calculate target for RPN network

    Attributes
    ----------
    is_cuda : bool
        Using GPU or not
    """

    def __init__(self, feat_stride, anchor_scales, is_cuda=True):
        """Summary

        Parameters
        ----------
        feat_stride : class::`numpy.array`
            The Ratio between original image size and the convolutional feature (feature maps),
            Used to calculate anchors from a point in convolutional feature into the original image.

            Example: np.array([16. ,])
        anchor_scales : class::`numpy.array`
            Description
        is_cuda : bool, optional
            Description
        """
        super(AnchorTargerLayer, self).__init__()
        self._feat_stride = feat_stride
        self._anchor_scales = anchor_scales
        self._anchors = generate_anchors(scales=np.array(self._anchor_scales))
        self._num_anchors = self._anchors.shape[0]
        self.is_cuda = is_cuda

        # allow boxes to sit over the edge by a small amount
        self._allow_border = 0

    def forward(self, rpn_cls_score, gt_boxes, batch_boxes_index, im_info):
        """ Generate all anchors, then filter anchors that lays outside original images. Calculate target base on overlaps values and bbox regression.

        Parameters
        ----------
        rpn_cls_score: :class:`torch.Tensor`
            The probability of each anchor contains object center
        gt_boxes: :class:`numpy.array`
            List all ground truth boxes across all the images in batch
        batch_boxes_index: :class:`numpy.array`
            Batch index where image belong to.
        im_info: :class:`torch.Tensor([[im_height, im_width]])`
            Original Image size

        Returns
        -------
        (:class:`torch.Tensor`, :class:`torch.Tensor`, :class:`torch.Tensor`, :class:`torch.Tensor`)
            Return labels, bbox_targets, bbox_inside_weights, bbox_outside_weights

        """

        # gt_boxes = gt_boxes.numpy()
        # im_info = im_info.numpy()
        # batch_boxes_index = batch_boxes_index.numpy()
        batch_boxes = gt_boxes[:, :4]

        feature_height, feature_width = rpn_cls_score.shape[2], rpn_cls_score.shape[3]
        batch_size = rpn_cls_score.shape[0]
        im_height, im_width = im_info[0][0], im_info[0][1]

        A = self._num_anchors

        # 1. Generate proposal from bbox deltas and shifted anchors
        all_anchors = self._create_anchors(feature_height, feature_width)
        total_anchors = all_anchors.shape[0]

        # only keep anchors inside the image
        inside_anchors, inside_anchor_indexes = self._filter_outside_anchors(
            all_anchors, im_height, im_width)

        # 2. Calculate overlap and assign corresponding label

        bbox_inside_weights = np.zeros(
            (batch_size, inside_anchor_indexes.shape[0], 4), dtype=np.float32)
        bbox_outside_weights = np.zeros(
            (batch_size, inside_anchor_indexes.shape[0], 4), dtype=np.float32)

        labels, bbox_targets = self.calculate_target(
            inside_anchors, batch_size, inside_anchor_indexes, batch_boxes, batch_boxes_index)

        # 3. calculate bbox_inside_weights, bbox_outside_weights
        bbox_inside_weights[labels == 1] = cfg.TRAIN.RPN_BBOX_INSIDE_WEIGHTS

        num_fg = int(cfg.TRAIN.RPN_FG_FRACTION * cfg.TRAIN.RPN_BATCHSIZE)

        sum_fg = np.sum(labels == 1, axis=1)
        sum_bg = np.sum(labels == 0, axis=1)

        for i in range(batch_size):
            current_batch_sum_fg = sum_fg[i]
            current_batch_sum_bg = sum_bg[i]
            current_batch_fg_index = np.where(labels[i] == 1)[0]
            current_batch_bg_index = np.where(labels[i] == 0)[0]
            if current_batch_sum_fg > num_fg:
                disable_inds = npr.choice(
                    current_batch_fg_index, size=(current_batch_sum_fg - num_fg), replace=False)
                labels[i][disable_inds] = -1

                # subsample negative labels if we have too many

            num_bg = cfg.TRAIN.RPN_BATCHSIZE - np.sum(labels[i] == 1)
            if current_batch_sum_bg > num_bg:
                disable_inds = npr.choice(
                    current_batch_bg_index, size=(current_batch_sum_bg - num_bg), replace=False)

                labels[i][disable_inds] = -1

        if cfg.TRAIN.RPN_POSITIVE_WEIGHT < 0:
            num_examples = np.sum(labels[i] >= 0)
            positive_weights = 1.0 / num_examples
            negative_weights = 1.0 / num_examples
        else:
            assert ((cfg.TRAIN.RPN_POSITIVE_WEIGHT > 0) &
                    (cfg.TRAIN.RPN_POSITIVE_WEIGHT < 1))
            positive_weights = (
                cfg.TRAIN.RPN_POSITIVE_WEIGHT / (np.sum(labels == 1) + 1))
            negative_weights = (
                (1.0 - cfg.TRAIN.RPN_POSITIVE_WEIGHT) / (np.sum(labels == 0) + 1))

        bbox_outside_weights[labels == 1] = np.array([positive_weights] * 4)
        bbox_outside_weights[labels == 0] = np.array([negative_weights] * 4)

        labels = self._unmap(labels, total_anchors,
                             inside_anchor_indexes, batch_size, fill=-1)
        bbox_targets = self._unmap(bbox_targets, total_anchors,
                                   inside_anchor_indexes, batch_size, fill=0)
        bbox_inside_weights = self._unmap(
            bbox_inside_weights, total_anchors, inside_anchor_indexes, batch_size, fill=0)
        bbox_outside_weights = self._unmap(
            bbox_outside_weights, total_anchors, inside_anchor_indexes, batch_size, fill=0)

        labels = labels.reshape((batch_size, feature_height, feature_width, A))
        labels = labels.transpose((0, 3, 1, 2))
        labels = labels.reshape(
            (batch_size, 1, A * feature_height, feature_width))

        bbox_targets = bbox_targets.reshape(
            (batch_size, feature_height, feature_width, A * 4)).transpose((0, 3, 1, 2))
        bbox_inside_weights = bbox_inside_weights.reshape(
            (batch_size, feature_height, feature_width, A * 4)).transpose((0, 3, 1, 2))

        bbox_outside_weights = bbox_outside_weights.reshape(
            (batch_size, feature_height, feature_width, A * 4)).transpose((0, 3, 1, 2))

        return np_to_tensor(labels, self.is_cuda, dtype=torch.LongTensor), np_to_tensor(bbox_targets, self.is_cuda), np_to_tensor(bbox_inside_weights, self.is_cuda), np_to_tensor(bbox_outside_weights, self.is_cuda)

    def backward(self, top, propagate_down, bottom):
        pass

    def reshape(self, bottom, top):
        pass

    def _create_anchors(self, feature_height, feature_width):
        """Create all anchors given features height, width.

        Parameters
        ----------
        feature_height : int
            feature map height
        feature_width : int
            feature map width

        Returns
        -------
        :class:`numpy.array`
            Anchors
        """
        shift_x = np.arange(0, feature_width) * self._feat_stride
        shift_y = np.arange(0, feature_height) * self._feat_stride

        shift_x, shift_y = np.meshgrid(shift_x, shift_y)
        shifts = np.vstack((shift_x.ravel(), shift_y.ravel(),
                            shift_x.ravel(), shift_y.ravel())).transpose()
        # generate shifted anchors
        A = self._num_anchors
        K = shifts.shape[0]

        # move to specific gpu.
        # self._anchors = self._anchors.type_as(gt_boxes)

        # add bbox deltas to shifted anchors to get proposal
        all_anchors = (self._anchors.reshape((1, A, 4)) +
                       shifts.reshape((1, K, 4)).transpose((1, 0, 2)))
        all_anchors = all_anchors.reshape((K * A, 4))
        return all_anchors

    def _filter_outside_anchors(self, all_anchors, im_height, im_width):
        """Remove outside anchors from generated anchors

        Parameters
        ----------
        all_anchors : :class:`numpy.array`
            All generated anchors
        im_height : int
            Origin image height.
        im_width : int
            Origin image width

        Returns
        -------
        tuple(inside_anchors, index_of_inside_anchors)
            Return all inside anchors and its indexes
        """
        inds_inside = np.where(
            (all_anchors[:, 0] >= - self._allow_border) &
            (all_anchors[:, 1] >= - self._allow_border) &
            (all_anchors[:, 2] < im_width + self._allow_border) &  # width
            (all_anchors[:, 3] < im_height + self._allow_border)  # height
        )[0]

        # keep only inside anchors
        anchors = all_anchors[inds_inside, :]
        return anchors, inds_inside

    def _unmap(self, data, count, inds, batch_size, fill=0):
        if len(data.shape) == 2:
            ret = np.empty((batch_size, count), dtype=np.float32)
            ret.fill(fill)
            ret[:, inds] = data
        else:
            ret = np.empty(
                (batch_size, count, data.shape[2]), dtype=np.float32)
            ret.fill(fill)
            ret[:, inds, :] = data
        return ret

    def calculate_target(self, inside_anchors, batch_size, inside_anchor_indexes, batch_boxes, batch_boxes_index):
        """Calculate bbox_targets and layer

        Notes
        -----
        |   Create empty label array.
        |   - `label`:

            >>> label.shape
            (A, batch_size)

        |   For each batch, there are `A` anchors and `G` batch boxes:
        |
        |   `current_batch_overlaps`: overlaps between anchors and boxes

            >>> current_batch_overlaps.shape
            (A, G)

        |    `argmax_overlaps` : List index of boxes, that have largest overlap w.r.t each Anchor.

            >>> argmax_overlaps.shape
            (A, 1)

        |    `max_overlaps` : List of largest overlap values between boxes w.r.t each Anchor.

            >>> max_overlaps.shape
            (A, 1)

        |    Set current batch `label` values:

            >>> labels[i, max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0
            >>> labels[i, max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1

        Parameters
        ----------
        inside_anchors : :class:`numpy.array`
            List all anchors that lay inside the original images.
            Shape: [number_of_anchor * 4]
        batch_size : int
            Current batch size
        inside_anchor_indexes : :class:`numpy.array`
            Indexes of inside anchors.
            Shape: [number_of_anchor * 1]
        batch_boxes : :class:`numpy.array`
            list all ground truth boxes across all the images in batch
            example:

            >>> batch_boxes
            [[  48.57142857  465.71428571  537.14285714 1774.28571429]
             [ 220.         1065.71428571  382.85714286 1771.42857143]
             [ 326.76056338  315.49295775  394.36619718  580.28169014]
             [  76.05633803  290.14084507  242.25352113  788.73239437]
             [  11.26760563    8.45070423  585.91549296 1769.01408451]
             [ 178.125       221.875       287.5         975.        ]
             [ 321.875       334.375       434.375       984.375     ]]

        batch_boxes_index : list[Int]
            |  Batch index where image belong to.
            |  Example:
            |  There 3 images in current batch, and 6 boxes inside that 3 images. Look at the `batch_boxes_index` we know:
            |  First 3 boxes belong to first image.
            |  Next 2 boxes belong to second image.
            |  Last box belong to last image.

            >>> [0, 0, 0, 1, 1, 2]

        Returns
        -------
        (:class:`numpy.array((A, batch_size))`, :class:`numpy.array((batch_size, A, 4))`)
            Return caculated labels , and bbox_targers
        """
        labels = np.empty(
            (batch_size, inside_anchor_indexes.shape[0]), dtype=np.float32)
        bbox_targets = np.zeros(
            (batch_size, inside_anchor_indexes.shape[0], 4), dtype=np.float32)
        labels.fill(-1)

        overlaps = bbox_overlaps(inside_anchors.astype(
            np.float), batch_boxes.astype(np.float))

        for i in range(batch_size):
            current_batch_overlaps = overlaps[:, batch_boxes_index == i]
            current_batch_boxes = batch_boxes[[batch_boxes_index == i]]

            argmax_overlaps = current_batch_overlaps.argmax(axis=1)  # (A)
            max_overlaps = current_batch_overlaps[np.arange(
                inside_anchor_indexes.shape[0]), argmax_overlaps]
            gt_argmax_overlaps = current_batch_overlaps.argmax(axis=0)  # G
            gt_max_overlaps = current_batch_overlaps[gt_argmax_overlaps, np.arange(
                current_batch_overlaps.shape[1])]
            gt_argmax_overlaps = np.where(
                current_batch_overlaps == gt_max_overlaps)[0]

            if not cfg.TRAIN.RPN_CLOBBER_POSITIVES:
                # assign bg labels first so that positive labels can clobber them
                labels[i, max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0

            # fg label: for each gt, anchor with highest overlap
            labels[i, gt_argmax_overlaps] = 1

            # fg label: above threshold IOU
            labels[i, max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1

            if cfg.TRAIN.RPN_CLOBBER_POSITIVES:
                # assign bg labels last so that negative labels can clobber positives
                labels[i, max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0
            current_batch_bbox_targets = bbox_transform(inside_anchors.astype(
                np.float), current_batch_boxes[argmax_overlaps, :]).astype(np.float32, copy=False)
            bbox_targets[i, :] = current_batch_bbox_targets

        return labels, bbox_targets


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Sep  8 14:55:31 2019

@author: Vik Jakkula
"""

import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.optim as optim
import numpy as np

idx2char = ['a','b','c','d','e', 'f']

# Teach acdbef -> bdface

x_data = [[0,2,3,1,4,5]] # acdbef
x_one_hot = [[1,0,0,0,0,0],
              [0,0,1,0,0,0],
              [0,0,0,1,0,0],
              [0,1,0,0,0,0],
              [0,0,0,0,1,0],
              [0,0,0,0,0,1]
        ]

y_data = [1,3,5,0,2,4] # bdface

inputs = Variable(torch.Tensor(x_one_hot))
labels = Variable(torch.LongTensor(y_data))

num_classes = 6
input_size = 6
hidden_size = 6
batch_size = 1 # One Sentence
sequence_length = 1 # Let go one by one
num_layers = 1

class LSTMModel(nn.Module):
    def __init__(self):
        super(LSTMModel, self).__init__()
        self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size,
                          batch_first=True)

    def forward(self, x, hidden, cell):
        x = x.view(batch_size, sequence_length, input_size)
        out, (hidden, cell) = self.lstm(x, hidden, cell)
        out= out.view(-1, num_classes)
        return hidden,out, cell

    def init_hidden(self):
        return Variable(torch.zeros(num_layers, batch_size, hidden_size))

model = LSTMModel()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.1)

for epoch in range(100):
    optimizer.zero_grad()
    loss = 0
    hidden = model.init_hidden()
    cell = model.init_hidden()
    print("Predicted String")
    for ins, label in zip(inputs,labels):
        hidden, output, cell = model(ins, hidden, cell)
        val, idx = output.max(1)
        print(idx2char[idx.data[0]])
        loss+=criterion(output, label)

    print(", epoch: %d, loss: %1.3f" % (epoch+1, loss.data[0]))

    loss.backward()
    optimizer.step()

ext: py

max_stars_repo_path: Pytorch LSTM.py

max_stars_repo_name: VikramadityaJakkula/Pytorch

max_issues_repo_path: Pytorch LSTM.py

max_issues_repo_name: VikramadityaJakkula/Pytorch

max_forks_repo_path: Pytorch LSTM.py

max_forks_repo_name: VikramadityaJakkula/Pytorch

reason: import numpy


text:
from spark_benchmark_hpc import *
import numpy as np

timers = SimpleTimer()

args = parse_args()

timers.init_and_start("generate")
data = generate(0, args.block_size * MEGA_MULTIPLIER)   # 0 is the RDD ID (can be anyting)
timers.stop("generate")

timers.init_and_start("shift")
shift = np.array([25.25, -12.125, 6.333], dtype=np.float64)
shifted = do_shift(data, shift)
timers.stop("shift")

timers.init_and_start("average")
average = do_average(shifted)
timers.stop("average")

results = {
    'timings' : timers.get_all(),
    'rows' : data.shape[0],
    'cols' : data.shape[1]
}
text = json.dumps(results, indent=4, sort_keys=True)
print(text + "\n")

max_stars_repo_name: jxy/simplemap-spark-python

max_issues_repo_name: jxy/simplemap-spark-python

max_forks_repo_name: jxy/simplemap-spark-python


text:
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   attention.py
@Author  :   Racle
@Version :   1.0
@Desc    :   None
'''

import numpy as np
import torch
import torch.nn as nn

class LuongAttention(nn.Module):
    def __init__(self, hidden_size, attn_dropout=0.1):
        super().__init__()
        self.linear_q = nn.Linear(hidden_size, hidden_size)
        self.linear_out = nn.Linear(hidden_size, hidden_size)
        self.dropout = nn.Dropout(attn_dropout)

    def forward(self, query, hidden):
        """
        :param query: (batch, 1,     hidden_size) decoder output
        :param hidden: (batch, t_len, hidden_size) encoder hidden state
        """
        query = self.linear_q(query)
        query = query.transpose(1, 2)
        attention = nn.Softmax(dim=-1)(torch.bmm(hidden, query))
        attention_t = attention.transpose(1, 2)
        if self.attn_dropout != 0:
            attention_t = self.dropout(attention_t)
        context = torch.bmm(attention_t, hidden)
        # (batch, t_len)  (batch, 1, hidden_size)
        return attention.squeeze(), context

class ScaledDotProductAtt(nn.Module):
    def __init__(self, attn_dropout=0.1):
        self.softmax = nn.Softmax(dim=2)

    def forward(self, q, k, v, mask=None):
        """
        q, k, v: (batch*n_head, t_len, hidden_size)
        mask: (batch*n_head, q_len, q_len)
        """
        attention = torch.bmm(q, k.transpose(1, 2))
        attention = attention / torch.sqrt(torch.cast(q.shape[-1], torch.float32))
        if mask is not None:
            attention = attention.masked_fill(mask, -np.inf)
        attention = self.softmax(attention)  # (batch*n_head, t_len, t_len)
        output = torch.bmm(attention, v)     # (batch*n_head, t_len, hidden_size)
        return output, attention

class MultiHeadAttention(nn.Module):
    def __init__(self, n_head, d_model, d_q, d_k, d_v, dropout=0.1):
        """一般 n_head * d_q = d_model; d_q/d_k/d_v 是相同的"""
        super().__init__()
        self.n_head = n_head
        self.d_q = d_q
        self.d_k = d_k
        self.d_v = d_v

        # 层与层之间全连接, 计算输入q/k/v
        self.w_q = nn.Linear(d_model, n_head * d_q)
        self.w_k = nn.Linear(d_model, n_head * d_k)
        self.w_v = nn.Linear(d_model, n_head * d_v)

        # Xavier(Glorot)
        nn.init.normal_(self.w_qs.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_k)))
        nn.init.normal_(self.w_ks.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_k)))
        nn.init.normal_(self.w_vs.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_v)))

        self.attention = ScaledDotProductAtt()

        self.layer_norm = nn.LayerNorm(d_model)

        self.fc = nn.Linear(n_head*d_v, d_model)
        nn.init.xavier_normal_(self.fc.weight)

        self.activation = nn.GELU()
        self.dropout = nn.Dropout(dropout)

    def forward(self, q, k, v, mask=None, is_last=False):
        """
        mask: (batch*n_head, q_len, k_len)
        is_last: 最后一层的decoder只输出到第二个multi head attention之后
        """
        d_q, d_k, d_v, n_head = self.d_q, self.d_k, self.d_v, self.n_head
        batch, q_len, _ = q.size()
        _, k_len, _ = k.size()
        _, v_len, _ = v.size()
        residual = q

        q = self.w_q(q).view(batch, q_len, n_head, d_q)
        k = self.w_k(k).view(batch, k_len, n_head, d_k)
        v = self.w_v(v).view(batch, v_len, n_head, d_v)

        # (batch*n_head, q_len, d_q)
        q = q.permute(2, 0, 1, 3).contiguous().view(-1, q_len, d_q)
        k = k.permute(2, 0, 1, 3).contiguous().view(-1, k_len, d_k)
        v = v.permute(2, 0, 1, 3).contiguous().view(-1, v_len, d_v)

        output, attention = self.attention(q, k, v, mask=mask)
        attention = attention.view(batch, n_head, q_len, d_v)
        attention = attention.premute(0, 2, 1, 3).contiguous()  # (batch, q_len, n_head, k_len)
        output = output.view(n_head, batch, q_len, d_v)
        output = output.permute(1, 2, 0, 3).contiguous().view(batch, q_len, -1)

        if is_last:
            return output, attention

        output = self.dropout(self.activation(self.fc(output)))
        output = self.layer_norm(output + residual)
        # (batch, q_len, d_v*n_head)  (batch, q_len, n_head, q_len)
        return output, attention

class PointWiseFeedForward(nn.Module):
    def __init__(self, d_model, d_affine, fc_dorpout=0.2):
        super().__init__()
        self.d_model = d_model
        self.d_affine = d_affine

        self.linear_1 = nn.Linear(self.d_model, self.d_affine)
        self.linear_2 = nn.Linear(self.d_affine, self.d_model)
        self.layer_norm = nn.LayerNorm(self.d_model)
        self.dropout_1 = nn.Dropout(fc_dorpout)
        self.dropout_2 = nn.Dropout(fc_dorpout)
        self.selu = nn.SELU()

    def forward(self, x):
        output = self.dropout_1(self.selu(self.linear_1(self.layer_norm(x))))
        output = self.dropout_2(self.linear_2(output))
        return output + x

max_stars_repo_name: RacleRay/TextSummerize

max_issues_repo_name: RacleRay/TextSummerize

max_forks_repo_name: RacleRay/TextSummerize


text:
# Authors: Hamza Tazi Bouardi (htazi@mit.edu), Michael L. Li (mlli@mit.edu), Omar Skali Lami (oskali@mit.edu)
import pandas as pd
import numpy as np
import json
from DELPHI_params import n_params_without_policy_params

def update_tracking_without_policy_change(
        df_updated: pd.DataFrame, yesterday: str,
) -> pd.DataFrame:
    df_updated.loc[0, "n_days_enacted_current_policy"] = (
        (pd.to_datetime(yesterday) - pd.to_datetime(df_updated.loc[0, "start_date_current_policy"])).days + 1
    )
    df_updated = update_n_params_fitted_without_policy_change(df_updated=df_updated, yesterday=yesterday)
    df_updated.loc[0, "n_params_constant_from_tree"] = (
            df_updated.loc[0, "n_policy_changes"] - df_updated.loc[0, "n_params_fitted"]
    )
    return df_updated

def update_tracking_when_policy_changed(
        df_updated: pd.DataFrame, df_previous: pd.DataFrame, yesterday: str,
        current_policy_in_tracking: str, current_policy_in_new_data: str,
        dict_normalized_policy_gamma_international: dict,
) -> pd.DataFrame:
    df_updated.loc[0, "n_policies_enacted"] = df_previous.loc[
        0, "n_policies_enacted"
    ] + 1
    df_updated.loc[0, "n_policy_changes"] = df_updated.loc[
        0, "n_policies_enacted"
    ] - 1
    # Modifying track of policy changes constant (adding the new one)
    policy_change_id_str = str(int(df_updated.loc[0, "n_policy_changes"] - 1))
    policy_changes_ids_constant = set(df_updated.loc[0, "policy_changes_ids_constant"].strip())
    policy_changes_ids_constant.add(policy_change_id_str)
    new_policy_changes_ids_constant = "".join(policy_changes_ids_constant)
    df_updated.loc[0, "policy_changes_ids_constant"] = new_policy_changes_ids_constant

    df_updated.loc[0, "last_policy"] = current_policy_in_tracking
    df_updated.loc[0, "current_policy"] = current_policy_in_new_data
    df_updated.loc[0, "last_policy_fitted"] = df_previous.loc[0, "current_policy_fitted"]
    df_updated.loc[0, "current_policy_fitted"] = False  # Necessarily as it's 1 day old
    df_updated.loc[0, "start_date_last_policy"] = df_previous.loc[
        0, "start_date_current_policy"
    ]
    df_updated.loc[0, "end_date_last_policy"] = str(pd.to_datetime(yesterday).date())
    df_updated.loc[0, "start_date_current_policy"] = str(pd.to_datetime(yesterday).date())
    df_updated.loc[0, "n_days_enacted_last_policy"] = df_previous.loc[
        0, "n_days_enacted_current_policy"
    ]
    df_updated.loc[0, "n_days_enacted_current_policy"] = 1
    df_updated.loc[0, "policy_shift_names"] = (
            df_updated.loc[0, "policy_shift_names"] + f"{current_policy_in_tracking}->{current_policy_in_new_data};"
    )
    df_updated.loc[0, "policy_shift_dates"] = (
            df_updated.loc[0, "policy_shift_dates"] + str(pd.to_datetime(yesterday).date()) + ";"
    )
    df_updated.loc[0, "policy_shift_initial_param_values"] = add_policy_shift_initial_param(
        df_previous=df_previous, current_policy_in_tracking=current_policy_in_tracking,
        current_policy_in_new_data=current_policy_in_new_data,
        dict_normalized_policy_gamma_international=dict_normalized_policy_gamma_international
    )
    df_updated.loc[0, "n_params_constant_from_tree"] = (
            df_updated.loc[0, "n_policy_changes"] - df_updated.loc[0, "n_params_fitted"]
    )
    return df_updated

def add_policy_shift_initial_param(
        df_previous: pd.DataFrame,
        current_policy_in_tracking: str,
        current_policy_in_new_data: str,
        dict_normalized_policy_gamma_international: dict,
) -> str:
    """
    This corresponds to the new value of k_0' (normalized difference of values in the tree for going from
    one policy to another)
    """
    previous_policy_shift_initial_params = df_previous.loc[0, "policy_shift_initial_param_values"]
    value_new_data = dict_normalized_policy_gamma_international[current_policy_in_new_data]
    value_tracking = dict_normalized_policy_gamma_international[current_policy_in_tracking]
    value_param_init = value_new_data - value_tracking
    updated_policy_shift_names = (
            previous_policy_shift_initial_params + f"{value_param_init};"
    )
    return updated_policy_shift_names

def update_n_params_fitted_without_policy_change(
        df_updated: pd.DataFrame, yesterday: str,
) -> pd.DataFrame:
    """
    When we do not update the policy (i.e. there hasn't been any change in policy) then both current_policy
    and last_policy are susceptible to change so we need to check if we have enough data to fit the parameter
    instead of using a constant value, and if so, update the flags for param fitting 'last/current_policy_fitted'
    """
    n_days_data_before_fitting = 10
    df_temp_updated = df_updated[
        (df_updated.n_days_enacted_current_policy >= n_days_data_before_fitting)
        & (df_updated.n_policy_changes >= 1)
        & (df_updated.current_policy_fitted == False)
    ]
    df_temp_updated_2 = df_updated[
        (df_updated.n_days_enacted_current_policy >= n_days_data_before_fitting)
        & (df_updated.n_policy_changes >= 1)
        & (df_updated.current_policy_fitted is False)
    ]
    n_days_enacted_current_policy_temp = df_updated.n_days_enacted_current_policy[0]
    n_policy_changes_temp = df_updated.n_policy_changes[0]
    current_policy_fitted_temp = df_updated.current_policy_fitted[0]
    flag_new_param_fitted_current_policy = len(df_temp_updated) > 0

    if flag_new_param_fitted_current_policy:
        print("Supposed to be updating the # fitting params as got in the if/else statement")
        # The policy change id is the n_policy_changes - 1 (so when there's the first policy change, its id will be '0')
        policy_change_id_str = str(int(df_updated.loc[0, "n_policy_changes"] - 1))
        df_updated.loc[0, "n_params_fitted"] = df_updated.loc[0, "n_params_fitted"] + 1
        df_updated.loc[0, "current_policy_fitted"] = True
        df_updated.loc[0, "params_fitting_starting_dates"] = (
                df_updated.loc[0, "params_fitting_starting_dates"] + str(pd.to_datetime(yesterday).date()) + ";"
        )
        # Modifying track of policy changes fitted (adding this one as it's now fitted)
        policy_changes_ids_fitted = set(df_updated.loc[0, "policy_changes_ids_fitted"].strip())
        policy_changes_ids_fitted.add(policy_change_id_str)
        new_policy_changes_ids_fitted = "".join(policy_changes_ids_fitted)
        df_updated.loc[0, "policy_changes_ids_fitted"] = new_policy_changes_ids_fitted

        # Modifying track of policy changes constant (popping it from constant as it's now fitted)
        policy_changes_ids_constant = set(df_updated.loc[0, "policy_changes_ids_constant"].strip())
        policy_changes_ids_constant.remove(policy_change_id_str)
        policy_changes_ids_constant.add(" ")
        new_policy_changes_ids_constant = "".join(policy_changes_ids_constant)
        df_updated.loc[0, "policy_changes_ids_constant"] = new_policy_changes_ids_constant

        # Adding the data to the end dates for this policy
        dict_policy_changes_constant_end_dates = json.loads(df_updated.loc[0, "policy_changes_constant_end_dates"])
        dict_policy_changes_constant_end_dates[policy_change_id_str] = str(pd.to_datetime(yesterday).date())
        df_updated.loc[0, "policy_changes_constant_end_dates"] = json.dumps(dict_policy_changes_constant_end_dates)

    return df_updated

def get_params_fitted_policies(
        df_updated: pd.DataFrame
) -> (list, list):
    params_fitted_present = len(df_updated[df_updated.n_params_fitted >= 1]) > 0
    if not params_fitted_present:
        return [], []
    else:
        policy_shifts_fitted_ids = list(set(df_updated.loc[0, "policy_changes_ids_fitted"].strip()))
        policy_shifts_fitted_ids = [int(x) for x in policy_shifts_fitted_ids]
        n_params_fitted = df_updated.n_params_fitted.iloc[0].astype(int)
        assert n_params_fitted == len(policy_shifts_fitted_ids), "Discrepancy between # fitted ids and n_params_fitted"
        params_fitted = df_updated.policy_shift_initial_param_values.iloc[0].strip().split(";")[:-1]
        # print(type(df_updated.policy_shift_initial_param_values.iloc[0]))
        # print(df_updated.policy_shift_initial_param_values)
        # print(params_fitted)
        params_fitted = [float(params_fitted[policy_id]) for policy_id in policy_shifts_fitted_ids]
        # Actually corresponds to the start date of the policy!
        start_date_fitting = df_updated.policy_shift_dates.iloc[0].strip().split(";")[:-1]
        start_date_fitting = [start_date_fitting[policy_id] for policy_id in policy_shifts_fitted_ids]
        return params_fitted, start_date_fitting

def get_constant_params_end_date(dict_end_dates_constant: dict, policy_shift_id: str, yesterday: str):
    try:
        end_date_policy_id = dict_end_dates_constant[policy_shift_id]
    except KeyError:
        end_date_policy_id = str(pd.to_datetime(yesterday).date())
    return end_date_policy_id

def get_params_constant_policies(
        df_updated: pd.DataFrame, yesterday: str
):
    yesterday_date = str(pd.to_datetime(yesterday).date())
    params_constant_present = len(df_updated[df_updated.n_params_constant_from_tree >= 1]) > 0
    if not params_constant_present:
        return [], []
    else:
        n_params_constant = df_updated.n_params_constant_from_tree.iloc[0].astype(int)
        policy_shifts_constant_ids = list(set(df_updated.loc[0, "policy_changes_ids_constant"].strip()))
        assert n_params_constant == len(policy_shifts_constant_ids), \
            "Discrepancy between # constant ids and n_params_constant"
        params_constant = df_updated.policy_shift_initial_param_values.iloc[0].strip().split(";")[:-1]
        params_constant = [float(params_constant[int(policy_id)]) for policy_id in policy_shifts_constant_ids]
        start_dates_constant = df_updated.policy_shift_dates.iloc[0].strip().split(";")[:-1]
        start_dates_constant = [start_dates_constant[int(policy_id)] for policy_id in policy_shifts_constant_ids]
        dict_policy_changes_constant_end_dates = json.loads(df_updated.loc[0, "policy_changes_constant_end_dates"])
        end_dates_constant = [
            get_constant_params_end_date(
                dict_end_dates_constant=dict_policy_changes_constant_end_dates,
                policy_shift_id=policy_shift_id,
                yesterday=yesterday
            )
            for policy_shift_id in policy_shifts_constant_ids
        ]
        assert len(end_dates_constant) == len(start_dates_constant), "# constant start dates != # constant end dates"
        start_end_dates_constant = [
            (start_date, end_date) for start_date, end_date
            in zip(start_dates_constant[-n_params_constant:], end_dates_constant[-n_params_constant:])
        ]
        return params_constant, start_end_dates_constant

def get_policy_names_before_after_fitted(df_updated: pd.DataFrame):
    """
    :return: a list of tuples like [(policy_str_before_shift, policy_str_after_shift)],
    here for the fitted params only
    """
    policy_shifts_fitted_ids = list(set(df_updated.loc[0, "policy_changes_ids_fitted"].strip()))
    policy_shifts_fitted_ids = [int(shift_id_str) for shift_id_str in policy_shifts_fitted_ids]
    all_policy_shifts_tuples = get_policy_shift_names_tuples(df_updated=df_updated)
    all_policy_shifts_tuples_fitted = [
        all_policy_shifts_tuples[shift_id_fitted] for shift_id_fitted in policy_shifts_fitted_ids
    ]
    return all_policy_shifts_tuples_fitted

def get_policy_names_before_after_constant(df_updated: pd.DataFrame):
    """
    :return: a list of tuples like [(policy_str_before_shift, policy_str_after_shift)],
    here for the constant params only
    """
    policy_shifts_constant_ids = list(set(df_updated.loc[0, "policy_changes_ids_constant"].strip()))
    policy_shifts_constant_ids = [int(shift_id_str) for shift_id_str in policy_shifts_constant_ids]
    all_policy_shifts_tuples = get_policy_shift_names_tuples(df_updated=df_updated)
    all_policy_shifts_tuples_constant = [
        all_policy_shifts_tuples[shift_id_constant] for shift_id_constant in policy_shifts_constant_ids
    ]
    return all_policy_shifts_tuples_constant

def get_policy_shift_names_tuples(df_updated: pd.DataFrame):
    policy_shift_names_raw = df_updated.loc[0, "policy_shift_names"].strip().split(";")[:-1]
    policy_shift_names_tuples = [
        (policy_shift.split("->")[0], policy_shift.split("->")[1])
        for policy_shift in policy_shift_names_raw
    ]
    return policy_shift_names_tuples

def get_list_and_bounds_params(
        df_updated: pd.DataFrame, parameter_list_line: list, param_MATHEMATICA: bool
):
    if param_MATHEMATICA:
        parameter_list_fitted = parameter_list_line[4:]
        parameter_list_fitted[3] = np.log(2) / parameter_list_fitted[3]
    else:
        parameter_list_fitted = parameter_list_line[5:]

    params_policies_fitted, start_dates_fitting_policies = get_params_fitted_policies(df_updated)
    parameter_list_fitted = parameter_list_fitted + params_policies_fitted
    param_bounds_list_lower = [x - 0.1 * abs(x) for x in parameter_list_fitted]
    param_bounds_list_upper = [x + 0.1 * abs(x) for x in parameter_list_fitted]
    bounds_params_fitted = tuple(
        [(lower, upper) for lower, upper in zip(param_bounds_list_lower, param_bounds_list_upper)]
    )
    assert len(start_dates_fitting_policies) == len(parameter_list_fitted) - n_params_without_policy_params, \
        "Should have as many extra fitted policy params as start fitting dates for policy params"
    return parameter_list_fitted, start_dates_fitting_policies, bounds_params_fitted

def add_policy_tracking_row_country(
        continent: str, country: str, province: str, yesterday: str, dict_current_policy_international:dict
):
    "continent, country, province, date, n_policies_enacted, n_policy_changes, last_policy, \
        start_date_last_policy, end_date_last_policy, n_days_enacted_last_policy, current_policy, \
        start_date_current_policy, end_date_current_policy, n_days_enacted_current_policy, policy_shift_names, \
        policy_shift_dates, policy_shift_initial_param_values, n_params_fitted, current_policy_fitted, \
        last_policy_fitted, params_fitting_starting_dates, n_params_constant_from_tree"
    yesterday_date = str(pd.to_datetime(yesterday).date())
    current_policy = dict_current_policy_international[(country, province)]
    dict_policy_change_tracking_area = {
        "continent": [continent], "country": [country], "province": [province], "date": [yesterday_date],
        "n_policies_enacted": [1], "n_policy_changes": [0], "last_policy": [current_policy],
        "start_date_last_policy": [yesterday_date], "end_date_last_policy": [" "], "n_days_enacted_last_policy": [1],
        "current_policy": [current_policy], "start_date_current_policy": [yesterday_date],
        "end_date_current_policy": [" "], "n_days_enacted_current_policy": [1], "policy_shift_names": [" "],
        "policy_shift_dates": [" "], "policy_shift_initial_param_values": [" "],"n_params_fitted": [0],
        "current_policy_fitted": [False], "last_policy_fitted": [False], "params_fitting_starting_dates": [" "],
        "n_params_constant_from_tree": [0], "policy_changes_ids_fitted": [" "], "policy_changes_ids_constant": [" "],
        "policy_changes_constant_end_dates": [json.dumps({})]  # This is going to be a json dictionary
    }
    df_policy_change_tracking_area = pd.DataFrame(dict_policy_change_tracking_area)
    return df_policy_change_tracking_area

def update_tracking_fitted_params(
        df_updated: pd.DataFrame,
        n_policy_shifts_fitted: int,
        new_best_params_fitted_policies: list,
):
    if len(new_best_params_fitted_policies) == 0:
        return df_updated
    else:
        policy_shifts_fitted_ids = list(set(df_updated.loc[0, "policy_changes_ids_fitted"].strip()))
        policy_shifts_fitted_ids = [int(x) for x in policy_shifts_fitted_ids]
        # print("Policy Shifts Fitted IDs (should be 1):", policy_shifts_fitted_ids)
        assert len(new_best_params_fitted_policies) == len(policy_shifts_fitted_ids) == n_policy_shifts_fitted, \
            "Supposed to have as many policy shift ids as new best params for fitted policies"
        params_all_saved = np.array(df_updated.policy_shift_initial_param_values.iloc[0].strip().split(";")[:-1])
        params_all_saved = np.array([float(x) for x in params_all_saved])
        # print("Parameters saved in policy tracking: ", params_all_saved)
        for i, j in zip(policy_shifts_fitted_ids, range(len(new_best_params_fitted_policies))):
            params_all_saved[i] = new_best_params_fitted_policies[j]
        # print("Parameters Updated with new value:", params_all_saved)
        params_all_updated = [str(x) for x in params_all_saved]
        params_all_updated_str = ";".join(params_all_updated) + ";"
        df_updated.loc[0, "policy_shift_initial_param_values"] = str(params_all_updated_str)
    return df_updated

def update_gamma_t_with_fitted_params():
    return ""

def update_gamma_t_with_constant_params(
        t, gamma_t, N_POLICIES_CONSTANT_TUPLE, t_start_end_constant_policy, date_yesterday_int,
        policy_shift_names_constant, dict_normalized_policy_gamma_countries, params_policies_constant,
        t_cases, future_time
):
    for param_policy_constant_i in range(N_POLICIES_CONSTANT_TUPLE):
        start_date_constant_i = t_start_end_constant_policy[param_policy_constant_i][0]
        end_date_constant_i = t_start_end_constant_policy[param_policy_constant_i][1]
        if end_date_constant_i != date_yesterday_int:
            assert end_date_constant_i < date_yesterday_int
            if (
                    (t >= start_date_constant_i)
                    and (t <= end_date_constant_i)
            ):
                past_policy_cst = policy_shift_names_constant[param_policy_constant_i][0]
                current_new_policy_cst = policy_shift_names_constant[param_policy_constant_i][1]
                normalized_gamma_current_policy = dict_normalized_policy_gamma_countries[current_new_policy_cst]
                normalized_gamma_past_policy = dict_normalized_policy_gamma_countries[past_policy_cst]
                gamma_t = gamma_t + min(
                    (2 - gamma_t) / (1 - normalized_gamma_current_policy),
                    (gamma_t / normalized_gamma_past_policy) * params_policies_constant[param_policy_constant_i]
                )
        else:
            # if end date of a constant param is set to yesterday,
            # it means that it continues even in the future
            if t >= start_date_constant_i:
                # however if t is after the future_time we are predicting for, then we should not use the same
                # list of parameters etc. but rather the previous way of predicting for policies (adding the constant
                # in the future)
                if t <= t_cases[-1] + future_time:
                    past_policy_cst = policy_shift_names_constant[param_policy_constant_i][0]
                    current_new_policy_cst = policy_shift_names_constant[param_policy_constant_i][1]
                    normalized_gamma_current_policy = dict_normalized_policy_gamma_countries[
                        current_new_policy_cst
                    ]
                    normalized_gamma_past_policy = dict_normalized_policy_gamma_countries[
                        past_policy_cst
                    ]
                    gamma_t = gamma_t + min(
                        (2 - gamma_t) / (1 - normalized_gamma_current_policy),
                        (gamma_t / normalized_gamma_past_policy) * params_policies_constant[
                            param_policy_constant_i
                        ]
                    )
    return gamma_t

max_stars_repo_path: archive/Adaptive Policy Model - Continuous Retraining/DELPHI_policies_utils_cr.py

max_stars_repo_name: EbTech/DELPHI

max_issues_repo_name: COVIDAnalytics/epidemic-model

max_forks_repo_name: COVIDAnalytics/epidemic-model


text:
#!/usr/bin/env python3
"""
Compute standard resistor divider pair (CLI).

(c) Marion Anderson, 2021
Provided under the MIT License.
"""
from argparse import ArgumentParser
from math import inf, floor, log10
from sys import exit

from numpy import (argmin, array, fill_diagonal, newaxis, tril, triu)

# Resistor Values (bless up for regex html downloads)
# src: https://www.electronics-notes.com/articles/electronic_components/resistors/standard-resistor-values-e-series-e3-e6-e12-e24-e48-e96.php
E12 = [1.0, 1.2, 1.5, 1.8, 2.2, 2.7, 3.3, 3.9, 4.7, 5.6, 6.8, 8.2]
E24 = [1.0, 1.1, 1.2, 1.3, 1.5, 1.6, 1.8, 2.0, 2.2, 2.4, 2.7, 3.0, 3.3, 3.6,
       3.9, 4.3, 4.7, 5.1, 5.6, 6.2, 6.8, 7.5, 8.2, 9.1]
E48 = [1.0, 1.05, 1.1, 1.15, 1.21, 1.27, 1.33, 1.4, 1.47, 1.54, 1.62, 1.69,
       1.78, 1.87, 1.96, 2.05, 2.15, 2.26, 2.37, 2.49, 2.61, 2.74, 2.87, 3.01,
       3.16, 3.32, 3.48, 3.65, 3.83, 4.02, 4.22, 4.42, 4.64, 4.87, 5.11, 5.36,
       5.62, 5.9, 6.19, 6.49, 6.81, 7.15, 7.5, 7.87, 8.25, 8.66, 9.09, 9.53]
E96 = [1.0, 1.02, 1.05, 1.07, 1.1, 1.13, 1.15, 1.18, 1.21, 1.24, 1.27, 1.3,
       1.33, 1.37, 1.4, 1.43, 1.47, 1.5, 1.54, 1.62, 1.65, 1.69, 1.74, 1.78,
       1.82, 1.87, 1.91, 1.96, 2.0, 2.05, 2.1, 2.16, 2.21, 2.26, 2.32, 2.37,
       2.43, 2.49, 2.55, 2.61, 2.67, 2.74, 2.8, 2.87, 2.94, 3.01, 3.09, 3.16,
       3.24, 3.32, 3.4, 3.48, 3.57, 3.65, 3.74, 3.83, 3.92, 4.02, 4.12, 4.22,
       4.32, 4.42, 4.53, 4.64, 4.75, 4.87, 4.99, 5.11, 5.23, 5.36, 5.49, 5.62,
       5.76, 5.9, 6.04, 6.19, 6.34, 6.49, 6.65, 6.98, 7.15, 7.32, 7.5, 7.68,
       7.87, 8.06, 8.25, 8.45, 8.66, 8.87, 9.09, 9.31, 9.53, 9.76]

parser = ArgumentParser('Standard Table 2-Resistor Divider Calculator')
parser.add_argument('Vin', type=float, help='divider input voltage')
parser.add_argument('Vout', type=float, help='divider output voltage')
parser.add_argument('--table', choices=['E12','E24','E48','E96'],
                    default='E12', help='resistor table, defaults to E12')
parser.add_argument('--prefer-over', action='store_true',
                    help='ignore solutions with lower Vout')
parser.add_argument('--prefer-under', action='store_true',
                    help='ignore solutions with higher Vout')

if __name__ == '__main__':
    # Parse Args
    args = parser.parse_args()
    if args.prefer_over and args.prefer_under:
        print('--prefer-over and --prefer-under are mutually exclusive')
        exit(1)
    Vin = args.Vin
    Vout = args.Vout
    rtable = eval(args.table)
    rtable += [10*v for v in rtable]  # include 10x higher resistors

    # Compute Resistor Ideal Ratio
    mult = Vin/Vout - 1
    powOf10 = floor(log10(mult))  # resistor tables are for 1 power of 10
    mult = mult * 10**(-powOf10)

    # Construct Difference-From-Ideal-R Matrix
    res_arr = array(rtable)[:,newaxis]             # convert to col vec
    diffmat = triu(mult*res_arr - res_arr.T)       # absolute difference is symm
    ignoremat = tril(diffmat+inf)                  # ignore below diag portion
    fill_diagonal(ignoremat, 0)
    diffmat += ignoremat
    if args.prefer_over:   # ignore voltage below desired Vout
        diffmat[diffmat<0] = inf
    if args.prefer_under:  # ignore voltage above desired Vout
        diffmat[diffmat>0] = inf
    diffmat = abs(diffmat)

    # Find Best R Pair
    mincols = argmin(diffmat,axis=1).astype('int')      # ndx of best for ech
    r1cands = diffmat[range(mincols.shape[0]),mincols]  # min rndx for each r
    minrows = argmin(r1cands).astype('int')             # best row

    # Output
    r1 = rtable[mincols[minrows]]  # top resistor
    r1 *= 10**powOf10
    r2 = rtable[minrows]           # bottom resistor
    Vo = Vin * r2 / (r1 + r2)      # actual vout
    print('R1={}, R2={}, Vout={}'.format(r1, r2, Vo))

max_stars_repo_name: lowdrant/rdiv

max_issues_repo_name: lowdrant/rdiv

max_forks_repo_name: lowdrant/rdiv

reason: from numpy


text:
from latmats.tasks.analogy_testing.data import load_analogies
from latmats.pretraining.data_loader import load_file
from latmats.pretraining.model import Word2VecPretrainingModel
from latmats.utils import example_generator_not_material

from gensim.models.keyedvectors import WordEmbeddingsKeyedVectors
from mat2vec.processing import MaterialsTextProcessor
import numpy as np
import time

word2index = load_file("word2index_3mil.pkl")
mat2index = load_file("material2index.pkl")

w2vr = Word2VecPretrainingModel(name="dense-2-128", n_layers=2)
w2vr.compile()
w2vr.load_weights()
w2v = w2vr.model_word2vec
embedding_layer = w2v.layers[3]

mproc = MaterialsTextProcessor()

print("generating word vector map")
word2vector = {word: embedding_layer(index) for word, index in list(word2index.items())[:100]}

print([k for k in word2vector.keys()])

print("normalizing all vectors")
word2vector = {word: vector / np.linalg.norm(vector) for word, vector in word2vector.items()}

av = word2vector["the"]
bv = word2vector["of"]
cv = word2vector["and"]

words = list(word2vector.keys())
vectors = list(word2vector.values())

test_vector = np.add(cv, np.subtract(bv, av))

t0 = time.time()
cosine_similarities = [0] * len(vectors)
for i, v in enumerate(vectors):
    cosine_similarities[i] = np.dot(v, test_vector) / (np.linalg.norm(v) * np.linalg.norm(test_vector))
t1= time.time()
print("python native", t1-t0)

# cosine_similarities_wekv = WordEmbeddingsKeyedVectors.cosine_similarities(test_vector, vectors)

t2 = time.time()
norm = np.linalg.norm(test_vector)
all_norms = np.linalg.norm(vectors, axis=1)
dot_products = np.dot(vectors, test_vector)
similarities_wekv = dot_products / (norm * all_norms)
t3 = time.time()
print("numpy way", t3-t2)

print(cosine_similarities)
print(similarities_wekv)

max_stars_repo_name: ardunn/latmats

max_issues_repo_name: ardunn/latmats

max_forks_repo_name: ardunn/latmats


text:
# Copyright 2020 The Q2 Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from collections import Counter
import numpy as np
import pandas as pd
import spacy

import sacrebleu
from bert_score import score

from run_pipeline import get_response_score

nlp = spacy.load("en_core_web_sm")

NO_VALID_Q = -1

def cross_annotated_scores(in_path, out_path):
    df = pd.read_csv(in_path)

    for sys_type in ['dodeca', 'memnet']:
        f1_scores, questions, cands, answers = [], [], [], []
        all_knowledge, responses, labels, ids = [], [], [], []
        for idx, row in df.iterrows():
            response = row[sys_type + '_response']
            knowledge = row['knowledge']

            res, res_questions, res_cands, res_answers, res_scores\
                = get_response_score(response, knowledge, 'beam', single=True, remove_personal=True)

            if res != NO_VALID_Q:
                f1_scores.extend(res_scores)
                questions.extend(res_questions)
                cands.extend(res_cands)
                answers.extend(res_answers)
                all_knowledge.extend([knowledge] * len(res_questions))
                responses.extend([response] * len(res_questions))
                labels.extend([row[sys_type + '_label']] * len(res_questions))
                ids.extend([idx] * len(res_questions))

            else:
                f1_scores.extend([NO_VALID_Q])
                questions.extend(['NO_Q'])
                cands.extend(['NO_Q'])
                answers.extend(['NO_Q'])
                all_knowledge.extend([knowledge])
                responses.extend([response])
                labels.extend([row[sys_type + '_label']])
                ids.extend([idx])

        res_dict = {'id': ids, 'score': f1_scores, 'response': responses, 'cand': cands, 'question': questions,
                    'knowledge_ans': answers, 'knowledge': all_knowledge, 'label': labels}

        res_df = pd.DataFrame(data=res_dict)
        res_df.to_csv(out_path + '_' + sys_type + '.csv')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--infile", type=str, required=True, help="Path to a csv file containing q^2 scores.")
    parser.add_argument("--outfile", type=str, required=True, help="Path to an output file")
    args = parser.parse_args()

    cross_annotated_scores(args.infile, args.outfile)

max_stars_repo_name: orhonovich/q-squared

max_issues_repo_name: orhonovich/q-squared

max_forks_repo_name: orhonovich/q-squared


text:
import cv2
import numpy as np

def overlay_transparent(background_img, img_to_overlay_t, bbox, x, y, overlay_size=None):
    """
    @brief      Overlays a transparant PNG onto another image using CV2

    @param      background_img    The background image
    @param      img_to_overlay_t  The transparent image to overlay (has alpha channel)
    @param      x                 x location to place the top-left corner of our overlay
    @param      y                 y location to place the top-left corner of our overlay
    @param      overlay_size      The size to scale our overlay to (tuple), no scaling if None

    @return     Background image with overlay on top
    """

    bg_img = background_img.copy()

    # ground truth background
    b_h, b_w, _ = bg_img.shape
    gt = np.full((b_h, b_w, 3), 0, dtype = np.uint8) # black
    gt_with_bbox = np.full((b_h, b_w, 3), 0, dtype = np.uint8)
    gt_with_bbox[bbox[1]:bbox[3], bbox[0]:bbox[2]] = 255

    if overlay_size is not None:
        img_to_overlay_t = cv2.resize(img_to_overlay_t.copy(), overlay_size)

    # Extract the alpha mask of the RGBA image, convert to RGB
    b,g,r,a = cv2.split(img_to_overlay_t)
    overlay_color = cv2.merge((b,g,r))

    # Apply some simple filtering to remove edge noise
    ret, mask = cv2.threshold(img_to_overlay_t[:, :, 3], 0, 255, cv2.THRESH_BINARY)

    h, w, _ = overlay_color.shape
    roi = bg_img[y:y+h, x:x+w]

    # this is for ground truth
    occ_white = np.full((h, w, 3), 255, dtype = np.uint8)

    # Black-out the area behind the logo in our original ROI
    img1_bg = cv2.bitwise_and(roi.copy(),roi.copy(),mask = cv2.bitwise_not(mask))

    # Mask out the logo from the logo image.
    img2_fg = cv2.bitwise_and(overlay_color,overlay_color,mask = mask)

    # ground truth
    occ_mask = cv2.bitwise_and(occ_white.copy(),occ_white.copy(),mask = cv2.bitwise_not(mask))
    gt[y:y+h, x:x+w] = occ_mask

    # Update the original image with our new ROI
    bg_img[y:y+h, x:x+w] = cv2.add(img1_bg, img2_fg)

    gt_with_bbox[y:y+h, x:x+w] = occ_mask

    # cv2.imshow("image", bg_img)
    # cv2.imshow("image1", gt)
    # cv2.imshow("image2", gt_with_bbox)
    # cv2.waitKey(0)

    return bg_img, gt_with_bbox

def overlay_flow(background_img, img_to_overlay_t, x, y, overlay_size=None):
    """
    @brief      Overlays a transparant PNG onto another image using CV2

    @param      background_img    The background image
    @param      img_to_overlay_t  The transparent image to overlay (has alpha channel)
    @param      x                 x location to place the top-left corner of our overlay
    @param      y                 y location to place the top-left corner of our overlay
    @param      overlay_size      The size to scale our overlay to (tuple), no scaling if None

    @return     Background image with overlay on top
    """

    bg_img = background_img.copy()
    flow_img = np.full((bg_img.shape[0], bg_img.shape[1], 3), (137, 131, 136), np.uint8)

    if overlay_size is not None:
        img_to_overlay_t = cv2.resize(img_to_overlay_t.copy(), overlay_size)

    # Extract the alpha mask of the RGBA image, convert to RGB
    b,g,r,a = cv2.split(img_to_overlay_t)
    overlay_color = cv2.merge((b,g,r))

    # Apply some simple filtering to remove edge noise
    ret, mask = cv2.threshold(img_to_overlay_t[:, :, 3], 0, 255, cv2.THRESH_BINARY)

    h, w, _ = overlay_color.shape
    roi = bg_img[y:y+h, x:x+w]
    flow_roi = flow_img[y:y+h, x:x+w]

    # Black-out the area behind the logo in our original ROI
    img1_bg = cv2.bitwise_and(roi.copy(),roi.copy(),mask = cv2.bitwise_not(mask))

    # Mask out the logo from the logo image.
    img2_fg = cv2.bitwise_and(flow_roi.copy(), flow_roi.copy(), mask=mask)

    # # Update the original image with our new ROI
    bg_img[y:y+h, x:x+w] = cv2.add(img1_bg, img2_fg)

    return bg_img
    # import sys
    # np.set_printoptions(threshold=sys.maxsize)
    # print(mask)
    # print(cv2.bitwise_not(mask))

max_stars_repo_name: delanoNieh/stado

max_issues_repo_name: delanoNieh/stado

max_forks_repo_name: delanoNieh/stado


text:
import numpy as np
from bo.bo import BO
from scipy.stats import norm

class GPUCB(BO):

    def __init__(self, gp_model, f, t):
        super().__init__(gp_model, f)
        self.f = f
        self.t = t

    def acquire(self, x):
        x = x.reshape(-1, self.f.dim)
        pred_mean, pred_var = self.gp_model.predict_noiseless(x)
        pred_std = np.sqrt(pred_var)
        res = pred_mean + np.sqrt(self.beta_func()) * pred_std
        return res

    def beta_func(self):
        beta = np.log((self.t**2) + 0.1)
        return beta

reason: import numpy,from scipy


text:
#! /usr/bin/python
# -*- coding: utf-8 -*-

# This file belongs to DWGranularSpeckles project.
# The software is realeased with MIT license.

import functools
import multiprocessing

import numpy as np
from scipy.signal import argrelextrema

def importMatrix(args):
    fullPath = "results/"+args.image_folder+"/full_correlation/"
    togePath = "results/"+args.image_folder+"/all_together/"
    return np.load(fullPath+args.block_size+".npy"), \
        np.load(togePath+args.block_size+".npy")

def space_correlate(matrix, shift):
    def column_correlation(matrix, shift, row):
        def check():
            if 0 <= row-shift < 120 and 0 <= row+shift < 120:
                return matrix[row+shift, shift:-shift] *\
                    matrix[row, shift:-shift] \
                    + matrix[row-shift, shift:-shift]\
                    * matrix[row, shift:-shift]
            else:
                return np.zeros((matrix.shape[1]-2*shift))
        return np.sum(matrix[row, shift:-shift]*matrix[row, :-2*shift] +
                      matrix[row, shift:-shift]*matrix[row, shift*2:] +
                      check())\
            * 1./(matrix.shape[1]-2*shift)/4
    if shift == 0:
        return map(lambda x: np.mean(np.vectorize(np.square)(matrix[x, :])),
                   range(matrix.shape[0]))
    else:
        return map(lambda x: column_correlation(matrix, shift, x),
                   range(matrix.shape[0]))

def space_correlation(matrix, time):
    mean = np.mean(matrix[:, :, time], axis=1)**2
    func = functools.partial(space_correlate, matrix[:, :, time])
    pool = multiprocessing.Pool(5)
    correlation = np.array(pool.map(func, range(0, matrix.shape[1]/10)))
    return np.nan_to_num(correlation/mean/mean.shape[0]).transpose()

def purify_row(matrix):
    return np.delete(matrix, np.where((np.all(matrix, axis=1) == 0) is True),
                     axis=0)

def halfheight(ave):
    z = []
    for count, a in enumerate(ave):
        z.append(np.argmax(a < np.max(a)/2.))
    return np.array(z)

def taulocalmin(ave):
    z = []
    for count, a in enumerate(ave):
        z.append(argrelextrema(a, np.less)[0][0])

    return np.array(z)

def spectrum(time_interval, sequence):
    return time_interval/(2*np.pi)*np.abs(np.fft.ifft(sequence \
                                    - np.mean(sequence))) \
                                    **2*len(sequence)

def spectrum_matrix(matrix, time_interval):
    """
    Fourier transform of a matrix

    Parameters:
    ===========
    matrix is the set of indipendent signals to be processed in batch
    time_interval set the physics scale of the process, in second

    Returns:
    ========
    spectres: matrix with spectre of each signal
    freq:     Frequency related to matrix ascissa, in Hertz

    """
    pool = multiprocessing.Pool(4)
    vmat, hmat, time = matrix.shape
    func = partial(spectrum, time_interval)
    sequences = [matrix[x,y,:] for x,y in product(np.arange(vmat),np.arange(hmat))]
    spectres = np.array(pool.map(func, sequences)).reshape(vmat,hmat,time)
    freq=np.fft.fftfreq(time,time_interval)
    return spectres, freq

def log_smooth_matrix(spectres, freq, minfreq, nbin):
    """
    Smooth signal with logaritmic binning for log scale plot
    """
    vmat, hmat, time = spectres.shape
    sequences = [spectres[x,y,:] for x,y in product(np.arange(vmat),np.arange(hmat))]
    results = []
    for seq in sequences:
        exp_time, smooth_seq = binLogMovAve5(freq, seq, minfreq, nbin)
        results.append(smooth_seq)
    return np.array(results).reshape(vmat, hmat, nbin), exp_time

def log_smooth(time, signal, minfreq, nbin):
    asc = asc[:int(len(asc)/2)]
    sig = sig[:int(len(sig)/2)]
    maxtime = len(sig)
    spec=[]
    exptime=[]
    logmaxtime=np.log(maxtime)
    logmintime=np.log(mintime)
    step=(logmaxtime-logmintime)/nbin
    for i in range(0,nbin):
        if(len(sig)>np.exp((i+1)*step)):
            exptime.append(np.mean(asc[np.exp(logmintime+i*step):np.exp(logmintime+(i+1)*step)]))
            spec.append(np.mean(sig[np.exp(logmintime+i*step):mintime+np.exp(logmintime+(i+1)*step)]))
        else:
            exptime.append(np.mean(asc[np.exp(logmintime+i*step):len(asc)]))
            spec.append(np.mean(sig[np.exp(logmintime+i*step):len(sig)]))
            break
    return np.asarray(exptime), np.asarray(spec)

def moving_average(a, n) :
    ret = np.cumsum(a, dtype=float)
    ret[n:] = ret[n:] - ret[:-n]
    return ret[n - 1:] / n

def moving_average_matrix(sequences, n):
    smooth_seq = []
    for seq in range(sequences.shape[0]):
        smoothed = moving_average(sequences[seq,:], n)
        smooth_seq.append(smoothed)
    return smooth_seq


text:
"""
======================
MultiTensor Simulation
======================

In this example we show how someone can simulate the signal and the ODF of a
single voxel using a MultiTensor.
"""

import numpy as np
from dipy.sims.voxel import (multi_tensor,
                             multi_tensor_odf,
                             single_tensor_odf,
                             all_tensor_evecs)
from dipy.data import get_sphere

"""
For the simulation we will need a GradientTable with the b-values and b-vectors
Here we use the one we created in :ref:`example_gradients_spheres`.
"""

from gradients_spheres import gtab

"""
In ``mevals`` we save the eigenvalues of each tensor.
"""

mevals = np.array([[0.0015, 0.0003, 0.0003],
                   [0.0015, 0.0003, 0.0003]])

"""
In ``angles`` we save in polar coordinates (:math:`\theta, \phi`) the principal
axis of each tensor.
"""

angles = [(0, 0), (60, 0)]

"""
In ``fractions`` we save the percentage of the contribution of each tensor.
"""

fractions = [50, 50]

"""
The function ``multi_tensor`` will return the simulated signal and an array
with the principal axes of the tensors in cartesian coordinates.
"""

signal, sticks = multi_tensor(gtab, mevals, S0=100, angles=angles,
                         fractions=fractions, snr=None)

"""
We can also add rician noise with a specific SNR.
"""

signal_noisy, sticks = multi_tensor(gtab, mevals, S0=100, angles=angles,
                         fractions=fractions, snr=20)

import matplotlib.pyplot as plt

plt.plot(signal, label='noiseless')

plt.plot(signal_noisy, label='with noise')
plt.legend()
plt.show()
plt.savefig('simulated_signal.png')

"""
.. figure:: simulated_signal.png
   :align: center

   **Simulated MultiTensor signal**
"""

"""
For the ODF simulation we will need a sphere. Because we are interetested in a
simulation of only a single voxel, we can use a sphere with very high
resolution. We generate that by subdividing the triangles of one of Dipy's
cached spheres, which we can read in the following way.
"""

sphere = get_sphere('symmetric724')
sphere = sphere.subdivide(2)

odf = multi_tensor_odf(sphere.vertices, mevals, angles, fractions)

from dipy.viz import fvtk

ren = fvtk.ren()

odf_actor = fvtk.sphere_funcs(odf, sphere)
odf_actor.RotateX(90)

fvtk.add(ren, odf_actor)

print('Saving illustration as multi_tensor_simulation')
fvtk.record(ren, out_path='multi_tensor_simulation.png', size=(300, 300))

"""
.. figure:: multi_tensor_simulation.png
   :align: center

   **Simulating a MultiTensor ODF**
"""

max_stars_repo_name: ChantalTax/dipy

max_issues_repo_name: ChantalTax/dipy

max_forks_repo_name: ChantalTax/dipy


text:
from sympy import Matrix
import liesym as ls

import particlezoo.validations.gauge_invariance as tp
from particlezoo import Field, Representation

def test_is_gauge_invariant_repr():
    q = Matrix([[1, 0]])
    G = Matrix([[1, 1]])
    qb = Matrix([[0, 1]])

    terms = (q, G, qb)
    group = ls.SU(3)
    assert tp.is_gauge_invariant_repr(terms, group)

    q = Matrix([[1, 0]])
    G = Matrix([[1, 0]])
    qb = Matrix([[0, 1]])

    terms = (q, G, qb)
    group = ls.SU(3)
    assert not tp.is_gauge_invariant_repr(terms, group)

def test_is_gauge_invariant():

    quark = Field(
        "quark",
        "1/2",
        {
            "QCD": Representation(Matrix([[1, 0]]), ls.SU(3)),
            "WEAK": Representation(Matrix([[1]]), ls.SU(2)),
            "Z2": Representation("Z_0", ls.Z(2)),
        }
    )

    quark_bar = Field(
        "quark_bar",
        "1/2",
        {
            "QCD": Representation(Matrix([[0, 1]]), ls.SU(3)),
            "WEAK": Representation(Matrix([[1]]), ls.SU(2)),
            "Z2": Representation("Z_1", ls.Z(2)),
        }
    )

    gluon = Field(
        "gluon",
        "1",
        {
            "QCD": Representation(Matrix([[1, 1]]), ls.SU(3)),
            "Z2": Representation("Z_1", ls.Z(2)),
        }
    )

    assert tp.is_gauge_invariant((quark, gluon, quark_bar)) == (True, "")

    assert tp.is_gauge_invariant((quark, gluon)) == (False, "QCD")

max_stars_repo_name: npapapietro/particlezoo

max_issues_repo_name: npapapietro/particlezoo

max_forks_repo_name: npapapietro/particlezoo

reason: from sympy


text:
from __future__ import absolute_import, print_function
import os
import numpy as np
from subprocess import Popen, PIPE
from Bio.PDB.Polypeptide import aa1 as AA_STANDARD
from ...seqToolManager import SeqToolManager
from utils import myMakeDir, tryToRemove #utils is at the root of the package

class Spider2Manager(SeqToolManager):
  '''
    Computes asa and secondary structure predictions and processes their outputs.
    Extends class seqToolManager
  '''
  VAR_LIST= ["pred_asa", "P_C", "P_E", "P_H"]
  BAD_SCORE_PREDS = ["-1048576",  "-1.0",  "-1.0", "-1.0" ] #Something went wrong tag
  def __init__(self, computedFeatsRootDir, winSize=None, statusManager=None):
    '''
      :param computedFeatsRootDir: str. root path where results will be saved
      :param winSize: int>=1 or None. The size of the windows for sliding window if desired
      :param statusManager: class that implements .setStatus(msg) to communicate
    '''
    SeqToolManager.__init__(self, computedFeatsRootDir, winSize)
    # self.spider2PyScript inherited from Config.py
    self.spider2OutPath= myMakeDir(self.computedFeatsRootDir,"SPIDER2_predAsaSS")
    if winSize:
      self.spider2PathWindowed= myMakeDir(self.computedFeatsRootDir,"spyder2_wSize"+str(winSize))
    else:
      self.spider2PathWindowed= None

  def getFinalPath(self):
    '''
      returns path where results are saved
    '''
    return self.spider2OutPath

  def getFNames(self, prefixExtended):
    '''
      Returns a dict that contains the fnames that will be used  by spyder2
      :param prefixExtended. prefix for output fnames.
      :return list of fnames: [ fname1, fnam2, ...]
    '''
    spyderProc= os.path.join(self.spider2OutPath, prefixExtended+".spyder2.gz")
    fNames=[spyderProc]
    if not self.winSize is None:
      spyder2WindowedOutName= os.path.join(self.spider2PathWindowed, prefixExtended+".wsize"+str(self.winSize)+".spyder2.gz")
      fNames+= [spyder2WindowedOutName]
    return fNames

  def computeFromSeqStructMapper(self, seqStructMap, prefixExtended, pssmOutNameRaw):
    '''
      Computes spyder2 for the sequence seqStr, that is contained at fastaInFname. This sequence is
      associated with prefixExtended as an unambiguous id
      :param seqStructMap: computeFeatures.seqStep.seqToolManagers.seqExtraction.SeqStructMapper
      :param prefixExtended: str. unambiguous id of the sequence that will be the prefix of output names
      :param pssmOutNameRaw: str. Path to psiblast pssms results
    '''
    if pssmOutNameRaw is not None:
      if not os.path.isfile(pssmOutNameRaw):
        pssmOutNameRaw= pssmOutNameRaw+".gz"
      if os.path.isfile(pssmOutNameRaw):
        uncompressFileName= self.uncompressFile(pssmOutNameRaw, self.tmp)
      else:
        uncompressFileName = None
    else:
      uncompressFileName= None
    try:
      prefix, chainType, chainId= self.splitExtendedPrefix(prefixExtended)[:3]
      seqStr, fastaFname= seqStructMap.getSeq(chainType, chainId) # repeat as psiBlastManager can modify seqs
      seqStructMap.setCurrentSeq(seqStr, chainType, chainId)
      if self.checkAlreayComputed(prefixExtended):
        print("spyder2 already computed for %s"%prefixExtended)
        return 0
      fNames= self.getFNames(prefixExtended)
      spider2ProcName= fNames[0]
      spider2RawName= os.path.join(self.spider2OutPath, prefixExtended+".spd3")
      print("launching spyder2 over %s"%prefixExtended)
      curWd= os.getcwd()
      os.chdir(self.spider2OutPath)
      if uncompressFileName is not None:
        cmd= ["python", self.spider2PyScript, uncompressFileName]
        process= Popen(cmd, stdout=PIPE, stderr=PIPE)
        processOut= process.communicate()
        os.chdir(curWd)
        if len(processOut[1])>0:
          print("Error computing spider2. Caught stdin/stderr:\n",processOut[0],processOut[1])
      else:
        spider2RawName= None
      dataList= self.processSpider2(seqStr, seqStructMap, prefixExtended, spider2RawName, spider2ProcName)

      if self.winSize:
        self.makeWindowed( dataList, ["asa", "P_C", "P_E", "P_H"], Spider2Manager.BAD_SCORE_PREDS, [None]*4,
                           fNames[1])
    except (Exception, KeyboardInterrupt):
      self.tryToRemoveAllFnames(prefixExtended)
      raise
    finally:
      if uncompressFileName is not None:
        tryToRemove(uncompressFileName)

  def processSpider2(self, seq, seqStructMap, prefixExtended, spider2Raw, spider2Proc):
    '''
      Reads spider2 output file and writes another one with tabulated format, headers and
      some error checking.
      :param: seq: str. Sequence of the chain
      :param prefixExtended: str. unambiguous id of the sequence that will be the prefix of output names
      :param spider2Raw: str. Path to spyder2 results
      :param spider2Proc: str. Path where formated results will be saved.
    '''

    if spider2Raw is not None:
      try:
        predictionsData = self.loadSpider2(spider2Raw)
      except IOError:
        predictionsData= [ (letter, type(self).BAD_SCORE_PREDS ) for letter in seq]
    else:
      predictionsData = [(letter, type(self).BAD_SCORE_PREDS) for letter in seq]

    prefix, chainType, chainId= self.splitExtendedPrefix(prefixExtended)[:3]
    try:
      spyderIx=0
      seqIx=0
      seqLen= len(seq)
      spyderLen= len(predictionsData)
      dataList=[]
      listOfRowsToPrint=[]

      while seqIx<seqLen and spyderIx<spyderLen:
        letter= seq[seqIx]
        letterSpider, predsVal= predictionsData[spyderIx]
        if letterSpider== letter:
          structIndex= seqStructMap.seqToStructIndex(chainType, chainId, seqIx, asString= True)
          if structIndex:
            if self.filterOutLabels and structIndex[-1].isalpha():
              continue
          else:
            structIndex=str(seqIx)+"?"
          dataList.append( ( (chainId, structIndex,letter), ( [predsVal[0]], predsVal[1:]) ) )
          listOfRowsToPrint.append( "%s %s %s %s %s %s %s"%(( chainId, structIndex, letter)+tuple(predsVal)) )
          spyderIx+=1
          seqIx+=1
        elif not letter in AA_STANDARD and letterSpider=="-":
          spyderIx+=1
          seqIx+=1
        elif letterSpider=="-":
          spyderIx+=1
        else:
          print(predictionsData)
          print(spyderIx, seqIx)
          raise ValueError("Spyder mismatch %s %s "%(letterSpider, letter))

      self.writeResultsFromDataDictSingleChain( {chainId: listOfRowsToPrint }, outName= spider2Proc)
      return dataList
    except (KeyboardInterrupt, Exception):
      print("Exception happend computing %s"%spider2Proc)
      tryToRemove(spider2Proc)
      raise
    finally:
      if spider2Raw is not None:
        tryToRemove(spider2Raw)
      pass

  def loadSpider2(self, fileName):
    '''
      Loads an spider2 file
      :param fileName: str. Path to spider2 file.

              #       AA      SS      ASA     Phi     Psi     Theta(i-1=>i+1) Tau(i-2=>i+1)   P(C)    P(E)    P(H)
        1       E       C       135.0    -89.3   131.8   116.5  -113.0  0.977   0.010   0.011

      :return  list of strings. ["row0_spider2","row1spider2"...]
    '''
    dataList= []
    if not os.path.isfile(fileName):
      fileName+=".gz"
    with self.openForReadingFnameOrGz(fileName) as f:
      f.readline()
      for line in f:
        lineArray=line.split()
        if lineArray[0][0].isdigit():
          dataList.append( (lineArray[1],  tuple([lineArray[3]] + lineArray[8:] )))
        else:
          break
    return dataList

max_stars_repo_name: rsanchezgarc/BIPSPI

max_issues_repo_name: rsanchezgarc/BIPSPI

max_forks_repo_name: rsanchezgarc/BIPSPI


text:
import pytest
import numpy as np
from copy import copy

from orbit.models.dlt import DLTFull, DLTAggregated, DLTMAP
from orbit.estimators.stan_estimator import StanEstimatorMCMC, StanEstimatorVI
from orbit.initializer.dlt import DLTInitializer

@pytest.mark.parametrize("model_class", [DLTMAP, DLTFull, DLTAggregated])
def test_base_ets_init(model_class):
    dlt = model_class()

    is_fitted = dlt.is_fitted()

    model_data_input = dlt.get_model_data_input()
    model_param_names = dlt.get_model_param_names()
    init_values = dlt.get_init_values()

    # model is not yet fitted
    assert not is_fitted
    # should only be initialized and not set
    assert not model_data_input
    # model param names should already be set
    assert model_param_names
    # callable is not implemented yet
    assert not init_values

@pytest.mark.parametrize("estimator_type", [StanEstimatorMCMC, StanEstimatorVI])
def test_dlt_full_univariate(synthetic_data, estimator_type):
    train_df, test_df, coef = synthetic_data

    dlt = DLTFull(
        response_col='response',
        date_col='week',
        prediction_percentiles=[5, 95],
        seasonality=52,
        num_warmup=50,
        verbose=False,
        estimator_type=estimator_type
    )

    dlt.fit(train_df)

    init_call = dlt.get_init_values()
    assert isinstance(init_call, DLTInitializer)
    assert init_call.s == 52
    init_values = init_call()
    assert init_values['init_sea'].shape == (51, )

    predict_df = dlt.predict(test_df)

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    expected_num_parameters = 13

    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns
    assert len(dlt._posterior_samples) == expected_num_parameters

@pytest.mark.parametrize("estimator_type", [StanEstimatorMCMC, StanEstimatorVI])
def test_dlt_aggregated_univariate(synthetic_data, estimator_type):
    train_df, test_df, coef = synthetic_data

    dlt = DLTAggregated(
        response_col='response',
        date_col='week',
        seasonality=52,
        num_warmup=50,
        verbose=False,
        estimator_type=estimator_type
    )

    dlt.fit(train_df)

    init_call = dlt.get_init_values()
    assert isinstance(init_call, DLTInitializer)
    assert init_call.s == 52
    init_values = init_call()
    assert init_values['init_sea'].shape == (51, )

    predict_df = dlt.predict(test_df)

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    expected_num_parameters = 13

    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns
    assert len(dlt._posterior_samples) == expected_num_parameters

def test_dlt_map_univariate(synthetic_data):
    train_df, test_df, coef = synthetic_data

    dlt = DLTMAP(
        response_col='response',
        date_col='week',
        seasonality=52,
        num_warmup=50,
        verbose=False,
    )

    dlt.fit(train_df)

    init_call = dlt.get_init_values()
    assert isinstance(init_call, DLTInitializer)
    assert init_call.s == 52
    init_values = init_call()
    assert init_values['init_sea'].shape == (51, )

    predict_df = dlt.predict(test_df)

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    expected_num_parameters = 12  # no `lp__` parameter in optimizing()

    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns
    assert len(dlt._posterior_samples) == expected_num_parameters

@pytest.mark.parametrize("estimator_type", [StanEstimatorMCMC, StanEstimatorVI])
def test_dlt_non_seasonal_fit(synthetic_data, estimator_type):
    train_df, test_df, coef = synthetic_data

    dlt = DLTFull(
        response_col='response',
        date_col='week',
        estimator_type=estimator_type,
        num_warmup=50,
    )

    dlt.fit(train_df)
    predict_df = dlt.predict(test_df)

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    expected_num_parameters = 11

    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns
    assert len(dlt._posterior_samples) == expected_num_parameters

@pytest.mark.parametrize("estimator_type", [StanEstimatorMCMC, StanEstimatorVI])
@pytest.mark.parametrize(
    "regressor_signs",
    [
        ["+", "+", "+", "+", "+", "+"],
        ["-", "-", "-", "-", "-", "-"],
        ["=", "=", "=", "=", "=", "="],
        ["+", "=", "+", "=", "-", "-"]
    ],
    ids=['positive_only', 'negative_only', 'regular_only', 'mixed_signs']
)
def test_dlt_full_with_regression(synthetic_data, estimator_type, regressor_signs):
    train_df, test_df, coef = synthetic_data

    dlt = DLTFull(
        response_col='response',
        date_col='week',
        regressor_col=train_df.columns.tolist()[2:],
        regressor_sign=regressor_signs,
        prediction_percentiles=[5, 95],
        seasonality=52,
        num_warmup=50,
        verbose=False,
        estimator_type=estimator_type
    )

    dlt.fit(train_df)
    init_call = dlt.get_init_values()
    assert isinstance(init_call, DLTInitializer)
    init_values = init_call()
    assert init_values['init_sea'].shape == (51, )

    if regressor_signs.count('+') > 0:
        assert init_values['pr_beta'].shape == (regressor_signs.count('+'), )
    if regressor_signs.count('-') > 0:
        assert init_values['nr_beta'].shape == (regressor_signs.count('-'), )
    if regressor_signs.count('=') > 0:
        assert init_values['rr_beta'].shape == (regressor_signs.count('='), )

    predict_df = dlt.predict(test_df)

    regression_out = dlt.get_regression_coefs()
    num_regressors = regression_out.shape[0]

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    expected_regression_shape = (6, 3)

    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns
    assert regression_out.shape == expected_regression_shape
    assert num_regressors == len(train_df.columns.tolist()[2:])

    assert np.sum(regression_out['coefficient'].values >= 0) <= \
           regressor_signs.count('+') + regressor_signs.count('=')
    assert np.sum(regression_out['coefficient'].values <= 0) <= \
           regressor_signs.count('-') + regressor_signs.count('=')

@pytest.mark.parametrize("estimator_type", [StanEstimatorMCMC, StanEstimatorVI])
@pytest.mark.parametrize(
    "regressor_signs",
    [
        ["+", "+", "+", "+", "+", "+"],
        ["-", "-", "-", "-", "-", "-"],
        ["=", "=", "=", "=", "=", "="],
        ["+", "=", "+", "=", "-", "-"]
    ],
    ids=['positive_only', 'negative_only', 'regular_only', 'mixed_signs']
)
def test_dlt_aggregated_with_regression(synthetic_data, estimator_type, regressor_signs):
    train_df, test_df, coef = synthetic_data

    dlt = DLTAggregated(
        response_col='response',
        date_col='week',
        regressor_col=train_df.columns.tolist()[2:],
        regressor_sign=regressor_signs,
        seasonality=52,
        num_warmup=50,
        verbose=False,
        estimator_type=estimator_type
    )

    dlt.fit(train_df)
    predict_df = dlt.predict(test_df)

    regression_out = dlt.get_regression_coefs()
    num_regressors = regression_out.shape[0]

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    expected_regression_shape = (6, 3)

    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns
    assert regression_out.shape == expected_regression_shape
    assert num_regressors == len(train_df.columns.tolist()[2:])

    predict_df = dlt.predict(test_df, decompose=True)
    assert any(predict_df['regression'].values)

@pytest.mark.parametrize("global_trend_option", ["linear", "loglinear", "logistic", "flat"])
def test_dlt_map_global_trend(synthetic_data, global_trend_option):
    train_df, test_df, coef = synthetic_data

    dlt = DLTMAP(
        response_col='response',
        date_col='week',
        seasonality=52,
        global_trend_option=global_trend_option,
    )

    dlt.fit(train_df)
    predict_df = dlt.predict(test_df)

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns

@pytest.mark.parametrize(
    "regressor_signs",
    [
        ['=', '=', '+'],
        ['=', '=', '-'],
        ['+', '-', '+'],
        ['+', '-', '=']
    ],
    ids=['positive_mixed', 'negative_mixed', 'positive_negative', 'mixed']
)
def test_dlt_mixed_signs_and_order(iclaims_training_data, regressor_signs):
    df = iclaims_training_data
    df['claims'] = np.log(df['claims'])
    raw_regressor_col = ['trend.unemploy', 'trend.filling', 'trend.job']
    new_regressor_col = [raw_regressor_col[idx] for idx in [1, 2, 0]]
    new_regressor_signs = [regressor_signs[idx] for idx in [1, 2, 0]]
    # mixiing ordering of cols in df of prediction
    new_df = df[['claims', 'week'] + new_regressor_col]

    dlt = DLTMAP(
        response_col='claims',
        date_col='week',
        regressor_col=raw_regressor_col,
        regressor_sign=regressor_signs,
        seasonality=52,
        seed=8888,
    )
    dlt.fit(df)
    predicted_df_v1 = dlt.predict(df)
    predicted_df_v2 = dlt.predict(new_df)

    # mixing ordering of signs
    dlt_new = DLTMAP(
        response_col='claims',
        date_col='week',
        regressor_col=new_regressor_col,
        regressor_sign=new_regressor_signs,
        seasonality=52,
        seed=8888,
    )
    dlt_new.fit(df)
    predicted_df_v3 = dlt_new.predict(df)
    predicted_df_v4 = dlt_new.predict(new_df)

    pred_v1 = predicted_df_v1['prediction'].values
    pred_v2 = predicted_df_v2['prediction'].values
    pred_v3 = predicted_df_v3['prediction'].values
    pred_v4 = predicted_df_v4['prediction'].values

    # they should be all identical; ordering of signs or columns in prediction show not matter
    assert np.allclose(pred_v1, pred_v2, atol=1e-3)
    assert np.allclose(pred_v1, pred_v3, atol=1e-3)
    assert np.allclose(pred_v1, pred_v4, atol=1e-3)

@pytest.mark.parametrize("estimator_type", [StanEstimatorMCMC, StanEstimatorVI])
@pytest.mark.parametrize(
    "regressor_signs",
    [
        ["+", "+", "+", "+", "+", "+"],
        ["=", "=", "=", "=", "=", "="],
        ["-", "-", "-", "-", "-", "-"],
        ["+", "=", "+", "=", "+", "+"],
        ["-", "=", "-", "=", "-", "="],
        ["+", "=", "+", "=", "-", "-"],
    ],
    ids=['positive_only', 'regular_only', 'negative_only',
         'positive_mixed', 'negative_mixed', 'mixed_signs']
)
@pytest.mark.parametrize("seasonality", [1, 52])
def test_dlt_full_reproducibility(synthetic_data, estimator_type, regressor_signs, seasonality):
    train_df, test_df, coef = synthetic_data

    dlt_first = DLTFull(
        response_col='response',
        date_col='week',
        regressor_col=train_df.columns.tolist()[2:],
        regressor_sign=regressor_signs,
        prediction_percentiles=[5, 95],
        seasonality=seasonality,
        num_warmup=50,
        verbose=False,
        estimator_type=estimator_type
    )

    # first fit and predict
    dlt_first.fit(train_df)
    posteriors_first = copy(dlt_first._posterior_samples)
    predict_df_first = dlt_first.predict(test_df)
    regression_out_first = dlt_first.get_regression_coefs()

    # second fit and predict
    # note a new instance must be created to reset the seed
    # note both fit and predict contain random generation processes
    dlt_second = DLTFull(
        response_col='response',
        date_col='week',
        regressor_col=train_df.columns.tolist()[2:],
        regressor_sign=regressor_signs,
        prediction_percentiles=[5, 95],
        seasonality=seasonality,
        num_warmup=50,
        verbose=False,
        estimator_type=estimator_type
    )

    dlt_second.fit(train_df)
    posteriors_second = copy(dlt_second._posterior_samples)
    predict_df_second = dlt_second.predict(test_df)
    regression_out_second = dlt_second.get_regression_coefs()

    # assert same posterior keys
    assert set(posteriors_first.keys()) == set(posteriors_second.keys())

    # assert posterior draws are reproducible
    for k, v in posteriors_first.items():
        assert np.allclose(posteriors_first[k], posteriors_second[k])

    # assert identical regression columns
    # this is also checked in posterior samples, but an extra layer just in case
    # since this one very commonly retrieved by end users
    assert regression_out_first.equals(regression_out_second)

    # assert prediction is reproducible
    assert predict_df_first.equals(predict_df_second)

@pytest.mark.parametrize("seasonality", [1, 52])
def test_dlt_map_reproducibility(synthetic_data, seasonality):
    train_df, test_df, coef = synthetic_data

    dlt1 = DLTMAP(
        response_col='response',
        date_col='week',
        prediction_percentiles=[5, 95],
        seasonality=seasonality,
    )

    # first fit and predict
    dlt1.fit(train_df)
    posteriors1 = copy(dlt1._aggregate_posteriors['map'])
    prediction1 = dlt1.predict(test_df)

    # second fit and predict
    # note a new instance must be created to reset the seed
    # note both fit and predict contain random generation processes
    dlt2 = DLTMAP(
        response_col='response',
        date_col='week',
        prediction_percentiles=[5, 95],
        seasonality=seasonality,
    )

    dlt2.fit(train_df)
    posteriors2 = copy(dlt2._aggregate_posteriors['map'])
    prediction2 = dlt2.predict(test_df)

    # assert same posterior keys
    assert set(posteriors1.keys()) == set(posteriors2.keys())

    # assert posterior draws are reproducible
    for k, v in posteriors1.items():
        assert np.allclose(posteriors1[k], posteriors2[k])

    # assert prediction is reproducible
    assert np.allclose(prediction1['prediction'].values, prediction2['prediction'].values)

@pytest.mark.parametrize("regression_penalty", ['fixed_ridge', 'lasso', 'auto_ridge'])
def test_dlt_regression_penalty(synthetic_data, regression_penalty):
    train_df, test_df, coef = synthetic_data

    dlt = DLTMAP(
        response_col='response',
        date_col='week',
        regressor_col=train_df.columns.tolist()[2:],
        regression_penalty=regression_penalty,
        seasonality=52,
        num_warmup=50,
        verbose=False,
    )

    dlt.fit(train_df)
    predict_df = dlt.predict(test_df)

    regression_out = dlt.get_regression_coefs()
    num_regressors = regression_out.shape[0]

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    expected_regression_shape = (6, 3)

    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns
    assert regression_out.shape == expected_regression_shape
    assert num_regressors == len(train_df.columns.tolist()[2:])

@pytest.mark.parametrize("level_sm_input", [0.0001, 0.5, 1.0])
@pytest.mark.parametrize("seasonality_sm_input", [0.0, 0.5, 1.0])
@pytest.mark.parametrize("slope_sm_input", [0.0, 0.5, 1.0])
def test_dlt_fixed_sm_input(synthetic_data, level_sm_input, seasonality_sm_input, slope_sm_input):
    train_df, test_df, coef = synthetic_data

    dlt = DLTMAP(
        response_col='response',
        date_col='week',
        regressor_col=train_df.columns.tolist()[2:],
        level_sm_input=level_sm_input,
        seasonality_sm_input=seasonality_sm_input,
        slope_sm_input=slope_sm_input,
        seasonality=52,
        num_warmup=50,
        verbose=False,
    )

    dlt.fit(train_df)
    predict_df = dlt.predict(test_df)

    regression_out = dlt.get_regression_coefs()
    num_regressors = regression_out.shape[0]

    expected_columns = ['week', 'prediction_5', 'prediction', 'prediction_95']
    expected_shape = (51, len(expected_columns))
    expected_regression_shape = (6, 3)

    assert predict_df.shape == expected_shape
    assert predict_df.columns.tolist() == expected_columns
    assert regression_out.shape == expected_regression_shape
    assert num_regressors == len(train_df.columns.tolist()[2:])

max_stars_repo_name: pochoi/orbit

max_issues_repo_name: pochoi/orbit

max_forks_repo_name: pochoi/orbit


text:
import os
import tensorflow as tf
import numpy as np
import json

from models_clevr.model import LCGNnet
from models_clevr.config import build_cfg_from_argparse
from models_clevr.vis import vis_batch_ref
from util.clevr_train.data_reader import DataReader

# Load config
cfg = build_cfg_from_argparse()

# Start session
os.environ["CUDA_VISIBLE_DEVICES"] = cfg.GPUS
sess = tf.Session(config=tf.ConfigProto(
    allow_soft_placement=True, gpu_options=tf.GPUOptions(allow_growth=True)))

def load_train_data(max_num=0):
    imdb_file = cfg.IMDB_FILE % cfg.TRAIN.SPLIT_REF
    data_reader = DataReader(
        imdb_file, shuffle=True, max_num=max_num,
        batch_size=cfg.TRAIN.BATCH_SIZE,
        vocab_question_file=cfg.VOCAB_QUESTION_FILE,
        T_encoder=cfg.T_ENCODER,
        vocab_answer_file=cfg.VOCAB_ANSWER_FILE,
        load_spatial_feature=True,
        spatial_feature_dir=cfg.SPATIAL_FEATURE_DIR,
        add_pos_enc=cfg.ADD_POS_ENC, img_H=cfg.IMG_H, img_W=cfg.IMG_W,
        pos_enc_dim=cfg.PE_DIM, pos_enc_scale=cfg.PE_SCALE)
    num_vocab = data_reader.batch_loader.vocab_dict.num_vocab
    num_choices = data_reader.batch_loader.answer_dict.num_vocab
    return data_reader, num_vocab, num_choices

def run_train_on_data(model, data_reader_train, saver_train, lr_start,
                      run_eval=False, data_reader_eval=None, saver_eval=None):
    lr = lr_start
    correct, total, loss_sum, batch_num = 0, 0, 0., 0
    prev_loss = None
    for batch, n_sample, e in data_reader_train.batches(one_pass=False):
        n_epoch = cfg.TRAIN.START_EPOCH + e
        if n_sample == 0 and n_epoch > cfg.TRAIN.START_EPOCH:
            print('')
            # save snapshot
            snapshot_file = cfg.SNAPSHOT_FILE % (cfg.EXP_NAME, n_epoch)
            saver_train.save(sess, snapshot_file, write_meta_graph=False)
            states_file = snapshot_file + '_states.npy'
            np.save(states_file, {'lr': lr})
            # run evaluation
            if run_eval:
                saver_eval.restore(sess, snapshot_file)
                run_eval_on_data(model, data_reader_eval)
                saver_train.restore(sess, snapshot_file)
            # adjust lr:
            curr_loss = loss_sum/batch_num
            if prev_loss is not None:
                lr = adjust_lr_clevr(curr_loss, prev_loss, lr)
            # clear stats
            correct, total, loss_sum, batch_num = 0, 0, 0., 0
            prev_loss = curr_loss
        if n_epoch >= cfg.TRAIN.MAX_EPOCH:
            break
        batch_res = model.run_batch(
            sess, batch, train=True, run_vqa=False, run_ref=True, lr=lr)
        correct += batch_res['bbox_num_correct']
        total += batch_res['batch_size']
        loss_sum += batch_res['loss']
        batch_num += 1
        print('\rTrain E %d S %d: avgL=%.4f, avgA=%.4f, lr=%.1e' % (
                n_epoch+1, total, loss_sum/batch_num, correct/total, lr),
              end='')

def adjust_lr_clevr(curr_los, prev_loss, curr_lr):
    loss_diff = prev_loss - curr_los
    not_improve = (
        (loss_diff < 0.015 and prev_loss < 0.5 and curr_lr > 0.00002) or
        (loss_diff < 0.008 and prev_loss < 0.15 and curr_lr > 0.00001) or
        (loss_diff < 0.003 and prev_loss < 0.10 and curr_lr > 0.000005))

    next_lr = curr_lr * cfg.TRAIN.SOLVER.LR_DECAY if not_improve else curr_lr
    return next_lr

def load_eval_data(max_num=0):
    imdb_file = cfg.IMDB_FILE % cfg.TEST.SPLIT_REF
    data_reader = DataReader(
        imdb_file, shuffle=False, max_num=max_num,
        batch_size=cfg.TEST.BATCH_SIZE,
        vocab_question_file=cfg.VOCAB_QUESTION_FILE,
        T_encoder=cfg.T_ENCODER,
        vocab_answer_file=cfg.VOCAB_ANSWER_FILE,
        load_spatial_feature=True,
        spatial_feature_dir=cfg.SPATIAL_FEATURE_DIR,
        add_pos_enc=cfg.ADD_POS_ENC, img_H=cfg.IMG_H, img_W=cfg.IMG_W,
        pos_enc_dim=cfg.PE_DIM, pos_enc_scale=cfg.PE_SCALE)
    num_vocab = data_reader.batch_loader.vocab_dict.num_vocab
    num_choices = data_reader.batch_loader.answer_dict.num_vocab
    return data_reader, num_vocab, num_choices

def run_eval_on_data(model, data_reader_eval, pred=False, vis=False,
                     vis_dir=None):
    predictions = []
    correct, total, loss_sum, batch_num = 0, 0, 0., 0
    for batch, _, _ in data_reader_eval.batches(one_pass=True):
        batch_res = model.run_batch(
            sess, batch, train=False, run_vqa=False, run_ref=True, vis=vis)
        if vis and total < cfg.TEST.NUM_VIS:
            vis_batch_ref(data_reader_eval, batch, batch_res, total, vis_dir)
        if pred:
            predictions.extend([
                {'questionId': q, 'prediction': [float(x) for x in p]}
                for q, p in zip(
                    batch['qid_list'], batch_res['bbox_predictions'])])
        correct += batch_res['bbox_num_correct']
        total += batch_res['batch_size']
        loss_sum += batch_res['loss']
        batch_num += 1
        print('\rEval S %d: avgL=%.4f, avgA=%.4f' % (
            total, loss_sum/batch_num, correct/total), end='')
    print('')
    eval_res = {
        'correct': correct,
        'total': total,
        'accuracy': correct/total,
        'loss': loss_sum/batch_num,
        'predictions': predictions}
    return eval_res

def dump_prediction_to_file(predictions, res_dir):
    pred_file = os.path.join(res_dir, 'pred_bbox_%s_%04d_%s.json' % (
        cfg.EXP_NAME, cfg.TEST.EPOCH, cfg.TEST.SPLIT_REF))
    with open(pred_file, 'w') as f:
        json.dump(predictions, f, indent=2)
    print('predictions written to %s' % pred_file)

def train():
    data_reader_train, num_vocab, num_choices = load_train_data()
    data_reader_eval, _, _ = load_eval_data(max_num=cfg.TRAIN.EVAL_MAX_NUM)

    # Load model
    model = LCGNnet(num_vocab, num_choices, gpusNum=len(cfg.GPUS.split(',')))

    # Save snapshot
    saver_train = tf.train.Saver(max_to_keep=None)  # keep all snapshots
    saver_eval = tf.train.Saver(model.emaDict if cfg.USE_EMA else None)
    snapshot_dir = os.path.dirname(cfg.SNAPSHOT_FILE % (cfg.EXP_NAME, 0))
    os.makedirs(snapshot_dir, exist_ok=True)
    with open(os.path.join(snapshot_dir, 'cfg.json'), 'w') as f:
        json.dump(cfg, f, indent=2)
    lr_start = cfg.TRAIN.SOLVER.LR
    if cfg.TRAIN.START_EPOCH > 0:
        print('resuming from epoch %d' % cfg.TRAIN.START_EPOCH)
        snapshot_file = cfg.SNAPSHOT_FILE % (
            cfg.EXP_NAME, cfg.TRAIN.START_EPOCH)
        saver_train.restore(sess, snapshot_file)
        states_file = snapshot_file + '_states.npy'
        if os.path.exists(states_file):
            lr_start = np.load(states_file, allow_pickle=True)[()]['lr']
            print('recovered previous lr %.1e' % lr_start)
        else:
            print('could not recover previous lr')
    else:
        sess.run(tf.global_variables_initializer())
    sess.graph.finalize()

    print('%s - train for %d epochs' % (cfg.EXP_NAME, cfg.TRAIN.MAX_EPOCH))
    run_train_on_data(
        model, data_reader_train, saver_train, lr_start,
        run_eval=cfg.TRAIN.RUN_EVAL, data_reader_eval=data_reader_eval,
        saver_eval=saver_eval)
    print('%s - train (done)' % cfg.EXP_NAME)

def test():
    data_reader_eval, num_vocab, num_choices = load_eval_data()

    # Load model
    model = LCGNnet(num_vocab, num_choices, gpusNum=len(cfg.GPUS.split(',')))

    # Load test snapshot
    saver_eval = tf.train.Saver(model.emaDict if cfg.USE_EMA else None)
    snapshot_file = cfg.SNAPSHOT_FILE % (cfg.EXP_NAME, cfg.TEST.EPOCH)
    saver_eval.restore(sess, snapshot_file)
    sess.graph.finalize()

    res_dir = cfg.TEST.RESULT_DIR % (cfg.EXP_NAME, cfg.TEST.EPOCH)
    vis_dir = os.path.join(
        res_dir, '%s_%s' % (cfg.TEST.VIS_DIR_PREFIX, cfg.TEST.SPLIT_REF))
    os.makedirs(res_dir, exist_ok=True)
    os.makedirs(vis_dir, exist_ok=True)
    pred = cfg.TEST.DUMP_PRED
    if not pred:
        print('NOT writing predictions (set TEST.DUMP_PRED True to write)')

    print('%s - test epoch %d' % (cfg.EXP_NAME, cfg.TEST.EPOCH))
    eval_res = run_eval_on_data(
        model, data_reader_eval, pred=pred, vis=True, vis_dir=vis_dir)
    print('%s - test epoch %d: accuracy = %.4f' % (
        cfg.EXP_NAME, cfg.TEST.EPOCH, eval_res['accuracy']))

    # write results
    if pred:
        dump_prediction_to_file(eval_res['predictions'], res_dir)
    eval_res.pop('predictions')
    res_file = os.path.join(res_dir, 'res_%s_%04d_%s.json' % (
        cfg.EXP_NAME, cfg.TEST.EPOCH, cfg.TEST.SPLIT_REF))
    with open(res_file, 'w') as f:
        json.dump(eval_res, f)

if __name__ == '__main__':
    if cfg.train:
        train()
    else:
        test()

max_stars_repo_name: geetickachauhan/lcgn

max_issues_repo_name: geetickachauhan/lcgn

max_forks_repo_name: geetickachauhan/lcgn