text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jul  9 10:54:32 2019

@author: nmei
"""

import os
from glob import glob
import pandas as pd
import numpy as np
import seaborn as sns
sns.set_style('whitegrid')
sns.set_context('poster')
import statsmodels.api as sm
from statsmodels.formula.api import ols
from shutil import copyfile
copyfile('../../../utils.py','utils.py')
import utils
from matplotlib.ticker import FormatStrFormatter

experiment = 'metasema'
here = 'encoding_model_15_ROIs'
working_dir = '../../../../results/{}/RP/{}'.format(experiment,here)
figure_dir = '../../../../figures/{}/RP/{}'.format(experiment,here)
if not os.path.exists(figure_dir):
    os.mkdir(figure_dir)

working_data = glob(os.path.join(working_dir,'*.csv'))
df = pd.concat([pd.read_csv(f) for f in working_data]).groupby(
        ['sub_name',
         'roi_name',
         'model_name',
         'condition',
         ]).mean().reset_index()
N = len(pd.unique(df['sub_name']))
alpha = 100

feature_dict = {'vgg19':'image2vec',
                'densenet121':'image2vec',
                'mobilenetv2_1':'image2vec',
                'Fast_Text':'Word2vec',
                'Glove':'Word2vec',
                'Word2Vec':'Word2vec',}
df['feature_type'] = df['model_name'].map(feature_dict)
hue_order = ['vgg19', 'densenet121', 'mobilenetv2_1',
             'Fast_Text', 'Glove', 'Word2Vec',
             ]
df = pd.concat([df[df['model_name'] == model_name] for model_name in hue_order])

temp = dict(
        F = [],
        df_nonimator = [],
        df_denominator = [],
        p = [],
        feature_type = [],
        condition = [],
        roi_name = [],
        )
for (feat,condition,roi),df_sub in df.groupby(['feature_type','condition','roi_name']):
    anova = ols('mean_variance ~ model_name',data = df_sub).fit()
    aov_table = sm.stats.anova_lm(anova,typ=2)
    print(aov_table)
    temp['F'].append(aov_table['F']['model_name'])
    temp['df_nonimator'].append(aov_table['df']['model_name'])
    temp['df_denominator'].append(aov_table['df']['Residual'])
    temp['p'].append(aov_table['PR(>F)']['model_name'])
    temp['feature_type'].append(feat)
    temp['condition'].append(condition)
    temp['roi_name'].append(roi)
anova_results = pd.DataFrame(temp)
temp = []
for roi,df_sub in anova_results.groupby('condition'):
    df_sub = df_sub.sort_values('p')
    converter = utils.MCPConverter(pvals = df_sub['p'].values)
    d = converter.adjust_many()
    df_sub['p_corrected'] = d['bonferroni'].values
    temp.append(df_sub)
anova_results = pd.concat(temp)
anova_results['stars'] = anova_results['p_corrected'].apply(utils.stars)
anova_results = anova_results.sort_values(['roi_name','condition','feature_type'])

g = sns.catplot(x = 'roi_name',
                y = 'mean_variance',
                hue = 'model_name',
                row = 'condition',
                data = df,
                kind = 'bar',
                aspect = 4,
                )
g._legend.set_title('Encoding Models')
(g.set_axis_labels("","Mean Variance Explained")
  .set_titles("{row_name}")
  .set(ylim=(0,0.05)))
g.axes[-1][0].set_xticklabels(g.axes[-1][0].xaxis.get_majorticklabels(),
      rotation = 45,
      horizontalalignment = 'center')
g.axes[0][0].set(title = 'Read')
g.axes[1][0].set(title = 'Think')
k = {'image2vec':-0.25,
     'Word2vec':0.175}
j = 0.15
l = 0.0005
height = 0.045
for ax,condition in zip(g.axes.flatten(),['read','reenact']):
    ax.yaxis.set_major_formatter(FormatStrFormatter('%.3f'))
    df_sub = anova_results[anova_results['condition'] == condition]
    for ii,((roi),df_star_sub) in enumerate(df_sub.groupby(['roi_name',])):
        for model,df_star_sub_model in df_star_sub.groupby(['feature_type']):
            ax.hlines(height,ii+k[model]-j,ii+k[model]+j)
            ax.vlines(ii+k[model]-j,height+l,height-l)
            ax.vlines(ii+k[model]+j,height+l,height-l)
            ax.annotate(df_star_sub_model['stars'].values[0],xy=(ii+k[model]-0.25,height + 0.002))
g.fig.suptitle("Comparison between Computer Vision and Word Embedding Models\nAverage Variance Explained\nN = {}, {} (alpha = {})\nBonforroni corrected for multiple one-way ANOVAs".format(
        N,"Ridge Regression",alpha),
            y = 1.15)

g.savefig(os.path.join(figure_dir,
                       'mean variance explained.png'),
        dpi = 400,
        bbox_inches = 'tight')

results = []
for image in ['densenet121','mobilenetv2_1','vgg19',]:
    for word in ['Fast_Text', 'Glove', 'Word2Vec',]:
        df_image = df[df['model_name'] == image]
        df_word = df[df['model_name'] == word]

        df_image = df_image.sort_values(['sub_name','roi_name','condition'])
        df_word = df_word.sort_values(['sub_name','roi_name','condition'])

        MV_diff = df_image['mean_variance'].values - df_word['mean_variance'].values
        BV_diff = df_image['best_variance'].values - df_word['best_variance'].values

        df_diff = df_image.copy()
        df_diff['mean_variance'] = MV_diff
        df_diff['best_variance'] = BV_diff
        df_diff['model_name'] = f"{image} - {word}"
        results.append(df_diff)
df_diff = pd.concat(results)

g = sns.catplot(x = 'roi_name',
                y = 'mean_variance',
                hue = 'model_name',
                row = 'condition',
                data = df_diff,
                kind = 'bar',
                aspect = 4,
                )
(g.set_axis_labels("","$\Delta$ Mean Variance Explained")
  .set_titles("{row_name}"))
g.axes[-1][0].set_xticklabels(g.axes[-1][0].xaxis.get_majorticklabels(),
      rotation = 45,
      horizontalalignment = 'center')
g.axes[0][0].set(title = 'Read')
g.axes[1][0].set(title = 'Think')
g.fig.suptitle('Difference of Computer Vision Models and Word Embedding Models\nBonforroni Corrected for multiple t-tests',
               y=1.05)

g.savefig(os.path.join(figure_dir,
                       'difference of variance explained by image2vec and word2vec.png'),
        dpi = 400,
        bbox_inches = 'tight')

df_condition = dict(
        roi = [],
        condition = [],
        ps_mean = [],
        ps_std = [],
        diff_mean = [],
        diff_std = [],)
for (roi,condition),df_sub in df.groupby(['roi_name','condition']):
    df_sub_img = df_sub[df_sub['feature_type'] == 'image2vec'].groupby(['sub_name']).mean().reset_index()
    df_sub_word = df_sub[df_sub['feature_type'] == 'Word2vec'].groupby(['sub_name']).mean().reset_index()
    a = df_sub_img['mean_variance'].values
    b = df_sub_word['mean_variance'].values
    ps = utils.resample_ttest_2sample(a,b,
                                      n_ps = 100,
                                      n_permutation = int(1e4),
                                      one_tail = False,
                                      match_sample_size = True)
    df_condition['roi'].append(roi)
    df_condition['condition'].append(condition)
    df_condition['ps_mean'].append(ps.mean())
    df_condition['ps_std'].append(ps.std())
    df_condition['diff_mean'].append(np.mean(a - b))
    df_condition['diff_std'].append(np.std(a - b))
df_condition = pd.DataFrame(df_condition)

temp = []
for condition, df_sub in df_condition.groupby(['condition']):
    df_sub = df_sub.sort_values(['ps_mean'])
    converter = utils.MCPConverter(pvals = df_sub['ps_mean'].values)
    d = converter.adjust_many()
    df_sub['ps_corrected'] = d['bonferroni'].values
    temp.append(df_sub)
df_condition = pd.concat(temp)

df_diff_diff = dict(
        roi = [],
        ps_mean = [],
        ps_std = [],
        diff_mean = [],
        diff_std = [],)
for roi,df_sub in df_diff.groupby(['roi_name']):
    df_read = df_sub[df_sub['condition'] == 'read']#.sort_values(['sub'])
    df_reenact = df_sub[df_sub['condition'] == 'reenact']#.sort_values(['sub'])

    df_read = df_read.sort_values(['sub_name','roi_name','feature_type']).reset_index()
    df_reenact = df_reenact.sort_values(['sub_name','roi_name','feature_type']).reset_index()

    a = df_read['mean_variance'].values
    b = df_reenact['mean_variance'].values
    ps = utils.resample_ttest_2sample(a,b,
                                      n_ps = 100,
                                      n_permutation = int(1e4),
                                      one_tail = False,
                                      match_sample_size = True)
    df_diff_diff['roi'].append(roi)
    df_diff_diff['ps_mean'].append(ps.mean())
    df_diff_diff['ps_std'].append(ps.std())
    df_diff_diff['diff_mean'].append(np.mean(np.abs(a - b)))
    df_diff_diff['diff_std'].append(np.std(np.abs(a - b)))
df_diff_diff = pd.DataFrame(df_diff_diff)
df_diff_diff = df_diff_diff.sort_values(['ps_mean'])
converter = utils.MCPConverter(pvals = df_diff_diff['ps_mean'].values)
d = converter.adjust_many()
df_diff_diff['ps_corrected'] = d['bonferroni'].values
df_diff_diff['star'] = df_diff_diff['ps_corrected'].apply(utils.stars)

roi_order = pd.unique(df['roi_name'])
df_diff = pd.concat([df_diff[df_diff['roi_name'] == name] for name in roi_order])
df_diff_diff = pd.concat([df_diff_diff[df_diff_diff['roi'] == name] for name in roi_order])

g = sns.catplot(x = 'roi_name',
                y = 'mean_variance',
                hue = 'condition',
                data = df_diff,
                kind = 'violin',
                aspect = 4,
                **dict(split = True,
                       cut = 0,
                       inner = 'quartile'))
(g.set_axis_labels('ROIs','$\Delta$ Variance Explained'))
g._legend.set_title('Conditions')
g._legend.texts[0].set_text('Read')
g._legend.texts[1].set_text('Think')
g.axes[-1][0].set_xticklabels(g.axes[-1][0].xaxis.get_majorticklabels(),
      rotation = 45,
      horizontalalignment = 'center')
for ii in range(len(pd.unique(df_diff['roi_name']))):
    g.axes[0][0].annotate(df_diff_diff['star'].values[ii],
          xy = (ii,0.045))
g.fig.suptitle('Difference of Computer Vision Models and Word Embedding Models\nBonforroni Corrected for multiple t-tests',
               y=1.05)
g.savefig(os.path.join(figure_dir,
                       'difference of difference.png'),
        dpi = 400,
        bbox_inches = 'tight')

max_stars_repo_path: scripts/encoding model 15 ROIs/post stats.py

reason: import numpy,import statsmodels,from statsmodels


text:
import os
import subprocess
from typing import Sequence

import numpy as np
import pytest

from bioimageio.core import load_resource_description

def run_subprocess(commands: Sequence[str], **kwargs) -> subprocess.CompletedProcess:
    return subprocess.run(commands, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", **kwargs)

def test_validate_model(unet2d_nuclei_broad_model):
    ret = run_subprocess(["bioimageio", "validate", unet2d_nuclei_broad_model])
    assert ret.returncode == 0, ret.stdout

def test_cli_package(unet2d_nuclei_broad_model, tmp_path):
    out_path = tmp_path / "model.zip"
    ret = run_subprocess(["bioimageio", "package", unet2d_nuclei_broad_model, str(out_path)])
    assert ret.returncode == 0, ret.stdout
    assert out_path.exists()

def test_cli_package_wo_cache(unet2d_nuclei_broad_model):
    env = os.environ.copy()
    env["BIOIMAGEIO_USE_CACHE"] = "false"
    ret = run_subprocess(["bioimageio", "package", unet2d_nuclei_broad_model], env=env)
    assert ret.returncode == 0, ret.stdout

def test_cli_test_model(unet2d_nuclei_broad_model):
    ret = run_subprocess(["bioimageio", "test-model", unet2d_nuclei_broad_model])
    assert ret.returncode == 0, ret.stdout

def test_cli_test_model_fail(stardist_wrong_shape):
    ret = run_subprocess(["bioimageio", "test-model", stardist_wrong_shape])
    assert ret.returncode == 1

def test_cli_test_model_with_weight_format(unet2d_nuclei_broad_model):
    ret = run_subprocess(
        ["bioimageio", "test-model", unet2d_nuclei_broad_model, "--weight-format", "pytorch_state_dict"]
    )
    assert ret.returncode == 0, ret.stdout

def test_cli_test_resource(unet2d_nuclei_broad_model):
    ret = run_subprocess(["bioimageio", "test-resource", unet2d_nuclei_broad_model])
    assert ret.returncode == 0, ret.stdout

def test_cli_test_resource_with_weight_format(unet2d_nuclei_broad_model):
    ret = run_subprocess(
        ["bioimageio", "test-resource", unet2d_nuclei_broad_model, "--weight-format", "pytorch_state_dict"]
    )
    assert ret.returncode == 0, ret.stdout

def _test_cli_predict_image(model, tmp_path, extra_kwargs=None):
    spec = load_resource_description(model)
    in_path = spec.test_inputs[0]
    out_path = tmp_path.with_suffix(".npy")
    cmd = ["bioimageio", "predict-image", model, "--inputs", str(in_path), "--outputs", str(out_path)]
    if extra_kwargs is not None:
        cmd.extend(extra_kwargs)
    ret = run_subprocess(cmd)
    assert ret.returncode == 0, ret.stdout
    assert out_path.exists()

def test_cli_predict_image(unet2d_nuclei_broad_model, tmp_path):
    _test_cli_predict_image(unet2d_nuclei_broad_model, tmp_path)

def test_cli_predict_image_with_weight_format(unet2d_nuclei_broad_model, tmp_path):
    _test_cli_predict_image(unet2d_nuclei_broad_model, tmp_path, ["--weight-format", "pytorch_state_dict"])

def _test_cli_predict_images(model, tmp_path, extra_kwargs=None):
    n_images = 3
    shape = (1, 1, 128, 128)
    expected_shape = (1, 1, 128, 128)

    in_folder = tmp_path / "inputs"
    in_folder.mkdir()
    out_folder = tmp_path / "outputs"
    out_folder.mkdir()

    expected_outputs = []
    for i in range(n_images):
        path = in_folder / f"im-{i}.npy"
        im = np.random.randint(0, 255, size=shape).astype("uint8")
        np.save(path, im)
        expected_outputs.append(out_folder / f"im-{i}.npy")

    input_pattern = str(in_folder / "*.npy")
    cmd = ["bioimageio", "predict-images", model, input_pattern, str(out_folder)]
    if extra_kwargs is not None:
        cmd.extend(extra_kwargs)
    ret = run_subprocess(cmd)
    assert ret.returncode == 0, ret.stdout

    for out_path in expected_outputs:
        assert out_path.exists()
        assert np.load(out_path).shape == expected_shape

def test_cli_predict_images(unet2d_nuclei_broad_model, tmp_path):
    _test_cli_predict_images(unet2d_nuclei_broad_model, tmp_path)

def test_cli_predict_images_with_weight_format(unet2d_nuclei_broad_model, tmp_path):
    _test_cli_predict_images(unet2d_nuclei_broad_model, tmp_path, ["--weight-format", "pytorch_state_dict"])

def test_torch_to_torchscript(unet2d_nuclei_broad_model, tmp_path):
    out_path = tmp_path.with_suffix(".pt")
    ret = run_subprocess(
        ["bioimageio", "convert-torch-weights-to-torchscript", str(unet2d_nuclei_broad_model), str(out_path)]
    )
    assert ret.returncode == 0, ret.stdout
    assert out_path.exists()

@pytest.mark.skipif(pytest.skip_onnx, reason="requires torch and onnx")
def test_torch_to_onnx(unet2d_nuclei_broad_model, tmp_path):
    out_path = tmp_path.with_suffix(".onnx")
    ret = run_subprocess(["bioimageio", "convert-torch-weights-to-onnx", str(unet2d_nuclei_broad_model), str(out_path)])
    assert ret.returncode == 0, ret.stdout
    assert out_path.exists()

def test_keras_to_tf(unet2d_keras, tmp_path):
    out_path = tmp_path / "weights.zip"
    ret = run_subprocess(["bioimageio", "convert-keras-weights-to-tensorflow", str(unet2d_keras), str(out_path)])
    assert ret.returncode == 0, ret.stdout
    assert out_path.exists()

ext: py

max_stars_repo_name: bioimage-io/python-core

max_issues_repo_name: bioimage-io/python-core

max_forks_repo_name: bioimage-io/python-core

reason: import numpy


text:
from typing import Dict, List

import numpy as np
from shapely.geometry.base import CAP_STYLE

from nuplan.common.actor_state.ego_state import EgoState
from nuplan.common.actor_state.state_representation import StateSE2
from nuplan.common.actor_state.tracked_objects import TrackedObjects
from nuplan.common.geometry.transform import rotate_angle
from nuplan.common.maps.abstract_map import AbstractMap
from nuplan.common.maps.abstract_map_objects import StopLine
from nuplan.common.maps.maps_datatypes import SemanticMapLayer, TrafficLightStatusType
from nuplan.planning.metrics.utils.expert_comparisons import principal_value
from nuplan.planning.simulation.observation.idm.idm_agent import IDMAgent
from nuplan.planning.simulation.observation.idm.idm_states import IDMLeadAgentState
from nuplan.planning.simulation.observation.idm.utils import path_to_linestring
from nuplan.planning.simulation.observation.observation_type import DetectionsTracks
from nuplan.planning.simulation.occupancy_map.abstract_occupancy_map import OccupancyMap

UniqueIDMAgents = Dict[str, IDMAgent]

class IDMAgentManager:
    """IDM smart-agents manager."""

    def __init__(self, agents: UniqueIDMAgents, agent_occupancy: OccupancyMap, map_api: AbstractMap):
        """
        Constructor for IDMAgentManager.
        :param agents: A dictionary pairing the agent's token to it's IDM representation.
        :param agent_occupancy: An occupancy map describing the spatial relationship between agents.
        :param map_api: AbstractMap API
        """
        self.agents: UniqueIDMAgents = agents
        self.agent_occupancy = agent_occupancy
        self._map_api = map_api

    def propagate_agents(
        self,
        ego_state: EgoState,
        tspan: float,
        iteration: int,
        traffic_light_status: Dict[TrafficLightStatusType, List[str]],
    ) -> None:
        """
        Propagate each active agent forward in time.

        :param ego_state: the ego's current state in the simulation.
        :param tspan: the interval of time to simulate.
        :param iteration: the simulation iteration.
        :param traffic_light_status: {traffic_light_status: lane_connector_ids} A dictionary containing traffic light information.
        """
        self.agent_occupancy.set("ego", ego_state.car_footprint.geometry)

        # Add stop lines into occupancy map if they have red light status.

        stop_lines = self._get_stop_line_with_status(traffic_light_status, TrafficLightStatusType.RED)
        # Keep track of the stop lines that were inserted. This is to remove them after all agents have been propagated.
        inactive_stop_line_tokens = self._insert_stop_lines_into_occupancy_map(stop_lines)

        for agent_token, agent in self.agents.items():
            if agent.is_active(iteration) and agent.has_valid_path():
                agent.plan_route(traffic_light_status)

                # Check for agents that intersects THIS agent's path
                agent_path = path_to_linestring(agent.get_path_to_go())
                intersecting_agents = self.agent_occupancy.intersects(
                    agent_path.buffer((agent.width / 2), cap_style=CAP_STYLE.flat)
                )
                assert intersecting_agents.contains(agent_token), "Agent's baseline does not intersect the agent itself"

                # Checking if there are agents intersecting THIS agent's baseline.
                # Hence, we are checking for at least 2 intersecting agents.
                if intersecting_agents.size > 1:
                    nearest_id, nearest_agent_polygon, relative_distance = intersecting_agents.get_nearest_entry_to(
                        agent_token
                    )
                    agent_heading = agent.to_se2().heading

                    if "ego" in nearest_id:
                        ego_velocity = ego_state.dynamic_car_state.rear_axle_velocity_2d
                        longitudinal_velocity = np.hypot(ego_velocity.x, ego_velocity.y)
                        relative_heading = ego_state.rear_axle.heading - agent_heading
                    elif 'stop_line' in nearest_id:
                        longitudinal_velocity = 0.0
                        relative_heading = 0.0
                    else:
                        nearest_agent = self.agents[nearest_id]
                        longitudinal_velocity = nearest_agent.velocity
                        relative_heading = nearest_agent.to_se2().heading - agent_heading

                    # Wrap angle to [-pi, pi]
                    relative_heading = principal_value(relative_heading)
                    # take the longitudinal component of the projected velocity
                    projected_velocity = rotate_angle(StateSE2(longitudinal_velocity, 0, 0), relative_heading).x

                    # relative_distance already takes the vehicle dimension into account.
                    # Therefore there is no need to pass in the length_rear.
                    length_rear = 0
                else:
                    # Free road case: no leading vehicle
                    projected_velocity = 0.0
                    relative_distance = agent.get_progress_to_go()
                    length_rear = agent.length / 2

                agent.propagate(
                    IDMLeadAgentState(progress=relative_distance, velocity=projected_velocity, length_rear=length_rear),
                    tspan,
                )
                self.agent_occupancy.set(agent_token, agent.projected_footprint)
        self.agent_occupancy.remove(inactive_stop_line_tokens)

    def get_active_agents(self, iteration: int, num_samples: int, sampling_time: float) -> DetectionsTracks:
        """
        Returns all agents as DetectionsTracks.
        :param iteration: the current simulation iteration.
        :param num_samples: number of elements to sample.
        :param sampling_time: [s] time interval of sequence to sample from.
        :return: agents as DetectionsTracks.
        """
        return DetectionsTracks(
            TrackedObjects(
                [
                    agent.get_agent_with_planned_trajectory(num_samples, sampling_time)
                    for agent in self.agents.values()
                    if agent.is_active(iteration)
                ]
            )
        )

    def _get_stop_line_with_status(
        self, traffic_light_status: Dict[TrafficLightStatusType, List[str]], status: TrafficLightStatusType
    ) -> List[StopLine]:
        """
        Retrieve the stop lines that has is labelled with the given TrafficLightStatusType.
        :param traffic_light_status: {traffic_light_status: lane_connector_ids} A dictionary containing traffic light information.
        :param status: Traffic light status.
        :return: A list of stop lines associated with the given traffic light status.
        """
        lane_connector_ids = traffic_light_status[status]
        lane_connectors = [
            self._map_api.get_map_object(lc_id, SemanticMapLayer.LANE_CONNECTOR) for lc_id in lane_connector_ids
        ]
        stop_lines = [stop_line for lc in lane_connectors if lc for stop_line in lc.get_stop_lines()]
        return stop_lines

    def _insert_stop_lines_into_occupancy_map(self, stop_lines: List[StopLine]) -> List[str]:
        """
        Insert stop lines into the occupancy map.
        :param stop_lines: A list of stop lines to be inserted.
        :return: A list of token corresponding to the inserted stop lines.
        """
        stop_line_tokens: List[str] = []
        for stop_line in stop_lines:
            stop_line_token = f"stop_line_{stop_line.id}"
            if not self.agent_occupancy.contains(stop_line_token):
                self.agent_occupancy.set(stop_line_token, stop_line.polygon)
                stop_line_tokens.append(stop_line_token)

        return stop_line_tokens

max_stars_repo_name: motional/nuplan-devkit

max_issues_repo_name: motional/nuplan-devkit

max_forks_repo_name: motional/nuplan-devkit


text:
"""
Modifications copyright (C) 2020 Michael Strobl
"""

import time
import tensorflow as tf
import numpy as np

from entity_linker.models.base import Model

class LabelingModel(Model):
    """Unsupervised Clustering using Discrete-State VAE"""

    def __init__(self, batch_size, num_labels, context_encoded_dim,
                 true_entity_embeddings,
                 word_embed_dim, context_encoded, mention_embed, scope_name, device):

        self.batch_size = batch_size
        self.num_labels = num_labels
        self.word_embed_dim = word_embed_dim

        with tf.variable_scope(scope_name) as s, tf.device(device) as d:
            if mention_embed == None:
                self.label_weights = tf.get_variable(
                  name="label_weights",
                  shape=[context_encoded_dim, num_labels],
                  initializer=tf.random_normal_initializer(mean=0.0,
                                                           stddev=1.0/(100.0)))
            else:
                context_encoded = tf.concat(
                  1, [context_encoded, mention_embed], name='con_ment_repr')
                self.label_weights = tf.get_variable(
                  name="label_weights",
                  shape=[context_encoded_dim+word_embed_dim, num_labels],
                  initializer=tf.random_normal_initializer(mean=0.0,
                                                           stddev=1.0/(100.0)))

            # [B, L]
            self.label_scores = tf.matmul(context_encoded, self.label_weights)
            self.label_probs = tf.sigmoid(self.label_scores)

            ### PREDICT TYPES FROM ENTITIES
            #true_entity_embeddings = tf.nn.dropout(true_entity_embeddings, keep_prob=0.5)
            self.entity_label_scores = tf.matmul(true_entity_embeddings, self.label_weights)
            self.entity_label_probs = tf.sigmoid(self.label_scores)

    def loss_graph(self, true_label_ids, scope_name, device_gpu):
        with tf.variable_scope(scope_name) as s, tf.device(device_gpu) as d:
            # [B, L]
            self.cross_entropy_losses = tf.nn.sigmoid_cross_entropy_with_logits(
              logits=self.label_scores,
              targets=true_label_ids,
              name="labeling_loss")

            self.labeling_loss = tf.reduce_sum(
              self.cross_entropy_losses) / tf.to_float(self.batch_size)

            self.enlabel_cross_entropy_losses = tf.nn.sigmoid_cross_entropy_with_logits(
              logits=self.entity_label_scores,
              targets=true_label_ids,
              name="entity_labeling_loss")

            self.entity_labeling_loss = tf.reduce_sum(
              self.enlabel_cross_entropy_losses) / tf.to_float(self.batch_size)

max_stars_repo_name: mjstrobl/WEXEA

max_issues_repo_name: mjstrobl/WEXEA

max_forks_repo_name: mjstrobl/WEXEA


text:
#!/usr/bin/env python
# coding: utf-8

##### PREPROCESSING AND SAMPLING #####

print('Starting preprocessing and sampling...')

# import libraries
from datetime import datetime,date
start_time = datetime.now()
print('Start time: ', start_time)

import sys
params = sys.argv

import numpy as np
import pandas as pd
import pickle

from helper_functions import *

# print selected training set size
print('Training set size:', params[1])

### LOAD DATA
print('Starting loading data...')

# input file
input_file = 'clickstream_0516-1016_prepared.pkl.gz'

# output files
output_file_train = 'train_'+params[1]+'.pkl.gz'
output_file_test = 'test_'+params[1]+'.pkl.gz'
output_file_descriptives = 'preprocessing_and_sampling_descriptives_'+str(params[1])+'.pkl.gz'

# load data
df = pd.read_pickle('../data/processed_data/'+input_file)

# set up dataframe for descriptives
columns = ['rows_pre',
'rows_post',
'columns_pre',
'columns_post',
'unique_visitors_pre',
'unique_visitors_post',
'run_time']
index = [input_file[:16]]
preprocessing_and_sampling_descriptives = pd.DataFrame(index=index, columns=columns)

# save pre descriptives
preprocessing_and_sampling_descriptives['rows_pre'] = df.shape[0]
preprocessing_and_sampling_descriptives['columns_pre'] = df.shape[1]
preprocessing_and_sampling_descriptives['unique_visitors_pre'] = df['visitor_id'].nunique()

print('Loading data complete.')

### PREPROCESSING
print('Starting preprocessing and sampling...')

# create training set (80%)
with open('../data/processed_data/unique_visitor_ids.pkl', 'rb') as f:
   unique_visitor_ids = pickle.load(f)
train = df[df['visitor_id'].isin(unique_visitor_ids[:(int(params[1]))])]

# create test set (20%)
test = df[df['visitor_id'].isin(unique_visitor_ids[(int(params[1])+1):(int(params[1])+1+(int(int(params[1])/0.8*0.2)))])]

# standardize numerical features
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
numerical_features = ['visit_page_num_max',
'product_view_boolean_sum',
'cart_addition_boolean_sum',
'cart_removal_boolean_sum',
'cart_view_boolean_sum',
'campaign_view_boolean_sum',
'cart_value_sum',
'page_view_boolean_sum',
'last_purchase_num_max',
'product_items_sum',
'product_item_price_sum',
'standard_search_results_clicked_sum',
'standard_search_started_sum',
'suggested_search_results_clicked_sum',
'visit_duration_seconds']
scaler.fit(train[numerical_features])
train[numerical_features] = scaler.transform(train[numerical_features])
test[numerical_features] = scaler.transform(test[numerical_features])

# keep only k best features
with open('../data/processed_data/k_best_features.pkl', 'rb') as f:
   k_best_features = pickle.load(f)
for column in train.columns:
    if column in ['visitor_id', 'visit_start_time_gmt', 'purchase_within_next_24_hours']:
        pass
    elif column not in k_best_features:
        train.drop(column, axis=1, inplace=True)
        test.drop(column, axis=1, inplace=True)
    else:
        pass

print('Preprocessing and sampling complete.')

### WRITE DATA
print('Starting writing data...')

# save post descriptives
preprocessing_and_sampling_descriptives['rows_post'] = train.shape[0] + test.shape[0]
preprocessing_and_sampling_descriptives['columns_post'] = train.shape[1] + test.shape[1]
preprocessing_and_sampling_descriptives['unique_visitors_post'] = train['visitor_id'].nunique() + test['visitor_id'].nunique()

train.to_pickle('../data/training_and_test_sets/'+output_file_train, compression='gzip')
test.to_pickle('../data/training_and_test_sets/'+output_file_test, compression='gzip')

print('Writing data complete.')

print('Preprocessing and sampling complete.')
run_time = datetime.now() - start_time
print('Run time: ', run_time)

# save run time and descriptives dataframe
preprocessing_and_sampling_descriptives['run_time'] = run_time.seconds
preprocessing_and_sampling_descriptives.to_pickle('../results/descriptives/'+output_file_descriptives, compression='gzip')

max_stars_repo_name: alexvmt/Master-Thesis

max_issues_repo_name: alexvmt/Master-Thesis

max_forks_repo_name: alexvmt/Master-Thesis


text:
# Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
import astropy.units as u
from numpy.testing import assert_allclose, assert_equal
from astropy.tests.helper import pytest, assert_quantity_allclose
from ...datasets import gammapy_extra
from ...utils.testing import requires_dependency, requires_data, data_manager
from ...irf.effective_area import EffectiveAreaTable2D, EffectiveAreaTable

@requires_dependency('scipy')
def test_EffectiveAreaTable2D_generic():
    # This tests NDData subclassing. Not needed for other IRF classes

    # Exercise __init__  method
    energy = np.logspace(0, 1, 4) * u.TeV
    offset = [0.2, 0.3] * u.deg
    effective_area = np.arange(6).reshape(3, 2) * u.cm * u.cm
    meta = dict(name='example')
    aeff = EffectiveAreaTable2D(offset=offset, energy=energy, data=effective_area,
                                meta=meta)
    assert (aeff.axes[0].data == energy).all()
    assert (aeff.axes[1].data == offset).all()
    assert (aeff.data == effective_area).all()
    assert aeff.meta.name == 'example'

    wrong_data = np.arange(8).reshape(4, 2) * u.cm * u.cm

    with pytest.raises(ValueError):
        aeff.data = wrong_data
        aeff = EffectiveAreaTable(offset=offset, energy=energy, data=wrong_data)

    # Test evaluate function
    # Check that nodes are evaluated correctly
    e_node = 1
    off_node = 0
    offset = aeff.offset.nodes[off_node]
    energy = aeff.energy.nodes[e_node]
    actual = aeff.evaluate(offset=offset, energy=energy, method='nearest')
    desired = aeff.data[e_node, off_node]
    assert_allclose(actual, desired)

    actual = aeff.evaluate(offset=offset, energy=energy, method='linear')
    desired = aeff.data[e_node, off_node]
    assert_allclose(actual, desired)

    # Check that values between node make sense
    energy2 = aeff.energy.nodes[e_node + 1]
    upper = aeff.evaluate(offset=offset, energy=energy)
    lower = aeff.evaluate(offset=offset, energy=energy2)
    e_val = (energy + energy2) / 2
    actual = aeff.evaluate(offset=offset, energy=e_val)
    assert_equal(lower > actual and actual > upper, True)

    # Test return shape
    # Case 0; offset = scalar, energy = scalar, done

    # Case 1: offset = scalar, energy = None
    offset = 0.234 * u.deg
    actual = aeff.evaluate(offset=offset).shape
    desired = aeff.energy.nodes.shape
    assert_equal(actual, desired)

    # Case 2: offset = scalar, energy = 1Darray

    offset = 0.564 * u.deg
    nbins = 10
    energy = np.logspace(3, 4, nbins) * u.GeV
    actual = aeff.evaluate(offset=offset, energy=energy).shape
    desired = np.zeros(nbins).shape
    assert_equal(actual, desired)

    # Case 3: offset = None, energy = scalar
    energy = 1.1 * u.TeV
    actual = aeff.evaluate(energy=energy).shape
    desired = tuple([aeff.offset.nbins])
    assert_equal(actual, desired)

    # Case 4: offset = 1Darray, energy = scalar
    energy = 1.5 * u.TeV
    nbins = 4
    offset = np.linspace(0, 1, nbins) * u.deg
    actual = aeff.evaluate(offset=offset, energy=energy).shape
    desired = np.zeros(nbins).shape
    assert_equal(actual, desired)

    # case 5: offset = 1Darray, energy = 1Darray
    nbinse = 5
    nbinso = 3
    offset = np.linspace(0.2, 0.3, nbinso) * u.deg
    energy = np.logspace(0, 1, nbinse) * u.TeV
    actual = aeff.evaluate(offset=offset, energy=energy).shape
    desired = np.zeros([nbinse, nbinso]).shape
    assert_equal(actual, desired)

    # case 6: offset = 2Darray, energy = 1Darray
    nbinse = 4
    nx, ny = (12, 3)
    offset = np.linspace(0.2, 0.3, nx * ny).reshape(nx, ny) * u.deg
    energy = np.logspace(0, 1, nbinse) * u.TeV
    actual = aeff.evaluate(offset=offset, energy=energy).shape
    desired = np.zeros([nbinse, nx, ny]).shape
    assert_equal(actual, desired)

    # Misc functions
    assert 'EffectiveAreaTable2D' in str(aeff)

@requires_dependency('scipy')
@requires_dependency('matplotlib')
@requires_data('gammapy-extra')
def test_EffectiveAreaTable2D(tmpdir):
    filename = gammapy_extra.filename(
        'datasets/hess-crab4-hd-hap-prod2/run023400-023599/run023523/hess_aeff_2d_023523.fits.gz')
    aeff = EffectiveAreaTable2D.read(filename)

    assert aeff.energy.nbins == 73
    assert aeff.offset.nbins == 6
    assert aeff.data.shape == (73, 6)

    assert aeff.energy.unit == 'TeV'
    assert aeff.offset.unit == 'deg'
    assert aeff.data.unit == 'm2'

    assert_allclose(aeff.high_threshold.value, 99.083, atol=1e-2)
    assert_allclose(aeff.low_threshold.value, 0.603, atol=1e-2)

    test_e = 14 * u.TeV
    test_o = 0.2 * u.deg
    test_val = aeff.evaluate(energy=test_e, offset=test_o)
    assert_allclose(test_val.value, 740929.645, atol=1e-2)

    aeff.plot_image()
    aeff.plot_energy_dependence()
    aeff.plot_offset_dependence()

    # Test ARF export
    offset = 0.236 * u.deg
    e_axis = np.logspace(0, 1, 20) * u.TeV
    effareafrom2d = aeff.to_effective_area_table(offset, e_axis)

    energy = np.sqrt(e_axis[:-1] * e_axis[1:])
    area = aeff.evaluate(offset=offset, energy=energy)
    effarea1d = EffectiveAreaTable(energy=e_axis, data=area)

    test_energy = 2.34 * u.TeV
    actual = effareafrom2d.evaluate(energy=test_energy)
    desired = effarea1d.evaluate(energy=test_energy)
    assert_equal(actual, desired)

    # Test ARF export #2
    offset = 1.2 * u.deg
    effareafrom2dv2 = aeff.to_effective_area_table(offset=offset)
    actual = effareafrom2dv2.data
    desired = aeff.evaluate(offset=offset)
    assert_equal(actual, desired)

@requires_dependency('scipy')
@requires_dependency('matplotlib')
@requires_data('gammapy-extra')
def test_EffectiveAreaTable(tmpdir, data_manager):
    store = data_manager['hess-crab4-hd-hap-prod2']
    aeff = store.obs(obs_id=23523).aeff
    arf = aeff.to_effective_area_table(offset=0.3 * u.deg)

    assert (arf.evaluate() == arf.data).all()

    arf.plot()

    filename = str(tmpdir / 'effarea_test.fits')
    arf.write(filename)

    arf2 = EffectiveAreaTable.read(filename)

    assert (arf.evaluate() == arf2.evaluate()).all()

    test_aeff = 0.6 * arf.max_area
    node_above = np.where(arf.data > test_aeff)[0][0]
    ener_above = arf.energy.nodes[node_above]
    ener_below = arf.energy.nodes[node_above - 1]
    test_ener = arf.find_energy(test_aeff)

    assert ener_below < test_ener and test_ener < ener_above

    elo_threshold = arf.find_energy(0.1 * arf.max_area)
    assert_quantity_allclose(elo_threshold, 0.4367 * u.TeV, rtol=1e-3)

    # Test evaluation outside safe range
    data = [np.nan, np.nan, 0, 0, 1, 2, 3, np.nan, np.nan]
    energy = np.logspace(0, 10, 10) * u.TeV
    aeff = EffectiveAreaTable(data=data, energy=energy)
    vals = aeff.evaluate(fill_nan=True)
    assert vals[1] == 0
    assert vals[-1] == 3

def test_EffectiveAreaTableParametrization():
    # Log center of this is 100 GeV
    energy = [80, 125] * u.GeV
    area_ref = 1.65469579e+07 * u.cm * u.cm

    area = EffectiveAreaTable.from_parametrization(energy, 'HESS')
    assert_allclose(area.data, area_ref)
    assert area.data.unit == area_ref.unit

    # Log center of this is 0.1, 2 TeV
    energy = [0.08, 0.125, 32] * u.TeV
    area_ref = [1.65469579e+07, 1.46451957e+09] * u.cm * u.cm

    area = EffectiveAreaTable.from_parametrization(energy, 'HESS')
    assert_allclose(area.data, area_ref)
    assert area.data.unit == area_ref.unit

    # TODO: Use this to test interpolation behaviour etc.

max_stars_repo_name: grburgess/gammapy

max_issues_repo_name: grburgess/gammapy

max_forks_repo_name: grburgess/gammapy

reason: import numpy,from numpy,import astropy,from astropy


text:
'''

define classes to be used in setting up and solving 2 body problems

@author: Maciej Tomasz Jarema ppymj11

'''

import numpy as np
import matplotlib.pyplot as plt
import scipy
from scipy import integrate

# %%

# deifne a function that will set up a class instance of a single body
def body_def(Mass, size, x, y, vx, vy):
    class body():
        def __init__(self, Mass, size, x, y, vx, vy):
            assert Mass >= 0, 'Mass must be positive'
            assert size >= 0, 'size must be positive'
            self.Mass = Mass
            self.size = size
            self.pos = [x, y]
            self.vel = [vx, vy]

        def size_change(self, new):
            '''
            Increases body size to newly chosen, as input, int
            '''
            self.size += new

    # run the class and return the instance of it to user
    instance = body(Mass, size, x, y, vx, vy)
    return instance

# define a function that will set up a class instance for a system of bodies
# for any number of input bodies
def system_def(*bodies):
    class system():
        def __init__(self, *bodies):
            # define system for however many bodies were input
            i = 0
            for body in [*bodies]:
                exec('self.body' + str(i) + ' = body')
                i += 1

            self.init = []  # list of initial parameters, to be generated
            self.length = i  # number of bodies

        # define a method to set up initials from all bodies:
        def initials(self):
            '''
            Takes no inputs, returns no outputs
            Sets up the intiial ocnditions of the system of bodies
            Stores these in the system isnatance that it acts on
            '''
            # append positions of each body
            for n in range(self.length):
                exec('self.init.extend(self.body' + str(n) + '.pos)')

            # append velocoties of each body
            for n in range(self.length):
                exec('self.init.extend(self.body' + str(n) + '.vel)')

        # define a function for the jacobian of this system:
        # this will be used by odeint, only for 2D, 2 body
        def jacobian_get(self, y, x):
            '''
            Acts as function for odeint to get derivatives for motion.
            takes in odeint array of variables and returns its derivastives
            the order is set by set up initial condition in system instance

            Only works for 2 bodies in 2D
            could generalise but no need for this investigation
            '''
            # grav constant
            G = 6.67e-11

            # extract masses for easier reading
            M = self.body0.Mass
            m = self.body1.Mass

            # set up an empty list to store derivaitves:
            derivs = np.zeros_like(y)

            # deal with substitution derivatives:
            derivs[0] = y[4]
            derivs[1] = y[5]
            derivs[2] = y[6]
            derivs[3] = y[7]

            # get the distance between the bodies
            dist = np.sqrt((y[0] - y[2])**2 + (y[1] - y[3])**2)

            # deal with equation dependant derivatives, from N2L and N grav. law. in 1D
            derivs[4] = -(G*m/dist**3) * (y[0] - y[2])
            derivs[5] = -(G*m/dist**3) * (y[1] - y[3])
            derivs[6] = -(G*M/dist**3) * (y[2] - y[0])
            derivs[7] = -(G*M/dist**3) * (y[3] - y[1])

            # return to odeint:
            return derivs

    # run the class and return the instance of it to user
    instance = system(*bodies)
    return instance

# #############################################################################
# example for how to use these
# #############################################################################

# Example of use:
if __name__ == '__main__':
    # define parameters for body 1:
    M1 = 2e30
    size1 = 20
    x1 = 0
    y1 = 0
    vx1 = 0
    vy1 = 0

    # set up a class instance for body 1:
    body_1 = body_def(M1, size1, x1, y1, vx1, vy1)

    # same for body 2:
    M2 = 6e24
    size2 = 6
    x2 = 1.496e11
    y2 = 0
    vx2 = 0
    vy2 = -29800
    body_2 = body_def(M2, size2, x2, y2, vx2, vy2)

    # run it with the above two bodies
    system_1 = system_def(body_1, body_2)
    system_1.initials()

    # set up simualtion time parameters
    year = 3.156e7  # numerical factor
    t_max = 1 * year
    t_number = 400
    t_arr = np.linspace(0, t_max, t_number)
    dt = t_arr[-1] - t_arr[-2]

    # call odeint to solve it:
    solution = scipy.integrate.odeint(system_1.jacobian_get, system_1.init, t_arr)

    # from it extract wanted positions
    xs_anim = solution[:, 0]
    xp_anim = solution[:, 2]

    ys_anim = solution[:, 1]
    yp_anim = solution[:, 3]

    # plot the motion of the two objects in the x-y plane
    fig = plt.figure()
    ax = fig.gca()
    ax.set_aspect('equal')
    ax.set_xlabel(r'$x \ [m]$')
    ax.set_ylabel(r'$y \ [m]$')
    ax.set_title('Simulation of Earth\'s orbit')
    plt.plot(xs_anim, ys_anim, 'ro')
    plt.plot(xp_anim, yp_anim, 'k-')

reason: import numpy,import scipy,from scipy


text:
"""
Pie Chart
===============================================================================

>>> directory = "/workspaces/techminer2/data/"
>>> from techminer2.annual_indicators import annual_indicators
>>> from techminer2._pie_chart import _pie_chart
>>> file_name = "/workspaces/techminer2/sphinx/images/_pie_chart.png"
>>> _pie_chart(
...     series=annual_indicators(directory).num_documents,
...     darkness=annual_indicators(directory).global_citations,
... ).savefig(file_name)

.. image:: images/_pie_chart.png
    :width: 700px
    :align: center

"""

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

TEXTLEN = 40

def _pie_chart(
    series,
    darkness=None,
    cmap="Greys",
    figsize=(6, 6),
    fontsize=9,
    wedgeprops={
        "width": 0.6,
        "edgecolor": "k",
        "linewidth": 0.5,
        "linestyle": "-",
        "antialiased": True,
    },
):
    """Plot a pie chart."""
    darkness = series if darkness is None else darkness

    cmap = plt.cm.get_cmap(cmap)
    colors = [
        cmap(0.1 + 0.90 * (d - min(darkness)) / (max(darkness) - min(darkness)))
        for d in darkness
    ]

    matplotlib.rc("font", size=fontsize)
    fig = plt.Figure(figsize=figsize)
    ax = fig.subplots()

    ax.pie(
        x=series,
        labels=series.index,
        colors=colors,
        wedgeprops=wedgeprops,
    )

    fig.set_tight_layout(True)

    return fig

max_stars_repo_name: jdvelasq/techminer-api

max_issues_repo_name: jdvelasq/techminer-api

max_forks_repo_name: jdvelasq/techminer-api


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: 01_electricmachinepu.ipynb (unless otherwise specified).

__all__ = ['spm', 'ipm']

# Cell
import numpy as np
import matplotlib.pyplot as plt

# Cell
class spm():
    def __init__(self, phi_m):
        self.lpu = np.sqrt(1-phi_m**2)
        self.phi_m = phi_m
        self.Vb = 0
        self.Pb = 0
        self.wb = 0
        self.speed = []
        self.torque = []
        self.voltage = []
        self.current = []
        self.gamma = []
        self.power = []
        self.values = dict.fromkeys(['speed', 'torque', 'power'])

    def motor_puprofile(self, gamma_limit = 85):
        """ this method calculates the per unit profile of the machine with current angle limit of 85 degrees
        """
        gamma_deg = 0
        gamma = gamma_deg*np.pi/180
        # constant torque region
        for o in np.arange(0,1.1,0.1):
            v = o * np.sqrt( (self.phi_m - np.sin(gamma)*self.lpu)**2 + (self.lpu * np.cos(gamma))**2 )
            t = self.phi_m * np.cos(gamma)
            p = t*o
            self.speed.append(o)
            self.voltage.append(v)
            self.gamma.append(gamma)
            self.torque.append(t)
            self.power.append(p)
        # constant voltage region
        for gamma_deg in range(1,85):
            gamma = gamma_deg*np.pi/180
            o = 1/np.sqrt( (self.phi_m - np.sin(gamma)*self.lpu)**2 + (self.lpu * np.cos(gamma))**2 )
            t = self.phi_m * np.cos(gamma)
            p = t*o
            self.speed.append(o)
            self.voltage.append(v)
            self.gamma.append(gamma_deg)
            self.torque.append(t)
            self.power.append(p)

    def motor_profile(self, Vb, Pb, wb):
        """This method takes Vb, Pb, wb as the base values for line voltage, KVA rating of the machine, base speed respectively
        """
        self.Vb = Vb
        self.Pb = Pb
        self.wb = wb
        Tb = Pb/(2*np.pi*wb/60)
        self.values['speed'] = np.array(self.speed)*wb
        self.values['torque'] = np.array(self.torque)*Tb
        self.values['power'] = np.array(self.power)*Pb

    def plot_puprofile(self):
        """ plots the motor performance profile in p.u
        """
        fig, axs = plt.subplots(3)
        fig.set_figheight(8)
        axs[0].plot(self.speed, self.torque)
        axs[0].set_title('torque vs speed')
        axs[1].plot(self.speed, self.power)
        axs[1].set_title('power vs speed')
        axs[2].plot(self.speed, self.gamma)
        axs[2].set_title('gamma vs speed')

        # Hide x labels and tick labels for top plots and y ticks for right plots.
        for ax in axs.flat:
            ax.label_outer()

        plt.tight_layout()
        plt.show()

    def plot_profile(self):
        """ plots the motor performance profile in actual values
        """
        fig, axs = plt.subplots(3)
        fig.set_figheight(8)
        axs[0].plot(self.values['speed'], self.values['torque'])
        axs[0].set_title('torque vs speed')
        axs[1].plot(self.values['speed'], self.values['power'])
        axs[1].set_title('power vs speed')
        axs[2].plot(self.values['speed'], self.gamma)
        axs[2].set_title('gamma vs speed')

        # Hide x labels and tick labels for top plots and y ticks for right plots.
        for ax in axs.flat:
            ax.label_outer()

        plt.tight_layout()
        plt.show()

# Cell
class ipm():
    def __init__(self, phi_m, ld):
        self.phi_m = phi_m
        self.sal = 0
        self.ld = ld
        self.Vb = 0
        self.Ib = 0
        self.Pb = 0
        self.wb = 0
        self.speed = []
        self.torque = []
        self.voltage = []
        self.current = []
        self.gamma = []
        self.power = []
        self.values = dict.fromkeys(['speed', 'torque', 'power'])
        self.valid = 0

    def calc_sal(self):
        """ calculate valid saliency ratio for given combination of magnet flux linkage and inductance
        """
        for eta in np.arange(1.1, 20, 0.1):
            gamma = np.arcsin((-self.phi_m + np.sqrt( (self.phi_m**2 + 8*(self.ld*(self.sal-1))**2 )))/(4*self.ld*(self.sal-1)))
            volt = np.sqrt( (self.phi_m - self.ld*np.sin(gamma))**2 + (eta*self.ld * np.cos(gamma))**2 )
            if eta == 1.1 and volt > 1.05:
                self.valid = 0
                return
            if abs(volt-1) <= 0.05:
                self.sal = round(eta,1)
                break
            self.sal = round(eta,1)

    def validate(self,):
        """ validate the input machine parameter combinations
        """
        sin_gamma = (-self.phi_m + np.sqrt( (self.phi_m**2 + 8*(self.ld*(self.sal-1))**2 )))/(4*self.ld*(self.sal-1))
        if sin_gamma <= 1:
            self.valid = 1

    def motor_puprofile(self, gamma_limit = 85):
        """ this method calculates the per unit profile of the machine with current angle limit of 85 degrees
        """
        gamma = np.arcsin((-self.phi_m + np.sqrt( (self.phi_m**2 + 8*(self.ld*(self.sal-1))**2 )))/(4*self.ld*(self.sal-1)))
        gamma_deg = gamma*180/np.pi
        speed_temp = []
        voltage_temp = []
        gamma_temp = []
        torque_temp = []
        power_temp = []
        # constant torque region
        for o in np.arange(0,1.1,0.1):
            v = o * np.sqrt( (self.phi_m - self.ld*np.sin(gamma))**2 + (self.sal*self.ld * np.cos(gamma))**2 )
            t = self.phi_m * np.cos(gamma) + (self.sal-1)*self.ld*np.sin(2*gamma)/2
            p = t*o
            speed_temp.append(o)
            voltage_temp.append(v)
            gamma_temp.append(gamma_deg)
            torque_temp.append(t)
            power_temp.append(p)
            #self.speed.append(o)
            #self.voltage.append(v)
            #self.gamma.append(gamma_deg)
            #self.torque.append(t)
            #self.power.append(p)

        # constant voltage region
        while (gamma_deg < gamma_limit):
            gamma_deg += 1
            gamma = gamma_deg*np.pi/180
            o = 1/np.sqrt( (self.phi_m - np.sin(gamma)*self.ld)**2 + (self.sal* self.ld * np.cos(gamma))**2 )
            t = self.phi_m * np.cos(gamma) + (self.sal-1)*self.ld*np.sin(2*gamma)/2
            p = t*o
            speed_temp.append(o)
            voltage_temp.append(v)
            gamma_temp.append(gamma_deg)
            torque_temp.append(t)
            power_temp.append(p)

        self.speed = speed_temp
        self.voltage = voltage_temp
        self.gamma = gamma_temp
        self.torque = torque_temp
        self.power = power_temp

    def motor_profile(self, Vb, Pb, wb):
        """This method takes Vb, Pb, wb as the base values for line voltage, KVA rating of the machine, base speed respectively
        """
        self.Vb = Vb
        self.Pb = Pb
        self.wb = wb
        self.Ib = round((Pb*2/(3*Vb))*np.sqrt(3/2),2)
        Tb = Pb/(2*np.pi*wb/60)
        self.values['speed'] = np.array(self.speed)*wb
        self.values['torque'] = np.array(self.torque)*Tb
        self.values['power'] = np.array(self.power)*Pb

    def plot_puprofile(self):
        """ plots the motor performance profile in p.u
        """
        fig, axs = plt.subplots(3)
        fig.set_figheight(8)
        axs[0].plot(self.speed, self.torque)
        axs[0].set_title('torque vs speed')
        axs[1].plot(self.speed, self.power)
        axs[1].set_title('power vs speed')
        axs[2].plot(self.speed, self.gamma)
        axs[2].set_title('gamma vs speed')

        # Hide x labels and tick labels for top plots and y ticks for right plots.
        for ax in axs.flat:
            ax.label_outer()

        plt.tight_layout()
        plt.show()

    def plot_profile(self):
        """ plots the motor performance profile in actual values
        """
        fig, axs = plt.subplots(3)
        fig.set_figheight(8)
        axs[0].plot(self.values['speed'], self.values['torque'])
        axs[0].set_title('torque vs speed')
        axs[0].set_xlim([0, 10000])
        axs[1].plot(self.values['speed'], self.values['power'])
        axs[1].set_title('power vs speed')
        axs[1].set_xlim([0, 10000])
        axs[2].plot(self.values['speed'], self.gamma)
        axs[2].set_title('gamma vs speed')
        axs[2].set_xlim([0, 10000])

        # Hide x labels and tick labels for top plots and y ticks for right plots.
        #for ax in axs.flat:
        #    ax.label_outer()

        plt.tight_layout()
        plt.show()

max_stars_repo_name: NaveenDeepak/EMDesigner

max_issues_repo_name: NaveenDeepak/EMDesigner

max_forks_repo_name: NaveenDeepak/EMDesigner


text:
# Copyright 2019 SanghunYun, Korea University.
# Copyright 2018 Dong-Hyun Lee, Kakao Brain.
#
# This file has been modified by SanghunYun, Korea University
# for add fucntion of _get_device and class of output_logging.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import csv
import random
import logging

import numpy as np
import torch

def torch_device_one():
    return torch.tensor(1.).to(_get_device())

def set_seeds(seed):
    "set random seeds"
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

def get_device():
    "get device (CPU or GPU)"
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    n_gpu = torch.cuda.device_count()
    print("%s (%d GPUs)" % (device, n_gpu))
    return device

def _get_device():
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    return device

def split_last(x, shape):
    "split the last dimension to given shape"
    shape = list(shape)
    assert shape.count(-1) <= 1
    if -1 in shape:
        shape[shape.index(-1)] = int(x.size(-1) / -np.prod(shape))
    return x.view(*x.size()[:-1], *shape)

def merge_last(x, n_dims):
    "merge the last n_dims to a dimension"
    s = x.size()
    assert n_dims > 1 and n_dims < len(s)
    return x.view(*s[:-n_dims], -1)

def truncate_tokens_pair(tokens_a, tokens_b, max_len):
    while True:
        if len(tokens_a) + len(tokens_b) <= max_len:
            break
        if len(tokens_a) > len(tokens_b):
            tokens_a.pop()
        else:
            tokens_b.pop()

def get_random_word(vocab_words):
    i = random.randint(0, len(vocab_words)-1)
    return vocab_words[i]

def get_logger(name, log_path):
    "get logger"
    logger = logging.getLogger(name)
    fomatter = logging.Formatter(
        '[ %(levelname)s|%(filename)s:%(lineno)s] %(asctime)s > %(message)s')

    if not os.path.isfile(log_path):
        f = open(log_path, "w+")

    fileHandler = logging.FileHandler(log_path)
    fileHandler.setFormatter(fomatter)
    logger.addHandler(fileHandler)

    #streamHandler = logging.StreamHandler()
    #streamHandler.setFormatter(fomatter)
    #logger.addHandler(streamHandler)

    logger.setLevel(logging.DEBUG)
    return logger

class output_logging(object):
    def __init__(self, mode, real_time=False, dump_dir=None):
        self.mode = mode
        self.real_time = real_time
        self.dump_dir = dump_dir if dump_dir else None

        if dump_dir:
            self.dump = open(os.path.join(dump_dir, 'logs/output.tsv'), 'w', encoding='utf-8', newline='')
            self.wr = csv.writer(self.dump, delimiter='\t')

            # header
            if mode == 'eval':
                self.wr.writerow(['Ground_truth', 'Predcit', 'sentence'])
            elif mode == 'test_str':
                self.wr.writerow(['Predict', 'sentence'])

    def __del__(self):
        if self.dump_dir:
            self.dump.close()

    def logs(self, sentence, pred, ground_turth=None):
        if self.real_time:
            if self.mode == 'eval':
                for p, g, s in zip(pred, ground_turth, sentence):
                    print('Ground_truth | Predict')
                    print(int(g), '         ', int(p))
                    print(s, end='\n\n')
            elif self.mode == 'test_str':
                for p, s in zip(pred, sentence):
                    print('predict : ', int(p))
                    print(s, end='\n\n')

        if self.dump_dir:
            if self.mode == 'eval':
                for p, g, s in zip(pred, ground_turth, sentence):
                    self.wr.writerow([int(p), int(g), s])
            elif self.mode == 'test_str':
                for p, s in zip(pred, sentence):
                    self.wr.writerow([int(p), s])

max_stars_repo_name: peng-yifei/FLiText


text:
import numpy as np
from meta_rand_envs.ant_changing_dir import AntChangingDirEnv

from . import register_env

@register_env('ant-changing-dir')
class AntChangingDirWrappedEnv(AntChangingDirEnv):
    def __init__(self, n_tasks=2, *args, **kwargs):
        super(AntChangingDirWrappedEnv, self).__init__(*args, **kwargs)
        self.tasks = self.sample_tasks(n_tasks)
        self.reset_task(0)

    def get_all_task_idx(self):
        return range(len(self.tasks))

    def reset_task(self, idx):
        self._task = self.tasks[idx]
        self.goal_direction_start = self._task['direction']
        self.reset()

max_stars_repo_name: BZSROCKETS/cemrl

max_issues_repo_name: BZSROCKETS/cemrl

max_forks_repo_name: BZSROCKETS/cemrl


text:
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

#!/usr/bin/env python
import json
import numpy as np

def compute_metrics(filename):
    parsed_results = parse_results(filename)
    results = extract_measures(parsed_results)
    return results

def get_loss_history(parsed_results):
    losses = np.array([r['loss'] for r in parsed_results])
    return losses

def get_switch_times(parsed_results):
    sequences = np.array([r['sequence'] for r in parsed_results])
    sequences_ids, switch_times = np.unique(sequences, return_index=True)
    return switch_times

def get_domain_history(parsed_results, switch_times):
    domain_history = np.array([parsed_results[t]['domain'] for t in switch_times])
    return domain_history

def get_domain_names(parsed_results):
    domain_names = {}
    for r in parsed_results:
        domain_names[r['domain']] = r['domain_name']
    return domain_names

def parse_results(filename):
    with open(filename) as f:
        parsed_results = []
        for line in f:
            parsed_line = json.loads(line)
            if parsed_line['sequence'] >= 50:
                parsed_results.append(parsed_line)
    if not parsed_results or parsed_line['sequence'] <  99:
        return []
    return parsed_results

def calc_surprisal_intensity(losses, _):
    return np.mean(losses[:10])

def calc_surprisal_duration(losses, prev_loss):
    duration = 0
    #loss_avg = np.average(losses)
    for i in range(len(losses)):
        if losses[i] < prev_loss:
            break
        duration += 1
    return duration

def get_mean_loss_by_domain(parsed_results):
    losses_by_domain = stitch_losses_by_domain(parsed_results)
    domain_names = get_domain_names(parsed_results)
    return {domain_names[d]: np.mean(losses) for d, losses in losses_by_domain.items()}

def stitch_losses_by_domain(parsed_results):
    loss_history = get_loss_history(parsed_results)
    switch_times = get_switch_times(parsed_results)
    dom_names = get_domain_history(parsed_results, switch_times)
    loss_per_domain = {}
    for i in range(len(switch_times)-1):
        if dom_names[i] not in loss_per_domain:
            loss_per_domain[dom_names[i]] = []
        local_losses = loss_history[switch_times[i]:switch_times[i+1]]
        loss_per_domain[dom_names[i]].extend(local_losses)
    loss_per_domain[dom_names[-1]].extend(loss_history[switch_times[-1]:])
    return loss_per_domain

def get_surprisal_by_domain(parsed_results, surprisal_measure):
    loss_history = get_loss_history(parsed_results)
    switch_times = get_switch_times(parsed_results)
    dom_names = get_domain_history(parsed_results, switch_times)
    real_domain_names = get_domain_names(parsed_results)
    surprisal_per_domain = {}
    avg_surprisal_per_domain = {}
    prev_losses = {}
    for i in range(len(switch_times)-1):
        if dom_names[i] not in surprisal_per_domain:
            surprisal_per_domain[dom_names[i]] = []
        local_losses = loss_history[switch_times[i]:switch_times[i+1]]
        if dom_names[i] in prev_losses:
            surprisal_per_domain[dom_names[i]].append(surprisal_measure(local_losses, prev_losses[dom_names[i]]))
        prev_losses[dom_names[i]] = np.mean(local_losses)
    #surprisal_per_domain[dom_names[-1]].append(surprisal_measure(local_losses))
    all_surprisals = []
    for el in surprisal_per_domain:
        avg_surprisal_per_domain[real_domain_names[el]] = np.average(surprisal_per_domain[el])
        all_surprisals.extend(surprisal_per_domain[el])
    gen_avg_surprisal = np.average(all_surprisals)
    return gen_avg_surprisal, avg_surprisal_per_domain

def extract_measures(parsed_results):
    loss_history = get_loss_history(parsed_results)
    if len(loss_history) == 0:
        return {}
    loss_per_domain = stitch_losses_by_domain(parsed_results)
    loss = np.mean(loss_history)
    total_pp = np.exp(loss) if loss < 20 else float('inf')
    results = {'loss': loss, 'total_pp': total_pp}
    loss_by_domain = get_mean_loss_by_domain(parsed_results)
    surprisal_intensity, surprisal_intensity_per_domain = get_surprisal_by_domain(parsed_results, calc_surprisal_intensity)
    results['ppl@sw'] = np.exp(surprisal_intensity)
    surprisal_duration, surprisal_duration_per_domain = get_surprisal_by_domain(parsed_results, calc_surprisal_duration)
    results['rec'] = surprisal_duration
    for domain, dloss in loss_by_domain.items():
        results[f'loss_{domain}'] = dloss
        results[f'total_pp_{domain}'] = np.exp(dloss) if loss < 20 else float('inf')
    for domain, dsurprisal in surprisal_intensity_per_domain.items():
        results[f'ppl@sw_{domain}'] = np.exp(dsurprisal)
    for domain, dsurprisal in surprisal_duration_per_domain.items():
        results[f'rec_{domain}'] = dsurprisal
    return results

max_stars_repo_name: germank/CALM

max_issues_repo_name: germank/CALM

max_forks_repo_name: germank/CALM


text:
import numpy as np
import astropy.constants as c
import astropy.units as u
from scipy.stats import truncnorm

def VelocitySampler(potential, r, sigma, with_units=False):
        '''
        Evaluates the potential at a given radius
        Parameters
        ----------
        potential:
          potential function of an initialized profile class
        r: astropy.units.Quantity
            radius to evaluate potential
        sigma: astropy.units.Quantity
            velocity dispersion
        with_units: boolean (default False)
            convert to astropy quantity in units of km/s

        Returns
        -------
        dict
            sampled radial and tangential velocities
        '''

        # Convert units for consistency
        r = (r.to(u.kpc))
        sigma = (sigma.to(u.km / u.s)).value

        v = {
            'vr': [],
            'vt': [],
            'vx': [],
            'vy': [],
            'vz': [],
            'vd': sigma
        }

        for i in np.arange(r.size):

            v_esc = (
                (2*abs(potential(r[i])))**(1/2)).to(u.km / u.s).value

            a, b = -1*v_esc/sigma[i], v_esc/sigma[i]

            # Sample from a truncated normal distribution s.t. we ignore energies that launch star out of system
            vr = truncnorm.rvs(a, b, scale=sigma[i], loc=0) * u.km / u.s

            # Using whatever energy we have left from the prior drawn sample, select a valid tangential velocity
            max_vt = ((2*abs(potential(r[i])) -
                       (vr**2))**(1/2)).to(u.km / u.s).value
            vt = truncnorm.rvs(-1*max_vt/sigma[i], max_vt /
                               sigma[i], scale=sigma[i], loc=0) * u.km / u.s

            if (not with_units):
                vr = vr.value
                vt = vt.value

            v['vr'].append(vr)
            v['vt'].append(vt)

            v_length = (vr**2 + vt**2)**(1/2)

            v_xyz = np.random.normal(size=3)
            v_xyz = (v_length * v_xyz) / np.linalg.norm(v_xyz, axis=0)

            v['vx'].append(v_xyz[0])
            v['vy'].append(v_xyz[1])
            v['vz'].append(v_xyz[2])

        return v

max_stars_repo_name: jaybaptista/dune

max_issues_repo_name: jaybaptista/dune

max_forks_repo_name: jaybaptista/dune

reason: import numpy,from scipy,import astropy


text:
import numpy as np
import os
import math
import matplotlib.cm as cm
import matplotlib.pylab as plt
from scipy import linalg as la

def getEpicenters(filename):
    infile = open(filename, 'r')
    data = infile.readlines()
    T = len(data)
    epicentersAngles = np.zeros((T, 2))
    epicentersEuclidean = np.zeros((T, 3))
    for i in xrange(T):
        substring1 = data[i][20:26]
        substring2 = data[i][26:33]
        epicentersAngles[i,:] = np.array([90. - float(substring1[0:5])*.001, float(substring2[0:6])*.001])
        if substring1[5] == 'S':
            epicentersAngles[i,0] = 180 - epicentersAngles[i,0]
        if substring2[6] == 'W':
            epicentersAngles[i,1] *= -1.
    return epicentersAngles

def getAllData(directory):
    filenames = os.listdir(directory)
    T = len(filenames)
    epicentersAngles = getEpicenters(directory + filenames[0])
    for i in xrange(1, T):
        temp = getEpicenters(directory + filenames[i])
        epicentersAngles = np.append(epicentersAngles, temp, 0)
    return epicentersAngles

def anglesToEuclidean(angles):
    T = angles.shape[0]
    euclidean = np.zeros((T, 3))
    for i in xrange(T):
        euclidean[i,:] = np.array([np.sin(math.radians(angles[i,0]))*np.cos(math.radians(angles[i,1])),
                                   np.sin(math.radians(angles[i,0]))*np.sin(math.radians(angles[i,1])),
                                   np.cos(math.radians(angles[i,0]))])
    return euclidean

def euclideanToAngles(euclidean):
    T = euclidean.shape[0]
    angles = np.zeros((T, 2))
    for i in xrange(T):
        angles[i,:] = np.array([math.degrees(math.acos(euclidean[i,2])),
                                math.degrees(math.atan2(euclidean[i,1], euclidean[i,0]))])
    return angles

def anglesToLL(angles):
    T = angles.shape[0]
    LL = np.zeros((T, 2))
    LL[:,0] = angles[:,1]
    LL[:,1] = -1 * (angles[:,0] - 90)
    return LL

def angleDistance(angle1,angle2):
    return math.acos(angle1.dot(angle2)/(la.norm(angle1)*la.norm(angle2)))

reason: import numpy,from scipy


text:
"""
Hardware performance analysis
"""

import os
import sys
import time
import math
import shutil
import tabulate
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torchvision

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def accuracy(output, target, topk=(1,)):
    """Computes the accuracy over the k top predictions for the specified values of k"""
    with torch.no_grad():
        maxk = max(topk)
        batch_size = target.size(0)

        _, pred = output.topk(maxk, 1, True, True)
        pred = pred.t()
        correct = pred.eq(target.view(1, -1).expand_as(pred))

        res = []
        for k in topk:
            correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
            res.append(correct_k.mul_(100.0 / batch_size))
        return res

def test(testloader, net, criterion):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    net.eval()
    test_loss = 0

    end = time.time()
    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(testloader):
            mean_loader = []
            inputs, targets = inputs.cuda(), targets.cuda(non_blocking=True)
            outputs = net(inputs)
            loss = criterion(outputs, targets)

            prec1, prec5 = accuracy(outputs.data, targets, topk=(1, 5))
            losses.update(loss.item(), inputs.size(0))
            top1.update(prec1.item(), inputs.size(0))
            top5.update(prec5.item(), inputs.size(0))
            test_loss += loss.item()

            batch_time.update(time.time() - end)
            end = time.time()

            # single image test
            if batch_idx == 0:
                break
    return top1.avg, losses.avg

class IMCEstimator(object):
    def __init__(self, model, wbit, hw_specs, p_specs):
        self.model = model
        self.state_dict = model.state_dict()

        # rram property
        self.hw_specs = hw_specs
        self.p_specs = p_specs
        self.array_size = hw_specs["array_size"]
        self.col_per_weight = wbit // hw_specs["cellbit"]

        assert len(self.array_size) == 2, "Array size must contains two dims (row, col)"

    def array_by_layer(self):
        num_array = {}
        num_rows = {}
        total_param = 0
        for k, layer in self.model.named_modules():
            if isinstance(layer, nn.Conv2d):
                weight = layer.weight
                o, c, kw, kh = weight.size()

                # consumption
                n_rows = math.ceil(c*kw*kh / self.array_size[0])
                n_cols = math.ceil(o / self.array_size[1])*self.col_per_weight
                n_array = n_rows * n_cols

                num_array[k] = n_array
                num_rows[k] = n_rows
                total_param += o*c*kw*kh
        return num_array, total_param, num_rows

    def nops(self, ofm_size, narr, spars):
        return narr*ofm_size[2]*ofm_size[3]*(1-spars)

    def cell_energy(self, N, G, V, T):
        return G*N*(V**2)*T

    def get_relu_energy(self, relu_energy, nunits, ofm_size):
        return ofm_size[1]*ofm_size[2]*ofm_size[3]*relu_energy / nunits

    def get_buffer_energy(self, buffer_energy, bw, ofm_size):
        return ofm_size[1]*ofm_size[2]*ofm_size[3] * buffer_energy * 4

    def get_adder_energy(self, add_energy, nunits, ofm_size):
        return ofm_size[1]*ofm_size[2]*ofm_size[3] * add_energy / nunits

    def get_model_energy(self, ofm, grp_spars=0):
        arrays, total_param, num_rows = self.array_by_layer()

        total_energy = 0
        total_arr_energy = 0
        total_relu_energy = 0
        total_buf_energy = 0
        total_add_energy = 0

        for k, v in ofm.items():
            ops = self.nops(v, arrays[k], grp_spars)
            add_pow = math.ceil(math.log2(math.sqrt(num_rows[k])))
            add_stage = 2**add_pow

            arr_e = ops*self.hw_specs["energy"]
            relu_e = self.get_relu_energy(self.p_specs["relu_energy"], self.p_specs["relu_nunits"], v)
            buf_e = self.get_buffer_energy(self.p_specs["buffer_energy"], 4, v)
            add_e = self.get_adder_energy(self.p_specs["add_tree"][add_stage], self.p_specs["add_nunits"], v)
            e = arr_e + relu_e + buf_e + add_e

            # logger.info("Layer: {}, Nrows: {}, Array Energy: {}, ReLU Energy: {}\n".format(k, num_rows[k], arr_e, relu_e))
            total_energy += e
            total_arr_energy += arr_e
            total_relu_energy += relu_e
            total_buf_energy += buf_e
            total_add_energy += add_e

        model_energy = {
            "total": total_energy*1e+6,
            "array": total_arr_energy*1e+6,
            "relu": total_relu_energy*1e+6,
            "buffer": total_buf_energy*1e+6,
            "adder": total_add_energy*1e+6,
            "m_buffer": self.p_specs["buffer_energy"]*40e+3
        }
        return model_energy

    def get_model_area(self, ofm):
        arrays, total_param, num_rows = self.array_by_layer()
        mem_area = 0
        for k, v in ofm.items():
            n_arrays = arrays[k]
            mem_area += self.hw_specs["arr_area"] * n_arrays

        total_area = mem_area + self.p_specs["add_area"] + self.p_specs["buffer_area"] + self.p_specs["mbuffer_area"]
        # convert to um^2
        model_area = {
            "total": total_area*1e+6,
            "array": mem_area*1e+6,
            "relu": self.p_specs["relu_area"]*1e+6,
            "add_area": self.p_specs["add_area"]*1e+6,
            "buffer_area": self.p_specs["buffer_area"]*1e+6,
            "mbuffer_area": self.p_specs["mbuffer_area"]*1e+6
        }

        return model_area

    def reprogram_energy(self, spars, cell_specs):
        """
        Element-wise sparsity for reprogramming
        """
        arrays, total_param, num_rows = self.array_by_layer()
        sparse_weight = round(total_param * spars)

        cell_energy = self.cell_energy(cell_specs["NPulse"], cell_specs["R"], cell_specs["wdv"], cell_specs["wdp"])
        rp_energy = sparse_weight * cell_energy * self.col_per_weight*1e+6
        return rp_energy

    def finetune_reprogram_energy(self, ref_dict, ft_dict, cell_specs):
        rp_energy = 0
        for k, v_ref in ref_dict.items():
            v_ft = ft_dict[k]

            # add offset (TODO: make the offset general)
            v_ref = v_ref + 7.
            v_ft = v_ft + 7.

            # compute the average difference
            diff = v_ft.sub(v_ref).mean().ceil()
            print("Layer {}; Average level change = {}".format(k, diff))
            cell_change = cell_specs["R"]*diff

            cell_energy = self.cell_energy(cell_specs["NPulse"], cell_change, cell_specs["wdv"], cell_specs["wdp"])
            rp_energy += cell_energy*v_ft.numel()*self.col_per_weight*1e+6
        return rp_energy


text:
# -*- coding: utf-8 -*-
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

"""
Common data processing utilities that are used in a
typical object detection data pipeline.
"""
import logging

import numpy as np
import torch
from PIL import Image, ImageOps

from dl_lib.structures import (BitMasks, Boxes, BoxMode,  # RotatedBoxes,
                               Instances, Keypoints, PolygonMasks)
from dl_lib.utils.file_io import PathManager

from . import transforms as T
from .catalog import MetadataCatalog

class SizeMismatchError(ValueError):
    """
    When loaded image has difference width/height compared with annotation.
    """

def read_image(file_name, format=None):
    """
    Read an image into the given format.
    Will apply rotation and flipping if the image has such exif information.

    Args:
        file_name (str): image file path
        format (str): one of the supported image modes in PIL, or "BGR"

    Returns:
        image (np.ndarray): an HWC image
    """
    with PathManager.open(file_name, "rb") as f:
        image = Image.open(f)

        image = ImageOps.exif_transpose(image)
        # capture and ignore this bug: https://github.com/python-pillow/Pillow/issues/3973
        try:
            image = ImageOps.exif_transpose(image)
        except Exception:
            pass

        if format is not None:
            # PIL only supports RGB, so convert to RGB and flip channels over below
            conversion_format = format
            if format == "BGR":
                conversion_format = "RGB"
            image = image.convert(conversion_format)
        image = np.asarray(image)
        if format == "BGR":
            # flip channels if needed
            image = image[:, :, ::-1]
        # PIL squeezes out the channel dimension for "L", so make it HWC
        if format == "L":
            image = np.expand_dims(image, -1)
        return image

def check_image_size(dataset_dict, image):
    """
    Raise an error if the image does not match the size specified in the dict.
    """
    if "width" in dataset_dict or "height" in dataset_dict:
        image_wh = (image.shape[1], image.shape[0])
        expected_wh = (dataset_dict["width"], dataset_dict["height"])
        if not image_wh == expected_wh:
            raise SizeMismatchError(
                "Mismatched (W,H){}, got {}, expect {}".format(
                    " for image " + dataset_dict["file_name"]
                    if "file_name" in dataset_dict
                    else "",
                    image_wh,
                    expected_wh,
                )
            )

    # To ensure bbox always remap to original image size
    if "width" not in dataset_dict:
        dataset_dict["width"] = image.shape[1]
    if "height" not in dataset_dict:
        dataset_dict["height"] = image.shape[0]

def transform_proposals(
    dataset_dict, image_shape, transforms, min_box_side_len, proposal_topk
):
    """
    Apply transformations to the proposals in dataset_dict, if any.

    Args:
        dataset_dict (dict): a dict read from the dataset, possibly
            contains fields "proposal_boxes", "proposal_objectness_logits", "proposal_bbox_mode"
        image_shape (tuple): height, width
        transforms (TransformList):
        min_box_side_len (int): keep proposals with at least this size
        proposal_topk (int): only keep top-K scoring proposals

    The input dict is modified in-place, with abovementioned keys removed. A new
    key "proposals" will be added. Its value is an `Instances`
    object which contains the transformed proposals in its field
    "proposal_boxes" and "objectness_logits".
    """
    if "proposal_boxes" in dataset_dict:
        # Transform proposal boxes
        boxes = transforms.apply_box(
            BoxMode.convert(
                dataset_dict.pop("proposal_boxes"),
                dataset_dict.pop("proposal_bbox_mode"),
                BoxMode.XYXY_ABS,
            )
        )
        boxes = Boxes(boxes)
        objectness_logits = torch.as_tensor(
            dataset_dict.pop("proposal_objectness_logits").astype("float32")
        )

        boxes.clip(image_shape)
        keep = boxes.nonempty(threshold=min_box_side_len)
        boxes = boxes[keep]
        objectness_logits = objectness_logits[keep]

        proposals = Instances(image_shape)
        proposals.proposal_boxes = boxes[:proposal_topk]
        proposals.objectness_logits = objectness_logits[:proposal_topk]
        dataset_dict["proposals"] = proposals

def transform_instance_annotations(
    annotation, transforms, image_size, *, keypoint_hflip_indices=None
):
    """
    Apply transforms to box, segmentation and keypoints annotations of a single instance.

    It will use `transforms.apply_box` for the box, and
    `transforms.apply_coords` for segmentation polygons & keypoints.
    If you need anything more specially designed for each data structure,
    you'll need to implement your own version of this function or the transforms.

    Args:
        annotation (dict): dict of instance annotations for a single instance.
            It will be modified in-place.
        transforms (TransformList):
        image_size (tuple): the height, width of the transformed image
        keypoint_hflip_indices (ndarray[int]): see `create_keypoint_hflip_indices`.

    Returns:
        dict:
            the same input dict with fields "bbox", "segmentation", "keypoints"
            transformed according to `transforms`.
            The "bbox_mode" field will be set to XYXY_ABS.
    """
    bbox = BoxMode.convert(
        annotation["bbox"], annotation["bbox_mode"], BoxMode.XYXY_ABS
    )
    # Note that bbox is 1d (per-instance bounding box)
    annotation["bbox"] = transforms.apply_box([bbox])[0]
    annotation["bbox_mode"] = BoxMode.XYXY_ABS

    if "segmentation" in annotation:
        # each instance contains 1 or more polygons
        polygons = [np.asarray(p).reshape(-1, 2) for p in annotation["segmentation"]]
        annotation["segmentation"] = [
            p.reshape(-1) for p in transforms.apply_polygons(polygons)
        ]

    if "keypoints" in annotation:
        keypoints = transform_keypoint_annotations(
            annotation["keypoints"], transforms, image_size, keypoint_hflip_indices
        )
        annotation["keypoints"] = keypoints

    return annotation

def transform_keypoint_annotations(
    keypoints, transforms, image_size, keypoint_hflip_indices=None
):
    """
    Transform keypoint annotations of an image.

    Args:
        keypoints (list[float]): Nx3 float in dl_lib Dataset format.
        transforms (TransformList):
        image_size (tuple): the height, width of the transformed image
        keypoint_hflip_indices (ndarray[int]): see `create_keypoint_hflip_indices`.
    """
    # (N*3,) -> (N, 3)
    keypoints = np.asarray(keypoints, dtype="float64").reshape(-1, 3)
    keypoints[:, :2] = transforms.apply_coords(keypoints[:, :2])

    # This assumes that HorizFlipTransform is the only one that does flip
    do_hflip = (
        sum(isinstance(t, T.HFlipTransform) for t in transforms.transforms) % 2 == 1
    )

    # Alternative way: check if probe points was horizontally flipped.
    # probe = np.asarray([[0.0, 0.0], [image_width, 0.0]])
    # probe_aug = transforms.apply_coords(probe.copy())
    # do_hflip = np.sign(probe[1][0] - probe[0][0]) != np.sign(probe_aug[1][0] - probe_aug[0][0])  # noqa

    # If flipped, swap each keypoint with its opposite-handed equivalent
    if do_hflip:
        assert keypoint_hflip_indices is not None
        keypoints = keypoints[keypoint_hflip_indices, :]

    # Maintain COCO convention that if visibility == 0, then x, y = 0
    # TODO may need to reset visibility for cropped keypoints,
    # but it does not matter for our existing algorithms
    keypoints[keypoints[:, 2] == 0] = 0
    return keypoints

def annotations_to_instances(annos, image_size, mask_format="polygon"):
    """
    Create an :class:`Instances` object used by the models,
    from instance annotations in the dataset dict.

    Args:
        annos (list[dict]): a list of instance annotations in one image, each
            element for one instance.
        image_size (tuple): height, width

    Returns:
        Instances:
            It will contain fields "gt_boxes", "gt_classes",
            "gt_masks", "gt_keypoints", if they can be obtained from `annos`.
            This is the format that builtin models expect.
    """
    boxes = [
        BoxMode.convert(obj["bbox"], obj["bbox_mode"], BoxMode.XYXY_ABS)
        for obj in annos
    ]
    target = Instances(image_size)
    boxes = target.gt_boxes = Boxes(boxes)
    boxes.clip(image_size)

    classes = [obj["category_id"] for obj in annos]
    classes = torch.tensor(classes, dtype=torch.int64)
    target.gt_classes = classes

    if len(annos) and "segmentation" in annos[0]:
        polygons = [obj["segmentation"] for obj in annos]
        if mask_format == "polygon":
            masks = PolygonMasks(polygons)
        else:
            assert mask_format == "bitmask", mask_format
            masks = BitMasks.from_polygon_masks(polygons, *image_size)
        target.gt_masks = masks

    if len(annos) and "keypoints" in annos[0]:
        kpts = [obj.get("keypoints", []) for obj in annos]
        target.gt_keypoints = Keypoints(kpts)

    return target

def filter_empty_instances(instances, by_box=True, by_mask=True):
    """
    Filter out empty instances in an `Instances` object.

    Args:
        instances (Instances):
        by_box (bool): whether to filter out instances with empty boxes
        by_mask (bool): whether to filter out instances with empty masks

    Returns:
        Instances: the filtered instances.
    """
    assert by_box or by_mask
    r = []
    if by_box:
        r.append(instances.gt_boxes.nonempty())
    if instances.has("gt_masks") and by_mask:
        r.append(instances.gt_masks.nonempty())

    # TODO: can also filter visible keypoints

    if not r:
        return instances
    m = r[0]
    for x in r[1:]:
        m = m & x
    return instances[m]

def create_keypoint_hflip_indices(dataset_names):
    """
    Args:
        dataset_names (list[str]): list of dataset names
    Returns:
        ndarray[int]: a vector of size=#keypoints, storing the
        horizontally-flipped keypoint indices.
    """

    check_metadata_consistency("keypoint_names", dataset_names)
    check_metadata_consistency("keypoint_flip_map", dataset_names)

    meta = MetadataCatalog.get(dataset_names[0])
    names = meta.keypoint_names
    # TODO flip -> hflip
    flip_map = dict(meta.keypoint_flip_map)
    flip_map.update({v: k for k, v in flip_map.items()})
    flipped_names = [i if i not in flip_map else flip_map[i] for i in names]
    flip_indices = [names.index(i) for i in flipped_names]
    return np.asarray(flip_indices)

def gen_crop_transform_with_instance(crop_size, image_size, instance):
    """
    Generate a CropTransform so that the cropping region contains
    the center of the given instance.

    Args:
        crop_size (tuple): h, w in pixels
        image_size (tuple): h, w
        instance (dict): an annotation dict of one instance, in dl_lib's
            dataset format.
    """
    crop_size = np.asarray(crop_size, dtype=np.int32)
    bbox = BoxMode.convert(instance["bbox"], instance["bbox_mode"], BoxMode.XYXY_ABS)
    center_yx = (bbox[1] + bbox[3]) * 0.5, (bbox[0] + bbox[2]) * 0.5

    min_yx = np.maximum(np.floor(center_yx).astype(np.int32) - crop_size, 0)
    max_yx = np.maximum(np.asarray(image_size, dtype=np.int32) - crop_size, 0)
    max_yx = np.minimum(max_yx, np.ceil(center_yx).astype(np.int32))

    y0 = np.random.randint(min_yx[0], max_yx[0] + 1)
    x0 = np.random.randint(min_yx[1], max_yx[1] + 1)
    return T.CropTransform(x0, y0, crop_size[1], crop_size[0])

def check_metadata_consistency(key, dataset_names):
    """
    Check that the datasets have consistent metadata.

    Args:
        key (str): a metadata key
        dataset_names (list[str]): a list of dataset names

    Raises:
        AttributeError: if the key does not exist in the metadata
        ValueError: if the given datasets do not have the same metadata values defined by key
    """
    if len(dataset_names) == 0:
        return
    logger = logging.getLogger(__name__)
    entries_per_dataset = [getattr(MetadataCatalog.get(d), key) for d in dataset_names]
    for idx, entry in enumerate(entries_per_dataset):
        if entry != entries_per_dataset[0]:
            logger.error(
                "Metadata '{}' for dataset '{}' is '{}'".format(
                    key, dataset_names[idx], str(entry)
                )
            )
            logger.error(
                "Metadata '{}' for dataset '{}' is '{}'".format(
                    key, dataset_names[0], str(entries_per_dataset[0])
                )
            )
            raise ValueError("Datasets have different metadata '{}'!".format(key))

def build_transform_gen(cfg, is_train):
    """
    Create a list of :class:`TransformGen` from config.
    Now it includes resizing and flipping.

    Returns:
        list[TransformGen]
    """
    logger = logging.getLogger(__name__)

    tfm_gens = []

    if is_train:
        for (aug, args) in cfg.INPUT.AUG.TRAIN_PIPELINES:
            if aug == "ResizeShortestEdge":
                check_sample_valid(args)
            tfm_gens.append(getattr(T, aug)(**args))
    else:
        for (aug, args) in cfg.INPUT.AUG.TEST_PIPELINES:
            if aug == "ResizeShortestEdge":
                check_sample_valid(args)
            tfm_gens.append(getattr(T, aug)(**args))

    logger.info("TransformGens used: " + str(tfm_gens))

    return tfm_gens

def check_sample_valid(args):
    if args["sample_style"] == "range":
        assert (
            len(args["min_size"]) == 2
        ), f"more than 2 ({len(args['min_size'])}) min_size(s) are provided for ranges"

max_stars_repo_name: harrylin-hyl/CenterNet-better

max_issues_repo_name: harrylin-hyl/CenterNet-better

max_forks_repo_name: harrylin-hyl/CenterNet-better


text:
import sys
import numpy as np
from mfd import sca
from matplotlib import pyplot as pl

n = int(sys.argv[1])
xr = np.linspace(0, 2, n)
yr = xr[:int(n*.66)]
w = abs(xr[1]-xr[0])
x, y = np.meshgrid(xr, yr)
r = np.sqrt(x*x+y*y)
z = np.exp(-r*r)
z[r > 1.75] = np.nan
a = sca(z, w)
e = 100 * (2 * a - r) / r
e = np.ma.masked_invalid(e)

pl.figure(1, (13.66, 7.68))
pl.title('Gaussian hill')
pl.pcolormesh(x, y, e,
        vmin = -100, vmax = 100,
        cmap = pl.cm.seismic)
cb = pl.colorbar()
cb.set_label(r'Rel. deviations [%]')
pl.axes().set_aspect('equal')
pl.savefig('%s.png' % sys.argv[0][:-3])

max_stars_repo_name: UP-RS-ESP/mfdrouting

max_issues_repo_name: UP-RS-ESP/mfdrouting

max_forks_repo_name: UP-RS-ESP/mfdrouting