text:
"""
The Effect of Taxes on Prices in Washington State
Copyright (c) 2022 Cannlytics

Authors: Keegan Skeate <keegan@cannlytics.com>
Created: 2/22/2022
Updated: 2/24/2022
License: MIT License <https://opensource.org/licenses/MIT>

Description: This script analyzes the relationship between price and
sales tax by county in Washington State.

Resources:

    - Notes on Partial Equilibrium Analysis
    https://cannlytics.page.link/partial-equilibrium-notes

Data sources:

    - WA State Traceability Data January 2018 - November 2021
    https://lcb.app.box.com/s/e89t59s0yb558tjoncjsid710oirqbgd

    - WA State Sales Taxes
    https://dor.wa.gov/get-form-or-publication/publications-subject/local-sales-use-tax-rates-excel-file-format#2021

    - WA Sate Cannabis Tax
    https://dor.wa.gov/about/statistics-reports/recreational-and-medical-marijuana-taxes

"""
# Standard imports.
import glob
from datetime import datetime, timedelta

# External imports.
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
import numpy as np
import pandas as pd
import statsmodels.formula.api as smf
import seaborn as sns

# Internal imports.
from utils import format_millions

# Define the plot style.
plt.style.use('fivethirtyeight')
plt.rcParams.update({
    'font.family': 'Times New Roman',
    'font.size': 24,
})

# Specify where your data lives.
DATA_DIR = 'D:\\leaf-data'

#--------------------------------------------------------------------------
# Read the sample sales data.
# Random sample of sales: https://cannlytics.page.link/cds53
#--------------------------------------------------------------------------

# Read in the data.
DATA_FILE = f'{DATA_DIR}/samples/random-sales-items-2022-02-16.csv'
data = pd.read_csv(DATA_FILE)

#------------------------------------------------------------------------------
# Get the retail city and county.
# Licensees data: https://cannlytics.com/data/market/augmented-washington-state-licensees
# Augment yourself: https://github.com/cannlytics/cannabis-data-science/blob/main/2022-01-26/geocode_licensees.py
#------------------------------------------------------------------------------

# Augment fields from the licensees data.
licensee_fields = {
    'global_id': 'string',
    'city': 'string',
    'county': 'string',
}
licensees = pd.read_csv(
    f'{DATA_DIR}/augmented/augmented-washington-state-licensees.csv',
    usecols=list(licensee_fields.keys()),
    dtype=licensee_fields,
)
data = pd.merge(
    left=data,
    right=licensees,
    how='left',
    left_on='mme_id',
    right_on='global_id',
)
data.drop(['global_id'], axis=1, inplace=True, errors='ignore')
print('Augmented the sales data with city and county.')

#--------------------------------------------------------------------------
# Clean the data.
#--------------------------------------------------------------------------

# Determine wholesale vs retail transactions.
# Note: It is necessary to find out if there are any medical sales and
# exclude any medical sales from taxes.
data = data.loc[data['sale_type'] != 'wholesale']

# Drop observations with negative prices and prices in the upper quantile.
data = data.loc[data.price_total > 0]
data = data[data.price_total < data.price_total.quantile(.95)]

# Add a date column.
data['date'] = pd.to_datetime(data['created_at'])
data['day'] = data['date'].dt.date

# Add quarter (e.g. 2022Q1) field.
data['quarter'] = data.date.dt.to_period('Q')

# Clean the city name and county.
data['city'] = data['city'].str.title()
data['county'] = data['county'].str.title().str.replace(' County', '')

# TODO: Calculate price per total cannabinoids ($/mg)

# Identify the time period for analysis.
start = '2021-01-01'
end = '2021-10-31'
data = data.loc[
    (data['date'] >= pd.to_datetime(start)) &
    (data['date'] <= pd.to_datetime(end))
]
print('Data cleaned and limited to the period of analysis.')
print('{:,} observations.'.format(len(data)))

#--------------------------------------------------------------------------
# Get the average tax by county.
#--------------------------------------------------------------------------

# Initialize tax rate for each observation.
excise_tax = 0.37
state_tax = 0.065
data['tax_rate'] = excise_tax + state_tax

# Assign the location code for each sales item.
data['location_code'] = data['postal_code'].astype(str).str[-4:]

# Read in the taxes, file by file for each quarter.
tax_datafiles = f'{DATA_DIR}/taxes/*.xlsx'
files = glob.glob(tax_datafiles)
files = [x for x in files if '~' not in x]
for file_name in files:

    # Identify the quarter.
    year = int(f'20{file_name[-10:-8]}')
    quarter = f'{year}{file_name[-7:-5]}'
    print('Matching tax rates by city for', quarter)
    iter_start = datetime.now()

    # Read in city tax rates for the quarter.
    sales_tax = pd.read_excel(file_name, skiprows=2)
    sales_tax = sales_tax[[
        'Location',
        'Location Code',
        'Combined Sales Tax'
    ]]

    # Match sales items for each city in the quarter.
    # Note: This is slow. Can this be optimized?
    for index, values in sales_tax.iterrows():
        code = str(values['Location Code']).replace('.0', '')
        data['match'] = data['city'].apply(lambda x: x in values['Location'].strip())
        data.loc[
            (data['location_code'] == code) |
            (data['match']) &
            (data['quarter'] == quarter),
            'tax_rate'
        ] = values['Combined Sales Tax'] + excise_tax

    iter_end = datetime.now()
    print('Matched %i cities in ', iter_end - iter_start)

# Identify the samples with a local tax rate.
has_local_tax = data.loc[data['tax_rate'] > excise_tax + state_tax]
print('Proportion of sample with local tax:', round(len(has_local_tax) / len(data), 2))

# Calculate the amount paid in tax.
data['tax'] = data['tax_rate'] * data['price_total']

# Calculate the price paid by the consumers.
data['consumer_price'] = data['price_total'] + data['tax']

# Save the data.
SAVE_FILE = f'{DATA_DIR}/samples/random-sales-items-with-taxes.xlsx'
data.to_excel(SAVE_FILE, index=False)

#--------------------------------------------------------------------------
# Estimate the amount of sales,
# assuming that our sample is 1/1000 of the population.
#--------------------------------------------------------------------------

# Estimate the total tax paid by day.
daily_sales = data.groupby('day').sum()['price_total'] * 1000
daily_sales.index = pd.to_datetime(daily_sales.index)

# Estimate the total tax paid by month.
monthly_sales = daily_sales.groupby(pd.Grouper(freq='M')).sum()

# Estimate the total tax paid by year.
annual_sales = monthly_sales.groupby(pd.Grouper(freq='Y')).sum()
print('Estimated sales in first 10 months of 2021:', format_millions(annual_sales))

#--------------------------------------------------------------------------
# Estimate the amount paid in taxes,
# assuming that our sample is 1/1000 of the population.
#--------------------------------------------------------------------------

# Estimate the total tax paid by day.
daily_tax = data.groupby('day').sum()['tax'] * 1000
daily_tax.index = pd.to_datetime(daily_tax.index)

# Estimate the total tax paid by month.
monthly_tax = daily_tax.groupby(pd.Grouper(freq='M')).sum()

# Estimate the total tax paid by year.
annual_tax = daily_tax.groupby(pd.Grouper(freq='Y')).sum()
print('Estimated tax in first 10 months of 2021:', format_millions(annual_tax))

#--------------------------------------------------------------------------
# Analysis: Regress the average price of cannabis on tax rate by
# county by month for the time period of January to November of 2021.
# Optionally control for:
# ✓ county fixed effects
# ✓ time fixied effects
# ✓ total cannabinoids
# - number of retailers in the county
# - population
# - number of tourists
# - median income
# - avg. education in number of years
# - Number of sunny days (of producer)?
# - rainfall (of producer)?
# - avg. electricity price (of producer)?
# - avg. temperature (of producer)?
# - proportion of economy that is agricultural (of producer)?
#--------------------------------------------------------------------------

# Identify flower sales.
sample_type = 'usable_marijuana'
sample_data = data.loc[data.intermediate_type == sample_type]

# Add trend variable before creating the panel.
sample_data['t'] = sample_data.date.dt.month
sample_data['tax_percent'] = sample_data['tax_rate'] * 100
sample_data['ln_price'] = np.log(sample_data['price_total'])
sample_data['ln_consumer_price'] = np.log(sample_data['consumer_price'])

# Create panel data (county i, time t).
group = sample_data.groupby(['county', pd.Grouper(key='date', freq='M')])
panel = group.mean()

# Make the county and date variables.
panel['county'] = panel.index.get_level_values('county')
panel['date'] = panel.index.get_level_values('date')

# Regress the average price, p_it, on tax rate, t_it, for the various markets
# with time and county fixed effects.
formula = 'price_total ~ tax_percent + total_cannabinoid_percent + C(date) + C(county)'
regression = smf.ols(formula, data=panel).fit()
print(regression.summary())

# Regress the log of average price by county on the tax rate.
formula = 'ln_price ~ tax_percent + total_cannabinoid_percent + C(date) + C(county)'
regression = smf.ols(formula, data=panel).fit()
print(regression.summary())

# Regress the average consumer price, p_it + t_it, on tax rate, t_it, for the various markets
# with time and county fixed effects.
formula = 'consumer_price ~ tax_percent + total_cannabinoid_percent + C(date) + C(county)'
regression = smf.ols(formula, data=panel).fit()
print(regression.summary())

# Regress the log of average price by county on the tax rate.
formula = 'ln_consumer_price ~ tax_percent + total_cannabinoid_percent + C(date) + C(county)'
regression = smf.ols(formula, data=panel).fit()
print(regression.summary())

#--------------------------------------------------------------------------
# Interpretation:
# What would be the effect on prices paid by
# consumers and prices received by producers
# from a marginal increase (or decrease) in
# cannabis sales tax in Washington State?
#--------------------------------------------------------------------------

# TODO: Interpret the estimated parameters.

#--------------------------------------------------------------------------
# Visualize the regression results (avg. price to tax rate) (expected decreasing)
# and price paid by consumers to tax rate (expected to be increasing).
#--------------------------------------------------------------------------

# TODO: Create a 4x4 visualization!

fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(60, 40))

# 1. Duel-regression plot.
# fig, ax = plt.subplots(figsize=(19.8, 12))
ax1 = axs[0, 0]
colors = sns.color_palette('Set1', n_colors=10)
sns.regplot(
    x='tax_percent',
    y='consumer_price',
    data=panel,
    ax=ax1,
    label='Average Price Paid',
    color=sns.color_palette('Set2', n_colors=10)[0],
)
sns.regplot(
    x='tax_percent',
    y='price_total',
    data=panel,
    ax=ax1,
    label='Average Price Received',
    color=sns.color_palette('Set1', n_colors=10)[3],
)
# plt.ylabel('Average Price ($)', fontsize=32)
# plt.xlabel('Tax Rate (%)',  fontsize=32)
# plt.legend(loc='upper right', fontsize=32)
plt.setp(ax1.get_yticklabels()[0], visible=False) # Hide first label.
ax1.set_title(
    'Average Price Paid and Received for Cannabis Flower by County by Month \nAgainst Tax Rate in Washington State Counties in 2021',
    fontsize=44,
    pad=24,
)
# plt.figtext(0.05, -0.175,
#     """Notes: Washington State levies a {} cannabis excise tax, a {} state sales tax,
# and each locale may levy a sales tax, ranging from 0.5% to 4%.
# Data: A random sample of {:,} “{}” sale items.
# The top {} of sale item observations by price were excluded as outliers.
# Data Source: Washington State Traceability Data from {} through {}.""".format(
#     '37%',
#     '6.5%',
#     len(sample_data),
#     sample_type.replace('_', ' '),
#     '5%',
#     'January 2021',
#     'October 2021',
# ),
#     fontsize=28,
#     ha='left',
# )
plt.ylim(0, 35)

# 2. TODO: Distribution plot.
# fig, ax = plt.subplots(figsize=(19.8, 12))
# colors = sns.color_palette('Set2', n_colors=10)
# data['price_total'].hist(
#     bins=250,
#     density=1,
#     alpha=0.4,
#     zorder=9,
#     color=sns.color_palette('Set1', n_colors=10)[1],
#     label='Price Total',
# )
# data['tax'].hist(
#     bins=250,
#     density=1,
#     alpha=0.4,
#     zorder=99,
#     color=colors[1],
#     label='Tax',
# )
# data['consumer_price'].hist(
#     bins=250,
#     density=1,
#     alpha=1,
#     color=colors[0],
#     label='Consumer Price',
# )
# plt.legend(loc='upper right')
# plt.xlim(0, 50)
# # plt.ylim(0, 35)
# plt.show()

# 3. TODO: Timeseries plot of monthly taxes, sales, and expenditure.
# TODO: Plot actual taxes as a line plot with dots on top of the bars.
ax3 = axs[1, 0]
width = 6
colors = sns.color_palette('Set2', n_colors=10)
total_expenditure = monthly_sales + monthly_tax
ax3.bar(
    monthly_tax.index - timedelta(days=6) - timedelta(days=30),
    monthly_tax,
    color=colors[1],
    width=width,
    edgecolor='black',
    label='Total Taxes',
)
ax3.bar(
    monthly_sales.index - timedelta(days=30),
    monthly_sales,
    color=colors[2],
    width=width,
    edgecolor='black',
    label='Total Sales',
)
ax3.bar(
    total_expenditure.index + timedelta(days=6) - timedelta(days=30),
    total_expenditure,
    color=colors[0],
    width=width,
    edgecolor='black',
    label='Total Expenditure',
)
ax3.set_title(
    'Estimated Cannabis Sales and Taxes in Washington State in 2021',
    fontsize=44,
    pad=24,
)
# plt.xlabel('Month', fontsize=32)
# plt.ylabel('Dollars ($)',  fontsize=32)
# plt.legend(loc='upper right', fontsize=32)
ax3.xaxis.set_major_formatter(mdates.DateFormatter('%b. %Y'))
ax3.yaxis.set_major_formatter(FuncFormatter(format_millions))
plt.setp(ax3.get_yticklabels()[0], visible=False) # Hide first label.
# plt.ylim(0, 220_000_000)
plt.show()

# 4. TODO: Data block? Put all notes here?
# format_millions(annual_tax)
# format_millions(annual_sales)
# format_millions(total_expenditure.sum())
# average prices?

# notes = """Notes: Washington State levies a {} cannabis excise tax, a {} state sales tax,
# # and each locale may levy a sales tax, ranging from 0.5% to 4%.
# # Data: A random sample of {:,} “{}” sale items.
# # The top {} of sale item observations by price were excluded as outliers.
# # Data Source: Washington State Traceability Data from {} through {}.""".format(
#     '37%',
#     '6.5%',
#     len(sample_data),
#     sample_type.replace('_', ' '),
#     '5%',
#     'January 2021',
#     'October 2021',
# )
# plt.margins(1, 1)
# fig.savefig(
#     f'{DATA_DIR}/figures/regression_price_on_tax_2021.png',
#     format='png',
#     dpi=300,
#     bbox_inches='tight',
#     pad_inches=0.75,
#     transparent=False,
# )

max_stars_repo_name: GrahamAnto/cannabis-data-science

max_issues_repo_name: GrahamAnto/cannabis-data-science

max_forks_repo_name: GrahamAnto/cannabis-data-science

reason: import numpy,import statsmodels


text:
"""Common utilities"""

import numpy as np
from sklearn.metrics import mean_squared_error

def print_metirc(text: str, score: float):
    """Print metric score in predefined format"""
    print(f"{text:40s} {score:.3f}")

def print_mse_and_pc(name: str, y_true, y_pred):
    """Print MSE and Pearson Correlation for signal"""
    mse_test = mean_squared_error(y_true, y_pred)
    cor_test = np.corrcoef(y_true, y_pred)[0, 1]

    print_metirc(f"{name} MSE", mse_test)
    print_metirc(f"{name} PC", cor_test)

ext: py

max_stars_repo_name: hofbi/vibromaf

max_issues_repo_name: hofbi/vibromaf

max_forks_repo_name: hofbi/vibromaf

reason: import numpy


text:
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import copy
from typing import Any, Dict, List, Optional, Union

import attr
import numpy as np

from habitat.core.dataset import Episode
from habitat.tasks.nav.nav import NavigationTask
from habitat.tasks.rearrange.rearrange_sim import RearrangeSim
from habitat.tasks.rearrange.utils import CollDetails, rearrange_collision

class RearrangeTask(NavigationTask):
    """
    Defines additional logic for valid collisions and gripping.
    """

    def __init__(self, *args, sim, dataset=None, **kwargs) -> None:
        self.n_objs = len(dataset.episodes[0].targets)

        super().__init__(sim=sim, dataset=dataset, **kwargs)
        self.is_gripper_closed = False
        self._sim: RearrangeSim = sim
        self.use_max_accum_force = self._config.MAX_ACCUM_FORCE
        self._ignore_collisions: List[Any] = []
        self.prev_force: Optional[float] = None

    def reset(self, episode: Episode):
        super_reset = True
        self._ignore_collisions = []
        if super_reset:
            observations = super().reset(episode)
        else:
            observations = None
        self.prev_measures = self.measurements.get_metrics()
        self.prev_picked = False
        self.n_succ_picks = 0
        self.coll_accum = CollDetails()
        self.prev_coll_accum = CollDetails()
        self.should_end = False
        self.accum_force = 0.0
        self.prev_force = None
        self._done = False
        self._get_collision_info()

        return observations

    def _get_collision_info(self):
        robot_force, _, overall_force = self._get_coll_forces()
        if self._config.COUNT_OBJ_COLLISIONS:
            self.cur_force = overall_force
        else:
            self.cur_force = robot_force

        if self.prev_force is not None:
            force_diff: float = self.cur_force - self.prev_force
            self.add_force = 0.0
            if force_diff > 20:
                self.add_force = force_diff
                self.accum_force += self.add_force
                self.prev_force = self.cur_force
            elif force_diff < 0:
                self.prev_force = self.cur_force
        else:
            self.add_force = 0.0
            self.prev_force = self.cur_force
        self.update_coll_count()

    def _is_violating_hold_constraint(self):
        if self._config.get("IGNORE_HOLD_VIOLATE", False):
            return False
        # Is the object firmly in the grasp of the robot?
        hold_obj = self._sim.snapped_obj_id
        ee_pos = self._sim.robot.ee_transform.translation
        if hold_obj is not None:
            obj_pos = self._sim.get_translation(hold_obj)
            if np.linalg.norm(ee_pos - obj_pos) >= self._config.HOLD_THRESH:
                return True

        if self._config.get("IGNORE_ART_HOLD_VIOLATE", False):
            return False

        return False

    def step(self, action: Dict[str, Any], episode: Episode):
        obs = super().step(action=action, episode=episode)

        self._get_collision_info()
        self.prev_coll_accum = copy.copy(self.coll_accum)

        return obs

    def _check_episode_is_active(
        self,
        *args: Any,
        action: Union[int, Dict[str, Any]],
        episode: Episode,
        **kwargs: Any,
    ) -> bool:
        from habitat.tasks.rearrange.rearrange_sensors import (
            RearrangePickSuccess,
        )

        done = False
        if self.measurements.get_metrics()[RearrangePickSuccess.cls_uuid]:
            done = True

        # If we have any sort of collision at all the episode is over.
        if (
            self._config.MAX_COLLISIONS > 0
            and self.cur_collisions > self._config.MAX_COLLISIONS
        ):
            done = True

        if self.should_end:
            done = True

        if self._is_violating_hold_constraint():
            done = True

        if (
            self._config.FORCE_BASED
            and self.use_max_accum_force > 0
            and self.accum_force > self.use_max_accum_force
        ):
            done = True
        return not done

    @property
    def _delta_coll(self):
        d_prev = attr.asdict(self.prev_coll_accum)
        d_cur = attr.asdict(self.coll_accum)
        delta = {}
        for k in d_prev:
            delta[k] = d_cur[k] - d_prev[k]
        return CollDetails(**delta)

    def _get_coll_forces(self):
        snapped_obj = self._sim.snapped_obj_id
        robot_id = self._sim.robot.sim_obj.object_id
        contact_points = self._sim.get_physics_contact_points()

        def get_max_force(contact_points, check_id):
            match_contacts = [
                x
                for x in contact_points
                if check_id in [x.object_id_a, x.object_id_b]
            ]
            match_contacts = [
                x for x in match_contacts if x.object_id_a != x.object_id_b
            ]

            max_force = 0
            if len(match_contacts) > 0:
                max_force = max([abs(x.normal_force) for x in match_contacts])

            return max_force

        forces = [
            abs(x.normal_force)
            for x in contact_points
            if (
                x.object_id_a not in self._ignore_collisions
                and x.object_id_b not in self._ignore_collisions
            )
        ]
        max_force = max(forces) if len(forces) > 0 else 0

        max_obj_force = get_max_force(contact_points, snapped_obj)
        max_robot_force = get_max_force(contact_points, robot_id)
        return max_robot_force, max_obj_force, max_force

    def update_coll_count(self):
        colls = self._sim.get_collisions()
        _, coll_details = rearrange_collision(
            colls,
            self._sim.snapped_obj_id,
            self._config.COUNT_OBJ_COLLISIONS,
        )

        self.coll_accum.obj_scene_colls += coll_details.obj_scene_colls
        self.coll_accum.robo_obj_colls += coll_details.robo_obj_colls
        self.coll_accum.robo_scene_colls += coll_details.robo_scene_colls

    @property
    def cur_collisions(self):
        ret = (
            self.coll_accum.obj_scene_colls + self.coll_accum.robo_scene_colls
        )
        if self._config.COUNT_ROBO_OBJ_COLLS:
            ret += self.coll_accum.robo_obj_colls
        return ret

    def get_n_targets(self):
        return self.n_objs


text:
"""
Copyright 2017-2018 yhenon (https://github.com/yhenon/)
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from generators.common import Generator
import cv2
import numpy as np
from PIL import Image
from six import raise_from
import csv
import sys
import os.path as osp
from collections import OrderedDict
from itertools import  repeat
from utils.bbox_transform import forward_convert
from utils.bbox_transform import backward_convert
from utils.bbox_transform import get_best_begin_point

def _parse(value, function, fmt):
    """
    Parse a string into a value, and format a nice ValueError if it fails.

    Returns `function(value)`.
    Any `ValueError` raised is catched and a new `ValueError` is raised
    with message `fmt.format(e)`, where `e` is the caught `ValueError`.
    """
    try:
        return function(value)
    except ValueError as e:
        raise_from(ValueError(fmt.format(e)), None)

def _read_classes(csv_reader):
    """
    Parse the classes file given by csv_reader.
    """
    result = OrderedDict()
    for line, row in enumerate(csv_reader):
        line += 1

        try:
            class_name, class_id = row
        except ValueError:
            raise_from(ValueError('line {}: format should be \'class_name,class_id\''.format(line)), None)
        class_id = _parse(class_id, int, 'line {}: malformed class ID: {{}}'.format(line))

        if class_name in result:
            raise ValueError('line {}: duplicate class name: \'{}\''.format(line, class_name))
        result[class_name] = class_id
    return result

def _read_quadrangle_annotations(csv_reader, classes, detect_text=False):
    """
    Read annotations from the csv_reader.
    Args:
        csv_reader: csv reader of args.annotations_path
        classes: list[str] all the class names read from args.classes_path

    Returns:
        result: dict, dict is like {image_path: [{'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2,
                                     'x3': x3, 'y3': y3, 'x4': x4, 'y4': y4, 'class': class_name}]}

    """
    result = OrderedDict()
    for line, row in enumerate(csv_reader, 1):
        try:
            img_file, x1, y1, x2, y2, x3, y3, x4, y4, class_name = row[:10]
            if img_file not in result:
                result[img_file] = []

            # If a row contains only an image path, it's an image without annotations.
            if (x1, y1, x2, y2, x3, y3, x4, y4, class_name) == ('', '', '', '', '', '', '', '', ''):
                continue

            x1 = _parse(x1, float, 'line {}: malformed x1: {{}}'.format(line))
            y1 = _parse(y1, float, 'line {}: malformed y1: {{}}'.format(line))
            x2 = _parse(x2, float, 'line {}: malformed x2: {{}}'.format(line))
            y2 = _parse(y2, float, 'line {}: malformed y2: {{}}'.format(line))
            x3 = _parse(x3, float, 'line {}: malformed x3: {{}}'.format(line))
            y3 = _parse(y3, float, 'line {}: malformed y3: {{}}'.format(line))
            x4 = _parse(x4, float, 'line {}: malformed x4: {{}}'.format(line))
            y4 = _parse(y4, float, 'line {}: malformed y4: {{}}'.format(line))

            # check if the current class name is correctly present
            if detect_text:
                if class_name == '###':
                    continue
                else:
                    class_name = 'text'

            if class_name not in classes:
                raise ValueError(f'line {line}: unknown class name: \'{class_name}\' (classes: {classes})')

            result[img_file].append({'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2,
                                     'x3': x3, 'y3': y3, 'x4': x4, 'y4': y4, 'class': class_name})
        except ValueError:
            raise_from(ValueError(
                f'line {line}: format should be \'img_file,x1,y1,x2,y2,x3,y3,x4,y4,class_name\' or \'img_file,,,,,\''),
                None)

    return result

def _read_annotations(csv_reader, classes):
    """
    Read annotations from the csv_reader.
    Args:
        csv_reader: csv reader of args.annotations_path
        classes: list[str] all the class names read from args.classes_path

    Returns:
        result: dict, dict is like {image_path: [{'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2, 'class': class_name}]}

    """
    result = OrderedDict()
    for line, row in enumerate(csv_reader, 1):
        try:
            img_file, x1, y1, x2, y2, class_name = row[:10]
            if img_file not in result:
                result[img_file] = []

            # If a row contains only an image path, it's an image without annotations.
            if (x1, y1, x2, y2, class_name) == ('', '', '', '', ''):
                continue

            x1 = _parse(x1, int, 'line {}: malformed x1: {{}}'.format(line))
            y1 = _parse(y1, int, 'line {}: malformed y1: {{}}'.format(line))
            x2 = _parse(x2, int, 'line {}: malformed x2: {{}}'.format(line))
            y2 = _parse(y2, int, 'line {}: malformed y2: {{}}'.format(line))

            if class_name not in classes:
                raise ValueError(f'line {line}: unknown class name: \'{class_name}\' (classes: {classes})')

            result[img_file].append({'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2, 'class': class_name})
        except ValueError:
            raise_from(ValueError(
                f'line {line}: format should be \'img_file,x1,y1,x2,y2,class_name\' or \'img_file,,,,,\''),
                None)

    return result

def _open_for_csv(path):
    """
    Open a file with flags suitable for csv.reader.

    This is different for python2 it means with mode 'rb', for python3 this means 'r' with "universal newlines".
    """
    if sys.version_info[0] < 3:
        return open(path, 'rb')
    else:
        return open(path, 'r', newline='')

def count_elements(seq) -> dict:
            """Tally elements from `seq`."""
            hist = {}
            for i in seq:
                hist[i] = hist.get(i, 0) + 1
            return hist

class CSVGenerator(Generator):
    """
    Generate data for a custom CSV dataset.

    See https://github.com/fizyr/keras-retinanet#csv-datasets for more information.
    """

    def __init__(
            self,
            csv_data_file,
            csv_class_file,
            base_dir=None,
            detect_quadrangle=False,
            detect_text=False,
            **kwargs
    ):
        """
        Initialize a CSV data generator.

        Args
            csv_data_file: Path to the CSV annotations file.
            csv_class_file: Path to the CSV classes file.
            detect_text: if do text detection
            base_dir: Directory w.r.t. where the files are to be searched (defaults to the directory containing the csv_data_file).
        """
        self.image_names = []
        self.image_data = {}
        self.base_dir = base_dir
        self.detect_quadrangle = detect_quadrangle
        self.detect_text = detect_text

        # Take base_dir from annotations file if not explicitly specified.
        if self.base_dir is None:
            if osp.exists(csv_data_file):
                self.base_dir = ''
            else:
                self.base_dir = osp.dirname(csv_data_file)

        # parse the provided class file
        try:
            with _open_for_csv(csv_class_file) as file:
                # class_name --> class_id
                self.classes = _read_classes(csv.reader(file, delimiter=','))
        except ValueError as e:
            raise_from(ValueError('invalid CSV class file: {}: {}'.format(csv_class_file, e)), None)

        self.labels = {}
        # class_id --> class_name
        for key, value in self.classes.items():
            self.labels[value] = key

        # csv with img_path, x1, y1, x2, y2, x3, y3, x4, y4, class_name
        try:
            with _open_for_csv(csv_data_file) as file:
                # {'img_path1':[{'x1':xx,'y1':xx,'x2':xx,'y2':xx,'x3':xx,'y3':xx,'x4':xx,'y4':xx, 'class':xx}...],...}
                if self.detect_quadrangle:
                    self.image_data = _read_quadrangle_annotations(csv.reader(file, delimiter=','), self.classes,
                                                                   self.detect_text)
                else:
                    self.image_data = _read_annotations(csv.reader(file, delimiter=','), self.classes)
        except ValueError as e:
            raise_from(ValueError('invalid CSV annotations file: {}: {}'.format(csv_data_file, e)), None)

        cls=list(self.classes.keys())
        o=dict()
        y=dict()
        s=[]
        for cl in cls:
          b=0
          p=list()
          for k,v in self.image_data.items():
            for vv in v:
              if vv['class']==cl:
                b+=1
                p.append(k)
                o[cl]=b
            y[cl]=p
        for k,v in y.items():
          y[k]=set(v)
          if (o[k])<5000:
            new_list=[]
            new_list.extend(
                repeat(list(y[k]),np.ceil((5000/o[k])).astype(np.int))
                )
            y[k]=np.array(new_list).reshape(-1)
          s.append(count_elements(y[k]))
        a={}
        for ss in s:
          for sss in ss.keys():
            if sss in a.keys():
              a[sss]=max(a[sss],ss[sss])
            else:
              a[sss]=ss[sss]
        image_list=[]
        for k,v in a.items():
          image_list.extend(repeat(k,v))
        self.image_names = image_list

#         self.image_names = list(self.image_data.keys())

        super(CSVGenerator, self).__init__(detect_text=detect_text, detect_quadrangle=detect_quadrangle, **kwargs)

    def size(self):
        """
        Size of the dataset.
        """
        return len(self.image_names)

    def num_classes(self):
        """
        Number of classes in the dataset.
        """
        return max(self.classes.values()) + 1

    def has_label(self, label):
        """
        Return True if label is a known label.
        """
        return label in self.labels

    def has_name(self, name):
        """
        Returns True if name is a known class.
        """
        return name in self.classes

    def name_to_label(self, name):
        """
        Map name to label.
        """
        return self.classes[name]

    def label_to_name(self, label):
        """
        Map label to name.
        """
        return self.labels[label]

    def image_path(self, image_index):
        """
        Returns the image path for image_index.
        """
        return osp.join(self.base_dir, self.image_names[image_index])

    def image_aspect_ratio(self, image_index):
        """
        Compute the aspect ratio for an image with image_index.
        """
        # PIL is fast for metadata
        image = Image.open(self.image_path(image_index))
        return float(image.width) / float(image.height)

    def load_image(self, image_index):
        """
        Load an image at the image_index.
        """
        image = cv2.imread(self.image_path(image_index))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        return image

    def load_annotations(self, image_index):
        """
        Load annotations for an image_index.
        """
        path = self.image_names[image_index]
        annotations = {'labels': np.empty((0,), dtype=np.int32),
                       'bboxes': np.empty((0, 4), dtype=np.float32),
                       'quadrangles': np.empty((0, 4, 2), dtype=np.float32),
                       }

        for idx, annot in enumerate(self.image_data[path]):
            annotations['labels'] = np.concatenate((annotations['labels'], [self.name_to_label(annot['class'])]))
            if self.detect_quadrangle:
                quadrangle = np.array([[float(annot['x1']), float(annot['y1'])],
                                       [float(annot['x2']), float(annot['y2'])],
                                       [float(annot['x3']), float(annot['y3'])],
                                       [float(annot['x4']), float(annot['y4'])]])

                # print('quadrangle\n',quadrangle)
                ordered_quadrangle =  get_best_begin_point(
                                      forward_convert(
                                      backward_convert(
                                      quadrangle.reshape(-1,8)))).reshape(4,2)

#                 ordered_quadrangle = self.xywhtheta_to_coords(self.coords_to_xywhtheta(quadrangle.reshape(1,8))).reshape(4,2)
#                 ordered_quadrangle = reorder_vertexes(ordered_quadrangle)
#                 print('ordered_quadrangle\n',ordered_quadrangle)
                annotations['quadrangles'] = np.concatenate((annotations['quadrangles'], ordered_quadrangle[None]))
                annotations['bboxes'] = np.concatenate((annotations['bboxes'], [[
                    float(min(annot['x1'], annot['x2'], annot['x3'], annot['x4'])),
                    float(min(annot['y1'], annot['y2'], annot['y3'], annot['y4'])),
                    float(max(annot['x1'], annot['x2'], annot['x3'], annot['x4'])),
                    float(max(annot['y1'], annot['y2'], annot['y3'], annot['y4'])),
                ]]))
            else:
                annotations['bboxes'] = np.concatenate((annotations['bboxes'], [[
                    float(annot['x1']),
                    float(annot['y1']),
                    float(annot['x2']),
                    float(annot['y2']),
                ]]))
        return annotations

max_stars_repo_name: nimaone/EfficientDet

max_issues_repo_name: nimaone/EfficientDet

max_forks_repo_name: nimaone/EfficientDet


text:
import os
import sys
import time

import numpy as np

all_slice = slice(None, None, None)

def split_tensor(col_list, tensor):
	"""
	Split an AbstractTensor vertically w.r.t a list of tensors or np.ndarrays
	"""
	new_arr = [0]*len(col_list)
	prev_idx = 0
	for i,x in enumerate(col_list):
		x_dim1 = x.shape[-1] if isinstance(x, np.ndarray) else x.shape()[-1]
		subset_slice = I_[:,prev_idx:prev_idx+x_dim1]
		new_arr[i] = tensor[subset_slice] if isinstance(x, np.ndarray) else tensor.index(subset_slice)
		prev_idx += x_dim1
	return new_arr

def rev_slice(i):
	"""
	"""
	return isinstance(i, slice) and i.step is not None and i.step < 0

def slice_to_range(i, hi, fn=range, invert=False):
	"""
	"""
	strt, stop, step = i.start, i.stop, i.step
	step = 1 if step is None else step
	strt = hi+strt if strt is not None and strt < 0 else strt
	stop = hi+stop if stop is not None and stop < 0 else stop
	if rev_slice(i):
		mn, mx = hi-1, -1
		strt = mn if strt is None else strt
		stop = mx if stop is None else stop
	else:
		mn, mx = 0, hi
		strt = mn if strt is None else strt
		stop = mx if stop is None else stop
	if invert:
		return fn(mn,strt,step), fn(stop,mx,step)
	return fn(strt,stop,step)

def exception_message(x, *args):
	"""
	"""
	x = x.replace('\n', ' ').replace('\t', '')
	return x.format(*args) if args else x

def is_class(T):
	"""
	"""
	try:
		return issubclass(T, T)
	except TypeError:
		return False

def is_subclass(T1, T2):
	"""
	"""
	try:
		return issubclass(T1, T2)
	except TypeError:
		return False

def list_type(l):
	"""
	"""
	return list_type(l[0]) if l and isinstance(l, list) else type(l)

def list_dims(l):
	"""
	"""
	return [len(l)] + list_dims(l[0]) if l and isinstance(l, list) else []

def list_flatten(l):
	"""
	"""
	out = []
	for o in l:
		if isinstance(o, list):
			for x in o:
				out.append(x)
		else:
			out.append(o)
	return out

class _PrettyIndex(object):
	"""
	"""
	def __getitem__(self, i):
		return i

I_ = _PrettyIndex()

max_stars_repo_name: jyhanna/tensorplane

max_issues_repo_name: jyhanna/tensorplane

max_forks_repo_name: jyhanna/tensorplane


text:
'''
Generate a catalog of syntetic images of mulitple
monolpoles. Based on Roger Fu's Matlab script
'''

import pickle
import sys
import numpy as np
import matplotlib
from matplotlib import rc
from matplotlib import cm
import matplotlib.pyplot as plt

# Constants
MU0 = 4 * np.pi * 1e-7
LIFTOFF = 5e-6 # Roger sez this should be: 5e-6
LEVELS = np.linspace(-1e-6, 1e-6, 30)
PIXELS = 300
FONT_SIZE = 14

def calc_dipole_parameters(declination, inclination, moment_scalar):
    '''Magnetization direction and dipole strength'''
    theta = np.deg2rad(-inclination + 90)
    phi = np.deg2rad(-declination + 90)

    moment_vector = np.zeros(3)
    moment_vector[0] = moment_scalar * np.sin(theta) * np.cos(phi)
    moment_vector[1] = moment_scalar * np.sin(theta) * np.sin(phi)
    moment_vector[2] = moment_scalar * np.cos(theta)
    return moment_vector

def calc_observation_grid(x_bound, y_bound):
    '''Generate observation coordinates'''
    n_x_points = PIXELS
    n_y_points = PIXELS
    x_grid, y_grid = np.meshgrid(np.linspace(-2 * x_bound, 2 * x_bound, n_x_points),
                                 np.linspace(-2 * y_bound, 2 * y_bound, n_y_points))
    return x_grid, y_grid

def calc_point_source_field(moment_vector, x_source, y_source, z_raw, x_grid, y_grid):
    '''Compute the field of a magnetic dipole point source'''
    # z_observed = z_raw * LIFTOFF
    z_observed = 30e-6 * np.random.uniform(1) + LIFTOFF
    squared_distance = (x_grid-x_source)**2.0 + \
                       (y_grid-y_source)**2.0 + \
                       z_observed**2.0

    aux = (moment_vector[0] * (x_grid - x_source) +
           moment_vector[1] * (y_grid - y_source) +
           moment_vector[2] * z_observed) / \
           squared_distance**(5.0 / 2.0)

    bz_dip = MU0 / (4.0 * np.pi) * \
             (3.0 * aux * z_observed - moment_vector[2] / squared_distance**(3.0 / 2.0))

    return bz_dip

def sample_sphere_angles(n_points):
    '''From: https://stackoverflow.com/questions/33976911/generate-a-random-sample-of-points-distributed-on-the-surface-of-a-unit-sphere'''
    declination = np.rad2deg(2*np.pi*np.random.rand(n_points))
    inclination = np.rad2deg(np.arccos(2*np.random.rand(n_points)-1)) - 90
    return declination, inclination

def gen_point_source_parameters(n_points, x_bound, y_bound):
    '''Generate diction of point source parameters'''
    point_source = dict()
    point_source['declination'], \
    point_source['inclination'] = sample_sphere_angles(n_points)
    point_source['x_source'] = np.random.uniform(-x_bound,
                                                 x_bound,
                                                 n_points)
    point_source['y_source'] = np.random.uniform(-y_bound,
                                                 y_bound,
                                                 n_points)
    point_source['moment_scalar'] = np.random.uniform(1e-14,
                                                      1e-12,
                                                      n_points)

    point_source['moment_vector'] = np.zeros((n_points, 3))
    for i in range(n_points):
        point_source['moment_vector'][i, :] = calc_dipole_parameters(point_source['declination'][i],
                                                                     point_source['inclination'][i],
                                                                     point_source['moment_scalar'][i])

    # Rescale moment_vector by a random number
    point_source['moment_vector'][i, :] = 1e4 * np.random.rand(1) * point_source['moment_vector'][i, :]

    return point_source

def field_from_point_source_dict(point_source, x_grid, y_grid, z_raw):
    '''Calculate and sum the point sources from the point_source dict'''
    bz_dip = np.zeros(x_grid.shape)
    for i in range(point_source['moment_scalar'].size):
        bz_dip += calc_point_source_field(point_source['moment_vector'][i, :],
                                          point_source['x_source'][i],
                                          point_source['y_source'][i],
                                          z_raw,
                                          x_grid,
                                          y_grid)

    point_source['bz_dip'] = bz_dip
    return point_source

def calc_feature_labels(field, n_bins):
    _, bins, _ = plt.hist(field, n_bins, facecolor='green')
    return np.digitize(field, bins)

def plot_field(grid):
    '''Simple visualization'''
    min_val = np.min(grid)
    max_val = np.max(grid)
    cbar_val = np.max(np.array([np.abs(min_val), max_val]))
    scale_factor = np.round(np.log10(cbar_val))
    cbar_val = cbar_val / (10**scale_factor)

    plt.imshow(grid / 10**(scale_factor),
               interpolation='nearest',
               cmap=cm.coolwarm,
               origin='lower')
    plt.clim(-cbar_val, cbar_val)
    plt.plot(np.array([-50, 50, 50, -50, -50])*PIXELS/200 + PIXELS/2,
             np.array([-50, -50, 50, 50, -50])*PIXELS/200 + PIXELS/2,
             '--k',
             linewidth=1.0)
    plt.xticks([0, PIXELS-1], ['-100', '100'])
    plt.yticks([0, PIXELS-1], ['-100', '100'])
    plt.xlabel('$x \; \mathrm{(microns)}$', fontsize=FONT_SIZE)
    plt.ylabel('$y \; \mathrm{(microns)}$', fontsize=FONT_SIZE)

    matplotlib.rc('xtick', labelsize=FONT_SIZE)
    matplotlib.rc('ytick', labelsize=FONT_SIZE)

    cbar = plt.colorbar(ticks=[-cbar_val, 0, cbar_val])
    exponent_string = '$10^{' + str(scale_factor) + '}$'
    cbar.ax.set_ylabel(r'field strength (units ' + r'$\times$' + ' ' + exponent_string + ')', fontsize=14, rotation=90)
    cbar.ax.tick_params(labelsize=14)
    plt.show(block=False)

def main(file_name):
    '''Generate random fields'''
    n_fields = 3
    n_points = 2400 # Roger says 2400
    x_bound = 100.0e-6 # microns, Roger says make this 50-100 microns
    y_bound = x_bound
    z_raw = 0 # microns, -z-coordinate: make a volume 0-30
    n_bins = 100 # Roger wants 1 degree...I say 10 degrees
    x_grid, y_grid = calc_observation_grid(x_bound, y_bound)

    frames_bzdip = np.zeros((n_fields, PIXELS, PIXELS)) # make 300x300
    frames_moment_scalar = np.zeros(n_fields)
    frames_moment_vector = np.zeros((n_fields, 3))

    for i in range(n_fields):
        sys.stdout.write('\r [ %f'%((i+1)/n_fields * 100)+'% ] ')
        sys.stdout.flush()
        point_source = gen_point_source_parameters(n_points, x_bound, y_bound)
        point_source = field_from_point_source_dict(point_source, x_grid, y_grid, z_raw)
        frames_bzdip[i, :, :] = point_source['bz_dip']
        frames_moment_vector[i, :] = np.sum(point_source['moment_vector'], 0)
        frames_moment_scalar[i] = np.linalg.norm(frames_moment_vector[i, :])

    print(' ')
    # plt.close('all')
    # plt.figure()
    # plt.hist(frames_moment_scalar, np.logspace(-15, -10, 200))
    # plt.ylabel('N')
    # plt.xlabel('moment vector magnitude')
    # plt.show(block=False)
    # import pdb; pdb.set_trace()

    np.savez(file_name + '.npz',
             frames_bzdip,
             frames_moment_scalar,
             frames_moment_vector)

if __name__ == '__main__':
    main(sys.argv[1])


text:
import tensorflow_addons as tfa
import tensorflow as tf
import tensorflow_datasets as tfds
import numpy as np
import time
import os
import shutil
import glob
import pandas as pd
from sklearn.model_selection import train_test_split
import cv2
from tensorflow.keras.utils import Sequence,to_categorical
from utils import safedirs
from tensorflow.keras.layers import Conv3D, MaxPool3D, Dense, Dropout, ReLU, BatchNormalization
from tensorflow.keras import Model,Sequential

class Residual(Model):
  expansion = 4
  def __init__(self, inDim, outDim, downsample=None, stride=1,head_conv=1):
    super(Residual,self).__init__()

    if head_conv == 1:
      self.conv1 = Conv3D(outDim, kernel_size=1,use_bias=False)
      self.bn1 = BatchNormalization()
    elif head_conv == 3:
      self.conv1 = Conv3D(outDim, kernel_size=(3,1,1), padding='same', use_bias=False)
      self.bn1 = BatchNormalization()
    else:
      raise ValueError("Invalid head convolution value")

    self.conv2 = Conv3D(outDim,kernel_size=(1,3,3), strides=(1,stride,stride), padding='same', use_bias=False)
    self.bn2 = BatchNormalization()

    self.conv3 = Conv3D(outDim*4, kernel_size=1, use_bias=False)
    self.bn3 = BatchNormalization()

    self.relu = ReLU()
    self.downsample = downsample

  def call(self, inputs):
    residual = inputs

    x = self.conv1(inputs)
    x = self.bn1(x)
    x = self.relu(x)

    x = self.conv2(x)
    x = self.bn2(x)
    x = self.relu(x)

    x = self.conv3(x)
    x = self.bn3(x)

    if self.downsample:
      residual = self.downsample(inputs)
    x += residual

    x = self.relu(x)
    return x

class SlowFast(tf.keras.Model):
  def __init__(self, block=Residual, layers=[3,4,6,3], class_num=10, dropout=0.5):
    super(SlowFast,self).__init__()

    self.fast_inplanes = 8
    self.fast_conv1 = Conv3D(8, kernel_size=(5,7,7), strides=(1,2,2), padding='same', use_bias=False)
    self.fast_bn1 = BatchNormalization()
    self.fast_relu = ReLU()
    self.fast_mpool = MaxPool3D(pool_size=(1,3,3), strides=(1,2,2), padding='same')

    self.fast_res2 = self._fastblock_(block, 8, layers[0], stride=1, head_conv=3)
    self.fast_res3 = self._fastblock_(block, 16, layers[1], stride=2, head_conv=3)
    self.fast_res4 = self._fastblock_(block, 32, layers[2], stride=2, head_conv=3)
    self.fast_res5 = self._fastblock_(block, 64, layers[3], stride=2, head_conv=3)

    self.lateral_p1 = Conv3D(8*2, kernel_size=(5,1,1), strides=(8,1,1), padding='same', use_bias=False)
    self.lateral_res2 = Conv3D(32*2, kernel_size=(5,1,1), strides=(8,1,1), padding='same', use_bias=False)
    self.lateral_res3 = Conv3D(64*2, kernel_size=(5,1,1), strides=(8,1,1), padding='same', use_bias=False)
    self.lateral_res4 = Conv3D(128*2, kernel_size=(5,1,1), strides=(8,1,1), padding='same', use_bias=False)

    self.slow_inplanes = 64+64//8*2
    self.slow_conv1 = Conv3D(64, kernel_size=(1,7,7), strides=(1,2,2), padding='same', use_bias=False)
    self.slow_bn1 = BatchNormalization()
    self.slow_relu = ReLU()
    self.slow_mpool = MaxPool3D(pool_size=(1,3,3), strides=(1,2,2), padding='same')

    self.slow_res2 = self._slowblock_(block, 64, layers[0], stride=1, head_conv=1)
    self.slow_res3 = self._slowblock_(block, 128, layers[1], stride=2, head_conv=1)
    self.slow_res4 = self._slowblock_(block, 256, layers[2], stride=2, head_conv=3)
    self.slow_res5 = self._slowblock_(block, 512, layers[3], stride=2, head_conv=3)

    self.dp = Dropout(dropout)
    self.fc1 = Dense(class_num, use_bias=False)

  def call(self,x):
    fast,lateral = self.fastpath(x[:,::2,:,:,:])
    slow = self.slowpath(x[:,::16,:,:,:], lateral)
    x = tf.concat([fast,slow],axis=1)
    x = self.dp(x)
    x = self.fc1(x)
    return x

  def fastpath(self,x):
    lateral = []

    x = self.fast_conv1(x)
    x = self.fast_bn1(x)
    x = self.fast_relu(x)
    pool1 = self.fast_mpool(x)

    lateral_pool1 = self.lateral_p1(pool1)
    lateral.append(lateral_pool1)

    res2 = self.fast_res2(pool1)
    lateral_fr2 = self.lateral_res2(res2)
    lateral.append(lateral_fr2)

    res3 = self.fast_res3(res2)
    lateral_fr3 = self.lateral_res3(res3)
    lateral.append(lateral_fr3)

    res4 = self.fast_res4(res3)
    lateral_fr4 = self.lateral_res4(res4)
    lateral.append(lateral_fr4)

    res5 = self.fast_res5(res4)
    x = tfa.layers.AdaptiveAveragePooling3D(1)(res5)
    x = tf.reshape(x,[-1,x.shape[4]])

    return x, lateral

  def slowpath(self, x, lateral):
    x = self.slow_conv1(x)
    x = self.slow_bn1(x)
    x = self.slow_relu(x)

    x = self.slow_mpool(x)
    x = tf.concat([x,lateral[0]],axis=4)

    x = self.slow_res2(x)
    x = tf.concat([x,lateral[1]],axis=4)

    x = self.slow_res3(x)
    x = tf.concat([x,lateral[2]],axis=4)

    x = self.slow_res4(x)
    x = tf.concat([x,lateral[3]],axis=4)

    x = self.slow_res5(x)
    x = tfa.layers.AdaptiveAveragePooling3D(1)(x)
    x = tf.reshape(x,[-1,x.shape[4]])
    return x

  def _fastblock_(self, block, outDim, blocks, stride=1, head_conv=1):
    douwnsample = None
    if stride != 1 or self.fast_inplanes != outDim * block.expansion:
      downsample = Sequential([
                    Conv3D(
                        outDim * block.expansion,
                        kernel_size = 1,
                        strides = (1,stride,stride),
                        padding='same',
                        use_bias = False),
                    BatchNormalization()])

    layers = []
    layers.append(block(self.fast_inplanes, outDim, downsample, stride, head_conv=head_conv))
    self.fast_inplanes = outDim * block.expansion
    for i in range(1,blocks):
      layers.append(block(self.fast_inplanes, outDim, head_conv=head_conv))
    return Sequential([*layers])

  def _slowblock_(self,block, outDim, blocks, stride=1, head_conv=1):
    downsample=None
    if stride != 1 or self.slow_inplanes != outDim * block.expansion:
      downsample = Sequential([
                    Conv3D(
                        outDim*block.expansion,
                        kernel_size = 1,
                        strides = (1,stride,stride),
                        padding='same',
                        use_bias=False),
                    BatchNormalization()])
    layers = []
    layers.append(block(self.slow_inplanes, outDim, downsample, stride, head_conv=head_conv))
    self.slow_inplanes = outDim * block.expansion
    for i in range(1,blocks):
      layers.append(block(self.slow_inplanes, outDim, head_conv=head_conv))

    self.slow_inplanes = outDim * block.expansion + outDim * block.expansion // 8 * 2
    return Sequential([*layers])

max_stars_repo_name: bmanikan/SlowFast

max_issues_repo_name: bmanikan/SlowFast

max_forks_repo_name: bmanikan/SlowFast


text:
import tensorflow as tf
from tensorflow.python.util import nest
from tensorflow.python.ops import array_ops
from tensorflow.python.framework import ops
import numpy as np
import json
from glob import glob

from . import base_image_fc, base_image_cnn

VALID_BASIC_CELL = dict(
    cnn=base_image_cnn,
    fc=base_image_fc
)

def dynamic_batch_size(inputs):
    """ Dynamic batch size, which is able to use in a model without deterministic batch size.
    See https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/python/ops/rnn.py
    """
    while nest.is_sequence(inputs):
        inputs = inputs[0]
    return array_ops.shape(inputs)[0]

class StepScheduler:
    """ step size scheduler """

    def __init__(self,
                 current_epoch: int,
                 initial_step: float=None,
                 multiplier: float=1.0,
                 power: float=1.0,
                 exponential: bool=False,
                 identity: bool=False
                 ):

        if not exponential and initial_step is None:
            raise ValueError('initial step is needed')
        self.__initial_st = initial_step
        self.__current_ep = current_epoch
        self.__multiplier = multiplier
        self.__power = power
        self.__exponential = exponential
        self.__identity = identity

    def __call__(self):
        self.__current_ep += 1

        if self.__identity:
            return self.__initial_st
        elif self.__exponential:
            new_step = 2 / (1 + np.exp(-self.__multiplier * self.__current_ep)) - 1
            return new_step
        else:
            new_step = self.__initial_st / (1 + self.__multiplier * self.__current_ep) ** self.__power
            return new_step

    @property
    def initial_step(self):
        return self.__initial_st

class FlipGradientBuilder(object):
    def __init__(self):
        self.num_calls = 0

    def __call__(self, x, scale=1.0):
        grad_name = "FlipGradient%d" % self.num_calls

        @ops.RegisterGradient(grad_name)
        def _flip_gradients(op, grad):
            return [tf.negative(grad) * scale]

        g = tf.get_default_graph()
        with g.gradient_override_map({"Identity": grad_name}):
            y = tf.identity(x)

        self.num_calls += 1
        return y

def variable_summaries(var, name):
    """Attach a lot of summaries to a Tensor (for TensorBoard visualization).
    https://www.tensorflow.org/guide/summaries_and_tensorboard """
    with tf.name_scope('var_%s' % name):
        mean = tf.reduce_mean(var)
        with tf.name_scope('stddev'):
            stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))

        return [tf.summary.scalar('mean', mean),
                tf.summary.scalar('stddev', stddev),
                tf.summary.scalar('max', tf.reduce_max(var)),
                tf.summary.scalar('min', tf.reduce_min(var)),
                tf.summary.histogram('histogram', var)]

def full_connected(x,
                   weight_shape,
                   scope=None,
                   bias=True,
                   reuse=None):
    """ fully connected layer
    - weight_shape: input size, output size
    - priority: batch norm (remove bias) > dropout and bias term
    """
    with tf.variable_scope(scope or "fully_connected", reuse=reuse):
        w = tf.get_variable("weight", shape=weight_shape, dtype=tf.float32)
        x = tf.matmul(x, w)
        if bias:
            b = tf.get_variable("bias", initializer=[0.0] * weight_shape[-1])
            return tf.add(x, b)
        else:
            return x

def convolution(x,
                weight_shape,
                stride,
                padding="SAME",
                scope=None,
                bias=True,
                reuse=None):
    """2d convolution
     Parameter
    -------------------
    weight_shape: width, height, input channel, output channel
    stride (list): [stride for axis 1, stride for axis 2]
    """
    with tf.variable_scope(scope or "2d_convolution", reuse=reuse):
        w = tf.get_variable('weight', shape=weight_shape, dtype=tf.float32)
        x = tf.nn.conv2d(x, w, strides=[1, stride[0], stride[1], 1], padding=padding)
        if bias:
            b = tf.get_variable("bias", initializer=[0.0] * weight_shape[-1])
            return tf.add(x, b)
        else:
            return x

def dynamic_batch_size(inputs):
    """ Dynamic batch size, which is able to use in a model without deterministic batch size.
    See https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/python/ops/rnn.py
    """
    while nest.is_sequence(inputs):
        inputs = inputs[0]
    return array_ops.shape(inputs)[0]

def checkpoint_version(checkpoint_dir: str,
                       config: dict = None,
                       version: int = None):
    """ Checkpoint versioner: Either of `config` or `version` need to be specified (`config` has priority)

     Parameter
    ---------------------
    checkpoint_dir: directory where specific model's checkpoints are (will be) saved, eg) `checkpoint/cnn`
    config: parameter configuration to find same setting checkpoint
    version: number of checkpoint to warmstart from

     Return
    --------------------
    path_to_checkpoint, config

    - if there are no checkpoints, having same config as provided one, return new version
        eg) in case there are 'checkpoint/cnn/{v0,v1,v2}`, path_to_checkpoint = 'checkpoint/cnn/v3'
    - if there is a checkpoint, which has same config as provided one, return that version
        eg) in case there are 'checkpoint/cnn/{v0,v1,v2}`, and `v2` has same config, path_to_checkpoint = 'checkpoint/cnn/v2'
    - if `config` is None, `version` is required.
        eg) in case there are 'checkpoint/cnn/{v0,v1,v2}`, path_to_checkpoint = 'checkpoint/cnn/v0' if `version`=0
    """

    if version is not None:
        checkpoints = glob('%s/v%i/hyperparameters.json' % (checkpoint_dir, version))
        if len(checkpoints) == 0:
            raise ValueError('No checkpoint: %s, %s' % (checkpoint_dir, version))
        elif len(checkpoints) > 1:
            raise ValueError('Multiple checkpoint found: %s, %s' % (checkpoint_dir, version))
        else:
            parameter = json.load(open(checkpoints[0]))
            target_checkpoints_dir = checkpoints[0].replace('/hyperparameters.json', '')
            return target_checkpoints_dir, parameter

    elif config is not None:
        # check if there are any checkpoints with same hyperparameters
        target_checkpoints = []
        for parameter_path in glob('%s/*/hyperparameters.json' % checkpoint_dir):
            # if not os.path.isdir(i):  # ignore files
            #     continue
            i = parameter_path.replace('/hyperparameters.json', '')
            json_dict = json.load(open(parameter_path))
            if config == json_dict:
                target_checkpoints.append(i)
        if len(target_checkpoints) == 1:
            return target_checkpoints[0], config
        elif len(target_checkpoints) == 0:
            new_checkpoint_id = len(glob('%s/*/hyperparameters.json' % checkpoint_dir))
            new_checkpoint_path = '%s/v%i' % (checkpoint_dir, new_checkpoint_id)
            os.makedirs(new_checkpoint_path, exist_ok=True)
            with open('%s/hyperparameters.json' % new_checkpoint_path, 'w') as outfile:
                json.dump(config, outfile)
            return new_checkpoint_path, config
        else:
            raise ValueError('Checkpoints are duplicated')


text:
import os
import sys

import Tkinter
import json
import numpy as np
from pprint import pprint
import itertools #for remove duplicate data
import matplotlib.pyplot as plt #pie

def main():
	if len(sys.argv)!=4:
		print 'inputan salah'
		print 'python fuzzy2.py  [Target Direktori][compound Direktori] [protein Direktori]'
		print 'Contoh inputan: python fuzzy1.py output dbscan#10-yamanishi#nr-compound-david-20170828135141  '
		return
	elif os.path.exists(sys.argv[1])==False:
		print "folder target tidak ditemukan"
		return

	TarDir=sys.argv[1]
	comDir=sys.argv[2]
	proDir=sys.argv[3]

	InDir=os.path.join(TarDir,proDir)

	InDir2=os.path.join(InDir,"analisis")
	os.makedirs(InDir2)

	jenis=['e','gpcr','ic','nr']

	for item in jenis:
		proPath='protein_'+item+'_index.json'
		proPath=os.path.join(InDir,proPath)
		status=os.path.exists(proPath)
		if status==True:
			jenis=item
			break

	#cluster protein
	with open(proPath) as data_file:
	    data = json.load(data_file)
	ProName= data.keys() #nama
	ProCluster= data.values() #cluster
	n_cluster=max(max(ProCluster))
	clusterPro={}
	for i in range(1,n_cluster+1):
		row=[]
		for j in range(0,len(ProCluster)):
			status= i in ProCluster[j]
			if status==True:
				ProName[j]=ProName[j].encode("utf-8")
				row.append(ProName[j])
		clusterPro[i]=row

	#cluster compound
	ComPath=os.path.join(TarDir,comDir)
	ComPath=os.path.join(ComPath,'compound_calinskiharabaz_bestlabels.json')
	if os.path.exists(ComPath)==False:
		print "label compound tidak di temukan di direktori"
		return
	with open(ComPath) as data_file:
	    data = json.load(data_file)
	ComName= data.keys() #nama
	comCluster= data.values() #cluster

	clusterCom={}
	for i in range (-1,len(set(comCluster))+1):
		row=[]
		for j in range (0,len(comCluster)):
			if comCluster[j]==i:
				ComName[j]=ComName[j].encode("utf-8")
				row.append(ComName[j])
		clusterCom[i]=row

	#SEDERHANAKAN BINDING ROW
	pathBin="../../dataset/connectivity/compound_vs_protein/yamanishi/ground-truth/"
	pathBin2="bind_orfhsa_drug_"+jenis+".txt"
	pathBin=os.path.join(pathBin,pathBin2)
	with open(pathBin,'r') as f:
		data=[line.strip() for line in f]
	data = map(lambda x: x.split("\t"), data)

	##ganti compound
	for i in range(-1,len(clusterCom)-1):
		for j in range(0,len(data)):
			status= data[j][1] in clusterCom[i]
			if status==True:
				data[j][1]=i
	data.sort()
	data=list(data for data,_ in itertools.groupby(data)) #remove duplicate data

	##ganti protein
	datanew=[]
	for j in range (0,len(data)):
		for x in range (1,len(clusterPro)+1):
			ttemp=data[j][0].replace(":","")
			status=ttemp in clusterPro[x]
			if status==True:
				temp=data[j]
				temp[0]=str(x)
				datanew.append(temp)
	datanew.sort()
	datanew=list(datanew for datanew,_ in itertools.groupby(datanew)) #remove duplicate data

	ConPath="konektivitas_protein_compound_"+jenis+".txt"
	konektivitas = open(os.path.join(InDir2,ConPath), 'a')

	judul="[cluster protein][cluster compound]"
	konektivitas.write("%s\n" % judul)
	for item in datanew:
  		konektivitas.write("%s\n" % item)

	#UPDATE MATRIX
	#read matrix
	PathMat="../../dataset/connectivity/compound_vs_protein/yamanishi/ground-truth/"
	PathMat2="admat_dgc_"+jenis+".txt"
	PathMat=os.path.join(PathMat,PathMat2)
	with open(PathMat,'r') as f:
		data=[line.strip() for line in f]
	dataMatrix = map(lambda x: x.split("\t"), data)

	for countP, protein in enumerate(ProName):
		for x in range (1,len(clusterPro)+1): #banyaknya cluster protein
				status= protein in clusterPro[x]
				if status==True:
					clusterP=x
					break
		for countC, compound in enumerate(ComName):
			for y in range(-1,len(clusterCom)-1): #banyaknya cluster compound
				status= compound in clusterCom[y]
				if status==True:
					clusterC=y
					break
			for data in datanew:
				if clusterP==int(data[0]) and clusterC==data[1] and int(dataMatrix[countP+1][countC+1])==0:
					dataMatrix[countP+1][countC+1]='-1' #semua 0 ke update jadi -1
					break

	#PIE CHART
	Data1=0
	Data0=0
	Datamin1=0
	fig=plt.figure()
	for i in range (1,len(dataMatrix)):
		for j in range (1, len(dataMatrix[1])):
			if dataMatrix[i][j]=='1':
				Data1+=1
			elif dataMatrix[i][j]=='0':
				Data0+=1
			else:
				Datamin1+=1
	plt.pie([Data1, Data0,Datamin1], 					#nilai
	        colors=["g","r", "b"],				#warna
	        labels=["1", "0 credible", "0 incredible"],	#label
	        autopct='%1.1f%%')
	plt.axis('equal') #ensure pie is round
	fig.savefig(os.path.join(InDir2,"pieChart.png"))
if __name__ == "__main__":
    main()

max_stars_repo_name: tttor/csipb-jamu-prj

max_issues_repo_name: tttor/csipb-jamu-prj

max_forks_repo_name: tttor/csipb-jamu-prj


text:
import numpy as np
from chainer import Chain
from chainer import links as L
from chainer import functions as F
from chainer.functions.connection import linear
from chainer.links.connection.linear import functools, operator

class LTLinear(L.Linear):
    """
    Lower Triangle Linear
    """
    def __init__(self, *args, **kwargs):
        super(LTLinear, self).__init__(*args, **kwargs)
        self.add_persistent('triu_indices', None)

    def _initialize_params(self, in_size):
        super(LTLinear, self)._initialize_params(in_size)
        self.triu_indices = np.array(np.triu_indices(in_size, m=self.out_size))
        self.W.data[self.triu_indices] = 0

    def __call__(self, x):
        if self.W.data is None:
            in_size = functools.reduce(operator.mul, x.shape[1:], 1)
            self._initialize_params(in_size)

        self.W.data[self.triu_indices] = 0
        return linear.linear(x, self.W, self.b)

class AutoregressiveLinear(Chain):

    def __init__(self, n_out):
        super(AutoregressiveLinear, self).__init__()
        with self.init_scope():
            self.l0 = LTLinear(None, n_out)
            self.l1 = LTLinear(None, n_out)

    def __call__(self, x):
        h = F.relu(self.l0(x))
        y = self.l1(h)
        return y


text:
# pylint: skip-file
import random
from typing import Optional, Dict
import numpy as np
from pyvisainstrument.testsuite.DummyTCPInstrument import DummyTCPInstrument

class DummyVNA(DummyTCPInstrument):

    def __init__(self, num_ports=4, *args, **kwargs):
        super(DummyVNA, self).__init__(*args, **kwargs)
        self.num_ports = num_ports
        self.cmd_tree = None
        self.state = {
            "*CLS": self.clear_status,
            "*RST": self.reset,
            "*IDN": "VNA",
            "*OPC": "1",
            "*ESR": "1",
            "SENSE": {
                "X": {
                    "VALUES": self._get_x_data
                },
                "AVERAGE": {
                    "STATE": 1,
                    "MODE": "SWEEP",
                    "COUNT": 1
                },
                "BWIDTH": "1000",
                "FREQUENCY": {
                    "START": "1E7",
                    "STOP": "2E10",
                    "CENTER": "5E8",
                    "CW": "5E8"
                },
                "SWEEP": {
                    "POINTS": "2000",
                    "STEP": "1E7",
                    "TYPE": "LINEAR",
                    "POWER": "10",
                    "MODE": "SINGLE",
                    "TIME": "0.1"
                },
                "CORRECTION": {
                    "CSET": {
                        "ACTIVATE": "",
                        "DEACTIVATE": "",
                        "CATALOG": "calset1,calset2",
                        "DESCRIPTION": "A calset description.",
                        "DATA": "1,2,3,4,5",
                        "ETERM": "Directivity(1,1)",
                        "NAME": "calset1",
                        "CREATE": ""
                    },
                    "STATE": 1,
                    "COLLECTION": {
                        "GUIDED": {
                            "STEPS": "3",
                            "DESCRIPTION": self.get_ecal_description,
                            "CONNECTOR": {
                                "PORT1": "",
                                "PORT2": "",
                                "PORT3": "",
                                "PORT4": ""
                            },
                            "CKIT": {
                                "PORT1": "",
                                "PORT2": "",
                                "PORT3": "",
                                "PORT4": ""
                            },
                            "ACQUIRE": None,
                            "SAVE": {"CSET": None},
                            "THRU": {"PORTS": [1, 2, 1, 3, 1, 4]},
                            "INITIATE": "+1"
                        }
                    },
                    "INTERPOLATE": "ON",
                    "PREFERENCE": {"ECAL": {"ORIENTATION": "ON"}},
                    "CKIT": {
                        "ECAL": {
                            "LIST": "+1"
                        },
                        "ECAL1": {
                            "INF": '"ModelNumber: N4691-60006, SerialNumber: 16194, ConnectorType: 35F 35F, PortAConnector: APC 3.5 female, PortBConnector: APC 3.5 female, MinFreq: 300000, MaxFreq: 26500000000, NumberOfPoints: 583, Calibrated: 19/Sep/2018, CharacterizedBy: 15N5239932, "\n'
                        }
                    }
                }
            },
            "CALCULATE": {
                "PARAMETER": {
                    "DELETE": {"ALL": None},
                    "DEFINE": "'sdd11',S11",
                    "SELECT": "'sdd11'",
                },
                "FSIMULATOR": {
                    "BALUN": {
                        "DEVICE": None,
                        "TOPOLOGY": {"BBALANCED": {"PPORTS": "1,2,3,4"}},
                        "PARAMETER": {"STATE": "ON", "BBALANCED": {"DEFINE": "SDD11"}}
                    }
                },
                "DATA": self._get_data,
                "FORMAT": "MLOG"
            },
            "DISPLAY": {
                "ENABLE": 1,
                "WIND1": {
                    "TRAC1": {"FEED": "sdd11"},
                    "TRAC2": {"FEED": "sdd12"},
                    "TRAC3": {"FEED": "sdd21"},
                    "TRAC4": {"FEED": "sdd22"},
                    "STATE": "OFF"
                },
                "WIND2": {
                    "TRAC1": {"FEED": "sdd11"},
                    "TRAC2": {"FEED": "sdd12"},
                    "TRAC3": {"FEED": "sdd21"},
                    "TRAC4": {"FEED": "sdd22"},
                    "STATE": "OFF"
                },
                "WIND3": {
                    "TRAC1": {"FEED": "sdd11"},
                    "TRAC2": {"FEED": "sdd12"},
                    "TRAC3": {"FEED": "sdd21"},
                    "TRAC4": {"FEED": "sdd22"},
                    "STATE": "OFF"
                },
                "WIND4": {
                    "TRAC1": {"FEED": "sdd11"},
                    "TRAC2": {"FEED": "sdd12"},
                    "TRAC3": {"FEED": "sdd21"},
                    "TRAC4": {"FEED": "sdd22"},
                    "STATE": "OFF"
                }
            },
            "TRIGGER": {
                "SOURCE": "IMMediate"
            },
            "MMEMORY": {
                "STORE": {
                    "TRACE": {
                        "FORMAT": {
                            "SNP": "RI"
                        }
                    }
                }
            },
            "FORMAT": {
                "DATA": "ASCii,+0"
            }
        }
        self.map_commands = dict(
            CALC1='CALCULATE', CALCULATE1='CALCULATE',
            SENS1='SENSE', SENSE1='SENSE',
            CALC='CALCULATE', CALCULATE='CALCULATE',
            CALP='CALPOD', CALPOD='CALPOD',
            CONT='CONTROL', CONTROL='CONTROL',
            DISP='DISPLAY', DISPLAY='DISPLAY',
            DEAC='DEACTIVATE', DEACTIVATE='DEACTIVATE',
            FORM='FORMAT', FORMAT='FORMAT',
            HCOP='HCOPY', HCOPY='HCOPY',
            INIT='INITIATE', INITIATE='INITIATE',
            MMEM='MMEMORY', MMEMORY='MMEMORY',
            OUTP='OUTPUT', OUTPUT='OUTPUT',
            ROUT='ROUTE', ROUTE='ROUTE',
            SENS='SENSE', SENSE='SENSE',
            STAT='STATUS', STATUS='STATUS',
            SYST='SYSTEM', SYSTEM='SYSTEM',
            TRIG='TRIGGER', TRIGGER='TRIGGER',
            CORR='CORRECTION', CORRECTION='CORRECTION',
            CUST='CUSTOM', CUSTOM='CUSTOM',
            EQU='EQUATION', EQUATION='EQUATION',
            FILT='FILTER', FILTER='FILTER',
            FSIM='FSIMULATOR', FSIMULATOR='FSIMULATOR',
            FUNC='FUNCTION', FUNCTION='FUNCTION',
            GDELA='GDELAY', GDELAY='GDELAY',
            LIM='LIMIT', LIMIT='LIMIT',
            MARK='MARKER', MARKER='MARKER',
            MIX='MIXER', MIXER='MIXER',
            NORM='NORMALIZE', NORMALIZE='NORMALIZE',
            OFFSet='OFFSET', OFFSET='OFFSET',
            PAR='PARAMETER', PARAMETER='PARAMETER',
            RDAT='RDATA', RDATA='RDATA',
            DATA='DATA', SDATA='SDATA',
            SMO='SMOOTHING', SMOOTHING='SMOOTHING',
            TRANS='TRANSFORM', TRANSFORM='TRANSFORM',
            UNC='UNCERTAINTY', UNCERTAINTY='UNCERTAINTY',
            COLL='COLLECTION', COLLECTION='COLLECTION',
            GUID='GUIDED', GUIDED='GUIDED',
            ABOR='ABORT', ABORT='ABORT',
            DEL='DELETE', DELETE='DELETE',
            DEF='DEFINE', DEFINE='DEFINE',
            CAT='CATALOG', CATALOG='CATALOG',
            MOD='MODIFY', MODIFY='MODIFY',
            SEL='SELECT', SELECT='SELECT',
            EXT='EXTENDED', EXTENDED='EXTENDED',
            DESC='DESCRIPTION', DESCRIPTION='DESCRIPTION',
            CONN='CONNECTOR', CONNECTOR='CONNECTOR',
            ACQ='ACQUIRE', ACQUIRE='ACQUIRE',
            CHAN='CHANNEL', CHANNEL='CHANNEL',
            PORT='PORTS', PORTS='PORTS',
            ACT='ACTIVATE', ACTIVATE='ACTIVATE',
            PREF='PREFERENCE', PREFERENCE='PREFERENCE',
            ORI='ORIENTATION', ORIENTATION='ORIENTATION',
            BAL='BALUN', BALUN='BALUN',
            DEV='DEVICE', DEVICE='DEVICE',
            TOP='TOPOLOGY', TOPOLOGY='TOPOLOGY',
            BBAL='BBALANCED', BBALANCED='BBALANCED',
            PPOR='PPORTS', PPORTS='PPORTS',
            POINT='POINTS', POINTS='POINTS',
            PORT1='PORT1', PORT2='PORT2',
            PORT3='PORT3', PORT4='PORT4',
            THRU='THRU', STEPS='STEPS',
            SOUR='SOURCE', SOURCE='SOURCE',
            CSET='CSET', INT='INTERPOLATE',
            COUN='COUNT', COUNT='COUNT',
            SNP='SNP',
            BWID='BWIDTH', BWIDTH='BWIDTH',
            BAND='BWIDTH', BANDWIDTH='BWIDTH',
            ETERM="ETERM", ETER="ETERM",
            WIND='WINDOW', WINDOW='WINDOW',
            TRAC='TRACE', TRACE='TRACE',
            STOR='STORE', STORE='STORE',
            AVER='AVERAGE', AVERAGE='AVERAGE',
            CRE='CREATE', CREATE='CREATE',
            ENAB='ENABLE', ENABLE='ENABLE',
        )

    def _get_x_data(self, params, is_query):
        if is_query:
            num_points = int(float(self.state["SENSE"]["SWEEP"]["POINTS"]))
            s_type = self.state["SENSE"]["SWEEP"]["TYPE"]
            f_start = float(self.state["SENSE"]["FREQUENCY"]["START"])
            f_stop = float(self.state["SENSE"]["FREQUENCY"]["STOP"])
            if s_type.upper().startswith('LIN'):
                data = np.linspace(f_start, f_stop, num_points)
            elif s_type.upper().startswith('LOG'):
                data = np.logspace(f_start, f_stop, num_points)
            else:
                data = np.linspace(f_start, f_stop, num_points)
            return ",".join('{:+.6E}'.format(v) for v in data)

    def _get_data(self, params, is_query):
        if is_query:
            num_points = int(float(self.state["SENSE"]["SWEEP"]["POINTS"]))
            # CALC:DATA:SNP:PORTs 1,2,3,4?
            if self.cmd_tree and 'SNP' in self.cmd_tree:
                nports = len(params[0].split(',')) if len(params) else self.num_ports
                #           freq + S11r,i + S12r,i, ... S22r,i ... SNNr,i
                num_points = num_points + num_points * (nports**2) * 2
            else:
                is_complex = len(params) and (params[0] in ["RDATA", "SDATA"])
                if is_complex:
                    num_points = 2 * num_points
            data = np.random.rand(num_points)
            data_str = ",".join('{:+.6E}'.format(v) for v in data)
            return data_str

    def clear_status(self, params, is_query):
        return

    def reset(self, params, is_query):
        return

    def get_ecal_description(self, params, is_query):
        ports = list(np.random.permutation(range(self.num_ports))[:2])
        return "Please connect e-cal module from port {} to port {}".format(ports[0] + 1, ports[-1] + 1)

    def process_command(self, cmd_tree, params, is_query):
        self.cmd_tree = cmd_tree
        rst = self.state
        prst: Optional[Dict] = None
        pcmd = None
        for cmd in cmd_tree:
            mapped_cmd = self.map_commands.get(cmd, cmd)
            if mapped_cmd in rst:
                prst = rst
                pcmd = mapped_cmd
                rst = rst[mapped_cmd]
                if callable(rst):
                    break
            else:
                break

        if is_query:
            if type(rst) in [str, int, float, bool]:
                return str(rst)
            elif type(rst) is None:
                return ''
            elif callable(rst):
                return rst(params, True)  # type: ignore
            else:
                return '-100'
        else:
            if callable(rst):
                rst(params, False)  # type: ignore
            elif isinstance(prst, dict) and (isinstance(rst, (str, int, float, bool, list)) or rst is None):
                if len(params):
                    castType = type(params[0]) if rst is None else type(prst[pcmd])
                    prst[pcmd] = castType(params[0])
                return None
            else:
                raise Exception('Unknown command')

max_stars_repo_name: Samtec-ASH/pyvisainstrument

max_issues_repo_name: Samtec-ASH/pyvisainstrument

max_forks_repo_name: Samtec-ASH/pyvisainstrument


text:
import os
import time
import torch
import numpy as np
import random

def make_file_path(file_path, is_pl = False):
    basepath = os.path.split(os.path.realpath(file_path))[0]
    fullpath = os.path.realpath(file_path)
    filename = os.path.split(os.path.realpath(file_path))[1]
    t = time.strftime("%b%d%H%M", time.localtime())
    os.system("mkdir -p _archives _outs")
    if not is_pl:
        os.system("mkdir -p _models/%s/best _models/%s/final" % (t, t))
    os.system("cp %s %s/_archives/%s.py" % (fullpath, basepath, t))
    modelpath = os.path.join(basepath, "_models", "%s.pt" % t)
    # plotpath = os.path.join(basepath, "_plots", "%s.png" % t)
    outpath = os.path.join(basepath, "_outs", "%s.out" % t)
    return modelpath, "", outpath, t, basepath

class Log():
    def __init__(self, outpath):
        self.outpath = outpath

    def __call__(self, *args):
        with open(self.outpath, 'a') as f:
            f.write(' '.join([str(arg) for arg in args]))
            f.write('\n')

def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.backends.cudnn.deterministic = True

max_stars_repo_name: CarnoZhao/utils

max_issues_repo_name: CarnoZhao/utils

max_forks_repo_name: CarnoZhao/utils


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 16 08:48:49 2017

@author: hao

K次交叉驗證，初始採樣分割成K個子樣本，一個單獨的子樣本被保留作為驗證模型的數據，其他K-1個樣本用來訓練。
交叉驗證重複K次，每個子樣本驗證一次，平均K次的結果或者使用其它結合方式，最終得到一個單一估測。
這個方法的優勢在於，同時重複運用隨機產生的子樣本進行訓練和驗證，每次的結果驗證一次，10次交叉驗證是最常用的。
"""
import numpy as np
from sklearn.model_selection import StratifiedKFold

# In[]
X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9,10], [11,12]])
y = np.array([0, 1, 1, 1, 1, 0])

# In[]
skf = StratifiedKFold(n_splits=2)
skf.get_n_splits(X, y)

print(skf)

# In[]
for train_index, test_index in skf.split(X, y):
   print("TRAIN:", train_index, "TEST:", test_index)
   X_train, y_tarin = X[train_index], y[train_index]
   X_test, y_test = X[test_index], y[test_index]

max_stars_repo_path: Ch 3/test_StratifiedKFold.py

max_stars_repo_name: IESSC/Machine-Learning

max_issues_repo_path: Ch 3/test_StratifiedKFold.py

max_issues_repo_name: IESSC/Machine-Learning

max_forks_repo_path: Ch 3/test_StratifiedKFold.py

max_forks_repo_name: IESSC/Machine-Learning


text:
import pandas as pd
import numpy as np

for item in range(0,120):
    if item % 3 == 1 and item % 5 == 3:
        print('Cracked')
    elif item % 3 == 1:
        print('Half Cracked')
    elif item % 5 == 0:
        print('High5')
    else :
        print(item)


text:
import matrices_new_extended as mne
import numpy as np
import sympy as sp
from equality_check import Point

x, y, z = sp.symbols("x y z")
Point.base_point = np.array([x, y, z, 1])

class Test_Axis_hex_3_00z:

    def test_matrix_hex_3_00z(self):
        expected = Point([ -y, x-y, z, 1])
        calculated = Point.calculate(mne._matrix_hex_3_00z)
        assert calculated == expected

    def test_matrix_hex_3_1_00z_00t(self):
        expected = Point([ -y, x-y, 2/3+z, 1])
        calculated = Point.calculate(mne._matrix_hex_3_1_00z_00t)
        assert calculated == expected

    def test_matrix_hex_3_2_00z_002t(self):
        expected = Point([ -y, x-y, 4/3+z, 1])
        calculated = Point.calculate(mne._matrix_hex_3_2_00z_002t)
        assert calculated == expected

reason: import numpy,import sympy


text:
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

__weights_dict = dict()

def load_weights(weight_file):
    if weight_file == None:
        return

    try:
        weights_dict = np.load(weight_file, allow_pickle=True).item()
    except:
        weights_dict = np.load(weight_file, encoding='bytes', allow_pickle=True).item()

    return weights_dict

class KitModel(nn.Module):

    def __init__(self, weight_file):
        super(KitModel, self).__init__()
        global __weights_dict
        __weights_dict = load_weights(weight_file)

        self.conv1 = self.__conv(2, name='conv1', in_channels=3, out_channels=96, kernel_size=(11, 11), stride=(4, 4),
                                 groups=1, bias=True)
        self.conv2 = self.__conv(2, name='conv2', in_channels=96, out_channels=256, kernel_size=(5, 5), stride=(1, 1),
                                 groups=2, bias=True)
        self.conv3 = self.__conv(2, name='conv3', in_channels=256, out_channels=384, kernel_size=(3, 3), stride=(1, 1),
                                 groups=1, bias=True)
        self.conv4 = self.__conv(2, name='conv4', in_channels=384, out_channels=384, kernel_size=(3, 3), stride=(1, 1),
                                 groups=2, bias=True)
        self.conv5 = self.__conv(2, name='conv5', in_channels=384, out_channels=256, kernel_size=(3, 3), stride=(1, 1),
                                 groups=2, bias=True)
        self.fc6_1 = self.__dense(name='fc6_1', in_features=9216, out_features=4096, bias=True)
        self.fc7_1 = self.__dense(name='fc7_1', in_features=4096, out_features=4096, bias=True)
        self.fc8_t_1 = self.__dense(name='fc8-t_1', in_features=4096, out_features=2089, bias=True)

    def forward(self, x):
        conv1_pad = F.pad(x, (0, 1, 0, 1))
        conv1 = self.conv1(conv1_pad)
        relu1 = F.relu(conv1)
        pool1_pad = F.pad(relu1, (0, 1, 0, 1), value=float('-inf'))
        pool1 = F.max_pool2d(pool1_pad, kernel_size=(3, 3), stride=(2, 2), padding=0, ceil_mode=False)
        norm1 = F.local_response_norm(pool1, size=5, alpha=9.999999747378752e-05, beta=0.75, k=1.0)
        conv2_pad = F.pad(norm1, (2, 2, 2, 2))
        conv2 = self.conv2(conv2_pad)
        relu2 = F.relu(conv2)
        pool2_pad = F.pad(relu2, (0, 1, 0, 1), value=float('-inf'))
        pool2 = F.max_pool2d(pool2_pad, kernel_size=(3, 3), stride=(2, 2), padding=0, ceil_mode=False)
        norm2 = F.local_response_norm(pool2, size=5, alpha=9.999999747378752e-05, beta=0.75, k=1.0)
        conv3_pad = F.pad(norm2, (1, 1, 1, 1))
        conv3 = self.conv3(conv3_pad)
        relu3 = F.relu(conv3)
        conv4_pad = F.pad(relu3, (1, 1, 1, 1))
        conv4 = self.conv4(conv4_pad)
        relu4 = F.relu(conv4)
        conv5_pad = F.pad(relu4, (1, 1, 1, 1))
        conv5 = self.conv5(conv5_pad)
        relu5 = F.relu(conv5)
        pool5_pad = F.pad(relu5, (0, 1, 0, 1), value=float('-inf'))
        pool5 = F.max_pool2d(pool5_pad, kernel_size=(3, 3), stride=(2, 2), padding=0, ceil_mode=False)
        fc6_0 = pool5.view(pool5.size(0), -1)
        fc6_1 = self.fc6_1(fc6_0)
        relu6 = F.relu(fc6_1)
        drop6 = F.dropout(input=relu6, p=0.5, training=self.training, inplace=True)
        fc7_0 = drop6.view(drop6.size(0), -1)
        fc7_1 = self.fc7_1(fc7_0)
        relu7 = F.relu(fc7_1)
        drop7 = F.dropout(input=relu7, p=0.5, training=self.training, inplace=True)
        fc8_t_0 = drop7.view(drop7.size(0), -1)
        fc8_t_1 = self.fc8_t_1(fc8_t_0)
        return fc8_t_1

    @staticmethod
    def __dense(name, **kwargs):
        layer = nn.Linear(**kwargs)
        layer.state_dict()['weight'].copy_(torch.from_numpy(__weights_dict[name]['weights']))
        if 'bias' in __weights_dict[name]:
            layer.state_dict()['bias'].copy_(torch.from_numpy(__weights_dict[name]['bias']))
        return layer

    @staticmethod
    def __conv(dim, name, **kwargs):
        if dim == 1:
            layer = nn.Conv1d(**kwargs)
        elif dim == 2:
            layer = nn.Conv2d(**kwargs)
        elif dim == 3:
            layer = nn.Conv3d(**kwargs)
        else:
            raise NotImplementedError()

        layer.state_dict()['weight'].copy_(torch.from_numpy(__weights_dict[name]['weights']))
        if 'bias' in __weights_dict[name]:
            layer.state_dict()['bias'].copy_(torch.from_numpy(__weights_dict[name]['bias']))
        return layer

max_stars_repo_name: arthurdecloedt/thesis

max_issues_repo_name: arthurdecloedt/thesis

max_forks_repo_name: arthurdecloedt/thesis


text:
# Copyright 2020 ByteDance Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import random
from abc import ABCMeta, abstractmethod

import numpy
import six
import yaml

from neurst.utils.flags_core import Flag

@six.add_metaclass(ABCMeta)
class DataSampler(object):
    REGISTRY_NAME = "data_sampler"

    def __init__(self, args):
        if isinstance(args["sample_sizes"], str):
            args["sample_sizes"] = yaml.load(args["sample_sizes"], Loader=yaml.FullLoader)
        assert isinstance(args["sample_sizes"], dict) and len(args["sample_sizes"]) > 0, (
            "Unknown `sample_sizes`={} with type {}".format(args["sample_sizes"], type(args["sample_sizes"])))
        self._sample_ratios = self.get_sample_ratios(args["sample_sizes"])
        total = sum(self._sample_ratios.values())
        self._normalized_sample_weights = {k: float(v) / total for k, v in self._sample_ratios.items()}
        self._sample_items = []
        self._sample_boundaries = []
        for k, v in self._sample_ratios.items():
            self._sample_items.append(k)
            if len(self._sample_boundaries) == 0:
                self._sample_boundaries.append(float(v) / total)
            else:
                self._sample_boundaries.append(self._sample_boundaries[-1] + float(v) / total)
        self._sample_boundaries = numpy.array(self._sample_boundaries)

    @staticmethod
    def class_or_method_args():
        return [
            Flag("sample_sizes", dtype=Flag.TYPE.STRING,
                 help="A dict. The key is the item name to be sampled, "
                      "while the value is the corresponding proportion.")
        ]

    @property
    def normalized_sample_weights(self):
        return self._normalized_sample_weights

    @abstractmethod
    def get_sample_ratios(self, sample_sizes) -> dict:
        raise NotImplementedError

    def __call__(self):
        ratio = random.random()
        for idx in range(len(self._sample_boundaries) - 1, -1, -1):
            if ratio > self._sample_boundaries[idx]:
                return self._sample_items[idx + 1]
        return self._sample_items[0]

max_stars_repo_name: ishine/neurst

max_issues_repo_name: ishine/neurst

max_forks_repo_name: ishine/neurst


text:
from scipy.stats import truncnorm
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.distributions import Categorical

#A truncated distribution has its domain (the x-values) restricted to a certain range of values. For example, you might restrict your x-values to between 0 and 100, written in math terminology as {0 > x > 100}. There are several types of truncated distributions:
def truncated_normal(shape, lower=-0.2, upper=0.2):
    size = 1
    for dim in shape:
        size *= dim
    w_truncated = truncnorm.rvs(lower, upper, size=size)
    w_truncated = torch.from_numpy(w_truncated).float()
    w_truncated = w_truncated.view(shape)
    return w_truncated

class Manager(nn.Module):
    def __init__(self, batch_size, hidden_dim, goal_out_size):
        super(Manager, self).__init__()
        self.batch_size = batch_size
        self.hidden_dim = hidden_dim
        self.goal_out_size = goal_out_size
        self.recurrent_unit = nn.LSTMCell(
            self.goal_out_size, #input size
            self.hidden_dim #hidden size
        )
        self.fc = nn.Linear(
            self.hidden_dim, #in_features
            self.goal_out_size #out_features
        )
        self.goal_init = nn.Parameter(torch.zeros(self.batch_size, self.goal_out_size))
        self._init_params()

    def _init_params(self):
        for param in self.parameters():
            nn.init.normal_(param, std=0.1)
        self.goal_init.data = truncated_normal(
            self.goal_init.data.shape
        )
    def forward(self, f_t, h_m_t, c_m_t):
        """
        f_t = feature of CNN from discriminator leaked at time t, it is input into LSTM
        h_m_t = ouput of previous LSTMCell
        c_m_t = previous cell state
        """
        #print("H_M size: {}".format(h_m_t.size()))
        #print("C_M size: {}".format(c_m_t.size()))
        #print("F_t size: {}".format(f_t.size()))
        h_m_tp1, c_m_tp1 = self.recurrent_unit(f_t, (h_m_t, c_m_t))
        sub_goal = self.fc(h_m_tp1)
        sub_goal = torch.renorm(sub_goal, 2, 0, 1.0) #Returns a tensor where each sub-tensor of input along dimension dim is normalized such that the p-norm of the sub-tensor is lower than the value maxnorm
        return sub_goal, h_m_tp1, c_m_tp1
class Worker(nn.Module):
    def __init__(self, batch_size, vocab_size, embed_dim, hidden_dim,
                    goal_out_size, goal_size):
        super(Worker, self).__init__()
        self.batch_size = batch_size
        self.vocab_size = vocab_size
        self.embed_dim = embed_dim
        self.hidden_dim = hidden_dim
        self.goal_out_size = goal_out_size
        self.goal_size = goal_size

        self.emb = nn.Embedding(self.vocab_size, self.embed_dim)
        self.recurrent_unit = nn.LSTMCell(self.embed_dim, self.hidden_dim)
        self.fc = nn.Linear(self.hidden_dim, self.goal_size*self.vocab_size)
        self.goal_change = nn.Parameter(torch.zeros(self.goal_out_size, self.goal_size))
        self._init_params()

    def _init_params(self):
        for param in self.parameters():
            nn.init.normal_(param, std=0.1)
    def forward(self, x_t, h_w_t, c_w_t):
        """
            x_t = last word
            h_w_t = last output of LSTM in Worker
            c_w_t = last cell state of LSTM in Worker
        """
        x_t_emb = self.emb(x_t)
        h_w_tp1, c_w_tp1 = self.recurrent_unit(x_t_emb, (h_w_t, c_w_t))
        output_tp1 = self.fc(h_w_tp1)
        output_tp1 = output_tp1.view(self.batch_size, self.vocab_size, self.goal_size)
        return output_tp1, h_w_tp1, c_w_tp1
class Generator(nn.Module):
    def __init__(self, worker_params, manager_params, step_size):
        super(Generator, self).__init__()
        self.step_size = step_size
        self.worker = Worker(**worker_params)
        self.manager = Manager(**manager_params)

    def init_hidden(self):
        h = Variable(torch.zeros(self.worker.batch_size, self.worker.hidden_dim))
        c = Variable(torch.zeros(self.worker.batch_size, self.worker.hidden_dim))
        return h, c

    def forward(self, x_t, f_t, h_m_t, c_m_t, h_w_t, c_w_t, last_goal, real_goal, t, temperature):
        sub_goal, h_m_tp1, c_m_tp1 = self.manager(f_t, h_m_t, c_m_t)
        output, h_w_tp1, c_w_tp1 = self.worker(x_t, h_w_t, c_w_t)
        last_goal_temp = last_goal + sub_goal
        w_t = torch.matmul(
            real_goal, self.worker.goal_change
        )
        w_t = torch.renorm(w_t, 2, 0, 1.0)
        w_t = torch.unsqueeze(w_t, -1)
        logits = torch.squeeze(torch.matmul(output, w_t))
        probs = F.softmax(temperature * logits, dim=1)
        x_tp1 = Categorical(probs).sample()
        return x_tp1, h_m_tp1, c_m_tp1, h_w_tp1, c_w_tp1,\
                last_goal_temp, real_goal, sub_goal, probs, t + 1

reason: from scipy


text:
import math
from itertools import count
import matplotlib.pyplot as plt

import gym_super_mario_bros
from gym_super_mario_bros.actions import SIMPLE_MOVEMENT, COMPLEX_MOVEMENT, RIGHT_ONLY
from nes_py.wrappers import JoypadSpace

import numpy as np
import torch
from torch import nn
from torch.distributions import Categorical
from torch.functional import F

from wrappers import wrapper

# env settings
WORLD = 1
STAGE = 1
LEVEL_NAME = "SuperMarioBros-{}-{}-v0".format(WORLD, STAGE)
FRAME_DIM = (120, 128, 4)  # original image size is 240x256
ACTION_SPACE = SIMPLE_MOVEMENT
RENDER_GAME = True

MODEL_PATH = "/Users/dominik/Desktop/Projects/deep-learning-super-mario/policy_gradient/models/v0/model_world-1-1"  # to create a new model set it to ""

# training hyperparameters
LEARNING_RATE = 0.000007
GAMMA = 0.99

VISUALIZATION_INTERVAL = 50

FIRST_LAYER_KERNEL_SIZE = 3
FIRST_LAYER_STRIDE = 2
FIRST_LAYER_OUT = 32

SECOND_LAYER_KERNEL_SIZE = 3
SECOND_LAYER_STRIDE = 1
SECOND_LAYER_OUT = 32

THIRD_LAYER_KERNEL_SIZE = 3
THIRD_LAYER_STRIDE = 1
THIRD_LAYER_OUT = 32

DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def create_environment():
    """Creates the environment, applies some wrappers and returns it."""
    tmp_env = gym_super_mario_bros.make(LEVEL_NAME)
    tmp_env = JoypadSpace(tmp_env, ACTION_SPACE)
    tmp_env = wrapper(tmp_env, FRAME_DIM)

    return tmp_env

def lazy_frame_to_tensor(lazy_frame):
    # pytorch expects the frames as height x width x depth
    return torch.from_numpy(
        np.expand_dims(np.asarray(lazy_frame).astype(np.float64).transpose((2, 1, 0)), axis=0)).float().to(DEVICE)

class Policy(nn.Module):
    def __init__(self, num_actions, frame_dim):
        super(Policy, self).__init__()

        self.conv1 = nn.Conv2d(in_channels=frame_dim[2], out_channels=FIRST_LAYER_OUT,
                               kernel_size=FIRST_LAYER_KERNEL_SIZE, stride=FIRST_LAYER_STRIDE)
        self.conv1_bn = nn.BatchNorm2d(FIRST_LAYER_OUT)

        self.conv2 = nn.Conv2d(in_channels=FIRST_LAYER_OUT, out_channels=SECOND_LAYER_OUT,
                               kernel_size=SECOND_LAYER_KERNEL_SIZE, stride=SECOND_LAYER_STRIDE)
        self.conv2_bn = nn.BatchNorm2d(SECOND_LAYER_OUT)

        self.conv3 = nn.Conv2d(in_channels=SECOND_LAYER_OUT, out_channels=THIRD_LAYER_OUT,
                               kernel_size=THIRD_LAYER_KERNEL_SIZE, stride=THIRD_LAYER_STRIDE)
        self.conv3_bn = nn.BatchNorm2d(THIRD_LAYER_OUT)

        def conv2d_size_out(size, kernel_size, stride):
            return (size - (kernel_size - 1) - 1) // stride + 1

        # calculate the output size of the last conv layer
        conv_width = conv2d_size_out(
            conv2d_size_out(
                conv2d_size_out(frame_dim[1], kernel_size=FIRST_LAYER_KERNEL_SIZE, stride=FIRST_LAYER_STRIDE),
                kernel_size=SECOND_LAYER_KERNEL_SIZE, stride=SECOND_LAYER_STRIDE),
            kernel_size=THIRD_LAYER_KERNEL_SIZE, stride=THIRD_LAYER_STRIDE)
        conv_height = conv2d_size_out(
            conv2d_size_out(
                conv2d_size_out(frame_dim[0], kernel_size=FIRST_LAYER_KERNEL_SIZE, stride=FIRST_LAYER_STRIDE),
                kernel_size=SECOND_LAYER_KERNEL_SIZE, stride=SECOND_LAYER_STRIDE),
            kernel_size=THIRD_LAYER_KERNEL_SIZE, stride=THIRD_LAYER_STRIDE)
        num_neurons = conv_width * conv_height * THIRD_LAYER_OUT
        self.fc1 = nn.Linear(in_features=num_neurons, out_features=1024)
        self.fc2 = nn.Linear(in_features=1024, out_features=num_actions)

    def forward(self, x):
        out = self.conv1(x)
        conv1 = F.elu(self.conv1_bn(out))

        out = self.conv2(conv1)
        conv2 = F.elu(self.conv2_bn(out))

        out = self.conv3(conv2)
        conv3 = F.elu(self.conv3_bn(out))
        out = conv3.view(conv3.size()[0], -1)

        out = F.elu(self.fc1(out))
        out = F.softmax(self.fc2(out), dim=-1)

        return out, conv1.detach(), conv2.detach(), conv3.detach()

def plot_activation_maps(state, conv_activations, file_name):
    fig = plt.figure(figsize=(64, 64))
    columns = int(math.ceil(math.sqrt(conv_activations.size(0))))
    rows = int(math.ceil(math.sqrt(conv_activations.size(0)))) + 1

    # display the current state in the first row
    fig.add_subplot(rows, columns, 1)
    plt.imshow(state[0].t(), cmap="gray")

    for i in range(0, conv_activations.size(0)):
        # normalize and transpose the image
        img = conv_activations[i] - conv_activations[i].min()
        img = img / img.max()
        img = img.t()

        # add it to the plot
        fig.add_subplot(rows, columns, columns + 1 + i)
        plt.imshow(img)

    plt.savefig(file_name)

env = create_environment()

policy = Policy(env.action_space.n, FRAME_DIM)
policy.load_state_dict(torch.load(MODEL_PATH, map_location=DEVICE))
policy.eval()

state = lazy_frame_to_tensor(env.reset())

for step in count(1):
    predicted_action_values, conv1, conv2, conv3 = policy.forward(state)
    action_dist = Categorical(predicted_action_values)
    action = action_dist.sample().item()

    next_state, reward, done, info = env.step(action)
    next_state = lazy_frame_to_tensor(next_state)

    if step % VISUALIZATION_INTERVAL == 0:
        plot_activation_maps(state.squeeze(), conv1.squeeze(), "activation_visualizations/conv1_{}.png".format(step))
        plot_activation_maps(state.squeeze(), conv2.squeeze(), "activation_visualizations/conv2_{}.png".format(step))
        plot_activation_maps(state.squeeze(), conv3.squeeze(), "activation_visualizations/conv3_{}.png".format(step))

    env.render()

    if done:
        break

    state = next_state