text:
# coding=utf-8
# Copyright 2021 The init2winit Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Max pooling convnet classifier.

This model can be used to implement the 3c3d architecture from:
https://github.com/fsschneider/DeepOBS/blob/master/deepobs/tensorflow/testproblems/_3c3d.py
"""

from flax import nn
from init2winit.model_lib import base_model
from init2winit.model_lib import model_utils
from jax.nn import initializers
import jax.numpy as jnp

from ml_collections.config_dict import config_dict

# small hparams used for unit tests
DEFAULT_HPARAMS = config_dict.ConfigDict(dict(
    num_filters=[64, 96, 128],
    kernel_sizes=[5, 3, 3],
    kernel_paddings=['VALID', 'VALID', 'SAME'],
    window_sizes=[3, 3, 3],
    window_paddings=['SAME', 'SAME', 'SAME'],
    strides=[2, 2, 2],
    num_dense_units=[512, 256],
    lr_hparams={
        'initial_value': 0.001,
        'schedule': 'constant'
    },
    layer_rescale_factors={},
    optimizer='momentum',
    opt_hparams={
        'momentum': 0.9,
    },
    batch_size=128,
    activation_fn='relu',
    normalizer='none',
    l2_decay_factor=.0005,
    l2_decay_rank_threshold=2,
    label_smoothing=None,
    rng_seed=-1,
    use_shallue_label_smoothing=False,
    model_dtype='float32',
))

class MaxPoolingCNN(nn.Module):
  """Defines a CNN model with max pooling.

  The model assumes the input shape is [batch, H, W, C].
  """

  def apply(self,
            x,
            num_outputs,
            num_filters,
            kernel_sizes,
            kernel_paddings,
            window_sizes,
            window_paddings,
            strides,
            num_dense_units,
            activation_fn,
            normalizer='none',
            kernel_init=initializers.lecun_normal(),
            bias_init=initializers.zeros,
            train=True):

    maybe_normalize = model_utils.get_normalizer(normalizer, train)
    for num_filters, kernel_size, kernel_padding, window_size, window_padding, stride in zip(
        num_filters, kernel_sizes, kernel_paddings, window_sizes,
        window_paddings, strides):
      x = nn.Conv(
          x,
          num_filters, (kernel_size, kernel_size), (1, 1),
          padding=kernel_padding,
          kernel_init=kernel_init,
          bias_init=bias_init)
      x = model_utils.ACTIVATIONS[activation_fn](x)
      x = maybe_normalize(x)
      x = nn.max_pool(
          x,
          window_shape=(window_size, window_size),
          strides=(stride, stride),
          padding=window_padding)
    x = jnp.reshape(x, (x.shape[0], -1))
    for num_units in num_dense_units:
      x = nn.Dense(
          x, num_units, kernel_init=kernel_init, bias_init=bias_init)
      x = model_utils.ACTIVATIONS[activation_fn](x)
      x = maybe_normalize(x)
    x = nn.Dense(x, num_outputs, kernel_init=kernel_init, bias_init=bias_init)
    return x

class MaxPoolingCNNModel(base_model.BaseModel):

  def build_flax_module(self):
    """CNN with a set of conv layers with max pooling followed by fully connected layers."""
    return MaxPoolingCNN.partial(
        num_outputs=self.hps['output_shape'][-1],
        num_filters=self.hps.num_filters,
        kernel_sizes=self.hps.kernel_sizes,
        kernel_paddings=self.hps.kernel_paddings,
        window_sizes=self.hps.window_sizes,
        window_paddings=self.hps.window_paddings,
        strides=self.hps.strides,
        num_dense_units=self.hps.num_dense_units,
        activation_fn=self.hps.activation_fn,
        normalizer=self.hps.normalizer,)

reason: import jax,from jax


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 19 03:49:39 2018

@author: Aman
"""

import os
import pandas as pd
from sklearn import linear_model
from sklearn import preprocessing
from sklearn import model_selection, metrics
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import math

def get_continuous_columns(df):
    return df.select_dtypes(include=['number']).columns

def get_categorical_columns(df):
    return df.select_dtypes(exclude=['number']).columns

def transform_cat_to_cont(df, features, mappings):
    for feature in features:
        null_idx = df[feature].isnull()
        df.loc[null_idx, feature] = None
        df[feature] = df[feature].map(mappings)

def transform_cont_to_cat(df, features):
    for feature in features:
        df[feature] = df[feature].astype('category')

def get_features_missing_data(df, cutoff):
    total_missing = df.isnull().sum()
    n = df.shape[0]
    to_delete = total_missing[(total_missing/n) > cutoff ]
    return list(to_delete.index)

def filter_features(df, features):
    df.drop(features, axis=1, inplace=True)

def get_imputers(df, features):
    all_cont_features = get_continuous_columns(df)
    cont_features = []
    cat_features = []
    for feature in features:
        if feature in all_cont_features:
            cont_features.append(feature)
        else:
            cat_features.append(feature)
    mean_imputer = preprocessing.Imputer()
    mean_imputer.fit(df[cont_features])
    mode_imputer = preprocessing.Imputer(strategy="most_frequent")
    mode_imputer.fit(df[cat_features])
    return mean_imputer, mode_imputer

def impute_missing_data(df, featuers, imputers):
    all_cont_features = get_continuous_columns(df)
    cont_features = []
    cat_features = []
    for feature in features:
        if feature in all_cont_features:
            cont_features.append(feature)
        else:
            cat_features.append(feature)

    df[cont_features] = imputers[0].transform(df[cont_features])
    df[cat_features] = imputers[1].transform(df[cat_features])

def get_heat_map_corr(df):
    corr = df.select_dtypes(include = ['number']).corr()
    sns.heatmap(corr, square=True)
    plt.xticks(rotation=70)
    plt.yticks(rotation=70)
    return corr

def get_target_corr(corr, target):
    return corr[target].sort_values(axis=0,ascending=False)

def one_hot_encode(df):
   features = get_categorical_columns(df)
   return pd.get_dummies(df, columns=features)

def rmse(y_orig, y_pred):
    return math.sqrt(metrics.mean_squared_error(y_orig,y_pred))

def fit_model(estimator, grid, X_train, y_train):
   grid_estimator = model_selection.GridSearchCV(estimator, grid, scoring = metrics.make_scorer(rmse), cv=10, n_jobs=1)
   grid_estimator.fit(X_train, y_train)
   print(grid_estimator.grid_scores_)
   print(grid_estimator.best_params_)
   print(grid_estimator.best_score_)
   print(grid_estimator.score(X_train, y_train))
   return grid_estimator.best_estimator_

def feature_importances(estimator):
    return estimator.feature_importances_

def predict(estimator, X_test):
    return estimator.predict(X_test)

#to uncover the mismatch of levels between train and test data
def merge(df1, df2):
    return pd.concat([df1, df2])

def split(df, ind):
    return (df[0:ind], df[ind:])

def viz_cont_cont(df, features, target):
    for feature in features:
        sns.jointplot(x = feature, y = target, data = df)

def viz_cat_cont_box(df, features, target):
    for feature in features:
        sns.boxplot(x = feature, y = target,  data = df)
        plt.xticks(rotation=45)

def viz_cont(df, features):
    for feature in features:
        sns.distplot(df[feature],kde=False)

os.chdir("F:/Data Science/House prices/Data/")

house_train = pd.read_csv("house_train.csv")
house_train.shape
house_train.info()

house_test = pd.read_csv("house_test.csv")
house_test.shape
house_test.info()
house_test['SalePrice'] = 0

house_data = merge(house_train, house_test)
house_data.shape
house_data.info()

print(get_continuous_columns(house_data))
print(get_categorical_columns(house_data))

#convert numerical columns to categorical type
features = ['MSSubClass']
transform_cont_to_cat(house_data, features)

#map string categoical values to numbers
ordinal_features = ["ExterQual", "ExterCond", "BsmtQual", "BsmtCond", "GarageQual", "GarageCond", "PoolQC", "FireplaceQu", "KitchenQual", "HeatingQC"]
quality_dict = {None: 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5}
transform_cat_to_cont(house_data, ordinal_features, quality_dict)

print(get_continuous_columns(house_data))
print(get_categorical_columns(house_data))

house_train, house_test = split(house_data, house_train.shape[0])
house_train.shape
house_test.shape

#filter missing data columns
missing_features = get_features_missing_data(house_train, 0)
filter_features(house_train, missing_features)
house_train.shape
house_train.info()

#smooth the sale price using log transformation(smoothening outlier data)
house_train['log_sale_price'] = np.log(house_train['SalePrice'])
features = ['SalePrice','log_sale_price']
viz_cont(house_train, features)

#explore relationship of neighborhood to saleprice
target = 'SalePrice'
features = ['Neighborhood']
viz_cat_cont_box(house_train, features, target)

#explore relationship of livarea and totalbsmt to saleprice
features = ['GrLivArea']
features = ['TotalBsmtSF']
viz_cont_cont(house_train, features, target)

filter_features(house_train, ['Id'])

#explore relation among all continuous features vs saleprice
corr = get_heat_map_corr(house_train)
get_target_corr(corr, 'SalePrice')
#In case if the data has outliers then some way we have to make them to Bell curve fasion. The same can be done by Log transformation.
get_target_corr(corr, 'log_sale_price')

#do one-hot-encoding for all the categorical features
print(get_categorical_columns(house_train))
house_train1 = one_hot_encode(house_train)
house_train1.shape
house_train1.info()

filter_features(house_train1, ['SalePrice','log_sale_price'])
X_train = house_train1
y_train = house_train['log_sale_price']

#L1 regulizer/Lasso
#Regulizers will assign zeros to un-important fetures
lasso_estimator = linear_model.Lasso()
lasso_grid = {'alpha':[0, 0.1, 0.5, 0.7, 1]}
model = fit_model(lasso_estimator, lasso_grid, X_train, y_train)
#0.194408382164
#0.19270196310743642
#Here you can see some of the coefficients as Zero and such fetures will be eliminated while predicting.
coef = model.coef_ #Weights of all the features (w1, w2, w3.....)
dir(coef.tolist)
intercept = model.intercept_ #W0, minimal value

ext: py

max_stars_repo_path: Lasso Regression.py

max_stars_repo_name: A-m-a-n/House-Prices

max_issues_repo_path: Lasso Regression.py

max_issues_repo_name: A-m-a-n/House-Prices

max_forks_repo_path: Lasso Regression.py

max_forks_repo_name: A-m-a-n/House-Prices

reason: import numpy


text:
"""
query() hits skatteverket's homepage
query takes payload argument
parse_response calls query
read_tax uses parse_response

The class can have its parameters set manually, but basic use-case is to have it driven off
a config file (se_test_cases.ini) with various scenarios together with what the correct tax should be (manually maintained).
There's also (perhaps more useful) to automatically query skatteverket to check what the correc tax should be.
This can help us capture whenever tax rules change.
The tests are run daily on github via test_swedish_tax.py

"""
import pdb
import copy
import datetime
import numpy as np
import pandas as pd
import tax_utils as tut
from tax_calculator import TaxCalculator

class SwedishTax(TaxCalculator):
    """
    to facilitate easy input

    """

    def __init__(self, salary=0, birth_year=1978, tax_year=None, listed_funds_and_shares_profit=0, listed_funds_and_shares_loss=0, int_inc_tax_withheld=0, int_inc_tax_not_withheld=0, rental_income=0, unlisted_funds_profit=0, sale_private_property_profit=0,
                 sale_private_property_loss=0, interest_expense=0, municipality='Partille', unlisted_funds_loss=0, standard_income=0, sale_commercial_property_profit=0, sale_commercial_property_loss=0, investor_deduction=0,
                 qualified_shares_profit_loss=0, unlisted_shares_profit_loss=0, case_idx=None):

        self._salary = salary
        self._birth_year = birth_year
        if tax_year is None:
            tax_year = pd.to_datetime('today').year

        self._listed_funds_and_shares_profit = listed_funds_and_shares_profit
        self._listed_funds_and_shares_loss = listed_funds_and_shares_loss
        self._int_inc_tax_withheld = int_inc_tax_withheld
        self._int_inc_tax_not_withheld = int_inc_tax_not_withheld
        self._rental_income = rental_income
        self._unlisted_funds_profit = unlisted_funds_profit
        self._sale_private_property_profit = sale_private_property_profit
        self._sale_private_property_loss = sale_private_property_loss
        self._interest_expense = interest_expense
        self._municipality = municipality
        self._unlisted_funds_loss = unlisted_funds_loss
        self._standard_income = standard_income
        self._sale_commercial_property_profit = sale_commercial_property_profit
        self._sale_commercial_property_loss = sale_commercial_property_loss
        self._investor_deduction = investor_deduction
        self._qualified_shares_profit_loss = qualified_shares_profit_loss
        self._unlisted_shares_profit_loss = unlisted_shares_profit_loss
        self._tax_breakdown_dict = {}

        self._jurisdiction = 'SWE'
        self._income_deduction = None
        self._job_deduction = None
        super().__init__(
            jurisdiction=self._jurisdiction,
            case_idx=case_idx,
            tax_url="https://app.skatteverket.se/gateway-skut-skatteutrakning/skatteberakning-fysisk/rakna-ut-skatt",
            tax_year=tax_year)

    @property
    def salary(self):
        return self._salary

    @salary.setter
    def salary(self, value):
        self._salary = value

    @property
    def headers(self):
        return self._headers

    @headers.setter
    def headers(self, value):
        assert isinstance(value, dict), "headers should be a dict"
        self._headers = value

    @property
    def tax_breakdown_dict(self):
        return self._tax_breakdown_dict

    @tax_breakdown_dict.setter
    def tax_breakdown_dict(self, value):
        assert isinstance(value, dict), "tax breakdown dict should be a dict"
        self._tax_breakdown_dict = value

    @property
    def birth_year(self):
        return self._birth_year

    @birth_year.setter
    def birth_year(self, value):
        self._birth_year = value

    @property
    def listed_funds_and_shares_profit(self):

        return self._listed_funds_and_shares_profit

    @listed_funds_and_shares_profit.setter
    def listed_funds_and_shares_profit(self, value):
        assert tut.value_is_numeric_type(
            value), "listed_funds_and_shares_profit should be a number!"
        self._listed_funds_and_shares_profit = value

    @property
    def listed_funds_and_shares_loss(self):
        return self._listed_funds_and_shares_loss

    @listed_funds_and_shares_loss.setter
    def listed_funds_and_shares_loss(self, value):
        assert tut.value_is_numeric_type(
            value), "listed_funds_and_shares_loss should be a number!"
        self._listed_funds_and_shares_loss = value

    @property
    def int_inc_tax_withheld(self):
        return self._int_inc_tax_withheld

    @int_inc_tax_withheld.setter
    def int_inc_tax_withheld(self, value):
        assert tut.value_is_numeric_type(
            value), "int_inc_tax_withheld should be a number!"
        self._int_inc_tax_withheld = value

    @property
    def int_inc_tax_not_withheld(self):
        return self._int_inc_tax_not_withheld

    @int_inc_tax_not_withheld.setter
    def int_inc_tax_not_withheld(self, value):
        assert tut.value_is_numeric_type(
            value), "int_inc_tax_not_withheld should be a number!"
        self._int_inc_tax_not_withheld = value

    @property
    def rental_income(self):
        return self._rental_income

    @rental_income.setter
    def rental_income(self, value):
        assert tut.value_is_numeric_type(
            value), "rental_income should be a number!"
        self._rental_income = value

    @property
    def unlisted_funds_profit(self):
        return self._unlisted_funds_profit

    @unlisted_funds_profit.setter
    def unlisted_funds_profit(self, value):
        assert tut.value_is_numeric_type(
            value), "unlisted_funds_profit should be a number!"
        self._unlisted_funds_profit = value

    @property
    def sale_private_property_profit(self):
        return self._sale_private_property_profit

    @sale_private_property_profit.setter
    def sale_private_property_profit(self, value):
        assert tut.value_is_numeric_type(
            value), "sale_private_property_profit should be a number!"
        self._sale_private_property_profit = value

    @property
    def sale_private_property_loss(self):
        return self._sale_private_property_loss

    @sale_private_property_loss.setter
    def sale_private_property_loss(self, value):
        assert tut.value_is_numeric_type(
            value), "sale_private_property_loss should be a number!"
        self._sale_private_property_loss = value

    @property
    def interest_expense(self):
        return self._interest_expense

    @interest_expense.setter
    def interest_expense(self, value):
        assert tut.value_is_numeric_type(
            value), "interest_expense should be a number!"
        self._interest_expense = value

    @property
    def municipality(self):
        return self._municipality

    @municipality.setter
    def municipality(self, value):
        params = tut.tax_parameters(jurisdiction='SE', tax_year=self.tax_year)
        assert 'kommunalskatt_%s' % value.lower() in dict(
            params), "Tax rate for '%s' not implemented yet!" % value
        self._municipality = value

    @property
    def unlisted_funds_loss(self):
        return self._unlisted_funds_loss

    @unlisted_funds_loss.setter
    def unlisted_funds_loss(self, value):
        assert tut.value_is_numeric_type(
            value), "unlisted_funds_loss should be a number!"
        self._unlisted_funds_loss = value

    @property
    def standard_income(self):
        return self._standard_income

    @standard_income.setter
    def standard_income(self, value):
        assert tut.value_is_numeric_type(
            value), "standard_income should be a number!"
        self._standard_income = value

    @property
    def sale_commercial_property_profit(self):
        return self._sale_commercial_property_profit

    @sale_commercial_property_profit.setter
    def sale_commercial_property_profit(self, value):
        assert tut.value_is_numeric_type(
            value), "sale_commercial_property_profit should be a number!"
        self._sale_commercial_property_profit = value

    @property
    def sale_commercial_property_loss(self):
        return self._sale_commercial_property_loss

    @sale_commercial_property_loss.setter
    def sale_commercial_property_loss(self, value):
        assert tut.value_is_numeric_type(
            value), "sale_commercial_property_loss should be a number!"
        self._sale_commercial_property_loss = value

    @property
    def investor_deduction(self):
        return self._investor_deduction

    @investor_deduction.setter
    def investor_deduction(self, value):
        assert tut.value_is_numeric_type(
            value), "investor_deduction should be a number!"
        if value > self.parameter('max_investor_deduction'):
            raise Exception(
                "The max investor_deduction (investeraravdrag) is %d" %
                self.parameter('max_investor_deduction'))
        self._investor_deduction = value

    @property
    def qualified_shares_profit_loss(self):
        return self._qualified_shares_profit_loss

    @qualified_shares_profit_loss.setter
    def qualified_shares_profit_loss(self, value):
        assert tut.value_is_numeric_type(
            value), "qualified_shares_profit_loss should be a number!"
        self._qualified_shares_profit_loss = value

    @property
    def unlisted_shares_profit_loss(self):
        return self._unlisted_shares_profit_loss

    @unlisted_shares_profit_loss.setter
    def unlisted_shares_profit_loss(self, value):
        assert tut.value_is_numeric_type(
            value), "unlisted_shares_profit_loss should be a number!"
        self._unlisted_shares_profit_loss = value

    @property
    def income_deduction(self):
        if self._income_deduction is None:
            return np.floor(min(max(self.parameter('skattereduktionForvarvsinkomst_procent', 100) * (self.taxable_income() -
                                                                                self.parameter('skattereduktionForvarvsinkomst_golv')), 0), self.parameter('skattereduktionForvarvsinkomst_tak')))
        return self._income_deduction

    @income_deduction.setter
    def income_deduction(self, value):
        # assert isinstance(value, float), "Income deduction needs to be float, not %s"%(str(type(value)))
        self._income_deduction = value

    @property
    def job_deduction(self):
        if self._job_deduction is None:
            return self.jobbskatteavdrag()
        return self._job_deduction

    @job_deduction.setter
    def job_deduction(self, value):
        self._job_deduction = value

    @property
    def lower_limit_for_income_tax(self):
        return np.floor(self.parameter('pris_bas_belopp') *
                        self.parameter('skatt_lagre_inkomst_multipel'))

    @property
    def funeral_rate(self):
        if self.municipality == 'Stockholm':
            return 0.065 / 100
        if self.municipality == 'Tranas':
            return 0.24 / 100

        return 0.253 / 100

    @property
    def funeral_fee(self):
        return np.floor(self.funeral_rate * self.taxable_income())

    def old_age_limit_year(self):
        """
        over a certain age, the various rates etc change
        """
        old_age_cutoff = datetime.date(
            self.tax_year, 1, 1) - pd.DateOffset(years=66)
        return old_age_cutoff.year

    def qualified_shares_income_taxation(self):
        """
        returns the extra income we need to add and then the extra capital_income we need to add
        Detta ar blankett K10
        """
        extra_unlisted_funds_loss = 0
        extra_salary = 0
        extra_listed_funds_and_shares_profit = 0

        if abs(self.qualified_shares_profit_loss) > 1e-4:

            if self.qualified_shares_profit_loss < 0:
                # this is point 3.4b in the K10 form:
                extra_unlisted_funds_loss += abs(
                    self.qualified_shares_profit_loss) * 2 / 3
            else:
                # we assume no dividend, so we end up at 3.7b of K10:
                # pdb.set_trace()
                # qualified_shares_income_limit =
                # upper_limit_employment_income =
                qualified_profit = max(
                    self.qualified_shares_profit_loss -
                    self.parameter('gransbelopp_forenkling'),
                    0)
                employment_income = min(
                    qualified_profit,
                    self.parameter('upper_limit_employment_income'))
                # gets taxed as employment income
                extra_salary += employment_income
                # residual = qualified_profit - employment_income
                # work out what should be tax as capital income (this we'll add to
                # the listed_funds_and_share_profit):
                capital_gain = max(
                    self.qualified_shares_profit_loss - employment_income, 0)
                # assume we don't have any "sparat utdelningsutrymme", the belowe
                # captures points 3.12-3.13 in the K10 form:
                to_be_taxed_as_capital = 0
                if self.parameter('gransbelopp_forenkling') > capital_gain:
                    to_be_taxed_as_capital += (2 / 3) * capital_gain
                else:
                    to_be_taxed_as_capital += (2 / 3) * \
                        self.parameter('gransbelopp_forenkling')
                to_be_taxed_as_capital += max(capital_gain -
                                              self.parameter('gransbelopp_forenkling'), 0)
                extra_listed_funds_and_shares_profit += to_be_taxed_as_capital

        return extra_salary, extra_listed_funds_and_shares_profit, extra_unlisted_funds_loss

    def adjusted_attribute_value(self, attribute='salary'):
        """
        returns the extra income we need to add and then the extra capital_income we need to add
        Detta ar blankett K10

        If you earn money through your company, some of that money could be reclassified as employment income etc,
        so we need this extra calculation step.
        """
        extra_unlisted_funds_loss = 0
        extra_salary = 0
        extra_listed_funds_and_shares_profit = 0
        extra_listed_loss = 0

        if abs(self.qualified_shares_profit_loss) > 1e-4:

            if self.qualified_shares_profit_loss < 0:
                # this is point 3.4b in the K10 form:
                extra_unlisted_funds_loss += abs(
                    self.qualified_shares_profit_loss) * 2 / 3
            else:
                # we assume no dividend, so we end up at 3.7b of K10:
                # pdb.set_trace()
                # qualified_shares_income_limit =
                # upper_limit_employment_income =
                qualified_profit = max(
                    self.qualified_shares_profit_loss -
                    self.parameter('gransbelopp_forenkling'),
                    0)
                employment_income = min(
                    qualified_profit,
                    self.parameter('upper_limit_employment_income'))
                # gets taxed as employment income
                extra_salary += employment_income
                # residual = qualified_profit - employment_income
                # work out what should be tax as capital income (this we'll add to
                # the listed_funds_and_share_profit):
                capital_gain = max(
                    self.qualified_shares_profit_loss - employment_income, 0)
                # assume we don't have any "sparat utdelningsutrymme", the belowe
                # captures points 3.12-3.13 in the K10 form:
                to_be_taxed_as_capital = 0
                if self.parameter('gransbelopp_forenkling') > capital_gain:
                    to_be_taxed_as_capital += (2 / 3) * capital_gain
                else:
                    to_be_taxed_as_capital += (2 / 3) * \
                        self.parameter('gransbelopp_forenkling')
                to_be_taxed_as_capital += max(capital_gain -
                                              self.parameter('gransbelopp_forenkling'), 0)
                extra_listed_funds_and_shares_profit += to_be_taxed_as_capital

        # return extra_salary, extra_listed_funds_and_shares_profit,
        # extra_unlisted_funds_loss

        if self.unlisted_shares_profit_loss < 0:
            extra_listed_loss += (5 / 6) * \
                abs(self.unlisted_shares_profit_loss)
        else:
            extra_listed_funds_and_shares_profit += (
                5 / 6) * self.unlisted_shares_profit_loss
        if attribute == 'salary':
            return self.salary + extra_salary
        if attribute == 'listed_funds_and_shares_profit':
            return self.listed_funds_and_shares_profit + \
                extra_listed_funds_and_shares_profit
        if attribute == 'unlisted_funds_loss':
            return self.unlisted_funds_loss + extra_unlisted_funds_loss
        if attribute == 'listed_funds_and_shares_loss':
            return self.listed_funds_and_shares_loss + extra_listed_loss
        raise Exception("Attributed == '%s' not handled!" % attribute)

    def unqualified_shares_income_taxation(self):
        """
        returns the extra income we need to add and then the extra capital_income we need to add
        Detta ar blankett K12
        Om vinst, 5/6 * belopp till 7.4 (where would we add it in code?) Profit should go to listed_funds_and_shares_profit
        Om förlust 5/6 * belopp till 8.3 goes to listed_loss
        """

        extra_listed_funds_and_shares_profit = 0
        extra_listed_loss = 0

        if self.unlisted_shares_profit_loss < 0:
            extra_listed_loss += (5 / 6) * \
                abs(self.unlisted_shares_profit_loss)
        else:
            extra_listed_funds_and_shares_profit += (
                5 / 6) * self.unlisted_shares_profit_loss

        return extra_listed_funds_and_shares_profit, extra_listed_loss

    def _basic_deduction_base_case(self):

        # TODO: Add these as parameters, will change
        if self.salary_rounded() <= self.parameter('ga_m1') * \
                self.parameter('pris_bas_belopp'):
            # A in s/s
            return 0.423 * self.parameter('pris_bas_belopp')
        if self.salary_rounded() <= self.parameter('ga_m2') * \
                self.parameter('pris_bas_belopp'):
            # B in s/s
            return 0.423 * self.parameter('pris_bas_belopp') + 0.2 * (
                self.salary_rounded() - self.parameter('ga_m1') * self.parameter('pris_bas_belopp'))
        if self.salary_rounded() <= self.parameter('ga_m3') * \
                self.parameter('pris_bas_belopp'):
            # C in s/s
            return 0.77 * self.parameter('pris_bas_belopp')
        if self.salary_rounded() <= self.parameter('ga_m4') * \
                self.parameter('pris_bas_belopp'):
            return 0.77 * self.parameter('pris_bas_belopp') - 0.1 * (
                self.salary_rounded() - self.parameter('ga_m3') * self.parameter('pris_bas_belopp'))

            # E in the spreadsheet
        return 0.293 * self.parameter('pris_bas_belopp')

    def _basic_deduction_extra(self):
        # extra = 0
        if self.birth_year <= self.old_age_limit_year():
            if self.salary_rounded() <= self.parameter('ega_m1') * \
                    self.parameter('pris_bas_belopp'):
                return 0.687 * self.parameter('pris_bas_belopp')
            if self.salary_rounded() <= self.parameter('ega_m2') * \
                    self.parameter('pris_bas_belopp'):
                return 0.885 * \
                    self.parameter('pris_bas_belopp') - \
                    0.2 * self.salary_rounded()
            if self.salary_rounded() <= self.parameter('ega_m3') * \
                    self.parameter('pris_bas_belopp'):
                return 0.6 * \
                    self.parameter('pris_bas_belopp') + \
                    0.057 * self.salary_rounded()
            if self.salary_rounded() <= self.parameter('ega_m4') * \
                    self.parameter('pris_bas_belopp'):
                return 0.34 * \
                    self.parameter('pris_bas_belopp') - \
                    0.169 * self.salary_rounded()
            if self.salary_rounded() <= self.parameter('ega_m5') * \
                    self.parameter('pris_bas_belopp'):
                return 0.44 * self.salary_rounded() - 0.48 * self.parameter('pris_bas_belopp')

            if self.salary_rounded() <= self.parameter('ega_m6') * \
                    self.parameter('pris_bas_belopp'):
                return 0.207 * \
                    self.parameter('pris_bas_belopp') + \
                    0.228 * self.salary_rounded()
            if self.salary_rounded() <= self.parameter('ega_m7') * \
                    self.parameter('pris_bas_belopp'):
                return 0.995 * \
                    self.parameter('pris_bas_belopp') + \
                    0.128 * self.salary_rounded()
            if self.salary_rounded() <= self.parameter('ega_m8') * \
                    self.parameter('pris_bas_belopp'):
                return 2.029 * self.parameter('pris_bas_belopp')
            if self.salary_rounded() <= self.parameter('ega_m9') * \
                    self.parameter('pris_bas_belopp'):
                return 9.023 * \
                    self.parameter('pris_bas_belopp') - \
                    0.62 * self.salary_rounded()
            if self.salary_rounded() <= self.parameter('ega_m10') * \
                    self.parameter('pris_bas_belopp'):
                # extra = 1.654 * basbelopp - 0.045 * salary
                return 1.253 * self.parameter('pris_bas_belopp')
            if self.salary_rounded() <= self.parameter('ega_m11') * \
                    self.parameter('pris_bas_belopp'):
                return 2.03 * \
                    self.parameter('pris_bas_belopp') - \
                    0.0574 * self.salary_rounded()

            return 0.215 * self.parameter('pris_bas_belopp')
        return 0

    def basic_deduction(self):
        """
        grundavdrag

        the extra for 65+ is described here:
        https://www.regeringen.se/4a6f30/contentassets/23ff11528fc54f918a144c067b44672e/ytterligare-skattesankningar-for-personer-over-65-ar.pdf
        """

        return min(np.ceil((self._basic_deduction_base_case(
        ) + self._basic_deduction_extra()) / 100) * 100, self.salary_rounded())

    def _jobbskatteavdrag_older(self):
        """
        older people have a different deduction than working-age people:
        """
        # TODO: get limits and rates from params file
        if self.birth_year > self.old_age_limit_year():
            return 0

        if self.salary_rounded() <= self.parameter('jsa_g1'):
            return 0.2 * self.salary_rounded()
        if self.salary_rounded() <= self.parameter('jsa_g2'):
            return 15e3 + 0.05 * self.salary_rounded()
        if self.salary_rounded() <= self.parameter('jsa_g3'):
            return 30e3

        return 30e3 - 0.03 * (self.salary_rounded() - self.parameter('jsa_g3'))

    def _jobbskatteavdrag_base_case(self):
        # TODO: get limits and rates from params file
        if self.birth_year <= self.old_age_limit_year():
            return 0
        if self.salary_rounded() <= self.parameter('jsa_m1') * \
                self.parameter('pris_bas_belopp'):
            return self.parameter('kommunalskatt_%s' % self.municipality,
                                  100) * (self.salary_rounded() - self.basic_deduction())
        if self.salary_rounded() <= self.parameter('jsa_m2') * \
                self.parameter('pris_bas_belopp'):
            return (self.parameter('jsa_m1') * self.parameter('pris_bas_belopp') + 0.3405 * (self.salary_rounded() - self.parameter('jsa_m1') * self.parameter(
                'pris_bas_belopp')) - self.basic_deduction()) * self.parameter('kommunalskatt_%s' % self.municipality, 100)
        if self.salary_rounded() <= self.parameter('jsa_m3') * \
                self.parameter('pris_bas_belopp'):
            return (1.703 * self.parameter('pris_bas_belopp') + 0.128 * (self.salary_rounded() - self.parameter('jsa_m2') * self.parameter(
                'pris_bas_belopp')) - self.basic_deduction()) * self.parameter('kommunalskatt_%s' % self.municipality, 100)
        if self.salary_rounded() <= self.parameter('jsa_m4') * \
                self.parameter('pris_bas_belopp'):
            return (2.323 * self.parameter('pris_bas_belopp') - self.basic_deduction()
                    ) * self.parameter('kommunalskatt_%s' % self.municipality, 100)

        return (2.323 * self.parameter('pris_bas_belopp') - self.basic_deduction()) * self.parameter('kommunalskatt_%s' %
                                                                                self.municipality, 100) - 0.03 * (self.salary_rounded() - self.parameter('jsa_m4') * self.parameter('pris_bas_belopp'))

    def jobbskatteavdrag(self):
        """
        break this out into own function since it's quite finicky
        """
        return np.floor(self._jobbskatteavdrag_older() +
                        self._jobbskatteavdrag_base_case())

    def net_interest(self):
        return self.int_inc_tax_not_withheld + \
            self.int_inc_tax_withheld - self.interest_expense

    def net_listed_funds_and_shares(self):
        return self.adjusted_attribute_value(
            'listed_funds_and_shares_profit') - self.adjusted_attribute_value('listed_funds_and_shares_loss')

    def net_unlisted_funds_and_shares(self):
        return self.unlisted_funds_profit - \
            self.parameter('olistad_forlust_avdrags_multipel') * \
            self.adjusted_attribute_value('unlisted_funds_loss')

    def net_property(self):
        ret_val = self.sale_private_property_profit * \
            self.parameter('vinst_avdrag_privat_fastighet_taljare') / \
            self.parameter('vinst_avdrag_privat_fastighet_namnare')
        ret_val -= self.sale_private_property_loss * \
            self.parameter('forlust_avdrag_privat_fastighet')
        ret_val += self.parameter('vinst_avdrag_kommers_fastighet') * \
            self.sale_commercial_property_profit
        ret_val -= self.parameter('forlust_avdrag_kommers_fastighet') * \
            self.sale_commercial_property_loss
        return ret_val

    def capital_surplus_deficit(self):
        """
        # overskottUnderskottKapital:
        # divide into different areas:
        #  standard_income  (no offsetting reduction)
        #  rental income (no offsetting reduction, since it's already net of costs)
        #  Ranta  (income vs expenses, net 100% multiplier)
        #  Fondandelar (profits vs loss, net is 70% multiplier)
        #  Fondandelar ej marknadsnoterade (profits vs loss, net is 70% multiplier)
        #  Forsaljning privatbostad (profit vs loss, net is 50% multipllier)
        #  Forsäljning näringsbostad (p vs l, net is 63%)
        """

        surplus_deficit = 0
        surplus_deficit += self.standard_income
        surplus_deficit += self.rental_income
        surplus_deficit += self.net_interest()
        if self.net_listed_funds_and_shares() < 0:
            surplus_deficit += self.parameter(
                'listade_instrument_forlust_avdrags_multipel') * self.net_listed_funds_and_shares()
        else:
            surplus_deficit += self.net_listed_funds_and_shares()

        surplus_deficit += self.net_unlisted_funds_and_shares()
        surplus_deficit += self.net_property()
        surplus_deficit -= self.investor_deduction

        return surplus_deficit

    def _skatte_reduktion(self, netto_underskott):
        """
        helper function
        """

        return np.floor(self.parameter('skatte_reduktion_forsta_niva') * min(self.parameter('tak_skattereduktion'), netto_underskott) +
                        self.parameter('skatte_reduktion_andra_niva') * max(netto_underskott - self.parameter('tak_skattereduktion'), 0))

    def capital_income_tax(self, verbose=False):
        """
        returns cap_tax, cap_tax_reduction, cap_tax_basis

        """
        surplus_deficit = self.capital_surplus_deficit()

        cap_tax_reduction = 0
        state_rate = self.parameter('statlig_skatt', 100)
        is_surplus = True
        # pdb.set_trace()
        if surplus_deficit < 0:
            cap_tax = 0
            is_surplus = False
            surplus_deficit = abs(surplus_deficit)
            cap_tax_reduction_calc = self._skatte_reduktion(surplus_deficit)
            if verbose:
                print("Calculated tax reduction cap is %f and the calculated deduction is %f" % (
                    self.tax_reduction_cap(), cap_tax_reduction_calc))
            cap_tax_reduction = max(
                min(self.tax_reduction_cap(), cap_tax_reduction_calc), 0)
        else:
            cap_tax = state_rate * surplus_deficit

        return cap_tax, cap_tax_reduction, np.ceil(surplus_deficit), is_surplus

    def taxable_income(self):
        return np.floor(self.adjusted_attribute_value(
            'salary') / 100) * 100 - self.basic_deduction()

    def municipal_tax(self):
        return np.floor(self.taxable_income(
        ) * self.parameter('kommunalskatt_%s' % self.municipality, 100))

    def salary_rounded(self):
        return np.floor(self.adjusted_attribute_value('salary') / 100) * 100

    def state_income_tax(self):
        return self.parameter('statlig_inkomst_skatt', 100) * \
            max(self.taxable_income() - self.parameter('skiktgrans'), 0)

    def absolute_pension_cutoff(self):
        return self.parameter('allman_pension_noll', 100) * \
            self.parameter('inkomst_bas_belopp')
        # abs_pension_cap = pension_ceil_mult * inkomst_bas_belopp

    def absolute_pension_cap(self):
        return self.parameter('allman_pension_tak') * \
            self.parameter('inkomst_bas_belopp')

    def pension(self):
        pension = 0 if self.adjusted_attribute_value('salary') < self.absolute_pension_cutoff() else min(
            self.parameter('allman_pension_procent', 100) * self.adjusted_attribute_value('salary'), self.absolute_pension_cap() * self.parameter('allman_pension_procent') / 100)

        return np.floor(pension / 100) * 100

    def tax_reduction_cap(self):
        # print('hej')
        return self.municipal_tax() + self.state_income_tax() - self.job_deduction - \
            self.income_deduction - self.pension()

    def public_service_cost(self):
        return min(self.parameter('public_service_avgift'), self.parameter(
            'public_service_percent', 100) * self.taxable_income())

    def tax(self):

        cap_tax, cap_tax_reduction, cap_tax_basis, is_surplus = self.capital_income_tax()

        # pdb.set_trace()
        hypo_pension_reduction = self.pension()
        if not is_surplus or abs(cap_tax_basis) < 1e-4:
            if self.tax_reduction_cap() < 0:
                # if birth_year > limit_year:
                orig_jsa = copy.deepcopy(self.job_deduction)
                residual = 0
                if abs(self.tax_reduction_cap()) > self.job_deduction:
                    residual = abs(self.tax_reduction_cap()) - \
                        self.job_deduction
                    self.job_deduction = 0
                else:
                    self.job_deduction = self.job_deduction + self.tax_reduction_cap()

                if residual > 0:
                    # pdb.set_trace()
                    if hypo_pension_reduction >= residual:
                        hypo_pension_reduction -= residual
                    else:
                        print("Not sure what deduction to decrease!?")
                        pdb.set_trace()
                else:
                    self.job_deduction = self.job_deduction + self.income_deduction
                    self.income_deduction = 0
                print(
                    "Changing jobbskatteavdrag from %.0f to %.0f" %
                    (orig_jsa, self.job_deduction))

        non_zero_surplus = False
        if is_surplus:
            non_zero_surplus = cap_tax_basis > 0

        if abs(self.municipal_tax()) > 1e-4:
            pension_reduction = hypo_pension_reduction
        else:
            if not non_zero_surplus:

                pension_reduction = 0
            else:
                pension_reduction = self.pension()

        # pdb.set_trace()
        if abs(self.municipal_tax()) < 1e-4:
            # we can't get this deduction if we don't pay any muni tax:
            # https://www4.skatteverket.se/rattsligvagledning/edition/2021.1/2940.html
            # ('avrakning av reduktionen')
            self.job_deduction = 0

        if self.adjusted_attribute_value(
                'salary') > self.lower_limit_for_income_tax:
            total_tax = self.municipal_tax() + cap_tax + self.pension() - pension_reduction + self.funeral_fee + \
                self.public_service_cost() - self.job_deduction - self.income_deduction + \
                self.state_income_tax() - cap_tax_reduction
        else:
            total_tax = 0

        # total_tax ties with 'slutligSkatt' in the skatteverket calculator, but that's not what we actually end up paying,
        # this catches that

        self.tax_breakdown_dict = {'slutligSkatt': total_tax - self.parameter('statlig_skatt', 100) * self.int_inc_tax_withheld, 'kommunalInkomstskatt': self.municipal_tax(), 'statligInkomstskattKapitalinkomst': cap_tax, 'allmanPensionsavgift': self.pension(), 'skattereduktionAllmanPensionsavgift': pension_reduction,
                                   'begravningsavgift': self.funeral_fee, 'publicServiceAvgift': self.public_service_cost(), 'skattereduktionArbetsinkomster': self.job_deduction, 'skattereduktionForvarvsinkomst': self.income_deduction,
                                   'statligInkomstskattForvarvsinkomst': self.state_income_tax(), 'overskottUnderskottKapital': cap_tax_basis, 'forvarvsinkomst': self.adjusted_attribute_value('salary'),
                                   'faststalldForvarvsinkomst': self.salary_rounded(), 'beskattningsbarForvarvsinkomst': self.taxable_income(), 'grundavdrag': self.basic_deduction(), 'kommunalLandstingsSkattesats': self.parameter('kommunalskatt_%s' % self.municipality, 100),
                                   'skattereduktionUnderskottKapital': cap_tax_reduction, 'avdragenSkattPaKapital': self.parameter('statlig_skatt', 100) * self.int_inc_tax_withheld}
        return total_tax - \
            self.parameter('statlig_skatt', 100) * self.int_inc_tax_withheld

    def payload(self):
        """
        FIXME can read municipal rate etc from api? Not important right now

        """
        data = {}
        data['allmannaAvdrag'] = {'socialforsakringsavgifter': 'null'}

        # calculate profit/loss from selling qualified shares in small companies:
        # parameters = tax_parameters(tax_year=tax_year)

        salary = self.salary
        unlisted_funds_loss = self.unlisted_funds_loss
        listed_funds_and_shares_profit = self.listed_funds_and_shares_profit
        listed_funds_and_shares_loss = self.listed_funds_and_shares_loss

        extra_salary, extra_listed_funds_and_shares_profit, extra_unlisted_funds_loss = self.qualified_shares_income_taxation()

        unlisted_funds_loss += extra_unlisted_funds_loss
        salary += extra_salary
        listed_funds_and_shares_profit += extra_listed_funds_and_shares_profit

        extra_listed_funds_and_shares_profit, extra_listed_loss = self.unqualified_shares_income_taxation()
        listed_funds_and_shares_profit += extra_listed_funds_and_shares_profit
        listed_funds_and_shares_loss += extra_listed_loss

        # pdb.set_trace()
        if abs(listed_funds_and_shares_loss) > 1e-4:
            listed_loss = '%d' % listed_funds_and_shares_loss
        else:
            listed_loss = 'null'

        vinstSmahusBostadsratt = 'null'
        forlustSmahusBostadsratt = 'null'
        if abs(self.sale_private_property_profit) > 1e-4:
            vinstSmahusBostadsratt = '%d' % self.sale_private_property_profit

        if abs(self.sale_private_property_loss) > 1e-4:
            forlustSmahusBostadsratt = '%d' % self.sale_private_property_loss

        inkomstrantorSkatteavdrag = 'null'
        inkomstrantorUtanSkatteavdrag = 'null'
        if abs(self.int_inc_tax_withheld) > 1e-4:
            inkomstrantorSkatteavdrag = '%d' % self.int_inc_tax_withheld

        if abs(self.int_inc_tax_not_withheld) > 1e-4:
            inkomstrantorUtanSkatteavdrag = '%d' % self.int_inc_tax_not_withheld

        ranteutgifter = 'null'

        if abs(self.interest_expense) > 1e-4:
            ranteutgifter = '%d' % self.interest_expense

        forlustFondandelarInteMarknadsnoterade = 'null'
        if abs(unlisted_funds_loss) > 1e-4:
            forlustFondandelarInteMarknadsnoterade = '%d' % unlisted_funds_loss

        schablonintakt = 'null'
        if abs(self.standard_income) > 1e-4:
            schablonintakt = '%d' % self.standard_income

        forlustNaringsfastighetNaringsbostadsratt = 'null'
        if abs(self.sale_commercial_property_loss) > 1e-4:
            forlustNaringsfastighetNaringsbostadsratt = '%d' % self.sale_commercial_property_loss

        vinstNaringsfastighetNaringsbostadsratt = 'null'
        if abs(self.sale_commercial_property_profit) > 1e-4:
            vinstNaringsfastighetNaringsbostadsratt = '%d' % self.sale_commercial_property_profit

        investeraravdrag = 'null'
        if abs(self.investor_deduction) > 1e-4:
            investeraravdrag = '%d' % self.investor_deduction
        # if tax_year is None:
        #     tax_year = pd.to_datetime('today').year
        income_start = datetime.date(self.tax_year, 1, 1).strftime('%Y-%m-%d')
        income_end = datetime.date(self.tax_year, 12, 1).strftime('%Y-%m-%d')

        data['avdragKapital'] = {'forlustAktierFondandelarMarknadsnoterade': listed_loss,
                                 'forlustFondandelarInteMarknadsnoterade': forlustFondandelarInteMarknadsnoterade,
                                 'forlustNaringsfastighetNaringsbostadsratt': forlustNaringsfastighetNaringsbostadsratt,
                                 'forlustSmahusBostadsratt': forlustSmahusBostadsratt,
                                 'investeraravdrag': investeraravdrag,
                                 'ranteutgifter': ranteutgifter}
        data['avdragTjanst'] = {'dubbelBosattning': 'null', 'ovrigaUtgifterBrutto': 'null',
                                'resorTillOchFranArbetetBrutto': 'null', 'tjansteresor': 'null'}

        data['uppgifterAvlidnaSjomanUtInvandrade'] = {'avlidenAr': 'null',
                                                      'bosattManader': '12',
                                                      'dagarFjarrfart': 'null',
                                                      'dagarNarfart': 'null',
                                                      'folkbokford': 'true',
                                                      'inUtvandrad': 'false',
                                                      'invandradManad': 'null',
                                                      'utvandradManad': 'null'}

        data['utlandskForsakringAvkastningsskatt'] = {'skatteunderlagKapitalforsakring': 'null',
                                                      'skatteunderlagPensionsforsakring': 'null'}

        data['underlagSkattreduktion'] = {'fackforeningsavgift': 'null',
                                          'gava': 'null',
                                          'hasAvdragFromArbetsgivare': 'false',
                                          'preliminarFornybarElkWh': 'null',
                                          'regionalSkattereduktion': 'false',
                                          'rotarbeteFaktura': 'null',
                                          'rotarbeteForman': 'null',
                                          'rutarbeteFaktura': 'null',
                                          'rutarbeteForman': 'null'}

        data['tjansteinkomster'] = {'hittillsUnderAret': {'allmanPensionTjanstepensionHittills': 'null',
                                                          'avdragenSkatt': 'null',
                                                          'kostnadsersattningarHittills': 'null',
                                                          'loneinkomsterHittills': 'null',
                                                          'period': {'fromDate': income_start, 'tomDate': income_end},
                                                          'sjukAktivitetsersattningLonHittills': 'null',
                                                          'sjukAktivitetsersattningPensionHittills': 'null',
                                                          'sjukpenningAKassaMmHittills': 'null'},
                                    'restenAvAret': {'allmanPensionTjanstepensionResten': 'null',
                                                     'inkomstFrom': 'null',
                                                     'isHelar': 'true',
                                                     'kostnadsersattningarResten': 'null',
                                                     'loneinkomsterResten': '%d' % salary,
                                                     'period': {'fromDate': '2021-12-01', 'tomDate': '2021-12-01'},
                                                     'sjukAktivitetsersattningLonResten': 'null',
                                                     'sjukAktivitetsersattningPensionResten': 'null',
                                                     'sjukpenningAKassaMmResten': 'null'}}

        data['grunduppgifter'] = {'fodelsear': self.birth_year,
                                  'inkomstar': pd.to_datetime('today').year,
                                  'kommunkod': 'null',
                                  'showRegionalSkattereduktion': 'false',
                                  'skatteAvgiftssatser': {'avgiftssatsBegravningsavgift': 0.253,
                                                          'avgiftssatsKyrkoavgift': 'null',
                                                          'avgiftssatsSamfundsavgift': 'null',
                                                          'kommunalLandstingsSkattesats': 31.44,
                                                          'summaKommunalskattAvgifter': 'null'},
                                  'skattesatserTyp': '0'}
        data['inkomsterKapital'] = {'egenInbetalningSkatt': 'null',
                                    'inkomstUthyrningPrivatbostad': '%d' % self.rental_income,
                                    'inkomstrantor': 'null',
                                    'inkomstrantorSkatteavdrag': inkomstrantorSkatteavdrag,
                                    'inkomstrantorUtanSkatteavdrag': inkomstrantorUtanSkatteavdrag,
                                    'schablonintakt': schablonintakt,
                                    'trettioProcentAvInkomstrantorSkatteavdrag': 0,
                                    'vinstAktierFondandelarMarknadsnoterade': '%d' % listed_funds_and_shares_profit,
                                    'vinstFondandelarInteMarknadsnoterade': '%d' % self.unlisted_funds_profit,
                                    'vinstNaringsfastighetNaringsbostadsratt': vinstNaringsfastighetNaringsbostadsratt,
                                    'vinstSmahusBostadsratt': vinstSmahusBostadsratt}
        data['naringsfastigheter'] = {'underlagFastighetsavgift': {'fastighetsavgiftHalvHyreshus': 'null',
                                                                   'fastighetsavgiftHelHyreshus': 'null'},
                                      'underlagFastighetsskatt': {'fastighetsskattHyreshusTomt': 'null',
                                                                  'fastighetsskattIndustri': 'null',
                                                                  'fastighetsskattLokal': 'null',
                                                                  'fastighetsskattVatten': 'null',
                                                                  'fastighetsskattVind': 'null'}}

        data['naringsverksamhet'] = {'aktivNaringsverksamhet': {'overskottAktivNaringsverksamhet': 'null',
                                                                'sjukpenningAktivNaringsverksamhet': ''},
                                     'allmannaAvdragDto': {'allmantAvdragUnderskottNaringsverksamhet': 'null'},
                                     'arbetsgivareSocialaAvgifter': {'inkomsterIngaSocialaAvgifter': 'null',
                                                                     'kostnaderIngaSocialaAvgifter': 'null'},
                                     'avkastningsskattPensionskostnader': {'underlagAvkastningsskattPension': 'null'},
                                     'nedsattningEgenavgifter': {'regionaltNedsattningsbelopp': 'null'},
                                     'passivNaringsverksamhet': {'overskottPassivNaringsverksamhet': 'null'},
                                     'rantefordelning': {'negativRantefordelning': 'null',
                                                         'positivRantefordelning': 'null'},
                                     'sarskildLoneskattPensionskostnader': {'underlagPensionskostnaderAnstallda': 'null',
                                                                            'underlagPensionskostnaderEgen': 'null'},
                                     'underlagExpansionsfondsskatt': {'minskningExpansionsfond': 'null',
                                                                      'okningExpansionsfond': 'null',
                                                                      'underlagAterforingExpansionsfondsskatt': 'null'}}
        data['ovrigaTjansteinkomster'] = {'ersattningFranVinstandelsstiftelseMmUtanPgiOchJobbskatteavdrag': 'null',
                                          'hobbyinkomster': 'null',
                                          'inkomsterFranFamansbolag': 'null',
                                          'ovrigaInkomsterUtanPgi': 'null'}
        data['pensionsforhallandenKarensuppgifter'] = {'allmanPensionHelaAret': 'false',
                                                       'antalDagarKarensEnDag': 'null',
                                                       'antalDagarKarensFjortonDagar': 'null',
                                                       'antalDagarKarensNittioDagar': 'null',
                                                       'antalDagarKarensSextioDagar': 'null',
                                                       'antalDagarKarensSjuDagar': 365,
                                                       'antalDagarKarensTrettioDagar': 'null',
                                                       'helSjukAktivitetsersattning': 'false',
                                                       'manuelltUnderlagForSlfPaAktivNrvJanTillJun2019': 'null'}
        data['skattOvrigt'] = {'avrakningUtlandskSkatt': 'null',
                               'egenInbetalningSkatt': 'null',
                               'preliminarSkatt': 'null'}
        data['smahusAgarlagenhet'] = {'underlagFastighetsavgift': {'fastighetsavgiftHalvSmahus': 'null',
                                                                   'fastighetsavgiftHelSmahus': 'null',
                                                                   'skrivenHelaAret': 'null',
                                                                   'underlagSkattereduktionFastighetsavgiftHalvAvgift': '',
                                                                   'underlagSkattereduktionFastighetsavgiftHelAvgift': ''},
                                      'underlagFastighetsskatt': {'fastighetsskattSmahusTomt': 'null'}}
        return data

    def tax_breakdown(self):
        """
        simple breakdown of the components making up tax. so far quite unstructured
        """
        self.tax()
        raw = self.tax_breakdown_dict
        return pd.DataFrame(raw, index=['0'])

    def official_tax(self, session=None, refresh=False):
        """
        would be great if we didn't have to do the extra step here...
        """
        resp = self.parsed_official_response(refresh=refresh, session=session)
        slutlig_skatt = resp.query("item == 'slutligSkatt'").value.item()

        return slutlig_skatt - \
            self.parameter('statlig_skatt', 100) * self.int_inc_tax_withheld

    def parsed_official_response(self, refresh=False, session=None):
        """
        make a dataframe from the structure returned by the web query
        """

        resp = self.query_web_for_tax_results(refresh=refresh, session=session)
        raw = resp.json()
        if 'leafs' not in raw:
            # this means something is wrong
            # pdb.set_trace()
            raise Exception(raw['technicalMessage'])

        res = raw['leafs']
        # pdb.set_trace()
        out = [[res[6]['apiText'], res[6]['leafs'][0]
                ['apiText'], float(res[6]['leafs'][0]['value'])]]

        skiplist = [
            'skillnadPrognos',
            'skatteavdragRestenPerManadPrognosHelar']

        out.append([res[4]['apiText'], res[4]['leafs'][0]['apiText'],
                    float(res[4]['leafs'][0]['value'].replace(',', '.')) / 100])

        for ele in res[5]['leafs']:
            out.append(
                [res[5]['apiText'], ele['apiText'], float(ele['value'])])
        for ele in res[7]['leafs']:
            if ele['apiText'] not in skiplist:
                out.append(
                    [res[7]['apiText'], ele['apiText'], float(ele['value'])])

        return pd.DataFrame(out, columns=['group', 'item', 'value'])


text:
import argparse
import numpy as np
import sys
import tensorflow as tf

from CAModel import CAModel
from DataLoader import float_to_note
from MIDIConverter import midi_to_chroma, midi_to_piano_roll

parser = argparse.ArgumentParser('Train a model on a midi file')
parser.add_argument('-a', '--piano-roll', action='store_true', dest='piano_roll', default=False, help='Use piano roll instead of chromagraph')
parser.add_argument('-b', '--batch-size', type=int, dest='batch_size', default=8, help='Set batch size')
parser.add_argument('-c', '--chorale', type=int, dest='chorale', default=0, help='Which chorale to use as a model')
parser.add_argument('-e', '--epochs', type=int, dest='epochs', default=8000, help='Number of learning epochs')
parser.add_argument('-f', '--framerate', type=int, dest='framerate', default=20, help='Number of epochs between graph updates')
parser.add_argument('-g', '--graphing', action='store_true', dest='graphing', default=False, help='Print chorale and exit')
parser.add_argument('-i', '--filters', type=int, dest='filters', default=128, help='Number of convolutional filters')
parser.add_argument('-l', '--load-model', type=str, dest='model', default=None, help='Continue learning from existing weights')
parser.add_argument('-m', '--midi-file', type=str, dest='midi_file', default=None, help='MIDI file to process, will override chorale')
parser.add_argument('-n', '--name', type=str, dest='output_name', default='output', help='Name of the weight output file')
parser.add_argument('-p', '--past-notes', type=int, dest='past_notes', default=16, help='How far into the past to stretch the convolutional window')
parser.add_argument('-o', '--output-destination', type=str, dest='output_destination', default='./outputs', help='Folder to save figures')
parser.add_argument('-r', '--chroma-frequency', type=int, dest='chroma_frequency', default=4, help='MIDI to chroma sampling frequency')
parser.add_argument('-s', '--slurm', action='store_true', dest='slurm', default=False, help='Just the learning')
parser.add_argument('-t', '--testing', type=int, default=None, dest='testing', help='How many rounds to test the model')
parser.add_argument('-w', '--width', type=int, dest='width', default=1, help='The width of the convolutional window, how many other notes the model can see')
args = parser.parse_args()

if not args.slurm:
    import ffmpeg
    import matplotlib.pyplot as plt

if args.midi_file is None:
    from DataLoader import list_chorales
    chorale = list_chorales()[args.chorale]
    note_chorale = [float_to_note(i) for i in chorale]
    notes = range(len(chorale))
    chorale = np.array(chorale).reshape((1, -1))
else:
    chorale = midi_to_piano_roll(args.midi_file, fs=args.chroma_frequency) if args.piano_roll else midi_to_chroma(args.midi_file, fs=args.chroma_frequency)
    note_chorale = (chorale - np.min(chorale))/(np.max(chorale) - np.min(chorale))
    notes = range(note_chorale.shape[1])

if args.graphing:
    import matplotlib.pyplot as plt
    if note_chorale is list:
        plt.plot(notes, note_chorale)
        plt.title(f'Chorale {args.chorale}')
        plt.ylim(55, 80)
        plt.xlabel('Time step')
        plt.ylabel('Note')
    else:
        fig, axs = plt.subplots(3, 4, sharex=True, sharey=True)
        for i, a in enumerate(np.asarray(axs).flatten()):
            a.plot(notes, note_chorale[i].tolist())
        fig.suptitle(args.midi_file)
        fig.text(0.5, 0.04, 'Time step', ha='center')
        fig.text(0.04, 0.5, 'Note', va='center', rotation='vertical')
    plt.show()
    sys.exit('Graphing complete! Exiting..')

target = tf.pad(np.array(note_chorale).astype('float32'), [(0, 0), (args.past_notes - 1, 0)])
seed = np.zeros([target.shape[0],target.shape[1],args.past_notes + 1], np.float32)
seed[:, args.past_notes-1, -1] = note_chorale[:, 0]

def loss_f(x): return tf.reduce_mean(tf.square(x[..., -1] - target))
def scale(x): return x if args.midi_file is not None else float_to_note(x)

ca = CAModel(past_notes=args.past_notes, width=args.width, filters=args.filters, piano_roll=args.piano_roll)
if not args.model is None:
    ca.load_weights(args.model)

loss_log = []

if not args.testing is None:
    from tqdm import tqdm
    import matplotlib.pyplot as plt
    x0 = np.repeat(seed[None, ...], args.batch_size, 0)
    for i in tqdm(range(args.testing)):
        x0 = ca(x0)
        loss_log.append(np.log10(tf.reduce_mean(loss_f(x0))))
    plt.plot(loss_log)
    plt.show()
    sys.exit('Testing complete')

lr = 2e-3
lr_sched = tf.keras.optimizers.schedules.PiecewiseConstantDecay([2000], [lr, lr * 0.1])
trainer = tf.keras.optimizers.Adam(lr_sched)

loss0 = loss_f(seed).numpy()

@tf.function
def train_step(x):
    iter_n = tf.random.uniform([], 64, 96, tf.int32)
    with tf.GradientTape() as g:
        for i in tf.range(iter_n):
            x = ca(x)
        loss = tf.reduce_mean(loss_f(x))
    grads = g.gradient(loss, ca.weights, unconnected_gradients='zero')
    grads = [g/(tf.norm(g)+1e-8) for g in grads]
    trainer.apply_gradients(zip(grads, ca.weights))
    return x, loss

if not args.slurm:
    plt.ion()
    if args.piano_roll:
        fig, axs = plt.subplots(2, 1)
        axs[0].imshow(target, aspect='auto')
        axs[1].imshow(tf.reduce_mean(np.repeat(seed[None, ...], args.batch_size, 0)[..., -1], 0), aspect='auto')
    else:
        lines = []
        plt.rcParams['axes.grid'] = True
        music_graphs = 1 if args.midi_file is None else 12
        batch_graphs = args.batch_size if args.midi_file is None else 0
        total_graphs = music_graphs + batch_graphs
        root = np.sqrt(total_graphs)
        rows = np.floor(root)
        cols = np.ceil(root)
        if rows * cols < total_graphs:
            rows += 1
        fig, axs = plt.subplots(int(rows), int(cols), sharex=True, sharey=True)
        for i, a in enumerate(np.asarray(axs).flatten()):
            if i < music_graphs:
                a.set_title(f'Music Channel {i + 1}')
                a.plot(notes, note_chorale[i])
                lines.append(a.plot(notes, [0] * max(chorale.shape))[0])
            elif i < total_graphs:
                a.set_title(f'Batch {i - music_graphs + 1}')
                a.plot(notes, np.mean(note_chorale, axis=0))
                lines.append(a.plot(notes, [0] * max(chorale.shape))[0])
            else:
                fig.delaxes(a)
    fig.suptitle('Epoch 0')
    fig.text(0.5, 0.04, 'Time step', ha='center')
    fig.text(0.04, 0.5, 'Note', va='center', rotation='vertical')

    mgr = plt.get_current_fig_manager().window.state('zoomed')
    plt.show()

framenum = 0

for i in range(1, args.epochs + 1):
    x0 = np.repeat(seed[None, ...], args.batch_size, 0)
    x, loss = train_step(x0)

    step_i = len(loss_log)
    loss_log.append(loss.numpy())

    print('\r step: %d, log10(loss): %.3f'%(i+1, np.log10(loss)), end='')

    if not args.slurm and step_i % args.framerate == 0:
        xn = x.numpy()
        if args.piano_roll:
            axs[1].imshow(tf.reduce_mean(xn[..., -1], 0), aspect='auto')
        else:
            for j in range(music_graphs):
                lines[j].set_ydata([scale(k) for k in np.mean(xn, axis=0)[j, :, -1].flatten().tolist()[args.past_notes - 1:]])
            for j in range(batch_graphs):
                lines[music_graphs + j].set_ydata([scale(k) for k in np.mean(xn, axis=1)[j, :, -1].flatten().tolist()[args.past_notes - 1:]])
        fig.suptitle(f'Epoch {i - 1}')
        plt.gcf().canvas.draw()
        plt.gcf().canvas.flush_events()
        plt.savefig(f'{args.output_destination}/frame-{str(framenum).zfill(5)}.jpg')
        framenum += 1

ca.save_weights(args.output_name or 'weights', overwrite=True)

if not args.slurm:
    #ffmpeg.input('/outputs/*.jpg', framerate=25).output('output.gif').run()
    input('\nPress ENTER to exit')

import json
with open((args.model or '') + '-loss.json', 'w') as filename:
    json.dump(np.array(loss_log).tolist(), filename)

max_stars_repo_name: royJackman/SineCircleMappings

max_issues_repo_name: royJackman/SineCircleMappings

max_forks_repo_name: royJackman/SineCircleMappings


text:
import argparse
import gc
import os

import numpy as np
import pandas as pd

parser = argparse.ArgumentParser(description='Make phenotype folder ready for HASE analysis')
parser.add_argument("-i", required=True, type=str, help="path to nparrays")
parser.add_argument("-id", type=str, help="csv data frame with two columns:"
                                          "1)id: ids identical to ids in genotype data"
                                          "2) exclude: 1 if exclude, 0 if include to analysis")

args = parser.parse_args()
print(args)

df = pd.read_csv(args.id, index_col=0)

print(df.head())
subject_id = []

for i in df.iterrows():
    if i[1].exclude == 1:
        subject_id.append('remove_' + str(i[1].id))
    else:
        subject_id.append(str(i[1].id))

dic = {}
dic['id'] = subject_id

files = os.listdir(args.i)

for f in files:
    print(f)
    if f.split('.')[-1] != 'npy':
        raise ValueError('In {} should be only nparrays, not {}'.format(args.i, f))

    d = np.load(os.path.join(args.i, f))
    ch = f.split('_')[0].split('reg')[1]
    index = f.split('_')[1].split('.npy')[0]
    n, m = d.shape
    if n != len(subject_id):
        raise ValueError(
            'Number of ids {} from data frame not equal to number of ids {} from file {} '.format(len(subject_id), n,
                                                                                f))
    dic[f] = [ch + "_" + index + "_" + str(i) for i in range(m)]
    d = None
    gc.collect()

np.save(os.path.join(args.i, 'info_dic.npy'), dic)

max_stars_repo_name: reneevdw/GenNet

max_issues_repo_name: reneevdw/GenNet

max_forks_repo_name: reneevdw/GenNet


text:
import numpy as np

## Here is a training function
def train(model,  X_train, y_train, X_valid, y_valid,
          num_epoch=20, batch_size=400, learning_rate=5e-4, learning_decay=0.95, optim='SGD', momentum=None, verbose=False):
    '''
    This function is for training
    Batch size is set to 400, learning rate to 0.00005.

    Inputs:
    - model: a neural network class object
    - X_train: (float32) input data, a tensor with shape (N, D1, D2, ...)
    - y_train: (int) label data for classification, a 1D array of length N
    - X_valid: (float32) input data, a tensor with shape (num_valid, D1, D2, ...)
    - y_valid: (int) label data for classification, a 1D array of length num_valid
    - num_epoch: (int) the number of training epochs
    - batch_size: (int) the size of a single batch for training
    - learning_rate: (float)
    - learning_decay: (float) reduce learning rate every epoch
    - verbose: (boolean) whether report training process
    '''
    num_train = X_train.shape[0]
    num_batch = num_train//batch_size
    print('number of batches for training: {}'.format(num_batch))
    train_acc_hist = []
    val_acc_hist = []
    for e in range(num_epoch):
        # Train stage
        for i in range(num_batch):
            ## Order selection
            #X_batch = X_train[i*batch_size:(i+1)*batch_size]
            #y_batch = y_train[i*batch_size:(i+1)*batch_size]
            ## Random selection
            sample_idxs = np.random.choice(num_train, batch_size)
            X_batch = X_train[sample_idxs,:]
            y_batch = y_train[sample_idxs]
            ## loss
            loss = model.loss(X_batch, y_batch)
            # update model
            if optim == 'SGD':
                model.step(learning_rate=learning_rate)
            else:
                model.step(learning_rate=learning_rate, optim=optim, momentum=momentum)

            if verbose and (i+1)%10==0:
                print('{}/{} loss: {}'.format(batch_size*(i+1), num_train, loss))

        # Validation stage
        sample_idxs = np.random.choice(num_train, 1000)
        train_acc = model.check_accuracy(X_train[sample_idxs,:], y_train[sample_idxs])
        train_acc_hist.append(train_acc)

        val_acc = model.check_accuracy(X_valid, y_valid)
        val_acc_hist.append(val_acc)
        # Shrink learning_rate
        learning_rate *= learning_decay
        print('epoch {}: valid acc = {}, new learning rate = {}'.format(e+1, val_acc, learning_rate))

    # Return Loss history
    return train_acc_hist, val_acc_hist

def test(model, X_test, y_test):
    """Test function for model accuracy"""
    test_acc = model.check_accuracy(X_test, y_test)
    print('test acc: {}'.format(test_acc))
    return test_acc


text:
#!/usr/bin/env python

import matplotlib
matplotlib.use('Agg') # Must be before importing matplotlib.pyplot or pylab!

import sys
import seaborn as sns
from pandas import read_pickle, qcut
from itertools import product
import matplotlib.pyplot as plt
from pandas import get_dummies
from pandas import groupby
import numpy as np
from djeval import *

sns.set_palette("deep", desat=.6)
sns.set_context(rc={"figure.figsize": (8, 4)})

def percentile_subset(x, min_pct, max_pct):
    min_value, max_value = np.percentile(x, [min_pct, max_pct])
    return x[(x >= min_value) & (x <= max_value)]

msg("Hello there, reading yy_df.")
yy_df = read_pickle(sys.argv[1])

yy_df['rfr_residual'] = (yy_df['rfr_prediction'] - yy_df['elo'])

with_elo = yy_df[yy_df['elo'].notnull()]

features = list(yy_df.columns.values)

# blunderrate graph looks terrible because its bin edges are: [ 0.        ,  0.        ,  0.        ,  0.        ,  0.01960784, 0.03125   ,  0.04651163,  0.06666667,  0.09090909,  0.12820513,        0.83333333]

plottables = ['elo', 'rfr_prediction', 'rfr_residual']
plottables = ['elo']
#features = ['opening_feature']

# this wasnt working for some reason
make_pairplot = False
if make_pairplot:
    g = sns.pairplot(with_elo[plottables], size=2.5)
    plt.savefig('/data/pairplot.png')
    plt.close('all')

for a, b in product(features, plottables):
    msg('Making %s %s' % (a, b))
    x = with_elo[a]
    y = with_elo[b]
    msg('type = %s' % x.dtype)
    if x.dtype == 'object':
        plt.figure()
        x.value_counts().plot(kind='bar')
        plt.savefig('/data/' + a + '_hist.png')
        plt.close('all')
    else:
        try:
            xlim = tuple(np.percentile(x, [1,99]))
            ylim = tuple(np.percentile(y, [1,99]))
            with sns.axes_style("white"):
                sns.jointplot(x, y, kind="hex", xlim=xlim, ylim=ylim)
            plt.savefig('/data/scatter_' + a + '_' + b + '.png')
            plt.close('all')
        except:
    #        sns.violinplot(x, y)
    #        plt.savefig('/data/' + a + '_' + b + '.png')
    #        plt.close()
            plt.figure()
            x.plot(kind='hist')
            plt.savefig('/data/' + a + '_hist.png')
            plt.close('all')

do_indivs = True
if do_indivs:
    for a, b in product(features, plottables):
        msg('Making %s %s' % (a, b))
        try:
            plt.figure()
            groupings, bins = qcut(with_elo[a], 10, labels=False, retbins=True)
            sns.violinplot(with_elo[b], groupings)
            plt.savefig('/data/' + a + '_' + b + '.png')
            plt.close('all')
        except:
            try:
                plt.figure()
                sns.violinplot(with_elo[b], with_elo[a])
                plt.savefig('/data/' + a + '_' + b + '.png')
                plt.close('all')
            except:
                msg("Couldnt manage for %s %s" % (a, b))

    #        f, ax = plt.subplots(figsize=(11, 6))
    #        sns.violinplot(with_elo[second], groupings, names=[str(b) + str(b+1) for b in bins[:-1]])
    #        ax.set(ylim=(-.7, 1.05))
    #        sns.despine(left=True, bottom=True)

max_stars_repo_name: Keesiu/meta-kaggle

max_issues_repo_name: Keesiu/meta-kaggle

max_forks_repo_name: Keesiu/meta-kaggle


text:
# =============================================================================
# Fog Graph Projection Unit Tests
# =============================================================================
import pytest
from pytest import approx

import networkx as nx
from test.clustering.utils import Clusters
from fog.graph.projection import monopartite_projection, MONOPARTITE_PROJECTION_METRICS
from fog.metrics import (
    sparse_cosine_similarity,
    sparse_binary_cosine_similarity,
    jaccard_similarity,
    overlap_coefficient,
    dice_coefficient
)

NODES = [
    ('John', 'people'),
    ('Mary', 'people'),
    ('Lucy', 'people'),
    ('Gabriel', 'people'),
    ('Meredith', 'people'),
    ('red', 'color'),
    ('blue', 'color'),
    ('yellow', 'color'),
    ('purple', 'color'),
    ('orange', 'color')
]

PEOPLE_PART = set([n[0] for n in NODES if n[1] == 'people'])

EDGES = [
    ('John', 'red', 1.0),
    ('John', 'purple', 3.0),
    ('Gabriel', 'yellow', 2.0),
    ('Gabriel', 'orange', 4.0),
    ('Mary', 'red', 2.0),
    ('Mary', 'yellow', 1.0),
    ('Lucy', 'red', 7.0),
    ('Lucy', 'purple', 1.0),
]

BIPARTITE = nx.Graph()

for key, part in NODES:
    BIPARTITE.add_node(key, part=part)

BIPARTITE.add_weighted_edges_from(EDGES)

def to_vector(g, u):
    return {n: w for _, n, w in g.edges(u, data='weight')}

class TestGraphProjection(object):
    def test_invalid_arguments(self):
        with pytest.raises(TypeError):
            monopartite_projection(BIPARTITE, 'people', part='part', metric='unknown')

        invalid_bipartite = BIPARTITE.copy()
        invalid_bipartite.add_edge("red", "purple")

        with pytest.raises(TypeError):
            monopartite_projection(invalid_bipartite, 'people', part='part', metric='unknown')

        invalid_bipartite = BIPARTITE.copy()
        invalid_bipartite.add_edge("John", "Lucy")

        with pytest.raises(TypeError):
            monopartite_projection(invalid_bipartite, 'people', part='part', metric='unknown')

    def test_basics(self):
        mono = monopartite_projection(BIPARTITE, 'people', part='part')

        assert set(mono.nodes) == PEOPLE_PART

        assert Clusters(mono.edges(data='weight')) == Clusters([
            ('John', 'Mary', 1),
            ('John', 'Lucy', 2),
            ('Mary', 'Lucy', 1),
            ('Mary', 'Gabriel', 1)
        ])

    def test_cosine(self):
        mono = monopartite_projection(BIPARTITE, 'people', part='part', metric='cosine')

        assert set(mono.nodes) == PEOPLE_PART

        for u, v, c in mono.edges(data='weight'):
            u = to_vector(BIPARTITE, u)
            v = to_vector(BIPARTITE, v)

            assert c == approx(sparse_cosine_similarity(u, v))

    def test_cosine_threshold(self):
        mono = monopartite_projection(BIPARTITE, 'people', part='part', metric='cosine', threshold=0.3)

        assert set(mono.nodes) == PEOPLE_PART

        assert Clusters(mono.edges) == Clusters([('John', 'Lucy'), ('Mary', 'Lucy')])

        for _, _, c in mono.edges(data='weight'):
            assert c >= 0.3

    def test_binary_cosine(self):
        mono = monopartite_projection(BIPARTITE, 'people', part='part', metric='binary_cosine')

        assert set(mono.nodes) == PEOPLE_PART

        for u, v, c in mono.edges(data='weight'):
            u = to_vector(BIPARTITE, u)
            v = to_vector(BIPARTITE, v)

            assert c == approx(sparse_binary_cosine_similarity(u, v))

    def test_jaccard(self):
        mono = monopartite_projection(BIPARTITE, 'people', part='part', metric='jaccard')

        assert set(mono.nodes) == PEOPLE_PART

        for u, v, c in mono.edges(data='weight'):
            u = to_vector(BIPARTITE, u)
            v = to_vector(BIPARTITE, v)

            assert c == approx(jaccard_similarity(u, v))

    def test_overlap(self):
        mono = monopartite_projection(BIPARTITE, 'people', part='part', metric='overlap')

        assert set(mono.nodes) == PEOPLE_PART

        assert mono['John']['Lucy']['weight'] == 1.0

        for u, v, c in mono.edges(data='weight'):
            u = to_vector(BIPARTITE, u)
            v = to_vector(BIPARTITE, v)

            assert c == approx(overlap_coefficient(u, v))

    def test_dice(self):
        mono = monopartite_projection(BIPARTITE, 'people', part='part', metric='dice')

        assert set(mono.nodes) == PEOPLE_PART

        for u, v, c in mono.edges(data='weight'):
            u = to_vector(BIPARTITE, u)
            v = to_vector(BIPARTITE, v)

            assert c == approx(dice_coefficient(u, v))

    def test_not_use_topology(self):

        for metric in [None, *MONOPARTITE_PROJECTION_METRICS]:
            mono1 = monopartite_projection(BIPARTITE, 'people', part='part', metric=metric)
            mono2 = monopartite_projection(BIPARTITE, 'people', part='part', metric=metric, use_topology=False)

            assert mono1.order() == mono2.order()
            assert mono1.size() == mono2.size()

            for u, v, w in mono1.edges(data='weight'):
                assert mono2.has_edge(u, v)
                assert mono2[u][v]['weight'] == approx(w), (u, v, w)

max_stars_repo_name: Yomguithereal/fog

max_issues_repo_name: Yomguithereal/fog

max_forks_repo_name: Yomguithereal/fog

reason: import networkx


text:
''' PolynomialFiltering.filters.ManagedScalarRecursiveFilter
 (C) Copyright 2019 - Blue Lightning Development, LLC.
 D. F. Linton. support@BlueLightningDevelopment.com

 SPDX-License-Identifier: MIT
 See separate LICENSE file for full text
'''

from abc import abstractmethod
from numpy import zeros
from numpy import array as vector
from polynomialfiltering.PythonUtilities import virtual;
from polynomialfiltering.AbstractComponentFilter import AbstractComponentFilter
from polynomialfiltering.filters.ManagedFilterBase import ManagedFilterBase;
from polynomialfiltering.filters.controls.judge.BaseScalarJudge import BaseScalarJudge
from polynomialfiltering.filters.controls.monitor.NullMonitor import NullMonitor

class ManagedScalarRecursiveFilter(ManagedFilterBase):

    def __init__(self, worker : AbstractComponentFilter):
        super().__init__(worker);
        self.setJudge( BaseScalarJudge(self, 0.0 ) );  # default is base judge with no smoothing
        self.setMonitor( NullMonitor() );              # default is do-nothing monitor
        self.iR = zeros([1,1]);

    @virtual
    def addaddObservation(self, t:float, y:vector) -> bool:
        self.iR = self.errorModel.getPrecisionMatrix(self, t, y)
        Zstar = self.worker.predict(t)
        e = y[0] - Zstar[0]
        if (self.judge.scalarUpdate(t, y, e, self.iR)) :
            innovation = self.worker.update(t, Zstar, e)
            self.monitor.accepted(t, y, innovation );
            return True;
        else :
            self.monitor.rejected(t, y );
            return False;

    @virtual
    def getCovariance(self):
        return self.worker.getVRF(self) * 1/self.iR[0,0]

max_stars_repo_name: lintondf/MorrisonPolynomialFiltering

max_issues_repo_name: lintondf/MorrisonPolynomialFiltering

max_forks_repo_name: lintondf/MorrisonPolynomialFiltering

reason: from numpy