"""Code for TCLR method."""

from os.path import isfile

import torch
import torch.nn as nn
import torchvision.models.video as models

from slowfast.models import head_helper
from slowfast.models.r2plus1d import video_resnet

class TCLR(nn.Module):
    """TCLR"""

    def __init__(self, cfg):
        """
        The `__init__` method of any subclass should also contain these
            arguments.

        Args:
            cfg (CfgNode): model building configs, details are in the
                comments of the config file.
        """
        super(TCLR, self).__init__()
        self.num_pathways = 1
        self._construct_network(cfg)

    def _construct_network(self, cfg):
        """
        TCLR network.

        Args:
            cfg (CfgNode): model building configs, details are in the
                comments of the config file.
        """

        # define the encoder
        pretrained = cfg.MODEL.PRETRAINED
        self.encoder = video_resnet.__dict__[cfg.MODEL.ARCH](pretrained=pretrained)
        self.encoder.layer4[0].conv1[0] = nn.Conv3d(256, 512, kernel_size=(3, 3, 3),\
                                    stride=(1, 2, 2), padding=(2, 1, 1),dilation = (2,1,1), bias=False)
        self.encoder.layer4[0].downsample[0] = nn.Conv3d(256, 512,\
                            kernel_size = (1, 1, 1), stride = (1, 2, 2), bias=False)

        self.init_weights_from_checkpoint(cfg.MODEL.CKPT)

        # add K = 2 heads (noun and verb prediction)
        # temporary hardcoding
        # this needs (8, 7, 7) pooling instead of (4, 7, 7) due to the change in the last conv layer
        pool_size=[
            [8, 7, 7]
        ]

        # temporary hardcoding
        self.head = head_helper.ResNetBasicHead(
            dim_in=[512],
            num_classes=cfg.MODEL.NUM_CLASSES,
            pool_size=pool_size,
            dropout_rate=cfg.MODEL.DROPOUT_RATE,
        )

    def forward(self, x):

        for pathway in range(self.num_pathways):
            x[pathway] = self.encoder(x[pathway])

        x = self.head(x)
        return x

    def init_weights_from_checkpoint(self, checkpoint_file):
        """Inits weights from given checkpoint."""
        ckpt = torch.load(checkpoint_file, map_location="cpu")
        csd = ckpt["state_dict"]

        csd_items = csd.items()
        csd_subset = dict()
        for layer_name, weights in csd_items:
            if 'module.1.' in layer_name:
                continue
            elif '1.' == layer_name[:2]:
                continue
            if 'module.0.' in layer_name:
                layer_name = layer_name.replace('module.0.','')
            if 'module.' in layer_name:
                layer_name = layer_name.replace('module.','')
            elif '0.' == layer_name[:2]:
                layer_name = layer_name[2:]
            if 'fc' in layer_name:
                continue
            csd_subset[layer_name] = weights

        msg = self.encoder.load_state_dict(csd_subset, strict=False)

        print(":::::::::::::::::::::: Loaded pretrained checkpoint from ::::::::::::::::::::::")
        print(f"Path:\t {checkpoint_file}")
        print(f"Message:\t {msg}")

    def freeze_fn(self, freeze_mode):

        if freeze_mode == 'bn_parameters':
            print("Freezing all BN layers\' parameters.")
            for m in self.modules():
                if isinstance(m, nn.BatchNorm3d):
                    # shutdown parameters update in frozen mode
                    m.weight.requires_grad_(False)
                    m.bias.requires_grad_(False)
        elif freeze_mode == 'bn_statistics':
            print("Freezing all BN layers\' statistics.")
            for m in self.modules():
                if isinstance(m, nn.BatchNorm3d):
                    # shutdown running statistics update in frozen mode
                    m.eval()

if __name__ == "__main__":
    from tools.run_net import parse_args, load_config

    # load cfg
    args = parse_args()
    args.cfg_file = "../../../configs/EPIC-KITCHENS/TCLR/tclr_32x112x112_R18_K400_LR0.0025.yaml"
    cfg = load_config(args)

    # load model
    model = TCLR(cfg)

    # test with sample inputs
    x = torch.randn(1, 3, 32, 112, 112)
    y = model([x])

    assert y[0].shape == (1, 97)
    assert y[1].shape == (1, 300)


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from n_pnd import *

n_two = 25
N = [n_two, n_two] # number of linked pendulums
SLOW_MOTION = 1 # slow motion factor in animation
dt = 0.001 # infinitesimal time increment in seconds
T = 35 # length of time to be simulated
L = 50.0
G = 9.8/L # effective gravity
coef = setup_coef(N)

# initial conditions
theta0 = []
omega0 = []
for i in range(len(N)):
    theta0.append(np.tile(2.0, N[i])) # counterclockwise angle relative to the vertical for each rod
    omega0.append(np.tile(0.0, N[i])) # counterclockwise angular velocity for each rod
theta0[1] += 1.0e-15

print("INITIAL THETA")
print(theta0[0])
print(theta0[1])

coef = setup_coef(N)

fig, ax = plt.subplots()
time_text = ax.text(-0.9*max(N), 0.9*max(N), "t=0")
data = []
init_data = [np.copy(theta0), np.copy(omega0)]
for i in range(len(N)):
    data.append(evolve_pnd(init_data[0][i], init_data[1][i], T, dt, G, coef))
lines = [ax.plot(data[i][0][0], data[i][1][0])[0] for i in range(len(data))]
draw = lines + [time_text]
nframes = int(len(data[0][0])*dt*SLOW_MOTION/0.02-1)
def animate(i):
    for j in range(len(lines)):
        lines[j].set_xdata(data[j][0][int(i*0.02/dt/SLOW_MOTION)])
        lines[j].set_ydata(data[j][1][int(i*0.02/dt/SLOW_MOTION)])
    time_text.set_text("t="+str(int(i*20)/SLOW_MOTION/1000.0))
    return draw
ani = animation.FuncAnimation(fig, animate, frames=nframes, interval=20, save_count=nframes)
plt.xlim([-1.1, 1.1])
plt.ylim([-1.1, 1.1])

if len(N) >= 2:
    fig2, ax2 = plt.subplots()
    print("Calculating Lyapunov")
    print(theta0)
    data_t_0 = [data[0][2], data[0][3]]
    data_t_1 = [data[1][2], data[1][3]]
    lyap = lyapunov(data_t_0, data_t_1, dt)
    print(lyap[1].shape)
    print(lyap[2])
    ax2.plot(lyap[0], lyap[1])
    fig3, ax3 = plt.subplots()
    ax3.plot(np.append(np.tile(0, int(T/dt)-lyap[0].size), lyap[0]), np.log(np.abs(np.asarray(data[0][2]).T[0] - np.asarray(data[1][2]).T[0])))

plt.show()


# Make matplotlib static. Use notebook instead of inline to make interactive
get_ipython().run_line_magic('matplotlib', 'inline')

# Import required libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# Load part of the dataset to have a look
df = pd.read_csv('kickstarter-projects/ks-projects-201801.csv', nrows=10)

# Let's have a look
df.head()

# We see that there are two date/time columns, we'll tell pandas to parse them when loading the full file.
df = pd.read_csv('kickstarter-projects/ks-projects-201801.csv',
                 parse_dates=['deadline', 'launched'],
                 encoding = "ISO-8859-1")

# Get some general info on the dataset.
# Int is for integer, float for floating point (number with decimal), object for string (text) or mixed types.
df.info()

# Describe is the basic summary stats function for numerical values.
df.describe()

stats_cols = ['backers', 'usd_pledged_real', 'usd_goal_real']
df[stats_cols].describe()

df.columns

df.main_category.unique()

df.state.unique()

df.groupby('main_category')['ID'].count()

df.groupby('state')['ID'].count()

# It would be nicer if those states were capitalized. We can easily do that
df['state'] = df['state'].str.capitalize()
df.groupby('state')['ID'].count()

# We'll also rename some of the columns so that the output is cleaner.
df.columns = ['ID', 'name', 'category', 'Main category', 'currency', 'deadline',
              'goal', 'launched', 'pledged', 'State', 'Backers', 'country',
              'usd pledged', 'Pledged (USD)', 'Goal (USD)']

# Let's add a column that compute the funding percentage
df['Funding %'] = df['pledged'] / df['goal']

stats_cols = ['Backers', 'Pledged (USD)', 'Goal (USD)', 'Funding %']
desc_stats = df[df.State.isin(['Successful', 'Failed'])].groupby('State')[stats_cols].describe()
desc_stats

desc_stats.transpose()

desc_stats.transpose().unstack(level=0)

# This here is what we want!
desc_stats = desc_stats.transpose().unstack(level=0).transpose()
desc_stats

# We still need to rename the columns
desc_stats.columns = ['Count', 'Mean', 'Std. Dev.', 'Min.', '25th Pct.', 'Median', '75th Pct.', 'Max']
desc_stats

# We can export that to Excel
desc_stats.to_excel('DescStats_v1.xlsx')

# Create a Pandas Excel writer using XlsxWriter as the engine.
writer = pd.ExcelWriter('DescStats_v2.xlsx', engine='xlsxwriter')
desc_stats.to_excel(writer, sheet_name='Sheet1')

# Get the xlsxwriter objects from the dataframe writer object.
workbook  = writer.book
worksheet = writer.sheets['Sheet1']

# Add some cell formats.
format1 = workbook.add_format({'num_format': '#,##0.00'})
format2 = workbook.add_format({'num_format': '#,##0'})

# Set the column width.
worksheet.set_column('B:B', 18, None)

# Set the column format.
worksheet.set_column('D:D', None, format1)
worksheet.set_column('F:I', None, format1)
worksheet.set_column('C:C', None, format2)

# Set the column width and format.
worksheet.set_column('E:E', 12, format1)
worksheet.set_column('J:J', 14, format1)

# Close the Pandas Excel writer and output the Excel file.
writer.save()

# First, create new columns with the year of each date columns
df['launched_year'] = df['launched'].dt.year
df['deadline_year'] = df['deadline'].dt.year

# Next, compute the duration of the campain, in weeks
df['duration'] = np.round((df['deadline'] - df['launched']).dt.days / 7)

df['duration'].describe()

pd.crosstab(df.launched_year, df.State)

# We can safely drop all events with a 1970 start date.
df = df[df.launched_year != 1970]

pd.crosstab(df.launched_year, df.State)

# Say we only want to compare fails and success, we can create a separate view of the dataset that
# contains only those states
df_result = df[df.State.isin(['Failed', 'Successful'])]
pd.crosstab(df_result.launched_year, df_result.State)

# Let's plot this instead
pd.crosstab(df_result.launched_year, df_result.State).plot()

# Ok, but maybe a bar plot would be better.
pd.crosstab(df_result.launched_year, df_result.State).plot.bar()

# Some additional imports - usually you would put these at the top of the notebook.
from matplotlib.ticker import FuncFormatter

# First change, let's capture the axis and set colors and hatches
# For the list of xkcd colors see https://xkcd.com/color/rgb/
ax = pd.crosstab(df_result.launched_year, df_result.State).plot.bar(
    color=['xkcd:dark gray', 'xkcd:light blue'])

# Set the label formatter of the y-axis to have thousand commas.
ax.yaxis.set_major_formatter(FuncFormatter(lambda x, p: format(int(x), ',')))

# Rotate the year labels
ax.xaxis.set_tick_params(rotation=0)

# Add some labels
ax.set_xlabel('Year', fontsize=13)
ax.set_ylabel('Number of projects', fontsize=13)
# Add a title
ax.set_title('Number of failed and successful projects per year', fontsize=14)

# Set the location of the legend
ax.legend(loc='upper left')

# Tell matplotlib to make it look sharp
plt.tight_layout()

# Export to pdf
plt.savefig('BarPlot_States.pdf')


"""Config Dictionary Dumpers."""

import json

from .__optional_imports import try_import
from .__typing import ConfigDict, FilePath

def json_dumper(config: ConfigDict, location: FilePath) -> None:
    """Dump config to JSON file.

    Args:
        config (ConfigDict): configuration
        location (FilePath): file to write
    """
    with open(location, "w", encoding="utf-8") as _file:
        json.dump(config, _file)

def yaml_dumper(config: ConfigDict, location: FilePath) -> None:
    """Dump config to YAML file.

    Args:
        config (ConfigDict): configuration
        location (FilePath): file to write

    Raises:
        ModuleNotFoundError: pyyaml is required
    """

    yaml = try_import("yaml")

    if yaml is None:  # pragma: no cover
        message = "Please install the pyyaml library."
        raise ModuleNotFoundError(message)

    with open(location, "w", encoding="utf-8") as _file:
        # NOTE: we must convert config from OrderedDict to dict because
        # pyyaml can't load OrderedDict for python <= 3.8
        yaml.dump(dict(config), _file)

def toml_dumper(config: ConfigDict, location: FilePath) -> None:
    """Dump config to TOML file.

    Args:
        config (ConfigDict): configuration
        location (FilePath): file to write

    Raises:
        ModuleNotFoundError: toml library is required for writing files
    """

    toml = try_import("toml")

    if toml is None:  # pragma: no cover
        message = "Please install the toml library to write TOML files."
        raise ModuleNotFoundError(message)

    with open(location, "w", encoding="utf-8") as _file:
        toml.dump(config, _file)  # type: ignore


from .abstract_model import AbstractModel
import numpy as np
import scipy as sp
from warnings import warn

class CoxModel(AbstractModel):

    def __init__(self, event_time, censoring_time, design):
        """

        Parameters
        ----------
        event_time : numpy array
            The lowest values indicate the earliest events. float('inf')
            indicates right-censoring.
        censoring_time : numpy array
            float('inf') indicates uncensored observations.
        """

        if np.any(event_time[:-1] > event_time[1:]):
            raise ValueError(
                "The observations need to be sorted so that the event times are "
                "in the increasing order, from the earliest to last events."
            )

        if np.any(censoring_time[:-1] < censoring_time[1:]):
            raise ValueError(
                "The observations need to be sorted so that the censoring times "
                "are in the decreasing order, from uncensored, last censored, "
                "to the earliest censored."
            )

        n_event = len(event_time) - np.sum(np.isinf(event_time))
        risk_set_start_index, risk_set_end_index = \
            self._find_risk_set_index(
                event_time[:n_event],
                np.flip(censoring_time[n_event:])
            )
        n_appearance = CoxModel.count_risk_set_appearance(
            len(event_time), risk_set_start_index, risk_set_end_index
        )
        if not np.all(n_appearance >= 1):
            raise ValueError(
                "Some individuals never appear in the risk set. They have to be"
                "removed before using the CoxModel class.")

        self.n_event = n_event
        self.event_time = event_time
        self.censoring_time = censoring_time
        self.n_appearance_in_risk_set = n_appearance
        self.risk_set_start_index = risk_set_start_index
        self.risk_set_end_index = risk_set_end_index
        self.design = design
        self.name = 'cox'

    @staticmethod
    def preprocess_data(event_time, censoring_time, X):
        event_time, censoring_time, X = \
            CoxModel._permute_observations_by_event_and_censoring_time(
                event_time, censoring_time, X
            )
        event_time, censoring_time, X = \
            CoxModel._drop_uninformative_observations(
                event_time, censoring_time, X
            )
        return event_time, censoring_time, X

    @staticmethod
    def _permute_observations_by_event_and_censoring_time(
            event_time, censoring_time, X):
        """
        Permute the observations so that they are ordered from the earliest
        to the last to experience events, followed by the last censored to the
        earliest censored.

        Params
        ------
        event_time : numpy array
            The lowest values indicate the earliest events. float('inf')
            indicates right-censoring.
        X : numpy array or scipy sparse matrix
        """
        if not np.all(np.equal(
                event_time == float("inf"),
                censoring_time < float('inf')
            )):
            raise ValueError(
                "Either event or censoring time must be infinity for each "
                "observation."
            )

        is_sorted = (
            np.all(event_time[:-1] <= event_time[1:])
            and np.all(censoring_time[:-1] >= censoring_time[1:])
        )
        if is_sorted:
            return event_time, censoring_time, X

        warn(
            "The observations and design matrix will be sorted so that the event "
            "times are in the ascending order and censoring times in the descending order."
        )

        n_event = np.sum(event_time < float('inf'))
        event_rank = CoxModel.np_rank_by_value(event_time)
        censoring_rank = CoxModel.np_rank_by_value(censoring_time)
        sort_ind = np.concatenate((
            np.argsort(event_rank)[:n_event],
            np.argsort(- censoring_rank)[n_event:]
        ))
        assert len(np.unique(sort_ind)) == len(sort_ind)

        event_time = event_time[sort_ind]
        censoring_time = censoring_time[sort_ind]
        if sp.sparse.issparse(X):
            X = X.tocsr()[sort_ind, :]
        else:
            X = X[sort_ind, :]

        return event_time, censoring_time, X

    @staticmethod
    def _drop_uninformative_observations(event_time, censoring_time, X):

        finite_event_time = event_time[event_time < float('inf')]
        finite_censoring_time = censoring_time[censoring_time < float('inf')]

        # Exclude those censored before the first event.
        is_uninformative = (censoring_time < np.min(event_time))

        if np.any(is_uninformative):
            warn(
                "Some observations do not contribute to the likelihood, so "
                "they are being removed."
            )
            is_informative = np.logical_not(is_uninformative)
            event_time = event_time[is_informative]
            censoring_time = censoring_time[is_informative]
            X = X[is_informative, :]

        return event_time, censoring_time, X

    @staticmethod
    def np_rank_by_value(arr):
        sort_arguments = np.argsort(arr)
        rank = np.arange(len(arr))[np.argsort(sort_arguments)]
        return rank.astype('float')

    @staticmethod
    def count_risk_set_appearance(n_obs, start_index, end_index):
        """ This function assumes that the observations are already sorted in
        the way required by the class. """

        # The calculation can be done more efficiently.
        n_appearance = np.zeros(n_obs, dtype=np.int)
        for i in range(len(start_index)):
            if start_index[i] <= end_index[i]:
                n_appearance[start_index[i]:(end_index[i] + 1)] += 1
        return n_appearance

    def _find_risk_set_index(self, event_time, censoring_time):
        """ The parameters are assumed to have 'inf' removed and in the ascending order. """

        n_event = len(event_time)
        start_index = np.zeros(n_event, dtype=np.int)
        for i in range(1, n_event):
            if event_time[i - 1] == event_time[i]:
                start_index[i] = start_index[i - 1]
            else:
                start_index[i] = i

        n_censored = np.array([
            np.searchsorted(censoring_time, t) for t in event_time
        ], dtype=np.int) # Tied censoring time is considered to be in the risk set.
        end_index = len(event_time) + len(censoring_time) - 1 - n_censored

        return start_index, end_index

    def compute_loglik_and_gradient(self, beta, loglik_only=False):

        grad = None # defalt return value

        log_rel_hazard, rel_hazard, hazard_sum_over_risk_set \
            = self._compute_relative_hazard(beta)
        if np.any(hazard_sum_over_risk_set == 0.):
            loglik = - float('inf')
            return loglik, grad

        loglik = np.sum(
            log_rel_hazard[:self.n_event] - np.log(hazard_sum_over_risk_set)
        )

        if not loglik_only:
            hazard_matrix = self._HazardMultinomialProbMatrix(
                rel_hazard, hazard_sum_over_risk_set,
                self.risk_set_start_index, self.risk_set_end_index, self.n_appearance_in_risk_set
            )
            v = np.zeros(self.design.shape[0])
            v[:self.n_event] = 1
            v -= hazard_matrix.sum_over_events()
            grad = self.design.Tdot(v)

        return loglik, grad

    def _compute_relative_hazard(self, beta):

        log_rel_hazard = self.design.dot(beta)
        log_rel_hazard = CoxModel._shift_log_hazard(log_rel_hazard)

        rel_hazard = np.exp(log_rel_hazard)

        hazard_sum_over_risk_set = self._sum_over_start_end(
            rel_hazard, self.risk_set_start_index, self.risk_set_end_index
        )

        return log_rel_hazard, rel_hazard, hazard_sum_over_risk_set

    @staticmethod
    def _sum_over_start_end(arr, start_index, end_index):
        """
        Returns
        -------
        numpy array whose k-th element equals
            np.sum(arr[start_index[k]:(1 + end_index[k])])
        """
        sum_from_right = \
            np.cumsum(arr[start_index[-1]:])[end_index - start_index[-1]]
        sum_from_left = np.concatenate((
            CoxModel.np_reverse_cumsum(arr[:start_index[-1]]), [0]
        ))
        total_sum = sum_from_right + sum_from_left
        return total_sum

    @staticmethod
    def np_reverse_cumsum(arr):
        return np.cumsum(arr[::-1])[::-1]

    @staticmethod
    def _shift_log_hazard(log_hazard_rate, log_offset=0):
        """
        Shift the values so that the max value equals 'log_offset' to
        prevent numerical under / over-flow before taking exponential.
        """
        log_hazard_rate += log_offset - np.max(log_hazard_rate)
        return log_hazard_rate

    def compute_hessian(self, beta):
        raise NotImplementedError()

    def get_hessian_matvec_operator(self, beta):

        _, rel_hazard, hazard_sum_over_risk_set \
            = self._compute_relative_hazard(beta)
        if np.any(hazard_sum_over_risk_set == 0.):
            raise ValueError(
                'Hessian operator cannot be computed likely due to an '
                'unreasonable value of regression coefficients. This could '
                'be caused by the likelihood and prior both being too weak '
                'or by a poor initialization of the Markov chain.'
            )
        W = self._HazardMultinomialProbMatrix(
            rel_hazard, hazard_sum_over_risk_set,
            self.risk_set_start_index, self.risk_set_end_index, self.n_appearance_in_risk_set
        )
        def hessian_op(beta):
            X_beta = self.design.dot(beta)
            result_vec = - self.design.Tdot(
                W.sum_over_events() * X_beta - W.Tdot(W.dot(X_beta))
            )
            return result_vec

        return hessian_op

    @staticmethod
    def simulate_outcome(X, beta, censoring_frac=.9, seed=None):
        """
        Simulate an outcome from a constant baseline hazard model i.e. the
        survival time is exponential.
        """
        if seed is not None:
            np.random.seed(seed)

        log_hazard_rate = X.dot(beta)
        log_hazard_rate = CoxModel._shift_log_hazard(log_hazard_rate)
        hazard_rate = np.exp(log_hazard_rate)
        event_time = np.random.exponential(scale=hazard_rate ** -1)

        scale = CoxModel._solve_for_exp_scale(
            np.quantile(event_time, 1 - censoring_frac), 1 - censoring_frac
        )
        censoring_time = np.random.exponential(
            scale=scale * np.ones(len(hazard_rate))
        )
        censoring_time[event_time < censoring_time] = float("inf")
        event_time[event_time >= censoring_time] = float('inf')

        return event_time, censoring_time

    @staticmethod
    def _solve_for_exp_scale(t, prob):
        """
        Computes the scale of an exponential random variable Z such that
            P(Z < t) == prob
        """
        return - t / np.log(1 - prob)

    class _HazardMultinomialProbMatrix():
        """
        Defines operations by a matrix whose each row represents the conditional
        probabilities of the event happening to the individuals in the risk set.
        """

        def __init__(self, rel_hazard, hazard_sum_over_risk_set,
                     risk_set_start_index, risk_set_end_index, n_appearance_in_risk_set):
            self.rel_hazard = rel_hazard
            self.hazard_sum_over_risk_set = hazard_sum_over_risk_set
            self.risk_set_start_index = risk_set_start_index
            self.risk_set_end_index = risk_set_end_index
            self.n_appearance_in_risk_set = n_appearance_in_risk_set
            self.n_event = len(hazard_sum_over_risk_set)

        def sum_over_events(self):
            """
            Returns the same value as the row sum of the explicitly computed
            the matrix (e.g. via the 'compute_matrix' method) but do it more
            efficiently.
            """
            normalizer_cumsum = np.cumsum(self.hazard_sum_over_risk_set ** -1)
            row_sum = normalizer_cumsum[self.n_appearance_in_risk_set - 1] \
                      * self.rel_hazard
            return row_sum

        def dot(self, v):
            return self.hazard_sum_over_risk_set ** - 1 * CoxModel._sum_over_start_end(
                self.rel_hazard * v, self.risk_set_start_index, self.risk_set_end_index
            )

        def Tdot(self, v):
            partial_inner_prod = np.cumsum(self.hazard_sum_over_risk_set ** -1 * v)
            return self.rel_hazard \
                   * partial_inner_prod[self.n_appearance_in_risk_set - 1]

        def compute_matrix(self):
            multinomial_prob = np.outer(
                self.hazard_sum_over_risk_set ** -1,
                self.rel_hazard
            )
            multinomial_prob = np.triu(multinomial_prob)
            for i in range(1, multinomial_prob.shape[0] + 1):
                if (self.risk_set_end_index[-i] + 1) >= len(self.rel_hazard):
                    break
                multinomial_prob[-i, (self.risk_set_end_index[-i] + 1):] = 0

            return multinomial_prob


import json
import requests

URL = 'http://localhost:8080/control_server/'

print('Get all control servers')
req_data = {
    'Action': 'GetControlServers',
}
r = requests.post(URL, data = req_data)
print(r.status_code, r.reason)
print(json.loads(r.text))

print('\nInsert control server')
req_data = {
    'Action': 'InsertControlServer',
    'Data': json.dumps({
        'IP': '127.0.0.1',
        'Port': '1',
        'Name': 'Test InsertControlServer'
    })
}
r = requests.post(URL, data = req_data)
print(r.status_code, r.reason)
print(json.loads(r.text))

print('\nUpdate control server')
json_resp = json.loads(r.text)
server = json_resp['Data']
req_data = {
    'Action': 'UpdateControlServer',
    'ID': server['id'],
    'Data': json.dumps({
        'IP': '127.0.0.2',
        'Port': '2',
        'Name': 'Test UpdateControlServer'
    })
}
r = requests.post(URL, data = req_data)
print(r.status_code, r.reason)
print(json.loads(r.text))

print('\nDelete control server')
json_resp = json.loads(r.text)
server = json_resp['Data']
req_data = {
    'Action': 'DeleteControlServer',
    'ID': server['id']
}
r = requests.post(URL, data = req_data)
print(r.status_code, r.reason)
print(json.loads(r.text))


from io import BytesIO
import json
import logging

import click

from pywcmp.wcmp2.kpi import (
    WMOCoreMetadataProfileKeyPerformanceIndicators as wcmp_kpis2
)
from pywcmp.util import (get_cli_common_options, parse_wcmp, setup_logger,
                         urlopen_)

LOGGER = logging.getLogger(__name__)

@click.group()
def kpi():
    """key performance indicators"""
    pass

@click.command()
@click.pass_context
@get_cli_common_options
@click.argument('file_or_url')
@click.option('--summary', '-s', is_flag=True, default=False,
              help='Provide summary of KPI test results')
@click.option('--kpi', '-k', help='KPI to run, default is all')
def validate(ctx, file_or_url, summary, kpi, logfile, verbosity):
    """run key performance indicators"""

    setup_logger(verbosity, logfile)

    if file_or_url.startswith('http'):
        content = BytesIO(urlopen_(file_or_url).read())
    else:
        content = file_or_url

    click.echo(f'Validating {file_or_url}')

    try:
        data = parse_wcmp(content)
    except Exception as err:
        raise click.ClickException(err)

    cls = wcmp_kpis2

    kpis = cls(data)

    try:
        kpis_results = kpis.evaluate(kpi)
    except ValueError as err:
        raise click.UsageError(f'Invalid KPI {kpi}: {err}')

    if not summary or kpi is not None:
        click.echo(json.dumps(kpis_results, indent=4))
    else:
        click.echo(json.dumps(kpis_results['summary'], indent=4))

kpi.add_command(validate)


""" Script that evaluates reaction coordinates using the SGOOP method.
Probabilites are calculated using MD trajectories. Transition rates are
found using the maximum caliber approach.
For unbiased simulations use rc_eval().
For biased simulations calculate unbiased probabilities and analyze then with sgoop().

The original method was published by Tiwary and Berne, PNAS 2016, 113, 2839.

Author: Zachary Smith                   zsmith7@terpmail.umd.edu
Original Algorithm: Pratyush Tiwary     ptiwary@umd.edu
Contributor: Pablo Bravo Collado        ptbravo@uc.cl"""

import numpy as np
import scipy.optimize as opt
import sgoop.analysis as analysis

# #####################################################################
# ########### Get probabilities along RC with KDE #####################
# #####################################################################

def md_prob(rc, md_traj, weights=None, rc_bins=20, kde_bw=None):
    """
    Calculate probability density along a given reaction coordinate.

    Calculate the value of the probability density function (for KDE) or probability
    mass function (for histogram) at discrete grid points along a given RC. For a
    biased simulation, frame weights should be supplied.

    Parameters
    ----------
    rc : array-like
    md_traj : array-like
    weights : array-like, None
    rc_bins : int, 20
    kde_bw : float, None

    Returns
    -------
    pdf : np.ndarray
    grid : np.ndarray

    Examples
    --------

    """
    # ensure rc and md_traj are numpy arrays before computation
    rc = np.array(rc)
    md_traj = np.array(md_traj)
    # calculate rc observable for each frame
    colvar_rc = np.sum(md_traj * rc, axis=1)

    if weights is not None:
        weights = np.array(weights)

    if kde_bw is not None:
        # evaluate pdf on a grid using KDE with Gaussian kernel
        grid = np.linspace(colvar_rc.min(), colvar_rc.max(), num=rc_bins)
        pdf = analysis.gaussian_density_estimation(colvar_rc, weights, grid, kde_bw)
        return pdf, grid

    # evaluate pdf using histograms
    pdf, bin_edges = analysis.histogram_density_estimation(colvar_rc, weights, rc_bins)
    # set grid points to center of bins
    bin_width = bin_edges[1] - bin_edges[0]
    grid = bin_edges[:-1] + bin_width

    return pdf, grid

# #####################################################################
# ###### Get binned RC value along unbiased traj for MaxCal ###########
# #####################################################################

def bin_max_cal(rc, md_traj, grid):
    """
    Calculate Reaction Coordinate bin index for each frame in max_cal_traj.

    Parameters
    ----------
    rc : np.ndarray
        Array of coefficients for one-dimensional reaction coordinate.
    md_traj : pd.DataFrame
        DataFrame storing COLVAR data from MaxCal trajectory.
    grid : np.ndarray
        Array of RC values at the center of each rc_bin.

    Returns
    ----------
    binned : np.ndarray

    """
    if rc is None or md_traj is None:
        return None

    # ensure rc and md_traj are ndarrays before computation
    rc = np.array(rc)
    md_traj = np.array(md_traj)
    # calculate rc observable for each frame
    colvar_rc = np.sum(md_traj * rc, axis=1)
    binned = analysis.find_closest_points(colvar_rc, grid)
    return binned

# #####################################################################
# ###### Calc transistion matrix from binned RC values from   #########
# ###### unbiased and probability from biased trajectory.     #########
# #####################################################################

def get_eigenvalues(binned_rc_traj, p, d, diffusivity=None):
    if diffusivity is None and binned_rc_traj is None:
        print("You must supply a MaxCal traj or diffusivity.")
        return

    n = diffusivity
    if binned_rc_traj is not None:
        # ensure binned traj is an ndarray before computation
        binned_rc_traj = np.array(binned_rc_traj)
        n = analysis.avg_neighbor_transitions(binned_rc_traj, d)

    with np.errstate(divide="ignore", invalid="ignore"):
        # ensure p is an ndarray before computation
        p = np.array(p)
        prob_matrix = analysis.probability_matrix(p, d)

    transition_matrix = n * prob_matrix
    eigenvalues = analysis.sorted_eigenvalues(transition_matrix)
    return eigenvalues

# ####################################################################
# ###### Evaluate a series of RCs or optimize from starting RC #######
# ####################################################################

def rc_eval(
    rc,
    probability_traj,
    sgoop_dict,
    weights=None,
    max_cal_traj=None,
    return_eigenvalues=False,
):
    # calculate prob for rc bins and binned rc value for MaxCal traj
    rc_bins = sgoop_dict.get("rc_bins")
    kde_bw = sgoop_dict.get("kde_bw")
    prob, grid = md_prob(rc, probability_traj, weights, rc_bins, kde_bw)
    # bin MaxCal trajectory. returns None if no trajectory is supplied.
    binned = bin_max_cal(rc, max_cal_traj, grid)
    # calculate spectral gap
    d = sgoop_dict.get("d")
    wells = sgoop_dict.get("wells")
    diffusivity = sgoop_dict.get("diffusivity")
    eigenvalues = get_eigenvalues(binned, prob, d, diffusivity)
    sg = analysis.spectral_gap(eigenvalues, wells)

    if return_eigenvalues:
        return sg, eigenvalues

    return sg

def optimize_rc(
    rc_0,
    probability_traj,
    sgoop_dict,
    weights=None,
    max_cal_traj=None,
    niter=50,
    annealing_temp=0.1,
    step_size=0.5,
):
    """
    Calculate optimal RC given an initial estimate for the coefficients
    and a Sgoop object containing a COLVAR file with CVs tracked over
    the course of a short unbiased simulation a COLVAR file with
    c(t) and CVs from a biased MetaD simulation.

    :param rc_0:
    :param single_sgoop:
    :param niter:
    :param annealing_temp:
    :return:
    """
    # pass trajectories and sgoop options through minimizer kwargs
    minimizer_kwargs = {
        "method": "BFGS",
        "options": {
            # "maxiter": 10
        },
        "args": (probability_traj, sgoop_dict, weights, max_cal_traj),
    }

    if max_cal_traj is None and sgoop_dict.get("diffusivity") is None:
        print(
            "A dynamical observable is required by the MaxCal framework. Please "
            "provide either a MaxCal trajectory or static diffusion constant."
        )
        return

    return opt.basinhopping(
        __opt_func,
        rc_0,
        niter=niter,
        T=annealing_temp,
        stepsize=step_size,
        minimizer_kwargs=minimizer_kwargs,
        disp=True,
        callback=__print_fun,
    )

def __opt_func(rc, metad_traj, sgoop_dict, weights, max_cal_traj):
    # normalize rc
    rc = rc / np.sqrt(np.sum(np.square(rc)))
    # calculate spectral gap for normalized rc
    sg = rc_eval(rc, metad_traj, sgoop_dict, weights, max_cal_traj)
    # return negative gap for minimization
    return -sg

def __print_fun(x, f, accepted):
    if accepted:
        rc = x / np.sqrt(np.sum(np.square(x)))
        print(f"RC with spectral gap {-f:} accepted.")
        print(", ".join([str(coeff) for coeff in rc]), "\n")
    else:
        print("")


from uuid import UUID
from components.models import Component
from django.db.models import Sum
from inventory.models import UserInventory
from inventory.serializers import UserInventorySerializer
from modules.models import ModuleBomListItem
from rest_framework import status
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.views import APIView
from django.shortcuts import get_object_or_404

class UserInventoryView(APIView):
    """
    Handles GET, POST, DELETE, and PATCH methods for the user's inventory.
    """

    permission_classes = [IsAuthenticated]

    def get(self, request):
        user = request.user
        inventory = UserInventory.objects.filter(user=user)
        serializer = UserInventorySerializer(inventory, many=True)
        return Response(serializer.data, status=status.HTTP_200_OK)

    def post(self, request, component_pk):
        user = request.user

        # Determine the editing mode from request
        edit_mode = request.data.get("editMode", True)

        # Filter the user inventory items by user and component_id
        user_inventory_items = UserInventory.objects.filter(
            user=user, component__id=component_pk
        )

        # If the user inventory item exists, update the quantity
        if user_inventory_items.exists():
            user_inventory_item = user_inventory_items.first()
            quantity = int(request.data.get("quantity", 0))
            if edit_mode:
                user_inventory_item.quantity = quantity
            else:
                user_inventory_item.quantity += quantity

            user_inventory_item.save()
            serializer = UserInventorySerializer(user_inventory_item)
            return Response(serializer.data, status=status.HTTP_200_OK)

        # If the user inventory item does not exist, create a new one
        quantity = int(request.data.get("quantity", 0))
        component = Component.objects.get(id=component_pk)

        user_inventory = UserInventory.objects.create(
            user=user, component=component, quantity=quantity
        )

        serializer = UserInventorySerializer(user_inventory)
        return Response(serializer.data, status=status.HTTP_201_CREATED)

    def delete(self, request, component_pk):
        user = request.user
        user_inventory_item = UserInventory.objects.filter(
            user=user, component__id=component_pk
        ).first()

        if not user_inventory_item:
            return Response(
                {"detail": "User inventory not found"}, status=status.HTTP_404_NOT_FOUND
            )

        user_inventory_item.delete()
        return Response(
            {"detail": "User inventory deleted successfully"},
            status=status.HTTP_204_NO_CONTENT,
        )

    def patch(self, request, component_pk):
        user = request.user
        user_inventory_item = UserInventory.objects.filter(
            user=user, component__id=component_pk
        ).first()

        if not user_inventory_item:
            return Response(
                {"detail": "User inventory not found"}, status=status.HTTP_404_NOT_FOUND
            )

        serializer = UserInventorySerializer(
            user_inventory_item, data=request.data, partial=True
        )

        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@permission_classes([IsAuthenticated])
@api_view(["GET"])
def get_user_inventory_quantity(request, component_pk):
    inventory = UserInventory.objects.filter(
        component__id=component_pk, user=request.user
    )

    # Check if inventory exists
    if inventory.exists():
        # Access the first inventory object in the QuerySet
        # and retrieve the 'quantity' attribute
        quantity = inventory.first().quantity
        return Response({"quantity": quantity}, status=status.HTTP_200_OK)
    else:
        return Response({"quantity": 0}, status=status.HTTP_200_OK)

@permission_classes([IsAuthenticated])
@api_view(["GET"])
def get_user_inventory_quantities_for_bom_list_item(request, modulebomlistitem_pk):
    """
    Get sum of components in user inventory that fulfill a given bom list item
    """
    bom_list_item = ModuleBomListItem.objects.get(id=modulebomlistitem_pk)
    inventory = UserInventory.objects.filter(
        component__in=bom_list_item.components_options.all(), user=request.user
    )

    # Check if inventory exists
    if inventory.exists():
        # Use aggregate function to get the sum of 'quantity' attribute
        quantity_sum = inventory.aggregate(Sum("quantity")).get("quantity__sum")
        return Response({"quantity": quantity_sum}, status=status.HTTP_200_OK)
    else:
        return Response({"quantity": 0}, status=status.HTTP_200_OK)


from pathlib import Path
import json
import requests as re

# looks up all .jl files in the data folder and downloads the pdf linked under mainFile/accessUrl
def download_pdfs():
    pathlist = Path("data/").glob("**/*.jl")
    for path in pathlist:
        jl_file = str(path)
        with open(jl_file, "r") as f:
            for paper in f:
                paper_json = json.loads(paper)
                if not paper_json.get("mainFile") or not paper_json.get("mainFile").get(
                    "accessUrl"
                ):  # manche von den jsons haben kein pdf als mainFile verlinkt – sollte man es da woanders suchen?
                    continue
                pdf_name = paper_json["mainFile"]["fileName"]
                pdf_path = Path(f"data/pdfs/{pdf_name}")
                if not pdf_path.is_file():
                    print("Processing " + f"data/pdfs/{pdf_name}")
                    if not pdf_name.startswith("1047735.pdf"):
                        pdf_url = paper_json["mainFile"]["accessUrl"]
                        response = re.get(pdf_url)
                        Path("data/pdfs").mkdir(parents=True, exist_ok=True)
                        with open(f"data/pdfs/{pdf_name}", "wb") as f:
                            print("Writing contents to " + f"data/pdfs/{pdf_name}")
                            f.write(response.content)
    print("finished!")

download_pdfs()


#!/usr/bin/env python
import glob
import multiprocessing.pool
import os
import tarfile
import urllib.request
import warnings
import stat
import torch

from setuptools import setup, find_packages, distutils
from setuptools.command.install import install
from distutils import log
from torch.utils.cpp_extension import BuildExtension
from torch.utils.cpp_extension import CppExtension, include_paths

def download_extract(url, dl_path):
    if not os.path.isfile(dl_path):
        # Already downloaded
        urllib.request.urlretrieve(url, dl_path)
    if dl_path.endswith(".tar.gz") and os.path.isdir(dl_path[:-len(".tar.gz")]):
        # Already extracted
        return
    tar = tarfile.open(dl_path)
    tar.extractall('third_party/')
    tar.close()

FILES_TO_MARK_EXECUTABLE = ["flac-linux-x86", "flac-linux-x86_64", "flac-mac", "flac-win32.exe"]

# Does gcc compile with this header and library?
def compile_test(header, library):
    dummy_path = os.path.join(os.path.dirname(__file__), "dummy")
    command = "bash -c \"g++ -include " + header + " -l" + library + " -x c++ - <<<'int main() {}' -o " + dummy_path \
              + " >/dev/null 2>/dev/null && rm " + dummy_path + " 2>/dev/null\""
    return os.system(command) == 0

compile_args = ['-O3', '-DKENLM_MAX_ORDER=6', '-std=c++14', '-fPIC']
ext_libs = []
if compile_test('zlib.h', 'z'):
    compile_args.append('-DHAVE_ZLIB')
    ext_libs.append('z')

if compile_test('bzlib.h', 'bz2'):
    compile_args.append('-DHAVE_BZLIB')
    ext_libs.append('bz2')

if compile_test('lzma.h', 'lzma'):
    compile_args.append('-DHAVE_XZLIB')
    ext_libs.append('lzma')

third_party_libs = ["kenlm", "openfst-1.6.7/src/include", "ThreadPool", "boost_1_67_0", "utf8"]
compile_args.extend(['-DINCLUDE_KENLM', '-DKENLM_MAX_ORDER=6'])
lib_sources = glob.glob('third_party/kenlm/util/*.cc') + glob.glob('third_party/kenlm/lm/*.cc') + glob.glob(
    'third_party/kenlm/util/double-conversion/*.cc') + glob.glob('third_party/openfst-1.6.7/src/lib/*.cc')
lib_sources = [fn for fn in lib_sources if not (fn.endswith('main.cc') or fn.endswith('test.cc'))]

third_party_includes = ["D:\\SpeechRecognition\\AutomaticSpeechRecognitionSystem\\neuralnet", "D:\\SpeechRecognition\\AutomaticSpeechRecognitionSystem\\ctcdecode"]
ctc_sources = glob.glob('ctcdecode/src/*.cpp')

extension = CppExtension(
    name='ctcdecode._ext.ctc_decode',
    package=True,
    with_cuda=False,
    sources=ctc_sources + lib_sources,
    include_dirs=third_party_includes + include_paths(),
    libraries=ext_libs,
    extra_compile_args=compile_args,
    language='c++'
)

# monkey-patch for parallel compilation
# See: https://stackoverflow.com/a/13176803
def parallelCCompile(self,
                     sources,
                     output_dir=None,
                     macros=None,
                     include_dirs=None,
                     debug=0,
                     extra_preargs=None,
                     extra_postargs=None,
                     depends=None):
    # those lines are copied from distutils.ccompiler.CCompiler directly
    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
        output_dir, macros, include_dirs, sources, depends, extra_postargs)
    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)

    # parallel code
    def _single_compile(obj):
        try:
            src, ext = build[obj]
        except KeyError:
            return
        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)

    # convert to list, imap is evaluated on-demand
    thread_pool = multiprocessing.pool.ThreadPool(os.cpu_count())
    list(thread_pool.imap(_single_compile, objects))
    return objects

# hack compile to support parallel compiling
distutils.ccompiler.CCompiler.compile = parallelCCompile

class BuildExtension(install):
    def run(self):
        install.run(self)  # do the original install steps

        # mark the FLAC executables as executable by all users (this fixes occasional issues when file permissions get messed up)
        for output_path in self.get_outputs():
            if os.path.basename(output_path) in FILES_TO_MARK_EXECUTABLE:
                log.info("setting executable permissions on {}".format(output_path))
                stat_info = os.stat(output_path)
                os.chmod(
                    output_path,
                    stat_info.st_mode |
                    stat.S_IRUSR | stat.S_IXUSR |  # owner can read/execute
                    stat.S_IRGRP | stat.S_IXGRP |  # group can read/execute
                    stat.S_IROTH | stat.S_IXOTH  # everyone else can read/execute
                )

setup(
    name="CTC_SpeechRecognition",
    version="1.0.0",
    description="Automatic Speech Recognition System with PyTorch and CTC Decoder based on Paddle Paddle's implementation",
    url="https://github.com/DvelopedByIkshaan/CTC-SpeechRecognition",
    author="Ikshaan Gupta",
    author_email="ikshaan@gmail.com",
    # Exclude the build files.
    packages=["CTC_SpeechRecognition"],
    include_package_data=True,
    include_dirs=third_party_includes,
    cmdclass={"install": BuildExtension}
)


"""
About Writer

The About Writer is a Python script that adds an "about" statement to the specified files.
It is designed to help add metadata information, such as author and timestamp, to source code files.
The script supports various file formats and can recursively process files in a specified folder.

Author: ZION

Usage:
    python about_writer.py [-h] [-x EXTS] [-v] path name

Arguments:
    path: The path to the file or folder.
    name: The name to be set as the author of the file.

Options:
    -x EXTS: Only modify files with the given extension(s). Separate multiple extensions with a comma (,).
    -v: Show verbose output.

Supported File Formats:
    - Python (.py)
    - Java (.java)
    - C (.c)
    - PHP (.php)
    - HTML (.html)
    - JSON (.json)
    - CSS (.css)
    - JavaScript (.js)
    - Perl (.pl)
    - Shell Script (.sh)
    - C++ (.cpp)
    - C# (.cs)
    - R (.r)
    - Text (.txt)
"""

import subprocess
import pathlib
import os
import argparse
import re

INCLUDED_EXTS = {
    "*.py": ("\"\"\"", "\"\"\""),
    "*.java": ("/*", "*/"),
    "*.c": ("/*", "*/"),
    "*.php": ("/*", "*/"),
    "*.html": ("<!--", "-->"),
    "*.json": ("/*", "*/"),
    "*.css": ("/*", "*/"),
    "*.js": ("/*", "*/"),
    "*.pl": ("=pod", "=cut"),
    "*.sh": (": '", "'"),
    "*.cpp": ("/*", "*/"),
    "*.cs": ("/*", "*/"),
    "*.r": ("#", ""),
    "*.txt": ("", ""),
}

EXCLUDED_EXTS = ["exe", "class", "pickle"]
FILES_LIST = []

def get_folder(foldername, exts=["*"], verbose=False):
    """
    Recursively gets all the files in the specified folder with the given extensions.

    :param foldername: The folder name
    :type foldername: str
    :param exts: The extensions of files to include, defaults to ["*"]
    :type exts: list, optional
    :param verbose: Flag to print verbose output, defaults to False
    :type verbose: bool, optional
    """
    for i in exts:
        for child in pathlib.Path(foldername).rglob(i):
            if child not in FILES_LIST and child.is_file():
                FILES_LIST.append(child)
                print(f"[*] Appended {child} to files_list") if verbose else print("")
            elif child.is_dir():
                get_folder(child, exts, verbose)

def perm_edit(name, verbose=False):
    """
    Applies the about statement to all the files in the files_list.

    :param name: The name to be set as the author of the file
    :type name: str
    :param verbose: Flag to print verbose output, defaults to False
    :type verbose: bool, optional
    """
    for child in FILES_LIST:
        if child.is_file():
            print(str(child).center(75, "="))
            try:
                edit_file(child, name, verbose)
            except Exception:
                ext = "*." + str(child).suffix
                print(f"[!] Couldn't write to \"{child}\" because file format \"{ext}\" is not supported") if verbose else print("")

def edit_file(filepath, name, verbose=False):
    """
    Edits the specified file to add the about statement.

    :param filepath: The path of the file to edit
    :type filepath: str
    :param name: The name to be set as the author of the file
    :type name: str
    :param verbose: Flag to print verbose output, defaults to False
    :type verbose: bool, optional
    """
    if os.sep in str(filepath):
        filepath = str(filepath).replace(os.sep, os.altsep)
    filename = str(filepath).split(os.altsep)[-1].upper()
    with open(filepath, "r") as file:
        content = file.read()
    print(f"[*] Reading content of \"{filepath}\"") if verbose else print("")
    if filename not in content:
        get_date_time_info = subprocess.run(["dir", "/Tc", filepath], shell=True, capture_output=True, text=True).stdout
        date = re.search(r"\r\n(\d+\W\w+\W\d+)", get_date_time_info)
        time = re.search(r"(\d+\W\d+\W+\w+)", get_date_time_info)
        ext = "*." + str(filepath).split(os.altsep)[-1].split(".")[-1]
        if ext not in INCLUDED_EXTS.keys():
            print(f"[!] Couldn't write to \"{filepath}\" because file format \"{ext}\" is not supported")
        else:
            for i in INCLUDED_EXTS:
                if i == ext:
                    print(f"[*] Writing about statement to \"{filepath}\"") if verbose else print("")
                    if "<DOCTYPE html>".lower() not in content.lower() and ext != "*.r":
                        text = f"{INCLUDED_EXTS[i][0]}\n{filename} Source Code\n{name}, {date[0].strip()} @ {time[0].strip()}.\n{INCLUDED_EXTS[i][-1]}\n\n\n"
                    elif ext == "*.r":
                        text = f"{INCLUDED_EXTS['*.r'][0]}\t{filename} Source Code\n{INCLUDED_EXTS['*.r'][0]}\t{name}, {date[0].strip()} @ {time[0].strip()}.\n\n\n"
                    else:
                        text = f"{INCLUDED_EXTS['*.html'][0]}\n{filename} Source Code\n{name}, {date[0].strip()} @ {time[0].strip()}.\n{INCLUDED_EXTS['*.html'][-1]}\n\n\n"
                    with open(filepath, "r", encoding="utf-8") as file:
                        if ext == "*.php":
                            if "<DOCTYPE html>".lower() not in content.lower():
                                file.seek(5)
                        content = file.read()
                    with open(filepath, "w", encoding="utf-8") as file:
                        if ext == "*.php":
                            if "<DOCTYPE html>".lower() not in content.lower():
                                text = "<?php\n" + text
                        content = text + content
                        file.write(content)
                    print(f"[*] Finished writing about statement to \"{filepath}\"") if verbose else print("")
    else:
        print(f"[!] About statement has already been written to \"{filepath}\"")

def main():
    parser = argparse.ArgumentParser(description="About Writer")
    parser.add_argument("path", help="Path to the file or folder")
    parser.add_argument("name", help="Value to be set as the author of the file")
    parser.add_argument("-x", "--exts", help="Only modify files with the given extension(s). Separate multiple extensions with comma (,)")
    parser.add_argument("-v", "--verbose", action="store_true", help="Show verbose output")
    args = parser.parse_args()

    if os.path.isdir(args.path):
        exts = ["*"] if not args.exts else args.exts.split(",")
        get_folder(foldername=args.path, exts=exts, verbose=args.verbose)
        perm_edit(name=args.name, verbose=args.verbose)
    elif os.path.isfile(args.path):
        edit_file(filepath=args.path, name=args.name, verbose=args.verbose)

if __name__ == "__main__":
    main()


import bpy
import os
import subprocess
import platform

bl_info = {
    "name": "Open Blend Directory",
    "description": "Opens the directory where the currently opened .blend file is stored similarly to how JetBrains handles it. Cross Platform!",
    "author": "Tobin Cavanaugh",
    "version": (1, 0),
    "blender": (3, 3, 1),
    "location": "View3D > Tool Shelf > My Addon",
    "category": "System",
}

addon_keymaps = []

# Thanks to
# https://blender.stackexchange.com/questions/717/is-it-possible-to-print-to-the-report-window-in-the-info-view
# Manu Järvinen & brockmann for this solution
def ErrorMessageUnsaved(self, context):
    self.layout.label(text="You haven't saved your blender file yet! That's bad! Save it!")

def ErrorMessageOS(self, context):
    self.layout.label(text="I don't know this operating system! It should be easy to add support for though")

#Actual function for opening the blend directory
def open_blend_directory():

    #Get the save data path
    blend_file_path = bpy.data.filepath

    #Choose the correct OS, if its a valid save path
    #No guarantees that these work on non windows platforms
    #According to Lous Brandy & Boris Verkhoviskiy this should be correct
    # https://stackoverflow.com/questions/1854/how-to-identify-which-os-python-is-running-on
    if blend_file_path:
        if platform.system() == 'Windows':
            subprocess.Popen(f'explorer /select, "{bpy.data.filepath}"')
        elif platform.system() == 'Linux':
            subprocess.Popen(['xdg-open', os.path.dirname(blend_file_path)])
        elif platform.system() == 'Darwin':
            subprocess.Popen(['open', '-R', blend_file_path])
        else:
            bpy.context.window_manager.popup_menu(ErrorMessageOS, title="Error", icon='ERROR')
    else:
        bpy.context.window_manager.popup_menu(ErrorMessageUnsaved, title="Error", icon='ERROR')

def register():

    #Register our class
    bpy.utils.register_class(OpenBlendDirectoryOperator)
    wm = bpy.context.window_manager

    #I dont really understand whats going on here tbh

    #Get our new keymap
    km = wm.keyconfigs.addon.keymaps.new(name='Window', space_type='EMPTY')

    #Create the keymap? if you want to change the keybinds for alt + shift + r, do it here!!
    kmi = km.keymap_items.new('wm.open_blend_directory', 'R', 'PRESS', alt=True, shift=True)

    #Append to the keymaps
    addon_keymaps.append((km, kmi))

def unregister():

    #Unregister our class
    bpy.utils.unregister_class(OpenBlendDirectoryOperator)
    wm = bpy.context.window_manager

    #Remove all keymaps
    for km, kmi in addon_keymaps:
        km.keymap_items.remove(kmi)

    #Clear our locally saved keymaps
    addon_keymaps.clear()

#Define our operator class
class OpenBlendDirectoryOperator(bpy.types.Operator):

    bl_idname = "wm.open_blend_directory"
    bl_label = "Open Blend Directory"
    bl_options = {'REGISTER'}

    #On execution, open the directory
    def execute(self, context):
        open_blend_directory()
        return {'FINISHED'}

classes = [
    OpenBlendDirectoryOperator,
]

if __name__ == "__main__":
    register()


from __future__ import division, print_function

import os

import numpy as np
import pandas as pd
import seaborn as sns

from gabor_experiment.misc import FIGS_DIRS
from gabor_experiment.misc import REPS_DIRS
from gabor_experiment.misc import RESULTS_DIR
from gabor_experiment.misc import STIMULI_DIRS
from gabor_experiment.misc import STIMULI_POSTFIXES
from gabor_experiment.misc import TITLES
import matplotlib.pyplot as plt
from utils.misc import LAYER_NAMES

def create_and_save_figures(just_original=True):
    """Create and save figures."""
    sns.set()
    sns.set(style="whitegrid")
    sns.set(font_scale=1.6, style="ticks")

    for stimuli_dir, reps_dir, figs_dir, stimuli_postfix, title \
            in zip(STIMULI_DIRS, REPS_DIRS, FIGS_DIRS,
                   STIMULI_POSTFIXES, TITLES):
        # Filename to load/save results database from/to:
        results_filename = RESULTS_DIR + 'pca' + stimuli_postfix + '.csv'
        correlations_results_filename = RESULTS_DIR + \
            'correlations_pca' + stimuli_postfix + '.csv'

        try:
            df = pd.read_csv(correlations_results_filename)
        except FileNotFoundError:
            print('Please run pca_permutations.py to have all files required'
                  'to create the figures.')
            continue

        # df.to_csv(correlations_results_filename)

        sns.set_palette(sns.color_palette('muted'))
        colors = ["bright sky blue", "purple"]
        sns.set_palette(sns.xkcd_palette(colors))

        fig, ax = plt.subplots()
        fig.subplots_adjust(bottom=0.15)

        ax = df.plot.line(x='Layer', y=['Frequency',
                                        'Orientation'],
                          ax=ax, lw=3,  alpha=0.75,
                          legend=False)
        # ax.set_ylim(0.45,1)
        # plt.legend(loc=3)
        # y_min = 0.39
        # y_max = 1.005
        # plt.axis([-1, 26, y_min, y_max])
        #
        # sns.despine(offset=10, trim=True)
        # ax.set_xlabel('Network Layer', size=20)
        # ax.set_ylabel('Correlation Coefficient', size=20)
        mean_permuted_frequency = []
        mean_permuted_orientation = []
        for l, layer_name in enumerate(LAYER_NAMES):
            try:
                del permutation_df
            except NameError:
                None
            permutations_filename = (RESULTS_DIR + 'permutations_pca/'
                                     + 'permuted_correlations'
                                     + stimuli_postfix + '_layer_' + str(l)
                                     + '.csv')

    #         print(permutations_filename)

            try:
                permutation_df = pd.read_csv(permutations_filename,
                                             index_col=0)
            except FileNotFoundError:
                print('File missing:', permutations_filename)
                print('Please run pca_permutations.py to have all files'
                      ' required to create the figures.')
                continue

            mean_permuted_frequency.append(
                permutation_df['Permuted Frequency'].mean())
            mean_permuted_orientation.append(
                permutation_df['Permuted Orientation'].mean())

    #         print(mean_permuted_frequency)
    #         print(mean_permuted_orientation)
        ax.plot(mean_permuted_orientation, color='#aaaaaa', lw=3,
                linestyle='--', label='Mean Permuted Orientation')
        ax.plot(mean_permuted_frequency, color='#000000', lw=3,
                linestyle=':', label='Mean Permuted Frequency')
        plt.legend(loc=3, frameon=False)
        y_min = 0.1
        y_max = 1.005
    #     plt.axis([-1, 26, y_min, y_max])
        sns.despine(offset=10, trim=True)
        ax.set_xlabel('Network Layer', size=20)
        ax.set_ylabel('Correlation Coefficient', size=20)

        general_figs_dir = os.sep.join(figs_dir.split(os.sep)[:-2])
        general_figs_dir = os.sep.join(
            [general_figs_dir, figs_dir.split(os.sep)[-2]])

        general_figs_dir = os.sep.join(figs_dir.split(os.sep)[:-2])
        general_figs_dir = os.sep.join(
            [general_figs_dir, figs_dir.split(os.sep)[-2]])

        ax.tick_params(axis='both', which='both', top='off', right='off')

        fig.savefig(general_figs_dir + '_correlation.pdf', bbox_inches='tight')
        fig.savefig(general_figs_dir + '_correlation.png', bbox_inches='tight')
        # if 'original' in figs_dir:
        # print(df[['Frequency', 'Orientation']])
        # print(df)

        if just_original:
            print(general_figs_dir + '_correlation.pdf')
            return general_figs_dir + '_correlation.pdf'

if __name__ == '__main__':
    create_and_save_figures(just_original=False)


import pandas as pd
import numpy as np
import torch
import utils
import config

import uncertainty_utilities
import pickle

def get_annotations_per_sample(params):
    annotations_per_vid = {}

    for annotator in range(1,18):
        try:
            data = utils.load_data(params, params.feature_set, params.emo_dim_set, params.normalize, params.label_preproc, params.norm_opts, params.segment_type, params.win_len, params.hop_len, save=params.cache, refresh=params.refresh, add_seg_id=params.add_seg_id, annotator=annotator)
        except Exception as e:
            print(f"Exception for annotator {annotator}: {e}")
            continue

        for partition in ["train", "devel", "test"]:
            metas = data[partition]["meta"]
            labels_ = data[partition]["label"]

            for emo_dim in range(labels_[0].shape[1]):
                labels = [l[:,emo_dim] for l in labels_]

                # NOTE iterate over samples and store them by their id
                for i, meta in enumerate(metas):

                    # NOTE meta list of shape (seq_len, 3); with 2nd dim: [vid_id,timestamp,segment_id]
                    vid_id = int(meta[0][0])

                    label_series = labels[i]

                    # NOTE only at the validation set for MuSe 2020 labels: for any reason, the raw annotations are always exactly by 3 timesteps longer than the fusioned annotations; therefore we have to cut it down, so that later the shapes of variances and subjectivites match
                    if partition == "devel" and not config.USE_2021_LABELS:
                        label_series = label_series[:-3]

                    # NOTE for training, samples get split up, so if we would not specify id, we would get multiple sub-series per sample and per annotator
                    if partition == "train":
                        first_timestamp = int(meta[0][1])
                        vid_id = f"{vid_id}_{first_timestamp}"
                    else:
                        vid_id = str(vid_id)

                    if vid_id not in annotations_per_vid.keys():
                        annotations_per_vid[vid_id] = {}
                    if emo_dim not in annotations_per_vid[vid_id].keys():
                        annotations_per_vid[vid_id][emo_dim] = []

                    annotations_per_vid[vid_id][emo_dim] += [label_series]

    return annotations_per_vid

def calculate_rolling_subjectivities(annotations_per_vid):
    subjectivities = {}
    for vid_id, emo_dims in annotations_per_vid.items():

        subjectivity_of_sample_all_emo_dims = []
        for emo_dim, annotations in emo_dims.items():

            subjectivity_of_sample = []
            for k, annotation_1 in enumerate(annotations):
                for j, annotation_2 in enumerate(annotations[k:]):
                    if k == j:
                        continue

                    # NOTE calculate rolling measuremt of subjectivity between each available annotation
                    rolling_window = 3
                    subjectivity = [
                        pd.Series(annotation_1[i - rolling_window : i]).corr(pd.Series(annotation_2[i - rolling_window : i]))
                            for i in range(rolling_window, len(annotation_1) + 1)
                        ]
                    subjectivity = [subjectivity[0]] * (rolling_window - 1) + subjectivity

                    # NOTE [0,0,0].corr([0,0,0]) = nan; therefore interpolate to fill nan
                    if np.isnan(subjectivity[0]):
                        subjectivity[0] = 0.
                    subjectivity = pd.Series(subjectivity).interpolate()

                    # NOTE maybe use rolling mean over subjectivity, that measurement becomes smoother
                    # rolling_mean_window = 3
                    # subjectivity = subjectivity.rolling(rolling_mean_window).mean()
                    # subjectivity[:rolling_mean_window-1] = subjectivity[rolling_mean_window-1]

                    subjectivity_of_sample += [subjectivity]

            assert len(subjectivity_of_sample) >= 1, f"too less annotations for sample {vid_id}"

            # NOTE calculate element-wise mean to get average subjectivity at each timestep
            subjectivity_of_sample = np.stack(subjectivity_of_sample).mean(axis=0)
            # NOTE convert to tensor and store to return
            subjectivity_of_sample = torch.Tensor(subjectivity_of_sample).float()

            # subjectivities[vid_id] = subjectivity_of_sample
            subjectivity_of_sample_all_emo_dims += [subjectivity_of_sample]

        # NOTE concatenate all dims of emotion (i.e. valence and arousal)
        subjectivity_of_sample_all_emo_dims = torch.Tensor(np.column_stack(subjectivity_of_sample_all_emo_dims))
        subjectivities[vid_id] = subjectivity_of_sample_all_emo_dims

    return subjectivities

def calculate_global_subjectivities(annotations_per_vid, window: int):
    subjectivities = {}
    max_len_global_subjectivities = None

    for vid_id, emo_dims in annotations_per_vid.items():

        subjectivity_of_sample_all_emo_dims = []
        for emo_dim, annotations in emo_dims.items():

            subjectivity_of_sample = []
            for k, annotation_1 in enumerate(annotations):
                for annotation_2 in annotations[k+1:]:

                    if window is None:
                        subjectivity = uncertainty_utilities.ccc_score(annotation_1, annotation_2)

                    else:
                        subjectivity = []
                        for i in range(0, len(annotation_1) + 1 - window, window):
                            if len(annotation_1[i : i + window]) < window:
                                continue
                            subjectivity += [uncertainty_utilities.ccc_score(annotation_1[i : i + window], annotation_2[i : i + window])]

                        if np.isnan(subjectivity[0]):
                            subjectivity[0] = 0.
                        subjectivity = pd.Series(subjectivity).interpolate().to_list()

                    subjectivity_of_sample += [subjectivity]

            if window is None:
                subjectivity_of_sample = np.mean(subjectivity_of_sample)
            else:
                subjectivity_of_sample = np.mean(subjectivity_of_sample, axis=0).tolist()

                if max_len_global_subjectivities is None or len(subjectivity_of_sample) > max_len_global_subjectivities:
                    max_len_global_subjectivities = len(subjectivity_of_sample)

            subjectivity_of_sample_all_emo_dims += [subjectivity_of_sample]

        if window is None:
            subjectivity_of_sample_all_emo_dims = torch.Tensor(subjectivity_of_sample_all_emo_dims)

        subjectivities[vid_id] = subjectivity_of_sample_all_emo_dims

    if not window is None:
        subjectivities_ = {}
        for vid_id, subjectivity_of_sample_all_emo_dims in subjectivities.items():
            subjectivity_of_sample_all_emo_dims_ = []
            for subjectivity_of_sample in subjectivity_of_sample_all_emo_dims:
                nans = [float("nan")] * (max_len_global_subjectivities - len(subjectivity_of_sample))
                subjectivity_of_sample_ = subjectivity_of_sample + nans
                subjectivity_of_sample_all_emo_dims_ += [subjectivity_of_sample_]
            subjectivities_[vid_id] = torch.Tensor(subjectivity_of_sample_all_emo_dims_)
        subjectivities = subjectivities_

    return subjectivities

def load_from_file(filename: str):
    with open(filename, "rb") as file:
        data = pickle.load(file)

    return data[0], data[1]

def save_to_file(short_term_subjectivities, global_subjectivities, filename: str):
    data = [short_term_subjectivities, global_subjectivities]
    with open(filename, "wb") as file:
        pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)

def build_from_scratch(params, filename: str):
    print("Calculating subjectivities among annotators from sratch...")
    annotations_per_vid = get_annotations_per_sample(params)
    short_term = calculate_rolling_subjectivities(annotations_per_vid)
    globals = calculate_global_subjectivities(annotations_per_vid, params.global_uncertainty_window)
    print("Subjectivities calculated.")

    if params.save_subjectivity_to_file:
        save_to_file(short_term, globals, filename)
        print("Subjectivities saved to file.")

    return short_term, globals

def calculate_subjectivities(params):
    filename = config.DATA_FOLDER + "/subjectivities_among_annotations.pickle"

    if params.load_subjectivity_from_file:
        try:
            short_term, globals = load_from_file(filename)
            print("Subjectivities deserialized from file.")
        except:
            short_term, globals = build_from_scratch(params, filename)

    else:
        short_term, globals = build_from_scratch(params, filename)

    return short_term, globals


# -*- encoding: utf-8 -*-
# @Time    :   2020/12/4
# @Author  :   Xiaolei Wang
# @email   :   wxl1999@foxmail.com

# UPDATE
# @Time   : 2020/1/3, 2021/1/4
# @Author : Xiaolei Wang, Yuanhang Zhou
# @email  : wxl1999@foxmail.com, sdzyh002@gmail.com

r"""
KBRD
====
References:
    Chen, Qibin, et al. `"Towards Knowledge-Based Recommender Dialog System."`_ in EMNLP 2019.

.. _`"Towards Knowledge-Based Recommender Dialog System."`:
   https://www.aclweb.org/anthology/D19-1189/

"""

import torch
import torch.nn.functional as F
from loguru import logger
from torch import nn
from torch_geometric.nn import RGCNConv

from crslab.model.base import BaseModel
from crslab.model.utils.functions import edge_to_pyg_format
from crslab.model.utils.modules.attention import SelfAttentionBatch
from crslab.model.utils.modules.transformer import TransformerDecoder, TransformerEncoder

class KBRDModel(BaseModel):
    """

    Attributes:
        vocab_size: A integer indicating the vocabulary size.
        pad_token_idx: A integer indicating the id of padding token.
        start_token_idx: A integer indicating the id of start token.
        end_token_idx: A integer indicating the id of end token.
        token_emb_dim: A integer indicating the dimension of token embedding layer.
        pretrain_embedding: A string indicating the path of pretrained embedding.
        n_entity: A integer indicating the number of entities.
        n_relation: A integer indicating the number of relation in KG.
        num_bases: A integer indicating the number of bases.
        kg_emb_dim: A integer indicating the dimension of kg embedding.
        user_emb_dim: A integer indicating the dimension of user embedding.
        n_heads: A integer indicating the number of heads.
        n_layers: A integer indicating the number of layer.
        ffn_size: A integer indicating the size of ffn hidden.
        dropout: A float indicating the dropout rate.
        attention_dropout: A integer indicating the dropout rate of attention layer.
        relu_dropout: A integer indicating the dropout rate of relu layer.
        learn_positional_embeddings: A boolean indicating if we learn the positional embedding.
        embeddings_scale: A boolean indicating if we use the embeddings scale.
        reduction: A boolean indicating if we use the reduction.
        n_positions: A integer indicating the number of position.
        longest_label: A integer indicating the longest length for response generation.
        user_proj_dim: A integer indicating dim to project for user embedding.

    """

    def __init__(self, opt, device, vocab, side_data):
        """

        Args:
            opt (dict): A dictionary record the hyper parameters.
            device (torch.device): A variable indicating which device to place the data and model.
            vocab (dict): A dictionary record the vocabulary information.
            side_data (dict): A dictionary record the side data.

        """
        self.device = device
        self.gpu = opt.get("gpu", [-1])
        # vocab
        self.pad_token_idx = vocab['pad']
        self.start_token_idx = vocab['start']
        self.end_token_idx = vocab['end']
        self.vocab_size = vocab['vocab_size']
        self.token_emb_dim = opt.get('token_emb_dim', 300)
        self.pretrain_embedding = side_data.get('embedding', None)
        # kg
        self.n_entity = vocab['n_entity']
        entity_kg = side_data['entity_kg']
        self.n_relation = entity_kg['n_relation']
        self.edge_idx, self.edge_type = edge_to_pyg_format(entity_kg['edge'], 'RGCN')
        self.edge_idx = self.edge_idx.to(device)
        self.edge_type = self.edge_type.to(device)
        self.num_bases = opt.get('num_bases', 8)
        self.kg_emb_dim = opt.get('kg_emb_dim', 300)
        self.user_emb_dim = self.kg_emb_dim
        # transformer
        self.n_heads = opt.get('n_heads', 2)
        self.n_layers = opt.get('n_layers', 2)
        self.ffn_size = opt.get('ffn_size', 300)
        self.dropout = opt.get('dropout', 0.1)
        self.attention_dropout = opt.get('attention_dropout', 0.0)
        self.relu_dropout = opt.get('relu_dropout', 0.1)
        self.embeddings_scale = opt.get('embedding_scale', True)
        self.learn_positional_embeddings = opt.get('learn_positional_embeddings', False)
        self.reduction = opt.get('reduction', False)
        self.n_positions = opt.get('n_positions', 1024)
        self.longest_label = opt.get('longest_label', 1)
        self.user_proj_dim = opt.get('user_proj_dim', 512)

        super(KBRDModel, self).__init__(opt, device)

    def build_model(self, *args, **kwargs):
        self._build_embedding()
        self._build_kg_layer()
        self._build_recommendation_layer()
        self._build_conversation_layer()

    def _build_embedding(self):
        if self.pretrain_embedding is not None:
            self.token_embedding = nn.Embedding.from_pretrained(
                torch.as_tensor(self.pretrain_embedding, dtype=torch.float), freeze=False,
                padding_idx=self.pad_token_idx)
        else:
            self.token_embedding = nn.Embedding(self.vocab_size, self.token_emb_dim, self.pad_token_idx)
            nn.init.normal_(self.token_embedding.weight, mean=0, std=self.kg_emb_dim ** -0.5)
            nn.init.constant_(self.token_embedding.weight[self.pad_token_idx], 0)
        logger.debug('[Build embedding]')

    def _build_kg_layer(self):
        self.kg_encoder = RGCNConv(self.n_entity, self.kg_emb_dim, self.n_relation, num_bases=self.num_bases)
        self.kg_attn = SelfAttentionBatch(self.kg_emb_dim, self.kg_emb_dim)
        logger.debug('[Build kg layer]')

    def _build_recommendation_layer(self):
        self.rec_bias = nn.Linear(self.kg_emb_dim, self.n_entity)
        self.rec_loss = nn.CrossEntropyLoss()
        logger.debug('[Build recommendation layer]')

    def _build_conversation_layer(self):
        self.register_buffer('START', torch.tensor([self.start_token_idx], dtype=torch.long))
        self.dialog_encoder = TransformerEncoder(
            self.n_heads,
            self.n_layers,
            self.token_emb_dim,
            self.ffn_size,
            self.vocab_size,
            self.token_embedding,
            self.dropout,
            self.attention_dropout,
            self.relu_dropout,
            self.pad_token_idx,
            self.learn_positional_embeddings,
            self.embeddings_scale,
            self.reduction,
            self.n_positions
        )
        self.decoder = TransformerDecoder(
            self.n_heads,
            self.n_layers,
            self.token_emb_dim,
            self.ffn_size,
            self.vocab_size,
            self.token_embedding,
            self.dropout,
            self.attention_dropout,
            self.relu_dropout,
            self.embeddings_scale,
            self.learn_positional_embeddings,
            self.pad_token_idx,
            self.n_positions
        )
        self.user_proj_1 = nn.Linear(self.user_emb_dim, self.user_proj_dim)
        self.user_proj_2 = nn.Linear(self.user_proj_dim, self.vocab_size)
        self.conv_loss = nn.CrossEntropyLoss(ignore_index=self.pad_token_idx)
        logger.debug('[Build conversation layer]')

    def encode_user(self, entity_lists, kg_embedding):
        user_repr_list = []
        for entity_list in entity_lists:
            if entity_list is None:
                user_repr_list.append(torch.zeros(self.user_emb_dim, device=self.device))
                continue
            user_repr = kg_embedding[entity_list]
            user_repr = self.kg_attn(user_repr)
            user_repr_list.append(user_repr)
        return torch.stack(user_repr_list, dim=0)  # (bs, dim)

    def recommend(self, batch, mode):
        context_entities, item = batch['context_entities'], batch['item']
        kg_embedding = self.kg_encoder(None, self.edge_idx, self.edge_type)
        user_embedding = self.encode_user(context_entities, kg_embedding)
        scores = F.linear(user_embedding, kg_embedding, self.rec_bias.bias)
        loss = self.rec_loss(scores, item)
        return loss, scores

    def _starts(self, batch_size):
        """Return bsz start tokens."""
        return self.START.detach().expand(batch_size, 1)

    def decode_forced(self, encoder_states, user_embedding, resp):
        bsz = resp.size(0)
        seqlen = resp.size(1)
        inputs = resp.narrow(1, 0, seqlen - 1)
        inputs = torch.cat([self._starts(bsz), inputs], 1)
        latent, _ = self.decoder(inputs, encoder_states)
        token_logits = F.linear(latent, self.token_embedding.weight)
        user_logits = self.user_proj_2(torch.relu(self.user_proj_1(user_embedding))).unsqueeze(1)
        sum_logits = token_logits + user_logits
        _, preds = sum_logits.max(dim=-1)
        return sum_logits, preds

    def decode_greedy(self, encoder_states, user_embedding):

        bsz = encoder_states[0].shape[0]
        xs = self._starts(bsz)
        incr_state = None
        logits = []
        for i in range(self.longest_label):
            scores, incr_state = self.decoder(xs, encoder_states, incr_state)  # incr_state is always None
            scores = scores[:, -1:, :]
            token_logits = F.linear(scores, self.token_embedding.weight)
            user_logits = self.user_proj_2(torch.relu(self.user_proj_1(user_embedding))).unsqueeze(1)
            sum_logits = token_logits + user_logits
            probs, preds = sum_logits.max(dim=-1)
            logits.append(scores)
            xs = torch.cat([xs, preds], dim=1)
            # check if everyone has generated an end token
            all_finished = ((xs == self.end_token_idx).sum(dim=1) > 0).sum().item() == bsz
            if all_finished:
                break
        logits = torch.cat(logits, 1)
        return logits, xs

    def decode_beam_search(self, encoder_states, user_embedding, beam=4):
        bsz = encoder_states[0].shape[0]
        xs = self._starts(bsz).reshape(1, bsz, -1)  # (batch_size, _)
        sequences = [[[list(), list(), 1.0]]] * bsz
        for i in range(self.longest_label):
            # at beginning there is 1 candidate, when i!=0 there are 4 candidates
            if i != 0:
                xs = []
                for d in range(len(sequences[0])):
                    for j in range(bsz):
                        text = sequences[j][d][0]
                        xs.append(text)
                xs = torch.stack(xs).reshape(beam, bsz, -1)  # (beam, batch_size, _)

            with torch.no_grad():
                if i == 1:
                    user_embedding = user_embedding.repeat(beam, 1)
                    encoder_states = (encoder_states[0].repeat(beam, 1, 1),
                                      encoder_states[1].repeat(beam, 1, 1))

                scores, _ = self.decoder(xs.reshape(len(sequences[0]) * bsz, -1), encoder_states)
                scores = scores[:, -1:, :]
                token_logits = F.linear(scores, self.token_embedding.weight)
                user_logits = self.user_proj_2(torch.relu(self.user_proj_1(user_embedding))).unsqueeze(1)
                sum_logits = token_logits + user_logits

            logits = sum_logits.reshape(len(sequences[0]), bsz, 1, -1)
            scores = scores.reshape(len(sequences[0]), bsz, 1, -1)
            logits = torch.nn.functional.softmax(logits)  # turn into probabilities,in case of negative numbers
            probs, preds = logits.topk(beam, dim=-1)
            # (candeidate, bs, 1 , beam) during first loop, candidate=1, otherwise candidate=beam

            for j in range(bsz):
                all_candidates = []
                for n in range(len(sequences[j])):
                    for k in range(beam):
                        prob = sequences[j][n][2]
                        score = sequences[j][n][1]
                        if score == []:
                            score_tmp = scores[n][j][0].unsqueeze(0)
                        else:
                            score_tmp = torch.cat((score, scores[n][j][0].unsqueeze(0)), dim=0)
                        seq_tmp = torch.cat((xs[n][j].reshape(-1), preds[n][j][0][k].reshape(-1)))
                        candidate = [seq_tmp, score_tmp, prob * probs[n][j][0][k]]
                        all_candidates.append(candidate)
                ordered = sorted(all_candidates, key=lambda tup: tup[2], reverse=True)
                sequences[j] = ordered[:beam]

            # check if everyone has generated an end token
            all_finished = ((xs == self.end_token_idx).sum(dim=1) > 0).sum().item() == bsz
            if all_finished:
                break
        logits = torch.stack([seq[0][1] for seq in sequences])
        xs = torch.stack([seq[0][0] for seq in sequences])
        return logits, xs

    def converse(self, batch, mode):
        context_tokens, context_entities, response = batch['context_tokens'], batch['context_entities'], batch[
            'response']
        kg_embedding = self.kg_encoder(None, self.edge_idx, self.edge_type)
        user_embedding = self.encode_user(context_entities, kg_embedding)
        encoder_state = self.dialog_encoder(context_tokens)
        if mode != 'test':
            self.longest_label = max(self.longest_label, response.shape[1])
            logits, preds = self.decode_forced(encoder_state, user_embedding, response)
            logits = logits.view(-1, logits.shape[-1])
            labels = response.view(-1)
            return self.conv_loss(logits, labels), preds
        else:
            _, preds = self.decode_greedy(encoder_state, user_embedding)
            return preds

    def forward(self, batch, mode, stage):
        if len(self.gpu) >= 2:
            self.edge_idx = self.edge_idx.cuda(torch.cuda.current_device())
            self.edge_type = self.edge_type.cuda(torch.cuda.current_device())
        if stage == "conv":
            return self.converse(batch, mode)
        if stage == "rec":
            return self.recommend(batch, mode)

    def freeze_parameters(self):
        freeze_models = [self.kg_encoder, self.kg_attn, self.rec_bias]
        for model in freeze_models:
            for p in model.parameters():
                p.requires_grad = False


import pygame

pygame.init()

screen = pygame.display.set_mode((1080,720))
p1col = (255,122,122)
p2col = (0,0,255)
def num_to_coords(x,y):
    if x == 1 and y == 1:
        return 7
    if x == 1 and y == 2:
        return 4
    if x == 1 and y == 3:
        return 1
    if x == 2 and y == 1:
        return 8
    if x == 2 and y == 2:
        return 5
    if x == 2 and y == 3:
        return 2
    if x == 3 and y == 1:
        return 9
    if x == 3 and y == 2:
        return 6
    if x == 1 and y == 1:
        return 3
def num_to_x(num):
    if num == 1 or num == 4 or num == 7:
        return 1
    if num == 2 or num == 5 or num == 8:
        return 2
    if num == 3 or num == 6 or num == 9:
        return 3
def num_to_y(num):
    if num == 1 or num == 2 or num == 3:
        return 3
    if num == 4 or num == 5 or num == 6:
        return 2
    if num == 7 or num == 8 or num == 9:
        return 1
def make_map():
    font = pygame.font.Font('freesansbold.ttf',64)
    p1txt = font.render('PLAYER',True,(10,10,10))
    p2txt = font.render('COMPUTER',True,(10,10,10))
    pygame.draw.rect(screen,p1col,(0,0,540,120))
    pygame.draw.rect(screen,p2col,(540,0,540,120))
    screen.blit(p1txt,(10,10))
    screen.blit(p2txt,(680,10))
    pygame.draw.rect(screen,(122,122,122),(235,110,610,610))
    pygame.draw.rect(screen,(0,0,0),(245,120,590,590))
    n = 1
    while n <= 2:
        pygame.draw.rect(screen,(122,122,122),(245,110+n*200,600,10))
        pygame.draw.rect(screen,(122,122,122),(235+n*200,110,10,600))

        n+=1
def cross(num):
    n = 0
    while n <= 156:
        pygame.draw.circle(screen, p1col, ((num_to_x(num) - 1) * 200 + 262+n, ((num_to_y(num) - 1)) * 200 + 137+n), 12)
        pygame.draw.circle(screen, p1col, ((num_to_x(num) - 1) * 200 + 262+n, ((num_to_y(num) - 1)) * 200 + 293-n), 12)
        n += 1
def circle(num):
    pygame.draw.circle(screen, p2col, (340 + (num_to_x(num)-1)*200, 215 + (num_to_y(num)-1)*200), 85)
    pygame.draw.circle(screen, (0,0,0), (340 + (num_to_x(num)-1)*200, 215 + (num_to_y(num)-1)*200), 61)

running = True
while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

    pygame.draw.rect(screen, (0, 0, 0), (0, 0, 1080, 720))
    make_map()
    circle(6)
    cross(1)

    pygame.display.flip()

pygame.quit()


import os
import sys
import numpy as np
import matplotlib.pyplot as plt
import pickle
import re

filePattern = re.compile('([^_]*)_([0-9]*).p');

algos = {}

# point this at the output directory
for root, subFolders, files in os.walk('./runtimes/'):
	for file in files:
		m = filePattern.match(file)
		if not m:
			continue
		algo = m.group(1)
		nprob = m.group(2)

		with open(os.path.join(root, file), "rb") as input:
			if nprob not in algos:
				algos[nprob] = {}
			data = pickle.load(input)
			algos[nprob][algo] = data

probNdx = [-1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, -1, 14, 15, 16, 17, -1, 19, 20, 21]
algoIdx = {
	'mine': 1,
	'pyOpt': 2
}

with open('../../octave/assign_to_matlab.m', 'w') as log:
	log.write('h = zeros(0,0,0);')

	for nprob in algos.keys():
		print(nprob)
		for algo in algos[nprob].keys():
			nfev = algos[nprob][algo]['nfev']
			fvals = algos[nprob][algo]['fvals']
			print('\t' + str(algo) + " " + str(nfev))
			for i in range(len(fvals)):
				log.write('h(' + str(i + 1) + ', ' + str(probNdx[int(nprob)]) + ', ' + str(algoIdx[algo]) + ') = ' + str(fvals[i][0]) + ';\n')
			xs = np.asarray(range(nfev))
			ys = np.zeros(nfev)
			for i in range(nfev):
				ys[i] = fvals[i][0]
			plt.plot(xs, ys, label=algo)

		plt.legend(loc='upper right')
		plt.savefig('plots/' + nprob + '_performance.png')
		plt.close()


'''
Created on May 6, 2019

@author: mac
'''

import os
import multiprocessing as mp
import time
import pandas as pd
from PathManager.StockPathManager import GetMergedFolder_Last

def _ReadAllMergedDailyData(fileName):
    stockID = fileName[fileName.rfind('/')+1:fileName.find('.')]
    df = pd.read_excel(fileName, index_col=None, encoding='utf_8_sig')
    print('Read file:', fileName, 'Done!')
    return {stockID: df}

def ReadAllMergedDailyData():
    folder = GetMergedFolder_Last()
    files = os.listdir(folder)
    dataFrames = []
    begin_time = time.time()
    pool = mp.Pool(mp.cpu_count()*2)
    for file_ in files:
        fullpath = os.path.join(folder, file_)
        if fullpath.find('.xlsx') == -1:
            continue
        dataFrames.append(pool.apply_async(
            _ReadAllMergedDailyData, (fullpath, )))
    pool.close()
    pool.join()
    endTime = time.time()
    print('ReadAllMergedDailyData cost:%s' % (endTime - begin_time))
    ret = {}
    for item in dataFrames:
        ret.update(item.get())
    endTime1 = time.time()
    print('ReadAllMergedDailyData to dict:%s' % (endTime1 - endTime))
    return ret

if __name__ == '__main__':
    ret = ReadAllMergedDailyData()
    for stockID in ret:
        print(stockID, ret[stockID])
        input()


import logging
import time
from collections import namedtuple
from urllib.robotparser import RobotFileParser

import requests
from bs4 import BeautifulSoup
from contexttimer import Timer
from dateutil import parser as date_parser
from lazy_property import LazyProperty as lazy_property

from articlix.crawler import USER_AGENT
from articlix.crawler.article import Article
from articlix.crawler.exception import FetchError
from articlix.crawler.url import Url

logger = logging.getLogger(__name__)

class Page:
    def __init__(self, url, headers, text):
        self.url = url
        self.head = headers
        self.text = text

    @lazy_property
    def date(self):
        date = self.head.get('Date', None)
        if date is not None:
            date = date_parser.parse(date)
        return date

    @lazy_property
    def last_modified(self):
        last_modified = self.head.get('Last-Modified', None)
        if last_modified is not None:
            last_modified = date_parser.parse(last_modified)
        return last_modified

    @lazy_property
    def allow_cache(self):
        return not self._have_perm('NOCACHE')

    @lazy_property
    def allow_follow(self):
        return not self._have_perm('NOFOLLOW')

    def links_gen(self):
        if not self.allow_follow:
            return

        for link_node in self.soup.find_all('a'):
            url_str = link_node.get('href')
            if url_str is None: continue
            url = Url(url_str)
            url = url if url.is_absolute else self.url + url
            if url.is_valid: yield url

    @lazy_property
    def soup(self):
        return BeautifulSoup(self.text, 'html.parser')

    def read(self):
        return Article(self)

    def _have_perm(self, perm):
        for tag in self.soup.find_all('ROBOTS', 'meta'):
            if perm in tag['content'].split(', '):
                return True
        return False

Response = namedtuple('Response', 'headers text elapsed ended')

def fetch_raw(url, method='GET', strict=True, timeout=3):
    try:
        with Timer() as t:
            r = requests.request(method, str(url),
                                 headers={'User-Agent': USER_AGENT},
                                 timeout=timeout)
            r.raise_for_status()
        return Response(r.headers, r.text, t.elapsed, t.end)
    except Exception as e:
        if strict:
            raise FetchError("Failed to get data") from e
    return None

class Fetcher:
    def __init__(self, delay=None, use_adaptive=True,
                 adaptive_scale=5, upper_bound=3):
        self.delay = delay
        self.use_adaptive = use_adaptive
        self.adaptive_scale = adaptive_scale
        self.upper_bound = upper_bound

        self._t = None
        self._last_fetched = None

    def __call__(self, url):
        self._cool_down()
        headers, text, self._t, self._last_fetched = fetch_raw(url)
        logging.info("Last fetched time is `%s`.", self._last_fetched)
        return Page(url, headers, text)

    def _cool_down(self):
        if self._last_fetched is None:
            return

        def cutb(t):
            return max(0, min(t, self.upper_bound))

        time_passed = time.time() - self._last_fetched

        if self.delay is not None:
            time.sleep(cutb(self.delay - time_passed))

        if self.delay is None and self.use_adaptive:
            time.sleep(cutb(self.adaptive_scale * self._t - time_passed))

class Site:
    DEFAULT_ROBOTS = 'User-Agent: *\nAllow: /\n'

    def __init__(self, url):
        self.url = url

    def allow_crawl(self, url):
        return self._robots.can_fetch(USER_AGENT, str(url))

    def fetch(self, url):
        return self._fetcher(url)

    @lazy_property
    def _robots(self):
        robots = RobotFileParser()
        r = fetch_raw(self.url.site + 'robots.txt', strict=False)
        if r is None:
            robots.parse(self.DEFAULT_ROBOTS.splitlines())
        else:
            robots.parse(r.text.splitlines())
        return robots

    @lazy_property
    def _fetcher(self):
        crawl_delay = self._crawl_delay
        if crawl_delay is None:
            request_rate = self._request_rate
            if request_rate is not None:
                crawl_delay = request_rate[1] / request_rate[0]
        return Fetcher(crawl_delay)

    @lazy_property
    def _crawl_delay(self):
        try:
            return self._robots.crawl_delay(USER_AGENT)
        except:
            return None

    @lazy_property
    def _request_rate(self):
        try:
            return self._robots.request_rate(USER_AGENT)
        except:
            return None


import wx
from FlowPanel import *
import globals
import os.path
import os

class Finish(FlowPanel):
    def __init__(self, parent, stepNumber):
        self.go = False
        FlowPanel.__init__(self, parent, stepNumber=stepNumber)
        self.parent = parent
        self.parent.finishBtn = wx.Button(self,id=wx.ID_ANY,label="Finish")
        self.parent.interruptBtn = wx.Button(self,id=wx.ID_ANY,label="Interrupt")
        self.parent.interruptBtn.Disable()

        actions = wx.BoxSizer(wx.HORIZONTAL)

        # event for moving forward
        def go(evt):
            self.finished()
            # disable the button once pressed
            self.parent.exit()

        self.parent.finishBtn.Bind(wx.EVT_BUTTON, go)
        actions.Add(self.parent.finishBtn, 1)

        self.parent.interruptBtn.Bind(wx.EVT_BUTTON, go)
        actions.Add(self.parent.interruptBtn, 1)

        self.sizer.Add(actions, flag=wx.EXPAND|wx.RIGHT|wx.LEFT, border=10)

    def do(self):
        self.setStatus(STATUS_READY)
        self.finished()
        return True