from __future__ import annotations

import os
import signal
from abc import abstractmethod
from typing import Optional
from subprocess import Popen

from .ContextBase import ContextBase
from autoinsight.common.CustomTyping import AutomationInstance
from autoinsight.decorator.Log import log
from ..IdentObjectBase import IdentObjectBase

class ProcessBase(ContextBase):
    def __init__(self,
                 processId: Optional[int] = 0,
                 processName: Optional[str] = None,
                 processHandle: Optional[Popen] = None,
                 cmdline: Optional[str] = None,
                 title: Optional[str] = None,
                 workdir: Optional[str] = None,
                 automationInstance: Optional[AutomationInstance] = None,
                 *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._processId: int = processId
        self._processHandle: Optional[Popen] = processHandle
        self._processName: str = processName
        self._cmdline: str = cmdline
        self._title: str = title
        self._workdir: str = workdir
        self._exitcode: int = 0

        if automationInstance:
            self.automationInstance: Optional[AutomationInstance] = automationInstance
            self._isStarted = True
        else:
            self._automationInstance: Optional[AutomationInstance] = None
            self._isStarted: bool = False

    def __repr__(self):
        if not self._repr:
            self._repr = f"name:{self._processName} id:{self.processId} cmdline:{self.cmdline} workdir:{self.workdir}"
        return self._repr

    def __str__(self):
        if not self._str:
            if self.cmdline:
                self._str = os.path.basename(self.cmdline)
            elif self._title:
                self._str = self._title
            elif self._processName:
                self._str = self._processName
            elif self._processId:
                self._str = str(self._processId)
            else:
                self._str = ""

        return self._str

    @property
    def parent(self) -> IdentObjectBase:
        return self._cms.os

    @property
    def automationInstance(self) -> Optional[AutomationInstance]:
        if not self._isStarted:
            self.start()

        if not self._automationInstance:
            self.automationInstance = self._cms.os.find(
                f"{self._processId} {self.cmdline} {self._title} {self.workdir}")
        return self._automationInstance

    @automationInstance.setter
    def automationInstance(self, value: AutomationInstance):
        if value and value != self._automationInstance:
            self._automationInstance = value
            self._processId = value.process_id()

    @property
    def exitcode(self) -> int:
        return self._exitcode

    @property
    def workdir(self) -> str:
        return self._workdir

    @property
    def cmdline(self) -> str:
        return self._cmdline

    @property
    def processName(self) -> str:
        return self._processName

    @property
    def processId(self) -> int:
        return self._processId

    def __enter__(self):
        super().__enter__()
        self.start()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def start(self) -> ProcessBase:
        self._isStarted = True

    @classmethod
    @abstractmethod
    def new(cls, *args, **kwargs) -> ProcessBase:
        pass

    @log
    def close(self):
        if self._processId:
            os.kill(self._processId, signal.SIGTERM)
            self.tearDown()
            return

        if self._processHandle:
            self._processHandle.terminate()
            return

        if self.automationInstance:
            self.automationInstance.close()
            self.tearDown()
            return

    @log
    def kill(self):
        if self._processId:
            os.kill(self._processId, signal.SIGKILL)
            self.tearDown()
            return

    def _printElementsTree(self):
        """
        Print debug element tree information to the console.
        """
        pass


# -*- coding: utf-8 -*-
"""
Conversion extensions for basic LOD models
"""

from datetime import datetime

class ConvertBase:
    """
    Common convert extension for basic LOD models
    """
    __index_sort_import__ = None
    __index_sort_export__ = None
    __load_from_file__ = True
    __load_to_file__ = True
    __load_to_db__ = True
    file_name = None

    @classmethod
    def export_file_name(cls) -> str:
        """
        Generate text file name for export class data
        :return:
        """
        return f"PG_{datetime.now().strftime('%y%m%d%H%M')}_{cls.file_name}"

    @classmethod
    def export_file_path(cls, export_directory) -> str:
        """
        Return full file path according export_file_name
        :param export_directory:
        :return:
        """
        return export_directory + cls.export_file_name()

class ConvertAuthor(ConvertBase):
    """
    Convert extension for basic Author LOD model
    """
    __index_sort_import__ = 1
    __index_sort_export__ = 1
    file_name = "Author.txt"

class ConvertEvent(ConvertBase):
    """
    Convert extension for basic Event LOD model
    """
    __index_sort_import__ = 2
    __index_sort_export__ = 3
    file_name = "LexEvent.txt"

class ConvertKey(ConvertBase):
    """
    Convert extension for basic Key LOD model
    """
    __index_sort_import__ = 3
    __index_sort_export__ = 8
    __load_from_file__ = False
    __load_to_file__ = False

    file_name = "LexEvent.txt"

class ConvertSetting(ConvertBase):
    """
    Convert extension for basic Setting LOD model
    """
    __index_sort_import__ = 4
    __index_sort_export__ = 4
    file_name = "Settings.txt"

class ConvertSyllable(ConvertBase):
    """
    Convert extension for basic Syllable LOD model
    """
    __index_sort_import__ = 5
    __index_sort_export__ = 5
    file_name = "Syllable.txt"

class ConvertType(ConvertBase):
    """
    Convert extension for basic Type LOD model
    """
    __index_sort_import__ = 6
    __index_sort_export__ = 6
    file_name = "Type.txt"

class ConvertDefinition(ConvertBase):
    """
    Convert extension for basic Definition LOD model
    """
    __index_sort_import__ = 8
    __index_sort_export__ = 2
    file_name = "WordDefinition.txt"

class ConvertWord(ConvertBase):
    """
    Convert extension for basic Word LOD model
    """
    __index_sort_import__ = 7
    __index_sort_export__ = 8
    file_name = "Words.txt"

class ConvertWordSpell(ConvertBase):
    """
    Convert extension for basic WordSpell LOD model
    """
    __index_sort_import__ = 9
    __index_sort_export__ = 7
    file_name = "WordSpell.txt"
    __load_to_db__ = False

all_models_convert = sorted(
    ConvertBase.__subclasses__(),
    key=lambda model: model.__index_sort_import__)


#!/usr/bin/env python
# coding: utf-8

# ## **Section 1: Library and Data Imports (Q1)**
#
# - Import your libraries and read the data into a dataframe. Print the head of the dataframe.

# In[1]:

import numpy as np
import pandas as pd
import seaborn as sns
sns.set_theme(color_codes = True)
sns.set_palette('Pastel1')
import matplotlib.pyplot as plt
import time

# In[2]:

from datetime import datetime, timedelta
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import permutation_test_score
from sklearn.metrics import mean_squared_log_error, make_scorer
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from scipy.stats import levene, ttest_ind

# In[3]:

train = pd.read_csv('rossmann-store-sales/train.csv')
store = pd.read_csv('rossmann-store-sales/store.csv')
train['Date'] = pd.to_datetime(train['Date'])
df = pd.merge(train, store, on = 'Store')
df.shape

# In[4]:

df.head()

# ## **Section 2: Effect of Holidays (Q2)**

# To begin with the data exploration process on the Rossmann Stores Sales, we are interested in how store sales are influenced during and before the holidays. Before doing any computation or plotting, it is essential to examine the crucial variables related to our interest. In particular, I believe variables `Open`, `StateHoliday` and `SchoolHoliday` are required for analyzing the relationship between holiday/non-holiday seasons and `Sales`. Thus, it is important to understand precisely the meaning of each variable:
#
# > `Sales` - the turnover for any given day
# >
# > `Open` - an indicator for whether the store was open: 0 = closed, 1 = open
# >
# > `StateHoliday` - indicates a state holiday. Normally all stores, with few exceptions, are closed on state holidays. Note that all schools are closed on public holidays and weekends. a = public holiday, b = Easter holiday, c = Christmas, 0 = None
# >
# > `SchoolHoliday` - indicates if the (Store, Date) was affected by the closure of public schools
#
# After confirming the interpretation of the variables, it is also essential to verify the data values and observe if they match our expectations.

# In[5]:

df1 = df[['Sales', 'StateHoliday', 'SchoolHoliday']]
df1['StateHoliday'].value_counts()

# In[6]:

df1[['SchoolHoliday', 'Sales']].value_counts()

# In[7]:

df1['Sales'].max()

# The variables `StateHoliday` and `SchoolHoliday` are encoded as categorical variables with explicit meanings, which provide a simple verification as shown above. `Sales` is a continuous variable with a minimum of 0 and a maximum of 41551. `Open` will be examined later on. Note that there are two zeros for `StateHoliday`. One of them is a numeric type and the other is a string type. Hence, I changed the 0s with a string variable type to a numeric variable type. Moreover, missing values may play a significant role in the calculation and analysis process and therefore inspection is needed.

# In[8]:

df1['StateHoliday'].replace('0', 0, inplace = True)
df1.isnull().sum()

# No missing value is found for the three variables. Therefore, we can proceed to analyze the relationship for the task. First, I investigated and visualized the store sales for different types of state holidays using boxplots. The first plot on the left shows the store sales distribution for state non-holiday seasons. The second and the third plots depict the store sales for state holiday seasons. In particular, the second boxplot considers sales only for stores having positive sales, and the other investigates store sales in general.

# In[9]:

fig, ax = plt.subplots(1, 3, figsize = (9.7, 6))
sns.boxplot(x = 'StateHoliday', y = 'Sales', data = df1[df1['StateHoliday'] == 0], linewidth = 1, ax = ax[0])
ax[0].set_title('State Non-holiday')
ax[0].set_ylim([0, 40000])

sns.boxplot(x = 'StateHoliday', y = 'Sales', data = df1[(df1['StateHoliday'] != 0) & (df1['Sales'] > 0)], linewidth = 1, ax = ax[1])
ax[1].set_title('State Holiday (Positive Sales)')
ax[1].set_ylim([0, 40000])

sns.boxplot(x = 'StateHoliday', y = 'Sales', data = df1[df1['StateHoliday'] != 0], linewidth = 1, ax = ax[2])
ax[2].set_title('State Holiday')
ax[2].set_ylim([0, 40000])

plt.tight_layout()

# Based on the boxplots, I observed that stores with positive sales during state holidays, in general, have more sales compared to those during state non-holidays. Yet, the mean sales differences between them is not dramatic. Moreover, comparing the second and the third boxplots, we observe that most stores do not have any sales during state holidays. Hence, the analysis result matches with the description of `StateHoliday`:
#
# > Normally all stores, with few exceptions, are closed on state holidays.
#
# We can further justify the conclusion by considering the variable `Open`.

# In[10]:

df1['Open'] = df['Open']
df1['Open'].value_counts()

# In[11]:

df1['Open'].isnull().sum()

# Note that `Open` is a binary variable without missing values. We can use the Pearson product-moment correlation coefficient to test the linear association between `StateHoliday` and `Open`. Remark that because `StateHoliday` is a categorical variable with values 0, a, b and c, cases a, b and c should be combined and denoted as 1 (state holiday) so that the correlation coefficient provides meaningful information.

# In[12]:

df1['StateHoliday'].replace(['a', 'b', 'c'], 1, inplace = True)
df1['StateHoliday'].corr(df1['Open'])

# The Pearson product-moment correlation coefficient is about -0.378. That is, stores tend to close as seasons move from a state non-holiday to a state holiday, which matches with our expectation from the boxplots. Hence, I conclude that people shop slightly more during state holidays for stores with positive sales (open stores in general). Moreover, people shop more before state holidays for stores having no sales (closed stores in general).

# Using a similar procedure, I analyzed whether school holiday seasons may affect store sales.

# In[13]:

fig, ax = plt.subplots(1, 3, figsize = (9.7, 6))
sns.boxplot(x = 'SchoolHoliday', y = 'Sales', data = df1[df1['SchoolHoliday'] == 0], linewidth = 1, ax = ax[0])
ax[0].set_title('School Non-holiday')
ax[0].set_ylim([0, 40000])

sns.boxplot(x = 'SchoolHoliday', y = 'Sales', data = df1[(df1['SchoolHoliday'] == 1) & (df1['Sales'] > 0)], linewidth = 1, ax = ax[1])
ax[1].set_title('School Holiday (Positive Sales)')
ax[1].set_ylim([0, 40000])

sns.boxplot(x = 'SchoolHoliday', y = 'Sales', data = df1[df1['SchoolHoliday'] != 0], linewidth = 1, ax = ax[2])
ax[2].set_title('School Holiday')
ax[2].set_ylim([0, 40000])

plt.tight_layout()

# The boxplots for school holidays above show some different sales patterns than state holidays. More specifically, the boxplot of the stores affected by the closure of public schools with positive sales shows, visually speaking, the same distribution as the boxplot of the stores not affected by school holiday seasons in general appears, with a little more on the average sales. On the other hand, stores have slightly higher sales for affected stores in school holiday seasons than not affected stores. Also, as mentioned in `StateHoliday` variable description:
#
# > Note that all schools are closed on public holidays and weekends.
#
# Thus, given the conclusion that stores having positive sales are more likely to have increased sales during state holidays, people shop a little more during school holidays than before.

# ## **Section 3: Most and Least selling stores (Q3a & Q3b)**

# For this task, we first select the stores having at least 6 months of sales data. I have changed the `Dates` string data type into datetime64 data type so that date time calculations become simpler. Note that the shortest 6 months is 182 days occurred in leap years. Again, we begin by looking into variable data values and missing values.
#
# > `Store` - a unique Id for each store
# >
# > `Date` - the date opening

# In[14]:

df['Date'].value_counts()

# In[15]:

df['Date'].isnull().sum()

# In[16]:

grouped1 = df['Date'].groupby(df['Store'])
all(grouped1.apply(lambda grp: grp.max() - grp.min()) >= timedelta(days = 182))

# Therefore, all stores have at least 6 months of sales data. Now, we can find the top and the bottom five store sales.

# In[17]:

grouped2 = df['Sales'].groupby(df['Store']).sum()
grouped2.nlargest(5)

# In[18]:

grouped2.nsmallest(5)

# The five stores with the highest cumulative sales are Store ID 262, 817, 562, 1114, and 251. The five stores with the least cumulative sales are Store ID 307, 543, 198, 208, and 263.

# ### Section 3(a): Sales Per Week over Time

# In particular, we are interested in plotting store sales per week over time for the two sets of stores found above. Since dates only record the year, month and day of the sales data, calculating the corresponding week number of the year, call it `WeekOfYear`, helps drastically to plot the sales per week over time. To save some effort for future tasks, `Date` is extracted into three separate variables `Year`, `Month` and `Day`.

# In[19]:

df['Year'] = df.Date.map(lambda x: x.year)
df['Month'] = df.Date.map(lambda x: x.month)
df['Day'] = df.Date.map(lambda x: x.day)
df['WeekOfYear'] = df['Date'].dt.isocalendar().week

# For best visualizations, I will plot a line chart for each year for each set of stores with horizontal axes labeling the week in the year.

# In[20]:

grouped3 = df[df['Store'].isin([262, 817, 562, 1114, 251]) & (df['Year'] == 2013)][['Store', 'Sales', 'WeekOfYear']].groupby(['Store', 'WeekOfYear']).mean().reset_index()

fig, ax = plt.subplots(figsize = (12.94, 8))
sns.lineplot(x = 'WeekOfYear', y = 'Sales', hue = 'Store', data = grouped3, palette = 'Set2')
ax.set_title('Sales Per Week for the Highest Five Cumulative Stores in 2013')

# In[21]:

grouped4 = df[df['Store'].isin([262, 817, 562, 1114, 251]) & (df['Year'] == 2014)][['Store', 'Sales', 'WeekOfYear']].groupby(['Store', 'WeekOfYear']).mean().reset_index()

fig, ax = plt.subplots(figsize = (12.94, 8))
sns.lineplot(x = 'WeekOfYear', y = 'Sales', hue = 'Store', data = grouped4, palette = 'Set2')
ax.set_title('Sales Per Week for the Highest Five Cumulative Stores in 2014')

# In[22]:

grouped5 = df[df['Store'].isin([262, 817, 562, 1114, 251]) & (df['Year'] == 2015)][['Store', 'Sales', 'WeekOfYear']].groupby(['Store', 'WeekOfYear']).mean().reset_index()

fig, ax = plt.subplots(figsize = (12.94, 8))
sns.lineplot(x = 'WeekOfYear', y = 'Sales', hue = 'Store', data = grouped5, palette = 'Set2')
ax.set_title('Sales Per Week for the Highest Five Cumulative Stores in 2015')

# In[23]:

grouped6 = df[df['Store'].isin([307, 543, 198, 208, 263]) & (df['Year'] == 2013)][['Store', 'Sales', 'WeekOfYear']].groupby(['Store', 'WeekOfYear']).mean().reset_index()

fig, ax = plt.subplots(figsize = (12.94, 8))
sns.lineplot(x = 'WeekOfYear', y = 'Sales', hue = 'Store', data = grouped6, palette = 'Set2')
ax.set_title('Sales Per Week for the Least Five Cumulative Stores in 2013')

# In[24]:

grouped7 = df[df['Store'].isin([307, 543, 198, 208, 263]) & (df['Year'] == 2014)][['Store', 'Sales', 'WeekOfYear']].groupby(['Store', 'WeekOfYear']).mean().reset_index()

fig, ax = plt.subplots(figsize = (12.94, 8))
sns.lineplot(x = 'WeekOfYear', y = 'Sales', hue = 'Store', data = grouped7, palette = 'Set2')
ax.set_title('Sales Per Week for the Least Five Cumulative Stores in 2014')

# In[25]:

grouped8 = df[df['Store'].isin([307, 543, 198, 208, 263]) & (df['Year'] == 2015)][['Store', 'Sales', 'WeekOfYear']].groupby(['Store', 'WeekOfYear']).mean().reset_index()

fig, ax = plt.subplots(figsize = (12.94, 8))
sns.lineplot(x = 'WeekOfYear', y = 'Sales', hue = 'Store', data = grouped8, palette = 'Set2')
ax.set_title('Sales Per Week for the Least Five Cumulative Stores in 2015')

# ### Section 3(b): Patterns of Sales

# Besides plotting sales per week over time for the two sets of stores, the patterns of sales are also of interest. In particular, the distributions of the two sets of store sales can be compared quantitatively by violin plots. For a better visualization for comparison, the variables are in ascending order of the cumulative sales for each set.

# In[26]:

grouped9 = pd.concat([grouped3, grouped4, grouped5])

fig, ax = plt.subplots(figsize = (9.7, 6))
sns.violinplot(x = 'Store', y = 'Sales', data = grouped9, order = [251, 1114, 562, 817, 262], linewidth = 1)
ax.set_title('Sales Per Week for the Highest Five Cumulative Stores')

# For the set of five stores with the highest cumulative sales, the distributions are all approximately normal, with some store distributions having longer tails. Unsurprisingly, the center/mean of the distributions is in increasing order.

# In[27]:

grouped10 = pd.concat([grouped6, grouped7, grouped8])

fig, ax = plt.subplots(figsize = (9.7, 6))
sns.violinplot(x = 'Store', y = 'Sales', data = grouped10, order = [307, 543, 198, 208, 263], linewidth = 1)
ax.set_title('Sales Per Week for the Least Five Cumulative Stores')

# The violin plot of the set of five stores with the least cumulative sales describes a different picture. Especially, the distributions are bimodal except for Store ID 263. The center/mean of the distributions does not deviate much, quantitatively speaking, except that Store ID 263 has slightly higher mean sales per week.

# ## **Section 4: Distance of the Closest Competitor (Q4)**

# In this task, I am interested in knowing whether the stores further than their closest competitors have better sales per week than the closer ones. The new variable required for this task is `CompetitionDistance`.
#
# `CompetitionDistance` - distance in meters to the nearest competitor store

# In[28]:

df2 = df[['Sales', 'CompetitionDistance', 'WeekOfYear']]
df2['CompetitionDistance'].value_counts()

# In[29]:

df2['CompetitionDistance'].isnull().sum()

# We first look at whether the variables of interest require data cleaning. There are 2642 missing values for `CompetitionDistance`. Because only 0.26% of the data is missing, dropping the missing value observations will not significantly impact the analysis process. We have examined the variables `Sales` and `WeekOfYear` in previous sections.

# In[30]:

df2.dropna(how = 'any', inplace = True)
grouped11 = df2.groupby(['CompetitionDistance', 'WeekOfYear']).mean().reset_index()

fig, ax = plt.subplots(figsize = (9.7, 6))
sns.regplot(x = 'CompetitionDistance', y = 'Sales', data = grouped11, scatter_kws = {'alpha': 0.5}, line_kws = {"color": "red"})
ax.set_title('Sales Per Week Versus Distance of the Closest Competitor')

# Given the above scatter plot with a regression line in red, since the regression line is about horizontal, we can conclude that the average sale per week is independent of the distance of the nearest competitor. In other words, the stores farther from competitors will not have a better sale per week than the closer ones.

# ## **Section 5: Pearson Correlation of Features (Q5)**

# Here I am interested in testing the correlation coefficients between the five selected variables (including sales) to find the feature pairs with the strongest correlations. The five variables are `DayOfWeek`, `Sales`, `Customers`, `Open` and `Promo`.
#
#
# > `DayOfWeek` - the day in a week (e.g. 1 corresponds to Monday and 7 corresponds to Sunday)
# >
# > `Customers` - the number of customers on a given day
# >
# > `Promo` - indicates whether a store is running a promo on that day
#
# The data and the missing values of the three new features above need to be examined.

# In[31]:

df4 = df[['DayOfWeek', 'Sales', 'Customers', 'Open', 'Promo']]
df4[['DayOfWeek', 'Customers', 'Promo']].value_counts()

# In[32]:

df4[['DayOfWeek', 'Customers', 'Promo']].isnull().sum()

# Now, we can calculate the correlation matrix, and plot its heatmap for the five variables. There are many existing metrics to calculate the correlation coefficient. First, we used the Pearson product-moment correlation coefficient metric.

# In[33]:

df4.corr()

# In[34]:

fig, ax = plt.subplots(figsize = (8, 6))
mask = np.triu(np.ones_like(df4.corr(), dtype = np.bool_))[1:, :-1]
sns.heatmap(df4.corr().iloc[1:,:-1], vmin = -1, vmax = 1, cmap = 'Blues', annot = True, fmt = ".2f", linewidth = 0.3, mask = mask)
ax.set_title('Correlation Heatmap (Pearson)')
plt.yticks(rotation = 0)

# From the above heatmap, we can see clearly that the strongest correlation using the Pearson product-moment correlation coefficient metric is the pair `Customers` and `Sales`. `Open` and `Promo` also have strong correlations with `Sales`.

# In[35]:

df4.corr(method = 'spearman')

# In[36]:

fig, ax = plt.subplots(figsize = (8, 6))
mask = np.triu(np.ones_like(df4.corr(method = 'spearman'), dtype = np.bool_))[1:, :-1]
sns.heatmap(df4.corr(method = 'spearman').iloc[1:,:-1], vmin = -1, vmax = 1, cmap = 'Blues', annot = True, fmt = ".2f", linewidth = 0.3, mask = mask)
ax.set_title('Correlation Heatmap (Spearman)')
plt.yticks(rotation = 0)

# Comparing with the correlation matrix using the Spearman's rank correlation coefficient metric, it is about the same as the correlation matrix using the Pearson product-moment correlation coefficient metric. The difference in precision is about 0.01.

# ## **Section 6: Permutation Testing (Q6)**

# In this section, three single-variable regression models will be built with three permutation tests to observe if the predictions on sales based on the regression models are better than chance. The three variables are one likely good, one presumably meaningless, and one at random. Because the regression models only contain one variable, I used simple linear regression to build the models. Moreover, `sklearn.model_selection.permutation_test_score` is a convenient function to perform permutation tests with p-values returned. Root Mean Squared Logarithmic Error (RMSLE) will be used as the statistic to score the models. $\text{RMSLE}=\sqrt{\frac{1}{n}\sum_{i=1}^{n}(\ln{(1 + y_i)}-\ln{(1+\hat{y}_i}))^2}$, where $y_i$ denotes the true sales of the $i$-th store on a single day and $\hat{y}_i$ denotes the corresponding predicted value.
#
# Note that currently since no scoring parameter corresponds to RMSLE in scikit-learn, it is necessary to make a scorer from an existing performance metric. The closest metric is `sklearn.metrics.mean_squared_log_error`. Also, remark from the below description that the internal scoring strategy in the permutation function is maximizing the score function. On the other hand, the rank statistic used here is an error function, which should be minimized. We need to negate the value of the metric in the permutation function because minimizing RMSLE is equivalent to maximizing -RMSLE.
#
# > **sklearn.model_selection.permutation_test_score(estimator, X, y, *, groups=None, cv=None, n_permutations=100, n_jobs=None, random_state=0, verbose=0, scoring=None, fit_params=None)**
# >
# > Evaluate the significance of a cross-validated score with permutations.
# >
# > Permutes targets to generate ‘randomized data’ and compute the empirical p-value against the null hypothesis that features and targets are independent.
# >
# > The p-value represents the fraction of randomized data sets where the estimator performed as well or better than in the original data. A small p-value suggests that there is a real dependency between features and targets which has been used by the estimator to give good predictions. A large p-value may be due to lack of real dependency between features and targets or the estimator was not able to use the dependency to give good predictions.

# In[37]:

linreg = LinearRegression()
df5 = df[['Sales', 'Customers']]
X1 = df5['Customers']
y1 = df5['Sales']

neg_root_mean_squared_log_error = make_scorer(mean_squared_log_error, greater_is_better = False, squared = False)
permutation_test_score(linreg, X1.values.reshape(-1, 1), y1, scoring = neg_root_mean_squared_log_error, random_state = 123)[2]

# I chose `Customers` for the likely good feature. At the 5% level of significance, the p-value is less than 0.05. Thus, we reject the null hypothesis that `Customers` and `Sales` are independent. That is, there is a real dependency between `Customers` and `Sales` which has been used by the estimator to give good predictions.

# In[38]:

df6 = df[['Sales', 'CompetitionDistance']]
df6.dropna(how = 'any', inplace = True)
X2 = df6['CompetitionDistance']
y2 = df6['Sales']

permutation_test_score(linreg, X2.values.reshape(-1, 1), y2, scoring = neg_root_mean_squared_log_error, random_state = 123)[2]

# I selected `CompetitionDistance` for the possibly useless feature. At the 5% level of significance, the p-value is larger than 0.05. Thus, we fail to reject the null hypothesis that `CompetitionDistance` and `Sales` are independent. That is, there is a lack of real dependency between `CompetitionDistance` and `Sales` or the estimator was not able to use the dependency to give good predictions.

# In[39]:

np.random.seed(123)
np.random.choice(train.columns.delete(2))

# In[40]:

df7 = df[['Sales', 'StateHoliday']]
df7['StateHoliday'].replace(['a', 'b', 'c'], 1, inplace = True)
X3 = df7['StateHoliday']
y3 = df7['Sales']

permutation_test_score(linreg, X3.values.reshape(-1, 1), y3, scoring = neg_root_mean_squared_log_error, random_state = 123)[2]

# Finally, the randomly selected variable is `StateHoliday`. `Date` is excluded from the sampling process because it is not a meaningful variable to be regressed on unless different variables are created to store the pieces of information of `Date`. At the 5% level of significance, the p-value is less than 0.05. Thus, we reject the null hypothesis that `StateHoliday` and `Sales` are independent. That is, there is a real dependency between `StateHoliday` and `Sales` which has been used by the estimator to give good predictions.

# ## **Section 7: Interesting findings (Q7)**

# Besides the information obtained from the previous analysis, there are additional interesting findings that can be revealed by visualizations. I listed five more observations based on personal judgment.

# In[41]:

grouped13 = df.groupby('Store')['Sales'].sum()/1000000

fig, ax = plt.subplots(figsize = (9.7, 6))
sns.distplot(grouped13)
ax.set_title('Probability Density Function of Sales')
ax.set(xlabel = 'Sales in Millions')

# In [Section 3(a)](#Section-3(a):-Sales-Per-Week-over-Time) and [Section 3(b)](#Section-3(b):-Patterns-of-Sales), the violin plots are not as normal as I thought based on the large sample statistical theory. In particular, the violin plots for the patterns of sales are mostly bimodally distributed rather than normally distributed. Therefore, I am interested in the distribution of sales. Based on the probability density function of the data, the distribution of `Sales` is approximately normal with some light tails on the right. Also, most stores have a sale around 5 million.

# In[42]:

grouped14 = df[['DayOfWeek', 'Sales']].groupby('DayOfWeek')['Sales'].mean().reset_index()

fig, ax = plt.subplots(figsize = (9.7, 6))
sns.pointplot(x = 'DayOfWeek', y = 'Sales', data = grouped14)
ax.set_title('Sales Versus Day of Week')

# I am not only interested in the distribution of sales in general, but also how many sales every day. I am curious about which day do stores have the highest sales in general. The answer is that stores have the most sales on Monday, and the least on Sunday. The average sale on Sunday is about 0, which can be explained by the fact that stores normally close on Sunday, similar to the explanation in the analysis of sales and holidays in [Section 2](#Section-2:-Effect-of-Holidays-(Q2)). Furthermore, I propose that Monday has the highest sales due to the closing on Sunday, and Friday the second due to the weekend after.

# In[43]:

grouped15 = df[['Sales', 'Year', 'Month']].groupby([df['Year'], df['Month']])['Sales'].mean().reset_index()

fig, ax = plt.subplots(figsize = (9.7, 6))
sns.pointplot(x = 'Month', y = 'Sales', hue = 'Year', data = grouped15)
ax.set_title('Sales Versus Month of Year')

# After analyzing the sales for each day in a week, it is reasonable to investigate which month yields the highest sales. Note that there are only six months of data in 2015. However, the sales drastically go up starting in October and ending in December. The sales go back to the lowest level in January. I believe that the significant increase in sales from October to December is due to the coming winter and Christmas. Moreover, summer holidays for students may be the factor of the sudden increase in sales in July. Last but not least, the lowest sales in January may suggest the shopping break after the shopping season.

# In[44]:

df8 = df[['Sales', 'Customers']]

fig, ax = plt.subplots(figsize = (9.7, 6))
sns.regplot(x = 'Customers', y = 'Sales', data = df8, scatter_kws = {'alpha': 0.5}, line_kws = {"color": "red"})
ax.set_title('Sales Versus Customers')

# Sometimes I observe that luxury stores, in general, do not have a lot of customers in a given time frame. Therefore, I wonder if the number of customers can affect the sales. Based on the scatterplot above, there is a strong positive relationship between the number of customers and sales, as ready showed in the correlation heatmap in [Section 5](#Section-5:-Pearson-Correlation-of-Features-(Q5)). The interesting property I want to discuss in the visualization is the outlier on the right. I am curious about the store information corresponding to that outlier. For example, what kind of store would there be a lot of customers, but not that many sales, comparatively speaking. Also, there is a chance that the outlier exists because the information is wrongly entered. Hence, additional investigation on that outlier is required.

# In[45]:

df9 = df[['Sales', 'StoreType']]

fig, ax = plt.subplots(figsize = (9.7, 6))
sns.barplot(x = 'StoreType', y = 'Sales', data = df9, order = ['a', 'b', 'c', 'd'])
ax.set_title('Sales Versus Store Type')

# In the previous graph, I thought about whether different kinds of stores can impact sales. There is a variable `StoreType` that provides additional information about the types of stores.
#
# > `StoreType` - differentiates between 4 different store models: a, b, c, d
#
# From the bar chart, store Type b has the highest sales, much higher than the rest. There is not a significant difference in sales among store Types a, c and d.

# ## **Section 8: Train Test Split and Modelling (Q8)**

# To begin with this task, I first selected some variables that can help predict sales based on my judgment in the previous tasks. The variables are `DayOfWeek`, `Customers`, `Open`, `Promo`, `StateHoliday` and `SchoolHoliday`.

# In[46]:

df10_X = train[['DayOfWeek', 'Customers', 'Open', 'Promo', 'StateHoliday', 'SchoolHoliday']]
df10_X['StateHoliday'].replace(['a', 'b', 'c'], 1, inplace = True)
df10_y = train['Sales']
df10_X.isnull().sum()

# It is a better idea to reduce the number of features for training models to predict `Sales`. I used `SelectKBest`, a univariate feature selection method, to choose only the $k$ highest scoring features. The scoring function used to evaluate the features is the sample's ANOVA F-value. I am interested in training four features.

# In[47]:

test = SelectKBest(score_func = f_classif, k = 4)
fit = test.fit(df10_X, df10_y)
np.set_printoptions(precision = 2, suppress = True)
fit.scores_

# The top four scoring features are `DayOfWeek`, `Customers`, `Open` and `Promo`. Next, we can split the dataset into a training and a validation set. For this task, The validation set will contain all the data from May, June, and July of 2015, and the training set will
# have of the rest of the data. For the following two models, some combinations of hyperparameters will be tried and see if those lead to a better model. The evaluation statistic used is the Root Mean Square Percentage Error (RMSPE) given by $\text{RMSPE} = \sqrt{\frac{1}{n}\sum_{i=1}^{n}\left(\frac{y_i-\hat{y}_i}{y_i}\right)^2}$, where $y_i$ denotes the true sales of the $i$-th store on a single day and $\hat{y}_i$ denotes the corresponding predicted value. Remark that the denominator in the above equation cannot be 0, and hence any day and store with 0 sales is ignored in scoring.

# In[48]:

train['Year'] = train.Date.map(lambda x: x.year)
train['Month'] = train.Date.map(lambda x: x.month)

X_train = train[['DayOfWeek', 'Customers', 'Open', 'Promo']][~((train['Year'] == 2015) & (train['Month'].isin([5, 6, 7])))]
y_train = pd.DataFrame(train['Sales'][~((train['Year'] == 2015) & (train['Month'].isin([5, 6, 7])))], columns = ['Sales'])
X_test = train[['DayOfWeek', 'Customers', 'Open', 'Promo']][(train['Year'] == 2015) & (train['Month'].isin([5, 6, 7]))]
y_test = pd.DataFrame(train['Sales'][(train['Year'] == 2015) & (train['Month'].isin([5, 6, 7]))], columns = ['Sales'])

# ### Section 8(a): Decision Tree

# I selected the decision tree algorithm to train the first prediction model.
#
# > **sklearn.tree.DecisionTreeRegressor(*, criterion='squared_error', splitter='best', max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=None, random_state=None, max_leaf_nodes=None, min_impurity_decrease=0.0, ccp_alpha=0.0)**

# In[49]:

model_1a = DecisionTreeRegressor(random_state = 123)
model_1a.get_params()

# The first set of hyperparameters I chose is the default, except for `random_state`, which is used to reproduce results.

# In[50]:

start = time.process_time()
model_1a.fit(X_train, y_train)
time.process_time() - start

# I also timed the training process for the default hyperparameters for the comparison of performance later on.

# In[51]:

y_predict1a = pd.DataFrame(model_1a.predict(X_test), index = y_test.index, columns = ['SalesPredicted'])
df11a = y_test.join(pd.DataFrame(y_predict1a))
df11a.drop(df11a[df11a['Sales'] == 0].index, inplace = True)
np.sqrt(np.mean(np.square(((df11a['Sales'] - df11a['SalesPredicted']) / df11a['Sales']))))

# The RMSPE for the default hyperparameters is about 0.205.

# In[52]:

model_1b = DecisionTreeRegressor(min_samples_split = 5, min_samples_leaf = 2, random_state = 123)
start = time.process_time()
model_1b.fit(X_train, y_train)
time.process_time() - start

# The second set of hyperparameters considers a different value for `min_samples_split` and `min_samples_leaf`. The default settings for the two hyperparameters are 2 and 1, respectively.

# In[53]:

y_predict1b = pd.DataFrame(model_1b.predict(X_test), index = y_test.index, columns = ['SalesPredicted'])
df11b = y_test.join(pd.DataFrame(y_predict1b))
df11b.drop(df11b[df11b['Sales'] == 0].index, inplace = True)
np.sqrt(np.mean(np.square(((df11b['Sales'] - df11b['SalesPredicted']) / df11b['Sales']))))

# In[54]:

model_1c = DecisionTreeRegressor(min_samples_split = 10, min_samples_leaf = 4, random_state = 123)
start = time.process_time()
model_1c.fit(X_train, y_train)
time.process_time() - start

# The last set of hyperparameters considers another pair of `min_samples_split` and `min_samples_leaf` to confirm whether the increasing values of the two hyperparameters can decrease the RMSPE.

# In[55]:

y_predict1c = pd.DataFrame(model_1c.predict(X_test), index = y_test.index, columns = ['SalesPredicted'])
df11c = y_test.join(pd.DataFrame(y_predict1c))
df11c.drop(df11c[df11c['Sales'] == 0].index, inplace = True)
np.sqrt(np.mean(np.square(((df11c['Sales'] - df11c['SalesPredicted']) / df11c['Sales']))))

# Below is a summary table for the three hyperparameter settings in the decision tree model with their respective RMSPE and timing performance. The third model with hyperparameters `min_samples_split` = 10 and `min_samples_leaf` = 4 gives the best performance because it has a slightly lower RMSPE.
#
# |   Model  | RMSPE  |  Time   |
# |:---------:|:------:|:--------:|
# | Model 1a  | 0.2049 | 0.781 s  |
# | Model 1b  | 0.2038 | 0.781 s  |
# | Model 1c  | 0.2030 | 0.797 s  |

# ### Section 8(b): Random Forest

# The next model will be trained by the random forest algorithm.
#
# > **sklearn.ensemble.RandomForestRegressor(n_estimators=100, *, criterion='squared_error', max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features='auto', max_leaf_nodes=None, min_impurity_decrease=0.0, bootstrap=True, oob_score=False, n_jobs=None, random_state=None, verbose=0, warm_start=False, ccp_alpha=0.0, max_samples=None)**

# In[56]:

model_2a = RandomForestRegressor(random_state = 123)
model_2a.get_params()

# Again, I used the default hyperparameters, except for `random_state`, to train the model at the beginning.

# In[57]:

start = time.process_time()
model_2a.fit(X_train, y_train)
time.process_time() - start

# In[58]:

y_predict2a = pd.DataFrame(model_2a.predict(X_test), index = y_test.index, columns = ['SalesPredicted'])
df12a = y_test.join(pd.DataFrame(y_predict2a))
df12a.drop(df12a[df12a['Sales'] == 0].index, inplace = True)
np.sqrt(np.mean(np.square(((df12a['Sales'] - df12a['SalesPredicted']) / df12a['Sales']))))

# In[59]:

model_2b = RandomForestRegressor(n_estimators = 10, random_state = 123)
start = time.process_time()
model_2b.fit(X_train, y_train)
time.process_time() - start

# The second set of hyperparameters considers `n_estimator` = 10. The default value for `n_estimator` is 100.

# In[60]:

y_predict2b = pd.DataFrame(model_2b.predict(X_test), index = y_test.index, columns = ['SalesPredicted'])
df12b = y_test.join(pd.DataFrame(y_predict2b))
df12b.drop(df12b[df12b['Sales'] == 0].index, inplace = True)
np.sqrt(np.mean(np.square(((df12b['Sales'] - df12b['SalesPredicted']) / df12b['Sales']))))

# In[61]:

model_2c = RandomForestRegressor(min_samples_split = 10, min_samples_leaf = 4, random_state = 123)
start = time.process_time()
model_2c.fit(X_train, y_train)
time.process_time() - start

# The last set of hyperparameters considers a different value for `min_samples_split` and `min_samples_leaf`. The default settings for the two hyperparameters are 2 and 1, respectively.

# In[62]:

y_predict2c = pd.DataFrame(model_2c.predict(X_test), index = y_test.index, columns = ['SalesPredicted'])
df12c = y_test.join(pd.DataFrame(y_predict2c))
df12c.drop(df12c[df12c['Sales'] == 0].index, inplace = True)
np.sqrt(np.mean(np.square(((df12c['Sales'] - df12c['SalesPredicted']) / df12c['Sales']))))

# Below is a summary table for the three hyperparameter settings in the random forest model with their respective RMSPE and timing performance. The third model with hyperparameters `min_samples_split` = 10 and `min_samples_leaf` = 4 gives the best RMSPE, but the time is much longer than the decision tree algorithms. On the other hand, Model 2b has a better time, but still, RMSPE is slightly higher than Model 1c.
#
# |  Model   | RMSPE  |  Time    |
# |:---------:|:------:|:---------:|
# | Model 2a  | 0.2034 | 70.047 s  |
# | Model 2b  | 0.2038 | 7.0313 s  |
# | Model 2c  | 0.2021 | 72.094 s  |

# ## **Section 9: t-test (Q9)**

# After we obtained the better models by tuning hyperparameters in the two algorithms above, we may want to test whether the difference of their predictions is statistically significant. In this case, performing a t-test is the best way to verify that. Remark that the two independent samples t-test requires to know if the variances of the two populations are the same. Thus, we need to test the equal-variance assumption for the two populations using Levene's test. To obtain a robust result, we have to select the best case among the three possible variations in Levene's test depending on the population distributions.
#
# > **scipy.stats.levene(*args, center='median', proportiontocut=0.05)**
# >
# > **scipy.stats.ttest_ind(a, b, axis=0, equal_var=True, nan_policy='propagate', permutations=None, random_state=None, alternative='two-sided', trim=0)**

# In[63]:

fig, ax = plt.subplots(1, 2, figsize = (12, 6))
sns.distplot(df11c, ax = ax[0])
ax[0].set_title('Probability Density Function of Predicted Sales (Decision Tree)')
ax[0].set(xlabel = 'Sales')

sns.distplot(df12c, ax = ax[1])
ax[1].set_title('Probability Density Function of Predicted Sales (Random Forest)')
ax[1].set(xlabel = 'Sales')

plt.tight_layout()

# Based on the above probability density functions, the predicted sales using the two algorithms are approximately normal with a small tail on the right. Therefore, the default setting `center='median'` is appropriate in Levene's test.

# In[64]:

levene(df11c['SalesPredicted'], df12c['SalesPredicted']).pvalue

# At the 5% level of significance, since the p-value is greater than 0.05, we fail to reject the null hypothesis that all input samples are from populations with equal variances. That is, we can assume an equal-variance assumption in the t-test below.

# In[65]:

ttest_ind(df11c['SalesPredicted'], df12c['SalesPredicted']).pvalue

# At the 5% level of significance, since the p-value is greater than 0.05, we fail to reject the null hypothesis that the two independent samples have identical average (expected) values. That is, we are 95% confident that the two predictions are not significantly different.

# ## **Section 10: Screenshots (Q10)**

# Lastly, we want to use the two prediction models built above to obtain a prediction for the observations in the testing set. Note that we have to make sure no missing values are present in the testing data so that we can move on to the prediction.

# In[66]:

test = pd.read_csv('rossmann-store-sales/test.csv')
result = test['Id']
X_test_1 = test[['Store', 'DayOfWeek', 'Open', 'Promo']]
X_test_1.isnull().sum()

# There are 11 missing values for `Open`. Because the number of missing values is small (0.035%), I will use random value imputation based on the distribution of `Open` data values.

# In[67]:

np.random.seed(123)
X_test_1['Open'].fillna(np.random.choice((0, 1), 1, p = [0.125, 0.875])[0], inplace = True)
grouped16 = train[['Store', 'DayOfWeek', 'Customers']].groupby(['Store', 'DayOfWeek']).mean()
X_test_1 = pd.merge(X_test_1, grouped16, on = ['Store', 'DayOfWeek'])
X_test_1 = X_test_1.drop('Store', 1)
X_test_1 = X_test_1[['DayOfWeek', 'Customers', 'Open', 'Promo']]

y_predict_test_1 = model_1c.predict(X_test_1)
result_1 = pd.concat([result, pd.DataFrame(y_predict_test_1, columns = ['Sales'])], axis = 1)
result_1.to_csv('model_1.csv', index = False)

y_predict_test_2 = model_2c.predict(X_test_1)
result_2 = pd.concat([result, pd.DataFrame(y_predict_test_2, columns = ['Sales'])], axis = 1)
result_2.to_csv('model_2.csv', index = False)

# Public Score: 0.72378
#
# Private Score: 0.70639
#
# Kaggle profile link: https://www.kaggle.com/garylikai
#
# Screenshot(s): ![Screenshot](Kaggle_Challenge.png)


# pylint: disable=no-member, wrong-import-position
"""
    Thermal-fluid specific material classes
"""
import xml.etree.ElementTree as ET

from jax.config import config

config.update("jax_enable_x64", True)
import jax.numpy as jnp

from srlife import materials

class ThermalFluidMaterial:
    """Thermal fluid material properties

    The goal of this class is to provide film coefficient as a function of
    fluid velocity, temperature, and tube inner radius.

    The current implementation works from the
    - Heat capacity (constant pressure)
    - Density
    - Dynamic viscosity
    - Conductivity

    as a function of temperature (in K).
    """

    def __init__(
        self,
        film_min=1e-8,
        T_max=2000.0,
        T_min=0.0,
        laminar_cutoff=2e3,
        laminar_value=4.01,
    ):
        self.film_min = film_min
        self.T_max = T_max
        self.T_min = T_min
        self.laminar_cutoff = laminar_cutoff
        self.laminar_value = laminar_value

    def T_effective(self, T):
        """
        Cutoff T to avoid issues with the correlations
        """
        return jnp.maximum(jnp.minimum(T, self.T_max), self.T_min)

    @classmethod
    def load(cls, fname, modelname):
        """Load model from an XML file

        Parameters:
            fname:      XML filename
            modelname:  model name in the file
        """
        tag, mtype = materials.find_name(fname, modelname)
        data = materials.load_node(tag)[modelname]

        if mtype == "PolynomialThermalFluidMaterial":
            return PolynomialThermalFluidMaterial.load(data)
        else:
            raise ValueError("Unknown ThermalFluidMaterial type %s" % mtype)

    def save(self, fname, modelname):
        """Save model to an XML file

        Parameters:
            fname:      XML filename
            modelname:  model name in the file
        """
        root = ET.Element("models")

        materials.save_node(
            modelname, self.data(), root, attrib={"type": self.model_type}
        )
        tree = ET.ElementTree(element=root)
        tree.write(fname)

    def film_coefficient(self, T, u, r):
        """Film coefficient

        Calculate the film coefficient as a function of
        temperature, flow velocity, and tube diameter

        Parameters:
            T:      temperature, in K
            u:      flow velocity, in mm/hr
            r:      tube inner radius, in mm
        """
        nu = self.nusselt(T, u, r)

        return jnp.maximum(nu * self.k(T) / (2.0 * r), self.film_min)

    def reynolds(self, T, u, r):
        """Reynolds number

        Parameters:
            T:      temperature, in K
            u:      flow velocity, in mm/hr
            r:      tube inner radius, in mm
        """
        return self.rho(T) * u * 2.0 * r / self.mu(T)

    def prandtl(self, T):
        """Prandtl number

        Parameters:
            T:      temperature, in K
        """
        return self.cp(T) * self.mu(T) / self.k(T)

    def nusselt(self, T, u, r):
        """Nusselt number

        This is really where the work gets done in calculating the film
        coefficient and different correlations are  possible.

        Right now we use the Gnielinski correlation for turbulent flow

        Parameters:
            T:      temperature, in K
            u:      flow velocity, in m/s
            r:      tube inner radius, in mm
        """
        re = self.reynolds(self.T_effective(T), u, r)
        pr = self.prandtl(self.T_effective(T))
        f = (0.79 * jnp.log(re) - 1.64) ** -2.0

        turbulent = ((f / 8.0) * (re - 1000.0) * pr) / (
            1.0 + 12.7 * (f / 8.0) ** 0.5 * (pr ** (2.0 / 3.0) - 1.0)
        )

        turbulent.at[re < self.laminar_cutoff].set(self.laminar_value)

        return turbulent

class PolynomialThermalFluidMaterial(ThermalFluidMaterial):
    """Thermal-fluid relations correlated with polynomials

    The polynomial definitions use C, the class handles converting K -> C

    Parameters:
        cp_poly (array): coefficients for the heat capacity, in numpy order
        rho_poly (array): coefficients for the density, in numpy order
        mu_poly (array): coefficients for the viscosity, in numpy order
        k_poly (array): coefficients for the conductivity, in numpy order
        film_min (Optional[float]): lower cutoff on the value of the film coefficient
        T_max (Optional[float]): upper cutoff on temperature used in finding values
        T_min (Optional[float]): lower cutoff on temperature used in finding values
        laminar_cutoff (Optional[float]): Reynolds number upper limit for laminar flow
        laminar_value (Optional[float]): Nusslet number to use in laminar flow
    """

    def __init__(self, cp_poly, rho_poly, mu_poly, k_poly, **kwargs):
        super().__init__(**kwargs)
        self.cp_poly = jnp.asarray(cp_poly)
        self.rho_poly = jnp.asarray(rho_poly)
        self.mu_poly = jnp.asarray(mu_poly)
        self.k_poly = jnp.asarray(k_poly)

        self.model_type = "PolynomialThermalFluidMaterial"

    def data(self):
        """
        Reduce the class down to a dictionary of strings for
        serialization
        """
        return {
            "cp_poly": materials.string_array(self.cp_poly),
            "rho_poly": materials.string_array(self.rho_poly),
            "mu_poly": materials.string_array(self.mu_poly),
            "k_poly": materials.string_array(self.k_poly),
            "film_min": str(self.film_min),
            "T_max": str(self.T_max),
            "T_min": str(self.T_min),
            "laminar_cutoff": str(self.laminar_cutoff),
            "laminar_value": str(self.laminar_value),
        }

    @classmethod
    def load(cls, values):
        """Create from a dictionary

        Parameters:
            values:     dictionary of values
        """
        # Assemble kwargs
        kwargs = {
            k: float(values[k])
            for k in ["film_min", "T_max", "T_min", "laminar_cutoff", "laminar_value"]
            if k in values
        }

        return cls(
            materials.destring_array(values["cp_poly"]),
            materials.destring_array(values["rho_poly"]),
            materials.destring_array(values["mu_poly"]),
            materials.destring_array(values["k_poly"]),
            **kwargs
        )

    def cp(self, T):
        """Heat capacity as a function of temperature in K

        Parameters:
            T:      temperature, in K
        """
        return jnp.polyval(self.cp_poly, T)

    def rho(self, T):
        """Density as a function of temperature in K

        Parameters:
            T:      temperature, in K
        """
        return jnp.polyval(self.rho_poly, self.T_effective(T))

    def mu(self, T):
        """Dynamic viscosity, as a function of temperature in K

        Parameters:
            T:  temperature, in K
        """
        return jnp.polyval(self.mu_poly, T)

    def k(self, T):
        """Conductivity, as a function of temperature in K

        Parameters:
            T:  temperature, in K
        """
        return jnp.polyval(self.k_poly, T)


import json
import logging
from pathlib import Path
from typing import Iterator, Union

from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import HumanMessage

from langchain_community.chat_loaders.base import BaseChatLoader

logger = logging.getLogger(__file__)

class SingleFileFacebookMessengerChatLoader(BaseChatLoader):
    """Load `Facebook Messenger` chat data from a single file.

    Args:
        path (Union[Path, str]): The path to the chat file.

    Attributes:
        path (Path): The path to the chat file.

    """

    def __init__(self, path: Union[Path, str]) -> None:
        super().__init__()
        self.file_path = path if isinstance(path, Path) else Path(path)

    def lazy_load(self) -> Iterator[ChatSession]:
        """Lazy loads the chat data from the file.

        Yields:
            ChatSession: A chat session containing the loaded messages.

        """
        with open(self.file_path) as f:
            data = json.load(f)
        sorted_data = sorted(data["messages"], key=lambda x: x["timestamp_ms"])
        messages = []
        for m in sorted_data:
            messages.append(
                HumanMessage(
                    content=m["content"], additional_kwargs={"sender": m["sender_name"]}
                )
            )
        yield ChatSession(messages=messages)

class FolderFacebookMessengerChatLoader(BaseChatLoader):
    """Load `Facebook Messenger` chat data from a folder.

    Args:
        path (Union[str, Path]): The path to the directory
            containing the chat files.

    Attributes:
        path (Path): The path to the directory containing the chat files.

    """

    def __init__(self, path: Union[str, Path]) -> None:
        super().__init__()
        self.directory_path = Path(path) if isinstance(path, str) else path

    def lazy_load(self) -> Iterator[ChatSession]:
        """Lazy loads the chat data from the folder.

        Yields:
            ChatSession: A chat session containing the loaded messages.

        """
        inbox_path = self.directory_path / "inbox"
        for _dir in inbox_path.iterdir():
            if _dir.is_dir():
                for _file in _dir.iterdir():
                    if _file.suffix.lower() == ".json":
                        file_loader = SingleFileFacebookMessengerChatLoader(path=_file)
                        for result in file_loader.lazy_load():
                            yield result


from django.urls import path, include
from . import views

urlpatterns = [
    path('', views.projects, name = 'projects'),
    path('homepage', views.homepage, name = 'homepage'),
    path('studentlist', views.liststudents, name='liststudent'),
    path('delete/<int:id>', views.delete, name='delete'),
    path('update/<int:id>', views.update, name='update'),
    path('update/updaterecord/<int:id>',
         views.updaterecord, name='updaterecord'),
    path('addstudent/', views.addstudent, name='addstudent'),
    path('addstudent/addrec/', views.addrec, name='addrec'),
    path('addCourses/', views.addCourses, name='addCourses'),
    path('addCourses/addrecord/', views.addrecord, name='addrecord'),
    path('searchStudent/', views.searchStudent, name='searchStudent'),
    path('searchStudent/registration/<int:id>',
         views.registration, name='registration'),
    path('searchStudent/registration/registration_update/<int:id>',
         views.registration_update, name='registration_update'),
    path('listcourses', views.listcourses, name='listcourses'),
    path('delete1/<int:id>', views.delete1, name='delete1'),
    path('update1/<int:id>', views.update1, name='update1'),
    path('update1/updaterecord1/<int:id>',
         views.updaterecord1, name='updaterecord1'),
]


import numpy as np
import matplotlib.pylab as plt
from sys import argv
s = [10,20,30,40,50,60,70]

data45  = np.genfromtxt("ode_45.txt", delimiter = ",")
x45 = data45[:,0]
y45 = data45[:,1]
f = plt.figure()
plt.plot(x45,y45)
plt.xlabel("x")
plt.ylabel("y")
plt.title("45 degrees")
f.savefig("45.png")
plt.close()

f = plt.figure()

for i in s:
    data  = np.genfromtxt("ode_"+str(i)+".txt", delimiter = ",")
    x = data[:,0]
    y = data[:,1]
    plt.plot(x,y, label = i)

plt.xlabel("x")
plt.ylabel("y")
plt.legend()
f.savefig("otherAngles.png")
plt.close()


import logging

from miro import app
from miro import transcode

class MiroLibraryManager:
    '''MiroLibraryManager
    Manages objects to transcode from miro's library
    '''

    def __init__(self, request_path_func):
        self.request_path_func = request_path_func
        self.transcodes = dict()

    def get_playlist(self, itemid):
        item_info = app.item_info_cache.get_info(itemid)

        if not item_info.video_path:
            logging.warn('No video_path for itemid %d' % itemid)
            return None

        transcode_object = self.start_transcode(item_info.video_path, itemid)
        self.transcodes[itemid] = transcode_object

        logging.debug('Playlist retreived for itemid %d' % itemid)
        return transcode_object.playlist

    def get_chunk(self, itemid, chunk):
        try:
            transcode_object = self.transcodes[itemid]
        except(KeyError):
            logging.warn('Invalid get_chunk for itemid %d' % itemid)
            return None

        logging.debug('Chunk %d retreived for itemid %d' % (chunk, itemid))
        return transcode_object.get_chunk()

    def start_transcode(self, media_path, itemid):
        transcode_object = self._setup_transcode(media_path, itemid)
        transcode_object.transcode()
        return transcode_object

    def _setup_transcode(self, media_path, itemid):
        yes, media_info = transcode.needs_transcode(media_path)
        logging.debug('media info: %s', media_info)
        assert(media_info)
        transcode_object = transcode.TranscodeObject(media_path, itemid, 0, None,
                                                     media_info, self.request_path_func)
        return transcode_object


import smtplib
import time
from datetime import datetime
import board
import busio
import adafruit_vl6180x

time.sleep(2)
i2c = busio.I2C(board.SCL, board.SDA)
sensor = adafruit_vl6180x.VL6180X(i2c)

door = False
count = 0

def sendMsg(msg):
    conn = smtplib.SMTP('smtp.gmail.com',587)
    conn.ehlo()
    conn.starttls()
    conn.login('from email', 'password')
    conn.sendmail('to email','to email',f'Subject: Alert! \n\n Door {msg} at {(datetime.now()).strftime("%H:%M:%S")}')
    conn.quit()

while True:
    range_mm = sensor.range
    print("Range: {0}mm".format(range_mm))
    if range_mm == 0  and door == False:
        count += 1
        print("Activated")
        if count % 2 == 0:
            msg = "Closed"
        else:
            msg = "Opened"
        sendMsg(msg)
        door = True

    elif range_mm == 255:
        door = False

    time.sleep(.25)


from django.shortcuts import get_object_or_404, render
from django.http import HttpResponseRedirect
from django.urls import reverse
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from ..models import Customer, ProductType, Product, ProductOrder, PaymentType, Order
from django.utils import timezone
from datetime import datetime, timedelta
from ecomm.forms import AddPayment
from django.db import connection
# from ..forms import UserSettings

@login_required
def editSettingsForm(request):
    """R Lancaster[Information from DB gets passed to the edit Settings Form]

    Arguments:
        request

    Returns:
        render sends info from the Customer table to the edit Settings form page.
    """
    categories = ProductType.objects.raw('''SELECT cat.id, cat.name FROM ecomm_producttype cat''')
    currentUserId = request.user.id
    customer = Customer.objects.raw('''SELECT * FROM ecomm_customer where user_id=%s''',[currentUserId])[0]
    context = {'customer' : customer, 'categories' : categories}
    return render(request, 'ecomm/editSettings.html', context)
    # userSettings = UserSettings()
    # return render(request, 'ecomm/editSettings.html', {"form": userSettings.as_p()})

@login_required
def editSettings(request):
    """R Lancaster[This method is executed when the user saves the updated user settings on the user settings update form page]

    Arguments:
        request

    Returns:
        User is redirected to main User Settings page.
    """

    currentUserId = request.user.id
    custSettings = Customer.objects.raw('''SELECT * FROM ecomm_customer where user_id=%s''',[currentUserId])[0]
    uSettings = User.objects.raw('''SELECT * FROM auth_user where id=%s''',[currentUserId])[0]
    uSettings.last_name = request.POST['lastName']
    custSettings.phone = request.POST['phone']
    custSettings.address = request.POST['address']
    custSettings.save()
    uSettings.save()
    return HttpResponseRedirect(reverse('ecomm:userSettings'))

def editPaymentsForm(request):
    """R Lancaster[method populates Edit Payment Methods form with existing payment options]

    Arguments:
        request

    Returns:
        render, context of existing payment options
    """

    currentUserId = request.user.id
    categories = ProductType.objects.raw('''SELECT cat.id, cat.name FROM ecomm_producttype cat''')
    payments = PaymentType.objects.raw('''
        SELECT * from ecomm_paymentType
        JOIN ecomm_customer
        ON ecomm_customer.user_id  = ecomm_paymentType.customer_id
        JOIN auth_user
        ON  auth_user.id = ecomm_customer.user_id
        WHERE auth_user.id =%s
        AND ecomm_paymentType.deletedOn = ''
    ''', [currentUserId])
    context = {'payments' : payments, 'categories': categories}
    return render(request, 'ecomm/editPayments.html', context)

def deletePayment(request, payment_id):
    """R Lancaster[method adds a deletedOn date to the PaymentType table, therefore "soft deleting" the payment option]

    Arguments:
        request
        payment_id

    Returns:
        redirects back to the Edit Payments form
    """

    todaysDate = datetime.now()
    formattedDate = str(todaysDate)[0:10]
    payment = PaymentType.objects.raw('''
        SELECT * FROM ecomm_paymentType
        WHERE id = %s
        ''', [payment_id])[0]
    payment.deletedOn = formattedDate
    payment.save()
    return HttpResponseRedirect(reverse('ecomm:editPaymentsForm'))

def addPaymentsForm(request):
    """R Lancaster[directs the user to the Add Payment template]

    Arguments:
        request

    Returns:
        render
    """
    categories = ProductType.objects.raw('''SELECT cat.id, cat.name FROM ecomm_producttype cat''')
    form = AddPayment()
    return render(request, 'ecomm/addPayment.html', {"form" : form, "categories": categories})

def addPayment(request):
    """R Lancaster[method takes values entered on Add Payment form and inserts a row into the PaymentType table]

    Arguments:
        request

    Returns:
        redirects user to the Edit Payments Form page
    """

    currentUserId = request.user.id
    name = request.POST['name']
    cardNum = request.POST['cardNum']
    # Creates a new payment type row
    payment_sql = ''' INSERT INTO ecomm_paymenttype VALUES (%s, %s, %s, %s, %s)'''
    with connection.cursor() as cursor:
        cursor.execute(payment_sql, [None, name, cardNum, '', currentUserId])
    return HttpResponseRedirect(reverse('ecomm:editPaymentsForm'))


import asyncio
import os
from datetime import datetime as dt

import aioredis
from aiogram import Bot, Dispatcher
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from aiogram.dispatcher.filters.state import State, StatesGroup
from aiogram.dispatcher.middlewares import LifetimeControllerMiddleware
from aiogram.types import ReplyKeyboardMarkup, ReplyKeyboardRemove
from dotenv import load_dotenv
from loguru import logger
from sqlalchemy.exc import IntegrityError

from database.redis_db.key_schema import KeySchema
from database.sql_db.data_manager import DataManager

logger.add(
    'logs.json', format='{time} {level} {message}',
    level='INFO', rotation='50 KB', compression='zip', serialize=True
)

load_dotenv()

class RedisMiddleware(LifetimeControllerMiddleware):
    def __init__(self):
        super().__init__()

    async def pre_process(self, obj, data, *args):
        pool = await aioredis.create_pool(
            "redis://localhost",
            encoding='utf-8'
        )
        data['redis'] = aioredis.Redis(pool_or_conn=pool)
        data['async_session'] = manager.async_session

TELEGRAM_TOKEN = os.environ.get('TELEGRAM_TOKEN')

if TELEGRAM_TOKEN is None:
    logger.exception('Не удалось получить токены')
    raise SystemExit()

try:
    bot = Bot(token=TELEGRAM_TOKEN)
    logger.info('Бот создан')
except Exception as e:
    logger.exception('Ошибка при создании бота: ', e)
    raise SystemExit()
storage = MemoryStorage()
try:
    dp = Dispatcher(bot=bot, storage=storage)
    logger.info('Диспетчер создан')
except Exception as e:
    logger.exception('Ошибка при создании диспетчера: ', e)
    raise SystemExit()

manager = DataManager('sqlite+aiosqlite:///compliment.db')
async_session = manager.async_session

async def on_startup(dispatcher):
    active_users = await manager.get_active_users()
    for user in active_users:
        chat_id = user.chat_id
        await dispatcher.bot.send_message(
            chat_id=chat_id,
            text='Мы перезапустили бота.\n'
                 'Что бы сново получить доступ к '
                 'его функционалу введите /rerun'
        )

class Form(StatesGroup):
    phone_number = State()
    conf_code = State()
    activated = State()

class FormAddContact(StatesGroup):
    username_input = State()
    username_confirm = State()
    birthday = State()

class FormDeleteContact(StatesGroup):
    whom_delete = State()
    delete_confirm = State()

@dp.message_handler(commands='start', state='*')
async def cmd_start(message):
    user = await manager.get_user(message.chat.id)
    if user:
        if user.is_active == False:
            await manager.set_user_active_status(
                message.chat.id,
                True
            )
            await Form.activated.set()
            message_reply = 'С возвращением!\n\n'
            contacts = await contacts_list(message.chat.id)
            if contacts is not None:
                message_reply += contacts
            return await message.reply(message_reply)
        return await message.reply('Вы уже зарегистрировались')
    await Form.phone_number.set()
    return await message.reply('Введите номер телефона')

@dp.message_handler(commands='rerun', state='*')
async def cmd_rerun(message):
    user = await manager.get_user(message.chat.id)
    if user is None:
        return await message.reply('Вы еще не зарегистрировались.\n'
                                   'Введите команду /start, что бы '
                                   'начать регистрацию')
    await Form.activated.set()
    await message.reply('Теперь вы сново можете использовать '
                        'весь функционал бота')

@dp.message_handler(commands='contacts', state=Form.activated)
async def cmd_contacts(message, data_manager=manager):
    message_contacts = await contacts_list(message.chat.id, data_manager)
    if message_contacts is None:
        return await message.reply('Вы еще не добавили ни один контакт.\n'
                                   'Что бы добавить контакт введите /add')
    await message.reply(message_contacts)

@dp.message_handler(
    commands='cancel',
    state=[FormDeleteContact.whom_delete, FormDeleteContact.delete_confirm]
)
async def cmd_cancel_delete_contact(message):
    await Form.activated.set()
    return await message.reply(
        'Удаление контакта отменено',
        reply_markup=ReplyKeyboardRemove()
    )

@dp.message_handler(
    commands='cancel',
    state=[FormAddContact.username_input, FormAddContact.birthday]
)
async def cmd_cancel_add_contact(message):
    await Form.activated.set()
    return await message.reply('Добавление нового контактка отменено')

@dp.message_handler(commands='delete', state=Form.activated)
async def cmd_delete_contact(message):
    reply_keyboard = ReplyKeyboardMarkup(resize_keyboard=True)
    user_contacts = await manager.get_client_contacts(message.chat.id)
    for contact in user_contacts:
        username = f'@{contact.contact_username}'
        reply_keyboard.add(username)
    await message.reply(
        'Выберите пользователя из списка добавленных контактов.\n'
        'Пользователь больше не будет получать от вас сообщения',
        reply_markup=reply_keyboard
    )
    await FormDeleteContact.whom_delete.set()

@dp.message_handler(state=FormDeleteContact.whom_delete)
async def delete_contact_whom(message, state):
    contact_username = message.text[1::]
    user_contact = await manager.get_contact(contact_username)
    if user_contact is None:
        return await message.reply('Воспользуйтесь клавиатурой для выбора '
                                   'контакта')
    async with state.proxy() as data:
        data['contact_delete_username'] = contact_username
    reply_keyboard = ReplyKeyboardMarkup(resize_keyboard=True)
    button_yes = 'Да'
    button_no = 'Нет'
    reply_keyboard.add(button_yes, button_no)
    await message.reply(
        f'Вы действительно хотите удалить '
        f'пользователя {message.text} из списка?',
        reply_markup=reply_keyboard
    )
    await FormDeleteContact.next()

@dp.message_handler(state=FormDeleteContact.delete_confirm)
async def delete_confirm(message, state):
    if message.text == 'Да':
        async with state.proxy() as data:
            contact_username = data['contact_delete_username']
        await manager.delete_contact(contact_username, message.chat.id)
        await Form.activated.set()
        return await message.reply(
            'Контакт успешно удален',
            reply_markup=ReplyKeyboardRemove()
        )
    elif message.text == 'Нет':
        await message.reply(
            'Удаление контакта отменено',
            reply_markup=ReplyKeyboardRemove()
        )
        return await Form.activated.set()
    await message.reply('Воспользуйтесь клавиатурой')

@dp.message_handler(commands='cancel', state=Form.activated)
async def cmd_cancel(message, state):
    await state.finish()
    await manager.set_user_active_status(message.chat.id, False)
    return await message.reply('С этого момента вы не '
                               'будете получать сообщения')

@dp.message_handler(commands='add', state=Form.activated)
async def add_contact(message):
    await FormAddContact.username_input.set()
    await message.reply('Отправьте имя пользователя в формате '
                        '"@username"')

@dp.message_handler(state=FormAddContact.username_input)
async def process_contact(message, state, redis):
    if message.text[0] != '@':

        return await message.reply('Введите имя пользователя в '
                                   'правильном формате')
    await redis.sadd(KeySchema().check_contact(), message.text[1::])
    while True:
        status_or_fullname = await redis.hget(
            KeySchema().check_contact_status(), message.text[1::]
        )
        if status_or_fullname:
            if status_or_fullname == 'False':
                return await message.reply('Данного контакта не существует')
            async with state.proxy() as data:
                data['contact_username'] = message.text[1::]
            reply_keyboard = ReplyKeyboardMarkup(resize_keyboard=True)
            button_yes = 'Да'
            button_no = 'Нет'
            reply_keyboard.add(button_yes, button_no)
            await FormAddContact.next()
            await redis.hdel(
                KeySchema().check_contact_status(), message.text[1::]
            )
            return await message.reply(
                f'Вы действительно хотите добавить контакт '
                f'"{status_or_fullname}"?',
                reply_markup=reply_keyboard
            )

@dp.message_handler(state=FormAddContact.username_confirm)
async def confirm_contact(message):
    if message.text == 'Да':
        await FormAddContact.next()
        return await message.reply(
            'Теперь введите день рождения данного '
            'контакта в формате ДД-ММ-ГГ',
            reply_markup=ReplyKeyboardRemove()
        )
    elif message.text == 'Нет':
        await message.reply(
            'Добавление контакта отменено',
            reply_markup=ReplyKeyboardRemove()
        )
        return await Form.activated.set()
    await message.reply('Воспользуйтесь клавиатурой')

@dp.message_handler(state=FormAddContact.birthday)
async def process_contact_birthday(message, state):
    async with state.proxy() as data:
        contact_username = data['contact_username']
    birthday = await validate_birthday(date=message.text)
    if birthday is None:
        await message.reply('Введите дату в правильном формате')
    try:
        await manager.create_contact(
            contact_username=contact_username,
            contact_birthday=birthday,
            user_chat_id=message.chat.id
        )
    except IntegrityError:
        return await message.reply('Данный контак уже есть в вашем списке')
    else:
        await message.reply('C этого момента дайнный контакт будет получать '
                            'от вас сообщения')
    finally:
        await Form.activated.set()

@dp.message_handler(state=Form.phone_number)
async def process_phone(message, state, redis, data_manager=manager):
    async with state.proxy() as data:
        data['phone_number'] = message.text
    try:
        await data_manager.create_user(
            user_chat_id=message.chat.id,
            user_phone_number=message.text
        )
    except IntegrityError:
        await data_manager.update_user_phone_number(
            message.chat.id,
            message.text
        )
    await redis.hdel('hash:phone_validation', message.chat.id)
    while True:
        await asyncio.sleep(1)
        phone_status = await redis.hget(
            'hash:phone_validation',
            message.chat.id
        )
        if phone_status:
            await redis.hdel('hash:phone_validation', message.chat.id)
            if phone_status == 'False':
                return await message.reply('Некорректный номер телефона')
            await redis.hdel('hash:phone_validation', message.chat.id)
            break
    await message.reply('Теперь введите код подтверждения.')
    await Form.next()

@dp.message_handler(state=Form.conf_code)
async def process_conf_code(message, redis, data_manager=manager):
    code = message.text[0:-1]
    await redis.hset(
        KeySchema().conf_code_by_id(), str(message.chat.id), code
    )
    while True:
        code_entered = await redis.smembers(KeySchema().code_entered())
        if str(message.chat.id) in code_entered:
            await asyncio.sleep(5)
            user = await data_manager.get_user(message.chat.id)
            if user.is_activated == True:
                await message.reply('Вы успешно зарегистрировались')
                await Form.next()
            else:
                await message.reply('Код неверный.\n'
                                    'Введите код еще раз')
            await redis.srem(KeySchema().code_entered(), message.chat.id)
            break
        await asyncio.sleep(0)

async def contacts_list(chat_id, data_manager=manager):
    """Returns str message of users contacts."""
    user_contacts = await data_manager.get_client_contacts(chat_id)
    if not user_contacts:
        return None
    message_contacts = 'Список ваших контактов:\n'
    counter = 1
    for contact in user_contacts:
        message_contacts += f'{counter}. @{contact.contact_username}\n'
        counter += 1
    return message_contacts

async def validate_birthday(date):
    """Is used to validate birthday."""
    try:
        dt.strptime(date, '%d-%m-%y')
    except ValueError:
        return None
    else:
        return date


from bpy.props import *
from .utils import *

maprange_rna = bpy.types.ShaderNodeMapRange.bl_rna
interpolation_type_items = enum_from_rna( maprange_rna, 'interpolation_type' )

class MaltNodeMapRange( EssentialsNode ):
    bl_idname = 'MaltNodeMapRange'
    bl_label = 'Map Range'
    menu_category = 'CONVERTOR'
    default_width = 160

    clamp : BoolProperty( name = 'Clamp', default = False, update = malt_update )
    interpolation_type : EnumProperty( name = 'Interpolation Type', items = interpolation_type_items, update = lambda s,c: s.update_interpolation( ))

    def update_socket_visibility( self ):
        self.inputs[ 'steps' ].enabled = self.interpolation_type == 'STEPPED'

    def update_interpolation( self ):
        self.update_socket_visibility( )
        self.update_tree( )

    def define_sockets( self ):
        return{
            'value' : I( 'float', 'Value', default = 0.5 ),
            'from_min' : I( 'float', 'From Min', default = 0.0 ),
            'from_max' : I( 'float', 'From Max', default = 1.0 ),
            'to_min' : I( 'float', 'To Min', default = 0.0 ),
            'to_max' : I( 'float', 'To Max', default = 1.0 ),
            'steps' : I( 'int', 'Steps', default = 4 ),
            'result' : O( 'float', 'Result' )
        }

    def get_function( self ):
        func = {
            'LINEAR': 'float_map_range_linear',
            'STEPPED': 'float_map_range_stepped',
            'SMOOTHSTEP': 'float_map_range_smoothstep',
            'SMOOTHERSTEP': 'float_map_range_smootherstep'
        }[ self.interpolation_type ]
        f = f'{func}( value, from_min, from_max, to_min, to_max, steps, result );\n'
        if self.clamp:
            f += 'result = clamp( result, min( to_min, to_max ), max( to_max, to_min ));\n'
        return f

    def draw_buttons( self, context, layout ):
        layout.prop( self, 'interpolation_type', text = '' )
        layout.prop( self, 'clamp' )

NODES = [ MaltNodeMapRange ]


from lxml import etree as et
from pathlib import Path
import sys

class BookmarkHandler:

    def __init__(self, qgis_path, profile_manager):
        self.profile_manager = profile_manager
        self.qgis_path = qgis_path
        self.bookmark_list = []
        self.source_bookmark_file = ""
        self.target_bookmark_file = ""
        self.parser = et.XMLParser(remove_blank_text=True)

    def parse_source_bookmarks(self):
        """Parses bookmarks from source xml"""
        # get the element tree of the source file
        try:
            source_tree = et.parse(self.source_bookmark_file, self.parser)
            self.insert_bookmarks_to_target_profile(source_tree)
        except:
            print("Oops!", sys.exc_info()[0], "occurred.")

    def insert_bookmarks_to_target_profile(self, source_tree):
        """Inserts bookmarks into target xml file"""
        # check if target file exists
        self.check_if_file_exists()
        # get the element tree of the target file
        # fill if empty
        try:
            target_tree = et.parse(self.target_bookmark_file, self.parser)
        except et.XMLSyntaxError:
            with open(self.target_bookmark_file, "w") as xmlTarget:
                xmlTarget.write("<Bookmarks></Bookmarks>")

            target_tree = et.parse(self.target_bookmark_file, self.parser)

        # find all bookmark elements
        source_root_tag = source_tree.findall('Bookmark')

        # get the root element "Bookmarks"
        target_tree_root = target_tree.getroot()

        # Remove duplicate entries to prevent piling data
        target_tree_root = self.remove_duplicates(source_root_tag, target_tree, target_tree_root)

        # append the elements
        for element in source_root_tag:
            target_tree_root.append(element)

        # overwrite the xml file
        et.ElementTree(target_tree_root)\
            .write(self.target_bookmark_file, pretty_print=True, encoding='utf-8', xml_declaration=True)

    @staticmethod
    def remove_duplicates(source_toot_tag, target_tree, target_tree_root):
        """Removes duplicate entries"""
        target_root_tag = target_tree.findall('Bookmark')
        for s_element in source_toot_tag:
            for t_element in target_root_tag:
                if s_element.attrib["name"] == t_element.attrib["name"]:
                    target_tree_root.remove(t_element)

        return target_tree_root

    def check_if_file_exists(self):
        """Checks if file exists"""
        target_file = Path(self.target_bookmark_file)
        if target_file.is_file():
            pass
        else:
            self.create_new_file()

    def create_new_file(self):
        """Creates a new file"""
        new_file = open(self.target_bookmark_file, "w")
        new_file.write("<Bookmarks></Bookmarks>")
        new_file.close()

    def set_path_files(self, source_bookmark_file, target_bookmark_file):
        """Sets file path's"""
        self.source_bookmark_file = source_bookmark_file
        self.target_bookmark_file = target_bookmark_file


import subprocess
import argparse
import time
import shlex
import signal

def test_foreground(args):
    launch_command = 'nnictl create --config {} --foreground'.format(args.config)
    print('nnictl foreground launch command: ', launch_command, flush=True)

    proc = subprocess.Popen(shlex.split(launch_command))

    time.sleep(args.timeout)
    proc.send_signal(signal.SIGINT)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", type=str, required=True)
    parser.add_argument("--timeout", type=int, default=45)
    args = parser.parse_args()

    test_foreground(args)


#!/usr/bin/env python
# coding: utf-8
"""
Object Detection (On Image) From TF2 Saved Model
=====================================
"""
import time
from object_detection.utils import label_map_util
from object_detection.utils import visualization_utils as viz_utils
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'    # Suppress TensorFlow logging (1)
import pathlib
import tensorflow as tf
import cv2
import argparse

tf.get_logger().setLevel('ERROR')           # Suppress TensorFlow logging (2)

parser = argparse.ArgumentParser()
parser.add_argument('--model', help='Folder that the Saved Model is Located In',
                    default='exported-models/my_mobilenet_model')
parser.add_argument('--labels', help='Where the Labelmap is Located',
                    default='exported-models/my_mobilenet_model/saved_model/label_map.pbtxt')
parser.add_argument('--image', help='Name of the single image to perform detection on',
                    default='WOB\WOB IPAD\Photo 21-3-22, 3 00 36 PM.jpg')
parser.add_argument('--threshold', help='Minimum confidence threshold for displaying detected objects',
                    default=0.5)

args = parser.parse_args()
# Enable GPU dynamic memory allocation
gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

# PROVIDE PATH TO IMAGE DIRECTORY
IMAGE_PATHS = args.image
# PROVIDE PATH TO MODEL DIRECTORY
PATH_TO_MODEL_DIR = args.model
# PROVIDE PATH TO LABEL MAP
PATH_TO_LABELS = args.labels
# PROVIDE THE MINIMUM CONFIDENCE THRESHOLD
MIN_CONF_THRESH = float(args.threshold)
# LOAD THE MODEL
PATH_TO_SAVED_MODEL = PATH_TO_MODEL_DIR + "/saved_model"
print('Loading model...', end='')
start_time = time.time()
# LOAD SAVED MODEL AND BUILD DETECTION FUNCTION
detect_fn = tf.saved_model.load(PATH_TO_SAVED_MODEL)

end_time = time.time()
elapsed_time = end_time - start_time
print('Done! Took {} seconds'.format(elapsed_time))
# LOAD LABEL MAP DATA FOR PLOTTING

category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS,
                                                                    use_display_name=True)
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')   # Suppress Matplotlib warnings

def process_image(image):
  kernel = np.ones((6, 6), np.uint8)
  binary_img = cv2.erode(image, kernel, iterations = 1)
  return binary_img

def load_image_into_numpy_array(path):
    """Load an image from file into a numpy array.

    Puts image into numpy array to feed into tensorflow graph.
    Note that by convention we put it into a numpy array with shape
    (height, width, channels), where channels=3 for RGB.

    Args:
      path: the file path to the image

    Returns:
      uint8 numpy array with shape (img_height, img_width, 3)
    """
    return np.array(Image.open(path))

def getWOB(image):
    imagecopy = image.copy()
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    thresh = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV,41,11)

    cv2.imshow("thresh", thresh)
    # Filter out all numbers and noise to isolate only boxes
    cnts = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    cnts = cnts[0] if len(cnts) == 2 else cnts[1]
    max_area = 0
    c = 0
    s = 0
    for i in cnts:
      area = cv2.contourArea(i)
      if area > 1000:
              if area > max_area:
                  max_area = area
                  best_cnt = i
                  image = cv2.drawContours(image, cnts, c, (0, 255, 0), 3)
                  s = s+1
      c+=1
    alist = best_cnt.reshape(best_cnt.shape[0], best_cnt.shape[2])
    xmax, ymax = np.max(alist, axis = 0)
    xmin, ymin = np.min(alist, axis = 0)
    rect = [[xmax, ymin], [xmin, ymin], [xmax, ymax], [xmin, ymax]]
    mask = np.zeros((gray.shape),np.uint8)
    cv2.drawContours(mask,[best_cnt],0,255,-1)
    cv2.drawContours(mask,[best_cnt],0,0,2)
    result = perspective_transform(mask, imagecopy, rect)
    result = result[ymin:ymax, xmin:xmax]
    cv2.imshow("mask", result)
    result = cv2.resize(result,(1000,1000))
    return result

def perspective_transform(mask, img, rect):
    corners = find_corners(mask)
    print(corners)
    rows,cols = mask.shape
    print(mask.shape)
    # pts1 = np.float32([[corners[0,0],corners[0,1]],[corners[1,0],corners[1,1]],[corners[2,0],corners[2,1]],[corners[3,0],corners[3,1]]])
    pts1 = np.float32([corners[1], corners[2], corners[3], corners[4]])
    # pts2 = np.float32([[rows,0], [0, 0], [0,cols], [rows,cols]])
    pts2 = np.float32(rect)
    M = cv2.getPerspectiveTransform(pts1,pts2)
    # print(M)
    dst = cv2.warpPerspective(img,M,(1000,1000))
    return dst

def find_corners(img):
    img = cv2.blur(img, (5,5))
    dst = cv2.cornerHarris(img,20,3,0.04)
    ret, dst = cv2.threshold(dst,0.1*dst.max(),255,0)
    dst = np.uint8(dst)
    ret, labels, stats, centroids = cv2.connectedComponentsWithStats(dst)
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.001)
    corners = cv2.cornerSubPix(img,np.float32(centroids),(5,5),(-1,-1),criteria)
    x = 0
    for i in range(1, len(corners)):
        # print(corners[i])
        cv2.circle(img, (int(corners[i,0]), int(corners[i,1])), 7, (255,255,255), 2)
        cv2.putText(img, str(x), (int(corners[i,0]), int(corners[i,1])), cv2.FONT_HERSHEY_COMPLEX, 0.5, (125,125,125), 2)
        x = x+1

    cv2.imshow('image', img)

    return corners

print('Running inference for {}... '.format(IMAGE_PATHS), end='')

def detector(image):
  image_rgb = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
  imH, imW, _ = image.shape
  image_expanded = np.expand_dims(image_rgb, axis=0)
  # The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
  input_tensor = tf.convert_to_tensor(image)
  # The model expects a batch of images, so add an axis with `tf.newaxis`.
  input_tensor = input_tensor[tf.newaxis, ...]
  # input_tensor = np.expand_dims(image_np, 0)
  detections = detect_fn(input_tensor)
  # All outputs are batches tensors.
  # Convert to numpy arrays, and take index [0] to remove the batch dimension.
  # We're only interested in the first num_detections.
  num_detections = int(detections.pop('num_detections'))
  detections = {key: value[0, :num_detections].numpy()
                for key, value in detections.items()}
  detections['num_detections'] = num_detections

  # detection_classes should be ints.
  detections['detection_classes'] = detections['detection_classes'].astype(np.int64)
  scores = detections['detection_scores']
  boxes = detections['detection_boxes']
  classes = detections['detection_classes']
  count = 0
  mid_point = []
  for i in range(len(scores)):
      if ((scores[i] > 0.99) and (scores[i] <= 1.0)):
          #increase count
          count += 1
          # Get bounding box coordinates and draw box
          # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
          ymin = int(max(1,(boxes[i][0] * imH)))
          xmin = int(max(1,(boxes[i][1] * imW)))
          ymax = int(min(imH,(boxes[i][2] * imH)))
          xmax = int(min(imW,(boxes[i][3] * imW)))

          cv2.rectangle(image, (xmin,ymin), (xmax,ymax), (10, 255, 0), 2)
          # Draw label
          object_name = category_index[int(classes[i])]['name'] # Look up object name from "labels" array using class index
          label = '%s: %d%%' % (object_name, int(scores[i]*100)) # Example: 'person: 72%'
          mid_point.append([(xmax+xmin)/2, (ymax+ymin)/2, label])
          labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size
          label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window
          cv2.rectangle(image, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), cv2.FILLED) # Draw white box to put label text in
          cv2.putText(image, label, (xmin, label_ymin-7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # Draw label text

  cv2.putText (image,'Total Detections : ' + str(count),(10,25),cv2.FONT_HERSHEY_SIMPLEX,1,(70,235,52),2,cv2.LINE_AA)
  print('Done')
  return image, mid_point

def sort_grid(image, items):
  x = []
  y = []
  name = []
  for i in range(len(items)):
    x.append(items[i][0])
    y.append(items[i][1])
    name.append(items[i][2])

  print(x,y,name)
  print(items)
  rows = 4
  cols = 6
  deliver = []
  deliver = [[0 for i in range(cols)] for j in range(rows)]
  rows = 2
  ret = []
  ret = [[0 for i in range(cols)] for j in range(rows)]
  rows = 6
  order = [[0 for i in range(cols)] for j in range(rows)]
  w,h,c = np.shape(image)
  colsize = w/7
  rowsize = h/7
  count = len(items)
  for i in range(0, cols):
    for j in range(0, rows):
      for counts in range(count):

        if x[counts] > colsize*(i+1) and x[counts] < colsize*(i+2) and y[counts] > rowsize*(j+1) and y[counts] < rowsize*(j+2):
          order[j][i] = name[counts]

  for i in range(4):
    deliver[i] = order[i]
  for i in range(2):
    ret[i] = order[i+4]
  with open('debug.txt', 'w') as f:
    f.write(str(deliver))
    f.write(str(ret))
  return deliver, ret

if __name__ == "__main__":
  image = cv2.imread(IMAGE_PATHS)
  image = cv2.resize(image, (1000,1000))
  image = getWOB(image)
  img, item = detector(image)
  deliver, ret = sort_grid(img,item)
  image = cv2.resize(image,(600,600))
  # DISPLAYS OUTPUT IMAGE
  cv2.imshow('Object Counter', image)
  # CLOSES WINDOW ONCE KEY IS PRESSED
  cv2.waitKey(0)
  # CLEANUP
  cv2.destroyAllWindows()