# Home work B
## BAX 443

Group Member: Jacqueline Huang, Jing Li, Jiayin Liu, Jack Ye

### Preparing data

```
import matplotlib.pyplot as plt#
import numpy as np
import statsmodels.api as sm
import statsmodels.formula.api as smf
from scipy.optimize import minimize
import math
import scipy
import scipy.stats as stats
from scipy.optimize import minimize_scalar
from scipy.optimize import minimize
from sklearn import preprocessing
import seaborn as sns
#sns.set(style="white")
#sns.set(style="whitegrid", color_codes=True)
import pandas as pd
import numpy as np
url = 'https://raw.githubusercontent.com/ormarketing/b2b/master/data.csv' #This is the link to my github profile with the data
df = pd.read_csv(url)#this load the data from the URL define above
df.head() #just looking at the data
#df['Stage'].unique()
```

      ID
      Close Date
      Created Date
      New Logo
      Type
      Stage
      Billing Plan
      ACV
      Discount Amount
      Amount
      Net Amount

      0
      1
      5/26/17
      5/4/17
      1
      Direct - Cloud Product
      Closed Lost
      3 Years
      431370

      225000
      225000

      1
      2
      12/29/16
      11/29/16
      1
      Partner - OnPremise Product
      Closed Won - Paid
      3 Years
      22050
      80850
      147000
      66150

      2
      3
      12/29/16
      12/9/16
      1
      Partner - Cloud Product
      Closed Won - Paid
      3 Years
      32750
      43200
      96000
      52800

      3
      4
      9/9/16
      4/20/16
      1
      Direct - Cloud Product
      Closed Won - Paid
      4 Years
      36694
      44800
      128000
      83200

      4
      5
      9/22/16
      9/1/16
      1
      Partner - OnPremise Product
      Closed Won - Paid
      3 Years
      18700
      9900
      66000
      56100

```
df['Partner cloud']=np.where(df['Type'] =='Partner - Cloud Product', 1.0,0.0)
df['Partner prem']=np.where(df['Type'] =='Partner - OnPremise Product', 1.0, 0.0)
df['Direct cloud']=np.where(df['Type'] =='Direct - Cloud Product', 1.0,0.0)
df['Direct prem']=np.where(df['Type'] =='Direct - OnPremise Product', 1.0, 0.0)
df['amount']=pd.to_numeric(df[' Amount '],errors='coerce')
df['acv']=pd.to_numeric(df[' ACV '],errors='coerce')
df['net amount']=pd.to_numeric(df[' Net Amount '],errors='coerce')
df['billing plan year']=pd.to_numeric(df['Billing Plan'].astype(str).str[0],errors='coerce') #turn billing plan to a bumber
df['discount']=df['amount']-df['net amount']
df['depth']=df['discount']/df['amount']
df.drop(' Discount Amount ',axis=1)
df.drop([' ACV ',' Discount Amount ',' Amount ', ' Net Amount ', 'ID', 'Type'],axis=1);
```

```
df['Stage']=np.where(df['Stage'] =='Closed Lost', 'Lost', df['Stage'])
df['Stage']=np.where(df['Stage'] =='Closed Won - Paid', 'Won', df['Stage'])
df['Stage']=np.where(df['Stage'] =='Closed Won - Not Paid', 'Won', df['Stage'])
df['Stage']=np.where(df['Stage'] =='Closed - Lost', 'Lost', df['Stage'])
df['Stage']=np.where(df['Stage'] =='Closed Lost - Not Renewing', 'Lost', df['Stage'])
sns.countplot(x='Stage',data=df)
df['Stage'].value_counts()
```

```
count_Lost = len(df[df['Stage']=='Lost'])
count_Won = len(df[df['Stage']=='Won'])
round(count_Won/(count_Lost+count_Won),3)
```

    0.355

```
# win/lose vs segment
%matplotlib inline
pd.crosstab(df.Type,df.Stage).plot(kind='bar')
plt.title('Frequency Won/Lost')
plt.xlabel('Direct vs. Partners')
plt.ylabel('Frequency of Win/Lost')
plt.savefig('WinLost')
```

#### seasonality plot

```
df_date_stage=df[['Close Date','Stage']]
df_date_stage['Close Date Month num']=pd.to_datetime(df_date_stage['Close Date']).dt.month
df_date_stage=pd.concat([df_date_stage,pd.get_dummies(df_date_stage.Stage,prefix='stage')],axis=1)

df_month_stage=df_date_stage.groupby(['Close Date Month num'],as_index=None).sum()
df_month_stage['won_prop']=df_month_stage['stage_Won']/(df_month_stage['stage_Won']+df_month_stage['stage_Lost'])
df_month_stage.head()

%matplotlib inline

plt.figure(figsize=(22,5))
plt.plot(df_month_stage['Close Date Month num'],df_month_stage['won_prop'])
plt.title('Proportion of won contract by month')
plt.xlabel('Close date by month')
plt.ylabel('Proportion of Won contract')
plt.savefig('WinLost vs date')
#There appears to be a seasonaility. at the end of the quarter, the close rate increase
```

plot avg discount vs time

```
df_date_discount=df[['Close Date','discount']]
df_date_discount['Close Date Month num']=pd.to_datetime(df_date_discount['Close Date']).dt.month
df_date_discount=df_date_discount.groupby('Close Date Month num',as_index=None).mean()
%matplotlib inline
plt.figure(figsize=(22,5))
plt.plot(df_date_discount['Close Date Month num'],df_date_discount['discount'])
plt.title('Discount avg amount by month')
plt.xlabel('Close date by month')
plt.ylabel('Discount avg amount')
plt.savefig('WinLost vs date')
#There appears to be a seasonaility. at the end of the quarter, the avg discount amount increase
```

The date in terms of month appears to differentiate, putting it as a dependent variable

```
df['Close Date Month']=pd.to_datetime(df['Close Date']).dt.month
df['End of Quarter']=np.where(df['Close Date Month'].isin([3,6,9,12]),1,0)
```

#### split into training and testing

```
df['y']=np.where(df['Stage'] =='Won', 1.0,0.0)
#spliting the dataset by ID
df_train = df.head(1200)
df_test = df.tail(305)

```

### Modeling

# **Price Optimization**

Using these results, I want to optimize on the price
p is the amount, d is the discount

\begin{equation}
R(p)=Proba(Y=1|p,d) \times (p-d)
\end{equation}

# **Enriching the Intercept**

##### Model from class as benchmark

```
# Original class Model: remove the constant variable from the list of independent variables and include instead:
# whether the lead is a new logo or not, and the types of product
y=df_train['y']
x2=df_train[['discount','amount','New Logo','Partner cloud','Partner prem','Direct cloud','Direct prem']]
model2 = sm.Logit(y, x2, missing='drop')
model2.fit
type(model2)
result2=model2.fit()
print(result2.summary()) #appears to be all significant
```

    Optimization terminated successfully.
             Current function value: 0.445041
             Iterations 8
                               Logit Regression Results
    ==============================================================================
    Dep. Variable:                      y   No. Observations:                 1151
    Model:                          Logit   Df Residuals:                     1144
    Method:                           MLE   Df Model:                            6
    Date:                Mon, 08 Jun 2020   Pseudo R-squ.:                  0.3318
    Time:                        19:44:48   Log-Likelihood:                -512.24
    converged:                       True   LL-Null:                       -766.56
    Covariance Type:            nonrobust   LLR p-value:                1.160e-106
    =================================================================================
                        coef    std err          z      P>|z|      [0.025      0.975]
    ---------------------------------------------------------------------------------
    discount       5.506e-05   6.59e-06      8.352      0.000    4.21e-05     6.8e-05
    amount        -1.804e-06    1.1e-06     -1.639      0.101   -3.96e-06    3.53e-07
    New Logo          2.1419      0.172     12.475      0.000       1.805       2.478
    Partner cloud    -2.6405      0.267     -9.891      0.000      -3.164      -2.117
    Partner prem     -1.7741      0.153    -11.630      0.000      -2.073      -1.475
    Direct cloud     -3.0963      0.420     -7.364      0.000      -3.920      -2.272
    Direct prem      -2.5071      0.214    -11.698      0.000      -2.927      -2.087
    =================================================================================

#### Model including acv`& end of quarter

```
# New Model2: remove the constant variable from the list of independent variables and include plus billing plan & acv :
# whether the lead is a new logo or not, and the types of product
# billing plan turns out insignificant
y=df_train['y']
x2=df_train[['discount','amount','New Logo','Partner cloud','Partner prem','Direct cloud','Direct prem','acv','End of Quarter']]
model2 = sm.Logit(y, x2, missing='drop')
model2.fit
type(model2)
result2=model2.fit()
print(result2.summary()) #appears to be all significant
#class R-square is  0.3048
```

    Optimization terminated successfully.
             Current function value: 0.370624
             Iterations 8
                               Logit Regression Results
    ==============================================================================
    Dep. Variable:                      y   No. Observations:                 1121
    Model:                          Logit   Df Residuals:                     1112
    Method:                           MLE   Df Model:                            8
    Date:                Mon, 08 Jun 2020   Pseudo R-squ.:                  0.4473
    Time:                        19:44:49   Log-Likelihood:                -415.47
    converged:                       True   LL-Null:                       -751.77
    Covariance Type:            nonrobust   LLR p-value:                5.628e-140
    ==================================================================================
                         coef    std err          z      P>|z|      [0.025      0.975]
    ----------------------------------------------------------------------------------
    discount         2.68e-05   5.65e-06      4.748      0.000    1.57e-05    3.79e-05
    amount          1.376e-05   2.38e-06      5.794      0.000    9.11e-06    1.84e-05
    New Logo           2.1085      0.199     10.614      0.000       1.719       2.498
    Partner cloud     -2.1364      0.358     -5.965      0.000      -2.838      -1.434
    Partner prem      -1.9009      0.213     -8.931      0.000      -2.318      -1.484
    Direct cloud      -2.6883      0.477     -5.635      0.000      -3.623      -1.753
    Direct prem       -2.5446      0.273     -9.326      0.000      -3.079      -2.010
    acv            -4.043e-05   4.84e-06     -8.353      0.000   -4.99e-05   -3.09e-05
    End of Quarter     1.6703      0.182      9.167      0.000       1.313       2.027
    ==================================================================================

Find the opimal discount on hold-out sample

```
#as the discount can be calculated as zero, we want to make it positive
#### with sqrt(x^2)
x2=df_test[['discount','amount','New Logo','Partner cloud','Partner prem','Direct cloud','Direct prem','acv','End of Quarter']]
y=df_test['y']
ID=0
ID_max=len(df_test)
z=np.empty([ID_max,])
for ID in range(ID_max):
    UT=np.matmul(x2.iloc[ID,1:10],result2.params[1:10])
    b=result2.params[0]
    p=x2.iloc[ID,1]
    def revenueD(d):
    #"""This is the negative revenue function"""
      return -np.exp(UT+b*np.sqrt(d**2))/(1+np.exp(UT+b*np.sqrt(d**2)))*(p-np.sqrt(d**2))
    sol3=minimize_scalar(revenueD)
    z[ID]=sol3.x
print('optimal discount is:',round(np.mean(z),0),'observed discount is ',np.round(np.mean(df['discount']),0))
# z is the optimal discount
```

    optimal discount is: 7240.0 observed discount is  14927.0

Calculate optimal revenue and compare

```
#xS for x star
df_test['optimal discount']=np.sqrt(z**2)
xS=df_test[['optimal discount','amount','New Logo','Partner cloud','Partner prem','Direct cloud','Direct prem','acv','End of Quarter']]
print(xS.shape,result2.params.shape)
ID=0
ID_max=len(df_test)
revOpt=np.empty([ID_max,])
while ID<=ID_max-1:
    j=0
    u=0
#    while j<=6:
#     u=u+xS.iloc[ID][j]*result2.params[j]
#     j=j+1
    UT=np.dot(xS.iloc[ID].T,result2.params[0:10].T)
    proba=np.exp(UT)/(1+np.exp(UT))
    revOpt[ID]=proba*(xS.iloc[ID,1]-z[ID])
    ID=ID+1
revOpt=np.nan_to_num(revOpt);
#Compute the expected revenue after optimization and compare it to the acual revenue
```

    (305, 9) (9,)

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

```
df_test['Revenue']=np.where(df_test['Stage'] =='Won',df_test['net amount'] ,0.0)
round(np.sum(revOpt)/np.sum(df_test['Revenue']),3)#This is the ratio of the optimized expected revenue / actual revenues
#the class example has 1.5
```

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      """Entry point for launching an IPython kernel.

    1.604

```
round(np.sum(revOpt)-np.sum(df_test['Revenue']),3)#This is the ratio of the optimized expected revenue / actual revenues

```

    2948896.389

#### Core Model improved with interactive terms

```
#To see correlation
df[['discount','amount','New Logo','Partner cloud','Partner prem','Direct cloud','Direct prem','acv','billing plan year','End of Quarter']].corr()
#billing plan year has high correlation
```

      discount
      amount
      New Logo
      Partner cloud
      Partner prem
      Direct cloud
      Direct prem
      acv
      billing plan year
      End of Quarter

      discount
      1.000000
      0.744895
      0.229336
      0.143729
      -0.046708
      0.027240
      -0.079695
      0.304594
      0.183610
      0.152034

      amount
      0.744895
      1.000000
      0.171924
      0.232642
      -0.063395
      0.044192
      -0.142931
      0.759000
      0.150175
      0.109886

      New Logo
      0.229336
      0.171924
      1.000000
      0.262359
      -0.114549
      0.111119
      -0.138820
      0.010098
      0.065821
      0.163964

      Partner cloud
      0.143729
      0.232642
      0.262359
      1.000000
      -0.467196
      -0.092392
      -0.248149
      0.238617
      0.056921
      0.086214

      Partner prem
      -0.046708
      -0.063395
      -0.114549
      -0.467196
      1.000000
      -0.239285
      -0.642677
      -0.109749
      -0.061087
      0.044530

      Direct cloud
      0.027240
      0.044192
      0.111119
      -0.092392
      -0.239285
      1.000000
      -0.127095
      0.091443
      0.003345
      -0.022233

      Direct prem
      -0.079695
      -0.142931
      -0.138820
      -0.248149
      -0.642677
      -0.127095
      1.000000
      -0.107158
      0.011227
      -0.111504

      acv
      0.304594
      0.759000
      0.010098
      0.238617
      -0.109749
      0.091443
      -0.107158
      1.000000
      -0.004352
      -0.011090

      billing plan year
      0.183610
      0.150175
      0.065821
      0.056921
      -0.061087
      0.003345
      0.011227
      -0.004352
      1.000000
      0.092949

      End of Quarter
      0.152034
      0.109886
      0.163964
      0.086214
      0.044530
      -0.022233
      -0.111504
      -0.011090
      0.092949
      1.000000

```
import statsmodels.formula.api as smf
formula = "y ~ 0+discount+amount+Q('New Logo')+Q('Partner cloud')+Q('Partner prem')+Q('Direct cloud')+Q('Direct prem')\
+Q('acv')+Q('End of Quarter')+discount:Q('New Logo')+discount:Q('Partner cloud')+discount:Q('Partner prem')+discount:Q('Direct cloud')+\
discount:Q('Direct prem')+discount:Q('End of Quarter')"
model_glm = smf.glm(formula = formula, data=df_train, family=sm.families.Binomial(),missing='drop')
result_glm = model_glm.fit()
print(result_glm.summary())

```

    /usr/local/lib/python3.6/dist-packages/statsmodels/genmod/families/family.py:890: RuntimeWarning: invalid value encountered in true_divide
      n_endog_mu = self._clean((1. - endog) / (1. - mu))

                     Generalized Linear Model Regression Results
    ==============================================================================
    Dep. Variable:                      y   No. Observations:                 1121
    Model:                            GLM   Df Residuals:                     1107
    Model Family:                Binomial   Df Model:                           13
    Link Function:                  logit   Scale:                          1.0000
    Method:                          IRLS   Log-Likelihood:                    nan
    Date:                Mon, 08 Jun 2020   Deviance:                          nan
    Time:                        19:44:50   Pearson chi2:                 8.84e+03
    No. Iterations:                   100
    Covariance Type:            nonrobust
    ================================================================================================
                                       coef    std err          z      P>|z|      [0.025      0.975]
    ------------------------------------------------------------------------------------------------
    discount                      2.012e-06   1.36e-05      0.148      0.883   -2.47e-05    2.87e-05
    amount                        1.201e-05   2.47e-06      4.868      0.000    7.18e-06    1.68e-05
    Q('New Logo')                    1.8746      0.219      8.561      0.000       1.445       2.304
    Q('Partner cloud')              -1.9063      0.374     -5.100      0.000      -2.639      -1.174
    Q('Partner prem')               -1.9883      0.235     -8.466      0.000      -2.449      -1.528
    Q('Direct cloud')               -2.6062      0.559     -4.662      0.000      -3.702      -1.510
    Q('Direct prem')                -2.3262      0.285     -8.176      0.000      -2.884      -1.769
    Q('acv')                     -3.532e-05   5.42e-06     -6.520      0.000   -4.59e-05   -2.47e-05
    Q('End of Quarter')              1.5188      0.211      7.190      0.000       1.105       1.933
    discount:Q('New Logo')         3.26e-05   1.87e-05      1.740      0.082   -4.11e-06    6.93e-05
    discount:Q('Partner cloud')  -1.528e-05   1.02e-05     -1.496      0.135   -3.53e-05    4.74e-06
    discount:Q('Partner prem')     3.51e-05   1.34e-05      2.626      0.009     8.9e-06    6.13e-05
    discount:Q('Direct cloud')    1.218e-06   1.43e-05      0.085      0.932   -2.69e-05    2.93e-05
    discount:Q('Direct prem')    -1.903e-05    1.3e-05     -1.463      0.143   -4.45e-05    6.46e-06
    discount:Q('End of Quarter')  1.477e-05   1.37e-05      1.077      0.281   -1.21e-05    4.16e-05
    ================================================================================================

    /usr/local/lib/python3.6/dist-packages/statsmodels/genmod/families/family.py:942: RuntimeWarning: divide by zero encountered in true_divide
      special.gammaln(n - y + 1) + y * np.log(mu / (1 - mu)) +
    /usr/local/lib/python3.6/dist-packages/statsmodels/genmod/families/family.py:943: RuntimeWarning: divide by zero encountered in log
      n * np.log(1 - mu)) * var_weights
    /usr/local/lib/python3.6/dist-packages/statsmodels/genmod/families/family.py:943: RuntimeWarning: invalid value encountered in add
      n * np.log(1 - mu)) * var_weights

\begin{equation}
Prob_{Win} = 2.012*10^{-6}*discount+1.20*10^{-5} *amount + 1.875*New\ Logo-1.9063*Partner\ cloud -1.9883*Partner\ prem -2.606*Direct\ cloud -2.326*Direct\ prem - 3.532*10^{-5} *acv +  1.5188*End\ of\ Quarter + 3.26*10^{-5}*(discount*New\ Logo) -1.528*10^{-5}*(discount*Partner\ cloud) +3.51*10^{-5}*(discount*Partner\ prem) +1.218*10^{-6}*(discount*Direct\ cloud)-1.903*10^{-5}*(discount*Direct\ prem) + 1.477*10^{-5}*(discount*End\ of\ Quarter)
\end{equation}

Find the opimal discount

```
#as the discount can be calculated as zero, we want to make it positive
#### with sqrt(x^2)
x2_interaction=df_test[['discount','amount','New Logo','Partner cloud','Partner prem','Direct cloud','Direct prem','acv','End of Quarter']]

ID=0
ID_max=len(x2_interaction)
z=[]
for ID in range(ID_max):
    UT=np.dot(x2_interaction.iloc[ID,1:9],result_glm.params[1:9])
    UT2=np.dot(x2_interaction.iloc[ID,2:7],result_glm.params[9:14])+x2_interaction.iloc[ID,8]*result_glm.params[14] #skipped acv
    b=result_glm.params[0]
    p=x2_interaction.iloc[ID,1]
    def revenueD(d):
    #"""This is the negative revenue function"""
      return -np.exp(UT+(UT2+b)*np.sqrt(d**2))/(1+np.exp(UT+(UT2+b)*np.sqrt(d**2)))*(p-np.sqrt(d**2))
    sol3=minimize_scalar(revenueD)
    z.append(sol3.x)
z = np.array(z)
print('optimal discount is:',round(np.mean(z),0),'observed discount is ',np.round(np.mean(df['discount']),0))
# z is the optimal discount
```

    optimal discount is: 6602.0 observed discount is  14927.0

Calculate optimal revenue and compare

```
#xS for x star
x2_interaction['optimal discount']=np.sqrt(z**2)
Summary_Opt=df_test
Summary_Opt['optimal discount']=np.sqrt(z**2)
xS=x2_interaction[['optimal discount','amount','New Logo','Partner cloud','Partner prem','Direct cloud','Direct prem','acv','End of Quarter']]

print(xS.shape,result_glm.params.shape)
ID=0
ID_max=len(x2_interaction)
revOpt=np.empty([ID_max,])
for ID in range(ID_max):
    dis=xS.iloc[ID,0]
    UT=np.dot(xS.iloc[ID,0:9],result_glm.params[0:9])
    UT2=np.dot(xS.iloc[ID,2:7],result_glm.params[9:14])+xS.iloc[ID,8]*result_glm.params[14] #skipped acv
    proba=np.exp(UT+UT2*np.sqrt(dis**2))/(1+np.exp(UT+UT2*np.sqrt(dis**2)))
    revOpt[ID]=proba*(xS.iloc[ID,1]-dis)
revOpt=np.nan_to_num(revOpt);
#Compute the expected revenue after optimization and compare it to the acual revenue

```

    (305, 9) (15,)

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      after removing the cwd from sys.path.

```
x2_interaction['Revenue']=np.where(df_test['Stage'] =='Won',df_test['net amount'] ,0.0)
round(np.sum(revOpt)/np.sum(x2_interaction['Revenue']),3)#This is the ratio of the optimized expected revenue / actual revenues
#the class example has 1.5
```

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      """Entry point for launching an IPython kernel.

    1.612

```
np.sum(revOpt) #optimized Revenue
```

    7870429.972939233

#### Performance metrics of the Improved model

#### a. Expected revenue improvement over un-optimized discounts (take the actual discounts in the data-set) for leads in the hold-out sample

```
#as the discount can be calculated as zero, we want to make it positive
#### with sqrt(x^2)
x2_interaction=df_test[['discount','amount','New Logo','Partner cloud','Partner prem','Direct cloud','Direct prem','acv','End of Quarter']]
xS=x2_interaction
print(xS.shape,result_glm.params.shape)
ID=0
ID_max=len(x2_interaction)
revUnOpt=np.empty([ID_max,])
for ID in range(ID_max):
    dis=xS.iloc[ID,0]
    UT=np.dot(xS.iloc[ID,0:9],result_glm.params[0:9])
    UT2=np.dot(xS.iloc[ID,2:7],result_glm.params[9:14])+xS.iloc[ID,8]*result_glm.params[14] #skipped acv
    proba=np.exp(UT+UT2*np.sqrt(dis**2))/(1+np.exp(UT+UT2*np.sqrt(dis**2)))
    revUnOpt[ID]=proba*(xS.iloc[ID,1]-dis)
revUnOpt=np.nan_to_num(revUnOpt)
# print('uoptimal discount is:',round(np.mean(z),0),'observed discount is ',np.round(np.mean(df['discount']),0))
# z is the optimal discount
```

    (305, 9) (15,)

```
x2_interaction['Revenue']=np.where(df_test['Stage'] =='Won',df_test['net amount'] ,0.0)
print("Ratio of unoptimized expected revenue improvement vs actual revenue: ")
round(np.sum(revUnOpt)/np.sum(x2_interaction['Revenue']),3)#This is the ratio of the unoptimized expected revenue / actual revenues

```

    Ratio of unoptimized expected revenue improvement vs actual revenue:

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      """Entry point for launching an IPython kernel.

    1.156

```
print('Expected revenue improvement unoptimized increased by: ')
a=round(np.sum(revUnOpt)-np.sum(x2_interaction['Revenue']),3)#This is the difference of the unoptimized expected revenue vs actual revenues
a
```

    Expected revenue improvement unoptimized increased by:

    761097.531

#### b. Expected revenue improvement over optimized discounts (based on your estimation results, optimize discounts offered to leads in the hold-out sample).

```
print("Ratio of expected revenue improvement vs actual revenue: ")
round(np.sum(revOpt)/np.sum(x2_interaction['Revenue']),3)#This is the ratio of the optimized expected revenue / actual revenues
```

    Ratio of expected revenue improvement vs actual revenue:

    1.612

```
print('Expected revenue improvement increased by: ')
b=round(np.sum(revOpt)-np.sum(x2_interaction['Revenue']),3) #This is the difference of the optimized expected revenue vs actual revenues
b
```

    Expected revenue improvement increased by:

    2986902.973

```
round(a/b,2)
```

    0.25

#### Summary

```
Summary_Opt.columns
```

    Index(['ID', 'Close Date', 'Created Date', 'New Logo', 'Type', 'Stage',
           'Billing Plan', ' ACV ', ' Discount Amount ', ' Amount ',
           ' Net Amount ', 'Partner cloud', 'Partner prem', 'Direct cloud',
           'Direct prem', 'amount', 'acv', 'net amount', 'billing plan year',
           'discount', 'depth', 'Close Date Month', 'End of Quarter', 'y',
           'optimal discount', 'Revenue'],
          dtype='object')

```
Summary_Opt['Revenue_optimized']=revOpt
Summary_Opt['depth_optimized']=Summary_Opt['optimal discount']/Summary_Opt['amount']
Summary_Opt['depth']=Summary_Opt['discount']/Summary_Opt['amount']
Summary_Opt['Revenue_optimized']=revOpt
Summary_Opt['Increased Revenue']=Summary_Opt['Revenue_optimized']-Summary_Opt['Revenue']
#type
round(Summary_Opt[['Type','depth','depth_optimized','amount','discount','Increased Revenue']].groupby('Type').mean(),2)
```

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      """Entry point for launching an IPython kernel.
    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      This is separate from the ipykernel package so we can avoid doing imports until
    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      after removing the cwd from sys.path.
    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      """

      depth
      depth_optimized
      amount
      discount
      Increased Revenue

      Type

      Direct - Cloud Product
      0.13
      0.15
      64117.65
      12471.76
      10767.75

      Direct - OnPremise Product
      0.04
      0.00
      33464.41
      2937.01
      2724.45

      Partner - Cloud Product
      0.22
      0.05
      141035.94
      52629.22
      37273.97

      Partner - OnPremise Product
      0.09
      0.14
      52695.51
      10492.98
      8531.55

```
#new logo
round(Summary_Opt[['New Logo','depth','depth_optimized','amount','discount','Increased Revenue']].groupby('New Logo').mean(),2)
```

      depth
      depth_optimized
      amount
      discount
      Increased Revenue

      New Logo

      0
      0.05
      0.08
      42581.66
      5617.06
      4365.66

      1
      0.15
      0.07
      75465.39
      22536.55
      17434.49

```
#end of quarter
round(Summary_Opt[['End of Quarter','depth','depth_optimized','amount','discount','Increased Revenue']].groupby('End of Quarter').mean(),2)
```

      depth
      depth_optimized
      amount
      discount
      Increased Revenue

      End of Quarter

      0
      0.05
      0.09
      47317.96
      6096.94
      3231.20

      1
      0.17
      0.04
      74158.21
      25733.13
      22884.31

# Additional Exploration: Optimize the revenue separately by customer segmentation

### Optimize each segment

###### direct cloud

```
df_seg=df_test
df_seg_direct_cloud=x2_interaction[(x2_interaction['Direct cloud']==1)]
df_seg_direct_cloud.head(1)
```

      discount
      amount
      New Logo
      Partner cloud
      Partner prem
      Direct cloud
      Direct prem
      acv
      End of Quarter
      Revenue

      1226
      0.0
      22000.0
      1
      0.0
      0.0
      1.0
      0.0
      41000.0
      1
      0.0

```
#Using the parameters from the new model with interactive terms
x2=df_seg_direct_cloud[['discount','amount','New Logo','acv','End of Quarter']]
ID=0
ID_max=len(df_seg_direct_cloud)
z=[]
for ID in range(ID_max):
    UT=np.dot(x2.iloc[ID,1:3],result_glm.params[1:3])+np.dot(x2.iloc[ID,3:5],result_glm.params[7:9])
    UT2=x2.iloc[ID,2]*result_glm.params[9]+x2.iloc[ID,4]*result_glm.params[14]+result_glm.params[12]#skipped dis:newlogo,dis:eof,dis:directcloud acv
    b=result_glm.params[0]
    p=x2.iloc[ID,1]
    def revenueD(d):
    #"""This is the negative revenue function"""
      return -np.exp(UT+(UT2+b)*np.sqrt(d**2))/(1+np.exp(UT+(UT2+b)*np.sqrt(d**2)))*(p-np.sqrt(d**2))
    sol3=minimize_scalar(revenueD)
    z.append(sol3.x)
z = np.array(z)
print('optimal discount is:',round(np.mean(z),0),'observed discount is ',np.round(np.mean(df['discount']),0))
# z is the optimal discount
```

    optimal discount is: 3272.0 observed discount is  14927.0

```
#xS for x star
df_seg_direct_cloud['optimal discount']=np.sqrt(z**2)
xS=df_seg_direct_cloud[['optimal discount','amount','New Logo','acv','End of Quarter']]
print(xS.shape,result2.params.shape)
ID=0
ID_max=len(df_seg_direct_cloud)
revOpt1=np.empty([ID_max,])
for ID in range(ID_max):
    j=0
    u=0
    UT=np.dot(xS.iloc[ID,0:3],result_glm.params[0:3])+np.dot(xS.iloc[ID,3:5],result_glm.params[7:9])
    UT2=xS.iloc[ID,2]*result_glm.params[9]+xS.iloc[ID,4]*result_glm.params[14]+result_glm.params[12] #skipped acv
    proba=np.exp(UT+UT2*np.sqrt(dis**2))/(1+np.exp(UT+UT2*np.sqrt(dis**2)))
    revOpt1[ID]=proba*(xS.iloc[ID,1]-z[ID]) #attention for the position
revOpt1=np.nan_to_num(revOpt1);
#the data points are few
```

    (17, 5) (9,)

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

###### direct on-premise

```
df_seg_direct_prem=x2_interaction[(x2_interaction['Direct prem']==1)]
df_seg_direct_prem.head(1)
```

      discount
      amount
      New Logo
      Partner cloud
      Partner prem
      Direct cloud
      Direct prem
      acv
      End of Quarter
      Revenue

      1202
      0.0
      75000.0
      0
      0.0
      0.0
      0.0
      1.0
      75000.0
      0
      0.0

```
x2=df_seg_direct_prem[['discount','amount','New Logo','acv','End of Quarter']]
ID=0
ID_max=len(df_seg_direct_prem)
z=[]
for ID in range(ID_max):
    UT=np.dot(x2.iloc[ID,1:3],result_glm.params[1:3])+np.dot(x2.iloc[ID,3:5],result_glm.params[7:9])
    UT2=x2.iloc[ID,2]*result_glm.params[9]+x2.iloc[ID,4]*result_glm.params[14]++result_glm.params[13] #skipped acv
    b=result_glm.params[0]
    p=x2.iloc[ID,1]
    def revenueD(d):
    #"""This is the negative revenue function"""
      return -np.exp(UT+(UT2+b)*np.sqrt(d**2))/(1+np.exp(UT+(UT2+b)*np.sqrt(d**2)))*(p-np.sqrt(d**2))
    sol3=minimize_scalar(revenueD)
    z.append(sol3.x)
z = np.array(z)
print('optimal discount is:',round(np.mean(z),0),'observed discount is ',np.round(np.mean(df['discount']),0))
# z is the optimal discount
```

    optimal discount is: 0.0 observed discount is  14927.0

```
#xS for x star
df_seg_direct_prem['optimal discount']=np.sqrt(z**2)
xS=df_seg_direct_prem[['optimal discount','amount','New Logo','acv','End of Quarter']]
print(xS.shape,result2.params.shape)
ID=0
ID_max=len(df_seg_direct_prem)
revOpt2=np.empty([ID_max,])
for ID in range(ID_max):
    j=0
    u=0
#    while j<=6:
#     u=u+xS.iloc[ID][j]*result2.params[j]
#     j=j+1
    UT=np.dot(xS.iloc[ID,0:3],result_glm.params[0:3])+np.dot(xS.iloc[ID,3:5],result_glm.params[7:9])
    UT2=xS.iloc[ID,2]*result_glm.params[9]+xS.iloc[ID,4]*result_glm.params[14]+result_glm.params[13] #skipped acv
    proba=np.exp(UT+UT2*np.sqrt(dis**2))/(1+np.exp(UT+UT2*np.sqrt(dis**2)))
    revOpt2[ID]=proba*(xS.iloc[ID,1]-z[ID]) #attention for the position
revOpt2=np.nan_to_num(revOpt2);
#Compute the expected revenue after optimization and compare it to the acual revenue
```

    (117, 5) (9,)

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

```
x2_interaction['Revenue']=np.where((df_test['Stage'] =='Won')&(df_test['Direct prem']==1),df_test['net amount'] ,0.0)
round(np.sum(revOpt2)/np.sum(x2_interaction['Revenue']),3)#This is the ratio of the optimized expected revenue / actual revenues

```

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      """Entry point for launching an IPython kernel.

    3.815

###### Partner on-prem

```
df_seg_partner_prem=x2_interaction[(x2_interaction['Partner prem']==1)]
df_seg_partner_prem.head(1)
```

      discount
      amount
      New Logo
      Partner cloud
      Partner prem
      Direct cloud
      Direct prem
      acv
      End of Quarter
      Revenue

      1200
      0.0
      32000.0
      0
      0.0
      1.0
      0.0
      0.0
      32000.0
      0
      0.0

```
#NEW as the discount can be calculated as zero, we want to make it positive
x2=df_seg_partner_prem[['discount','amount','New Logo','acv','End of Quarter']]

ID=0
ID_max=len(df_seg_partner_prem)
z=[]
for ID in range(ID_max):
    UT=np.dot(x2.iloc[ID,1:3],result_glm.params[1:3])+np.dot(x2.iloc[ID,3:5],result_glm.params[7:9])
    UT2=x2.iloc[ID,2]*result_glm.params[9]+x2.iloc[ID,4]*result_glm.params[14]+result_glm.params[11] #skipped acv
    b=result_glm.params[0]
    p=x2.iloc[ID,1]
    def revenueD(d):
    #"""This is the negative revenue function"""
      return -np.exp(UT+(UT2+b)*np.sqrt(d**2))/(1+np.exp(UT+(UT2+b)*np.sqrt(d**2)))*(p-np.sqrt(d**2))
    sol3=minimize_scalar(revenueD)
    z.append(sol3.x)
z = np.array(z)
print('optimal discount is:',round(np.mean(z),0),'observed discount is ',np.round(np.mean(df['discount']),0))
# z is the optimal discount
```

    optimal discount is: 4822.0 observed discount is  14927.0

```
#xS for x star
df_seg_partner_prem['optimal discount']=np.sqrt(z**2)
xS=df_seg_partner_prem[['optimal discount','amount','New Logo','acv','End of Quarter']]
print(xS.shape,result2.params.shape)
ID=0
ID_max=len(df_seg_partner_prem)
revOpt3=np.empty([ID_max,])
for ID in range(ID_max):
    j=0
    u=0
#    while j<=6:
#     u=u+xS.iloc[ID][j]*result2.params[j]
#     j=j+1
    UT=np.dot(xS.iloc[ID,0:3],result_glm.params[0:3])+np.dot(xS.iloc[ID,3:5],result_glm.params[7:9])
    UT2=xS.iloc[ID,2]*result_glm.params[9]+xS.iloc[ID,4]*result_glm.params[14]+result_glm.params[11] #skipped acv
    proba=np.exp(UT+UT2*np.sqrt(dis**2))/(1+np.exp(UT+UT2*np.sqrt(dis**2)))
    revOpt3[ID]=proba*(xS.iloc[ID,1]-z[ID]) #attention for the position
revOpt3=np.nan_to_num(revOpt3);
#Compute the expected revenue after optimization and compare it to the acual revenue
```

    (135, 5) (9,)

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

```
x2_interaction['Revenue']=np.where((df_test['Stage'] =='Won')&(df_test['Partner prem']==1),df_test['net amount'] ,0.0)
round(np.sum(revOpt3)/np.sum(x2_interaction['Revenue']),2)#This is the ratio of the optimized expected revenue / actual revenues
#the class example has 1.5
```

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      """Entry point for launching an IPython kernel.

    2.08

###### partner cloud

```
df_seg_partner_cloud=x2_interaction[(x2_interaction['Partner cloud']==1)]
df_seg_partner_cloud.head(1)
```

      discount
      amount
      New Logo
      Partner cloud
      Partner prem
      Direct cloud
      Direct prem
      acv
      End of Quarter
      Revenue

      1201
      1125000.0
      1875000.0
      1
      1.0
      0.0
      0.0
      0.0
      327251.0
      1
      0.0

```
#NEW as the discount can be calculated as zero, we want to make it positive
x2=df_seg_partner_cloud[['discount','amount','New Logo','acv','End of Quarter']]
ID=0
ID_max=len(df_seg_partner_cloud)
z=[]
for ID in range(ID_max):
    UT=np.dot(x2.iloc[ID,1:3],result_glm.params[1:3])+np.dot(x2.iloc[ID,3:5],result_glm.params[7:9])
    UT2=x2.iloc[ID,2]*result_glm.params[9]+x2.iloc[ID,4]*result_glm.params[14]+result_glm.params[10] #skipped acv
    b=result_glm.params[0]
    p=x2.iloc[ID,1]
    def revenueD(d):
    #"""This is the negative revenue function"""
      return -np.exp(UT+(UT2+b)*np.sqrt(d**2))/(1+np.exp(UT+(UT2+b)*np.sqrt(d**2)))*(p-np.sqrt(d**2))
    sol3=minimize_scalar(revenueD)
    z.append(sol3.x)
z = np.array(z)
print('optimal discount is:',round(np.mean(z),0),'observed discount is ',np.round(np.mean(df['discount']),0))
# z is the optimal discount
```

    optimal discount is: 2995.0 observed discount is  14927.0

```
#xS for x star
df_seg_partner_cloud['optimal discount']=np.sqrt(z**2)
xS=df_seg_partner_cloud[['optimal discount','amount','New Logo','acv','End of Quarter']]
print(xS.shape,result2.params.shape)
ID=0
ID_max=len(df_seg_partner_cloud)
revOpt4=np.empty([ID_max,])
for ID in range(ID_max):
    UT=np.dot(xS.iloc[ID,0:3],result_glm.params[0:3])+np.dot(xS.iloc[ID,3:5],result_glm.params[7:9])
    UT2=xS.iloc[ID,2]*result_glm.params[9]+xS.iloc[ID,4]*result_glm.params[14]+result_glm.params[10] #skipped acv
    proba=np.exp(UT+UT2*np.sqrt(dis**2))/(1+np.exp(UT+UT2*np.sqrt(dis**2)))
    revOpt4[ID]=proba*(xS.iloc[ID,1]-z[ID]) #attention for the position
revOpt4=np.nan_to_num(revOpt4);
#Compute the expected revenue after optimization and compare it to the acual revenue
```

    (36, 5) (9,)

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

```
x2_interaction['Revenue']=np.where((df_test['Stage'] =='Won')&(df_test['Partner cloud']==1),df_test['net amount'] ,0.0)
round(np.sum(revOpt4)/np.sum(x2_interaction['Revenue']),3)#This is the ratio of the optimized expected revenue / actual revenues
#the class example has 1.5
```

    /usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      """Entry point for launching an IPython kernel.

    2.028

## Performance Metrics:

#### b. Expected revenue improvement over optimized discounts (based on your estimation results, optimize discounts offered to leads in the hold-out sample).

```
print("Ratio of expected revenue improvement vs actual revenue: ")
(np.sum(revOpt1)+np.sum(revOpt2)+np.sum(revOpt3)+np.sum(revOpt4))/np.sum(df_test['Revenue']) #ratio
```

    Ratio of expected revenue improvement vs actual revenue:

    2.348735587114301

```
print('Expected revenue improvement increased by: ')
round(np.sum(revOpt1)+np.sum(revOpt2)+np.sum(revOpt3)+np.sum(revOpt4)-np.sum(df_test['Revenue']),2) #difference
```

    Expected revenue improvement increased by:

    6586586.66


```python
# In mathematics, the exponential integral Ei is a special function on the complex plane.
# It is defined as one particular definite integral of the ratio between an exponential function and its argument.

from sympy import *
from sympy import E
from sympy.abc import x,omega,u,m,g
f = lambda x: E**(E**x)
expr = f(x)
Eq(omega,f(x))
```

$\displaystyle \omega = e^{e^{x}}$

```python
dexpr = Derivative(expr)
Eq(dexpr,dexpr.doit())
```

$\displaystyle \frac{d}{d x} e^{e^{x}} = e^{x} e^{e^{x}}$

```python
dexprs = Eq(dexpr,dexpr.doit()).rhs.simplify()
Eq(dexpr.doit(),dexprs)
```

$\displaystyle e^{x} e^{e^{x}} = e^{x + e^{x}}$

```python
Eq(Integral(expr),Derivative(expr)) # is this true in terms of density? (a very thin string = large object in essence)
                                    # trying to show that derivative/integral are inverse operations
```

$\displaystyle \int e^{e^{x}}\, dx = \frac{d}{d x} e^{e^{x}}$

```python
# if the above is true then the below expression must also be true
Eq(Integral(expr),dexprs)
#https://en.wikipedia.org/wiki/Exponential_integral
```

$\displaystyle \int e^{e^{x}}\, dx = e^{x + e^{x}}$

```python
Eq(Integral(expr),Integral(expr).doit())   # .doit() is a method that evaluate objects that are not evaluated by default.
```

$\displaystyle \int e^{e^{x}}\, dx = \operatorname{Ei}{\left(e^{x} \right)}$

```python
# therefore
Eq(Integral(expr).doit(),dexprs)   # reads as "the exponential integral of e**x equals e**(x+e**x)"
```

$\displaystyle \operatorname{Ei}{\left(e^{x} \right)} = e^{x + e^{x}}$

```python
# and
Eq(Derivative(Integral(expr).doit()),Derivative(Integral(expr).doit()).doit())
```

$\displaystyle \frac{d}{d x} \operatorname{Ei}{\left(e^{x} \right)} = e^{e^{x}}$

```python
# here we succeeded in calculating some definite exponential integrals. The calculator got too slow around m>4
u = Integral(Ei(exp(x)),(x,0,m))
Eq(g,u)
```

$\displaystyle g = \int\limits_{0}^{m} \operatorname{Ei}{\left(e^{x} \right)}\, dx$

```python
import timeit # This returns the time it takes to execute the main statement a number of times, measured in seconds as a float.
values = []
total_time = []
from collections import defaultdict
Points = defaultdict(list)

for i in range (1,7):
    start0 = timeit.default_timer()
    print(u.subs(m,i).evalf(),"-------------------------------------------------------------when m = ",i)
    stop0 = timeit.default_timer()
    values.append(Integral(Ei(exp(x)),(x,0,)).evalf())
    print('                        Compute Time: ', stop0 - start0,'Seconds')
    print('                                      ')
    total_time.append(stop0 - start0)
    Points[total_time[i-1]].append(i)
print('                                                           ')
print('                                                           ')
print("                        Average Speed =",sum(total_time)/len(values),"Seconds     ")
print('                                                           ')
print('                                                           ')
print('                                                           ')
print(Points)
```

    4.15775030018700 -------------------------------------------------------------when m =  1
                            Compute Time:  0.4002557999999681 Seconds

    58.4176824523972 -------------------------------------------------------------when m =  2
                            Compute Time:  0.6176943000000392 Seconds

    1552871.13084787 -------------------------------------------------------------when m =  3
                            Compute Time:  0.8336731999999074 Seconds

    1.82897127952040e+20 -------------------------------------------------------------when m =  4
                            Compute Time:  6.680456499999991 Seconds

    1.32124004569319e+60 -------------------------------------------------------------when m =  5
                            Compute Time:  17.255297300000052 Seconds

    0.e+170 -------------------------------------------------------------when m =  6
                            Compute Time:  14.982998400000042 Seconds

                            Average Speed = 6.795062583333333 Seconds

    defaultdict(<class 'list'>, {0.4002557999999681: [1], 0.6176943000000392: [2], 0.8336731999999074: [3], 6.680456499999991: [4], 17.255297300000052: [5], 14.982998400000042: [6]})

```python
from IPython.display import Image
from IPython.core.display import HTML
from sympy import *
Image(url= "https://i.imgur.com/8HOr9wu.png")
```


## 2-2. アダマールテスト

最も簡単な量子アルゴリズムとして、アダマールテストとよばれる以下のような量子回路（図1）を考える。つまり、第1ビットは$|0\rangle$に、第2ビット以降は状態$|\psi\rangle$に初期化されていて、まず第1ビットにアダマールゲートをかける。そして、全体に制御ユニタリ演算子$\Lambda(U)$（後述）を作用させ、再び第1ビットにアダマールゲートをかけて、最後にその第1ビットを測定する。

ここで制御ユニタリ演算子$\Lambda(U)$というのは、第1量子ビットが$|0\rangle$の場合にはなにもせず、$|1\rangle$の場合には$U$を作用させるユニタリ演算である。

$$
\Lambda (U) = |0\rangle \langle 0| \otimes I + |1\rangle \langle 1|  \otimes U.
$$

つまり、１つ目の量子ビットが$|0\rangle$か$|1\rangle$かによって条件分岐して、「なにもしない」または「$U$を作用させる」という演算が実行される。従来のコンピュータでは条件分岐は同時に実行することができないが、量子コンピュータでは状態の重ね合わせを利用して、条件分岐を**同時並列的に実行することができる**。

このアダマールテストの動作について考えていく。最初は簡単のために、量子状態$|\psi \rangle$が
ユニタリー演算（行列）$U$の固有値$e^{i \lambda}$の固有状態（固有ベクトル）である場合を考える：
\begin{eqnarray}
U|\psi \rangle = e^{i \lambda} |\psi\rangle.
\end{eqnarray}

１つ目の量子ビットにアダマール演算$H$を作用させることで
\begin{eqnarray}
\frac{1}{\sqrt{2}} (|0\rangle  + |1\rangle) \otimes |\psi  \rangle
\end{eqnarray}
が得られる。
その後、制御$U$演算を作用させることによって、
**固有値**$e^{i\lambda}$**が１つめの量子ビットの相対位相として得られる**（このことを**位相キックバック**と呼ぶ）:
\begin{eqnarray}
&&\frac{1}{\sqrt{2}} (|0\rangle \otimes |\psi  \rangle
+ |1\rangle \otimes U|\psi \rangle )
\\
&=&\frac{1}{\sqrt{2}} (|0\rangle \otimes |\psi  \rangle
+e^{i \lambda} |1\rangle \otimes |\psi \rangle )
\\
&=&
\frac{1}{\sqrt{2}} (|0\rangle
+e^{i \lambda} |1\rangle )\otimes |\psi  \rangle.
\end{eqnarray}
最後に、１つ目の量子ビットに再度アダマール演算を行い
\begin{eqnarray}
\left(\frac{1+e^{i\lambda}}{2}|0\rangle
+\frac{1-e^{i\lambda}}{2} |1\rangle \right)\otimes |\psi  \rangle
\label{eq01}
\end{eqnarray}
が得られる。
１つ目の量子ビットを測定すると測定結果$m=0,1$を得る確率は
\begin{eqnarray}
p_{m}=\left|\frac{1+(-1)^m e^{i\lambda}}{2}\right|^2 =\frac{1+(-1)^m \cos \lambda}{2}
\end{eqnarray}
となる。
$|\psi \rangle$、$U$、$\langle \psi |$は
それぞれ$2^n$次元の列ベクトル、$2^n \times 2^n$行列、
$2^n$次元の行ベクトルなので、
このアダマールテストを古典コンピュータ上で愚直に計算すると
指数的に大きなメモリーの確保と演算回数が必要になる。
一方で、量子コンピューターでは、
確率分布$p_m$のもとで$m$がサンプルされる。
$\cos \lambda$を
ある誤差$\epsilon$で推定したい場合は、
その逆数$1/\epsilon$の多項式回程度サンプルすればよいことになる。

同じ計算を、必ずしも固有ベクトルとは限らない、一般の入力に対して行うと、測定前の状態は、

$$
 |0\rangle \frac{I+U}{2} |\psi \rangle  +  |1\rangle  \frac{I-U}{2} |\psi \rangle
$$

となり、0もしくは1が得られる確率は、

\begin{align}
p_0 &= \frac{1+ {\rm Re} \langle \psi | U | \psi \rangle }{2} \\
p_1 &= \frac{1- {\rm Re} \langle \psi | U | \psi \rangle }{2} \tag{1}
\end{align}

となる。つまり、量子コンピュータ上でアダマールテストを実行すれば、その測定結果のサンプル平均をとることで**ベクトル**$|\psi \rangle$**でユニタリ行列**$U$**を挟んだ値を推定することができる**。同じ値を古典コンピュータで求めようとした場合、量子ビット数$n$が大きくなるにつれベクトルや行列の次元は指数的に大きくなるので、指数的な時間を要する。

なお、１つ目の量子ビットを測定した後の、２つ目の量子ビットの状態は、測定結果$m = 0, 1$に応じて以下の状態になる（規格化因子は省略）：

$$
|\psi_0\rangle = \frac{I + U}{2}|\psi\rangle,
\quad
|\psi_1\rangle = \frac{I - U}{2}|\psi\rangle.
$$

ここで、$U$が１量子ビットのユニタリ演算で、かつその固有値が$\pm 1$であるような場合を考える。固有値$\pm 1$に対応する固有ベクトル$|u_1\rangle$, $|u_{-1}\rangle$を使って$|\psi\rangle = c_1|u_1\rangle + c_2|u_2\rangle$と展開し代入することで、測定後の状態$|\psi_0\rangle$, $|\psi_1\rangle$はそれぞれ固有値$\pm 1$に対応する固有状態であることが分かる。固有値が$\pm 1$ではない場合も、アダマールテストの出力を入力として繰り返すと$U$の固有状態に状態が収束していく（興味のある人は、以下の例を参考にして試してもらいたい）。

### SymPyでの実装
具体的な例として、$U=H$（アダマールゲート）の場合を考えてみよう。補助量子ビットを$|0\rangle$、アダマールテストの入力$|\psi\rangle$も$|0\rangle$とする。

```python
from sympy import *
from sympy.physics.quantum import *
from sympy.physics.quantum.qubit import Qubit,QubitBra
init_printing() # ベクトルや行列を綺麗に表示するため
from sympy.physics.quantum.gate import X,Y,Z,H,S,T,CNOT,SWAP,CPHASE,CGateS
```

```python
# Google Colaboratory上でのみ実行してください
from IPython.display import HTML
def setup_mathjax():
    display(HTML('''

    '''))
get_ipython().events.register('pre_run_cell', setup_mathjax)
```

```python
state = Qubit('00')
```

制御H演算は、`CGateS()`を用いて

```python
ctrlH = CGateS(1,H(0))
represent(ctrlH,nqubits=2)
```

と行列表示される。
測定前の状態は、

```python
H(1)*ctrlH*H(1)*state
```

とかけるが、SymPyに計算させてみると

```python
qapply(H(1)*ctrlH*H(1)*state)
```

となる。第１章で紹介したSymPyの`measure_partial`関数を用いて、1つ目の量子ビットの測定結果が0だった場合の量子状態と確率を求めると、

```python
from sympy.physics.quantum.qubit import measure_all, measure_partial, measure_all_oneshot, measure_partial_oneshot
measured_state_and_probability_zero = measure_partial(qapply(H(1)*ctrlH*H(1)*state),(1,))[0]
simplify(measured_state_and_probability_zero)
```

となる。$\langle 0 | H | 0\rangle = 1/\sqrt{2}$だから、測定確率が式(1)と一致していることが分かる。
また、２つめの量子ビットに$H$を作用させてみると、測定後の状態が$H$の固有ベクトルになっていることが分かる（SymPyのインデックスは左から１つ目が1、２つ目が0になるよう対応させていることに注意）。

```python
measured_state_zero = measured_state_and_probability_zero[0]
simplify(qapply(H(0)*measured_state_zero))
```

同様に1の測定結果を得た場合は、固有値−1の固有状態であることも確認できるので試してもらいたい。

```python
measured_state_one = measure_partial(qapply(H(1)*ctrlH*H(1)*state),(1,))[1][0]
simplify(qapply(H(0)*measured_state_one))
```


# **Фильтр Калмана для системы ДУ второго порядка**

## Филаткин Алексей

Построим фильтр Калмана для системы
 \begin{cases}
   \dot x(t) = \begin{pmatrix} 0 & 1\\ 1 & 0 \end{pmatrix}x(t) + \begin{pmatrix} 1\\ 0\end{pmatrix}u(t) + \widetilde{w}(t)\\
   z(t) = \begin{pmatrix} 1& 0\end{pmatrix}x(t) + v(t)
 \end{cases}
где $u(t)$ - внешнее воздействие, $w$ и $v$ - не коррелирующие между собой белые шумы с матрицей ковариции $Q =\begin{pmatrix} 3 & 0 \\ 0 & 3 \end{pmatrix}$ и дисперсией $R = 1$ соответственно, z(t) - измерение $x_{1}$

Белый шум моделируется нормальным распределением с нулевой медианой.

Внешнее воздействие также имеет нормальное распределение.

Тогда разностная схема в случае применения метода Эйлера для численного дифференцирования имеет вид
\begin{cases}
   x_{k} = Ax_{k-1} + Bu_{k} + w_{k}\\
   z_{k} = Hx_{k} + v_{k}
 \end{cases}
 где $A = \begin{pmatrix} 1 & \tau\\ \tau & 1 \end{pmatrix}$, $B = \begin{pmatrix} \tau\\ 0 \end{pmatrix}$, $H = \begin{pmatrix} 1 & 0 \end{pmatrix}$, w - белый шум с матрицей ковариации $Q =\begin{pmatrix} 3\tau^2 & 0 \\ 0 & 3\tau^2 \end{pmatrix}$, $\tau$ - шаг по времени

```python
import numpy as np
import matplotlib.pyplot as plt

# Инициализация
tau = 0.01
T = 3 # Суммарное время работы
A = np.array([[1, tau], [tau, 1]])
B = np.array([tau, 0])
H = np.array([1, 0]).T
Q = np.array([[3 * tau ** 2, 0], [0, 3 * tau ** 2]])
R = 1
I = np.eye(2)
P = I
x = np.array([1, 1])
x_estimate = x
l = int(T / tau)
# Задание шумов и внешнего воздействия
u = np.random.normal(0, 1, l)
v = np.random.normal(0, 1, l)
w = tau*np.array([np.random.normal(0, 3**0.5, l), np.random.normal(0, 3**0.5, l)])
# Сюда будут записываться данные
x_plot1 = np.zeros((l, 2))
x_plot2 = np.zeros((l, 2))
z_plot = np.zeros(l)
z = x[0]
```

Фильтр Калмана - классический двухшаговый алгоритм предиктор-корректор, реализующийся на каждом такте управления.

1) Оценка вектора состояния x по модели
\begin{equation}
  \bar x_{k} = A x_{k-1} + B u_{k}
\end{equation}
Матрица ковариации ошибки оценивается следующим образом:
\begin{equation}
  \bar P_{k} = A \hat P_{k-1} A^{T} + Q
\end{equation}
где $\hat P_{k-1}$ - оценка матрицы ковариации ошибки с учётом коррекции на предыдущем шаге

2) Коррекция предсказания с учетом измерений

Матрица коррекции:
\begin{equation}
  \bar K_{k} = \bar P_{k} H^{T}(H \bar P_{k-1} H^{T} + R)^{-1}
\end{equation}
Оценка состояния:
\begin{equation}
  \hat x_{k} = \bar x_{k} + K_{k}(z_{k} - H \bar x_{k})
\end{equation}
Матрица ковариации ошибки с учётом коррекции:
\begin{equation}
  \hat P_{k} = (I - K_{k} H) \bar P_{k}
\end{equation}
Точное решение системы без внешнего воздействия и без шумов имеет вид
\begin{equation}
   x = C_{1} e^t \begin{pmatrix} 1 \\ 1 \end{pmatrix} + C_{2} e^{-t} \begin{pmatrix} 1 \\ -1 \end{pmatrix}
\end{equation}
Поэтому при несильных внешних воздействиях графики должны быть похоже на экспоненциальные

```python
for i in range(l):
    # Сохраняем в массив данных
    x_plot1[i, :] = x_estimate
    x_plot2[i, :] = x
    z_plot[i] = z
    # Новый шаг процесса
    x = np.dot(A, x_estimate) + np.dot(B, u[i]) + w[:, i]
    # Измеряем вектор x
    z = np.dot(H, x) + v[i]
    # Предиктор
    x_predict = np.dot(A, x_estimate) + np.dot(B, u[i])
    P_predict = np.dot(A, np.dot(P, A.T)) + Q
    # Корректор
    K = np.dot(P_predict, H.T) / (np.dot(H, np.dot(P_predict, H.T)) + R)
    x_estimate = x_predict + np.dot(K, z - np.dot(H, x_predict))
    P = np.dot(I - np.dot(K, H), P_predict)
# Строим графики
t = np.linspace(0, T, l)
fig = plt.figure(figsize=(25, 15))

ax = fig.add_subplot(2, 3, 1)
ax.plot(t, z_plot, label='Измерение x1')
ax.plot(t, x_plot2[:, 0], label='Модельное значение x1')
ax.grid()
ax.legend()
ax.set_xlabel('Время, с')
ax.set_ylabel('x1')

ax = fig.add_subplot(2, 3, 2)
ax.plot(t, x_plot1[:, 0], label='Скорректрированное фильтром значение x1')
ax.plot(t, x_plot2[:, 0], label='Модельное значение x1')
ax.set_xlabel('Время, с')
ax.set_ylabel('x1')
ax.grid()
ax.legend()

ax = fig.add_subplot(2, 3, 3)
ax.plot(t, x_plot1[:, 1], label='Скорректрированное фильтром значение x2')
ax.plot(t, x_plot2[:, 1], label='Модельное значение x2')
ax.set_xlabel('Время, с')
ax.set_ylabel('x2')
ax.grid()
ax.legend()

ax = fig.add_subplot(2, 3, 4)
ax.plot(t, abs(x_plot1[:, 0] - x_plot2[:, 0]))
ax.set_xlabel('Время, с')
ax.set_ylabel('Ошибка коррекции x1')
ax.grid()

ax = fig.add_subplot(2, 3, 5)
ax.plot(t, abs(x_plot1[:, 1] - x_plot2[:, 1]))
ax.set_xlabel('Время, с')
ax.set_ylabel('Ошибка коррекции x2')
ax.grid()
plt.show()
```

Усилим шум модели $\widetilde{Q} = 100 Q$ а заодно для разнообразия зададим постоянное воздействие.

```python
import numpy as np
import matplotlib.pyplot as plt

# Инициализация
tau = 0.01
T = 5 # Суммарное время работы
A = np.array([[1, tau], [tau, 1]])
B = np.array([tau, 0])
H = np.array([1, 0]).T
Q = np.array([[3 * tau ** 2, 0], [0, 3 * tau ** 2]])
R = 1
I = np.eye(2)
P = I
x = np.array([1, 1])
x_estimate = x
l = int(T / tau)
# ИЗМЕНИЛИ ЗАДАНИЕ ШУМОВ И ВОЗДЕЙСТВИЯ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
u = np.random.normal(0, 1, l)
v = -2*np.ones(l)
w = 100*tau*np.array([np.random.normal(0, 3 ** 0.5, l), np.random.normal(0, 3 ** 0.5, l)])
# Сюда будут записываться данные
x_plot1 = np.zeros((l, 2))
x_plot2 = np.zeros((l, 2))
z_plot = np.zeros(l)
z = x[0]
for i in range(l):
    # Сохраняем в массив данных
    x_plot1[i, :] = x_estimate
    x_plot2[i, :] = x
    z_plot[i] = z
    # Новый шаг процесса
    x = np.dot(A, x_estimate) + np.dot(B, u[i]) + w[:, i]
    # Измеряем вектор x
    z = np.dot(H, x) + v[i]
    # Предиктор
    x_predict = np.dot(A, x_estimate) + np.dot(B, u[i])
    P_predict = np.dot(A, np.dot(P, A.T)) + Q
    # Корректор
    K = np.dot(P_predict, H.T) / (np.dot(H, np.dot(P_predict, H.T)) + R)
    x_estimate = x_predict + np.dot(K, z - np.dot(H, x_predict))
    P = np.dot(I - np.dot(K, H), P_predict)
# Строим графики
t = np.linspace(0, T, l)
fig = plt.figure(figsize=(25, 15))

ax = fig.add_subplot(2, 3, 1)
ax.plot(t, z_plot, label='Измерение x1')
ax.plot(t, x_plot2[:, 0], label='Модельное значение x1')
ax.grid()
ax.legend()
ax.set_xlabel('Время, с')
ax.set_ylabel('x1')

ax = fig.add_subplot(2, 3, 2)
ax.plot(t, x_plot2[:, 0], label='Модельное значение x1')
ax.plot(t, x_plot1[:, 0], label='Скорректрированное фильтром значение x1')
ax.set_xlabel('Время, с')
ax.set_ylabel('x1')
ax.grid()
ax.legend()

ax = fig.add_subplot(2, 3, 3)
ax.plot(t, x_plot2[:, 1], label='Модельное значение x2')
ax.plot(t, x_plot1[:, 1], label='Скорректрированное фильтром значение x2')
ax.set_xlabel('Время, с')
ax.set_ylabel('x2')
ax.grid()
ax.legend()

ax = fig.add_subplot(2, 3, 4)
ax.plot(t, abs(x_plot1[:, 0] - x_plot2[:, 0]))
ax.set_xlabel('Время, с')
ax.set_ylabel('Ошибка коррекции x1')
ax.grid()

ax = fig.add_subplot(2, 3, 5)
ax.plot(t, abs(x_plot1[:, 1] - x_plot2[:, 1]))
ax.set_xlabel('Время, с')
ax.set_ylabel('Ошибка коррекции x2')
ax.grid()
plt.show()
```

Эффективность фильтра теперь наглядна. Поскольку внешнее воздействие слабее экспоненциального, то естественно с ростом времени решение снова приходит к классическому виду.

Если шум модели сравним с ней самой(экспоненциальный по времени), то фильтр все равно отлично работает.

Если же шум сильнее решения, то, как и ожидается, точно установить значения нельзя и в среднем будет ноль(из начальных условий)

```python
import numpy as np
import matplotlib.pyplot as plt

# Инициализация
tau = 0.01
T = 5 # Суммарное время работы
A = np.array([[1, tau], [tau, 1]])
B = np.array([tau, 0])
H = np.array([1, 0]).T
Q = np.array([[3 * tau ** 2, 0], [0, 3 * tau ** 2]])
R = 1
I = np.eye(2)
P = I
x = np.array([1, 1])
x_estimate = x
l = int(T / tau)
# ИЗМЕНИЛИ ЗАДАНИЕ ШУМОВ И ВОЗДЕЙСТВИЯ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
t = np.linspace(0, T, l)
u = np.zeros((l, 2))
v = np.random.normal(0, 1, l)
w = np.array([np.exp(0.5*t) * np.random.normal(0, 1, l), t**2*np.exp(t) * np.random.normal(0, 3 ** 0.5, l)])
# Сюда будут записываться данные
x_plot1 = np.zeros((l, 2))
x_plot2 = np.zeros((l, 2))
z_plot = np.zeros(l)
z = x[0]
for i in range(l):
    # Сохраняем в массив данных
    x_plot1[i, :] = x_estimate
    x_plot2[i, :] = x
    z_plot[i] = z
    # Новый шаг процесса
    x = np.dot(A, x_estimate) + np.dot(B, u[i]) + w[:, i]
    # Измеряем вектор x
    z = np.dot(H, x) + v[i]
    # Предиктор
    x_predict = np.dot(A, x_estimate) + np.dot(B, u[i])
    P_predict = np.dot(A, np.dot(P, A.T)) + Q
    # Корректор
    K = np.dot(P_predict, H.T) / (np.dot(H, np.dot(P_predict, H.T)) + R)
    x_estimate = x_predict + np.dot(K, z - np.dot(H, x_predict))
    P = np.dot(I - np.dot(K, H), P_predict)
# Строим графики
fig = plt.figure(figsize=(25, 15))

ax = fig.add_subplot(2, 3, 1)
ax.plot(t, z_plot, label='Измерение x1')
ax.plot(t, x_plot2[:, 0], label='Модельное значение x1')
ax.grid()
ax.legend()
ax.set_xlabel('Время, с')
ax.set_ylabel('x1')

ax = fig.add_subplot(2, 3, 2)
ax.plot(t, x_plot2[:, 0], label='Модельное значение x1')
ax.plot(t, x_plot1[:, 0], label='Скорректрированное фильтром значение x1')
ax.set_xlabel('Время, с')
ax.set_ylabel('x1')
ax.grid()
ax.legend()

ax = fig.add_subplot(2, 3, 3)
ax.plot(t, x_plot2[:, 1], label='Модельное значение x2')
ax.plot(t, x_plot1[:, 1], label='Скорректрированное фильтром значение x2')
ax.set_xlabel('Время, с')
ax.set_ylabel('x2')
ax.grid()
ax.legend()

ax = fig.add_subplot(2, 3, 4)
ax.plot(t, abs(x_plot1[:, 0] - x_plot2[:, 0]))
ax.set_xlabel('Время, с')
ax.set_ylabel('Ошибка коррекции x1')
ax.grid()

ax = fig.add_subplot(2, 3, 5)
ax.plot(t, abs(x_plot1[:, 1] - x_plot2[:, 1]))
ax.set_xlabel('Время, с')
ax.set_ylabel('Ошибка коррекции x2')
ax.grid()
plt.show()
```

Действительно, вследствие слишком сильного шума $x_{2}$($\sim t^2 e^{t}$ ) коррекция невозможна

# **Выводы**

Фильтр Калмана довольно неплохо справляется  с определением значения переменных при корректной постановки задачи. С увеличением времени работы программы стабильность увеличивается.


```python
import numpy
import math
from matplotlib import pyplot
```

# Line segment source

## Generate meshgrid for pressure and velocity field

```python
N = 100                               # Number of points in each direction
x_start, x_end = -5, 7            # x-direction boundaries
y_start, y_end = -7, 7            # y-direction boundaries
x = numpy.linspace(x_start, x_end, N)    # computes a 1D-array for x
y = numpy.linspace(y_start, y_end, N)    # computes a 1D-array for y
X, Y = numpy.meshgrid(x, y)              # generates a mesh grid
```

## Line segment source integration

### Line source solution for pressure and velocity (Datta-Gupta, 2007)

\begin{equation}
P(x,y)=B{{Q}_{w}}=-\frac{70.60\mu }{h\sqrt{{{k}_{x}}{{k}_{y}}}}\ln \left\{ {{(x-{{x}_{w}})}^{2}}+\frac{{{k}_{x}}}{{{k}_{y}}}{{(y-{{y}_{w}})}^{2}} \right\}{{Q}_{w}}+{{P}_{avg}}
\end{equation}

\begin{equation}
\frac{\partial P}{\partial x}=u=\frac{0.8936}{h\phi }\sqrt{\frac{{{k}_{x}}}{{{k}_{y}}}}\sum\limits_{k=1}^{{{N}_{w}}}{{{Q}_{k}}}\frac{x-{{x}_{k}}}{{{\left( x-{{x}_{k}} \right)}^{2}}+\frac{{{k}_{x}}}{{{k}_{y}}}{{(y-{{y}_{k}})}^{2}}}
\end{equation}

\begin{equation}
\frac{\partial P}{\partial y}=v=\frac{0.8936}{h\phi }\sqrt{\frac{{{k}_{x}}}{{{k}_{y}}}}\sum\limits_{k=1}^{{{N}_{w}}}{{{Q}_{k}}}\frac{y-{{y}_{k}}}{{{\left( x-{{x}_{k}} \right)}^{2}}+\frac{{{k}_{x}}}{{{k}_{y}}}{{(y-{{y}_{k}})}^{2}}}
\end{equation}

### Local coordinate transformation

Figure 1. Nomenclature of the boundary element in the local coordinates

Based on the nomenclatures in Fig. 1, a boundary element could be determined as its start point (A), end point (B), length (lj) and inclination (αj). The coordinate transformation from global system to local system can be given as follows:

\begin{matrix}
   {x}'=x-{{x}_{A}} & {y}'=y-{{y}_{A}}  \\
\end{matrix}

Where (xA,yA) is the location of start point (point A in Fig. A-1) of a BE in global coordinate system. Trigonometric sine and cosine function for a BE could be determined from start point (A) and end point (B) of the boundary elements, which can be expressed as follows:

\begin{matrix}
   \sin {{\alpha }_{j}}=\frac{{{y}_{B}}-{{y}_{A}}}{{{l}_{j}}} & \cos {{\alpha }_{j}}=\frac{{{x}_{B}}-{{x}_{A}}}{{{l}_{j}}}  \\
\end{matrix}

In local coordinate system, line source are placed along the boundary element at a point (x’j,y’j). The location can be easily given as follows:

\begin{equation}
\begin{matrix}
   \begin{matrix}
   {{{{x}'}}_{j}}=t\cos {{\alpha }_{j}} & {{{{y}'}}_{j}}=t\sin {{\alpha }_{j}}  \\
\end{matrix} & 0\le t  \\
\end{matrix}\le {{l}_{j}}
\end{equation}

```python
#coordinate transformation
origin_new_x=0            # A point in Fig 1
origin_new_y=0

X_local=X-origin_new_x
Y_local=Y-origin_new_y
```

### Line segment source integration
Integrating the line source solution (Eqs. 3-4) from A to B over a length l, the velocity can be given as follows:

\begin{equation}
P({x}',{y}')=-\frac{70.60\mu }{h\sqrt{{{k}_{x}}{{k}_{y}}}}{{Q}_{w}}\int_{t=0}^{t={{l}_{j}}}{\ln \left\{ {{({x}'-t\cos {{\alpha }_{j}})}^{2}}+\frac{{{k}_{x}}}{{{k}_{y}}}{{({y}'-t\sin {{\alpha }_{j}})}^{2}} \right\}dt}
\end{equation}

\begin{equation}
u\left( {x}',{y}' \right)=\frac{0.8936}{h\phi }\sqrt{\frac{{{k}_{x}}}{{{k}_{y}}}}{{Q}_{j}}\int_{t=0}^{t={{l}_{j}}}{\frac{{x}'-t\cos {{\alpha }_{j}}}{{{\left( {x}'-t\cos {{\alpha }_{j}} \right)}^{2}}+\frac{{{k}_{x}}}{{{k}_{y}}}{{({y}'-t\sin {{\alpha }_{j}})}^{2}}}dt}
\end{equation}

\begin{equation}
v\left( {x}',{y}' \right)=\frac{0.8936}{h\phi }\sqrt{\frac{{{k}_{x}}}{{{k}_{y}}}}{{Q}_{j}}\int_{t=0}^{t={{l}_{j}}}{\frac{{y}'-t\sin {{\alpha }_{j}}}{{{\left( {x}'-t\cos {{\alpha }_{j}} \right)}^{2}}+\frac{{{k}_{x}}}{{{k}_{y}}}{{({y}'-t\sin {{\alpha }_{j}})}^{2}}}dt}
\end{equation}

The integrals can be rewritten as follows:

\begin{equation}
\left. P({x}',{y}')={{Q}_{w}}\frac{{{d}_{p}}}{a}\left[ b\ln ({{{{x}'}}^{2}}-2bt+a{{t}^{2}}+{{k}_{r}}{{{{y}'}}^{2}})-ta\ln \left\{ {{({x}'-t\cos {{\alpha }_{j}})}^{2}}+{{k}_{r}}{{({y}'-t\sin {{\alpha }_{t}})}^{2}} \right\}+2\sqrt{{{k}_{r}}}c\arctan \left( \frac{b-at}{\sqrt{{{k}_{r}}}c} \right) \right] \right|_{t=0}^{t={{l}_{j}}}
\end{equation}

\begin{equation}
\left. u\left( {x}',{y}' \right)={{Q}_{j}}\frac{{{d}_{v}}}{a}\left[ \cos {{\alpha }_{j}}\ln ({{{{x}'}}^{2}}-2bt+a{{t}^{2}}+{{k}_{r}}{{{{y}'}}^{2}})+2\sqrt{{{k}_{r}}}\sin {{\alpha }_{j}}\arctan (\frac{at-b}{\sqrt{{{k}_{r}}}c}) \right] \right|_{t=0}^{t={{l}_{j}}}
\end{equation}

\begin{equation}
v\left( {x}',{y}' \right)=\left. {{Q}_{j}}\frac{{{d}_{v}}}{a}\left[ \sin {{\alpha }_{j}}\ln ({{{{x}'}}^{2}}-2tb+a{{t}^{2}}+{{k}_{r}}{{{{y}'}}^{2}})+2\sqrt{\frac{1}{{{k}_{r}}}}\cos {{\alpha }_{j}}\arctan (\frac{b-at}{\sqrt{{{k}_{r}}}c}) \right] \right|_{t=0}^{t={{l}_{j}}}
\end{equation}

where,
\begin{equation}
a={{\cos }^{2}}{{\alpha }_{j}}+{{k}_{r}}{{\sin }^{2}}{{\alpha }_{j}}\end{equation}
\begin{equation}
b={x}'\cos {{\alpha }_{j}}+{y}'{{k}_{r}}\sin {{\alpha }_{j}}\end{equation}
\begin{equation}
c={y}'\cos {{\alpha }_{j}}-{x}'\sin {{\alpha }_{j}}\end{equation}
\begin{equation}
{{d}_{v}}=-\frac{0.4468}{h\phi }\sqrt{{{k}_{r}}}\begin{matrix}
   {} & {{d}_{p}}=\frac{70.60\mu }{h\sqrt{{{k}_{x}}{{k}_{y}}}}  \\
\end{matrix}\end{equation}
\begin{equation}
{{k}_{r}}={{k}_{x}}/{{k}_{y}}\end{equation}

## Code implementation

```python
#More General method
Q = 2000    #strength of the source-sheet,stb/d
h=25.26        #thickness of local gridblock,ft
phi=0.2        #porosity
kx=200         #pemerability in x direction,md
ky=200         #pemerability in y direction,md
kr=kx/ky       #pemerability ratio
miu=1          #viscosity,cp

#Line segment source parameters
L1=1                      #Line segment source length
alpha=30*numpy.pi/180     #Line segment angle from x direction, angle alpha in Fig.1

# computes the pressure-velcity field generated by the source-sheet
p_sheet1 = numpy.empty((N, N), dtype=float)
u_sheet1 = numpy.empty((N, N), dtype=float)
v_sheet1 = numpy.empty((N, N), dtype=float)
for i in range(N):
    for j in range(N):
        a=math.cos(alpha)**2+kr*math.sin(alpha)**2
        b=X_local[i,j]*math.cos(alpha)+kr*math.sin(alpha)*Y_local[i,j]
        c=Y_local[i,j]*math.cos(alpha)-X_local[i,j]*math.sin(alpha)
        dp=70.6*Q*miu/h/math.sqrt(kx*ky)
        dv=-0.4468*Q/h/phi*math.sqrt(kx/ky)

        p_sheet1[i,j] = dp/a*(
                             (
                              b*math.log(X_local[i,j]**2-2*b*L1+a*L1**2+kr*Y_local[i,j]**2)
                             -L1*a*math.log((X_local[i,j]-L1*math.cos(alpha))**2+kr*(Y_local[i,j]-L1*math.sin(alpha))**2)
                             +2*math.sqrt(kr)*c*math.atan((b-a*L1)/math.sqrt(kr)/c)
                             )
                             -
                             (
                               b*math.log(X_local[i,j]**2+kr*Y_local[i,j]**2)
                               +2*math.sqrt(kr)*c*math.atan((b)/math.sqrt(kr)/c)
                             )
                            )
        u_sheet1[i,j] = dv/a*(
                             (
                            math.cos(alpha)*math.log(X_local[i,j]**2-2*b*L1+a*L1**2+kr*Y_local[i,j]**2)+ 2*math.sqrt(kr)*math.sin(alpha)*math.atan((a*L1-b)/math.sqrt(kr)/c)
                             )
                            -
                             (
                            math.cos(alpha)*math.log(X_local[i,j]**2+kr*Y_local[i,j]**2)+2*math.sqrt(kr)*math.sin(alpha)*math.atan((-b)/math.sqrt(kr)/c)
                             )
                            )
        v_sheet1[i,j] = dv/a*(
                             (
                            math.sin(alpha)*math.log(X_local[i,j]**2-2*b*L1+a*L1**2+kr*Y_local[i,j]**2)+ 2*math.sqrt(1/kr)*math.cos(alpha)*math.atan((b-a*L1)/math.sqrt(kr)/c)
                             )
                            -
                             (
                            math.sin(alpha)*math.log(X_local[i,j]**2+kr*Y_local[i,j]**2)+2*math.sqrt(1/kr)*math.cos(alpha)*math.atan((b)/math.sqrt(kr)/c)
                             )
                            )

# superposition of the source-sheet to the uniform flow
p1 = p_sheet1+ p_sheet1.min()*-1 #make it looks better
#p1 = p_sheet1 #make it looks better
u1 = u_sheet1
v1 = v_sheet1
```

## Plot the results

```python
# plots the streamlines
%matplotlib inline

size = 8
pyplot.figure(figsize=(size, size-1))
pyplot.title('Streamline field')
pyplot.grid(True)
pyplot.xlabel('x', fontsize=16)
pyplot.ylabel('y', fontsize=16)
pyplot.xlim(x_start, x_end)
pyplot.ylim(y_start, y_end)
pyplot.plot((0+origin_new_x,L1*math.cos(alpha)+origin_new_x),(0+origin_new_y,L1*math.sin(alpha)+origin_new_y), color='#CD2305', linewidth=4) #red source line

#stream =pyplot.streamplot(X, Y, u, v, color=numpy.sqrt(u**2+v**2),cmap='jet',density=2, linewidth=1, arrowsize=1, arrowstyle='->') #streamline
stream =pyplot.streamplot(X, Y, u1, v1, color=numpy.sqrt(u1**2+v1**2),cmap='jet',density=2, linewidth=1, arrowsize=1, arrowstyle='->') #streamline

cbar=pyplot.colorbar(orientation='vertical')
equipotential=pyplot.contour(X, Y, p1, extend='both')
```

```python
size = 8
pyplot.figure(figsize=(size, size-1))
pyplot.title('Velocity field')
pyplot.xlabel('x', fontsize=16)
pyplot.ylabel('y', fontsize=16)
pyplot.xlim(x_start, x_end)
pyplot.ylim(y_start, y_end)

Vtotal= numpy.sqrt(u1**2+v1**2)

pyplot.contour(X, Y, Vtotal, 10, linewidths=0.5, colors='k')
pyplot.contourf(X, Y, Vtotal, 10, cmap='rainbow')
pyplot.colorbar()  # draw colorbar
```


# Manoeuvring parameter identification

Can the method that was introduced in the [Ball drop example](00.01_intro.ipynb#Example) be applied to a more complex system? This will be attempted in this chapter based on the results from the manoeuvring simulation from the [previous chapter](01.01_manoeuvring_simulation.ipynb#Simulate-data).

```python
# %load imports.py
## Local packages:

%matplotlib inline
%load_ext autoreload
%autoreload 2
%config Completer.use_jedi = False  ## (To fix autocomplete)

## External packages:
import pandas as pd
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
pd.set_option("display.max_columns", None)

import numpy as np
import os
import plotly.express as px
import plotly.graph_objects as go

import seaborn as sns

import matplotlib.pyplot as plt
if os.name == 'nt':
    plt.style.use('book.mplstyle')  # Windows

import sympy as sp
from sympy.physics.mechanics import (dynamicsymbols, ReferenceFrame,
                                      Particle, Point)
from sympy.physics.vector.printing import vpprint, vlatex
from IPython.display import display, Math, Latex, Markdown
from src.substitute_dynamic_symbols import run, lambdify

import pyro

import sklearn
import pykalman
from statsmodels.sandbox.regression.predstd import wls_prediction_std
import statsmodels.api as sm

from scipy.integrate import solve_ivp

## Local packages:
from src.data import mdl

from src.symbols import *
from src.parameters import *
import src.symbols as symbols
from src import prime_system
from src.models import regression
from src.visualization.regression import show_pred
from src.visualization.plot import track_plot
from src.visualization.equation import Equation
from myst_nb import glue

## Load models:
# (Uncomment these for faster loading):
import src.models.vmm_nonlinear_EOM  as vmm

## Examples
from example_1 import ship_parameters, df_parameters, ps, ship_parameters_prime
```

## Decoupling
The hydrodynamic derivatives in a mathematical model for ship manoeuvring should be identified by expressing the ODE as a OLS regression. There is a coupling between the sway and yaw equation in this model. These equations need to be decoupled {cite:p}`wang_parameter_2021`, in a similar way as how it was done in the previous chapter. The simulation model can be expressed as:

```python
A = vmm.simulator.A
b = vmm.simulator.b
acceleration = sp.matrices.MutableDenseMatrix([u1d,v1d,r1d])
eq_simulator = sp.Eq(sp.UnevaluatedExpr(A)*sp.UnevaluatedExpr(acceleration),sp.UnevaluatedExpr(b))
Math(vlatex(eq_simulator))
```

$X_{qs}$, $Y_{qs}$, $N_{qs}$ are the quasi static hydrodynamic force models, that contains the hydrodynamic derivatives ($Y_{uv}$ etc.) that should be identified.

The PIT should be expressed as three regressions based on $\dot{u}$, $\dot{v}$ and $\dot{r}$:

```python
coeff_matrix = sp.matrices.MutableDenseMatrix([A_coeff*X_X,B_coeff*X_Y,C_coeff*X_N])
eq_regression = sp.Eq(acceleration, coeff_matrix)
Math(vlatex(eq_regression))
```

Where $A_{coeff}$, $B_{coeff}$ and $C_{coeff}$ are coefficient vectors from where the hydrodynamic derivatives can be obtained with decoupling as follows, where eq.{eq}`eq_coeff` has been inserted into eq.{eq}`eq_simulator`:

```python
Math(vlatex(sp.Eq(sp.UnevaluatedExpr(A)*sp.UnevaluatedExpr(coeff_matrix),sp.UnevaluatedExpr(b))))
```

```{glue:math} sym_eq
:label: eq_simulator_coeff
```

And the right hand side of this equation can also be expressed with vectors:

```python
X_qs_, Y_qs_, N_qs_ = sp.symbols('X_qs, Y_qs, N_qs')
eq_X_qs = sp.Eq(X_qs_*X_X, b[0])
Math(vlatex(eq_X_qs))
```

```python
eq_Y_qs = sp.Eq(X_qs_*X_Y, b[1])
Math(vlatex(eq_Y_qs))
```

```python
eq_N_qs = sp.Eq(N_qs_*X_N, b[2])
Math(vlatex(eq_N_qs))
```

So that the parameter vectors $X_{qs}$, $Y_{qs}$, $N_{qs}$ can be calculated from $A_{coeff}$, $B_{coeff}$ and $C_{coeff}$ in the following way.

```python
Math(vlatex(sp.Eq(sp.UnevaluatedExpr(A)*sp.UnevaluatedExpr(sp.matrices.MutableDenseMatrix([A_coeff,B_coeff,C_coeff])),
                  sp.UnevaluatedExpr(sp.matrices.MutableDenseMatrix([X_qs_,Y_qs_,N_qs_])))))
```

## Regression
The parameter vectors : $A_{coeff}$, $B_{coeff}$ and $C_{coeff}$ are determined with OLS regression:

### Load simulation results:

```python
df_result = pd.read_csv('first_simulation.csv', index_col=0)  # (gnereated by:01.01_manoeuvring_simulation.ipynb)
df = ps.prime(df_result, U=df_result['U'])
```

### N

```python
N_ = sp.symbols('N_')

diff_eq_N = regression.DiffEqToMatrix(ode=vmm.N_qs_eq.subs(N_D,N_),
                                      label=N_, base_features=[delta,u,v,r])
```

```python
X = diff_eq_N.calculate_features(data=df)
y = diff_eq_N.calculate_label(y=df['r1d'])

model_N = sm.OLS(y,X)
results_N = model_N.fit()

show_pred(X=X,y=y,results=results_N, label=r'$\dot{r}$')
```

### Y

```python
Y_ = sp.symbols('Y_')
diff_eq_Y = regression.DiffEqToMatrix(ode=vmm.Y_qs_eq.subs(Y_D,Y_),
                                      label=Y_, base_features=[delta,u,v,r])
```

```python
X = diff_eq_Y.calculate_features(data=df)
y = diff_eq_Y.calculate_label(y=df['v1d'])

model_Y = sm.OLS(y,X)
results_Y = model_Y.fit()

show_pred(X=X,y=y,results=results_Y, label=r'$\dot{v}$')
```

### X

```python
X_ = sp.symbols('X_')
diff_eq_X = regression.DiffEqToMatrix(ode=vmm.X_qs_eq.subs(X_D,X_),
                                      label=X_, base_features=[delta,u,v,r,thrust])
```

```python
X = diff_eq_X.calculate_features(data=df)
y = diff_eq_X.calculate_label(y=df['u1d'])

model_X = sm.OLS(y,X)
results_X = model_X.fit()

show_pred(X=X,y=y,results=results_X, label=r'$\dot{u}}$')
```

### Decoupling

```python
subs = {value:key for key,value in p.items()}
A_ = A*sp.matrices.MutableDenseMatrix([A_coeff,B_coeff,C_coeff])
A_lambda=lambdify(A_.subs(subs))
```

```python
results_summary_X = regression.results_summary_to_dataframe(results_X)
results_summary_Y = regression.results_summary_to_dataframe(results_Y)
results_summary_N = regression.results_summary_to_dataframe(results_N)

A_coeff_ = results_summary_X['coeff']
B_coeff_ = results_summary_Y['coeff']
C_coeff_ = results_summary_N['coeff']

coeffs = run(A_lambda,A_coeff=A_coeff_.values, B_coeff=B_coeff_.values, C_coeff=C_coeff_.values,
    **df_parameters['prime'], **ship_parameters_prime)
```

The way that the regression is formulated, inertial forces, such as centrifugal force will be included into the derivatives (I think) which means that centrifugal force : $-m \cdot r \cdot u$ will be included into $Y_{ur}$ coefficient. This coefficient is therefore not pure hydrodynamic, and can potentially be counted twice..?
The coefficients are recalculated below to avooid this:

```python
results_summary_X['decoupled'] = coeffs[0][0]
results_summary_Y['decoupled'] = coeffs[1][0]
results_summary_N['decoupled'] = coeffs[2][0]

x_G_ = ship_parameters_prime['x_G']
m_ = ship_parameters_prime['m']

results_summary_X.loc['Xrr','decoupled']+=(-m_*x_G_)
results_summary_X.loc['Xvr','decoupled']+=(-m_)
results_summary_Y.loc['Yur','decoupled']+=m_
results_summary_N.loc['Nur','decoupled']+=m_*x_G_
```

### Add the regressed parameters
Hydrodynamic derivatives that depend on acceleration cannot be obtained from the VCT regression. They are however essential if a time simulation should be conducted. These values have then been taken from Brix semi empirical formulas for the simulations below.

```python
df_parameters_all = df_parameters.copy()
for other in [results_summary_X, results_summary_Y, results_summary_N]:
    df_parameters_all = df_parameters_all.combine_first(other)

df_parameters_all.rename(columns={'decoupled':'regressed'}, inplace=True)
df_parameters_all.drop(columns=['brix_lambda'], inplace=True)

df_parameters_all['regressed'] = df_parameters_all['regressed'].combine_first(df_parameters_all['prime'])  # prefer regressed
#df_parameters_all['regressed'].fillna(0,inplace=True)
```

# Simulation

A simulation with the regressed hydrodynamic coefficients can now be conducted. The figures below compare results from these simulation and the original data.

```python
parameters=df_parameters_all['regressed'].copy()

result_regression = vmm.simulator.simulate(df_=df_result,
                                           parameters=parameters,
                                           ship_parameters=ship_parameters,
                                           control_keys=['delta'],
                                           primed_parameters=True,
                                           prime_system=ps,
                                           name='regressed')

result_regression.df_model_test=df_result  # dirty
```

```python
result_regression.track_plot(compare=True);
```

```python
result_regression.plot(compare=True);
```

And the regressed hydrodynamic derivatives can be compared with the original parameters, refered to as "prime" in the bar chart below:

```python
df_plot_parameters = df_parameters_all[['regressed','prime']].copy()
mask = ((df_plot_parameters['regressed'].abs() > 0) &
        (df_plot_parameters['regressed'].abs() < 0.99)
       )

df_plot_parameters.loc[mask].plot(kind='bar')

mask = (
        (df_plot_parameters['regressed'].abs() > 0.99)
       )
if mask.any():
    df_plot_parameters.loc[mask].plot(kind='bar')
```

## Measurement noise
The regression seems to be able to identify the parameters in the model if it is given perfect data, but what will happen if the data has some measurement or process noise? A new regression with gausian noise added to the data will be conducted to investigate this.

```python
df_measure = df_result.copy()

scales = df_measure.max()-df_measure.min()
noise = np.random.normal(scale=0.04*scales, size=df_measure.shape)
df_noise = pd.DataFrame(data=noise, columns=df_measure.columns, index=df_measure.index)
df_measure+=df_noise

df = ps.prime(df_measure, U=df_result['U'])
```

```python
for key in df_measure:

    fig,ax=plt.subplots()
    fig.set_size_inches(10,1)
    df_measure.plot(y=key, ax=ax)

```

### N

```python
X = diff_eq_N.calculate_features(data=df)
y = diff_eq_N.calculate_label(y=df['r1d'])

model_N = sm.OLS(y,X)
results_N = model_N.fit()

show_pred(X=X,y=y,results=results_N, label=r'$\dot{r}$')
```

## Y

```python
X = diff_eq_Y.calculate_features(data=df)
y = diff_eq_Y.calculate_label(y=df['v1d'])

model_Y = sm.OLS(y,X)
results_Y = model_Y.fit()

show_pred(X=X,y=y,results=results_Y, label=r'$\dot{v}$')
```

### X

```python
X = diff_eq_X.calculate_features(data=df)
y = diff_eq_X.calculate_label(y=df['u1d'])

model_X = sm.OLS(y,X)
results_X = model_X.fit()

show_pred(X=X,y=y,results=results_X, label=r'$\dot{u}}$')
```

```python
results_summary_X = regression.results_summary_to_dataframe(results_X)
results_summary_Y = regression.results_summary_to_dataframe(results_Y)
results_summary_N = regression.results_summary_to_dataframe(results_N)

A_coeff_ = results_summary_X['coeff']
B_coeff_ = results_summary_Y['coeff']
C_coeff_ = results_summary_N['coeff']

coeffs = run(A_lambda,A_coeff=A_coeff_.values, B_coeff=B_coeff_.values, C_coeff=C_coeff_.values,
    **df_parameters['prime'], **ship_parameters_prime)
```

```python
results_summary_X['decoupled'] = coeffs[0][0]
results_summary_Y['decoupled'] = coeffs[1][0]
results_summary_N['decoupled'] = coeffs[2][0]

x_G_ = ship_parameters_prime['x_G']
m_ = ship_parameters_prime['m']

results_summary_X.loc['Xrr','decoupled']+=(-m_*x_G_)
results_summary_X.loc['Xvr','decoupled']+=(-m_)
results_summary_Y.loc['Yur','decoupled']+=m_
results_summary_N.loc['Nur','decoupled']+=m_*x_G_
```

```python
df_parameters_all = df_parameters.copy()
for other in [results_summary_X, results_summary_Y, results_summary_N]:
    df_parameters_all = df_parameters_all.combine_first(other)

df_parameters_all.rename(columns={'decoupled':'regressed'}, inplace=True)
df_parameters_all.drop(columns=['brix_lambda'], inplace=True)

df_parameters_all['regressed'] = df_parameters_all['regressed'].combine_first(df_parameters_all['prime'])  # prefer regressed
#df_parameters_all['regressed'].fillna(0,inplace=True)
```

# Simulation

A simulation with the regressed hydrodynamic coefficients can now be conducted. The figures below compare results from these simulation and the original data.

```python
parameters=df_parameters_all['regressed'].copy()

result_regression = vmm.simulator.simulate(df_=df_result,
                                           parameters=parameters,
                                           ship_parameters=ship_parameters,
                                           control_keys=['delta'],
                                           primed_parameters=True,
                                           prime_system=ps,
                                           name='regressed')

result_regression.df_model_test=df_result  # dirty
```

```python
result_regression.track_plot(compare=True);
```

```python
result_regression.plot(compare=True);
```

And the regressed hydrodynamic derivatives can be compared with the original parameters, refered to as "prime" in the bar chart below:

```python
df_plot_parameters = df_parameters_all[['regressed','prime']].copy()
mask = ((df_plot_parameters['regressed'].abs() > 0) &
        (df_plot_parameters['regressed'].abs() < 0.99)
       )

df_plot_parameters.loc[mask].plot(kind='bar')

mask = (
        (df_plot_parameters['regressed'].abs() > 0.99)
       )
if mask.any():
    df_plot_parameters.loc[mask].plot(kind='bar')
```

## Summary
It seams that the hydrodynamic coefficients in a nonlinear manoeuvring simulation model can be identified by regressing the reversed simulation model. The example in this chapter used exact positions, velocities and accelerations, without any measurment error. This is ofcourse an idealized scenario, when using real data from model test or full scale ship, measurement noise will always be present. A gausian noise was added to see if this would cause any problems in the parameter identification. It was found that the regression still identifies a model that recreates the time series pretty accuratelly. The identified parameters are however very different when compared to the original "real" parameters. This is another important finding from this example: eventhough the model recreates the time series well, that does not necessarily mean that the identified parameters are correct.

This is referred to as *parameter drift*. The identified coefficients might be not physically correct but be mathematically correct [2], which implies that even if the identified parameters deviate from their true values, the manoeuvring model can predict the manoeuvring motion well, compared with the test results. The drift of nonlinear hydrodynamic coefficients is due to the so-called multicollinearity {cite:p}`luo_parameter_2016-1`.

```python

```

```python

```


# Sistemas de EDOs

Considere o sistema abaixo:

\begin{equation}
\begin{aligned}
\dot{x} &= f(t,x(t),y(t)) \\
\dot{y} &= g(t,x(t),y(t))
\end{aligned}
\end{equation}

com $x(t=0)=x_0$ e $y(t=0)=y_0$.

Como podemos resolver esse sistema de EDOs numéricamente? A seguir, vamos mostrar alguns exemplos.

```python
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-poster')
```

## Exemplo 1
Resolva o seguinte sistema de equações diferenciais ordinárias com valores iniciais:
\begin{equation}
\begin{aligned}
\dot{x} &= -y(t) \\
\dot{y} &= x(t)
\end{aligned}
\end{equation}
com $x(t=0)=1$ e $y(t=0)=0$.
Use o método de Euler para achar a sua solução aproximada.

```python
f = lambda t,x,y: -y
g = lambda t,x,y: x
```

```python
# Parametros

h = 1e-4
tfim = 10
nt = int(tfim/h)
x0 = 1
y0 = 0
s0 = [x0,y0]
```

```python
def Euler2D(f,g,h,s0,tmax=10):
  nt = int(tmax/h)
  x = np.zeros(nt)
  y = np.zeros(nt)
  tempo = np.linspace(0,tmax,nt)
  x[0] = s0[0]
  y[0] = s0[1]

  for k in np.arange(1,nt):
      x[k] = x[k-1] + h*f(tempo[k-1],x[k-1],y[k-1])
      y[k] = y[k-1] + h*g(tempo[k-1],x[k-1],y[k-1])

  return tempo,x,y
```

```python
tempo, x_num, y_num = Euler2D(f,g,h,s0,tfim)
```

```python
  plt.figure(figsize = (12, 8))
  plt.plot(tempo, y_num, 'bo--', label='Numérica com h={}'.format(h))
  plt.title('Solução Aproximada de um Sistema de EDO')
  plt.xlabel('x')
  plt.ylabel('y')
  plt.grid()
  plt.legend(loc='lower left')
  plt.show()
```

```python
  plt.figure(figsize = (12, 8))
  plt.plot(tempo, x_num, 'bo--', label='x')
  plt.plot(tempo, y_num, 'go--', label='y')
  plt.title('Solução Aproximada de um Sistema de EDO')
  plt.xlabel('t')
  plt.ylabel('y')
  plt.grid()
  plt.legend(loc='lower left')
  plt.show()
```

Para generalizar, vamos criar uma função *derivadas* para colocar as funções na forma vetorial:

```python
def derivadas(t,X):
  F = np.zeros(len(X))
  F[0] = -X[1] # xdot = -y
  F[1] = X[0]  # ydot = x

  return F

```

```python
def Euler_geral(derivadas,h,s0,tmax=10):
  nt = int(tmax/h)
  X = np.zeros([nt,len(s0)])
  tempo = np.linspace(0,tmax,nt)
  X[0,:] = s0

  for k in np.arange(1,nt):
      F = derivadas(tempo[k-1],X[k-1,:])
      X[k,:] = X[k-1,:] + h*F[:]

  return tempo,X
```

```python
tempo,X = Euler_geral(derivadas,h,[1,0],tmax=1)
```

```python
  plt.figure(figsize = (12, 8))
  plt.plot(X[:,0],X[:,1], 'bo--', label='Numérica com h={}'.format(h))
  plt.title('Solução Aproximada de um Sistema de EDO')
  plt.xlabel('x')
  plt.ylabel('y')
  plt.grid()
  plt.legend(loc='lower left')
  plt.show()
```

## Exemplo 2

Resolva o seguinte sistema de equações diferenciais ordinárias com valores iniciais:
\begin{equation}
\begin{aligned}
\dot{x} &= -y(t) \\
\dot{y} &= x(t)
\end{aligned}
\end{equation}
com $x(t=0)=1$ e $y(t=0)=0$.
Use o método de Euler **modificado** para achar a sua solução aproximada.

```python
def Euler_mod_geral(derivadas,h,s0,tmax=10):
  nt = int(tmax/h)
  X = np.zeros([nt,2])
  tempo = np.linspace(0,tmax,nt)
  X[0,:] = s0

  for k in np.arange(1,nt):
      F = derivadas(tempo[k-1],X[k-1,:])
      kx1 = F[0]
      ky1 = F[1]

      x_euler = X[k-1,0] + h*kx1
      y_euler = X[k-1,1] + h*ky1

      Fnew = derivadas(tempo[k],[x_euler,y_euler])

      kx2 = Fnew[0]
      ky2 = Fnew[1]

      X[k,0] = X[k-1,0] + h*(kx1+kx2)/2
      X[k,1] = X[k-1,1] + h*(ky1+ky2)/2
  return tempo,X

```

```python
# Parametros

h = .1
tfim = 10
nt = int(tfim/h)
x0 = 1
y0 = 0
s0 = [x0,y0]

# Rodar o integrador
tempo, X  = Euler_mod_geral(derivadas,h,s0,tmax=10)
```

```python
  plt.figure(figsize = (12, 8))
  plt.plot(X[:,0],X[:,1], 'bo--', label='Numérica com h={}'.format(h))
  plt.title('Solução Aproximada de um Sistema de EDO')
  plt.xlabel('x')
  plt.ylabel('y')
  plt.grid()
  plt.legend(loc='lower left')
  plt.show()
```

## Tarefa
Resolva o seguinte sistema de equações diferenciais ordinárias com valores iniciais:
\begin{equation}
\begin{aligned}
\dot{x} &= -x(t)-y(t) \\
\dot{y} &= x(t)-y(t)^3
\end{aligned}
\end{equation}

com $x(t=0)=1$ e $y(t=0)=0$.
Use o método de Euler **modificado** para achar a sua solução aproximada.

```python
def derivadas(t,X):
  F = np.zeros(len(X))
  F[0] = -X[0]-X[1]
  F[1] = X[0]-X[1]**3
  return F
```

```python
# Parametros

h = 1e-2
tfim = 1
nt = int(tfim)/h
x0 = 1
y0 = 0
s0 = [x0,y0]
X=[]
# Rodar o integrador
tempo, X  = Euler_mod_geral(derivadas,h,s0,tmax=10)
```

```python
  plt.figure(figsize = (12, 8))
  plt.plot(X[:,0],X[:,1], 'bo--', label='Numérica com h={}'.format(h))
  plt.title('Solução Aproximada de um Sistema de EDO')
  plt.xlabel('x')
  plt.ylabel('y')
  plt.grid()
  plt.legend(loc='lower left')
  plt.show()
```

# Modelo SIR

O Sistema de EDOs a seguir modela epidemeologicamente a propagação de uma doença (como a covid19). Considere $S(t)$ o número de indivíduos *susceptíveis* à infecção. $I(t)$ corresponde ao número de pessoas infectadas e $R(t)$ o número de indivíduos recuperados.

\begin{equation}
\begin{aligned}
S^{\prime } &= -\beta S I \\
I^{\prime } &= \beta S I - \gamma I \\
R^{\prime } &= \gamma I
\end{aligned}
\end{equation}

onde o parâmetros $\beta$ correspode à taxa de contaminação e $\gamma$ à probabilidade de um indivíduo se recuperar em um intervalo de tempo.

```python
def derivadas_SIR(t,X,params=[0,0]):
  F = np.zeros(len(X))
  beta = params[0]
  gamma = params[1]
  F[0] = -beta*X[0]*X[1]
  F[1] = beta*X[0]*X[1]-gamma*X[1]
  F[2] = gamma*X[1]
  return F

```

```python
def Euler_geral_D(derivadas,params,h,s0,tmax=10):
  nt = int(tmax/h)
  X = np.zeros([nt,len(s0)])
  tempo = np.linspace(0,tmax,nt)
  X[0,:] = s0

  for k in np.arange(1,nt):
      F = derivadas(tempo[k],X[k-1,:],params)
      X[k,:] = X[k-1,:] + h*F[:]

  return tempo,X
```

```python
beta = 10/(40*8*24)
gamma = 3/(15*24)
params = [beta,gamma]
s0 = [50,1,0]
h=1e-2
tmax = 800

tempo,X = Euler_geral_D(derivadas_SIR,params,h,s0,tmax)

```

```python

plt.figure(figsize = (12, 8))
plt.plot(tempo,X[:,0], 'bo--', label='S')
plt.plot(tempo,X[:,1], 'ro--', label='I')
plt.plot(tempo,X[:,2], 'go--', label='R')
plt.title('Modelo SIR')
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.legend(loc='lower left')
plt.show()
```

```python

```