## Clustering

In order to extract useful insights about different types of potential groups of blocks in Yucatan with similar characteristics we performed a clustering analysis. We used three different types of clustering algorithms, namely, K Means, Gaussian Mixture Models, and DBSCAN. Our goal is to identify similarities and try to assess the living conditions particularly in blocks with high density of disabled people, compared to those with low density of disabled people.

First, we import some useful libraries.

```python
import matplotlib

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sb
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances_argmin_min

plt.rcParams['figure.figsize'] = (16, 9)
plt.style.use('ggplot')
```

Then, we load our previously cleaned dataset

```python
input_file = './drive/Shareddrives/CS5056.500 Data Analytics/datos INEGI/datos_clean.csv'
df = pd.read_csv(input_file)
df = df[df['POBTOT'] > 0]
df
```

      index
      MUN
      NOM_MUN
      LOC
      NOM_LOC
      AGEB
      MZA
      POBTOT
      PNACENT
      PHOG_IND
      PCON_DISC
      PSIND_LIM
      P18YM_PB
      GRAPROES
      PEA
      PE_INAC
      PSINDER
      PDER_SS
      P12YM_SOLT
      P12YM_CASA
      PCATOLICA
      POTRAS_REL
      TOTHOG
      POBHOG
      VIVTOT
      TVIVHAB
      OCUPVIVPAR
      VPH_2YMASD
      VPH_3YMASC
      VPH_C_ELEC
      VPH_AGUADV
      VPH_AEASP
      VPH_TINACO
      VPH_EXCSA
      VPH_LETR
      VPH_C_SERV
      VPH_REFRI
      VPH_TV
      VPH_CEL

      0
      4
      1
      Abalá
      7
      Uayalceh
      18
      1
      62.0
      62.000000
      59.000000
      4.000000
      51.000000
      4.000000
      5.720000
      31.000000
      16.000000
      12.000000
      50.000000
      10.0000
      32.000000
      34.000000
      0.000000
      21.000000
      62.000000
      23.0
      21.0
      62.000000
      6.000000
      14.667958
      21.000000
      21.000000
      21.000000
      8.000000
      11.000000
      0.000000
      10.000000
      17.00000
      21.000000
      15.0000

      1
      5
      1
      Abalá
      7
      Uayalceh
      18
      2
      59.0
      59.000000
      59.000000
      0.000000
      46.000000
      24.407191
      3.930000
      30.000000
      18.000000
      21.000000
      38.000000
      17.0000
      28.000000
      23.000000
      0.000000
      19.000000
      59.000000
      21.0
      19.0
      59.000000
      11.000000
      9.000000
      19.000000
      19.000000
      19.000000
      6.000000
      8.000000
      0.000000
      7.000000
      9.00000
      16.000000
      9.0000

      2
      6
      1
      Abalá
      7
      Uayalceh
      18
      3
      71.0
      71.000000
      71.000000
      0.000000
      57.000000
      24.407191
      5.940000
      34.000000
      20.000000
      27.000000
      44.000000
      17.0000
      34.000000
      54.000000
      0.000000
      18.000000
      71.000000
      23.0
      18.0
      71.000000
      10.000000
      9.000000
      18.000000
      18.000000
      18.000000
      11.000000
      13.000000
      0.000000
      12.000000
      12.00000
      17.000000
      15.0000

      3
      7
      1
      Abalá
      7
      Uayalceh
      18
      6
      13.0
      13.000000
      13.000000
      4.740691
      9.000000
      0.000000
      5.000000
      8.000000
      3.000000
      16.155265
      11.000000
      18.6207
      6.000000
      0.000000
      0.000000
      5.000000
      13.000000
      6.0
      5.0
      13.000000
      12.771432
      4.000000
      5.000000
      5.000000
      5.000000
      16.062434
      3.000000
      0.000000
      3.000000
      3.00000
      5.000000
      3.0000

      4
      8
      1
      Abalá
      7
      Uayalceh
      18
      7
      86.0
      86.000000
      86.000000
      4.740691
      74.000000
      5.000000
      6.020000
      37.000000
      29.000000
      13.000000
      73.000000
      15.0000
      42.000000
      15.000000
      0.000000
      18.000000
      86.000000
      19.0
      18.0
      86.000000
      10.000000
      11.000000
      18.000000
      16.000000
      16.000000
      4.000000
      7.000000
      0.000000
      7.000000
      13.00000
      17.000000
      12.0000

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      33844
      35493
      105
      Yaxkukul
      1
      Yaxkukul
      46
      38
      41.0
      40.000000
      4.000000
      4.740691
      38.000000
      4.000000
      8.260000
      15.000000
      9.000000
      4.000000
      37.000000
      4.0000
      19.000000
      25.000000
      0.000000
      12.000000
      41.000000
      13.0
      12.0
      41.000000
      5.000000
      4.000000
      10.000000
      10.000000
      8.000000
      5.000000
      8.000000
      3.000000
      6.000000
      7.00000
      10.000000
      9.0000

      33845
      35494
      105
      Yaxkukul
      1
      Yaxkukul
      46
      41
      9.0
      57.125439
      25.357365
      4.740691
      54.213825
      24.407191
      9.880247
      33.608914
      20.773406
      16.155265
      51.162475
      18.6207
      29.860776
      49.777662
      0.052387
      18.903129
      65.973919
      2.0
      2.0
      65.973919
      12.771432
      14.667958
      18.813104
      18.751216
      18.224804
      16.062434
      18.110843
      0.042897
      17.965593
      17.09095
      17.780906
      17.4779

      33846
      35495
      105
      Yaxkukul
      1
      Yaxkukul
      46
      42
      37.0
      32.000000
      7.000000
      4.740691
      36.000000
      11.000000
      9.680000
      20.000000
      8.000000
      5.000000
      32.000000
      11.0000
      17.000000
      25.000000
      0.000000
      11.000000
      37.000000
      14.0
      11.0
      37.000000
      8.000000
      8.000000
      11.000000
      11.000000
      11.000000
      11.000000
      11.000000
      0.000000
      11.000000
      10.00000
      11.000000
      11.0000

      33847
      35496
      105
      Yaxkukul
      1
      Yaxkukul
      46
      43
      31.0
      30.000000
      0.000000
      0.000000
      31.000000
      10.000000
      10.550000
      14.000000
      13.000000
      15.000000
      16.000000
      12.0000
      14.000000
      23.000000
      0.000000
      9.000000
      31.000000
      9.0
      9.0
      31.000000
      6.000000
      7.000000
      9.000000
      9.000000
      9.000000
      9.000000
      9.000000
      0.000000
      9.000000
      9.00000
      7.000000
      9.0000

      33848
      35498
      105
      Yaxkukul
      1
      Yaxkukul
      65
      1
      54.0
      54.000000
      4.000000
      4.740691
      46.000000
      7.000000
      8.020000
      25.000000
      19.000000
      11.000000
      43.000000
      14.0000
      26.000000
      23.000000
      0.000000
      14.000000
      54.000000
      16.0
      14.0
      54.000000
      7.000000
      6.000000
      14.000000
      14.000000
      13.000000
      4.000000
      8.000000
      0.000000
      8.000000
      10.00000
      13.000000
      12.0000

32642 rows × 39 columns

```python
df = df[df['POBTOT'] >= df['PCON_DISC']]
df
```

      index
      MUN
      NOM_MUN
      LOC
      NOM_LOC
      AGEB
      MZA
      POBTOT
      PNACENT
      PHOG_IND
      PCON_DISC
      PSIND_LIM
      P18YM_PB
      GRAPROES
      PEA
      PE_INAC
      PSINDER
      PDER_SS
      P12YM_SOLT
      P12YM_CASA
      PCATOLICA
      POTRAS_REL
      TOTHOG
      POBHOG
      VIVTOT
      TVIVHAB
      OCUPVIVPAR
      VPH_2YMASD
      VPH_3YMASC
      VPH_C_ELEC
      VPH_AGUADV
      VPH_AEASP
      VPH_TINACO
      VPH_EXCSA
      VPH_LETR
      VPH_C_SERV
      VPH_REFRI
      VPH_TV
      VPH_CEL

      0
      4
      1
      Abalá
      7
      Uayalceh
      18
      1
      62.0
      62.000000
      59.000000
      4.000000
      51.000000
      4.000000
      5.720000
      31.000000
      16.000000
      12.000000
      50.000000
      10.0000
      32.000000
      34.000000
      0.000000
      21.000000
      62.000000
      23.0
      21.0
      62.000000
      6.000000
      14.667958
      21.000000
      21.000000
      21.000000
      8.000000
      11.000000
      0.000000
      10.000000
      17.00000
      21.000000
      15.0000

      1
      5
      1
      Abalá
      7
      Uayalceh
      18
      2
      59.0
      59.000000
      59.000000
      0.000000
      46.000000
      24.407191
      3.930000
      30.000000
      18.000000
      21.000000
      38.000000
      17.0000
      28.000000
      23.000000
      0.000000
      19.000000
      59.000000
      21.0
      19.0
      59.000000
      11.000000
      9.000000
      19.000000
      19.000000
      19.000000
      6.000000
      8.000000
      0.000000
      7.000000
      9.00000
      16.000000
      9.0000

      2
      6
      1
      Abalá
      7
      Uayalceh
      18
      3
      71.0
      71.000000
      71.000000
      0.000000
      57.000000
      24.407191
      5.940000
      34.000000
      20.000000
      27.000000
      44.000000
      17.0000
      34.000000
      54.000000
      0.000000
      18.000000
      71.000000
      23.0
      18.0
      71.000000
      10.000000
      9.000000
      18.000000
      18.000000
      18.000000
      11.000000
      13.000000
      0.000000
      12.000000
      12.00000
      17.000000
      15.0000

      3
      7
      1
      Abalá
      7
      Uayalceh
      18
      6
      13.0
      13.000000
      13.000000
      4.740691
      9.000000
      0.000000
      5.000000
      8.000000
      3.000000
      16.155265
      11.000000
      18.6207
      6.000000
      0.000000
      0.000000
      5.000000
      13.000000
      6.0
      5.0
      13.000000
      12.771432
      4.000000
      5.000000
      5.000000
      5.000000
      16.062434
      3.000000
      0.000000
      3.000000
      3.00000
      5.000000
      3.0000

      4
      8
      1
      Abalá
      7
      Uayalceh
      18
      7
      86.0
      86.000000
      86.000000
      4.740691
      74.000000
      5.000000
      6.020000
      37.000000
      29.000000
      13.000000
      73.000000
      15.0000
      42.000000
      15.000000
      0.000000
      18.000000
      86.000000
      19.0
      18.0
      86.000000
      10.000000
      11.000000
      18.000000
      16.000000
      16.000000
      4.000000
      7.000000
      0.000000
      7.000000
      13.00000
      17.000000
      12.0000

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      33844
      35493
      105
      Yaxkukul
      1
      Yaxkukul
      46
      38
      41.0
      40.000000
      4.000000
      4.740691
      38.000000
      4.000000
      8.260000
      15.000000
      9.000000
      4.000000
      37.000000
      4.0000
      19.000000
      25.000000
      0.000000
      12.000000
      41.000000
      13.0
      12.0
      41.000000
      5.000000
      4.000000
      10.000000
      10.000000
      8.000000
      5.000000
      8.000000
      3.000000
      6.000000
      7.00000
      10.000000
      9.0000

      33845
      35494
      105
      Yaxkukul
      1
      Yaxkukul
      46
      41
      9.0
      57.125439
      25.357365
      4.740691
      54.213825
      24.407191
      9.880247
      33.608914
      20.773406
      16.155265
      51.162475
      18.6207
      29.860776
      49.777662
      0.052387
      18.903129
      65.973919
      2.0
      2.0
      65.973919
      12.771432
      14.667958
      18.813104
      18.751216
      18.224804
      16.062434
      18.110843
      0.042897
      17.965593
      17.09095
      17.780906
      17.4779

      33846
      35495
      105
      Yaxkukul
      1
      Yaxkukul
      46
      42
      37.0
      32.000000
      7.000000
      4.740691
      36.000000
      11.000000
      9.680000
      20.000000
      8.000000
      5.000000
      32.000000
      11.0000
      17.000000
      25.000000
      0.000000
      11.000000
      37.000000
      14.0
      11.0
      37.000000
      8.000000
      8.000000
      11.000000
      11.000000
      11.000000
      11.000000
      11.000000
      0.000000
      11.000000
      10.00000
      11.000000
      11.0000

      33847
      35496
      105
      Yaxkukul
      1
      Yaxkukul
      46
      43
      31.0
      30.000000
      0.000000
      0.000000
      31.000000
      10.000000
      10.550000
      14.000000
      13.000000
      15.000000
      16.000000
      12.0000
      14.000000
      23.000000
      0.000000
      9.000000
      31.000000
      9.0
      9.0
      31.000000
      6.000000
      7.000000
      9.000000
      9.000000
      9.000000
      9.000000
      9.000000
      0.000000
      9.000000
      9.00000
      7.000000
      9.0000

      33848
      35498
      105
      Yaxkukul
      1
      Yaxkukul
      65
      1
      54.0
      54.000000
      4.000000
      4.740691
      46.000000
      7.000000
      8.020000
      25.000000
      19.000000
      11.000000
      43.000000
      14.0000
      26.000000
      23.000000
      0.000000
      14.000000
      54.000000
      16.0
      14.0
      54.000000
      7.000000
      6.000000
      14.000000
      14.000000
      13.000000
      4.000000
      8.000000
      0.000000
      8.000000
      10.00000
      13.000000
      12.0000

31381 rows × 39 columns

To calculate the density of disabled people, we divide the number of people with disability per block (`PCON_DISC`) by the total population of that block (`POBTOT`).

```python
density = df[['POBTOT', 'PCON_DISC']]
density = density[density['POBTOT'] > 0]
density['DISC_DENS'] = density['PCON_DISC'] / density['POBTOT']
density.describe()
```

      POBTOT
      PCON_DISC
      DISC_DENS

      count
      31381.000000
      31381.000000
      31381.000000

      mean
      63.460342
      4.740842
      0.116302

      std
      52.906426
      3.912955
      0.158340

      min
      3.000000
      0.000000
      0.000000

      25%
      29.000000
      3.000000
      0.039506

      50%
      54.000000
      4.740691
      0.074766

      75%
      84.000000
      5.000000
      0.126437

      max
      1679.000000
      89.000000
      0.948138

```python
df.describe()
```

      index
      MUN
      LOC
      MZA
      POBTOT
      PNACENT
      PHOG_IND
      PCON_DISC
      PSIND_LIM
      P18YM_PB
      GRAPROES
      PEA
      PE_INAC
      PSINDER
      PDER_SS
      P12YM_SOLT
      P12YM_CASA
      PCATOLICA
      POTRAS_REL
      TOTHOG
      POBHOG
      VIVTOT
      TVIVHAB
      OCUPVIVPAR
      VPH_2YMASD
      VPH_3YMASC
      VPH_C_ELEC
      VPH_AGUADV
      VPH_AEASP
      VPH_TINACO
      VPH_EXCSA
      VPH_LETR
      VPH_C_SERV
      VPH_REFRI
      VPH_TV
      VPH_CEL

      count
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000
      31381.000000

      mean
      17633.124566
      54.513973
      5.643224
      20.982792
      63.460342
      57.137608
      25.362118
      4.740842
      54.220640
      24.410666
      9.879979
      33.612880
      20.775296
      16.156810
      51.171715
      18.624165
      29.864582
      49.786574
      0.052402
      18.907786
      65.991885
      22.756190
      18.141806
      65.991885
      12.772653
      14.669732
      18.817230
      18.755327
      18.228391
      16.064195
      18.113828
      0.042909
      17.968550
      17.093291
      17.784298
      17.480302

      std
      10135.754267
      22.369419
      29.859006
      36.347191
      52.906426
      44.863486
      30.263367
      3.912955
      43.799827
      23.474408
      2.617485
      27.134894
      16.400174
      14.359371
      40.518386
      14.715481
      23.379683
      40.386475
      0.473738
      14.566539
      51.449347
      18.271922
      14.998739
      51.449347
      10.201466
      12.261515
      14.486812
      14.443322
      14.191626
      12.894209
      14.136406
      0.491981
      14.058819
      13.553961
      13.725708
      13.721925

      min
      4.000000
      1.000000
      1.000000
      1.000000
      3.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      3.000000
      3.000000
      0.000000
      0.000000
      3.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000
      0.000000

      25%
      9086.000000
      50.000000
      1.000000
      8.000000
      29.000000
      28.000000
      6.000000
      3.000000
      27.000000
      10.000000
      7.970000
      16.000000
      10.000000
      7.000000
      26.000000
      9.000000
      15.000000
      24.000000
      0.000000
      10.000000
      33.000000
      11.000000
      8.000000
      33.000000
      7.000000
      7.000000
      10.000000
      10.000000
      9.000000
      8.000000
      9.000000
      0.000000
      9.000000
      9.000000
      9.000000
      9.000000

      50%
      17398.000000
      50.000000
      1.000000
      17.000000
      54.000000
      50.000000
      17.000000
      4.740691
      48.000000
      21.000000
      9.700000
      30.000000
      19.000000
      15.000000
      45.000000
      17.000000
      27.000000
      44.000000
      0.000000
      17.000000
      59.000000
      20.000000
      16.000000
      59.000000
      12.000000
      14.000000
      17.000000
      17.000000
      17.000000
      15.000000
      16.000000
      0.000000
      16.000000
      16.000000
      16.000000
      16.000000

      75%
      26205.000000
      59.000000
      1.000000
      28.000000
      84.000000
      73.000000
      31.000000
      5.000000
      69.000000
      31.000000
      11.560000
      43.000000
      26.000000
      20.000000
      65.000000
      23.000000
      38.000000
      63.000000
      0.000000
      24.000000
      84.000000
      30.000000
      24.000000
      84.000000
      15.000000
      18.000000
      24.000000
      24.000000
      23.000000
      20.000000
      23.000000
      0.000000
      23.000000
      22.000000
      23.000000
      22.000000

      max
      35498.000000
      105.000000
      509.000000
      800.000000
      1679.000000
      1373.000000
      536.000000
      89.000000
      1515.000000
      646.000000
      19.440000
      885.000000
      435.000000
      434.000000
      1236.000000
      413.000000
      765.000000
      1123.000000
      13.000000
      508.000000
      1679.000000
      592.000000
      508.000000
      1679.000000
      380.000000
      447.000000
      508.000000
      506.000000
      503.000000
      477.000000
      504.000000
      30.000000
      501.000000
      468.000000
      470.000000
      486.000000

## Scaling data

In general, data scaling in fundamental when dealing with distance-based clustering algorithms. Since most of the existing distance functions are scale sensible, data scaling is introduced in order to make sure that calculations will not be biased either to the very high or to the very low values. In other words, attribute scaling is used to make sure that all the data is at the same level.

In our case, we proceed to apply Min-max scaling, which compacts our data into a 0 to 1 range of values. The formula for Min-Max scaling is:

\begin{equation}
    x_{scaled} = \frac{x - x_{min}}{x_{max} - x_{min}}
\end{equation}

Since the first 7 attributes in our dataset are either categorical or just identifiers, we will exclude them for future experiments.

```python
from sklearn import preprocessing

x = df.iloc[:, 7:].values
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)
df_scaled = pd.DataFrame(x_scaled)
```

## PCA

Without the first 7 attributes, our dataset is made up of 32 attributes. Realistically, although that number of variables is manageable, less dimensions are always easier to manage, and are in fact desirable.

To reduce the dimensionality of our dataset, we performed Principal Component Analysis (PCA). PCA is a dimensionality reduction method that allows to simplify the complexity of multi-dimensional data while conserving its information. Therefore, PCA allows to condensate the information provided by multiple variables into just a few components.

Furthermore, by reducing the dimensions of the dataset using PCA, we are able to better visualize our information, as well as our output clusters. For this reason, we chose to keep the first two components of our data, since visualizing it in 2D would be far more intuitive.

```python
from sklearn.decomposition import PCA

pca = PCA(n_components = 2)
reduced = pca.fit_transform(df_scaled)
reduced = pd.DataFrame(reduced)
reduced
```

      0
      1

      0
      -0.058371
      0.216382

      1
      -0.090600
      0.297210

      2
      -0.040614
      0.210754

      3
      -0.184123
      0.211561

      4
      -0.034813
      0.221085

      ...
      ...
      ...

      31376
      -0.117130
      0.058729

      31377
      -0.018511
      -0.003191

      31378
      -0.089317
      -0.011438

      31379
      -0.103643
      -0.061578

      31380
      -0.083024
      0.076023

31381 rows × 2 columns

## K Means

K-Means Clustering may be the most widely known clustering algorithm and involves assigning examples to clusters in an effort to minimize the variance within each cluster. It aims to partition $n$ observations into $k$ clusters in which each observation belongs to the cluster with the nearest mean (cluster centers or cluster centroid), serving as a prototype of the cluster.

To find the optimal number of clusters (k), we perform K Means starting from 1 target cluster and increasing one by one the number of clusters. For each model we calculate the sum of squares error and plot them in a graph. The 'optimal' number of clusters is located at the poin of maximum curvature in the graph. In this case, at 5 clusters.

```python
Nc = range(1, 20)
kmeans = [KMeans(n_clusters=i) for i in Nc]
kmeans
score = [kmeans[i].fit(reduced).score(reduced) for i in range(len(kmeans))]
score
plt.plot(Nc,score)
plt.xlabel('Number of Clusters')
plt.ylabel('Score')
plt.title('Elbow Curve')
plt.show()
```

```python
kmeans = KMeans(n_clusters=5).fit(reduced)
centroids = kmeans.cluster_centers_
print(centroids)
```

    [[-0.0340806  -0.01464435]
     [ 0.19314064  0.06891364]
     [-0.0935422   0.13023781]
     [ 1.01740346  0.14014465]
     [ 0.00848577 -0.20471383]]

Now we plot our clusters

```python
# Predicting the clusters
labels = kmeans.predict(reduced)

# Calculate density
clustered = df.copy(0)
clustered['LABEL'] = labels

densities = []
for label in np.unique(labels):
    cluster = clustered[(clustered['LABEL'] == label) & (clustered['POBTOT'] > 0)]
    density = np.mean(cluster['PCON_DISC'] / cluster['POBTOT'])
    densities.append(density)

# Assign a color to a cluster
colors=['red','green','violet','cyan','yellow']

assign =[]
for row in labels:
    assign.append(colors[row])

fig = plt.figure()
plt.scatter(reduced[0], reduced[1], c=assign, s=60)

# We annotate the cluster density in the cluster centroid
for i in range(len(densities)):
    plt.annotate(str(round(densities[i], 2)), (centroids[i][0], centroids[i][1]), color='black', ha='center', fontweight=1000)

plt.title('K Means Clustering (k=5)')
```

```python
from scipy.spatial.distance import cdist

def intra_cluster_dist(clustered):
    distances = []
    for cluster_idx in clustered.LABEL.unique():
        if cluster_idx == -1: continue
        cluster = clustered[clustered['LABEL'] == cluster_idx][[0, 1]]
        dist = cdist(cluster, cluster, 'euclid')
        distances.append(np.mean(dist.mean(axis=0)))
    return distances

def inter_cluster_dist(clustered):
    distances = []
    for idx_a in clustered.LABEL.unique():
        cluster_a = clustered[clustered['LABEL'] == idx_a][[0, 1]]
        for idx_b in clustered.LABEL.unique():
            if idx_a == -1 or idx_b == -1 or idx_a == idx_b:
                continue
            cluster_b = clustered[clustered['LABEL'] == idx_b][[0, 1]]
            dist = cdist(cluster_a, cluster_b, 'euclid')
            distances.append(np.min(dist))
    return distances

x = reduced.copy()
x['LABEL'] = labels
distances = intra_cluster_dist(x)
print('Total intra cluster distance: ', np.sum(distances))
#distances
```

    Total intra cluster distance:  1.0701735175679252

```python
distances = inter_cluster_dist(x)
print('Total inter cluster distance: ', np.sum(distances))
#distances
```

    Total inter cluster distance:  2.973526878061844

## Gaussian Mixture Model Clustering

Gaussian Mixture Models is another popular clustering technique. In contrast to K Means, which can only generate circular clusters due to its nature of defining centroids and point neighborhoods, Gaussian Mixture Models are able to create stretched clusters, which provide more freedom when creating clusters.

Similarly to K Means, Gaussia Mixture clustering requires the user to provide the number of components (clusters) to generate. Since it is impossible to know the optimal number a priori, we follow a similar approach to the elbow curve. However, instead of calculating and plotting the mean squared error, we claculate and plot the Bayesian information criterion (BIC) score.

The BIC score gives us an estimation on how much is good the Gaussian Mixture Model is in terms of predicting the data we actually have. The lower is the BIC, the better is the model to actually predict the data we have, and by extension, the true, unknown, distribution. In order to avoid overfitting, this technique penalizes models with big number of clusters.

We start from 1 target cluster and progressively increment the number of target clusters. For each model, we calculate and plot the BIC score. To select the best model, we choose the point of maximum curvature of the BIC curve.

In this case, we select 5 clusters.

```python
from sklearn.mixture import GaussianMixture

Nc = range(1, 11)
gmm = [GaussianMixture(n_components=i) for i in Nc]

score = [gmm[i].fit(reduced).bic(reduced) for i in range(len(gmm))]

plt.plot(Nc,score)
plt.xlabel('Number of Clusters')
plt.ylabel('Score')
plt.title('BIC Curve')
plt.show()
print(score)
```

```python
model = GaussianMixture(n_components=4)
# fit the model
model.fit(reduced)
# assign a cluster to each example
yhat = model.predict(reduced)

# Calculate density
clustered = df.copy()
centers = reduced.copy()
clustered['LABEL'] = yhat
centers['LABEL'] = yhat

densities = []
center = []
for label in np.unique(yhat):
    cluster = clustered[(clustered['LABEL'] == label) & (clustered['POBTOT'] > 0)]
    density = np.mean(cluster['PCON_DISC'] / cluster['POBTOT'])
    densities.append(density)

    cluster = centers[centers['LABEL'] == label]
    x = np.mean(cluster[0])
    y = np.mean(cluster[1])
    center.append((x, y))

# retrieve unique clusters
clusters = np.unique(yhat)
# create scatter plot for samples from each cluster
for cluster in clusters:
	# get row indexes for samples with this cluster
	row_ix = np.where(yhat == cluster)
	# create scatter of these samples
	plt.scatter(reduced.iloc[row_ix][0], reduced.iloc[row_ix][1], cmap='viridis')
# show the plot

# We annotate the cluster density in the cluster centroid
for i in range(len(densities)):
    plt.annotate(str(round(densities[i], 2)), center[i], color='black', ha='center', fontweight=1000)

plt.title('GMM Clustering (n=4)')
```

```python
x = reduced.copy()
x['LABEL'] = yhat
distances = intra_cluster_dist(x)
print('Total intra cluster distance: ', np.sum(distances))
print(np.sum(distances))
#distances
```

    Total intra cluster distance:  0.8290737353923373
    0.8290737353923373

```python
distances = inter_cluster_dist(x)
print('Total inter cluster distance: ', np.sum(distances))
#distances
```

    Total inter cluster distance:  0.1370332916670486

## DBSCAN

DBSCAN clusters data points by measuring the distance from each data point to another while considering a minimum number of points. If enough data points are close enough together, DBSCAN will classify it as a new cluster. In contrast to K Means, which fails to accurately cluster data points that are not linearly separable, DBSCAN is able to find intricate clusters based on the density of points.

One of the main parameters of DBSCAN is called Epsilon, which indicates the maximum distance a point can be from another point to be considered a neighbor. To estimate the optimal value of Epsilon, we carried out a k nearest neighbors search. For each data point, we calculated its nearest neighbor and measured their distance. The computed distances were then sorted and plotted. Similarly to the previous approaches, the value for epsilon was selected corresponding to the point of maximum curvature.

For the case of Min Points, as a rule of thumb, $MinPts = 2 \times dim$ can be used, but it may be necessary to choose larger values for very large data, for noisy data or for data that contains many duplicates.

```python
from sklearn.neighbors import NearestNeighbors

neigh = NearestNeighbors(n_neighbors=2)
nbrs = neigh.fit(reduced)
distances, indices = nbrs.kneighbors(reduced)

distances = np.sort(distances, axis=0)
distances = distances[:,1]
plt.plot(distances)
plt.hlines(y=0.007, xmin=0, xmax=35000, linestyles='dashed')
plt.scatter( x = [ 30600 ], y = [ 0.007 ], color = 'red', s = 100  )
plt.text( 24500, 0.01, 'distance threshold = 0.007' )
#plt.xlim([30000, 35000])
plt.ylim([0, 0.1])

plt.title('Nearest Neighbor distance')
```

```python
from sklearn.cluster import DBSCAN

model = DBSCAN(eps=0.007, min_samples=60) #45, 55, 60
# fit the model
yhat = model.fit_predict(reduced)

# Calculate density
clustered = df.copy()
centers = reduced.copy()
clustered['LABEL'] = yhat
centers['LABEL'] = yhat

densities = []
center = []
for label in np.unique(yhat):
    cluster = clustered[(clustered['LABEL'] == label) & (clustered['POBTOT'] > 0)]
    density = np.mean(cluster['PCON_DISC'] / cluster['POBTOT'])
    densities.append(density)

    cluster = centers[centers['LABEL'] == label]
    x = np.mean(cluster[0])
    y = np.mean(cluster[1])
    center.append((x, y))

# retrieve unique clusters
clusters = np.unique(yhat)
# create scatter plot for samples from each cluster
for cluster in clusters:
    if cluster == -1:
        continue
    # get row indexes for samples with this cluster
    row_ix = np.where(yhat == cluster)
    # create scatter of these samples
    plt.scatter(reduced.iloc[row_ix][0], reduced.iloc[row_ix][1])

# We annotate the cluster density in the cluster centroid
for i in range(len(densities)):
    plt.annotate(str(round(densities[i], 2)), center[i], color='black', ha='center', fontweight=1000)

plt.title('DBSCAN Clustering')
```

```python
clusters
```

    array([-1,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11])

```python
x = clustered[clustered['LABEL'] == 0]
x = x[['NOM_MUN', 'NOM_LOC', 'MZA', 'POBTOT', 'PCON_DISC']]
x = x[df.PCON_DISC != 4.740691423780638]
x['DISC_DENS'] = x['PCON_DISC'] / x['POBTOT']
x.sort_values(by='DISC_DENS', ascending=False, inplace=True)
x.head(50)
```

    /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
      This is separate from the ipykernel package so we can avoid doing imports until

      NOM_MUN
      NOM_LOC
      MZA
      POBTOT
      PCON_DISC
      DISC_DENS

      8756
      Mérida
      Mérida
      28
      50.0
      13.0
      0.260000

      9434
      Mérida
      Mérida
      9
      50.0
      10.0
      0.200000

      2902
      Dzidzantún
      Dzidzantún
      10
      47.0
      9.0
      0.191489

      2995
      Dzidzantún
      Dzidzantún
      18
      58.0
      11.0
      0.189655

      15527
      Mérida
      Mérida
      17
      60.0
      11.0
      0.183333

      9586
      Mérida
      Mérida
      16
      56.0
      10.0
      0.178571

      2894
      Dzidzantún
      Dzidzantún
      2
      61.0
      10.0
      0.163934

      5410
      Kanasín
      Kanasín
      9
      62.0
      10.0
      0.161290

      19098
      Mérida
      Mérida
      28
      57.0
      9.0
      0.157895

      9114
      Mérida
      Mérida
      38
      57.0
      9.0
      0.157895

      7759
      Maxcanú
      Maxcanú
      23
      59.0
      9.0
      0.152542

      12326
      Mérida
      Mérida
      25
      55.0
      8.0
      0.145455

      11270
      Mérida
      Mérida
      13
      62.0
      9.0
      0.145161

      17645
      Mérida
      Mérida
      1
      49.0
      7.0
      0.142857

      31827
      Umán
      Umán
      23
      56.0
      8.0
      0.142857

      13764
      Mérida
      Mérida
      11
      55.0
      7.0
      0.127273

      12356
      Mérida
      Mérida
      25
      64.0
      8.0
      0.125000

      2863
      Dzemul
      Dzemul
      20
      56.0
      7.0
      0.125000

      25294
      Progreso
      Progreso
      16
      58.0
      7.0
      0.120690

      30314
      Tizimín
      Tizimín
      31
      50.0
      6.0
      0.120000

      9677
      Mérida
      Mérida
      5
      59.0
      7.0
      0.118644

      3787
      Halachó
      Halachó
      3
      59.0
      7.0
      0.118644

      30810
      Tizimín
      Tizimín
      40
      61.0
      7.0
      0.114754

      9453
      Mérida
      Mérida
      30
      62.0
      7.0
      0.112903

      20360
      Mérida
      Mérida
      16
      54.0
      6.0
      0.111111

      6608
      Kanasín
      Kanasín
      52
      55.0
      6.0
      0.109091

      23406
      Motul
      Motul de Carrillo Puerto
      35
      57.0
      6.0
      0.105263

      417
      Baca
      Baca
      13
      57.0
      6.0
      0.105263

      30638
      Tizimín
      Tizimín
      31
      58.0
      6.0
      0.103448

      9330
      Mérida
      Mérida
      7
      58.0
      6.0
      0.103448

      28660
      Telchac Pueblo
      Telchac
      27
      58.0
      6.0
      0.103448

      24777
      Peto
      Peto
      19
      68.0
      7.0
      0.102941

      23668
      Muna
      Muna
      12
      59.0
      6.0
      0.101695

      12549
      Mérida
      Mérida
      6
      59.0
      6.0
      0.101695

      13703
      Mérida
      Mérida
      16
      61.0
      6.0
      0.098361

      30801
      Tizimín
      Tizimín
      25
      51.0
      5.0
      0.098039

      9155
      Mérida
      Mérida
      38
      62.0
      6.0
      0.096774

      5257
      Izamal
      Izamal
      6
      52.0
      5.0
      0.096154

      12972
      Mérida
      Mérida
      17
      63.0
      6.0
      0.095238

      8685
      Mérida
      Mérida
      29
      63.0
      6.0
      0.095238

      17845
      Mérida
      Mérida
      2
      63.0
      6.0
      0.095238

      9343
      Mérida
      Mérida
      21
      53.0
      5.0
      0.094340

      16131
      Mérida
      Mérida
      15
      64.0
      6.0
      0.093750

      12611
      Mérida
      Mérida
      11
      64.0
      6.0
      0.093750

      15632
      Mérida
      Mérida
      7
      64.0
      6.0
      0.093750

      13711
      Mérida
      Mérida
      25
      65.0
      6.0
      0.092308

      23156
      Motul
      Motul de Carrillo Puerto
      6
      55.0
      5.0
      0.090909

      13696
      Mérida
      Mérida
      9
      67.0
      6.0
      0.089552

      13038
      Mérida
      Mérida
      12
      58.0
      5.0
      0.086207

      32366
      Umán
      Umán
      8
      58.0
      5.0
      0.086207

```python
x = reduced.copy()
x['LABEL'] = yhat
distances = intra_cluster_dist(x)
print('Total intra cluster distance: ', np.sum(distances))
#distances
```

    Total intra cluster distance:  0.13337129806795028

```python
distances = inter_cluster_dist(x)
print('Total inter cluster distance: ', np.sum(distances))
#distances
```

    Total inter cluster distance:  7.915159479738801

```python

```


# Spherical Mean Technique

The recently proposed Spherical Mean Technique (SMT) model *(Kaden et al. 2015)* is a spherical convolution-based technique, which instead of a model to multi-shell DWIs, fits the *spherical mean* of the model to the *spherical mean* of the signal per shell.

SMT observes that if the FOD is a probability density (i.e. integrated to unity) then spherical mean of the signal and the convolution kernel must be the same

\begin{equation}
\int_{\mathbb{S}^2}E_b(\textbf{g})d\textbf{g}=\int_{\mathbb{S}^2}(\operatorname{FOD}\,*_{\mathbb{S}^2}\,K)_b(\textbf{g})d\textbf{g}=\int_{\mathbb{S}^2}K_b(\textbf{g})d\textbf{g}=\epsilon_K(b,\lambda_\perp,\lambda_\parallel).
\end{equation}

The estimation of the multi-compartment kernel using SMT enables the characterization of per-axon micro-environments, as the effects of axon dispersion and crossings are only contained in the FOD.

Advantages:
- Insensitive to axon dispersion and crossings.

Limitations:
- Only accounts for average of tissue properties in crossing configurations, which potentially each have different properties.

# Using Dmipy to set up the SMT Model

To set up the SMT model we start by calling the regular zeppelin model.

```python
from dmipy.signal_models import gaussian_models
zeppelin = gaussian_models.G2Zeppelin()
```

To initialize a spherical mean model, instead of calling the MultiCompartmentModel, we call MultiCompartmentSphericalMeanModel.

```python
from dmipy.core import modeling_framework
smt_mod = modeling_framework.MultiCompartmentSphericalMeanModel(
    models=[zeppelin])
```

Notice how the zeppelin's spherical mean representation has no orientation parameter 'mu'.

```python
smt_mod.parameter_names
```

    ['G2Zeppelin_1_lambda_perp', 'G2Zeppelin_1_lambda_par']

The only constraint on the Zeppelin model is that $\lambda_\parallel\geq\lambda_\perp$. We can impose this parameter constraint using the model.set_fractional_parameter() function.

```python
smt_mod.set_fractional_parameter('G2Zeppelin_1_lambda_perp', 'G2Zeppelin_1_lambda_par')
smt_mod.parameter_names
```

    ['G2Zeppelin_1_lambda_perp_fraction', 'G2Zeppelin_1_lambda_par']

The lambda_perp parameter has now been replaced with lambda_perp_fraction.
In essence, what this function did is replace the original parameter with a different optimization parameter that operates as a fraction $f$ of $\lambda_{\parallel}$, such that $\lambda_{\perp}=f \times \lambda_{\parallel}$, where $1\geq f \geq 0$. Internally, the $\lambda_{\perp}$ parameter will now always respect the relative "smaller than" constraint to $\lambda_{\parallel}$, and the true value of $\lambda_{\perp}$ can be recovered after fitting the model.

Visualize the model, notice that it's just single model:

```python
from IPython.display import Image
smt_mod.visualize_model_setup(view=False, cleanup=False)
Image('Model Setup.png')
```

## Fitting SMT to Human Connectome Project data

```python
from dmipy.data import saved_data
scheme_hcp, data_hcp = saved_data.wu_minn_hcp_coronal_slice()
sub_image = data_hcp[70:90,: , 70:90]
```

    This data slice originates from Subject 100307 of the Human Connectome Project, WU-Minn Consortium (Principal Investigators: David Van Essen and Kamil Ugurbil; 1U54MH091657) funded by the 16 NIH Institutes and Centers that support the NIH Blueprint for Neuroscience Research; and by the McDonnell Center for Systems Neuroscience at Washington University.

```python
import matplotlib.pyplot as plt
import matplotlib.patches as patches
%matplotlib inline

fig, ax = plt.subplots(1)
ax.imshow(data_hcp[:, 0, :, 0].T, origin=True)
rect = patches.Rectangle((70,70),20,20,linewidth=1,edgecolor='r',facecolor='none')
ax.add_patch(rect)
ax.set_axis_off()
ax.set_title('HCP coronal slice B0 with ROI');
```

```python
# Fitting SMT is very fast, half the time is actually spent estimating the spherical mean of the data.
smt_fit_hcp = smt_mod.fit(scheme_hcp, data_hcp, Ns=30, mask=data_hcp[..., 0]>0, use_parallel_processing=False)
```

    Setup brute2fine optimizer in 1.50449085236 seconds
    Fitting of 8181 voxels complete in 22.991079092 seconds.
    Average of 0.0028103018081 seconds per voxel.

```python
fitted_parameters = smt_fit_hcp.fitted_parameters

fig, axs = plt.subplots(1, len(fitted_parameters), figsize=[15, 15])
axs = axs.ravel()

for i, (name, values) in enumerate(fitted_parameters.items()):
    cf = axs[i].imshow(values.squeeze().T, origin=True)
    axs[i].set_title(name)
    axs[i].set_axis_off()
    fig.colorbar(cf, ax=axs[i], shrink=0.2)
```

## Estimating Parametric FODs using SMT kernel

Dmipy allows for the estimation of parametric Fiber Orientation Distributions (FODs) using the fitted spherical mean model parameters as a convolution kernel. At this time, it is possible to choose either 'watson' or 'bingham' distributions with any number of compartments.

### Watson FOD estimation
First we fit SMT on a patch where we want to estimate FODs.

```python
smt_fit_patch = smt_mod.fit(scheme_hcp, sub_image, Ns=30)
```

    Using parallel processing with 8 workers.
    Setup brute2fine optimizer in 0.0670688152313 seconds
    Fitting of 400 voxels complete in 1.86997413635 seconds.
    Average of 0.00467493534088 seconds per voxel.

Then we call the fitted SMT model to return a multi-compartment model that is designed to fit $N$ parametric FODs to the data. Note that the returned model is no different from a self-designed regular MultiCompartmentModel - it only differs in that as the kernel properties are fixed to the estimated SMT parameters.

```python
parametric_fod_model= smt_fit_patch.return_parametric_fod_model(
    distribution='watson', Ncompartments=1)
```

Notice that the FOD optimizer has the same design as a regular MultiCompartmentModel, and all the same solvers and options are available for FOD estimation. In this way, it is possible to have completely different approaches to estimating the kernel using a spherical mean model, and estimating the subsequent parametric FODs.

```python
smt_fod_fit = parametric_fod_model.fit(scheme_hcp, sub_image)
```

    Using parallel processing with 8 workers.
    Cannot estimate signal grid with voxel-dependent x0_vector.
    Setup brute2fine optimizer in 0.000311851501465 seconds
    Fitting of 400 voxels complete in 27.5659279823 seconds.
    Average of 0.0689148199558 seconds per voxel.

Fitting an FOD optimizer returns a FittedMultiCompartmentModel, where now all the same functions are available. We will show the estimated FODs in the next example.

### Watson FOD visualization

Here we use the same Dipy procedure as in the previous examples to visualize the FODs.

```python
from dipy.data import get_sphere
from dipy.viz.actor import slicer
sphere = get_sphere(name='symmetric724').subdivide()
fods = smt_fod_fit.fod(sphere.vertices, visual_odi_lower_bound=0.1)
```

```python
import numpy as np
affine = np.eye(4)
affine[0,3] = -10
affine[1,3] = -10

# lambda_perp = lambda_perp_fraction * lambda_par
volume_res =  (smt_fod_fit.fitted_parameters['SD1WatsonDistributed_1_G2Zeppelin_1_lambda_perp_fraction'] *
               smt_fod_fit.fitted_parameters['SD1WatsonDistributed_1_G2Zeppelin_1_lambda_par'])

volume_im = slicer(volume_res[:, 0, :, None], interpolation='nearest', affine=affine, opacity=0.7)
```

```python
from dipy.viz import fvtk
ren = fvtk.ren()
fod_spheres = fvtk.sphere_funcs(fods, sphere, scale=1., norm=False)
fod_spheres.RotateX(90)
fod_spheres.RotateZ(180)
fod_spheres.RotateY(180)
fvtk.add(ren, fod_spheres)
fvtk.add(ren, volume_im)
fvtk.record(ren=ren, size=[700, 700])
```

```python
import matplotlib.image as mpimg
img = mpimg.imread('dipy.png')

plt.figure(figsize=[10, 10])
plt.imshow(img[100:-97, 100:-85])
plt.title('SMT Watson FODs lambda_perp background', fontsize=20)
plt.axis('off');
```

## Estimating Error Metrics: MSE and $R^2$

It is also possible to calculate the Mean Squared Error (MSE) and the $R^2$ coefficient of determination.
In MSE, the lower the better, while $R^2$ ranges between 0 and 1, with 1 being a perfect model fit.

```python
mse = smt_fit_hcp.mean_squared_error(data_hcp)
R2 = smt_fit_hcp.R2_coefficient_of_determination(data_hcp)

fig, axs = plt.subplots(1, 2, figsize=[15, 15])
cf = axs[0].imshow(mse.squeeze().T, origin=True, vmax=1e-3, cmap='Greys_r')
fig.colorbar(cf, ax=axs[0], shrink=0.33)
axs[0].set_title('Mean Squared Error', fontsize=20)
axs[0].set_axis_off()
cf = axs[1].imshow(R2.squeeze().T, origin=True, vmin=.98, cmap='Greys_r')
fig.colorbar(cf, ax=axs[1], shrink=0.33)
axs[1].set_title('R2 - Coefficient of Determination', fontsize=20)
axs[1].set_axis_off();
```

The MSE shows that the fitting error is overall very low, with higher errors in the CSF and the skull. The $R^2$ agree with the MSE results, having values very close to 1 overall, with lower values in the CSF and skull.

## References
- Kaden, Enrico, et al. "Multi-compartment microscopic diffusion imaging." NeuroImage 139 (2016): 346-359.


## TD;DR

After practicing with elastic material model, we'll continue to nonlinear material modelling: viscoplasticity. In this post we'll explore Norton and Birgin viscoplastic potential equations, von Mises yield function and automatic derivation using ForwardDiff. Most of the theory can be found from [here](http://mms2.ensmp.fr/msi_paris/plasticity/transparents/Expo2.pdf) and [here](http://mms2.ensmp.fr/msi_paris/archives-transparents/JLC-Plasticity.pdf).

**Note! Equations are written in tensor notation, but the implementated code is in matrix notation!**

## Recap to linear material modelling

Here's a small recap example from elastic material modelling with matrix notation. Do note that we're using now cyclic loading.

\begin{equation}
\sigma = C : \epsilon
\end{equation}

```julia
using PyPlot
using ForwardDiff

num = 100                  # Number of steps
E = 200000.0             # Young's modulus [GPa]
ν = 0.3                  # Poisson's ratio

L = 1000.0               # Initial length [mm]
ΔL = linspace(0, 5, num)   # Change of length [mm]

ϵ_ = collect(ΔL / L)
ϵ_total = vec([ϵ_; reverse(ϵ_[2:end-1]); -ϵ_; reverse(-ϵ_[2:end-1])])

Δt = 0.1
accumulator = 0.0
I = eye(3)
total_num = length(ϵ_total)
time_ = zeros(total_num)
σ = zeros(total_num, 6)

C = E / ((1 + ν)*(1 - 2*ν)) * [
    1-ν   ν   ν         0         0         0
      ν 1-ν   ν         0         0         0
      ν   ν 1-ν         0         0         0
      0   0   0 (1-2*ν)/2         0         0
      0   0   0         0 (1-2*ν)/2         0
      0   0   0         0         0 (1-2*ν)/2]

for i=1:(total_num)
    ϵ = zeros(6)
    ϵ[1] = ϵ_total[i]
    ϵ[2] = -ϵ_total[i] * ν
    ϵ[3] = -ϵ_total[i] * ν

    # The most general form of Hooke's law for isotropic materials
    σ[i, :] = C * ϵ

    accumulator += Δt
    time_[i] = accumulator
end

σ_11 = vec(σ[:, 1])

fig = figure()
grid()
plot(ϵ_total, σ_11)
ylabel("Stress [MPa]")
xlabel("Strain")
title("Stress-strain cyclic curve")
fig = figure()
grid()
title("Strain history")
plot(time_, ϵ_total)
xlabel("Time [s]")
ylabel("Strain")
```

## Nonlinear material

Stress is always related only to the elastic part of the the strain. The theory of plasticity is used to define the non-linear region of the material behaviour. Moving from linear to nonlinear modelling, we'll have to introduce a new variable: plastic strain $\epsilon^p$. Here's a definition for plasticity from [wiki](https://en.wikipedia.org/wiki/Plasticity_(physics)):

> In physics and materials science, plasticity describes the deformation of a (solid) material undergoing non-reversible changes of shape in response to applied forces. For example, a solid piece of metal being bent or pounded into a new > shape displays plasticity as permanent changes occur within the material itself. In engineering, the transition from > elastic behavior to plastic behavior is called yield.

### Decomposition of strain
Theory presented here can be found from [here](http://eu.wiley.com/WileyCDA/WileyTitle/productCd-1118632702.html). Strain increment $de$ is decomposed into the elastic part de e and the plastic part de p using kinematics. Additive decomposition is used for small strains and multiplicative decomposition for large strains. In elasticplastic materials the strain increment is decomposed into an elastic and a plastic
part:

\begin{equation}
d\epsilon = d\epsilon^e + d\epsilon^{vp}
\end{equation}

Dividing both sides with the differential time increment $dt$ produces the rate form:

\begin{equation}
\dot\epsilon = \dot\epsilon^e + \dot\epsilon^{vp}
\end{equation}

Diving previous equation with time increment and substituting elastic strain rate produces relation for linear region:

\begin{equation}
\dot\sigma = C : \dot\epsilon^e = C : (\dot\epsilon - \dot\epsilon^{vp})
\end{equation}

In the non-linear regime, the stress-strain relation is defined as:

\begin{equation}
\dot\sigma = C : \dot\epsilon^e = C^{tan} : \dot\epsilon
\end{equation}

Stress can now be expressed as:
\begin{equation}
\sigma = \int_0^t \dot\sigma dt'
\end{equation}

but usually, stress is more conveniently written as:

\begin{equation}
\sigma_{n+1} = \sigma_n + C : (\dot\epsilon - \dot\epsilon^{vp})
\end{equation}

### Yield function

Yield function defines the transition between elastic and plastic zones. One of the most common yield functions is [von Mises yield function](https://en.wikipedia.org/wiki/Von_Mises_yield_criterion), which is defined as:

\begin{equation}
f(\sigma) = \sigma_{eq}(\sigma) - \sigma_y
\end{equation}

in which $\sigma_y$ is yield stress, which defines the limit, when plastic strain starts and $\sigma_{eq}$ is equivalent stress, which is expressed as:

\begin{equation}
\sigma_{eq}(\sigma) = \sqrt{3J_2(\sigma)}
\end{equation}

in which $J_2$ is second deviatoric stress invariant:

\begin{equation}
J_2(\sigma) = \frac{1}{2}s:s
\end{equation}

and $s$ is deviatoric stress, which is defined as:

\begin{equation}
s = \sigma - \frac{1}{3}\rm{tr}(\sigma)I
\end{equation}

in which tr is a trace operator and $\textbf{I}$ is second order identity tensor. What the yield function basically tells us: if the return value of given stress tensor is negative, we are in the elastic region and if the returned value is positive, we have exceeded the yield limit and ventured into plastic region. In order to take this into account in the implementation, we'll make a simple comparison operation with:

\begin{equation}
f(\sigma) > 0
\end{equation}

which will define, if we apply viscoplastic strain or not to our stress.

```julia
""" Deviatoric stress tensor
"""
function deviatoric_stress(σ)
    σ_tensor = [σ[1] σ[4] σ[6];
                σ[4] σ[2] σ[5];
                σ[6] σ[5] σ[3]]
    σ_dev = σ_tensor - 1/3 * trace(σ_tensor) * eye(3)
    return [σ_dev[1, 1], σ_dev[2, 2], σ_dev[3, 3], σ_dev[1, 2], σ_dev[2, 3], σ_dev[1, 3]]
end

""" Second deviatoric stress invariant
"""
function J_2_stress(σ)
    s = deviatoric_stress(σ)
    s_vec = vec([s; s[4]; s[5]; s[6]])  # Adding missing shear elements for double dot product
    return 1/2 * dot(s_vec, s_vec)      # equivalent to sqrt(1/2 * s : s), in tensor notation
end

""" Equivalent stress
"""
function equivalent_stress(σ)
    J_2 = J_2_stress(σ)
    return sqrt(3 * J_2)
end

""" Von mises yield function
"""
function von_mises_yield(σ, σ_y)
    return equivalent_stress(σ) - σ_y
end
```

    von_mises_yield (generic function with 1 method)

```julia
using IJulia.display

HTML("""
<div>

</div>
""")
```

<div>

</div>

Here are presented Von mises yield surfaces both in 2D and 3D. Used yield stress is 200 MPa. As you can see, von Mises yield surface is an oval in 2D plane. Inside the circle represents the elastic region and outside the circle the plastic region. Do note, that x- and y-axis are eigenstresses and not $\sigma_{x}$ and $\sigma_{y}$.

3D presentation reveals the whole truth and we can see that the oval shape, which we saw in 2D plane, is marely a cross-section of the cylinder, which is the von Mises yield surface. The axis of the cylinder resides at hte hydrostatic pressure axis.

### Viscoplastic potential

Next we'll need to define the viscoplastic strain rate $\dot\epsilon^{vp}$:

\begin{equation}
\dot\epsilon^{vp} = \frac{\partial\Phi}{\partial\sigma}
\end{equation}
in which $\Phi$ is a plastic potential, which is a function of the stress and the material parameters. In the post, we'll look at both Norton rule and Bingham model. But first, the Norton rule:

\begin{equation}
\Phi = \frac{K}{n + 1} \left(\frac{J}{K}\right)^{n+1}, \hspace{1cm} J = f(\sigma)
\end{equation}

in which $K$ and $n$ are material parameters and $f$ is an yield function. When we use von Mises yield function, we can derivate this function with $\sigma$:

\begin{equation}
\frac{\partial\Phi}{\partial\sigma} = \left(\frac{J}{K}\right)^{n}\frac{3}{2}\frac{s}{\sigma_{eq}}
\end{equation}

```julia
""" Norton rule
"""
function norton_plastic_potential(σ, K, n, f)
    f_ = f(σ)
    return K/(n+1) * (f_ / K) ^ (n + 1)
end

""" Analytically derivated norton rule
"""
function analytical_dΦdσ(σ, K, n, f)
    f_ = f(σ)
    σ_v = equivalent_stress(σ)
    σ_dev_vec = deviatoric_stress(σ)
    return (f_ / K) ^ n * 3 / 2 * σ_dev_vec / σ_v
end
```

    analytical_dΦdσ (generic function with 1 method)

### Summary of needed functions

Here's a summarize of the main functions, which we need:

\begin{equation}
\sigma_{n+1} = \sigma_n + C : (\dot\epsilon - \dot\epsilon^{vp}) \hspace{1.5cm}\dot\epsilon^{vp} = \left(\frac{J}{K}\right)^{n}\frac{3}{2}\frac{s}{\sigma_{eq}}\hspace{1.5cm} J = f(\sigma) = \sigma_{eq}(\sigma) - \sigma_y
\end{equation}

Now, we can proceed to the simulation:

```julia
# Create an empty history vectors
σ_hist = zeros(total_num, 6)
ϵ_hist = zeros(total_num, 6)

# Initialize arrays
ϵ_last = zeros(6)
σ = zeros(6)
ϵ = zeros(6)

# We'll define our yield limit to be 200 MPa
σ_y = 200.0

# Define Δt, K and n
Δt = 0.1
n = 0.92
K = 180.0e3

# Let's make a currier function to make it pretty!
f = x->von_mises_yield(x, 200.)

# Ok, going through time steps
for i=2:total_num
    ϵ = zeros(6)
    ϵ[1] = ϵ_total[i]
    ϵ[2] = -ϵ_total[i] * ν
    ϵ[3] = -ϵ_total[i] * ν

    # Trial stress
    dϵdt = (ϵ - ϵ_last) / Δt
    dσdt = C * dϵdt
    σ_tria = σ + dσdt * Δt

    # Checking if yielded
    yield_ = f(σ_tria)
    if yield_ > 0

        # If yielded, we'll add the viscoplastic strain to stress
        dϵ_vp = analytical_dΦdσ(σ_tria, K, n, f)
        σ += C * (dϵdt - dϵ_vp) * Δt
    else
        σ[:] = σ_tria
    end

    # Collect data
    σ_hist[i, :, :] = σ
    ϵ_last[:] = ϵ
    ϵ_hist[i, :] = ϵ
end

# Plot
plot(ϵ_hist[:, 1], σ_hist[:, 1])
grid()
xlabel("Strain")
ylabel("Stress x [MPa]")
```

As you can see, plot has now changed. At 200 MPa our stress exceeds the yield limit and viscoplastic strain is added. After reaching 0.004 strain level, we'll start to go back to the zero and return to the elastic strain region. At ~20 MPa we again exceed the yield limit again and start adding viscoplastic strain. I also wanted to show the power of automatic derivation using ForwardDiff in the same calculation.

```julia
σ_hist = zeros(total_num, 6)
ϵ_hist = zeros(total_num, 6)

ϵ_last = zeros(6)
ϵ = zeros(6)
σ = zeros(6)

σ_y = 200.0
Δt = 0.1
n = 0.92
K = 180.0e3

f = x->von_mises_yield(x, σ_y)

for i=2:total_num
    ϵ = zeros(6)
    ϵ[1] = ϵ_total[i]
    ϵ[2] = -ϵ_total[i] * ν
    ϵ[3] = -ϵ_total[i] * ν

    dϵdt = (ϵ - ϵ_last) / Δt
    dσdt = C * dϵdt
    σ_tria = σ + dσdt * Δt
    yield_ = f(σ_tria)
    if yield_ > 0
        wrap = x-> norton_plastic_potential(x, K, n, f)

        # This part is different. As you can see, we'll take a numerical derivate
        # Way more convenient compared to hand derivated potential function
        dϵ_vp = ForwardDiff.gradient(wrap, σ_tria)
        σ += C * (dϵdt - dϵ_vp) * Δt
    else
        σ[:] = σ_tria
    end
    σ_hist[i, :, :] = σ
    ϵ_last[:] = ϵ
    ϵ_hist[i, :] = ϵ
end
plot(ϵ_hist[:, 1], σ_hist[:, 1])
grid()
```

Using the ForwardDiff now enables me to derivate quite complex looking function and also saves time from debugging hand derivated function. Lastly, we'll try another viscoplastic potential (Bingham model) for our plastic strain rate, which is defined as:

\begin{equation}
\Phi = \frac{1}{2} \left(\frac{J}{\eta}\right)^2, \hspace{1cm} J=f(\sigma)
\end{equation}

```julia
function bingham_plastic_potential(σ, η, f)
    return 0.5 * (f(σ) / η) ^2
end
```

    bingham_plastic_potential (generic function with 1 method)

Now that we've discovered the power of ForwardDiff, we'll use automatic derivation also in this simlation.

```julia
ϵ_last = zeros(6)
ϵ_hist = zeros(total_num, 6)
σ_hist = zeros(total_num, 6)
σ = zeros(6)
σ_y = 200.0

Δt = 0.01
ϵ = zeros(6)

# Parameter for Bingham model
η=190.

f = x->von_mises_yield(x, σ_y)

for i=2:total_num
    ϵ = zeros(6)
    ϵ[1] = ϵ_total[i]
    ϵ[2] = -ϵ_total[i] * ν
    ϵ[3] = -ϵ_total[i] * ν

    dϵdt = (ϵ - ϵ_last) / Δt
    dσdt = C * dϵdt
    σ_tria = σ + dσdt * Δt
    yield_ = f(σ_tria)
    if yield_ > 0
        wrap = x->bingham_plastic_potential(x, η, f)

        # Again, I'm a bit lazy and just use ForwardDiff
        dϵ_vp = ForwardDiff.gradient(wrap, σ_tria)
        σ += C * (dϵdt - dϵ_vp) * Δt
    else
        σ[:] = σ_tria
    end
    σ_hist[i, :, :] = σ
    ϵ_last[:] = ϵ
    ϵ_hist[i, :] = ϵ
end
plot(ϵ_hist[:, 1], σ_hist[:, 1])
grid()
```

## The End

In this post we have seen how to implement non-linear viscoplastic material model. Varying yield function or viscoplastic potential we can create different material responses.


# Mie Scattering Function

**Scott Prahl**

**1 Aug 2019, Version 6**

```python
%matplotlib inline

import numpy as np
import matplotlib.pyplot as plt

import miepython as mp
```

Mie scattering describes the special case of the interaction of light passing through a non-absorbing medium with a single embedded spherical object.  The sphere itself can be non-absorbing, moderately absorbing, or perfectly absorbing.

## Goals for this notebook:

* show how to plot the phase function
* explain the units for the scattering phase function
* describe normalization of the phase function
* show a few examples from classic Mie texts

## Geometry

Specifically, the scattering function $p(\theta_i,\phi_i,\theta_o,\phi_o)$ describes the amount of light scattered by a particle for light incident at an angle $(\theta_i,\phi_i)$ and exiting the particle (in the far field) at an angle $(\theta_o,\phi_o)$.  For simplicity, the scattering function is often assumed to be rotationally symmetric (it is, obviously, for spherical scatterers) and that the angle that the light is scattered into only depends the $\theta=\theta_o-\theta_i$.  In this case, the scattering function can be written as $p(\theta)$.  Finally, the angle is often replaced by $\mu=\cos\theta$ and therefore the phase function becomes just $p(\mu)$.

The figure below shows the basic idea.  An incoming monochromatic plane wave hits a sphere and produces *in the far field* two separate monochromatic waves — a slightly attenuated unscattered planar wave and an outgoing spherical wave.

Obviously. the scattered light will be cylindrically symmetric about the ray passing through the center of the sphere.

```python
t = np.linspace(0,2*np.pi,100)
xx = np.cos(t)
yy = np.sin(t)
fig,ax=plt.subplots(figsize=(10,8))
plt.axes().set_aspect('equal')
plt.plot(xx,yy)
plt.plot([-5,7],[0,0],'--k')

plt.annotate('incoming irradiance', xy=(-4.5,-2.3),ha='left',color='blue',fontsize=14)
for i in range(6):
    y0 = i -2.5
    plt.annotate('',xy=(-1.5,y0),xytext=(-5,y0),arrowprops=dict(arrowstyle="->",color='blue'))

plt.annotate('unscattered irradiance', xy=(3,-2.3),ha='left',color='blue',fontsize=14)
for i in range(6):
    y0 = i -2.5
    plt.annotate('',xy=(7,y0),xytext=(3,y0),arrowprops=dict(arrowstyle="->",color='blue',ls=':'))

plt.annotate('scattered\nspherical\nwave', xy=(0,1.5),ha='left',color='red',fontsize=16)
plt.annotate('',xy=(2.5,2.5),xytext=(0,0),arrowprops=dict(arrowstyle="->",color='red'))
plt.annotate(r'$\theta$',xy=(2,0.7),color='red',fontsize=14)
plt.annotate('',xy=(2,2),xytext=(2.7,0),arrowprops=dict(connectionstyle="arc3,rad=0.2", arrowstyle="<->",color='red'))

plt.xlim(-5,7)
plt.ylim(-3,3)
plt.axis('off')
plt.show()
```

## Scattered Wave

```python
fig,ax=plt.subplots(figsize=(10,8))
plt.axes().set_aspect('equal')
plt.scatter([0],[0],s=30)

m = 1.5
x = np.pi/3
theta = np.linspace(-180,180,180)
mu = np.cos(theta/180*np.pi)
scat = 15 * mp.i_unpolarized(m,x,mu)

plt.plot(scat*np.cos(theta/180*np.pi),scat*np.sin(theta/180*np.pi))
for i in range(12):
    ii = i*15
    xx = scat[ii]*np.cos(theta[ii]/180*np.pi)
    yy = scat[ii]*np.sin(theta[ii]/180*np.pi)
#    print(xx,yy)
    plt.annotate('',xy=(xx,yy),xytext=(0,0),arrowprops=dict(arrowstyle="->",color='red'))

plt.annotate('incident irradiance', xy=(-4.5,-2.3),ha='left',color='blue',fontsize=14)
for i in range(6):
    y0 = i -2.5
    plt.annotate('',xy=(-1.5,y0),xytext=(-5,y0),arrowprops=dict(arrowstyle="->",color='blue'))

plt.annotate('unscattered irradiance', xy=(3,-2.3),ha='left',color='blue',fontsize=14)
for i in range(6):
    y0 = i -2.5
    plt.annotate('',xy=(7,y0),xytext=(3,y0),arrowprops=dict(arrowstyle="->",color='blue',ls=':'))

plt.annotate('scattered\nspherical wave', xy=(0,1.5),ha='left',color='red',fontsize=16)

plt.xlim(-5,7)
plt.ylim(-3,3)
#plt.axis('off')
plt.show()
```

## Normalization of the scattered light

So the scattering function or phase function has at least three reasonable normalizations that involve integrating over all $4\pi$ steradians.  Below $d\Omega=\sin\theta d\theta\,d\phi$ is a differential solid angle

$$
\begin{align}
\int_{4\pi} p(\theta,\phi) \,d\Omega &= 1\\[2mm]
\int_{4\pi} p(\theta,\phi) \,d\Omega &= 4\pi \\[2mm]
\int_{4\pi} p(\theta,\phi) \,d\Omega &= a  \qquad\qquad \mbox{Used by miepython}\\[2mm]
\end{align}
$$

where $a$ is the single scattering albedo,

$$
a = \frac{\sigma_s}{\sigma_s+\sigma_a}
$$

and $\sigma_s$ is the scattering cross section, and $\sigma_a$ is the absorption cross section.

*The choice of normalization was made because it accounts for light lost through absorption by the sphere.*

If the incident light has units of watts, then the values from the scattering function $p(\theta,\phi)$ have units of radiant intensity or W/sr.

For example, a circular detector with radius $r_d$ at a distance $R$  will subtend an angle

$$
\Omega = \frac{\pi r_d^2}{R^2}
$$

(assuming $r_d\ll R$). Now if $P_0$ of light is scattered by a sphere then the scattered power on the detector will be

$$
P_d = P_0 \cdot \Omega \cdot p(\theta,\phi)
$$

## Examples

### Unpolarized Scattering Function

If unpolarized light hits the sphere, then there are no polarization effects to worry about.  It is pretty easy to generate a plot to show how scattering changes with angle.

```python
m = 1.5
x = np.pi/3
theta = np.linspace(-180,180,180)
mu = np.cos(theta/180*np.pi)
scat = mp.i_unpolarized(m,x,mu)

fig,ax = plt.subplots(1,2,figsize=(12,5))
ax=plt.subplot(121, projection='polar')
ax.plot(theta/180*np.pi,scat)
ax.set_rticks([0.05, 0.1,0.15])
ax.set_title("m=1.5, Sphere Diameter = $\lambda$/3")

plt.subplot(122)
plt.plot(theta,scat)
plt.xlabel('Exit Angle [degrees]')
plt.ylabel('Unpolarized Scattered light [1/sr]')
plt.title('m=1.5, Sphere Diameter = $\lambda$/3')
plt.ylim(0.00,0.2)

plt.show()
```

### Rayleigh Scattering

Classic Rayleigh scattering treats small particles with natural (unpolarized) light.
The solid black line denotes the total scattered intensity. The red dashed line is light scattered that is polarized perpendicular to the plane of the graph and the blue dotted line is for light parallel to the plane of the graph. (Compare with van de Hult, Figure 10)

```python
m = 1.3
x = 0.01
theta = np.linspace(-180,180,180)
mu = np.cos(theta/180*np.pi)
ipar = mp.i_par(m,x,mu)/2
iper = mp.i_per(m,x,mu)/2
iun  = mp.i_unpolarized(m,x,mu)

fig,ax = plt.subplots(1,2,figsize=(12,5))
ax=plt.subplot(121, projection='polar')
ax.plot(theta/180*np.pi,iper,'r--')
ax.plot(theta/180*np.pi,ipar,'b:')
ax.plot(theta/180*np.pi,iun,'k')

ax.set_rticks([0.05, 0.1,0.15])
plt.title('m=%.2f, Sphere Parameter = %.2f' %(m,x))

plt.subplot(122)
plt.plot(theta,iper,'r--')
plt.plot(theta,ipar,'b:')
plt.plot(theta,iun,'k')

plt.xlabel('Exit Angle [degrees]')
plt.ylabel('Normalized Scattered light [1/sr]')
plt.title('m=%.2f, Sphere Parameter = %.2f' %(m,x))
plt.ylim(0.00,0.125)
plt.text(130,0.02,r"$0.5I_{per}$",color="blue", fontsize=16)
plt.text(120,0.062,r"$0.5I_{par}$",color="red", fontsize=16)
plt.text(30,0.11,r"$I_{unpolarized}$",color="black", fontsize=16)

plt.show()
```

## Differential Scattering Cross Section

Sometimes one would like the scattering function normalized so that the integral over all $4\pi$ steradians to be the scattering cross section

$$
\sigma_{sca} = \frac{\pi d^2}{4} Q_{sca}
$$

The *differential scattering cross section* \frac{d\sigma_{sca}}{d\Omega}

$$
\sigma_{sca} = \int_{4\pi} \frac{d\sigma_{sca}}{d\Omega}\,d\Omega
$$

Since the unpolarized scattering normalized so its integral is the single scattering albedo, this means that

$$
\frac{Q_{sca}}{Q_{ext}} = \int_{4\pi} p(\mu) \sin\theta\,d\theta d\phi
$$

and therefore the differential scattering cross section can be obtained `miepython` using

$$
\frac{d\sigma_{sca}}{d\Omega} = \frac{\pi d^2 Q_{ext}}{4} \cdot p(\theta,\phi)
$$

Note that this is $Q_{ext}$ and *not* $Q_{sca}$ because of the choice of normalization!

For example, here is a replica of [figure 4](http://plaza.ufl.edu/dwhahn/Rayleigh%20and%20Mie%20Light%20Scattering.pdf)

```python
m = 1.4-0j
lambda0 = 532e-9  # m
theta = np.linspace(0,180,1000)
mu = np.cos(theta* np.pi/180)

d = 1700e-9        # m
x = 2 * np.pi/lambda0 * d/2
geometric_cross_section = np.pi * d**2/4 * 1e4  # cm**2
qext, qsca, qback, g = mp.mie(m,x)
sigma_sca = geometric_cross_section * qext * mp.i_unpolarized(m,x,mu)
plt.semilogy(theta, sigma_sca*1e-3, color='blue')
plt.text(15, sigma_sca[0]*3e-4, "%.0fnm\n(x10$^{-3}$)" % (d*1e9), color='blue')

d = 170e-9         # m
x = 2 * np.pi/lambda0 * d/2
geometric_cross_section = np.pi * d**2/4 * 1e4  # cm**2
qext, qsca, qback, g = mp.mie(m,x)
sigma_sca = geometric_cross_section * qext * mp.i_unpolarized(m,x,mu)
plt.semilogy(theta, sigma_sca, color='red')
plt.text(110, sigma_sca[-1]/2, "%.0fnm" % (d*1e9), color='red')

d = 17e-9                                                       # m
x = 2 * np.pi/lambda0 * d/2
geometric_cross_section = np.pi * d**2/4 * 1e4                 # cm**2
qext, qsca, qback, g = mp.mie(m,x)
sigma_sca = geometric_cross_section * qext * mp.i_unpolarized(m,x,mu)
plt.semilogy(theta, sigma_sca*1e6, color='green')
plt.text(130, sigma_sca[-1]*1e6, "(x10$^6$)\n%.0fnm" % (d*1e9), color='green')

plt.title("Refractive index m=1.4, $\lambda$=532nm")
plt.xlabel("Scattering Angle (degrees)")
plt.ylabel("Diff. Scattering Cross Section (cm$^2$/sr)")
plt.grid(True)
plt.show()
```

## Normalization revisited

### Evenly spaced $\mu=\cos\theta$

Start with uniformly distributed scattering angles that are evenly spaced over the cosine of the scattered angle.

#### Verifying normalization numerically

Specifically, to ensure proper normalization, the integral of the scattering function over all solid angles must be unity

$$
a = \int_0^{2\pi}\int_0^\pi \, p(\theta,\phi)\,\sin\theta\,d\theta\,d\phi
$$

or with a change of variables $\mu=\cos\theta$ and using the symmetry to the integral in $\phi$

$$
a = 2\pi \int_{-1}^1 \, p(\mu)\,d\mu
$$

This integral can be done numerically by simply summing all the rectangles

$$
a = 2\pi \sum_{i=0}^N p(\mu_i)\,\Delta\mu_i
$$

and if all the rectanges have the same width

$$
a = 2\pi\Delta\mu \sum_{i=0}^N p(\mu_i)
$$

#### Case 1. n=1.5, x=1

The total integral `total=` in the title should match the albedo `a=`.

For this non-strongly peaked scattering function, the simple integration remains close to the expected value.

```python
m = 1.5
x = 1
mu = np.linspace(-1,1,501)
intensity = mp.i_unpolarized(m,x,mu)
qext, qsca, qback, g = mp.mie(m,x)
a = qsca/qext

#integrate over all angles
dmu = mu[1] - mu[0]
total = 2 * np.pi * dmu * np.sum(intensity)

plt.plot(mu,intensity)
plt.xlabel(r'$\cos(\theta)$')
plt.ylabel('Unpolarized Scattering Intensity [1/sr]')
plt.title('m=%.3f%+.3fj, x=%.2f, a=%.3f, total=%.3f'%(m.real,m.imag,x,a, total))
plt.show()
```

#### Case 2: m=1.5-1.5j, x=1

Aagin the total integral `total=` in the title should match the albedo `a=`.

For this non-strongly peaked scattering function, the simple integration remains close to the expected value.

```python
m = 1.5 - 1.5j
x = 1
mu = np.linspace(-1,1,501)
intensity = mp.i_unpolarized(m,x,mu)
qext, qsca, qback, g = mp.mie(m,x)
a = qsca/qext

#integrate over all angles
dmu = mu[1] - mu[0]
total = 2 * np.pi * dmu * np.sum(intensity)

plt.plot(mu,intensity)
plt.xlabel(r'$\cos(\theta)$')
plt.ylabel('Unpolarized Scattering Intensity [1/sr]')
plt.title('m=%.3f%+.3fj, x=%.2f, a=%.3f, total=%.3f'%(m.real,m.imag,x,a, total))
plt.show()
```

## Normalization, evenly spaced $\theta$

The total integral total in the title should match the albedo $a$.

For this non-strongly peaked scattering function, even spacing in $\theta$ improves the accuracy of the integration.

```python
m = 1.5-1.5j
x = 1
theta = np.linspace(0,180,361)*np.pi/180
mu = np.cos(theta)

intensity = mp.i_unpolarized(m,x,mu)
qext, qsca, qback, g = mp.mie(m,x)
a = qsca/qext

#integrate over all angles
dtheta = theta[1]-theta[0]
total = 2 * np.pi * dtheta * np.sum(intensity* np.sin(theta))

plt.plot(mu,intensity)
plt.xlabel(r'$\cos(\theta)$')
plt.ylabel('Unpolarized Scattering Intensity [1/sr]')
plt.title('m=%.3f%+.3fj, x=%.2f, a=%.3f, total=%.3f'%(m.real,m.imag,x,a, total))
plt.show()

```

## Comparison to Wiscombe's Mie Program

Wiscombe normalizes as

$$
\int_{4\pi} p(\theta,\phi) \,d\Omega = \pi x^2  Q_{sca}
$$

where $p(\theta)$ is the scattered light.

Once corrected for differences in phase function normalization, Wiscombe's test cases match those from `miepython` exactly.

### Wiscombe's Test Case 14

```python
"""
MIEV0 Test Case 14:  Refractive index:  real    1.500  imag  -1.000E+00,  Mie size parameter =     1.000
  Angle    Cosine           S-sub-1                     S-sub-2               Intensity  Deg of Polzn
   0.00  1.000000   5.84080E-01   1.90515E-01   5.84080E-01   1.90515E-01   3.77446E-01        0.0000
  30.00  0.866025   5.65702E-01   1.87200E-01   5.00161E-01   1.45611E-01   3.13213E-01       -0.1336
  60.00  0.500000   5.17525E-01   1.78443E-01   2.87964E-01   4.10540E-02   1.92141E-01       -0.5597
  90.00  0.000000   4.56340E-01   1.67167E-01   3.62285E-02  -6.18265E-02   1.20663E-01       -0.9574
"""

x=1.0
m=1.5-1.0j
mu=np.cos(np.linspace(0,90,4) * np.pi/180)

qext, qsca, qback, g = mp.mie(m,x)
albedo = qsca/qext
unpolar = mp.i_unpolarized(m,x,mu) # normalized to a
unpolar /= albedo                  # normalized to 1

unpolar_miev = np.array([3.77446E-01,3.13213E-01,1.92141E-01,1.20663E-01])
unpolar_miev /= np.pi * qsca * x**2                         # normalized to 1
ratio = unpolar_miev/unpolar

print("MIEV0 Test Case 14:  m=1.500-1.000j,  Mie size parameter =     1.000")
print()
print("             %9.1f°%9.1f°%9.1f°%9.1f°"%(0,30,60,90))
print("MIEV0        %9.5f %9.5f %9.5f %9.5f"%(unpolar_miev[0],unpolar_miev[1],unpolar_miev[2],unpolar_miev[3]))
print("miepython    %9.5f %9.5f %9.5f %9.5f"%(unpolar[0],unpolar[1],unpolar[2],unpolar[3]))
print("ratio        %9.5f %9.5f %9.5f %9.5f"%(ratio[0],ratio[1],ratio[2],ratio[3]))
```

    MIEV0 Test Case 14:  m=1.500-1.000j,  Mie size parameter =     1.000

                       0.0°     30.0°     60.0°     90.0°
    MIEV0          0.18109   0.15027   0.09218   0.05789
    miepython      0.18109   0.15027   0.09218   0.05789
    ratio          1.00000   1.00000   1.00000   1.00000

### Wiscombe's Test Case 10

```python
"""
MIEV0 Test Case 10:  Refractive index:  real    1.330  imag  -1.000E-05,  Mie size parameter =   100.000
  Angle    Cosine           S-sub-1                     S-sub-2               Intensity  Deg of Polzn
   0.00  1.000000   5.25330E+03  -1.24319E+02   5.25330E+03  -1.24319E+02   2.76126E+07        0.0000
  30.00  0.866025  -5.53457E+01  -2.97188E+01  -8.46720E+01  -1.99947E+01   5.75775E+03        0.3146
  60.00  0.500000   1.71049E+01  -1.52010E+01   3.31076E+01  -2.70979E+00   8.13553E+02        0.3563
  90.00  0.000000  -3.65576E+00   8.76986E+00  -6.55051E+00  -4.67537E+00   7.75217E+01       -0.1645
"""

x=100.0
m=1.33-1e-5j
mu=np.cos(np.linspace(0,90,4) * np.pi/180)

qext, qsca, qback, g = mp.mie(m,x)
albedo = qsca/qext
unpolar = mp.i_unpolarized(m,x,mu) # normalized to a
unpolar /= albedo                  # normalized to 1

unpolar_miev = np.array([2.76126E+07,5.75775E+03,8.13553E+02,7.75217E+01])
unpolar_miev /= np.pi * qsca * x**2                         # normalized to 1
ratio = unpolar_miev/unpolar

print("MIEV0 Test Case 10:  m=1.330-0.00001j,  Mie size parameter =     100.000")
print()
print("             %9.1f°%9.1f°%9.1f°%9.1f°"%(0,30,60,90))
print("MIEV0        %9.5f %9.5f %9.5f %9.5f"%(unpolar_miev[0],unpolar_miev[1],unpolar_miev[2],unpolar_miev[3]))
print("miepython    %9.5f %9.5f %9.5f %9.5f"%(unpolar[0],unpolar[1],unpolar[2],unpolar[3]))
print("ratio        %9.5f %9.5f %9.5f %9.5f"%(ratio[0],ratio[1],ratio[2],ratio[3]))
```

    MIEV0 Test Case 10:  m=1.330-0.00001j,  Mie size parameter =     100.000

                       0.0°     30.0°     60.0°     90.0°
    MIEV0        419.22116   0.08742   0.01235   0.00118
    miepython    419.22168   0.08742   0.01235   0.00118
    ratio          1.00000   1.00000   1.00000   1.00000

### Wiscombe's Test Case 7

```python
"""
MIEV0 Test Case  7:  Refractive index:  real    0.750  imag   0.000E+00,  Mie size parameter =    10.000
  Angle    Cosine           S-sub-1                     S-sub-2               Intensity  Deg of Polzn
   0.00  1.000000   5.58066E+01  -9.75810E+00   5.58066E+01  -9.75810E+00   3.20960E+03        0.0000
  30.00  0.866025  -7.67288E+00   1.08732E+01  -1.09292E+01   9.62967E+00   1.94639E+02        0.0901
  60.00  0.500000   3.58789E+00  -1.75618E+00   3.42741E+00   8.08269E-02   1.38554E+01       -0.1517
  90.00  0.000000  -1.78590E+00  -5.23283E-02  -5.14875E-01  -7.02729E-01   1.97556E+00       -0.6158
"""

x=10.0
m=0.75
mu=np.cos(np.linspace(0,90,4) * np.pi/180)

qext, qsca, qback, g = mp.mie(m,x)
albedo = qsca/qext
unpolar = mp.i_unpolarized(m,x,mu) # normalized to a
unpolar /= albedo                  # normalized to 1

unpolar_miev = np.array([3.20960E+03,1.94639E+02,1.38554E+01,1.97556E+00])
unpolar_miev /= np.pi * qsca * x**2                          # normalized to 1
ratio = unpolar_miev/unpolar

print("MIEV0 Test Case 7:  m=0.75,  Mie size parameter =     10.000")
print()
print("             %9.1f°%9.1f°%9.1f°%9.1f°"%(0,30,60,90))
print("MIEV0        %9.5f %9.5f %9.5f %9.5f"%(unpolar_miev[0],unpolar_miev[1],unpolar_miev[2],unpolar_miev[3]))
print("miepython    %9.5f %9.5f %9.5f %9.5f"%(unpolar[0],unpolar[1],unpolar[2],unpolar[3]))
print("ratio        %9.5f %9.5f %9.5f %9.5f"%(ratio[0],ratio[1],ratio[2],ratio[3]))
```

    MIEV0 Test Case 7:  m=0.75,  Mie size parameter =     10.000

                       0.0°     30.0°     60.0°     90.0°
    MIEV0          4.57673   0.27755   0.01976   0.00282
    miepython      4.57673   0.27755   0.01976   0.00282
    ratio          1.00000   1.00000   1.00000   1.00000

## Comparison to Bohren & Huffmans's Mie Program

Bohren & Huffman normalizes as

$$
\int_{4\pi} p(\theta,\phi) \,d\Omega = 4 \pi x^2  Q_{sca}
$$

### Bohren & Huffmans's Test Case 14

```python
"""
BHMie Test Case 14, Refractive index = 1.5000-1.0000j, Size parameter = 1.0000

Angle  Cosine              S1                        S2
0.00   1.0000  -8.38663e-01 -8.64763e-01  -8.38663e-01 -8.64763e-01
0.52   0.8660  -8.19225e-01 -8.61719e-01  -7.21779e-01 -7.27856e-01
1.05   0.5000  -7.68157e-01 -8.53697e-01  -4.19454e-01 -3.72965e-01
1.57   0.0000  -7.03034e-01 -8.43425e-01  -4.44461e-02  6.94424e-02
"""

x=1.0
m=1.5-1j
mu=np.cos(np.linspace(0,90,4) * np.pi/180)

qext, qsca, qback, g = mp.mie(m,x)
albedo = qsca/qext
unpolar = mp.i_unpolarized(m,x,mu) # normalized to a
unpolar /= albedo                  # normalized to 1

s1_bh = np.empty(4,dtype=np.complex)
s1_bh[0] = -8.38663e-01 - 8.64763e-01*1j
s1_bh[1] = -8.19225e-01 - 8.61719e-01*1j
s1_bh[2] = -7.68157e-01 - 8.53697e-01*1j
s1_bh[3] = -7.03034e-01 - 8.43425e-01*1j

s2_bh = np.empty(4,dtype=np.complex)
s2_bh[0] = -8.38663e-01 - 8.64763e-01*1j
s2_bh[1] = -7.21779e-01 - 7.27856e-01*1j
s2_bh[2] = -4.19454e-01 - 3.72965e-01*1j
s2_bh[3] = -4.44461e-02 + 6.94424e-02*1j

# BHMie seems to normalize their intensities to 4 * pi * x**2 * Qsca
unpolar_bh = (abs(s1_bh)**2+abs(s2_bh)**2)/2
unpolar_bh /= np.pi * qsca * 4 * x**2                          # normalized to 1
ratio = unpolar_bh/unpolar

print("BHMie Test Case 14:  m=1.5000-1.0000j, Size parameter = 1.0000")
print()
print("             %9.1f°%9.1f°%9.1f°%9.1f°"%(0,30,60,90))
print("BHMIE        %9.5f %9.5f %9.5f %9.5f"%(unpolar_bh[0],unpolar_bh[1],unpolar_bh[2],unpolar_bh[3]))
print("miepython    %9.5f %9.5f %9.5f %9.5f"%(unpolar[0],unpolar[1],unpolar[2],unpolar[3]))
print("ratio        %9.5f %9.5f %9.5f %9.5f"%(ratio[0],ratio[1],ratio[2],ratio[3]))
print()
print("Note that this test is identical to MIEV0 Test Case 14 above.")
print()
print("Wiscombe's code is much more robust than Bohren's so I attribute errors all to Bohren")
```

    BHMie Test Case 14:  m=1.5000-1.0000j, Size parameter = 1.0000

                       0.0°     30.0°     60.0°     90.0°
    BHMIE          0.17406   0.14780   0.09799   0.07271
    miepython      0.18109   0.15027   0.09218   0.05789
    ratio          0.96118   0.98353   1.06296   1.25600

    Note that this test is identical to MIEV0 Test Case 14 above.

    Wiscombe's code is much more robust than Bohren's so I attribute errors all to Bohren

### Bohren & Huffman, water droplets

Tiny water droplet (0.26 microns) in clouds has pretty strong forward scattering!  A graph of this is figure 4.9 in Bohren and Huffman's *Absorption and Scattering of Light by Small Particles*.

A bizarre scaling factor of $16\pi$ is needed to make the `miepython` results match those in the figure 4.9.

```python
x=3
m=1.33-1e-8j

theta = np.linspace(0,180,181)
mu = np.cos(theta*np.pi/180)

scaling_factor = 16*np.pi
iper = scaling_factor*mp.i_per(m,x,mu)
ipar = scaling_factor*mp.i_par(m,x,mu)

P = (iper-ipar)/(iper+ipar)

plt.subplots(2,1,figsize=(8,8))
plt.subplot(2,1,1)
plt.semilogy(theta,ipar,label='$i_{par}$')
plt.semilogy(theta,iper,label='$i_{per}$')
plt.xlim(0,180)
plt.xticks(range(0,181,30))
plt.ylabel('i$_{par}$ and i$_{per}$')
plt.legend()
plt.title('Figure 4.9 from Bohren & Huffman')
plt.subplot(2,1,2)
plt.plot(theta,P)
plt.ylim(-1,1)
plt.xticks(range(0,181,30))
plt.xlim(0,180)
plt.ylabel('Polarization')
plt.plot([0,180],[0,0],':k')
plt.xlabel('Angle (Degrees)')
plt.show()
```

## van de Hulst Comparison

This graph (see figure 29 in *Light Scattering by Small Particles*) was obviously constructed by hand.  In this graph, van de Hulst worked hard to get as much information as possible

```python
x=5
m=10000

theta = np.linspace(0,180,361)
mu = np.cos(theta*np.pi/180)

fig, ax = plt.subplots(figsize=(8,8))

x=10
s1,s2 = mp.mie_S1_S2(m,x,mu)
sone = 2.5*abs(s1)
stwo = 2.5*abs(s2)
plt.plot(theta,sone,'b')
plt.plot(theta,stwo,'--r')
plt.annotate('x=%.1f  '%x,xy=(theta[-1],sone[-1]),ha='right',va='bottom')

x=5
s1,s2 = mp.mie_S1_S2(m,x,mu)
sone = 2.5*abs(s1) + 1
stwo = 2.5*abs(s2) + 1
plt.plot(theta,sone,'b')
plt.plot(theta,stwo,'--r')
plt.annotate('x=%.1f  '%x,xy=(theta[-1],sone[-1]),ha='right',va='bottom')

x=3
s1,s2 = mp.mie_S1_S2(m,x,mu)
sone = 2.5*abs(s1) + 2
stwo = 2.5*abs(s2) + 2
plt.plot(theta,sone,'b')
plt.plot(theta,stwo,'--r')
plt.annotate('x=%.1f  '%x,xy=(theta[-1],sone[-1]),ha='right',va='bottom')

x=1
s1,s2 = mp.mie_S1_S2(m,x,mu)
sone = 2.5*abs(s1) + 3
stwo = 2.5*abs(s2) + 3
plt.plot(theta,sone,'b')
plt.plot(theta,stwo,'--r')
plt.annotate('x=%.1f  '%x,xy=(theta[-1],sone[-1]),ha='right',va='bottom')

x=0.5
s1,s2 = mp.mie_S1_S2(m,x,mu)
sone = 2.5*abs(s1) + 4
stwo = 2.5*abs(s2) + 4
plt.plot(theta,sone,'b')
plt.plot(theta,stwo,'--r')
plt.annotate('x=%.1f  '%x,xy=(theta[-1],sone[-1]),ha='right',va='bottom')

plt.xlim(0,180)
plt.ylim(0,5.5)
plt.xticks(range(0,181,30))
plt.yticks(np.arange(0,5.51,0.5))
plt.title('Figure 29 from van de Hulst, Non-Absorbing Spheres')
plt.xlabel('Angle (Degrees)')
ax.set_yticklabels(['0','1/2','0','1/2','0','1/2','0','1/2','0','1/2','5',' '])
plt.grid(True)
plt.show()
```

## Comparisons with Kerker, Angular Gain

Another interesting graph is figure 4.51 from *The Scattering of Light* by Kerker.

The angular gain is
$$
G_1 = \frac{4}{x^2} |S_1(\theta)|^2
\qquad\mbox{and}\qquad
G_2 = \frac{4}{x^2} |S_2(\theta)|^2
$$

```python
## Kerker, Angular Gain

x=1
m=10000

theta = np.linspace(0,180,361)
mu = np.cos(theta*np.pi/180)

fig, ax = plt.subplots(figsize=(8,8))

s1,s2 = mp.mie_S1_S2(m,x,mu)

G1 = 4*abs(s1)**2/x**2
G2 = 4*abs(s2)**2/x**2

plt.plot(theta,G1,'b')
plt.plot(theta,G2,'--r')
plt.annotate('$G_1$',xy=(50,0.36),color='blue',fontsize=14)
plt.annotate('$G_2$',xy=(135,0.46),color='red',fontsize=14)

plt.xlim(0,180)
plt.xticks(range(0,181,30))
plt.title('Figure 4.51 from Kerker, Non-Absorbing Spheres, x=1')
plt.xlabel('Angle (Degrees)')
plt.ylabel('Angular Gain')
plt.show()
```

```python

```