```python
from astropy.io import fits
from astropy.table import Table

import numpy as np
import numpy.ma as ma

import matplotlib.pyplot as plt
```

```python
G = 6.674e-11 # m^3 kg^-1 s^-2
c = 3e5 # km/s
Msun = 1.989e30 # kg

h = 1
H0 = 100*h # km/s/Mpc
```

# Import galaxy data

```python
data_directory = '/Users/kellydouglass/Documents/Research/data/SDSS/dr16/manga/spectro/analysis/v2_4_3/2.2.1/'
DRP_directory = '/Users/kellydouglass/Documents/Research/data/SDSS/dr16/manga/spectro/redux/v2_4_3/'
```

```python
galaxy_ID = '7443-1902'

plate, fiber = galaxy_ID.split('-')
galaxy_data = fits.open(data_directory + 'HYB10-GAU-MILESHC/' + plate + '/' + fiber + '/manga-' + galaxy_ID + '-MAPS-HYB10-GAU-MILESHC.fits.gz')

# Extract H-alpha velocity map (C19)
Ha_vel = galaxy_data['EMLINE_GVEL'].data[18]

# H-alpha mask extension
Ha_mask_extension = galaxy_data['EMLINE_GVEL'].header['QUALDATA']

# Extract stellar velocity map
star_vel = galaxy_data['STELLAR_VEL'].data

# Stellar velocity mask extension
star_vel_mask_extension = galaxy_data['STELLAR_VEL'].header['QUALDATA']

# Extract (and correct for instrumental resolution effects) stellar velocity dispersion map
star_sigma = np.sqrt(galaxy_data['STELLAR_SIGMA'].data**2
                     - galaxy_data['STELLAR_SIGMACORR'].data**2)
# See https://www.sdss.org/dr16/manga/manga-data/working-with-manga-data/ for correction
# details

# Stellar velocity dispersion mask extension
star_sigma_mask_extension = galaxy_data['STELLAR_SIGMA'].header['QUALDATA']

# Extract stellar mass density map
#sMass_density = galaxy_data[1].data[19]

# Extract average mass-to-light ratio map
#MtoL = galaxy_data[1].data[17]

# Extract average r-band image
r_band = galaxy_data['SPX_MFLUX'].data
```

    /Users/kellydouglass/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:20: RuntimeWarning: invalid value encountered in sqrt

```python
galaxy_data.info()
```

    Filename: /Users/kellydouglass/Documents/Research/data/SDSS/dr16/manga/spectro/analysis/v2_4_3/2.2.1/HYB10-GAU-MILESHC/7443/1902/manga-7443-1902-MAPS-HYB10-GAU-MILESHC.fits.gz
    No.    Name      Ver    Type      Cards   Dimensions   Format
      0  PRIMARY       1 PrimaryHDU     146   ()
      1  SPX_SKYCOO    1 ImageHDU        43   (32, 32, 2)   float32
      2  SPX_ELLCOO    1 ImageHDU        46   (32, 32, 3)   float32
      3  SPX_MFLUX     1 ImageHDU        36   (32, 32)   float32
      4  SPX_MFLUX_IVAR    1 ImageHDU        37   (32, 32)   float32
      5  SPX_SNR       1 ImageHDU        34   (32, 32)   float32
      6  BINID         1 ImageHDU        45   (32, 32, 5)   int32
      7  BIN_LWSKYCOO    1 ImageHDU        43   (32, 32, 2)   float32
      8  BIN_LWELLCOO    1 ImageHDU        46   (32, 32, 3)   float32
      9  BIN_AREA      1 ImageHDU        35   (32, 32)   float32
     10  BIN_FAREA     1 ImageHDU        34   (32, 32)   float32
     11  BIN_MFLUX     1 ImageHDU        37   (32, 32)   float32
     12  BIN_MFLUX_IVAR    1 ImageHDU        38   (32, 32)   float32
     13  BIN_MFLUX_MASK    1 ImageHDU        37   (32, 32)   int32
     14  BIN_SNR       1 ImageHDU        34   (32, 32)   float32
     15  STELLAR_VEL    1 ImageHDU        37   (32, 32)   float32
     16  STELLAR_VEL_IVAR    1 ImageHDU        38   (32, 32)   float32
     17  STELLAR_VEL_MASK    1 ImageHDU        37   (32, 32)   int32
     18  STELLAR_SIGMA    1 ImageHDU        37   (32, 32)   float32
     19  STELLAR_SIGMA_IVAR    1 ImageHDU        38   (32, 32)   float32
     20  STELLAR_SIGMA_MASK    1 ImageHDU        37   (32, 32)   int32
     21  STELLAR_SIGMACORR    1 ImageHDU        35   (32, 32)   float32
     22  STELLAR_CONT_FRESID    1 ImageHDU        42   (32, 32, 2)   float32
     23  STELLAR_CONT_RCHI2    1 ImageHDU        34   (32, 32)   float32
     24  EMLINE_SFLUX    1 ImageHDU        65   (32, 32, 22)   float32
     25  EMLINE_SFLUX_IVAR    1 ImageHDU        66   (32, 32, 22)   float32
     26  EMLINE_SFLUX_MASK    1 ImageHDU        65   (32, 32, 22)   int32
     27  EMLINE_SEW    1 ImageHDU        65   (32, 32, 22)   float32
     28  EMLINE_SEW_IVAR    1 ImageHDU        66   (32, 32, 22)   float32
     29  EMLINE_SEW_MASK    1 ImageHDU        65   (32, 32, 22)   int32
     30  EMLINE_GFLUX    1 ImageHDU        65   (32, 32, 22)   float32
     31  EMLINE_GFLUX_IVAR    1 ImageHDU        66   (32, 32, 22)   float32
     32  EMLINE_GFLUX_MASK    1 ImageHDU        65   (32, 32, 22)   int32
     33  EMLINE_GEW    1 ImageHDU        65   (32, 32, 22)   float32
     34  EMLINE_GEW_IVAR    1 ImageHDU        66   (32, 32, 22)   float32
     35  EMLINE_GEW_MASK    1 ImageHDU        65   (32, 32, 22)   int32
     36  EMLINE_GVEL    1 ImageHDU        65   (32, 32, 22)   float32
     37  EMLINE_GVEL_IVAR    1 ImageHDU        66   (32, 32, 22)   float32
     38  EMLINE_GVEL_MASK    1 ImageHDU        65   (32, 32, 22)   int32
     39  EMLINE_GSIGMA    1 ImageHDU        65   (32, 32, 22)   float32
     40  EMLINE_GSIGMA_IVAR    1 ImageHDU        66   (32, 32, 22)   float32
     41  EMLINE_GSIGMA_MASK    1 ImageHDU        65   (32, 32, 22)   int32
     42  EMLINE_INSTSIGMA    1 ImageHDU        63   (32, 32, 22)   float32
     43  EMLINE_TPLSIGMA    1 ImageHDU        63   (32, 32, 22)   float32
     44  SPECINDEX     1 ImageHDU       134   (32, 32, 46)   float32
     45  SPECINDEX_IVAR    1 ImageHDU       135   (32, 32, 46)   float32
     46  SPECINDEX_MASK    1 ImageHDU        89   (32, 32, 46)   int32
     47  SPECINDEX_CORR    1 ImageHDU        86   (32, 32, 46)   float32

```python
'''
master_filename = '../master_file.txt'

general_data = Table.read(master_filename, format='ascii.ecsv')
'''

drp_filename = DRP_directory + 'drpall-v2_4_3.fits'

general_data = Table.read(drp_filename, format='fits')
```

```python
general_data.colnames
```

    ['plate',
     'ifudsgn',
     'plateifu',
     'mangaid',
     'versdrp2',
     'versdrp3',
     'verscore',
     'versutil',
     'versprim',
     'platetyp',
     'srvymode',
     'objra',
     'objdec',
     'ifuglon',
     'ifuglat',
     'ifura',
     'ifudec',
     'ebvgal',
     'nexp',
     'exptime',
     'drp3qual',
     'bluesn2',
     'redsn2',
     'harname',
     'frlplug',
     'cartid',
     'designid',
     'cenra',
     'cendec',
     'airmsmin',
     'airmsmed',
     'airmsmax',
     'seemin',
     'seemed',
     'seemax',
     'transmin',
     'transmed',
     'transmax',
     'mjdmin',
     'mjdmed',
     'mjdmax',
     'gfwhm',
     'rfwhm',
     'ifwhm',
     'zfwhm',
     'mngtarg1',
     'mngtarg2',
     'mngtarg3',
     'catidnum',
     'plttarg',
     'manga_tileid',
     'nsa_iauname',
     'ifudesignsize',
     'ifutargetsize',
     'ifudesignwrongsize',
     'z',
     'zmin',
     'zmax',
     'szmin',
     'szmax',
     'ezmin',
     'ezmax',
     'probs',
     'pweight',
     'psweight',
     'psrweight',
     'sweight',
     'srweight',
     'eweight',
     'esweight',
     'esrweight',
     'nsa_field',
     'nsa_run',
     'nsa_camcol',
     'nsa_version',
     'nsa_nsaid',
     'nsa_nsaid_v1b',
     'nsa_z',
     'nsa_zdist',
     'nsa_sersic_absmag',
     'nsa_elpetro_absmag',
     'nsa_elpetro_amivar',
     'nsa_sersic_mass',
     'nsa_elpetro_mass',
     'nsa_elpetro_ba',
     'nsa_elpetro_phi',
     'nsa_extinction',
     'nsa_elpetro_th50_r',
     'nsa_petro_th50',
     'nsa_petro_flux',
     'nsa_petro_flux_ivar',
     'nsa_elpetro_flux',
     'nsa_elpetro_flux_ivar',
     'nsa_sersic_ba',
     'nsa_sersic_n',
     'nsa_sersic_phi',
     'nsa_sersic_th50',
     'nsa_sersic_flux',
     'nsa_sersic_flux_ivar']

```python
general_data['nsa_elpetro_th50_r']

general_data['plateifu']

general_data['nsa_elpetro_absmag'][10][4]
```

    -18.5437

# Mask "bad" data

H$\alpha$ velocity map

```python
mHa_vel = ma.array(Ha_vel, mask=galaxy_data[Ha_mask_extension].data[18]>0)

Ha_vel_max = np.max(np.abs(mHa_vel))
```

DAP stellar velocity map

```python
mStar_vel = ma.array(star_vel, mask=galaxy_data[star_vel_mask_extension].data>0)

star_vel_max = np.max(np.abs(mStar_vel))
```

DAP stellar velocity dispersion map

```python
mStar_sigma = ma.array(star_sigma, mask=galaxy_data[star_sigma_mask_extension].data>0)

mStar_sigma = ma.masked_invalid(mStar_sigma)
```

```python
mr_band = ma.masked_equal(r_band, 0)
```

# Plot velocity map

```python
%matplotlib notebook

plt.figure(figsize=(10,4))

plt.subplot(121)
plt.title(r'H$\alpha$ velocity map')
plt.imshow(mHa_vel, cmap='RdBu_r', origin='lower')
cbar1 = plt.colorbar()
plt.clim(-Ha_vel_max, Ha_vel_max)
cbar1.ax.set_ylabel(r'H$\alpha$ velocity [km/s]')
plt.xlabel('spaxel')
plt.ylabel('spaxel')

plt.subplot(122)
plt.title('Stellar velocity map')
plt.imshow(mStar_vel, cmap='RdBu', origin='lower')
cbar2 = plt.colorbar()
plt.clim(-star_vel_max, star_vel_max)
cbar2.ax.set_ylabel(r'$v_*$ [km/s]')
plt.xlabel('spaxel')
plt.ylabel('spaxel')

plt.tight_layout();
```

    <IPython.core.display.Javascript object>

# Velocity dispersion

```python
%matplotlib inline

plt.figure(figsize=(12,4))

plt.subplot(131)
plt.title(r'Distribution of $v^2$ in H$\alpha$')
plt.hist(mHa_vel.flatten()**2, bins=50)
plt.xlabel(r'H$\alpha$ 6564 Gaussian velocity [km/s]')

plt.subplot(132)
plt.title('Stellar velocity dispersion map')
plt.imshow(mStar_sigma, origin='lower')
cbar = plt.colorbar()
cbar.ax.set_ylabel(r'$\sigma_*$ [km/s]')
plt.xlabel('spaxel')
plt.ylabel('spaxel')

plt.subplot(133)
plt.title(r'Distribution of $\sigma_*$')
plt.hist(mStar_sigma.flatten(), bins=50)
plt.xlabel(r'$\sigma_*$ [km/s]')

plt.tight_layout();
```

Segev recommends that we use the median of the stellar velocity dispersion distribution for our mass calculations.

```python
sigma_Ha = np.mean(mHa_vel*mHa_vel)

sigma_Ha
```

    59150.18579234973

```python
avg_star_sigma = np.nanmean(mStar_sigma)

median_star_sigma = ma.median(mStar_sigma)

# Convert from km/s to m/s
median_star_sigma_mpers = median_star_sigma*1000

median_star_sigma
```

    166.4339

I should check this velocity dispersion against the Faber-Jackson relation.

For this galaxy, according to Figure 9 of Aguerri12, the median $\sigma_0$ is closer to the expected value for the velocity dispersion for a galaxy with this stellar mass than the central $\sigma_0$.

##### What if we use the velocity dispersion at the center of the galaxy?

```python
# Center of galaxy is defined as the spaxel with the maximum luminosity
center_spaxel = np.unravel_index(ma.argmax(mr_band, axis=None), mr_band.shape)

center_star_sigma = mStar_sigma[center_spaxel]

# Convert from km/s to m/s
center_star_sigma_mpers = center_star_sigma*1000

center_star_sigma
```

    257.1611

# Mass distribution

```python
plt.figure(figsize=(18,5))

plt.subplot(131)
plt.title('Mean r-band image')
plt.imshow(mr_band, origin='lower')
cbar1 = plt.colorbar()
cbar1.ax.set_ylabel(r'Average r-band flux [10$^{-17}$ erg/s/cm$^2$]')
'''
plt.subplot(132)
plt.title('Pipe3d stellar mass density')
plt.imshow(sMass_density)
cbar2 = plt.colorbar()
cbar2.ax.set_ylabel(r'Stellar mass density [log($M_\odot$)]')

plt.subplot(133)
plt.title('Pipe3d stellar mass-to-light ratio')
plt.imshow(MtoL)
cbar3 = plt.colorbar()
cbar3.ax.set_ylabel(r'M/L [log($M_\odot$/$L_\odot$)]')
''';
```

# Galaxy mass

We can use the virial theorem to calculate the total mass in an elliptical galaxy, since it is primarily supported by the stars' pressure rather than rotation.

\begin{equation}
  M \approx 2.5 \frac{\left< v^2 \right> r_h}{G} \approx 7.5 \frac{\sigma^2 r_h}{G}
\end{equation}

(Eqn. 20.17 from *Foundations of Astrophysics* by Ryden & Peterson) where $r_h$ is the half-light radius (really, the half-mass radius, but we cannot measure this).

So, first, we need to find $r_h$, presumably from the r-band image.

The total flux from the galaxy is not necessarily contained within the MaNGA footprint.  Ideally, then, we should use a total flux value from the legacy SDSS.  Then, we count out in consecutive annuli (using the NSA b/a, phi angles to start) until we reach 50% of this flux.  The resulting annulus is our $r_h$.

I have the 'NSA_index' in the master file, so I should be able to quickly extract other parameters from the NSA as needed.  At the moment, quantities that the NSA has that we might need for this analysis are:
* SERSICFLUX or nsa_sersic_flux - the 2D Sersic fit flux in FNugriz
* SERSIC_N or nsa_sersic_n - the 2D Sersic index
* SERSIC_BA or nsa_sersic_ba - the b/a axis ratio (already using)
* SERSIC_PHI or nsa_sersic_phi - the angle (E of N) of the major axis (already using)
* SERSIC_TH50 or nsa_sersic_th50 [arcsec] - 50% light radius along major axis from Sersic 2D fit

Or, we use the circular Petrosian fits:
* PETROTH50 or nsa_petro_th50 [arcsec] - 50% light radius from Petrosian fit

Or the **elliptical Petrosian** fits:
* **nsa_elpetro_th50_r**
* nsa_elpetro_ba
* nsa_elpetro_phi

## Locate galaxy in DRP table

```python
gal_idx = general_data['plateifu'] == galaxy_ID
```

```python
general_data['mangaid'][gal_idx]
```

<Column name='mangaid' dtype='bytes11' length=1>

12-49536

```python
Mstar = general_data[1].data['nsa_elpetro_mass'][gal_idx] # Solar masses
z = general_data[1].data['z'][gal_idx]
r_half = general_data[1].data['nsa_elpetro_th50_r'][gal_idx] # arcsec

# Convert arcsec to kpc
r_half_kpc = (1000*c*z/H0)*np.tan(r_half*np.pi/(60*60*180))

# Convert kpc to m
r_half_m = 3.0857e19*r_half_kpc
```

## Calculate mass

```python
def virial_mass(star_sigma, r_half):
    '''
    Calculate the virial mass of a galaxy

    PARAMETERS
    ==========

    star_sigma : float or numpy array
        Stellar velocity dispersion in m/s

    r_half : float or numpy array
        Half-light radius in m

    RETURNS
    =======

    Mvir_Msun : float or numpy array
        Virial mass in solar masses
    '''

    # Virial mass (in kg)
    Mvir = 7.5*star_sigma*star_sigma*r_half/G

    # Convert kg to Msun
    Mvir_Msun = Mvir/Msun

    return Mvir_Msun
```

#### Using median velocity dispersion

```python
Mtot_Msun_median = virial_mass(median_star_sigma_mpers, r_half_m)

Mdark_median = Mtot_Msun_median - Mstar
```

#### Using central velocity dispersion

```python
Mtot_Msun_center = virial_mass(center_star_sigma_mpers, r_half_m)

Mdark_center = Mtot_Msun_center - Mstar
```

# Mass ratio

```python
MassRatio_median = Mdark_median/Mstar

MassRatio_median
```

    array([1.51404181])

```python
MassRatio_center = Mdark_center/Mstar

MassRatio_center
```

    array([5.00204821])

```python
np.log10(Mstar)
```

    array([10.68767073])

```python
np.log10(median_star_sigma)
```

    2.2212417

```python

```


[Leo's Home page](https://leomrocha.github.com) -- [Github Page](https://github.com/leomrocha/minibrain/blob/master/predictors/sequence/midi_encoding.ipynb) -- License: [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)

# MIDI encoding analysis

[Leonardo M. Rocha](https://leomrocha.github.com)

[Contact Me](https://leomrocha.github.io/contact/)

This document contains a complete description step by step of the process and ideas behind the proposed encodings, it might be long.

The ideas here were inspired by the work I was doing to help with [Few Shot Music Generation](https://github.com/AI-ON/Few-Shot-Music-Generation) project. My fork is located [here](https://github.com/leomrocha/Few-Shot-Music-Generation)

## TLDR:

The encoding instead of being a huge one-hot vector (dimension 4708) it can be separated in several inputs each of them a one-hot vector of lower dimensionality and a time encoding given by a sum of sinusoids with different period.

The source code is provided to automatically do the encoding as a python package with numpy.

## Introduction

This current analysis tries to find another alternative encoding to try in conjunction with the previous one.

Current encoding based in the Magenta project has the following properties:
 - Time is detached as a separate TIME_SHIFT event
 - code is one hot for all the elements giving vectors of dimension 4708

The idea of the current analysis is to discuss the following points:
 - Try to find a code that takes less space
 - Argue that alternative time-coding could be worth trying

The current encoding separates a TIME_SHIFT event as a separate event, in my opinion the time of an event is an intrinsic property of it, so separating it might make more difficult for the network to learn what the time means.

The other issue is that similar structures (in music as well as in other kind of languages) might repeat at different time scales, the fact that the time event is treated as one separate event with it's own duration (actually 100 different types of TIME_SHIFT event) might make it harder to the network to detect patterns. This added to the high number of input features (4708) makes me think that the training time and number of samples needed for the training might be big.

```python
# current input one-hot vector dimension
16*128*2+32*16+100
```

    4708

In this study I propose another way of encoding that I would like to try reducing the number of features between 1 and 2 orders of magnitude and providing a Fourier series approximation inspired time and velocity approximation.

I know that is not a traditional encoding, but I think might be worth trying (specially as I don't have so many computational resources)

## Basic Idea Description

The basic idea is that a feature vector will be composed of several different features (7) encoded in different ways

From a midi file extract:

Event = [instrument, octave, note(s), action (on|off), velocity, time]

### Codes:

* instrument: (one|multi?) hot encoding: 16 instruments, many instruments can start or finish at the same time
* octave: (one|multi?) hot encoding: 11 octaves, many octaves can have an event at the same time
* note(s): (one|multi?) hot encoding: 12 notes per octave, many notes can start or finish at the same time
* action (on|off): one hot
* velocity: either a sampled (maybe 32 as the current approach) one-hot encoding, or a fourier inspired approach
* time: Fourier inspired approach to encoding

### Encoding Problems & Questions

As there might be more than 1 note, octave or instrument event at the same this could be dealt as:
1. doing an actual one-hot encoding and separating all the events, but writing the same event time -> can this confuse the network??
2. doing a multi-hot encoding and trying to find out a way ouf of this during the decoding stage (something like auto-correlation or a sort of dynamic-routing ? ) This would also mix the signals of the velocity of each note, which is problematic for the purpose of the exercise.

### Features candidates for One-hot Encoding

As MIDI events are independent (each note on|off event of a single instrument is a different event) and for the purpose of this encoding proposal the solution would be to do actual one-hot encoding and separate all the events. A single event will correspond to a single instrument and note event with the corresponding velocity and time encoding.

```python
# computation of the number of octaves needed for the encoding
import numpy as np
np.ceil(128/12.)
```

    11.0

```python
# dimensionality analysis -> basic (one)-hot encoding
16 + 11 + 12 + 2
```

    41

The basic coding of the event will contain a minimum of 41 dimensions to which must be added the encoding of the velocity and time encoding

### Velocity encoding

Both solutions are presented here, One-Hot Encoding and Fourier Inspired Encoding.

#### One-Hot encoding of Velocity

In this case the same number of velocities will be used as in the current approach, this means that the velocity vector will have a dimension of 32

#### Sinusoidal (Fourier Inspired) Encoding of Velocity

The resolution that is looked at is to be able to differentiate at least 32 different velocities.

As a description later on time encoding analysis will show, we can deal with a resolution of 256 velocities with $ 256 = 2^8 $ sinusoidals which would take the dimensionality of the input to $ 41 + 8 = 49 $

For a resolution of 32 velocities we need only $ 32 = 2^5 $ for a the dimensionality of the input to $ 41 + 5 = 46 $

```python
2**8, 2**5
```

    (256, 32)

## Time encoding

### Why not linear encoding?

Let's do a simple linear function, this can not hold as it diverges

Let's imaging an encoding where the time is cutted in 3 different parts:
* milliseconds
* seconds
* minutes

the functions will be linear in a range, but will have undefined elements at the borders, the derivatives are undefined there and the NNs are shown that they do not behave nice with these kind of functions

Then we can think of a continuous function, sinusoidal ones are really good for this, well studied and implemented in every standard library.

### Time resolution Analysis

Now let's think for a moment from the point of view of listening to  music (instead of time passing) for this we'd like a time resolution:

* maximum (desired) 10 milliseconds.
* minimum about 10 minutes ... ? let's take 10 minutes for the sake of this example (could be more or less depending on the needed domain)

_Note_: There are studies on real-time [latency issue](https://dl.acm.org/citation.cfm?id=1085892) but is not of our concern here.

For a perfect reconstruction the sampling frequency to discretize a function is given by the [Nyquist theorem](https://en.wikipedia.org/wiki/Nyquist%E2%80%93Shannon_sampling_theorem):

    If a function x(t) contains no frequencies higher than B hertz,
    it is completely determined by giving its ordinates at a series
    of points spaced 1/(2B) seconds apart.

Which means that for a *perfect* reconstruction for a given sample rate $fs$ is guaranteed for a *Bandlimit* $B < fs/2$

Let's imagine that we want to approximate the time for a minimum and maximum time interval, we could use sum of sinusoids to approximate it.

In our case $B = 100Hz $ for the maximum frequency (100 times 10 ms for 1 sec), hence the sampling rate would be *5ms*, taking in account that also there will be reconstruction errors and some other issues we could argue that $ fs<5ms $ let's say $ 4ms $ to be sure

And for the case of the 10 minutes one, this gives us $ 10*60 = 600 s $ so $ 1/600 ~ 0.00167Hz =>  fs < 0.000833Hz $

As a reference the human hearing resolution is about $ 20KHz $ and the CD sampling is $ 44.1KHz $

**With a sampling in each resolution level we should be able to reconstruct the patterns at different time-scales.**

_____

```python
f = (1./600)
fs = f/2
f,fs
```

    (0.0016666666666666668, 0.0008333333333333334)

### Time encoding details

#### Intuition

We are using time as reference, Not all the existing time but just a part of it, a range that is big enough to accomodate our work. In the case of music a few minutes (~10?) would suffice. In the case of lyrics they are even shorter if we count the number of letters than the number of milliseconds in a music piece.

Periodic functions should be easier for a NN to learn to include periodic elements (languages have periodic elements) than a linear one.

The Fourier Series can represent periodic functions with sums of sinusoidal functions.

To be able to represent a linear function with a Fourier Series, we can think of a sawtooth wave and just take only one period.

(TODO graph here)

#### Reference Formulas

*Euler's Formula*
\begin{equation}
 e^{ix} = cos(x) + i\sin(x)
\end{equation}

\begin{equation}
 cos(x) = \frac{e^{ix} + e^{-ix} } {2}
\end{equation}

\begin{equation}
 sin(x) = \frac{e^{ix} - e^{-ix} } {2 i }
\end{equation}

*Fourier Series*

\begin{equation}
f(x) = \sum_{n=-\infty}^\infty(c_n(f) e^{i2\pi {n/T} x} )
\end{equation}
where
\begin{equation}
c_n(f) = \frac{1}{T} \int_{-T/2}^{T/2} ( f(t) e^{-i2\pi {n/T} x} )
\end{equation}

In our case there is no need to go so far as using the Fourier Series but this gives an inspiration on how to encode the values.
Also we don't want (or even need):
 - the complete series $ [-\infty,\infty] $ but enough values that allows us to reconstruct well enough our function
 - to know the * Fourier Coefficients* $ c_n $ we don't need to compute the integral but we can estimate their values (done later)

We could use either the *Complex Domain* values or go for the *Discrete Cosine Transform* but we'll do something simpler that is choosing sinusoidals

Let's come back to time and think how we can encode the time, which is a linear function that just grows.

For this analysis we constrain the time to be only a segment between $ 0s $ and a max value $ Ms $ in seconds

To be able to recreate the original function we would need a series of functions that can represent elements that go from the maximum to the minimum resolution.

Let the first sinusoidal represent the values up to the period $ T = 5ms = 0.005s $ and the last one $ Ms $

Now let's work directly in milliseconds and forget about the unit, which for the purpose of the experiment does not affect the result as long as we can reconstruct the input correctly.

_____

#### Hypothesis .... (not proved yet and in working progress...)

_The main idea:_

To **mix Fourier Series with the Shannon sampling**, trying to reconstruct not only the input signal patterns at each time scale, but being able to reconstruct the time signal (a linear function) restrained to a period interpreting it as a SawTooth-Wave, which means that we should be able not only to give different time scales to the network but also be able to reconstruct the (locally) linear signal in an easy way, which should be needed to be able to re-encode a MIDI or text file from the output of the network.

We can encode any value between $ [0,5]ms $ , for the next we can duplicate the period $ T $ and we'll get the next resolution level. Advancing like this we need at least $ n $ values such as $ 2^n >= M $ and for the minimum resolution we need at most $ n $ values such as $ 2^n < 5 $

Finding the exponents can be quickly achieved with a bit shifting trick:

## WARNING

I've found an error in the reasoning (to verify) I need to do one approximation PER SCALE not for all together ....

```python
def get_upper_pow2(M):
    i = 1
    n = 0
    while (i <= M):
        i = i << 1
        n+=1
    return n,i

def get_lower_pow2(M):
    i = 1
    n = -1
    while (i <= M):
        i = i << 1
        n+=1
    return n,i
```

```python
for m in range (1,11): # iterating to show the vector dimension needed for each minute increment resolution
    M = m * 60 * 1000 # 10 minutes, 60 seconds per minute 1000 ms -> putting all into
    n,i = get_upper_pow2(M)
    print( m, n, i)
```

    1 16 65536
    2 17 131072
    3 18 262144
    4 18 262144
    5 19 524288
    6 19 524288
    7 19 524288
    8 19 524288
    9 20 1048576
    10 20 1048576

```python
# and the smallest resolution:
get_lower_pow2(5)
```

    (2, 8)

Which means that we can encode 10 minutes worth of music with a $ 20 - 2 = 18 $ dimensional vector, $ 36 $ if we count a complex-value vector.

For a total of $ 46 + 36 = 82 $ dimensions instead of $ 4708 $

Let's think as an encoding as the $sin$ and $cos$ values, and the network works directly on this domain giving only the $sin$ and $cos$ values as results, this allows the network to work in sinusoidal periodic functions instead of the time domain.

The decoding will take place just later to be able to do a reconstruction.

The series needed then is one representing a SawTooth Wave but restrained only to the time period we need. This allows us to represent periodic elements inside a linear function.

The other advantage of this function is that we can approximate it only with a real valued function, avoiding the complex domain and restraining the number of coefficients (and so dimensions) needed to only the real valued ones.

This gives is a total of $ 46 + 18 = 64 $ dimensions for a complete representation of the input vector

Which should be also great about this encoding is that if we create hierarchical representations, and each layer in the hierarchy corresponds to a different base resolution (ms, s, minutes, ...) then with limited amount of extra dimensions we could encode increasingly longer time periods.

_____

```python

def get_upper_pow2(M):
    i = 1
    n = 0
    while (i <= M):
        i = i << 1
        n+=1
    return n

def get_lower_pow2(m):
    i = 1
    n = -1
    while (i <= m):
        i = i << 1
        n+=1
    return n

class Lin2WaveEncoder():
    """
    Fourier Series Inspired Approximation of a linear function encoding.
    Takes the linear function as one period of a SawTooth-Wave and encodes
    the period with enough elements to approximate the function wiht the
    resolution needed for a (close to) perfect reconstruction of the input
    """
    def __init__(self, min_val, max_val, neg_allowed=True):
        """
        @param min_val: minimum value to encode
        @param max_val: maximum value to encode
        @param neg_allowed=True : allow negative values in the encoded vecto. If False
                        will move the sinusoids to the range [0,1]
        """
        self.min = min_val
        self.max = max_val
        self.neg_allowed = neg_allowed
        self.n_low = np.max(1, get_lower_pow2(min_val))
        self.n_high = get_upper_pow2(max_val)
        self.n = self.n_high - self.n_low
        # Each period indicates a resolution level, this allows for different (time) scales
        self.periods = [2**i for i in range(self.n_low, self.n_high)]  # choose periods acording to sampling -> maybe we should add 3rds and 5ths of each to allow for more complex time pattern compositions
        # Fourier divisor coefficients of the Series
        self.coefficients = [2*i for i in range(self.n_low, self.n_high)]  # just counting the elements
        print(self.coefficients, self.periods)

    def encode(self, x):
        """
        @param x: input vector to encode
        @return vector encoding the input x in the value points of x for the sinusoidal encoders
        """
        vec = []
        for n,T in zip(self.coefficients, self.periods):
            val = np.stack( np.sin(n * x / T) ) # base term of a SawTooth Wave of period T Fourier Series
            #val = np.stack( np.sin( x / T) ) # it seems to work better for the mix without the constant
            vec.append(val)
        ret = np.stack(vec)
        if not self.neg_allowed:
            ret = (ret + 1. ) / 2.  # pull all the encoded values in range [0,1]
        return ret

    def decode(self, vec):
        """
        @param vec: multi-dimensional input vector to decode into the original signal
        """
        x = 0
        scale_factor = []
        for n,T in zip(self.coefficients, self.periods):
            scale_factor.append((1./n)*T)  # Scale factor of each term
            #scale_factor.append(T)  # Scale factor of each term
        scale_factor = np.array(scale_factor)
        tx = vec
        tx = tx * scale_factor[:,None]
        x = np.sum(tx, axis=0)
        if not self.neg_allowed:  # here as is commutative to make less computations
            x = (x * 2.) -1 # go back to the original range
        return x

```

```python
we = Lin2WaveEncoder(1,1000000) # 10 seconds max -> 13 parameters
```

    [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38] [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288]

```python
l_short = np.linspace(0,np.pi,10000)  # for short periods the reconstruction works great -> this is already usable if the input is scaled to these kind of elements
l_long = np.linspace(0,10000,1000)  # for long periods this starts to work less and less well
```

```python
import matplotlib.pyplot as plt
%matplotlib inline
```

```python
plt.plot(l_short)
```

```python
plt.plot(l_long)
```

```python
el_short = we.encode(l_short)
el_long  = we.encode(l_long)
```

```python
el_short.shape, el_long.shape
```

    ((13, 10000), (13, 1000))

```python
dl_short = we.decode(l_short)
dl_long  = we.decode(l_long)
```

```python
dl_short.shape, dl_long.shape
```

    ((10000,), (1000,))

```python
plt.plot(dl_short)
```

```python
plt.clf()
plt.plot(dl_long)
```

**WARNING** : there is stil something wrong with the scale of the reconstruction.
Also, there is an issue that now I have not managed to reproduce and is the deformation that occurred before with the samples .... I have to find out what is there

```python
def fourier10000(x):
    ret = -10000*np.sin(np.pi*x/5000)/np.pi - 5000*np.sin(np.pi*x/2500)/np.pi - 10000*np.sin(3*np.pi*x/5000)/(3*np.pi) - 2500*np.sin(np.pi*x/1250)/np.pi - 2000*np.sin(np.pi*x/1000)/np.pi - 5000*np.sin(3*np.pi*x/2500)/(3*np.pi) - 10000*np.sin(7*np.pi*x/5000)/(7*np.pi) - 1250*np.sin(np.pi*x/625)/np.pi - 10000*np.sin(9*np.pi*x/5000)/(9*np.pi) - 1000*np.sin(np.pi*x/500)/np.pi
    return ret
```

```python
fel_short = fourier10000(l_short)
fel_long = fourier10000(l_long)
```

```python
plt.plot(fel_short)
```

```python
plt.plot(fel_long)
```

### Other tools (python)

There are already available libraries that can compute the fourier series approximation for a function, so this would make it easier to compute.

Here there are some tests with sympy to compute it.

```python
import sympy
from sympy import fourier_series, pi
from sympy import abc
from sympy.plotting import plot as spplot
```

```python
M = 1 # seconds
sm = fourier_series(abc.x, (abc.x, 0, M))
sm10k = fourier_series(abc.x, (abc.x, 0, 10000))
```

```python
sm.truncate(10)
```

    -sin(2*pi*x)/pi - sin(4*pi*x)/(2*pi) - sin(6*pi*x)/(3*pi) - sin(8*pi*x)/(4*pi) - sin(10*pi*x)/(5*pi) - sin(12*pi*x)/(6*pi) - sin(14*pi*x)/(7*pi) - sin(16*pi*x)/(8*pi) - sin(18*pi*x)/(9*pi) - sin(20*pi*x)/(10*pi)

```python
from sympy import lambdify
```

```python
fsm = lambdify(abc.x, sm.truncate(10), "numpy")
```

```python
fsm(1)
```

    7.796343665038751e-16

```python
sm.truncate(10).args
```

    (-sin(2*pi*x)/pi,
     -sin(4*pi*x)/(2*pi),
     -sin(6*pi*x)/(3*pi),
     -sin(8*pi*x)/(4*pi),
     -sin(10*pi*x)/(5*pi),
     -sin(12*pi*x)/(6*pi),
     -sin(14*pi*x)/(7*pi),
     -sin(16*pi*x)/(8*pi),
     -sin(18*pi*x)/(9*pi),
     -sin(20*pi*x)/(10*pi))

```python
sm.truncate(10).args[0].args
```

    (-1, 1/pi, sin(2*pi*x))

```python
sm10k.truncate(10).args
```

    (-10000*sin(pi*x/5000)/pi,
     -5000*sin(pi*x/2500)/pi,
     -2500*sin(pi*x/1250)/pi,
     -2000*sin(pi*x/1000)/pi,
     -1250*sin(pi*x/625)/pi,
     -1000*sin(pi*x/500)/pi,
     -10000*sin(3*pi*x/5000)/(3*pi),
     -10000*sin(7*pi*x/5000)/(7*pi),
     -10000*sin(9*pi*x/5000)/(9*pi),
     -5000*sin(3*pi*x/2500)/(3*pi))

```python
f1 = sm10k.truncate(10).args[0]
```

```python
np.product(f1.args[:-1])
```

    -10000/pi

```python
coef_fact = [ (np.product(f.args[:-1]),  lambdify(abc.x, f.args[-1], "numpy")  ) for f in sm10k.truncate(10).args]
```

```python
z1,z2 = zip(*coef_fact)
```

```python
z1
```

    (-10000/pi,
     -5000/pi,
     -2500/pi,
     -2000/pi,
     -1250/pi,
     -1000/pi,
     -10000/(3*pi),
     -10000/(7*pi),
     -10000/(9*pi),
     -5000/(3*pi))

```python
lambdify(abc.x, z1[0], "numpy")(0)
```

    -3183.098861837907

```python
pi.evalf()
```

    3.14159265358979

```python
sm10k.truncate(10)
```

    -10000*sin(pi*x/5000)/pi - 5000*sin(pi*x/2500)/pi - 10000*sin(3*pi*x/5000)/(3*pi) - 2500*sin(pi*x/1250)/pi - 2000*sin(pi*x/1000)/pi - 5000*sin(3*pi*x/2500)/(3*pi) - 10000*sin(7*pi*x/5000)/(7*pi) - 1250*sin(pi*x/625)/pi - 10000*sin(9*pi*x/5000)/(9*pi) - 1000*sin(pi*x/500)/pi

```python
sm.scale(10000).truncate(10)
```

    -10000*sin(2*pi*x)/pi - 5000*sin(4*pi*x)/pi - 10000*sin(6*pi*x)/(3*pi) - 2500*sin(8*pi*x)/pi - 2000*sin(10*pi*x)/pi - 5000*sin(12*pi*x)/(3*pi) - 10000*sin(14*pi*x)/(7*pi) - 1250*sin(16*pi*x)/pi - 10000*sin(18*pi*x)/(9*pi) - 1000*sin(20*pi*x)/pi

```python
sm.truncate(10).evalf(1)
```

    -0.3*sin(2*pi*x) - 0.2*sin(4*pi*x) - 0.1*sin(6*pi*x) - 0.08*sin(8*pi*x) - 0.06*sin(10*pi*x) - 0.05*sin(12*pi*x) - 0.05*sin(14*pi*x) - 0.04*sin(16*pi*x) - 0.04*sin(18*pi*x) - 0.03*sin(20*pi*x)

```python
s = fourier_series(abc.x, (abc.x, 0, pi))
```

```python
s.scale(10).truncate(10)
```

    -10*sin(2*x) - 5*sin(4*x) - 10*sin(6*x)/3 - 5*sin(8*x)/2 - 2*sin(10*x) - 5*sin(12*x)/3 - 10*sin(14*x)/7 - 5*sin(16*x)/4 - 10*sin(18*x)/9 - sin(20*x)

```python
s.truncate(20)
```

    -sin(2*x) - sin(4*x)/2 - sin(6*x)/3 - sin(8*x)/4 - sin(10*x)/5 - sin(12*x)/6 - sin(14*x)/7 - sin(16*x)/8 - sin(18*x)/9 - sin(20*x)/10 - sin(22*x)/11 - sin(24*x)/12 - sin(26*x)/13 - sin(28*x)/14 - sin(30*x)/15 - sin(32*x)/16 - sin(34*x)/17 - sin(36*x)/18 - sin(38*x)/19 - sin(40*x)/20

______
______
______

# MIDI Encoding tests

```python
import string
import collections
import math
import numpy as np
import pandas as pd

import pretty_midi

import matplotlib.pyplot as plt

%matplotlib inline
```

```python
test_midi = "/home/leo/DeepLearning/Few-Shot-Music-Generation/Datasets/freemidi/freemidi_data/Aerosmith/crazy.mid"
```

```python
midi_data = pretty_midi.PrettyMIDI(test_midi)
```

    /home/leo/DeepLearning/venv3/lib/python3.5/site-packages/pretty_midi/pretty_midi.py:100: RuntimeWarning: Tempo, Key or Time signature change events found on non-zero tracks.  This is not a valid type 0 or type 1 MIDI file.  Tempo, Key or Time Signature may be wrong.
      RuntimeWarning)

```python
midi_data.instruments
```

    [Instrument(program=17, is_drum=False, name="Vocal"),
     Instrument(program=27, is_drum=False, name="Guitar 1"),
     Instrument(program=27, is_drum=False, name="Guitar 2"),
     Instrument(program=33, is_drum=False, name="Bass"),
     Instrument(program=0, is_drum=False, name="Piano"),
     Instrument(program=50, is_drum=False, name="Chord Changes"),
     Instrument(program=0, is_drum=True, name="Drums"),
     Instrument(program=81, is_drum=False, name="\"Crazy\"  by  AEROSMITH"),
     Instrument(program=0, is_drum=True, name="Sequenced by Ryan109")]

```python
i1 = midi_data.instruments[0]
```

```python
type(i1)
```

    pretty_midi.instrument.Instrument

```python
i1.name
```

    'Vocal'

```python
i1.get_end_time()
```

    261.555294

```python
i1.program
```

    17

```python
n1 = i1.notes[0]
```

```python
n1
```

    Note(start=17.333316, end=17.444427, pitch=73, velocity=39)

TODO ....

```python

```


# Logistic Regression

```python
import pandas as pd
import numpy as np

pd.options.display.max_columns=None
pd.options.display.max_rows=None
```

## 알고 가야 할 것 들

### 1. Evidence of Weight
### 2. Information Value
### 3. VIF
### 4. C 통계량
### 5. AIC
### 6. ROC, AUC

## 데이터 셋을 로딩합니다.

```python
path = 'https://archive.ics.uci.edu/ml/machine-learning-databases/statlog/german/german.data'
dataset = pd.read_csv(path, delimiter=' ', header=None)
```

```python
dataset.shape
```

    (1000, 21)

```python
dataset.head()
```

      0
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      19
      20

      0
      A11
      6
      A34
      A43
      1169
      A65
      A75
      4
      A93
      A101
      4
      A121
      67
      A143
      A152
      2
      A173
      1
      A192
      A201
      1

      1
      A12
      48
      A32
      A43
      5951
      A61
      A73
      2
      A92
      A101
      2
      A121
      22
      A143
      A152
      1
      A173
      1
      A191
      A201
      2

      2
      A14
      12
      A34
      A46
      2096
      A61
      A74
      2
      A93
      A101
      3
      A121
      49
      A143
      A152
      1
      A172
      2
      A191
      A201
      1

      3
      A11
      42
      A32
      A42
      7882
      A61
      A74
      2
      A93
      A103
      4
      A122
      45
      A143
      A153
      1
      A173
      2
      A191
      A201
      1

      4
      A11
      24
      A33
      A40
      4870
      A61
      A73
      3
      A93
      A101
      4
      A124
      53
      A143
      A153
      2
      A173
      2
      A191
      A201
      2

```python
COL = [
    'Status_of_existing_checking_account',
    'Duration_in_month',
    'Credit_history',
    'Purpose',
    'Credit_amount',
    'Savings_account_bonds',
    'Present_employment_since',
    'Installment_rate_in_percentage_of_disposable_income',
    'Personal_status_and_sex',
    'Other_debtors_guarantors',
    'Present_residence_since',
    'Property',
    'Age_in_years',
    'Other_installment_plans',
    'Housing',
    'Number_of_existing_credits_at_this_bank',
    'Job',
    'Number_of_people_being_liable_to_provide_maintenance_for',
    'Telephone',
    'foreign_worker',
    'Target'
]
```

```python
dataset.columns = COL
```

```python
# 총 20개의 설명 변수와 1개의 종속 변수
dataset.head()
```

      Status_of_existing_checking_account
      Duration_in_month
      Credit_history
      Purpose
      Credit_amount
      Savings_account_bonds
      Present_employment_since
      Installment_rate_in_percentage_of_disposable_income
      Personal_status_and_sex
      Other_debtors_guarantors
      Present_residence_since
      Property
      Age_in_years
      Other_installment_plans
      Housing
      Number_of_existing_credits_at_this_bank
      Job
      Number_of_people_being_liable_to_provide_maintenance_for
      Telephone
      foreign_worker
      Target

      0
      A11
      6
      A34
      A43
      1169
      A65
      A75
      4
      A93
      A101
      4
      A121
      67
      A143
      A152
      2
      A173
      1
      A192
      A201
      1

      1
      A12
      48
      A32
      A43
      5951
      A61
      A73
      2
      A92
      A101
      2
      A121
      22
      A143
      A152
      1
      A173
      1
      A191
      A201
      2

      2
      A14
      12
      A34
      A46
      2096
      A61
      A74
      2
      A93
      A101
      3
      A121
      49
      A143
      A152
      1
      A172
      2
      A191
      A201
      1

      3
      A11
      42
      A32
      A42
      7882
      A61
      A74
      2
      A93
      A103
      4
      A122
      45
      A143
      A153
      1
      A173
      2
      A191
      A201
      1

      4
      A11
      24
      A33
      A40
      4870
      A61
      A73
      3
      A93
      A101
      4
      A124
      53
      A143
      A153
      2
      A173
      2
      A191
      A201
      2

Label
 - 1 : Good (갚을 수 있다 / 신용도가 좋다) $\to$ 0으로 변경
 - 2 : Bad (갚을 수 없다 / 신용도가 나쁘다) $\to$ 1로 변경

- Ham(0) / Spam(1)
- 정상사람(0) / 암환자(1)
- 이벤트가 발생하지 않음(0) / 이벤트가 발생함(1)
- 귀무가설(0) / 대립가설(1)

```python
# Label을 1/2 -> 0/1로 변경
dataset['Target'] = dataset['Target'] - 1
```

## 1. Information Value (정보 가치)
모델에서 변수의 사용유무를 판단하는 feature selection에서 유용한 방법입니다. 주로 모델을 학습하기전 첫 단계에서 변수들을 제거하는 데 사용합니다. 최종 모델에서는 대략 10개 내외의 변수를 사용하도록 합니다. IV와 WOE 신용채무능력이 가능한(good) 고객과 불가능한(bad) 고객을 예측하는 로지스틱 회귀 모델링과 밀접한 관계가 있습니다. 신용 정보 관련분야에서는 good customer는 부채를 갚을 수 있는 고객, bad customer는 부채를 갚을 수 없는 고객을 뜻합니다. 일반적으로 이야기할 때는 good customer는 non-events를 의미하고 bad customer는 events를 의미합니다.

> 신용 관련 분야

$
\begin{align}
&\text{WOE} = ln{\frac{\text{distribution of good}}{\text{distribution of bad}}} \\
&\text{IV} = \sum{(\text{WOE} \times (\text{distribution of good} - \text{distribution of bad}))}\\
\end{align}
$

>일반적

$
\begin{align}
&\text{WOE} = ln{\frac{\text{distribution of non-events}}{\text{distribution of events}}} \\
&\text{IV} = \sum{(\text{WOE} \times (\text{distribution of non-events} - \text{distribution of events}))}\\
\end{align}
$

- 구간을 나눌 때, `pd.qcut`을 사용하였습니다. bin의 갯수만큼 값의 구간을 나누고 값을 해당 구간에 mapping 시켜줍니다.
- Information value를 측정할 때, EOW가 inf, -inf가 나올 수 있습니다. 분포값이 0이 될 수 있기 때문입니다. 이 때는 EOW를 0으로 변경합니다.

|Information Value|예측력|
|:-----------:|:------------------------------:|
|0 to 0.02|무의미|
|0.02 to 0.1|낮은 예측|
|0.1 to 0.3|중간 예측|
|0.3 to 0.5|강한 예측|
|0.5 to 1|너무 강한 예측(의심되는 수치)|

예제)

```python
max_bin = 10  # 전체 데이터의 예측력에 해를 가하지 않는 한에서 구간을 테스트하였습니다.
def calc_iv(df, col, label, max_bin = max_bin):
    """IV helper function"""
    bin_df = df[[col, label]].copy()
    # Categorical column
    if bin_df[col].dtype == 'object':
        bin_df = bin_df.groupby(col)[label].agg(['count', 'sum'])
    # Numerical column
    else:
        bin_df.loc[:, 'bins'] = pd.qcut(bin_df[col].rank(method='first'), max_bin)
#         bin_df.loc[:, 'bins'] = pd.cut(bin_df[col], max_bin)
        bin_df = bin_df.groupby('bins')[label].agg(['count', 'sum'])

    bin_df.columns = ['total', 'abuse']
    bin_df['normal'] = bin_df['total'] - bin_df['abuse']
    bin_df['normal_dist'] = bin_df['normal'] / sum(bin_df['normal'])
    bin_df['abuse_dist'] = bin_df['abuse'] / sum(bin_df['abuse'])
    bin_df['woe'] = np.log(bin_df['normal_dist'] / bin_df['abuse_dist'])
    bin_df['iv'] = bin_df['woe'] * (bin_df['normal_dist'] - bin_df['abuse_dist'])

    bin_df.replace([np.inf, -np.inf], 0, inplace=True)
    bin_df = bin_df[bin_df['total'] > 0]
    iv_val = sum(filter(lambda x: x != float('inf'), bin_df['iv']))

    return bin_df, col, iv_val
```

```python
ch_df, ch, ch_i_val = calc_iv(dataset,'Credit_history', 'Target')
ch_df
```

      total
      abuse
      normal
      normal_dist
      abuse_dist
      woe
      iv

      Credit_history

      A30
      40
      25
      15
      0.021429
      0.083333
      -1.358123
      0.084074

      A31
      49
      28
      21
      0.030000
      0.093333
      -1.134980
      0.071882

      A32
      530
      169
      361
      0.515714
      0.563333
      -0.088319
      0.004206

      A33
      88
      28
      60
      0.085714
      0.093333
      -0.085158
      0.000649

      A34
      293
      50
      243
      0.347143
      0.166667
      0.733741
      0.132423

```python
# 중간 정도의 예측 능력
print('information value', ch_i_val)
```

    information value 0.2932335473908263

```python
dim_df, dim, dim_i_val = calc_iv(dataset,'Duration_in_month', 'Target')
dim_df
```

      total
      abuse
      normal
      normal_dist
      abuse_dist
      woe
      iv

      bins

      (0.999, 100.9]
      100
      11
      89
      0.127143
      0.036667
      1.243443
      0.112502

      (100.9, 200.8]
      100
      20
      80
      0.114286
      0.066667
      0.538997
      0.025667

      (200.8, 300.7]
      100
      27
      73
      0.104286
      0.090000
      0.147325
      0.002105

      (300.7, 400.6]
      100
      25
      75
      0.107143
      0.083333
      0.251314
      0.005984

      (400.6, 500.5]
      100
      26
      74
      0.105714
      0.086667
      0.198671
      0.003784

      (500.5, 600.4]
      100
      38
      62
      0.088571
      0.126667
      -0.357750
      0.013629

      (600.4, 700.3]
      100
      32
      68
      0.097143
      0.106667
      -0.093526
      0.000891

      (700.3, 800.2]
      100
      31
      69
      0.098571
      0.103333
      -0.047179
      0.000225

      (800.2, 900.1]
      100
      42
      58
      0.082857
      0.140000
      -0.524524
      0.029973

      (900.1, 1000.0]
      100
      48
      52
      0.074286
      0.160000
      -0.767255
      0.065765

```python
# 높은 예측 능력
dim_i_val
```

    0.2605225223321392

```python
# 함수를 만들어서 전체 iv를 살펴보자
col_iv = {}
for col in [idx for idx in dataset.columns.tolist()]:
    if col == 'Target':
        continue
    _, col, iv = calc_iv(dataset, col, 'Target')
    col_iv[col] = iv
```

```python
col_iv
```

    {'Status_of_existing_checking_account': 0.6660115033513336,
     'Duration_in_month': 0.2605225223321392,
     'Credit_history': 0.2932335473908263,
     'Purpose': 0.16919506567307832,
     'Credit_amount': 0.11342803024552867,
     'Savings_account_bonds': 0.19600955690422672,
     'Present_employment_since': 0.086433631026641,
     'Installment_rate_in_percentage_of_disposable_income': 0.061554683786294126,
     'Personal_status_and_sex': 0.04467067763379073,
     'Other_debtors_guarantors': 0.032019322019485055,
     'Present_residence_since': 0.04874371881018562,
     'Property': 0.11263826240979674,
     'Age_in_years': 0.10267245670259074,
     'Other_installment_plans': 0.057614541955647885,
     'Housing': 0.08329343361549926,
     'Number_of_existing_credits_at_this_bank': 0.09779114631307396,
     'Job': 0.008762765707428294,
     'Number_of_people_being_liable_to_provide_maintenance_for': 0.03408883520785682,
     'Telephone': 0.0063776050286746735,
     'foreign_worker': 0.04387741201028899,
     'bin_var': 0.03408883520785682}

```python
import operator
candidates = sorted(col_iv.items(), key=operator.itemgetter(1), reverse=True)
display(candidates)
```

    [('Status_of_existing_checking_account', 0.6660115033513336),
     ('Credit_history', 0.2932335473908263),
     ('Duration_in_month', 0.2605225223321392),
     ('Savings_account_bonds', 0.19600955690422672),
     ('Purpose', 0.16919506567307832),
     ('Credit_amount', 0.11342803024552867),
     ('Property', 0.11263826240979674),
     ('Age_in_years', 0.10267245670259074),
     ('Number_of_existing_credits_at_this_bank', 0.09779114631307396),
     ('Present_employment_since', 0.086433631026641),
     ('Housing', 0.08329343361549926),
     ('Installment_rate_in_percentage_of_disposable_income', 0.061554683786294126),
     ('Other_installment_plans', 0.057614541955647885),
     ('Present_residence_since', 0.04874371881018562),
     ('Personal_status_and_sex', 0.04467067763379073),
     ('foreign_worker', 0.04387741201028899),
     ('Number_of_people_being_liable_to_provide_maintenance_for',
      0.03408883520785682),
     ('bin_var', 0.03408883520785682),
     ('Other_debtors_guarantors', 0.032019322019485055),
     ('Job', 0.008762765707428294),
     ('Telephone', 0.0063776050286746735)]

Status_of_existing_checking_account가 가장 강력한 예측 능력(의심 될 정도로)를 가졌고, 다음이 Credit_history이다. Telephone은 가장 낮은 예측력을 가지고 있다.

사용할 컬럼은 본인이 선택해야한다. 나은 예측을 위해, 모델을 생성하고, 테스트하고 다시 돌아와서 컬럼을 생성하고를 반복한다.

우선 상위 15개만을 선택하자

```python
# 분석하려는 feature의 갯수와 예측력의 trade-off를 조정하였습니다.
iv_cols = [key for key, iv in candidates if iv >= 0.044]
display(len(iv_cols))
display(sorted(iv_cols))
```

    15

    ['Age_in_years',
     'Credit_amount',
     'Credit_history',
     'Duration_in_month',
     'Housing',
     'Installment_rate_in_percentage_of_disposable_income',
     'Number_of_existing_credits_at_this_bank',
     'Other_installment_plans',
     'Personal_status_and_sex',
     'Present_employment_since',
     'Present_residence_since',
     'Property',
     'Purpose',
     'Savings_account_bonds',
     'Status_of_existing_checking_account']

```python
# 상수로 설정합니다.
IV_COL = iv_cols[:]
```

Information Value에 의하여 상위 15개의 feature를 선정하였습니다. 전체 feature를 사용하여 로지스틱 회귀를 구했을 때의 accuracy와 신용불량자에 대한 precision의 감소폭이 적으면서 (전체 데이터셋을 가장 잘 대변해 줄 수 있으면서) feature의 수를 줄이는 과정을 몇 번 테스트 하면서 얻은 값입니다.

먼저, IV에서 선택된 컬럼들로 이루어진 모델에서부터 시작합니다.

```python
# Category column들은 one hot vectore로 변환합니다.
cate_features = {}
num_fetures = []
for col in IV_COL:
    if dataset[col].dtype == 'object':
        cate_features[col] = pd.get_dummies(dataset[col], prefix=col)
    else:
        num_fetures.append(col)
```

```python
cate_features.keys()
```

    dict_keys(['Status_of_existing_checking_account', 'Credit_history', 'Savings_account_bonds', 'Purpose', 'Property', 'Present_employment_since', 'Housing', 'Other_installment_plans', 'Personal_status_and_sex'])

```python
cate_features['Status_of_existing_checking_account'].head()
```

      Status_of_existing_checking_account_A11
      Status_of_existing_checking_account_A12
      Status_of_existing_checking_account_A13
      Status_of_existing_checking_account_A14

      0
      1
      0
      0
      0

      1
      0
      1
      0
      0

      2
      0
      0
      0
      1

      3
      1
      0
      0
      0

      4
      1
      0
      0
      0

더미 변수를 생성하게 되면, 자유도는 더미 변수가 가질 수 있는 unique 값이 $n$ 이라고 했을때 $n-1$이 되므로, 각 변수에서 더미변수 축을 한개씩 제거해준다. 제거하지 않으면 `NaN`값이 발생합니다.

후진 제거법을 통해 무의미한 변수를 하나씩 제거해 나가기 위해 별도의 리스트를 하나 생성합니다. 이 리스트는 실행이 반복될 때마다 가장 무의미한 변수(p-value가 큰 변수)와 다중공선성을 갖는 변수를 제거 한다.

```python
removed_features = []
```

```python
for col, dummies in cate_features.items():
    dropped_col = dummies.columns[-1]
    removed_features.append(dropped_col)
    cate_features[col] = dummies.drop(dropped_col, axis=1)
```

```python
removed_features
```

    ['Status_of_existing_checking_account_A14',
     'Credit_history_A34',
     'Savings_account_bonds_A65',
     'Purpose_A49',
     'Property_A124',
     'Present_employment_since_A75',
     'Housing_A153',
     'Other_installment_plans_A143',
     'Personal_status_and_sex_A94']

```python
cate_features['Status_of_existing_checking_account'].head()
```

      Status_of_existing_checking_account_A11
      Status_of_existing_checking_account_A12
      Status_of_existing_checking_account_A13

      0
      1
      0
      0

      1
      0
      1
      0

      2
      0
      0
      0

      3
      1
      0
      0

      4
      1
      0
      0

```python
final_dataset = dataset[num_fetures]
```

```python
for col, df in cate_features.items():
    final_dataset = pd.concat([final_dataset, df], axis=1)
```

```python
final_dataset.shape
```

    (1000, 40)

```python
final_dataset.head()
```

      Duration_in_month
      Credit_amount
      Age_in_years
      Number_of_existing_credits_at_this_bank
      Installment_rate_in_percentage_of_disposable_income
      Present_residence_since
      Status_of_existing_checking_account_A11
      Status_of_existing_checking_account_A12
      Status_of_existing_checking_account_A13
      Credit_history_A30
      Credit_history_A31
      Credit_history_A32
      Credit_history_A33
      Savings_account_bonds_A61
      Savings_account_bonds_A62
      Savings_account_bonds_A63
      Savings_account_bonds_A64
      Purpose_A40
      Purpose_A41
      Purpose_A410
      Purpose_A42
      Purpose_A43
      Purpose_A44
      Purpose_A45
      Purpose_A46
      Purpose_A48
      Property_A121
      Property_A122
      Property_A123
      Present_employment_since_A71
      Present_employment_since_A72
      Present_employment_since_A73
      Present_employment_since_A74
      Housing_A151
      Housing_A152
      Other_installment_plans_A141
      Other_installment_plans_A142
      Personal_status_and_sex_A91
      Personal_status_and_sex_A92
      Personal_status_and_sex_A93

      0
      6
      1169
      67
      2
      4
      4
      1
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1

      1
      48
      5951
      22
      1
      2
      2
      0
      1
      0
      0
      0
      1
      0
      1
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1
      0
      0
      1
      0
      0
      0
      1
      0

      2
      12
      2096
      49
      1
      2
      3
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      1
      0
      1
      0
      0
      0
      0
      0
      1
      0
      1
      0
      0
      0
      0
      1

      3
      42
      7882
      45
      1
      2
      4
      1
      0
      0
      0
      0
      1
      0
      1
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      1

      4
      24
      4870
      53
      2
      3
      4
      1
      0
      0
      0
      0
      0
      1
      1
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      1
      0
      0
      0
      0
      0
      0
      0
      1

## 2. Training set, test set을 나눕니다.

```python
from sklearn.model_selection import train_test_split
```

```python
train_x, test_x, train_y, test_y = \
    train_test_split( \
        final_dataset, dataset['Target'], test_size=0.2, random_state=42)
```

```python
print('train_set', train_x.shape)
print('test_set', test_x.shape)
```

    train_set (800, 40)
    test_set (200, 40)

## 3. Statsmodels를 이용하여 로지스틱 회귀를 돌립니다.

```python
import statsmodels.api as sm
```

    /anaconda3/lib/python3.6/site-packages/statsmodels/compat/pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.
      from pandas.core import datetools

```python
logistic_model = sm.Logit(
    train_y,
    sm.add_constant(train_x)
).fit()
```

    Optimization terminated successfully.
             Current function value: 0.460159
             Iterations 7

### 성능 측정

```python
from sklearn.metrics import auc
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
```

```python
train_pred = pd.DataFrame({
    'probs': logistic_model.predict(sm.add_constant(train_x)),
    'class': train_y
})

train_pred['y_pred'] = 0
train_pred.loc[train_pred['probs'] > 0.5, 'y_pred'] = 1

# Test prediction
test_pred = pd.DataFrame({
    'probs': logistic_model.predict(sm.add_constant(test_x)),
    'class': test_y
})

test_pred['y_pred'] = 0
test_pred.loc[test_pred['probs'] > 0.5, 'y_pred'] = 1
```

```python
print('\nTraining Confusion matrix:')
display(pd.crosstab(train_pred['y_pred'], train_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))
print('\nTraining Accuracy: ', round(accuracy_score(train_pred['class'], train_pred['y_pred']), 4))
print('\nTraining classification report:\n', classification_report(train_pred['class'], train_pred['y_pred'], digits=4))
```

    Training Confusion matrix:

      Actual
      0
      1
      All

      Predict

      0
      503
      120
      623

      1
      56
      121
      177

      All
      559
      241
      800

    Training Accuracy:  0.78

    Training classification report:
                  precision    recall  f1-score   support

              0     0.8074    0.8998    0.8511       559
              1     0.6836    0.5021    0.5789       241

    avg / total     0.7701    0.7800    0.7691       800

```python
print('Test Confusion matrix:')
display(pd.crosstab(test_pred['y_pred'], test_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))
print('\nTest Accuracy: ', round(accuracy_score(test_pred['class'], test_pred['y_pred']), 4))
print('\nTest classification report:\n', classification_report(test_pred['class'], test_pred['y_pred'], digits=4))
```

    Test Confusion matrix:

      Actual
      0
      1
      All

      Predict

      0
      123
      27
      150

      1
      18
      32
      50

      All
      141
      59
      200

    Test Accuracy:  0.775

    Test classification report:
                  precision    recall  f1-score   support

              0     0.8200    0.8723    0.8454       141
              1     0.6400    0.5424    0.5872        59

    avg / total     0.7669    0.7750    0.7692       200

## 2. 후진제거법
 - 가장 무의미한 변수 (p-value)
 - 다중공선성을 갖는 변수 (Variance Inflation Factor)

### (A) P-value
P-value가 가장 높은 변수를 제거합니다. 여기서 P-value는 각 독립변수의 유의성 검정(t-test)에서 얻어진 값입니다.

```python
unnecesarries = []
```

```python
logistic_model.summary2()
```

        Model:	              Logit	       No. Iterations:	   7.0000

  Dependent Variable:	      Target	      Pseudo R-squared:	   0.248

         Date:	        2019-02-25 22:10	       AIC:	        818.2538

   No. Observations:	         800	             BIC:	        1010.3229

       Df Model:	             40	         Log-Likelihood:	   -368.13

     Df Residuals:	           759	           LL-Null:	       -489.54

      Converged:	           1.0000	           Scale:	        1.0000

                           	                            Coef.	  Std.Err.	    z	     P>|z|	 [0.025	  0.975]

  const	                                               -3.6675	  1.0471	  -3.5026	 0.0005	 -5.7198	 -1.6153

  Duration_in_month	                                   0.0230	   0.0101	  2.2759	  0.0228	 0.0032	  0.0429

  Credit_amount	                                       0.0001	   0.0000	  2.4648	  0.0137	 0.0000	  0.0002

  Age_in_years	                                        -0.0315	  0.0104	  -3.0316	 0.0024	 -0.0519	 -0.0111

  Number_of_existing_credits_at_this_bank	             0.3461	   0.2075	  1.6682	  0.0953	 -0.0605	 0.7528

  Installment_rate_in_percentage_of_disposable_income	 0.3172	   0.0945	  3.3582	  0.0008	 0.1321	  0.5024

  Present_residence_since	                             0.0320	   0.0948	  0.3373	  0.7359	 -0.1538	 0.2178

  Status_of_existing_checking_account_A11	             1.5522	   0.2525	  6.1468	  0.0000	 1.0573	  2.0472

  Status_of_existing_checking_account_A12	             1.1125	   0.2474	  4.4969	  0.0000	 0.6276	  1.5974

  Status_of_existing_checking_account_A13	             0.5329	   0.4364	  1.2211	  0.2220	 -0.3224	 1.3881

  Credit_history_A30	                                  1.5371	   0.4723	  3.2545	  0.0011	 0.6114	  2.4628

  Credit_history_A31	                                  1.4798	   0.4815	  3.0734	  0.0021	 0.5361	  2.4235

  Credit_history_A32	                                  0.8741	   0.2821	  3.0988	  0.0019	 0.3212	  1.4269

  Credit_history_A33	                                  0.5364	   0.3691	  1.4532	  0.1462	 -0.1871	 1.2599

  Savings_account_bonds_A61	                           0.6851	   0.2741	  2.4997	  0.0124	 0.1479	  1.2224

  Savings_account_bonds_A62	                           0.5970	   0.3698	  1.6144	  0.1064	 -0.1278	 1.3217

  Savings_account_bonds_A63	                           0.2964	   0.4700	  0.6305	  0.5284	 -0.6249	 1.2176

  Savings_account_bonds_A64	                           -0.2078	  0.5857	  -0.3547	 0.7228	 -1.3558	 0.9402

  Purpose_A40	                                         0.6410	   0.3599	  1.7810	  0.0749	 -0.0644	 1.3464

  Purpose_A41	                                         -0.9585	  0.4654	  -2.0593	 0.0395	 -1.8707	 -0.0462

  Purpose_A410	                                        -0.6272	  0.8378	  -0.7486	 0.4541	 -2.2693	 1.0149

  Purpose_A42	                                         -0.2957	  0.3846	  -0.7686	 0.4421	 -1.0496	 0.4582

  Purpose_A43	                                         -0.3352	  0.3618	  -0.9263	 0.3543	 -1.0443	 0.3740

  Purpose_A44	                                         0.1854	   0.8089	  0.2292	  0.8187	 -1.4000	 1.7709

  Purpose_A45	                                         0.2867	   0.6258	  0.4581	  0.6469	 -0.9398	 1.5131

  Purpose_A46	                                         0.9372	   0.5224	  1.7941	  0.0728	 -0.0867	 1.9611

  Purpose_A48	                                         -1.2412	  1.2458	  -0.9964	 0.3191	 -3.6829	 1.2005

  Property_A121	                                       -0.6366	  0.4530	  -1.4053	 0.1599	 -1.5244	 0.2513

  Property_A122	                                       -0.2833	  0.4461	  -0.6349	 0.5255	 -1.1576	 0.5911

  Property_A123	                                       -0.2635	  0.4302	  -0.6126	 0.5401	 -1.1067	 0.5796

  Present_employment_since_A71	                        0.2897	   0.4022	  0.7203	  0.4713	 -0.4986	 1.0781

  Present_employment_since_A72	                        0.1534	   0.3273	  0.4686	  0.6394	 -0.4881	 0.7948

  Present_employment_since_A73	                        0.1896	   0.2776	  0.6830	  0.4946	 -0.3545	 0.7337

  Present_employment_since_A74	                        -0.3076	  0.3278	  -0.9385	 0.3480	 -0.9500	 0.3348

  Housing_A151	                                        0.2009	   0.5064	  0.3967	  0.6916	 -0.7916	 1.1934

  Housing_A152	                                        -0.3416	  0.4803	  -0.7112	 0.4770	 -1.2829	 0.5997

  Other_installment_plans_A141	                        0.7211	   0.2690	  2.6809	  0.0073	 0.1939	  1.2483

  Other_installment_plans_A142	                        0.5466	   0.4384	  1.2467	  0.2125	 -0.3127	 1.4059

  Personal_status_and_sex_A91	                         0.4998	   0.5226	  0.9563	  0.3389	 -0.5245	 1.5241

  Personal_status_and_sex_A92	                         0.0181	   0.3437	  0.0527	  0.9580	 -0.6555	 0.6918

  Personal_status_and_sex_A93	                         -0.4830	  0.3432	  -1.4074	 0.1593	 -1.1558	 0.1897

가장 무의미한 변수는 P값이 0.9580인 Personal_status_and_sex_A92이다.

체크하고 변수를 1개씩 제거하고를 반복해야한다.

```python
# Help 함수
def get_unrelated_cols(model, pvalue):
    cols = model.pvalues[model.pvalues >= pvalue].keys().tolist()
    print(len(cols))
    print(cols)

    return cols
```

```python
unrelated_cols = get_unrelated_cols(logistic_model, 0.8187)
```

    1
    ['Personal_status_and_sex_A92']

Personal_status_and_sex_A92을 삭제한다.

```python
unnecesarries.append('Personal_status_and_sex_A92')
```

### (B) 다중공선성
테스트와 상관없다고 판단되는 변수들을 제거한후에 다중공선성을 체크합니다. **다중공선성(multicollinearity)** 이란 독립변수들간의 선형관계가 존재하는 것을 나타냅니다. 독립변수 전체의 set에서 하나의 독립변수를 골라 종속변수로 보고, 그 독립변수를 제외한 set을 이용하여 선형회귀분석을 하여 구할 수 있습니다.

사실 변수간의 선형관계 알아보기 위해서는 산점도 or 상관계수를 사용해도 되지만 **VIF(분산 팽창 지수, Variance Inflation Factor)** 를 이용하여 구한다.

```python
def get_max_vif(df, removal_cols):
    vifs = []
    cnames = df.drop(removal_cols, axis=1).columns.tolist()
    for i in range(len(cnames)):
        xvar = cnames[:]
        yvar = xvar.pop(i)
        model = sm.OLS(
            df.drop(removal_cols, axis=1)[yvar],
            sm.add_constant(df.drop(removal_cols, axis=1)[xvar]))
        res = model.fit()
        vif = 1 / (1 - res.rsquared)
        vifs.append((yvar, round(vif, 3)))
    vifs = sorted(vifs, key=operator.itemgetter(1), reverse=True)
    return vifs
```

```python
vifs = get_max_vif(train_x, unnecesarries)
```

```python
vifs
```

    [('Housing_A152', 5.846),
     ('Property_A123', 4.999),
     ('Property_A121', 4.793),
     ('Housing_A151', 4.633),
     ('Property_A122', 4.281),
     ('Personal_status_and_sex_A93', 3.643),
     ('Personal_status_and_sex_A92', 3.302),
     ('Purpose_A43', 3.165),
     ('Purpose_A40', 2.983),
     ('Purpose_A42', 2.614),
     ('Credit_amount', 2.406),
     ('Purpose_A41', 2.121),
     ('Credit_history_A32', 2.011),
     ('Duration_in_month', 1.974),
     ('Present_employment_since_A73', 1.923),
     ('Savings_account_bonds_A61', 1.879),
     ('Present_employment_since_A72', 1.838),
     ('Present_employment_since_A74', 1.62),
     ('Purpose_A46', 1.564),
     ('Number_of_existing_credits_at_this_bank', 1.558),
     ('Personal_status_and_sex_A91', 1.53),
     ('Savings_account_bonds_A62', 1.478),
     ('Status_of_existing_checking_account_A11', 1.465),
     ('Age_in_years', 1.459),
     ('Credit_history_A31', 1.368),
     ('Status_of_existing_checking_account_A12', 1.365),
     ('Present_residence_since', 1.338),
     ('Savings_account_bonds_A63', 1.334),
     ('Credit_history_A33', 1.326),
     ('Present_employment_since_A71', 1.314),
     ('Installment_rate_in_percentage_of_disposable_income', 1.305),
     ('Savings_account_bonds_A64', 1.27),
     ('Purpose_A45', 1.26),
     ('Credit_history_A30', 1.259),
     ('Purpose_A410', 1.216),
     ('Purpose_A44', 1.191),
     ('Purpose_A48', 1.146),
     ('Status_of_existing_checking_account_A13', 1.14),
     ('Other_installment_plans_A141', 1.134),
     ('Other_installment_plans_A142', 1.107)]

```python
unnecesarries.append(vifs[0][0])
```

```python
unnecesarries
```

    ['Personal_status_and_sex_A92', 'Housing_A152']

무의미한 변수(가장 높은 p-value)는 Personal_status_and_sex_A92였고 pvalue가 0.9580, VIF가 3.302이다.

다중공선성이 가장 높았던 변수는 Housing_A152였고 pvalue가 0.4770, VIF가 5.846이다.

**무의미한 변수를 먼저 삭제해야하며, 지우기 전에, 두 변수의 pvalue와 VIF를 각각 비교해보는것이 좋다**.

확실히 Personal_status_and_sex_A92는 삭제해야 한다.

모델을 삭제하기 앞서 모델 채택 기준을 살펴봐야한다.

## 3. 모델 채택 기준
### (A) C통계량 (concordance statistics)
 - 로지스틱 회귀 모델에서 이항 결과의 적합도(goodness of fit)에 관한 품질을 측정하는 척도
 - ROC curve와 동일하다.

$$C=Pr[\pi(B|x_{i}) > \pi(B|x_{j}) | Y_{i} = 1, Y_{j} = 0]$$
$$C 통계량(C 인덱스) = 0.5 + (\frac{일치쌍 비율 - 불일치쌍 비율}{2})$$

```python
def get_c_stat(iv_pred):
    noraml_test_df = iv_pred[iv_pred['class'] == 0][['class', 'probs']]
    spammer_test_df = iv_pred[iv_pred['class'] == 1][['class', 'probs']]

    noraml_test_df['key'] = 0
    spammer_test_df['key'] = 0

    cross_join_df = noraml_test_df.merge(spammer_test_df, how='outer', on='key').drop('key', axis=1)

    cross_join_df['concordance'] = cross_join_df['probs_x'] < cross_join_df['probs_y']
    cross_join_df['in_concordance'] = cross_join_df['probs_x'] > cross_join_df['probs_y']
    cross_join_df['tie'] = cross_join_df['probs_x'] == cross_join_df['probs_y']

    results = cross_join_df.agg({'concordance': np.sum, 'in_concordance': np.sum, 'tie': np.sum}) / len(cross_join_df)
    c_stat = 0.5 + (results['concordance'] - results['in_concordance']) / 2

    return c_stat
```

### (B) AIC, Likelihood

- $AIC=-2 \times ln(L) + 2 \times k$

Akaike information criterion는 주어진 데이터 집합에 관해 통계 모델의 상대적 품질을 측정한다. 이 척도는 편향과 분산의 트레이드 오프이다. 더 작은 AIC를 선호한다.

- $L = \Pi_{i=1}^{n}p(x_i)^{y_i}(1-p(x_i))^{1-y_i}$

log likelihood는 큰 값을 선호한다.

```python
def get_aic_value(model):
    return -2 * model.llf + 2 * (len(model.params) - 1)
```

> 무의미한 변수를 지우기 전

```python
c_stat, aic = get_c_stat(train_pred), get_aic_value(logistic_model)
print('c_stat:', c_stat)
print('aic:', aic)
print('loglikehood:', logistic_model.llf)
```

    c_stat: 0.8227198836095874
    aic: 816.2537855406207
    loglikehood: -368.12689277031035

```python
c_stat, aic = get_c_stat(test_pred), get_aic_value(logistic_model)
print('c_stat:', c_stat)
print('aic:', aic)
print('loglikehood:', logistic_model.llf)
```

    c_stat: 0.8067075369635773
    aic: 816.2537855406207
    loglikehood: -368.12689277031035

> 지우고 난 후

```python
unnecesarries.extend(['Personal_status_and_sex_A92', 'Housing_A152'])
```

```python
drop_logistic_model = sm.Logit(
    train_y,
    sm.add_constant(train_x.drop(unnecesarries, axis=1))
).fit()
```

    Optimization terminated successfully.
             Current function value: 0.460475
             Iterations 7

```python
train_pred = pd.DataFrame({
    'probs': drop_logistic_model.predict(sm.add_constant(train_x.drop(unnecesarries, axis=1))),
    'class': train_y
})

train_pred['y_pred'] = 0
train_pred.loc[train_pred['probs'] > 0.5, 'y_pred'] = 1

# Test prediction
test_pred = pd.DataFrame({
    'probs': drop_logistic_model.predict(sm.add_constant(test_x.drop(unnecesarries, axis=1))),
    'class': test_y
})

test_pred['y_pred'] = 0
test_pred.loc[test_pred['probs'] > 0.5, 'y_pred'] = 1
```

```python
c_stat, aic = get_c_stat(train_pred), get_aic_value(drop_logistic_model)
print('c_stat:', c_stat)
print('aic:', aic)
print('loglikehood:', drop_logistic_model.llf)
```

    c_stat: 0.8227198836095874
    aic: 812.7605967161102
    loglikehood: -368.3802983580551

```python
c_stat, aic = get_c_stat(test_pred), get_aic_value(drop_logistic_model)
print('c_stat:', c_stat)
print('aic:', aic)
print('loglikehood:', drop_logistic_model.llf)
```

    c_stat: 0.8101935328765477
    aic: 812.7605967161102
    loglikehood: -368.3802983580551

```python
drop_logistic_model.summary2()
```

        Model:	              Logit	       No. Iterations:	   7.0000

  Dependent Variable:	      Target	      Pseudo R-squared:	   0.247

         Date:	        2019-02-25 22:24	       AIC:	        814.7606

   No. Observations:	         800	             BIC:	        997.4605

       Df Model:	             38	         Log-Likelihood:	   -368.38

     Df Residuals:	           761	           LL-Null:	       -489.54

      Converged:	           1.0000	           Scale:	        1.0000

                           	                            Coef.	  Std.Err.	    z	     P>|z|	 [0.025	  0.975]

  const	                                               -3.8574	  0.9727	  -3.9656	 0.0001	 -5.7638	 -1.9509

  Duration_in_month	                                   0.0231	   0.0101	  2.2787	  0.0227	 0.0032	  0.0429

  Credit_amount	                                       0.0001	   0.0000	  2.4931	  0.0127	 0.0000	  0.0002

  Age_in_years	                                        -0.0306	  0.0103	  -2.9737	 0.0029	 -0.0508	 -0.0104

  Number_of_existing_credits_at_this_bank	             0.3431	   0.2072	  1.6559	  0.0977	 -0.0630	 0.7492

  Installment_rate_in_percentage_of_disposable_income	 0.3180	   0.0944	  3.3681	  0.0008	 0.1330	  0.5031

  Present_residence_since	                             0.0415	   0.0935	  0.4440	  0.6570	 -0.1417	 0.2248

  Status_of_existing_checking_account_A11	             1.5561	   0.2523	  6.1684	  0.0000	 1.0617	  2.0506

  Status_of_existing_checking_account_A12	             1.1076	   0.2469	  4.4869	  0.0000	 0.6238	  1.5915

  Status_of_existing_checking_account_A13	             0.5488	   0.4351	  1.2612	  0.2072	 -0.3041	 1.4016

  Credit_history_A30	                                  1.5543	   0.4713	  3.2982	  0.0010	 0.6307	  2.4780

  Credit_history_A31	                                  1.4789	   0.4809	  3.0756	  0.0021	 0.5365	  2.4214

  Credit_history_A32	                                  0.8853	   0.2813	  3.1471	  0.0016	 0.3340	  1.4367

  Credit_history_A33	                                  0.5490	   0.3686	  1.4893	  0.1364	 -0.1735	 1.2715

  Savings_account_bonds_A61	                           0.6956	   0.2737	  2.5413	  0.0110	 0.1591	  1.2321

  Savings_account_bonds_A62	                           0.5991	   0.3697	  1.6203	  0.1052	 -0.1256	 1.3237

  Savings_account_bonds_A63	                           0.2976	   0.4698	  0.6335	  0.5264	 -0.6231	 1.2183

  Savings_account_bonds_A64	                           -0.2150	  0.5863	  -0.3668	 0.7138	 -1.3641	 0.9340

  Purpose_A40	                                         0.6528	   0.3596	  1.8154	  0.0695	 -0.0520	 1.3576

  Purpose_A41	                                         -0.9272	  0.4621	  -2.0067	 0.0448	 -1.8329	 -0.0216

  Purpose_A410	                                        -0.6423	  0.8345	  -0.7698	 0.4414	 -2.2779	 0.9932

  Purpose_A42	                                         -0.2835	  0.3841	  -0.7380	 0.4605	 -1.0364	 0.4694

  Purpose_A43	                                         -0.3321	  0.3613	  -0.9192	 0.3580	 -1.0402	 0.3760

  Purpose_A44	                                         0.1707	   0.8124	  0.2102	  0.8335	 -1.4216	 1.7630

  Purpose_A45	                                         0.2766	   0.6279	  0.4406	  0.6595	 -0.9540	 1.5073

  Purpose_A46	                                         0.9581	   0.5227	  1.8327	  0.0668	 -0.0665	 1.9826

  Purpose_A48	                                         -1.2385	  1.2279	  -1.0086	 0.3132	 -3.6451	 1.1682

  Property_A121	                                       -0.8475	  0.3403	  -2.4908	 0.0127	 -1.5144	 -0.1806

  Property_A122	                                       -0.5012	  0.3230	  -1.5515	 0.1208	 -1.1342	 0.1319

  Property_A123	                                       -0.4812	  0.3012	  -1.5973	 0.1102	 -1.0716	 0.1092

  Present_employment_since_A71	                        0.3068	   0.4011	  0.7649	  0.4444	 -0.4793	 1.0929

  Present_employment_since_A72	                        0.1538	   0.3268	  0.4706	  0.6380	 -0.4867	 0.7943

  Present_employment_since_A73	                        0.1967	   0.2771	  0.7097	  0.4779	 -0.3464	 0.7398

  Present_employment_since_A74	                        -0.3067	  0.3273	  -0.9372	 0.3487	 -0.9482	 0.3348

  Housing_A151	                                        0.5122	   0.2563	  1.9984	  0.0457	 0.0099	  1.0146

  Other_installment_plans_A141	                        0.7157	   0.2689	  2.6619	  0.0078	 0.1887	  1.2427

  Other_installment_plans_A142	                        0.5353	   0.4381	  1.2218	  0.2218	 -0.3234	 1.3939

  Personal_status_and_sex_A91	                         0.4805	   0.4459	  1.0775	  0.2813	 -0.3935	 1.3544

  Personal_status_and_sex_A93	                         -0.4934	  0.2122	  -2.3255	 0.0200	 -0.9092	 -0.0776

## 4. 무의미한 변수와 다중공선성이 있는 변수가 없을때까지 반복한다. (기준은 C통계량, AIC, Likelihood)

원래는 이 과정을 각각 변수를 확인하면서, 변수를 하나씩 제거해야 한다. 하지만 시간 관계상..

#### Step 1. Pvalue

```python
# Pvalue를 확인하고, 상한선을 구한다.
unrelated_cols = get_unrelated_cols(drop_logistic_model, 0.5)
```

    6
    ['Present_residence_since', 'Savings_account_bonds_A63', 'Savings_account_bonds_A64', 'Purpose_A44', 'Purpose_A45', 'Present_employment_since_A72']

```python
unnecesarries.extend(unrelated_cols)
```

#### Step 2. VIF

```python
vifs = get_max_vif(train_x, unnecesarries)
```

```python
vifs[:10]
```

    [('Property_A121', 2.58),
     ('Property_A123', 2.487),
     ('Purpose_A43', 2.431),
     ('Credit_amount', 2.354),
     ('Purpose_A40', 2.32),
     ('Property_A122', 2.283),
     ('Purpose_A42', 2.111),
     ('Credit_history_A32', 1.977),
     ('Duration_in_month', 1.947),
     ('Purpose_A41', 1.79)]

```python
unnecesarries.extend([elem[0] for elem in vifs[:10]])
```

```python
drop_logistic_model = sm.Logit(
    train_y,
    sm.add_constant(train_x.drop(unnecesarries, axis=1))
).fit()
```

    Optimization terminated successfully.
             Current function value: 0.506885
             Iterations 6

```python
train_pred = pd.DataFrame({
    'probs': drop_logistic_model.predict(sm.add_constant(train_x.drop(unnecesarries, axis=1))),
    'class': train_y
})

train_pred['y_pred'] = 0
train_pred.loc[train_pred['probs'] > 0.5, 'y_pred'] = 1

# Test prediction
test_pred = pd.DataFrame({
    'probs': drop_logistic_model.predict(sm.add_constant(test_x.drop(unnecesarries, axis=1))),
    'class': test_y
})

test_pred['y_pred'] = 0
test_pred.loc[test_pred['probs'] > 0.5, 'y_pred'] = 1
```

```python
c_stat, aic = get_c_stat(train_pred), get_aic_value(drop_logistic_model)
print('c_stat:', c_stat)
print('aic:', aic)
print('loglikehood:', drop_logistic_model.llf)
```

    c_stat: 0.7741112983320838
    aic: 855.0167324752877
    loglikehood: -405.5083662376438

```python
c_stat, aic = get_c_stat(test_pred), get_aic_value(drop_logistic_model)
print('c_stat:', c_stat)
print('aic:', aic)
print('loglikehood:', drop_logistic_model.llf)
```

    c_stat: 0.7756941940137035
    aic: 855.0167324752877
    loglikehood: -405.5083662376438

```python
drop_logistic_model.summary2()
```

        Model:	              Logit	       No. Iterations:	   6.0000

  Dependent Variable:	      Target	      Pseudo R-squared:	   0.172

         Date:	        2019-02-25 22:41	       AIC:	        857.0167

   No. Observations:	         800	             BIC:	        964.7628

       Df Model:	             22	         Log-Likelihood:	   -405.51

     Df Residuals:	           777	           LL-Null:	       -489.54

      Converged:	           1.0000	           Scale:	        1.0000

                           	                            Coef.	  Std.Err.	    z	     P>|z|	 [0.025	  0.975]

  const	                                               -2.1483	  0.5231	  -4.1067	 0.0000	 -3.1737	 -1.1230

  Age_in_years	                                        -0.0256	  0.0087	  -2.9609	 0.0031	 -0.0426	 -0.0087

  Number_of_existing_credits_at_this_bank	             -0.0049	  0.1631	  -0.0303	 0.9758	 -0.3245	 0.3146

  Installment_rate_in_percentage_of_disposable_income	 0.1913	   0.0806	  2.3732	  0.0176	 0.0333	  0.3494

  Status_of_existing_checking_account_A11	             1.6818	   0.2321	  7.2451	  0.0000	 1.2268	  2.1367

  Status_of_existing_checking_account_A12	             1.2142	   0.2300	  5.2782	  0.0000	 0.7633	  1.6650

  Status_of_existing_checking_account_A13	             0.4189	   0.4152	  1.0090	  0.3130	 -0.3948	 1.2326

  Credit_history_A30	                                  1.4395	   0.4109	  3.5034	  0.0005	 0.6342	  2.2449

  Credit_history_A31	                                  1.0135	   0.3945	  2.5689	  0.0102	 0.2402	  1.7868

  Credit_history_A33	                                  0.3081	   0.3093	  0.9960	  0.3193	 -0.2982	 0.9143

  Savings_account_bonds_A61	                           0.5528	   0.2132	  2.5935	  0.0095	 0.1350	  0.9706

  Savings_account_bonds_A62	                           0.6598	   0.3173	  2.0793	  0.0376	 0.0379	  1.2817

  Purpose_A410	                                        0.0894	   0.6874	  0.1301	  0.8965	 -1.2578	 1.4367

  Purpose_A46	                                         1.1777	   0.3936	  2.9919	  0.0028	 0.4062	  1.9492

  Purpose_A48	                                         -1.6748	  1.2032	  -1.3919	 0.1639	 -4.0330	 0.6835

  Present_employment_since_A71	                        0.3893	   0.3416	  1.1397	  0.2544	 -0.2802	 1.0589

  Present_employment_since_A73	                        0.0832	   0.2046	  0.4069	  0.6841	 -0.3177	 0.4842

  Present_employment_since_A74	                        -0.2333	  0.2595	  -0.8990	 0.3686	 -0.7420	 0.2754

  Housing_A151	                                        0.4031	   0.2296	  1.7556	  0.0791	 -0.0469	 0.8531

  Other_installment_plans_A141	                        0.6645	   0.2541	  2.6149	  0.0089	 0.1664	  1.1625

  Other_installment_plans_A142	                        0.4337	   0.4208	  1.0306	  0.3027	 -0.3911	 1.2585

  Personal_status_and_sex_A91	                         0.4378	   0.4224	  1.0365	  0.3000	 -0.3900	 1.2657

  Personal_status_and_sex_A93	                         -0.2726	  0.1908	  -1.4281	 0.1533	 -0.6466	 0.1015

```python
print('Train Confusion matrix:')
display(pd.crosstab(train_pred['y_pred'], train_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))
```

    Train Confusion matrix:

      Actual
      0
      1
      All

      Predict

      0
      504
      150
      654

      1
      55
      91
      146

      All
      559
      241
      800

```python
print('Test Confusion matrix:')
display(pd.crosstab(test_pred['y_pred'], test_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))
```

    Test Confusion matrix:

      Actual
      0
      1
      All

      Predict

      0
      122
      34
      156

      1
      19
      25
      44

      All
      141
      59
      200

## 5. ROC curve, AUC
 - ROC = FPR vs TPR(Recall) 의 비율
 - AUC = ROC 곡선 아래의 면적

```python
import matplotlib.pyplot as plt
from sklearn import metrics
from sklearn.metrics import auc

plt.rcParams["figure.figsize"] = (7,7)
```

```python
fpr, tpr, thresholds = metrics.roc_curve(train_pred['class'], train_pred['probs'], pos_label=1)
```

```python
roc_auc = auc(fpr, tpr)
```

```python
plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.4f)' % roc_auc)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.xlabel('False Postive Ratio')
plt.ylabel('True Postive Ratio(Recall)')
plt.title('ROC curve')
plt.legend(loc='lower right')
plt.show()
```

## 6. 결정경계

```python
for i in np.arange(0, 1, 0.1):
    train_pred['y_pred'] = 0
    train_pred.loc[train_pred['probs'] > i, 'y_pred'] = 1
    acc = round(accuracy_score(train_pred['class'], train_pred['y_pred']), 4)
    print("Threshold", round(i,1), "Train accuracy:",acc)
```

    Threshold 0.0 Train accuracy: 0.3012
    Threshold 0.1 Train accuracy: 0.4625
    Threshold 0.2 Train accuracy: 0.63
    Threshold 0.3 Train accuracy: 0.6975
    Threshold 0.4 Train accuracy: 0.735
    Threshold 0.5 Train accuracy: 0.7438
    Threshold 0.6 Train accuracy: 0.74
    Threshold 0.7 Train accuracy: 0.7275
    Threshold 0.8 Train accuracy: 0.715
    Threshold 0.9 Train accuracy: 0.7

```python
for i in np.arange(0, 1, 0.1):
    test_pred['y_pred'] = 0
    test_pred.loc[test_pred['probs'] > i, 'y_pred'] = 1
    acc = round(accuracy_score(test_pred['class'], test_pred['y_pred']), 4)
    print("Threshold", round(i,1), "Test accuracy:",acc)
```

    Threshold 0.0 Test accuracy: 0.295
    Threshold 0.1 Test accuracy: 0.455
    Threshold 0.2 Test accuracy: 0.625
    Threshold 0.3 Test accuracy: 0.675
    Threshold 0.4 Test accuracy: 0.725
    Threshold 0.5 Test accuracy: 0.735
    Threshold 0.6 Test accuracy: 0.735
    Threshold 0.7 Test accuracy: 0.72
    Threshold 0.8 Test accuracy: 0.705
    Threshold 0.9 Test accuracy: 0.71

```python
# Test prediction
test_pred = pd.DataFrame({
    'probs': drop_logistic_model.predict(sm.add_constant(test_x.drop(unnecesarries, axis=1))),
    'class': test_y
})

test_pred['y_pred'] = 0
test_pred.loc[test_pred['probs'] > 0.4, 'y_pred'] = 1
```

```python
print('Test Confusion matrix:')
display(pd.crosstab(test_pred['y_pred'], test_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))
```

    Test Confusion matrix:

      Actual
      0
      1
      All

      Predict

      0
      106
      20
      126

      1
      35
      39
      74

      All
      141
      59
      200


```python
from multiprocessing import Pool
import matplotlib.pyplot as plt
from eddy import rotationmap
import numpy as np
```

The data is a little big (30.5MB per file) so I've always had them loaded up on the Harvard Dataverse: https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/KXELJL.

```python
cube = rotationmap(path='../../cubes/TWHya_CO_cube_v0.fits',
                   uncertainty='../../cubes/TWHya_CO_cube_dv0.fits',
                   downsample='beam',
                   FOV=7.0)
cube.plot_data()
```

This initial run gives an idea of what the 'good' parameters are.

```python
# Dictionary to contain the disk parameters.

params = {}

# Start with the free variables in p0.

params['x0'] = 0
params['y0'] = 1
params['PA'] = 2
params['mstar'] = 3
params['vlsr'] = 4

# Provide starting guesses for these values.

p0 = [0.0, 0.0, 151., 0.81, 2.8e3]

# Fix the other parameters. All values which are to be fixed must be floats.

params['inc'] = 5.8    # degrees
params['dist'] = 60.1  # parsec
params['r_min'] = 0.2  # arcsec
params['r_max'] = 3.0  # arcsec

# Run the sampler.

samples = cube.fit_map(p0=p0,
                       params=params,
                       optimize=False,
                       nwalkers=128,
                       nburnin=200,
                       nsteps=1000)
```

For this dataset, when using $i = 5.8^{\circ}$ (inclination) and $d = 60.1~{\rm pc}$ (distance) it would be considered a 'pass' if:

\begin{align}
|x0| &\leq 0.1^{\prime\prime} \\
|y0| &\leq 0.1^{\prime\prime} \\
|{\rm PA} - 151.0| &\leq 2^{\circ} \\
|M_{*} - 0.8| &\leq 0.5~M_{\rm sun} \\
|v_{\rm LSR} - 2840| &\leq 10~{\rm m\,s^{-1}} \\
\end{align}

There is a large degeneracy between $\{i,\, M_{*},\, d\}$ so the standard is to hold $i$ and $d$ fixed.

```python

```