## Imports
### Se cargan todas las librerias a utilizar

```python
import pathlib
```

```python
# Data
import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score, mean_squared_error
from scipy.stats import skew, skewtest
```

```python
# Charting
import matplotlib.pyplot as plt
from matplotlib import gridspec
from matplotlib import pyplot
from statsmodels.graphics.gofplots import qqplot
import tensorflow as tf
import seaborn as sns
```

```python
# Deep Learning
from keras.models import Sequential
from keras.layers import Conv1D, Dropout, MaxPool1D, Dense, Flatten, BatchNormalization, Conv2D, MaxPooling2D
from keras.layers import SimpleRNN, LSTM, TimeDistributed, Reshape
from tensorflow.keras.optimizers import Adam, RMSprop
from keras.callbacks import TensorBoard
```

```python
# OpenSees Py
import openseespy.opensees as op
import openseespy.postprocessing.ops_vis as opsv
import sympy as sp

#import the os module
import os
import math
from tabulate import tabulate

import scipy.linalg
import socket, struct
import sys
```

## Preprocessing functions
Esta función es utilizada para generar las base de datos de entrenamiento y testeo a utilizar

```python
def walk_forward(df, x_labels, y_labels, n, n_input_labels,
                 split_per=None, split_data=False, y_offset=0):

    td_shape = df.shape
    td_l = td_shape[0]

    x_indices = [range(i, i+n) for i in range(td_l-n)]
    y_indices = [range(i+y_offset, i+n) for i in range(td_l-n)]

    x = df.loc[:, x_labels].values[x_indices]
    y = df.loc[:, y_labels].values[y_indices]

    if split_data:
        split_index = int(len(x)*split_per)
        x = x.reshape(-1, n, n_input_labels, 1)
        y = y.reshape(-1, n-y_offset)
        X_train, X_test, y_train, y_test = (x[:split_index], x[split_index:], y[:split_index], y[split_index:])
        return X_train, X_test, y_train, y_test

    return x, y
```

## Generate Data
- A continuación, se resuelve un sistema lineal elástico cuyo input es la aceleración basal, la cual se genera de forma aleatoria.
- Se resuelve de esta forma ya que no tenemos los registros sísmicos para ser ingresados y esto funciona como un primer aproach

```python
# -*- coding: utf-8 -*-
"""
Created on Sun Oct 17 16:43:00 2021

@author: diego.mera
"""

##########################################################################################################################################################################
op.wipe()
#########################################################################################################################################################################
#All results in Inch, Kip and Sec
# Define ELEMENTS & SECTIONS
inch = 1.0
kip = 1.0
sec = 1.0
LunitTXT = 'inch'
FunitTXT = 'kip'
TunitTXT = 'sec'
ft = 12*inch
ksi = kip/math.pow(inch,2)
psi = ksi/1000
lbf = psi*inch*inch
pcf = lbf/math.pow(ft,3)
inch2 = inch*inch
inch4 = math.pow(inch,4)
cm = inch/2.54
PI = 2 * math.asin(1.0)
g = 32.2 * ft/math.pow(sec,2)
Ubig = 1e10
Usmall = 1/Ubig

op.model('basic', '-ndm', 2, '-ndf', 3)
LCol = 36.0*ft       # column length
LBeam = 42.0*ft      # beam length
Weight = 200.0*kip   # superstructure weight

# define section geometry
HCol = 5.0*ft   # Column Depth
BCol = 5.0*ft   # Column Width
HBeam = 8.0*ft  # Beam Depth
BBeam = 5.0*ft  # Beam Width

# calculated parameters
PCol =Weight/2  # nodal dead-load weight per column
g = 386.4       # Gravity in/s2
Mass =  PCol/g  # mass lb
MCol = ((Weight/LBeam)*math.pow(LBeam,2))/12

# calculated geometry parameters
ACol = HCol*BCol                        # cross-sectional area column
ABeam = HBeam*BBeam                     # cross-sectional area beam
IzCol = (BCol*math.pow(HCol,3))/12      # Column moment of inertia
IzBeam =  (BBeam*math.pow(HBeam,3))/12  # Beam moment of inertia

# Nodes
op.node(1, 0.0, 0.0)
op.node(2, LBeam, 0.0);
op.node(3, 2*LBeam, 0.0);
op.node(4, 0.0, LCol)
op.node(5, LBeam, LCol)
op.node(6, 2*LBeam, LCol)
op.node(7, 0.0, 2*LCol)
op.node(8, LBeam, 2*LCol)
op.node(9, 2*LBeam, 2*LCol)
op.node(10, 0.0, 3*LCol)
op.node(11, LBeam, 3*LCol)
op.node(12, 2*LBeam, 3*LCol)

# Create a homogeneous SP constriant.
op.fix(1, 1, 1, 0)
op.fix(2, 1, 1, 0);
op.fix(3, 1, 1, 0);

IDctrlNode = 2
IDctrlDOF = 1

# set the mass at a node
op.mass(4, Mass, 0.0, 0.0)
op.mass(5, Mass, 0.0, 0.0)
op.mass(6, Mass, 0.0, 0.0)
op.mass(7, Mass, 0.0, 0.0)
op.mass(8, Mass, 0.0, 0.0)
op.mass(9, Mass, 0.0, 0.0)
op.mass(10, Mass, 0.0, 0.0)
op.mass(11, Mass, 0.0, 0.0)
op.mass(12, Mass, 0.0, 0.0)

ColSecTag = 1			 # assign a tag number to the column section
BeamSecTag = 2           # assign a tag number to the beam section

seccion = 'Fibra' # Fibra or Agregaton

if seccion == 'Fibra':
    coverCol = 6.0*inch      # Column cover to reinforcing steel NA.
    numBarsCol = 10          # number of longitudinal-reinforcement bars in column. (symmetric top & bot)
    barAreaCol = 2.25*inch2  # area of longitudinal-reinforcement bars

    # MATERIAL parameters
    IDconcU = 1 			 # material ID tag -- unconfined cover concrete (here used for complete section)
    IDreinf = 2 			 # material ID tag -- reinforcement

    # nominal concrete compressive strength
    fc = -4.0*ksi 				   # CONCRETE Compressive Strength (+Tension, -Compression)
    Ec = 57*ksi*math.sqrt(-fc/psi) # Concrete Elastic Modulus (the term in sqr root needs to be in psi

    # unconfined concrete
    fc1U = fc			    # UNCONFINED concrete (todeschini parabolic model), maximum stress
    eps1U = -0.003			# strain at maximum strength of unconfined concrete
    fc2U =  0.2*fc1U		# ultimate stress
    eps2U = -0.05			# strain at ultimate stress
    Lambda = 0.1		    # ratio between unloading slope at $eps2 and initial slope $Ec

    # tensile-strength properties
    ftU = -0.14* fc1U		# tensile strength +tension
    Ets = ftU/0.002			# tension softening stiffness

    Fy = 66.8*ksi			# STEEL yield stress
    Es = 29000.0*ksi		# modulus of steel
    Bs = 0.01				# strain-hardening ratio
    R0 = 18.0				# control the transition from elastic to plastic branches
    cR1 = 0.925				# control the transition from elastic to plastic branches
    cR2 = 0.15				# control the transition from elastic to plastic branches

    op.uniaxialMaterial('Concrete02', IDconcU, fc1U, eps1U, fc2U, eps2U, Lambda, ftU, Ets) # build cover concrete (unconfined)
    op.uniaxialMaterial('Steel02', IDreinf, Fy, Es, Bs, R0,cR1,cR2) # build reinforcement material
    # FIBER SECTION properties -------------------------------------------------------------
    # symmetric section
    #                        y
    #                        ^
    #                        |
    #             ---------------------     --   --
    #             |   o     o     o    |     |    -- cover
    #             |                    |     |
    #             |                    |     |
    #    z <---   |          +         |     H
    #             |                    |     |
    #             |                    |     |
    #             |   o     o     o    |     |    -- cover
    #             ---------------------     --   --
    #             |-------- B --------|
    #
    # RC section:
    coverY = HCol/2.0	# The distance from the section z-axis to the edge of the cover concrete -- outer edge of cover concrete
    coverZ = BCol/2.0	# The distance from the section y-axis to the edge of the cover concrete -- outer edge of cover concrete
    coreY = coverY-coverCol
    coreZ = coverZ-coverCol
    nfY = 16        # number of fibers for concrete in y-direction
    nfZ = 4			# number of fibers for concrete in z-direction

    op.section('Fiber', ColSecTag)
    op.patch('quad', IDconcU, nfZ, nfY, -coverY,coverZ, -coverY,-coverZ, coverY,-coverZ, coverY,coverZ) # Define the concrete patch
    op.layer('straight', IDreinf, numBarsCol, barAreaCol, -coreY,coreZ,-coreY,-coreZ)
    op.layer('straight', IDreinf, numBarsCol, barAreaCol, coreY,coreZ, coreY,-coreZ)

    # BEAM section:
    op.section('Elastic', BeamSecTag,Ec,ABeam,IzBeam)	# elastic beam section)

elif seccion == 'Agregator':
    #Define Elements and Sections
    ColMatTagFlex  = 2
    ColMatTagAxial = 3

    Fy = 66.8*ksi			# STEEL yield stress
    E0 = 29000.0*ksi		# modulus of steel
    Es = E0;
    b = 0.01
    R0=18.5
    cR1=0.925
    cR2=0.15
    params=[R0,cR1,cR2]
    a2 = 1.0
    a1=a2*Fy/E0
    a4 = 1.0
    a3=a4*Fy/E0

    op.uniaxialMaterial('Steel02', ColMatTagFlex, Fy, E0, b, R0, cR1, cR2, a1, a2, a3, a4) #steel moment curvature isused for Mz of the section only, # bilinear behavior for flexure
    op.uniaxialMaterial('Elastic', ColMatTagAxial, E0) # this is not used as a material, this is an axial-force-strain response
    op.section('Aggregator', ColSecTag, ColMatTagAxial, 'P', ColMatTagFlex, 'Mz')  # combine axial and flexural behavior into one section (no P-M interaction here)

    # BEAM section:
    op.section('Elastic', BeamSecTag,E0,ABeam,IzBeam)	# elastic beam section)

ColTransfTag = 1
BeamTransfTag = 2
op.geomTransf('Linear', ColTransfTag)
op.geomTransf('Linear', BeamTransfTag)

numIntgrPts = 5

IntegTagCol = 1
IntegTagBeam = 2

# We are using gauss-Legendre  integration as it is the default integration scheme used in opensees tcl
op.beamIntegration('Legendre', IntegTagCol , ColSecTag, 3)
op.beamIntegration('Legendre', IntegTagBeam , BeamSecTag, 3)

# Select the section Type
sectiontype = 'elastic' # elastic - nonlinear - disp - force

if sectiontype == 'nonlinear':
    # columns
    op.element('nonlinearBeamColumn', 1, 1, 4, numIntgrPts, ColSecTag, ColTransfTag)
    op.element('nonlinearBeamColumn', 2, 2, 5, numIntgrPts, ColSecTag, ColTransfTag)
    op.element('nonlinearBeamColumn', 3, 4, 7, numIntgrPts, ColSecTag, ColTransfTag)
    op.element('nonlinearBeamColumn', 4, 5, 8, numIntgrPts, ColSecTag, ColTransfTag)
    op.element('nonlinearBeamColumn', 5, 6, 9, numIntgrPts, ColSecTag, ColTransfTag)
    op.element('nonlinearBeamColumn', 6, 7, 10, numIntgrPts, ColSecTag, ColTransfTag)
    op.element('nonlinearBeamColumn', 7, 8, 11, numIntgrPts, ColSecTag, ColTransfTag)
    op.element('nonlinearBeamColumn', 8, 9, 12, numIntgrPts, ColSecTag, ColTransfTag)

    op.element('nonlinearBeamColumn', 15, 3, 6, numIntgrPts, ColSecTag, ColTransfTag)
    # beam
    op.element('nonlinearBeamColumn', 9, 4, 5, numIntgrPts, BeamSecTag, BeamTransfTag)
    op.element('nonlinearBeamColumn', 10, 5, 6, numIntgrPts, BeamSecTag, BeamTransfTag)
    op.element('nonlinearBeamColumn', 11, 7, 8, numIntgrPts, BeamSecTag, BeamTransfTag)
    op.element('nonlinearBeamColumn', 12, 8, 9, numIntgrPts, BeamSecTag, BeamTransfTag)
    op.element('nonlinearBeamColumn', 13, 10, 11, numIntgrPts, BeamSecTag, BeamTransfTag)
    op.element('nonlinearBeamColumn', 14, 11, 12, numIntgrPts, BeamSecTag, BeamTransfTag)

elif sectiontype == 'disp':
    # columns
    op.element('dispBeamColumn', 1, 1, 4, ColTransfTag, IntegTagCol)
    op.element('dispBeamColumn', 2, 2, 5, ColTransfTag, IntegTagCol)
    op.element('dispBeamColumn', 3, 4, 7, ColTransfTag, IntegTagCol)
    op.element('dispBeamColumn', 4, 5, 8, ColTransfTag, IntegTagCol)
    op.element('dispBeamColumn', 5, 6, 9, ColTransfTag, IntegTagCol)
    op.element('dispBeamColumn', 6, 7, 10, ColTransfTag, IntegTagCol)
    op.element('dispBeamColumn', 7, 8, 11, ColTransfTag, IntegTagCol)
    op.element('dispBeamColumn', 8, 9, 12, ColTransfTag, IntegTagCol)

    op.element('dispBeamColumn', 15, 3, 6, ColTransfTag, IntegTagCol)
    # beam
    op.element('dispBeamColumn', 9, 4, 5, BeamTransfTag, IntegTagBeam)
    op.element('dispBeamColumn', 10, 5, 6, BeamTransfTag, IntegTagBeam)
    op.element('dispBeamColumn', 11, 7, 8, BeamTransfTag, IntegTagBeam)
    op.element('dispBeamColumn', 12, 8, 9, BeamTransfTag, IntegTagBeam)
    op.element('dispBeamColumn', 13, 10, 11, BeamTransfTag, IntegTagBeam)
    op.element('dispBeamColumn', 14, 11, 12, BeamTransfTag, IntegTagBeam)

elif sectiontype == 'force':
    # columns
    # op.element('forceBeamColumn', eleTag, *eleNodes, transfTag, integrationTag, '-iter', maxIter=10, tol=1e-12, '-mass', mass=0.0)
    op.element('forceBeamColumn', 1, 1, 4, ColTransfTag, IntegTagCol)
    op.element('forceBeamColumn', 2, 2, 5, ColTransfTag, IntegTagCol)
    op.element('forceBeamColumn', 3, 4, 7, ColTransfTag, IntegTagCol)
    op.element('forceBeamColumn', 4, 5, 8, ColTransfTag, IntegTagCol)
    op.element('forceBeamColumn', 5, 6, 9, ColTransfTag, IntegTagCol)
    op.element('forceBeamColumn', 6, 7, 10, ColTransfTag, IntegTagCol)
    op.element('forceBeamColumn', 7, 8, 11, ColTransfTag, IntegTagCol)
    op.element('forceBeamColumn', 8, 9, 12, ColTransfTag, IntegTagCol)

    op.element('forceBeamColumn', 15, 3, 6, ColTransfTag, IntegTagCol)
    # beam
    op.element('forceBeamColumn', 9, 4, 5, BeamTransfTag, IntegTagBeam)
    op.element('forceBeamColumn', 10, 5, 6, BeamTransfTag, IntegTagBeam)
    op.element('forceBeamColumn', 11, 7, 8, BeamTransfTag, IntegTagBeam)
    op.element('forceBeamColumn', 12, 8, 9, BeamTransfTag, IntegTagBeam)
    op.element('forceBeamColumn', 13, 10, 11, BeamTransfTag, IntegTagBeam)
    op.element('forceBeamColumn', 14, 11, 12, BeamTransfTag, IntegTagBeam)
elif sectiontype == 'elastic':
    # columns
    op.element('elasticBeamColumn', 1, 1, 4, ACol, Es, IzCol, ColTransfTag)
    op.element('elasticBeamColumn', 2, 2, 5, ACol, Es, IzCol, ColTransfTag)
    op.element('elasticBeamColumn', 3, 4, 7, ACol, Es, IzCol, ColTransfTag)
    op.element('elasticBeamColumn', 4, 5, 8, ACol, Es, IzCol, ColTransfTag)
    op.element('elasticBeamColumn', 5, 6, 9, ACol, Es, IzCol, ColTransfTag)
    op.element('elasticBeamColumn', 6, 7, 10, ACol, Es, IzCol, ColTransfTag)
    op.element('elasticBeamColumn', 7, 8, 11, ACol, Es, IzCol, ColTransfTag)
    op.element('elasticBeamColumn', 8, 9, 12, ACol, Es, IzCol, ColTransfTag)

    op.element('elasticBeamColumn', 15, 3, 6, ACol, Es, IzCol, ColTransfTag)
    # beam
    op.element('elasticBeamColumn', 9, 4, 5, ABeam, Es, IzBeam, BeamTransfTag)
    op.element('elasticBeamColumn', 10, 5, 6, ABeam, Es, IzBeam, BeamTransfTag)
    op.element('elasticBeamColumn', 11, 7, 8, ABeam, Es, IzBeam, BeamTransfTag)
    op.element('elasticBeamColumn', 12, 8, 9, ABeam, Es, IzBeam, BeamTransfTag)
    op.element('elasticBeamColumn', 13, 10, 11, ABeam, Es, IzBeam, BeamTransfTag)
    op.element('elasticBeamColumn', 14, 11, 12, ABeam, Es, IzBeam, BeamTransfTag)
else:
    print('Undefined section')

#defining gravity loads
WzBeam = Weight/LBeam
op.timeSeries('Linear', 1)
op.pattern('Plain', 1, 1)
op.eleLoad('-ele', [9, 10, 11, 12, 13, 14], '-type', '-beamUniform', -WzBeam, 0.0, 0.0)
Tol = 1e-8                       # convergence tolerance for test
op.integrator('LoadControl', 1)  # determine the next time step for an analysis
op.numberer('Plain')             # renumber dof's to minimize band-width (optimization), if you want to
op.system('BandGeneral')         # how to store and solve the system of equations in the analysis
op.constraints('Plain')          # how it handles boundary conditions
op.test('NormDispIncr', Tol, 6)  # determine if convergence has been achieved at the end of an iteration step
op.algorithm('Newton')           # use Newton's solution algorithm: updates tangent stiffness at every iteration
op.analysis('Static')            # define type of analysis static or transient
op.analyze(1)                    # apply gravity

op.loadConst('-time', 0.0)       # maintain constant gravity loads and reset time to zero

#%%   SEISMIC RECORD APPLIED IN A FOR LOOP
op.wipeAnalysis()

#applying Dynamic Ground motion analysis
Tol = 1e-3
maxNumIter = 10
GMdirection = 1
GMfact = 1.5
GMfatt = g*GMfact

Lambda = op.eigen('-fullGenLapack', 9) # eigenvalue mode 1
Omega = math.pow(Lambda[0], 0.5)
omega = np.sqrt(Lambda)
period = 2.0*np.pi/omega
freq = 1/period
betaKcomm = 2 * (0.02/Omega)

Damping = 'Modal' # Modal - Rayleigh

if Damping == 'Rayleigh':
    xDamp = 0.05				                        # 5% damping ratio
    alphaM = 0.0				                        # M-prop. damping; D = alphaM*M
    betaKcurr = 0.0		                                # K-proportional damping;      +beatKcurr*KCurrent
    betaKinit = 0.0                                     # initial-stiffness proportional damping      +beatKinit*Kini
    op.rayleigh(alphaM,betaKcurr, betaKinit, betaKcomm) # RAYLEIGH damping
elif Damping == 'Modal':
    zeta = 0.05
    op.modalDamping(zeta) # Modal Damping
else:
    print('Unspecified damping')

import scipy.io
excitation = scipy.io.loadmat('excitation.mat')
register = np.transpose(excitation['exc'][::1])
Lista = []
for i in range(register.shape[0]):
    Lista.extend(register[i])
register = np.array(np.transpose(Lista))
time_steps = register.shape[0]

#time_steps = 120000
#register = 0.25 * (np.random.rand(time_steps) - 0.5)
dt = 0.02

i=1
motion = register[0:-1]
values = list(-1 * motion)  # should be negative
op.timeSeries('Path', i+20000, '-dt', dt, '-values', *values, '-factor', GMfatt)
IDloadTag = i+4000		# load tag
op.pattern('UniformExcitation', IDloadTag, GMdirection, '-accel', i+20000)

index = list(np.linspace(0, (len(motion)-1)*dt, len(motion)))

time2 = [0.0]

u44 = [0.0]
u77= [0.0]

okk = 0

op.constraints('Transformation')
op.numberer('Plain')
op.system('BandGeneral')
op.test('EnergyIncr', Tol, maxNumIter)
op.algorithm('Linear')
op.integrator('AlphaOS',0.9)
op.analysis('Transient')

for i in range(0,time_steps-1):
    okk = op.analyze(1, dt)
    if okk == 0 :
        tCurrent = dt*i
        time2.append(tCurrent)
        u44.append(op.nodeDisp(4,1))
        u77.append(op.nodeDisp(7,1))
```

```python
# Displat Period and Frequency of the model
d = [ ["Mode 1", period[0], freq[0]],
      ["Mode 2", period[1], freq[1]],
      ["Mode 3", period[2], freq[2]],
      ["Mode 4", period[3], freq[3]],
      ["Mode 5", period[4], freq[4]],
      ["Mode 6", period[5], freq[5]],
      ["Mode 7", period[6], freq[6]],
      ["Mode 8", period[7], freq[7]],
      ["Mode 9", period[8], freq[8]]]

print(tabulate(d, headers=["Mode", "Period [s]", "Frequency [Hz]"]))
```

    Mode      Period [s]    Frequency [Hz]
    ------  ------------  ----------------
    Mode 1    0.194866             5.13173
    Mode 2    0.0504181           19.8341
    Mode 3    0.0281166           35.5662
    Mode 4    0.00555031         180.17
    Mode 5    0.00552443         181.014
    Mode 6    0.00545223         183.411
    Mode 7    0.00320521         311.992
    Mode 8    0.0032             312.5
    Mode 9    0.00318628         313.846

```python
# 1. plot model with tag lebels
opsv.plot_model()
plt.show()
op.wipe()
```

```python
d = {'exc': register, 'disp': u44}
gen_data = pd.DataFrame(data=d)
gen_data.head()
```

      exc
      disp

      0
      0.000000
      0.000000

      1
      -0.008678
      -0.000432

      2
      -0.008670
      -0.001764

      3
      -0.008663
      -0.003683

      4
      -0.008659
      -0.005487

```python
x = gen_data['disp']
a = gen_data['exc']
t = np.linspace(0,dt*time_steps,time_steps)

fig = plt.figure(figsize=(12, 8))
gs = gridspec.GridSpec(2, 2, width_ratios=[3, 1])

ax0 = plt.subplot(gs[0])
ax0.plot(t, x)
ax0.set_xlim([0, 160])
ax0.title.set_text('Displacement')
ax0.set_ylim([-0.6,0.6])
ax0.grid()

ax1 = plt.subplot(gs[1])
ax1.plot(t, x)
ax1.set_xlim([10,20])
ax1.title.set_text('Zoom')
ax1.set_ylim([-0.6,0.6])
ax1.grid()

ax2 = plt.subplot(gs[2])
ax2.plot(t, a)
ax2.set_xlim([0,160])
ax2.title.set_text('Excitation')
ax2.set_ylim([-1.1,1.1])
ax2.grid()

ax3 = plt.subplot(gs[3])
ax3.plot(t, a)
ax3.set_xlim([10,20])
ax3.title.set_text('Zoom')
ax3.set_ylim([-1.1,1.1])
ax3.grid()

plt.show()
```

```python
for col in gen_data.columns:
    mean = gen_data[col].mean()
    minimum = gen_data[col].min()
    maximum = gen_data[col].max()
    gen_data[col] = (gen_data[col]-mean)/(maximum - minimum)
    gen_data[col] = gen_data[col] / 2.0 + 0.5
```

## Preprocessing

```python
n = 100
```

```python
train_df = gen_data.iloc[:200000, :]
delayed_disp = train_df['disp'][:len(train_df)-n].rename('del_disp')
train_df = train_df.iloc[n:, :].reset_index(drop=True).join(delayed_disp)
train_df.head(105)
```

      exc
      disp
      del_disp

      0
      0.498362
      0.498996
      0.500027

      1
      0.498502
      0.499096
      0.499888

      2
      0.498689
      0.499199
      0.499459

      3
      0.498863
      0.499303
      0.498840

      4
      0.499022
      0.499407
      0.498259

      ...
      ...
      ...
      ...

      100
      0.492884
      0.495776
      0.498996

      101
      0.493220
      0.496147
      0.499096

      102
      0.493719
      0.496447
      0.499199

      103
      0.494469
      0.496687
      0.499303

      104
      0.495512
      0.496951
      0.499407

105 rows × 3 columns

```python
train_df.loc[:3500, :].plot()
plt.show()
```

### Some config params

```python
td_shape = train_df.shape
td_l = td_shape[0]
```

```python
x_labels = ['exc', 'del_disp']
y_labels = ['disp']
```

```python
n_input_labels = len(x_labels)
y_offset = 1
```

```python
train_percentage = 0.8 # porcentaje de la base de datos que será utilizado como entrenamiento.
length_train_data = len(train_df)
split_index = int(length_train_data*train_percentage)
```

```python
split_index
```

    159920

## Get train and test data

### Walk forward
Las bases de datos de entrenamiento y testeo son reorganizadas. Como se aprecio previamente se resolve el sistema dinámico de forma tal que se obtiene una base de datos con inputs y outputs. El reordenamiento es para generar un conjunto de datos de input/output pero múltiple. Así se obtienen varios registros (series temporales) de input/output para entrenamiento y testeo y no una única serie temporal.

```python
X_train, X_test, y_train, y_test = walk_forward(train_df,
                                                x_labels, y_labels,
                                                n, n_input_labels, y_offset=y_offset,
                                                split_per=0.7, split_data=True)

X_train = X_train.reshape(-1, n, n_input_labels, 1)
y_train = y_train.reshape(-1, n-y_offset)
```

## Model
A continuación, se define el modelo de la Red Neuronal

```python
tb_callback = TensorBoard(log_dir='./graph', histogram_freq=0, write_graph=True, write_images=True)
```

```python
class Model():

    def __init__(self):
        self.mdl = None
        self.hist = None

    def create_model(self, input_shape, n_outputs):

        # Optimiser
        opt = Adam(learning_rate=0.001)

        # define model
        self.mdl = Sequential()

        self.mdl.add(Conv2D(9, (2,1), input_shape=input_shape))
        self.mdl.add(Dropout(0.2))
        self.mdl.add(BatchNormalization())

        self.mdl.add(Flatten())
        self.mdl.add(Dense(200, activation='relu'))
        self.mdl.add(Dropout(0.2))
        self.mdl.add(BatchNormalization())

        self.mdl.add(Dense(n_outputs))

        # Prepare model for training
        self.mdl.compile(loss='mae',
                      optimizer=opt,
                      metrics=['mse'])

    def run_model(self, X_train, X_test, y_train, y_test, callback):

        # Fit model
        self.hist = self.mdl.fit(X_train, y_train,
                                batch_size=1000,
                                epochs=30,
                                verbose=True,
                                validation_data=(X_test, y_test),
                                callbacks=[callback])
```

```python
# Create Model
model = Model()
model.create_model(X_train[0].shape, n-y_offset)
```

### Sumarry
Keras proporciona una forma de resumir un modelo.El resumen es textual e incluye información sobre:
- Las capas y su orden en el modelo.
- La forma de salida de cada capa.
- El número de parámetros (pesos) en cada capa.
- El número total de parámetros (pesos) en el modelo.

```python
model.mdl.summary()
```

    Model: "sequential"
    _________________________________________________________________
     Layer (type)                Output Shape              Param #
    =================================================================
     conv2d (Conv2D)             (None, 99, 2, 9)          27

     dropout (Dropout)           (None, 99, 2, 9)          0

     batch_normalization (BatchN  (None, 99, 2, 9)         36
     ormalization)

     flatten (Flatten)           (None, 1782)              0

     dense (Dense)               (None, 200)               356600

     dropout_1 (Dropout)         (None, 200)               0

     batch_normalization_1 (Batc  (None, 200)              800
     hNormalization)

     dense_1 (Dense)             (None, 99)                19899

    =================================================================
    Total params: 377,362
    Trainable params: 376,944
    Non-trainable params: 418
    _________________________________________________________________

- Una vez creado el modelo, se puede configurar el modelo con pérdidas y métricas con model.compile(), entrenar el modelo con model.fit() o usar el modelo para hacer predicciones model.predict().

```python
model.run_model(X_train, X_test, y_train, y_test, tb_callback)
```

    Epoch 1/30
    140/140 [==============================] - 15s 94ms/step - loss: 0.3656 - mse: 0.3747 - val_loss: 0.0222 - val_mse: 0.0010
    Epoch 2/30
    140/140 [==============================] - 13s 93ms/step - loss: 0.0337 - mse: 0.0249 - val_loss: 0.0258 - val_mse: 0.0012
    Epoch 3/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0112 - mse: 3.9139e-04 - val_loss: 0.0172 - val_mse: 6.5404e-04
    Epoch 4/30
    140/140 [==============================] - 13s 91ms/step - loss: 0.0096 - mse: 3.1336e-04 - val_loss: 0.0132 - val_mse: 4.0527e-04
    Epoch 5/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0093 - mse: 3.0144e-04 - val_loss: 0.0094 - val_mse: 2.0998e-04
    Epoch 6/30
    140/140 [==============================] - 13s 93ms/step - loss: 0.0092 - mse: 3.0018e-04 - val_loss: 0.0071 - val_mse: 1.2329e-04
    Epoch 7/30
    140/140 [==============================] - 13s 94ms/step - loss: 0.0092 - mse: 2.9815e-04 - val_loss: 0.0060 - val_mse: 9.4556e-05
    Epoch 8/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0091 - mse: 2.9774e-04 - val_loss: 0.0061 - val_mse: 9.8988e-05
    Epoch 9/30
    140/140 [==============================] - 13s 91ms/step - loss: 0.0092 - mse: 2.9899e-04 - val_loss: 0.0060 - val_mse: 9.7227e-05
    Epoch 10/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0091 - mse: 3.0082e-04 - val_loss: 0.0060 - val_mse: 9.6170e-05
    Epoch 11/30
    140/140 [==============================] - 13s 93ms/step - loss: 0.0091 - mse: 2.9549e-04 - val_loss: 0.0066 - val_mse: 1.0764e-04
    Epoch 12/30
    140/140 [==============================] - 13s 92ms/step - loss: 0.0091 - mse: 2.9713e-04 - val_loss: 0.0060 - val_mse: 9.7176e-05
    Epoch 13/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0092 - mse: 2.9730e-04 - val_loss: 0.0067 - val_mse: 1.0929e-04
    Epoch 14/30
    140/140 [==============================] - 13s 91ms/step - loss: 0.0092 - mse: 2.9641e-04 - val_loss: 0.0063 - val_mse: 1.0031e-04
    Epoch 15/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0091 - mse: 2.9582e-04 - val_loss: 0.0067 - val_mse: 1.0788e-04
    Epoch 16/30
    140/140 [==============================] - 13s 93ms/step - loss: 0.0093 - mse: 3.0072e-04 - val_loss: 0.0061 - val_mse: 9.9491e-05
    Epoch 17/30
    140/140 [==============================] - 13s 93ms/step - loss: 0.0091 - mse: 2.9629e-04 - val_loss: 0.0063 - val_mse: 9.8743e-05
    Epoch 18/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0090 - mse: 2.9556e-04 - val_loss: 0.0065 - val_mse: 1.0731e-04
    Epoch 19/30
    140/140 [==============================] - 13s 91ms/step - loss: 0.0091 - mse: 2.9573e-04 - val_loss: 0.0064 - val_mse: 1.0207e-04
    Epoch 20/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0090 - mse: 2.9685e-04 - val_loss: 0.0065 - val_mse: 1.0499e-04
    Epoch 21/30
    140/140 [==============================] - 13s 93ms/step - loss: 0.0091 - mse: 2.9704e-04 - val_loss: 0.0070 - val_mse: 1.1599e-04
    Epoch 22/30
    140/140 [==============================] - 13s 92ms/step - loss: 0.0091 - mse: 2.9551e-04 - val_loss: 0.0061 - val_mse: 1.0131e-04
    Epoch 23/30
    140/140 [==============================] - 12s 87ms/step - loss: 0.0090 - mse: 2.9586e-04 - val_loss: 0.0060 - val_mse: 9.4639e-05
    Epoch 24/30
    140/140 [==============================] - 13s 89ms/step - loss: 0.0089 - mse: 2.8692e-04 - val_loss: 0.0055 - val_mse: 7.8835e-05
    Epoch 25/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0087 - mse: 2.7854e-04 - val_loss: 0.0061 - val_mse: 8.7666e-05
    Epoch 26/30
    140/140 [==============================] - 13s 94ms/step - loss: 0.0087 - mse: 2.7882e-04 - val_loss: 0.0059 - val_mse: 9.2527e-05
    Epoch 27/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0087 - mse: 2.7877e-04 - val_loss: 0.0061 - val_mse: 9.9149e-05
    Epoch 28/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0086 - mse: 2.7727e-04 - val_loss: 0.0070 - val_mse: 1.0411e-04
    Epoch 29/30
    140/140 [==============================] - 13s 90ms/step - loss: 0.0087 - mse: 2.8015e-04 - val_loss: 0.0061 - val_mse: 8.6987e-05
    Epoch 30/30
    140/140 [==============================] - 12s 89ms/step - loss: 0.0086 - mse: 2.7688e-04 - val_loss: 0.0073 - val_mse: 1.0618e-04

## Evaluation

```python
# Gráfico de error (%)
# Histograma de la diferencia entre el valor predecido y el de testeo.
preds = model.mdl.predict(X_test)
resids_all = preds-y_test
resids = pd.DataFrame((preds - y_test).reshape(-1))
resids.plot(kind='hist', bins=100, title='Residuals Histogram')
print(resids.describe())
```

```python
# Test D'Agostino u Pearson's
from numpy.random import seed
from numpy.random import randn
from scipy.stats import normaltest

# Observaciones
data = resids.values

# Test de normalidad
stat, p = normaltest(data)

print('Statistics=%.3f, p=%.3f' % (stat, p))

# interpretar
alpha = 0.05
if p > alpha:
    print('La muestra parece gaussiana (no rechaza H0)')
else:
    print('La muestra no parece gaussiana (rechazar H0)')

# q-q plot
qqplot(data, line='s')
pyplot.show()
```

- Al ejecutar, se crea el gráfico QQ que muestra el gráfico de dispersión de puntos como una nube de puntos, que no se ajusta al patrón diagonal esperado para una muestra de una distribución gaussiana.

```python
# Devuelve el valor de pérdida y los valores de las métricas del modelo en modo de prueba.
names = model.mdl.metrics_names
value = model.mdl.evaluate(X_test, y_test, verbose=0)

print('\nMetricas:\n ')
print(names[0],': ',value[0])
print(names[1],': ',value[1])
print('r2 Score:   ',r2_score(y_test, preds))
print("")
```

    Metricas:

    loss :  0.0072527979500591755
    mse :  0.00010617761290632188
    r2 Score:    0.8847122959986324

```python
l1 = 1990
l2 = 2060

a = np.transpose(preds)
pred = a.reshape(-1)

c = np.transpose(y_test)
testeo = c.reshape(-1)

error = (abs(testeo-pred))

RMS = np.sqrt(np.mean((testeo-pred)**2))/np.sqrt(np.mean(testeo**2))*100

pasos = pred.shape[0]
t = np.linspace(0,dt*pasos,pasos)

fig = plt.figure(figsize=(16, 10))
gs = gridspec.GridSpec(2, 1)

ax0 = plt.subplot(gs[0])
ax0.plot(t, testeo,'r',label="Test")
ax0.plot(t, pred,'k--',label="Prediction")
ax0.title.set_text('Displacement comparison Pred v/s Test')
ax0.legend(loc="upper right")
ax0.set_xlim([l1, l2])
ax0.grid()

ax1 = plt.subplot(gs[1])
ax1.plot(t, error,label="RMSE: "+str(round(RMS,2))+"[%]")
ax1.title.set_text('Error [in]')
ax1.legend(loc="upper right")
ax1.set_xlim([l1, l2])
ax1.set_ylim([0, 0.1])
ax1.grid()

plt.show()
```

```python
comp_ind = 2000
comparison = pd.concat([pd.DataFrame(preds[comp_ind].reshape(-1), columns=['preds']),
                        pd.DataFrame(y_test[comp_ind].reshape(-1), columns=['true'])], axis=1)
comparison.plot(figsize=(15,10))
plt.show()
```

```python
indices = np.random.randint(low=0, high=len(resids_all), size=1)
pd.DataFrame((resids_all[indices])).T.plot(figsize=(15,10), legend=False)
plt.show()
```

```python
pred = preds[comp_ind]
testeo = y_test[comp_ind]
error = abs((testeo-pred)/max(abs(testeo))*100)
RMSE = mean_squared_error(testeo, pred, squared=False)

pasos = preds[comp_ind].shape[0]
t = np.linspace(0,dt*pasos,pasos)

fig = plt.figure(figsize=(12, 8))
gs = gridspec.GridSpec(2, 1)

ax0 = plt.subplot(gs[0])
ax0.plot(t, testeo,'r',label="Test")
ax0.plot(t, pred,'k--',label="Prediction")
ax0.title.set_text('Displacement comparison Pred v/s Test')
ax0.grid()

ax1 = plt.subplot(gs[1])
ax1.plot(t, error,label="RMSE: "+str(round(RMSE*100,2))+"[%]")
ax1.title.set_text('Error [%]')
ax1.grid()

plt.legend()
plt.show()
```

```python

```


# Construction of Regression Models using Data

    Author: Jerónimo Arenas García (jarenas@tsc.uc3m.es)
            Jesús Cid Sueiro (jcid@tsc.uc3m.es)

    Notebook version: 2.1 (Sep 27, 2019)

    Changes: v.1.0 - First version. Extracted from regression_intro_knn v.1.0.
             v.1.1 - Compatibility with python 2 and python 3
             v.2.0 - New notebook generated. Fuses code from Notebooks R1, R2, and R3
             v.2.1 - Updated index notation

```python
# Import some libraries that will be necessary for working with data and displaying plots

# To visualize plots in the notebook
%matplotlib inline

import numpy as np
import scipy.io       # To read matlab files
import pandas as pd   # To read data tables from csv files

# For plots and graphical results
import matplotlib
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import pylab

# For the student tests (only for python 2)
import sys
if sys.version_info.major==2:
    from test_helper import Test

# That's default image size for this interactive session
pylab.rcParams['figure.figsize'] = 9, 6
```

## 1. The regression problem

The goal of regression methods is to predict the value of some *target* variable $S$ from the observation of one or more *input* variables $X_0, X_1, \ldots, X_{K-1}$ (that we will collect in a single vector $\bf X$).

Regression problems arise in situations where the value of the target variable is not easily accessible, but we can measure other dependent variables, from which we can try to predict $S$.

The only information available to estimate the relation between the inputs and the target is a *dataset* $\mathcal D$ containing several observations of all variables.

$$\mathcal{D} = \{{\bf x}_k, s_k\}_{k=0}^{K-1}$$

The dataset $\mathcal{D}$ must be used to find a function $f$ that, for any observation vector ${\bf x}$, computes an output $\hat{s} = f({\bf x})$ that is a good predition of the true value of the target, $s$.

Note that for the generation of the regression model, we exploit the statistical dependence between random variable $S$ and random vector ${\bf X}$. In this respect, we can assume that the available dataset $\mathcal{D}$ consists of i.i.d. points from the joint distribution $p_{S,{\bf X}}(s,{\bf x})$. If we had access to the true distribution, a statistical approach would be more accurate; however, in many situations such knowledge is not available, but using training data to do the design is feasible (e.g., relying on historic data, or by manual labelling of a set of patterns).

## 2. Examples of regression problems.

The scikit-learn package contains several  datasets related to regression problems.

*  Boston dataset: the target variable contains housing values in different suburbs of Boston. The goal is to predict these values based on several social, economic and demographic variables taken frome theses suburbs (you can get more details in the  UCI repository ).

*  Diabetes dataset.

We can load these datasets as follows:

```python
from sklearn import datasets

# Load the dataset. Select it by uncommenting the appropriate line
D_all = datasets.load_boston()
#D_all = datasets.load_diabetes()

# Extract data and data parameters.
X = D_all.data         # Complete data matrix (including input and target variables)
S = D_all.target      # Target variables
n_samples = X.shape[0] # Number of observations
n_vars = X.shape[1]    # Number of variables (including input and target)
```

This dataset contains

```python
print(n_samples)
```

    506

observations of the target variable and

```python
print(n_vars)
```

    13

input variables.

## 3. Scatter plots

### 3.1. 2D scatter plots

When the instances of the dataset are multidimensional, they cannot be visualized directly, but we can get a first rough idea about the regression task if we plot the target variable versus one of the input variables. These representations are known as scatter plots

Python methods `plot` and `scatter` from the `matplotlib` package can be used for these graphical representations.

```python
# Select a dataset
nrows = 4
ncols = 1 + (X.shape[1]-1)/nrows

# Some adjustment for the subplot.
pylab.subplots_adjust(hspace=0.2)

# Plot all variables
for idx in range(X.shape[1]):
    ax = plt.subplot(nrows,ncols,idx+1)
    ax.scatter(X[:,idx], S)    # <-- This is the key command
    ax.get_xaxis().set_ticks([])
    ax.get_yaxis().set_ticks([])
    plt.ylabel('Target')

```

## 4. Evaluating a regression task

In order to evaluate the performance of a given predictor, we need to quantify the quality of predictions. This is usually done by means of a loss function $l(s,\hat{s})$. Two common losses are

   - Square error: $l(s, \hat{s}) = (s - \hat{s})^2$
   - Absolute error: $l(s, \hat{s}) = |s - \hat{s}|$

Note that both the square and absolute errors are functions of the estimation error $e = s-{\hat s}$. However, this is not necessarily the case. As an example, imagine a situation in which we would like to introduce a penalty which increases with the magnitude of the estimated variable. For such case, the following cost would better fit our needs: $l(s,{\hat s}) = s^2 \left(s-{\hat s}\right)^2$.

```python
# In this section we will plot together the square and absolute errors
grid = np.linspace(-3,3,num=100)
plt.plot(grid, grid**2, 'b-', label='Square error')
plt.plot(grid, np.absolute(grid), 'r--', label='Absolute error')
plt.xlabel('Error')
plt.ylabel('Cost')
plt.legend(loc='best')
plt.show()
```

In general, we do not care much about an isolated application of the regression model, but instead, we are looking for a generally good behavior, for which we need to average the loss function over a set of samples. In this notebook, we will use the average of the square loss, to which we will refer as the `mean-square error` (MSE).

$$\text{MSE} = \frac{1}{K}\sum_{k=0}^{K-1} \left(s^{(k)}- {\hat s}^{(k)}\right)^2$$

The following code fragment defines a function to compute the MSE based on the availability of two vectors, one of them containing the predictions of the model, and the other the true target values.

```python
# We start by defining a function that calculates the average square error
def square_error(s, s_est):
    # Squeeze is used to make sure that s and s_est have the appropriate dimensions.
    y = np.mean(np.power((np.squeeze(s) - np.squeeze(s_est)), 2))
    return y
```

### 4.1. Training and test data

The major goal of the regression problem is that the predictor should make good predictions for arbitrary new inputs, not taken from the dataset used by the regression algorithm.

Thus, in order to evaluate the prediction accuracy of some regression algorithm, we need some data, not used during the predictor design, to *test* the performance of the predictor under new data. To do so, the original dataset is usually divided in (at least) two disjoint sets:

* **Training set**, $\cal{D}_{\text{train}}$: Used by the regression algorithm to determine predictor $f$.
* **Test set**, $\cal{D}_{\text{test}}$: Used to evaluate the performance of the regression algorithm.

A good regression algorithm uses $\cal{D}_{\text{train}}$ to obtain a predictor with small average loss based on $\cal{D}_{\text{test}}$
$$
{\bar R}_{\text{test}} = \frac{1}{K_{\text{test}}}
\sum_{  ({\bf x},s) \in \mathcal{D}_{\text{test}}} l(s, f({\bf x}))
$$
where $K_{\text{test}}$ is the size of the test set.

As a designer, you only have access to training data. However, for illustration purposes, you may be given a test dataset for many examples in this course. Note that in such a case, using the test data to adjust the regression model is completely forbidden. You should work as if such test data set were not available at all, and recur to it just to assess the performance of the model after the design is complete.

To model the availability of a train/test partition, we split next the boston dataset into a training and test partitions, using 60% and 40% of the data, respectively.

```python
from sklearn.model_selection import train_test_split

X_train, X_test, s_train, s_test = train_test_split(X, S, test_size=0.4, random_state=0)
```

### 4.2. A first example: A baseline regression model

A first very simple method to build the regression model is to use the average of all the target values in the training set as the output of the model, discarding the value of the observation input vector.

This approach can be considered as a baseline, given that any other method making an effective use of the observation variables, statistically related to $s$, should improve the performance of this method.

The following code fragment uses the train data to compute the baseline regression model, and it shows the MSE calculated over the test partitions.

```python
S_baseline = np.mean(s_train)

print('The baseline estimator is:', S_baseline)

#Compute MSE for the train data
#MSE_train = square_error(s_train, S_baseline)

#Compute MSE for the test data. IMPORTANT: Note that we still use
#S_baseline as the prediction.
MSE_test = square_error(s_test, S_baseline)

#print('The MSE for the training data is:', MSE_train)
print('The MSE for the test data is:', MSE_test)
```

    The baseline estimator is: 22.7465346535
    The MSE for the test data is: 83.0143622257

## 5. Parametric and non-parametric regression models

Generally speaking, we can distinguish two approaches when designing a regression model:

   - Parametric approach: In this case, the estimation function is given a priori a parametric form, and the goal of the design is to find the most appropriate values of the parameters according to a certain goal

   For instance, we could assume a linear expression
   $${\hat s} = f({\bf x}) = {\bf w}^\top {\bf x}$$
   and adjust the parameter vector in order to minimize the average of the quadratic error over the training data. This is known as least-squares regression, and we will study it in Section 8 of this notebook.

   - Non-parametric approach: In this case, the analytical shape of the regression model is not assumed a priori.

## 6. Non parametric method: Regression with the $k$-nn method

The principles of the $k$-nn method are the following:

   - For each point where a prediction is to be made, find the $k$ closest neighbors to that point (in the training set)
   - Obtain the estimation averaging the labels corresponding to the selected neighbors

The number of neighbors is a hyperparameter that plays an important role in the performance of the method. You can test its influence by changing $k$ in the following piece of code.

```python
from sklearn import neighbors

n_neighbors = 1

knn = neighbors.KNeighborsRegressor(n_neighbors)
knn.fit(X_train, s_train)

s_hat_train = knn.predict(X_train)
s_hat_test = knn.predict(X_test)

print('The MSE for the training data is:', square_error(s_train, s_hat_train))
print('The MSE for the test data is:', square_error(s_test, s_hat_test))
```

    The MSE for the training data is: 0.0
    The MSE for the test data is: 57.2859605911

```python
max_k = 25
n_neighbors_list = np.arange(max_k)+1

MSE_train = []
MSE_test = []

for n_neighbors in n_neighbors_list:
    knn = neighbors.KNeighborsRegressor(n_neighbors)
    knn.fit(X_train, s_train)

    s_hat_train = knn.predict(X_train)
    s_hat_test = knn.predict(X_test)

    MSE_train.append(square_error(s_train, s_hat_train))
    MSE_test.append(square_error(s_test, s_hat_test))

plt.plot(n_neighbors_list, MSE_train,'bo', label='Training square error')
plt.plot(n_neighbors_list, MSE_test,'ro', label='Test square error')
plt.xlabel('$k$')
plt.axis('tight')

plt.legend(loc='best')
plt.show()
```

Although the above figures illustrate evolution of the training and test MSE for different selections of the number of neighbors, it is important to note that **this figure, and in particular the red points, cannot be used to select the value of such parameter**. Remember that it is only legal to use the test data to assess the final performance of the method, what includes also that any parameters inherent to the method should be adjusted using the train data only.

## 7. Hyperparameter selection via cross-validation

An inconvenient of the application of the $k$-nn method is that the selection of $k$ influences the final error of the algorithm. In the previous experiments, we kept the value of $k$ that minimized the square error on the training set. However, we also noticed that the location of the minimum is not necessarily the same from the perspective of the test data. Ideally, we would like that the designed regression model works as well as possible on future unlabeled patterns that are not available during the training phase. This property is known as generalization. Fitting the training data is only pursued in the hope that we are also indirectly obtaining a model that generalizes well. In order to achieve this goal, there are some strategies that try to guarantee a correct generalization of the model. One of such approaches is known as cross-validation

Since using the test labels during the training phase is not allowed (they should be kept aside to simultate the future application of the regression model on unseen patterns), we need to figure out some way to improve our estimation of the hyperparameter that requires only training data. Cross-validation allows us to do so by following the following steps:

   - Split the training data into several (generally non-overlapping) subsets. If we use $M$ subsets, the method is referred to as $M$-fold cross-validation. If we consider each pattern a different subset, the method is usually referred to as leave-one-out (LOO) cross-validation.
   - Carry out the training of the system $M$ times. For each run, use a different partition as a validation set, and use the restating partitions as the training set. Evaluate the performance for different choices of the hyperparameter (i.e., for different values of $k$ for the $k$-NN method).
   - Average the validation error over all partitions, and pick the hyperparameter that provided the minimum validation error.
   - Rerun the algorithm using all the training data, keeping the value of the parameter that came out of the cross-validation process.

**Exercise**: Use `Kfold` function from the `sklearn` library to validate parameter `k`. Use a 10-fold validation strategy. What is the best number of neighbors according to this strategy? What is the corresponding MSE averaged over the test data?

```python
from sklearn.model_selection import KFold

max_k = 25
n_neighbors_list = np.arange(max_k)+1

MSE_val = np.zeros((max_k,))

nfolds = 10
kf = KFold(n_splits=nfolds)
for train, val in kf.split(X_train):
    for idx,n_neighbors in enumerate(n_neighbors_list):
        knn = neighbors.KNeighborsRegressor(n_neighbors)
        knn.fit(X_train[train,:], s_train[train])

        s_hat_val = knn.predict(X_train[val,:])

        MSE_val[idx] += square_error(s_train[val], s_hat_val)

MSE_val = [el/10 for el in MSE_val]

selected_k = np.argmin(MSE_val) + 1

plt.plot(n_neighbors_list, MSE_train,'bo', label='Training square error')
plt.plot(n_neighbors_list, MSE_val,'ro', label='Validation square error')
plt.plot(selected_k, MSE_test[selected_k-1],'gs', label='Test square error')
plt.xlabel('$k$')
plt.axis('tight')

plt.legend(loc='best')
plt.show()

print('Cross-validation selected the following value for the number of neighbors:', selected_k)
print('Test MSE:', MSE_test[selected_k-1])

```

## 8. A parametric regression method: Least squares regression

### 8.1. Problem definition

   - The goal is to learn a (possibly non-linear) regression model from a set of $L$ labeled points, $\{{\bf x}_k,s_k\}_{k=0}^{K-1}$.

   - We assume a parametric function of the form:

   $${\hat s}({\bf x}) = f({\bf x}) = w_0 z_0({\bf x}) + w_1 z_1({\bf x}) + \dots w_{m-1} z_{m-1}({\bf x})$$

   where $z_i({\bf x})$ are particular transformations of the input vector variables.

Some examples are:

   - If ${\bf z} = {\bf x}$, the model is just a linear combination of the input variables

   - If ${\bf z} = \left[\begin{array}{c}1\\{\bf x}\end{array}\right]$, we have again a linear combination with the inclusion of a constant term.

   - For unidimensional input $x$, ${\bf z} = [1, x, x^2, \dots,x^{M}]^\top$ would implement a polynomia of degree $m-1$.

   - Note that the variables of ${\bf z}$ could also be computed combining different variables of ${\bf x}$. E.g., if ${\bf x} = [x_1,x_2]^\top$, a degree-two polynomia would be implemented with
      $${\bf z} = \left[\begin{array}{c}1\\x_1\\x_2\\x_1^2\\x_2^2\\x_1 x_2\end{array}\right]$$

   - The above expression does not assume a polynomial model. For instance, we could consider ${\bf z} = [\log(x_1),\log(x_2)]$

Least squares (LS) regression finds the coefficients of the model with the aim of minimizing the square of the residuals. If we define ${\bf w} = [w_0,w_1,\dots,w_M]^\top$, the LS solution would be defined as

\begin{equation}{\bf w}_{LS} = \arg \min_{\bf w} \sum_{k=0}^{K-1} e_k^2 = \arg \min_{\bf w} \sum_{k=0}^{K-1} \left[s_k - {\hat s}_k \right]^2 \end{equation}

### 8.2. Vector Notation

In order to solve the LS problem it is convenient to define the following vectors and matrices:

   - We can group together all available target values to form the following vector

   $${\bf s} = \left[s_0, s_1, \dots, s_{K-1} \right]^\top$$

   - The estimation of the model for a single input vector ${\bf z}_k$ (which would be computed from ${\bf x}_k$), can be expressed as the following inner product

   $${\hat s}_k = {\bf z}_k^\top {\bf w}$$

   - If we now group all input vectors into a matrix ${\bf Z}$, so that each row of ${\bf Z}$ contains the transpose of the corresponding ${\bf z}_k$, we can express

$$
\hat{{\bf s}}
    = \left[{\hat s}_0, {\hat s}_1, \dots, {\hat s}_{K-1} \right]^\top
    = {\bf Z} {\bf w}, \;\;\;\; \text{with} \;\;
{\bf Z} = \left[\begin{array}{c} {\bf z}_0^\top \\
                                 {\bf z}_1^\top \\
                                 \vdots  \\
                                 {\bf z}_{K-1}^\top \\
                \end{array}\right]$$

### 8.3. Least-squares solution

   - Using the previous notation, the cost minimized by the LS model can be expressed as

   $$C({\bf w}) = \sum_{k=0}^{K-1} \left[s_k - {\hat s}_k \right]^2 = \|{\bf s} - {\hat{\bf s}}\|^2 = \|{\bf s} - {\bf Z}{\bf w}\|^2$$

   - Since the above expression depends quadratically on ${\bf w}$ and is non-negative, we know that there is only one point where the derivative of $C({\bf w})$ becomes zero, and that point is necessarily a minimum of the cost

   $$\nabla_{\bf w} \|{\bf s} - {\bf Z}{\bf w}\|^2\Bigg|_{{\bf w} = {\bf w}_{LS}} = {\bf 0}$$

Exercise:
Solve the previous problem to show that
$${\bf w}_{LS} = \left( {\bf Z}^\top{\bf Z} \right)^{-1} {\bf Z}^\top{\bf s}$$

The next fragment of code adjusts polynomia of increasing order to randomly generated training data.

```python
n_points = 20
n_grid = 200
frec = 3
std_n = 0.2
max_degree = 20

colors = 'brgcmyk'

#Location of the training points
X_tr = (3 * np.random.random((n_points,1)) - 0.5)

#Labels are obtained from a sinusoidal function, and contaminated by noise
S_tr = np.cos(frec*X_tr) + std_n * np.random.randn(n_points,1)

#Equally spaced points in the X-axis
X_grid = np.linspace(np.min(X_tr),np.max(X_tr),n_grid)

#We start by building the Z matrix
Z = []
for el in X_tr.tolist():
    Z.append([el[0]**k for k in range(max_degree+1)])
Z = np.matrix(Z)

Z_grid = []
for el in X_grid.tolist():
    Z_grid.append([el**k for k in range(max_degree+1)])
Z_grid = np.matrix(Z_grid)

plt.plot(X_tr,S_tr,'b.')

for k in [1, 2, n_points]: # range(max_degree+1):
    Z_iter = Z[:,:k+1]

    # Least square solution
    #w_LS = (np.linalg.inv(Z_iter.T.dot(Z_iter))).dot(Z_iter.T).dot(S_tr)

    # Least squares solution, with leass numerical errors
    w_LS, resid, rank, s = np.linalg.lstsq(Z_iter, S_tr)
    #estimates at all grid points
    fout = Z_grid[:,:k+1].dot(w_LS)
    fout = np.array(fout).flatten()
    plt.plot(X_grid,fout,colors[k%len(colors)]+'-',label='Degree '+str(k))

plt.legend(loc='best')
plt.ylim(1.2*np.min(S_tr), 1.2*np.max(S_tr))
plt.show()
```

It may seem that increasing the degree of the polynomia is always beneficial, as we can implement a more expressive function. A polynomia of degree $M$ would include all polynomia of lower degrees as particular cases. However, if we increase the number of parameters without control, the polynomia would eventually get expressive enough to adjust any given set of training points to arbitrary precision, what does not necessarily mean that the solution is obtaining a model that can be extrapolated to new data.

The conclusions is that, when adjusting a parametric model using least squares, we need to validate the model, for which we can use the cross-validation techniques we introudece in Section 7. In this contexts, validating the model implies:
   - Validating the kind of model that will be used, e.g., linear, polynomial, logarithmic, etc ...
   - Validating any additional parameters that the nodel may have, e.g., if selecting a polynomial model, the degree of the polynomia.

The code below shows the performance of different models. However, no validation process is considered, so the reported test MSEs could not be used as criteria to select the best model.

```python
# Linear model with no bias
w_LS, resid, rank, s = np.linalg.lstsq(X_train, s_train)
s_hat_test = X_test.dot(w_LS)
print('Test MSE for linear model without bias:', square_error(s_test, s_hat_test))

# Linear model with no bias
Z_train = np.hstack((np.ones((X_train.shape[0],1)), X_train))
Z_test = np.hstack((np.ones((X_test.shape[0],1)), X_test))

w_LS, resid, rank, s = np.linalg.lstsq(Z_train, s_train)
s_hat_test = Z_test.dot(w_LS)
print('Test MSE for linear model with bias:', square_error(s_test, s_hat_test))

# Polynomial model degree 2
Z_train = np.hstack((np.ones((X_train.shape[0],1)), X_train, X_train**2))
Z_test = np.hstack((np.ones((X_test.shape[0],1)), X_test, X_test**2))

w_LS, resid, rank, s = np.linalg.lstsq(Z_train, s_train)
s_hat_test = Z_test.dot(w_LS)
print('Test MSE for polynomial model (order 2):', square_error(s_test, s_hat_test))

```

    Test MSE for linear model without bias: 25.8546625564
    Test MSE for linear model with bias: 25.7971648592
    Test MSE for polynomial model (order 2): 19.2626529893

```python

```


# Axion-neutron coupling limits vs axion mass

Axion mass-PQ scale relation:
\begin{equation}
m_{a}=5.70(7) \mu \mathrm{eV}\left(\frac{10^{12} \,\mathrm{GeV}}{f_{a}}\right)
\end{equation}

Axion-proton coupling
\begin{equation}
g_{an} \equiv \frac{C_{ap} m_{p}}{f_{a}} = 1.64\times 10^{-7} C_{ap}\frac{m_a}{\mathrm{eV}}
\end{equation}

Model dependent constant:
\begin{equation}
C_{an}  =
\begin{cases}
  -0.47 & {\rm KSVZ} \\
  [-0.2,-0.6] & {\rm DFSZ\,I} \\
  [-0.2,-0.6] & {\rm DFSZ\,II}
\end{cases}
\end{equation}
Like the axion-electron coupling the range of values for DFSZ I and II come from the perturbativity of the Yukawa couplings with sets the range $0.28<v_{u} / v_{d}<140$ for the Higgs vevs.

```python
%matplotlib inline
from PlotFuncs import BlackHoleSpins, AxionProton, MySaveFig, FigSetup
import matplotlib.pyplot as plt

fig,ax = FigSetup(Shape='Rectangular',ylab='$|g_{ap}|$',mathpazo=True,\
                              g_min=1e-17,g_max=1e-2,m_min=1e-22,m_max=1e-2,FrequencyAxis=True,N_Hz=1)
AxionProton.QCDAxion(ax)
AxionProton.Haloscopes(ax,projection=False)
AxionProton.StellarBounds(ax)
AxionProton.LabExperiments(ax,projection=False)

BlackHoleSpins(ax,0.93828,[3e-11,2e-15],text_on=False)
plt.text(1.5e-12,1e-14,r'{\bf Black}',fontsize=25,ha='center')
plt.text(1.5e-12,1e-14/4,r'{\bf Hole}',fontsize=25,ha='center')
plt.text(1.5e-12,1e-14/16,r'{\bf Spins}',fontsize=25,ha='center');

MySaveFig(fig,'AxionProton')

```

```python
fig,ax = FigSetup(Shape='Rectangular',ylab='$|g_{ap}|$',mathpazo=True,\
                              g_min=1e-17,g_max=1e-2,m_min=1e-22,m_max=1e-2,FrequencyAxis=True,N_Hz=1)
AxionProton.QCDAxion(ax)
AxionProton.Haloscopes(ax,projection=True)
AxionProton.StellarBounds(ax)
AxionProton.LabExperiments(ax,projection=True)

BlackHoleSpins(ax,0.93828,[3e-11,2e-15],text_on=False)
plt.text(1.5e-12,0.2e-14,r'{\bf Black}',fontsize=25,ha='center')
plt.text(1.5e-12,0.2e-14/4,r'{\bf Hole}',fontsize=25,ha='center')
plt.text(1.5e-12,0.2e-14/16,r'{\bf Spins}',fontsize=25,ha='center');

MySaveFig(fig,'AxionProton_with_Projections')
```

```python

```


#### _Speech Processing Labs 2020: Module 1_

```python
## Run this first!
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np

import cmath
from math import floor
from matplotlib.animation import FuncAnimation
from IPython.display import HTML
plt.style.use('ggplot')

from dspMisc import *
```

# 5 Interpreting the  Discrete Fourier Transform

### Learning Outcomes
* Understand how sampling rate effects the DFT output
* Understand what the DFT leakage is.

### Need to know
* Topic Videos: Fourier Analysis, Frequency Domain
* [Digital Signals: Sampling sinusoids](./sp-m1-3-sampling-sinusoids.ipynb)
* [The Discrete Fourier Transform](./sp-m1-4-discrete-fourier-transform.ipynb) _(desirable)_

Equation alert: If you're viewing this on github, please note that the equation rendering is not always perfect. You should view the notebooks through a jupyter notebook server for an accurate view.

## 5.1 A Very Quick DFT recap
The previous worksheet went through the mechanics of the Discrete Fourier Transform (DFT).  The DFT input and output are broadly:

* **Input:**  $N$ amplitude samples over time
    * $x[n]$, for $n=1..N$ (i.e. a time series of $N$ samples)

* **Output:** the dot product (i.e., the similiarity) between the input and $N$ sinusoids with different frequencies
    * DFT[k] $= Me^{j\phi}$, i.e. a complex number with **magnitude** $M$ and **phase** angle $\phi$

The outputs are calculated using the following formula for $k=0,...N-1$.

$$
\begin{align}
DFT[k] &= \sum_{n=0}^{N-1} x[n] e^{-j \frac{2\pi nk}{N}} \\
&= \sum_{n=0}^{N-1} x[n]\big[\cos\big(\frac{2\pi nk}{N} \big) - j \sin\big(\frac{2\pi nk}{N} \big) \big]
\end{align}
$$

We saw how the complex numbers in this equation $e^{-j2\pi nk/N}$ for $k=1...N-1$ represent sinusoids with specific frequencies as projections of phasors. You can just think of a phasor as an analogue clock with one hand (vector) ticking (rotating) around a clockface (i.e. a circle), where the length of the hand is the amplitude of that wave, and how fast it goes around the clock is it's frequency.

Each DFT[k] output essentially tells us how strongly the input is repeats itself (i.e. is periodic) at the frequency of the $k$th DFT phasor. So, we talk about the DFT outputs as providing a **frequency response**.

Since the the DFT outputs are complex numbers, we can talk about them in terms of magnitude and phase angle: The magnitude of DFT[k] tells us how much we'd weight the $k$-th phasor if we were to try to reconstruct the original input by adding all the DFT phasors together.  The phase of DFT[k] tells use whether we need to shift that wave along the time axis.

## 5.2 The DFT Frequency Response:  Which Frequencies?

You will have noticed in the previous worksheet that we only talked about relative frequencies above, e.g. the $e^{-j2\pi nk/N}$ phasor has $k$ times the frequency of the $e^{-j2\pi n/N}$ phasor.

Since the frequencies of the DFT phasors are all multiples of the $k=1$ phasor, we just have to figure out the frequency of that phasor and we'll know all the rest. Now, the $k=1$ phasor takes $N$ steps to complete a full cycle of the unit circle.  So, it's frequency depends on how long each of these steps takes.  That is, the **sampling rate**.

Assume we are sampling at a rate of $f_s$ samples per second. Then, we can calculate:

* How long between each sample (i.e. the **sampling time**):
    * $t_s = 1/f_s$  (seconds)

* How long does it take to take $N$ samples?
    * $t_s \times N$  (seconds x samples = seconds)

* How long will it take the $k=1$ phasor to make 1 complete cycle?
    * $T_1 = t_s \times N$  (seconds)
    * This is the **period** or **wavelength** of the phasor

* What is the **frequency** of the $k=1$ phasor?
    * $f_{min} = 1/T_1 = 1/(t_s N) $ (cycles/second)

* What is the **frequency** of the $k$th phasor?
    * $kf_{min}$ (cycles/second)

So, the important thing to remember is that the DFT outputs depend on:

* The **number of samples** in the input sequence, $N$
* The **sampling rate**, $f_s$ samples/second

### Example
Assume we have a sampling rate of $f_s = 32$ samples/second, and an input length of $N=16$.

We can then calculate the $k=1$ DFT output frequency by noting the following:

* The sampling time is: $t_s = 1/f_s = 1/32$ seconds
* So, the time it takes to complete one cycle is: $T_1 = t_sN = 1/32 \times 16 = 1/2$ seconds
* So, the frequency of DFT[1] is $f_{min} = 1/1/2 = 2$ cycles/second

### Exercise

* What's the frequency of the $k=5$ phasor in the previous example?

### Notes

```python
# 10 cycles per second
```

### Exercise

Try running this code which sets the input to be a sine wave with the same frequency as the the 1st DFT component. That is, `f_in = 1/wavlen_in`.

```python
N=64

#sampling rate:
f_s = 128
f_s = 256

## sample time
t_s = 1/f_s

## Notice that the DFT component frequencies as completely determined by the number of samples N
## and the sampling frequency f_s.
## So if we wanted the input frequency to exactly match the frequency of the 1st DFT component we
## could just do this:

wavlen_in = t_s*N
f_in = 1/wavlen_in

print("f_s: %f\nt_s: %f\nwavlen: %f\nf_in %f" % (f_s, t_s, wavlen_in, f_in))

## indices of input sequence of size N
nsteps = np.array(range(N))

samples_per_cycle = wavlen_in/t_s
print("samples per cycle %f" % samples_per_cycle)

## Generate a sine wave with f_in matching the frequency of the k=1 DFT phasor
## You can look at the code for this function in the file XXX.py

x, time_steps = gen_sinusoid(frequency=f_in, phase=0, amplitude=1, sample_rate=f_s, seq_length=N, gen_function=np.sin)

## Plot the input
fig, timedom = plt.subplots(figsize=(16, 4))
timedom.scatter(time_steps, x, color='magenta')
timedom.plot(time_steps, x, color='magenta')
timedom.set_xlabel("time (s)")
timedom.set_ylabel("Amplitude")
timedom.set_title("Input: $x[n]$ in the time domain")
```

```python
## Calculate the frequencies associated with each DFT output
def get_dft_freqs_all(sample_rate, seq_len):
    ## Get the minimum frequency
    f_min = sample_rate/seq_len

    ## All the other frequencies are just integer multiples of the minimum frequency
    ## We have 0,...,seq_len-1 outputs
    dft_outs = np.arange(seq_len)

    return f_min * dft_outs
```

```python
## Find the DFT output frequencies for our current sample rate and input length
dft_freqs = get_dft_freqs_all(sample_rate=f_s, seq_len=N)
print(dft_freqs)

```

    [  0.   4.   8.  12.  16.  20.  24.  28.  32.  36.  40.  44.  48.  52.
      56.  60.  64.  68.  72.  76.  80.  84.  88.  92.  96. 100. 104. 108.
     112. 116. 120. 124. 128. 132. 136. 140. 144. 148. 152. 156. 160. 164.
     168. 172. 176. 180. 184. 188. 192. 196. 200. 204. 208. 212. 216. 220.
     224. 228. 232. 236. 240. 244. 248. 252.]

```python
## Apply the DFT to the input and return the magnitudes and phases for each DFT output
mags, phases = get_dft_mag_phase(x, seq_len=N)

## Plot the magnitudes
fig, fdom = plt.subplots(figsize=(16, 4))
fdom.scatter(dft_freqs, mags)
fdom.set_xlabel("Frequency (Hz)")
fdom.set_ylabel("Magnitude")
fdom.set_title("Magnitude response")

#print(mags)

## Plot the phases
fig, fdom = plt.subplots(figsize=(16, 4))
fdom.scatter(dft_freqs, phases)
fdom.set_xlabel("Frequency (Hz)")
fdom.set_ylabel("Phase angle (radians)")
fdom.set_title("Phase response")

#print(phases)

```

### Exercise

* Is the output what you'd expect given the input?
* Why is the phase non-zero for the DFT output's 1 and 63?
* What happens if you change the `amplitude` of the input sine wave? What about the `phase`?
* What happens to the range of DFT output frequencies if you increase the sampling rate `f_s` but keep the input sequence length `N` fixed?
* What happens to the range of DFT output frequencies for $k > N/2$? Can the DFT actually model those frequencies?

### Notes

```python
#
# 1 and 63 are the same phase turning in opposite directions
# amplitude: the magnitude response increases by the same factor,
# phase: the phase response changes
# increase f_s: the output frequency increases by the same amount
#
```

## 5.3 DFT on a compound waveform

Now, let's see what happens when we use a compound waverform as input to the DFT.  First we need to set the length of the sequence and the sample rate.

```python
## Set the number of samples N, sampling rate f_s
N=64

#sampling rate:
f_s = 64

## sample time
t_s = 1/f_s

print("N: %d\nf_s: %f\nt_s: %f" % (N, f_s, t_s))
```

    N: 64
    f_s: 64.000000
    t_s: 0.015625

Let's create a compound waveform as input with frequency components of 4Hz and 12Hz:

```python
## Now let's create a compound waveform as input
## We're setting gen_function=np.cos, meaning that each of our sinusoids is a cosine wave
x1, time_steps = gen_sinusoid(frequency=4, phase=0, amplitude=1, sample_rate=f_s, seq_length=N, gen_function=np.cos)

## Let's shift the phase of this one by 90 degrees (pi/2 rads)
x2, time_steps = gen_sinusoid(frequency=12, phase=np.pi/2, amplitude=1, sample_rate=f_s, seq_length=N, gen_function=np.cos)

x3, time_steps = gen_sinusoid(frequency=0.5, phase=0, amplitude=1, sample_rate=f_s, seq_length=N, gen_function=np.cos)

x_compound = x1 + x2 + x3

## Plot the input
fig, timedom = plt.subplots(figsize=(16, 4))
timedom.scatter(time_steps, x_compound, color='magenta')
timedom.plot(time_steps, x_compound, color='magenta')
timedom.set_xlabel("time (s)")
timedom.set_ylabel("Amplitude")
timedom.set_title("A compound waveform: $x_compound$ in the time domain")
```

You should see a somewhat complicated waveform whose overall pattern repeats 4 times in a second.  Now we do the DFT and look at the magnitude and phase values for the different DFT outputs.

```python
## Do the DFT on the compound waveform as above:
mags, phases = get_dft_mag_phase(x_compound, seq_len=N)
dft_freqs = get_dft_freqs_all(sample_rate=f_s, seq_len=N)

## Plot the magnitudes
fig, fdom = plt.subplots(figsize=(16, 4))
#timedom.scatter(dft_freqs[:round(N)], mags[:round(N)])
fdom.scatter(dft_freqs[:round(N/2)], mags[:round(N/2)])
fdom.set_xlabel("Frequency (Hz)")
fdom.set_ylabel("Magnitude")
fdom.set_title("Magnitude response")
#print(mags)

## Plot the phases - let's just skip this for the moment
fig, fdom = plt.subplots(figsize=(16, 4))
fdom.scatter(dft_freqs[:round(N/2)], phases[:round(N/2)])
fdom.set_xlabel("Frequency (Hz)")
fdom.set_ylabel("Phase (rad)")
fdom.set_title("Phase response")
#print(phases)

```

### Exercise

* Do you see what you would expect given the input?
* Why does the number of non-zero phases differ from the number of non-zero magnitudes?

### Notes

```python
# yes
# only the 12 Hz wave was phase-shifted
```

### Exercise

Explore the outputs of the DFT when you change the following parameters
* sampling rate `f_s`
* sequence length `N`
* frequency of the input sequence(s) `frequency` (in the function `gen_sinusoid`)

**Question**
* What happens if you set the frequency of one of the input components to be greater than the Nyquist frequency?
* What happens when the frequency of the input components doesn't match any of the DFT component frequencies?
(e.g. for the example above what if one of the input frequencies is not a whole number?)

### Notes

```python
# it won't appear in the output
# the output is a combination of component frequencies
```

## 5.4  Leakage

One of the main things to remember about the DFT is that you're calculating the correlation between the input and phasors with specific frequencies.  If your input exactly matches one of those phasor frequencies the magnitude response will show a positive magnitude for that phasor and zero for everything else.  However, if the input frequency falls between phasor frequencies (i.e. doesn't correspond to a specific DFT output), then you'll see spectral **leakage**.  The DFT outputs close to the input frequency will also get positive magnitudes, with the DFT output closest to the input frequency getting the highest magnitude.

The following code gives an example

```python
N=64
f_s = 64
f_in1 = 4.5 ## In between DFT output frequencies
f_in2 = 12  ## One of the DFT outputs

x1, time_steps = gen_sinusoid(frequency=f_in1, phase=0, amplitude=1, sample_rate=f_s, seq_length=N, gen_function=np.cos)
x2, time_steps = gen_sinusoid(frequency=f_in2, phase=np.pi/2, amplitude=0.4, sample_rate=f_s, seq_length=N, gen_function=np.cos)
x_compound = x1  + x2

## Plot the input
fig, timedom = plt.subplots(figsize=(16, 4))
timedom.scatter(time_steps, x_compound, color='magenta')
timedom.plot(time_steps, x_compound, color='magenta')
timedom.set_xlabel("time (s)")
timedom.set_ylabel("Amplitude")
timedom.set_title("Leakage example (time domain)")

```

```python
## Do the DFT on the compound waveform as above:
mags, phases = get_dft_mag_phase(x_compound, seq_len=N)
dft_freqs = get_dft_freqs_all(sample_rate=f_s, seq_len=N)

## Plot the magnitudes
fig, fdom = plt.subplots(figsize=(16, 4))

## Just plot the first N/2 frequencies since we know that they are the mirrored for k>N/2
fdom.scatter(dft_freqs[:round(N/2)], mags[:round(N/2)])
fdom.set_xlabel("Frequency (Hz)")
fdom.set_ylabel("Magnitude")
fdom.set_title("Leakage example: Magnitude response")
#print(mags)

```

### Leakage as the normalized sinc function

Leakage makes the DFT harder to interpret. However, we can derive the shape that leakage will have from the the DFT equation and some algebra about rectangular functions.  It turns out that leakage for a particular frequency follows the normalized **sinc** function:

$$
\begin{align}
X(m) &= \Big|\frac{AN}{2} \cdot \mathop{sinc}(c-m)\Big|\\
&= \Big|\frac{AN}{2} \cdot \frac{\sin(\pi(c-m))}{2\pi(c-m))}\Big|\\
\end{align}
$$

Where $A$ is the peak amplitude of the input, $N$ is the input sequence length, $c$ is the number of cycles completed in the input sequence time.  If $c$ is a whole number we just get the usual DFT response, but if $c$ is not a whole number, we get a spread across output frequency bins.

Let's check whether the sinc function matches what we get in the DFT. First we write a function to evaluate the leakage function in between our DFT outputs.

```python
## Calculate the approximated leakage as the sinc function
def calc_leakage(freq,  sample_rate, seqlen, amplitude=1):
    sequence_time = (1/sample_rate)*seqlen

    ## number of cycles in input for component
    c = freq * sequence_time
    print("c=", c)

    ## Interpolate between DFT ouput indices
    ms = np.arange(0, seqlen, 0.05)

    ## Approximated response - we could actually just return a function here, but
    ## let's just keep things concrete for now.
    leakage = np.abs(amplitude * seqlen * 0.5 * np.sinc((c-ms)))

    return leakage, ms * (sample_rate/seqlen)

```

Now let's plot the leakage predicted for our two input components separately (top) and added together (bottom)

```python
## Calculate the leakage function for our know input wave components
leakage1, ms = calc_leakage(f_in1, f_s, N)
leakage2, ms = calc_leakage(f_in2, f_s, N)

## Plot the magnitude response and the leakage function for each of our input components
fig, fdom = plt.subplots(figsize=(16, 4))
fdom.set(xlim=(-1, N/2), ylim=(-1, N))
fdom.scatter(dft_freqs, mags)

fdom.plot(ms, leakage1)
fdom.plot(ms, leakage2)
fdom.set_xlabel("Frequency (Hz)")
fdom.set_ylabel("Magnitude")
fdom.set_title("Leakage function for each input component frequency")

## Plot the magnitude response and the sum of the leakage functions
fig, fdom = plt.subplots(figsize=(16, 4))
fdom.set(xlim=(-1, N/2), ylim=(-1, N))
fdom.scatter(dft_freqs, mags)
fdom.plot(ms, leakage1 + leakage2, color='C5')
fdom.set_xlabel("Frequency (Hz)")
fdom.set_ylabel("Magnitude")
fdom.set_title("Sum of leakage functions for input components")

## It fits, though not perfectly!
```

In the top figure, you should see that peaks (**main lobes**) of each leakage function are aligned with our input component frequencies. The peaks are at the same points as the DFT outputs when the sinusoidal component has a frequency matching the DFT output frequency (i.e. 12 above Hz).  Otherwise we see the spread of leakage around the input component frequency (i.e. around 4.5 Hz).

You'll also notice that our DFT magnitudes points don't always line up perfectly with our sinc curves.  Part of this is because the leakage function is an _approximation_.  Nevertheless, it's a very good approximation!

### Exercise

* What happens if the frequencies of the two components making the compound waveform are very close together?
    * e.g. make `f_in2=6`
* What if one of the components has a relatively small amplitude?
    * e.g. change `amplitude` of the second input to 0.4

### Notes

```python
# f_in2=6: the two peaks merge
# amplitude: the output decreases and doesn't match the sinc function
```

### Shaping the lobes
The leakage sinc function has a peak around a specific frequency. If we want our DFT to be better able to distinguish between close frequencies, we need that peak, the **main lobe** to be narrower.  We also want the other peaks, the **side lobes** to be flatter.  We can achieve this using different **windowing methods** on our input.  This is why you see 'Hanning' as the default option for window method in the advanced spectrogram settings in praat.

We'll see some more examples of this when we look at different types of filters later.  But for now the main thing to observe is that leakage might give you the impression that specific frequency components are present in your waveform, when what's actually happening is that your waveform has frequencies that don't match the DFT phasors.  Another thing that can happen is that the peak for a low amplitude component gets subsumed into the main lobe of a nearby frequency component.  This might make you miss frequency components in your input!

```python

```

## 5.6 Extra:

Here's the composition and decomposition for the compound waveform one you saw in Notebook 2!

```python
N=1028
f_s = 1028
f_in1 = 8 ## In between DFT output frequencies
f_in2 = 20  ## One of the DFT outputs
f_in3 = 36  ## One of the DFT outputs

x1, time_steps = gen_sinusoid(frequency=f_in1, phase=0, amplitude=0.5, sample_rate=f_s, seq_length=N, gen_function=np.cos)
x2, time_steps = gen_sinusoid(frequency=f_in2, phase=np.pi/2, amplitude=1, sample_rate=f_s, seq_length=N, gen_function=np.cos)
x3, time_steps = gen_sinusoid(frequency=f_in3, phase=0, amplitude=0.3, sample_rate=f_s, seq_length=N, gen_function=np.cos)

x_compound = x1  + x2 + x3

## Plot the input
fig, timedom = plt.subplots(figsize=(16, 4))
#timedom.scatter(time_steps, x_compound, color='magenta')
#timedom.set(xlim=(0, 0.2))
timedom.plot(time_steps, x_compound, color='magenta')
```

```python
## Plot the input

fig = plt.figure(figsize=(15,15))
gs = fig.add_gridspec(3,2)

ymax=2
timedom = fig.add_subplot(gs[0, 0])
timedom.set(xlim=(-0.1, 1), ylim=(-ymax,ymax))
timedom.plot(time_steps, x_compound, color='magenta')
timedom.set_title("waveform made from adding 3 sine waves")

s1 = fig.add_subplot(gs[0, 1])
s1.set(xlim=(-0.1, 1), ylim=(-ymax,ymax))
s1.plot(time_steps, x1)
s1.set_title("component 1: 8 Hz")

s2 = fig.add_subplot(gs[1, 1])
s2.set(xlim=(-0.1, 1), ylim=(-ymax,ymax))
s2.plot(time_steps, x2)
s2.set_title("component 2: 20 Hz")

s3 = fig.add_subplot(gs[2, 1])
s3.set(xlim=(-0.1, 1), ylim=(-ymax,ymax))
s3.plot(time_steps, x3)
s3.set_title("component 3: 36 Hz")

fig.savefig("./fig/compound_waveform.png")
```

```python
## Do the DFT on the compound waveform as above:
mags, phases = get_dft_mag_phase(x_compound, seq_len=N)
dft_freqs = get_dft_freqs_all(sample_rate=f_s, seq_len=N)

## Plot the magnitudes
fig,fdom = plt.subplots(figsize=(16, 4))
fdom.set(xlim=(-1, N/2))
## Just plot the first N/2 frequencies since we know that they are the mirrored for k>N/2
fdom.scatter(dft_freqs[:round(N/2)], mags[:round(N/2)])
fdom.set_xlabel("Frequency (Hz)")
fdom.set_ylabel("Magnitude")
fdom.set_title("Magnitude Response")
#print(mags)

```

```python

```

```python

```


2019年度
線形代数　I　講座資料

資料番号　No.4-1

## [4-1] 　行列とその演算

行列 $A$ がm行n列のとき (　$m$x$n$行列 のとき　）

$$
A = \left(
\begin{array}{ccc}
a_{11} & a_{12} & \ldots & a_{1n} \\
a_{21} & a_{22} & \ldots & a_{2n} \\
\vdots & \vdots & \ddots & \vdots \\
a_{m1} & a_{m2} & \ldots & a_{mn}
\end{array}
\right)
$$

行列　$A$ は$(a_{ij})$と書く

$A$の$(i,j)$成分は$a_{ij}$と書く　ここで、$i$は行の数字、$j$は列の数字

1xn 行列　 （n次行ベクトル）
$$ ( a_1   a_2 \ldots a_n　)  $$

nx1 行列　（n次列ベクトル）

$$
\left(  \begin{array}{c}    a_1 \\    a_2 \\    \vdots \\    a_n  \end{array}\right)
$$

行列の等号（＝）は
対応する各成分が等しいとき、$ A=B$とかく

$$(a_{ij}) =(b_{ij})$$では、
$$ \begin{array}{ccc}  a_{11} = b_{11}\\a_{12} = b_{12}\\ a_{13} = b_{13} \\ \vdots \end{array}$$

###  行列の積
$A = (a_{ij}) $  (　$m$x$n$行列 のとき　）
$B = (b_{ij}) $  (　$n$x$p$行列 のとき　）
$C = (c_{ij}) $  (　$m$x$p$行列 のとき　）

$AB = C $

($\color{red}{m}$x$\boxed{n}$)($\boxed{n}$x($\color{blue}{p}$) $\Rightarrow$ ($\color{red}{m}$x$\color{blue}{p}$)

積の前の行列の列数と後の行列の行数が等しい（　$\boxed{n}$が等しい　）ことが積が成分する条件である

例

$A = (a_{ij}) $  (　$4$x$5$行列 のとき　）
$B = (b_{ij}) $  (　$5$x$7$行列 のとき　）
$C = (c_{ij}) $  (　$4$x$7$行列 のとき　）

$AB = C $

($\color{red}{4}$x$\boxed{5}$)($\boxed{5}$x$\color{blue}{7}$) $\Rightarrow$ ($\color{red}{4}$x$\color{blue}{7}$)

積の前の行列の列数と後の行列の行数が等しい（$\boxed{5}$が等しい）ことが積が成分する条件である

$\begin{pmatrix} a & b & c\\ d & e & f \end{pmatrix}\begin{pmatrix} \color{red}g &\color{blue}h \\ \color{red}i & \color{blue}j\\ \color{red}k &\color{blue}l \\ \end{pmatrix} =\begin{pmatrix} ag+bi+ck & ah+bj+cl \\ dg+ei+fk & dh+ej+fl\\ \end{pmatrix}$

この計算の手順を図で示したのが下図です，ただし，これは説明のための図であることに注意してください．

fig-4-1.png

## 積の結合法則

###  $ (AB)C = A(BC) $
### $ AB \neq BA $  　　　 一般には交換法則は成立しない．

## 連立方程式 について

$
  \left\{
  \begin{array}{l}
      a_{11}x_1+a_{12}x_2+a_{13}x_3 + \cdots + a_{1n}x_n & = b_1\\
      a_{21}x_1+a_{22}x_2+a_{23}x_3 + \cdots + a_{2n}x_n & = b_2\\
        \vdots                                           &  \vdots \\
      a_{m1}x_1+a_{m2}x_2+a_{m3}x_3 + \cdots + a_{mn}x_n & = b_m\\
  \end{array}
  \right.
$

上記の連立方程式より

$
\begin{eqnarray}
A = \left(
  \begin{array}{cccc}
    a_{ 11 } & a_{ 12 } & \ldots & a_{ 1n } \\
    a_{ 21 } & a_{ 22 } & \ldots & a_{ 2n } \\
    \vdots & \vdots & \ddots & \vdots \\
    a_{ m1 } & a_{ m2 } & \ldots & a_{ mn }
  \end{array}
\right)
\end{eqnarray}
$
，$\boldsymbol{x}$ = $\left(  \begin{array}{c}    x_1 \\    x_2 \\    \vdots \\    x_n  \end{array}\right)$，$\boldsymbol{b}$ = $\left(  \begin{array}{c}    b_1 \\    b_2 \\    \vdots \\    b_n  \end{array}\right)$

$A$を係数行列という．上記より

$A$ $\boldsymbol{x}$=$\boldsymbol{b}$

拡大係数行列は

$
\begin{eqnarray}
(Ab) = \left(
  \begin{array}{cccc}
    a_{ 11 } & a_{ 12 } & \ldots & a_{ 1n }& \color{red}{b_1} \\
    a_{ 21 } & a_{ 22 } & \ldots & a_{ 2n }&\color{red}{b_2}\\
    \vdots & \vdots & \ddots & \vdots \\
    a_{ m1 } & a_{ m2 } & \ldots & a_{ mn }&\color{red}{b_n}\\
  \end{array}
\right)
\end{eqnarray}
$

### 行列の和，スカラー倍

行列の和は同じ形の行列$A,B$で定義されている，和は対応する成分を加えることでできる行列を$A+B$と書いて行列$A,B$の和と定める

$A+B = C $

$a_{ij}+b_{ij} =c_{ij} $

行列のスカラー倍（実数:$k$） 行列のすべての成分を$k$倍する

$kA=(ka_{ij})$

$(-1)A=-A$と書く

ゼロ行列

$O$はすべての成分が0 (ゼロ)である行列である

$
\begin{eqnarray}
O = \left(
  \begin{array}{cccc}
    0 &\ldots & 0\\
    0 &\ldots &0 \\
    \vdots &  & \vdots \\
   0 & \ldots &0\\
  \end{array}
\right)
\end{eqnarray}
$

行列のルール

(1)　$(kA)B =A(kB)=k(AB)$

(2)　$A(B+C)=AB+AC$

　　　$(A+B)C=AC+BC$

(3)　$AO=O,OA=O$

### 転置行列

mxn行列$A$の行と列を入れ替えて作られる行列を転置行列といい，${}^t \!A$と書く

内積を転置で表現すると，例えば
$ \boldsymbol{a}=(a_1　a_2)，　\boldsymbol{b}=(b_1　b_2)$とおくと

$  \boldsymbol{a} {}^t\boldsymbol{b}=(a_1　a_2)$
$ \left(  \begin{array}{c} b_1 \\ b_2 \\ \end{array}\right)$ = $a_1b_1 + a_2 b_2$

となり，内積の結果と同じとなる．

#### 転置行列の性質

(1)　${}^t({}^t \!A)=A$

(2)　${}^t(A+B)={}^tA+{}^tB$

(3)　${}^t(cA)=c{}^tA$

(4)　${}^t(AB)={}^tB{}^tA$

```python
# 初めに1回だけ実行させる
from sympy import *
init_printing()
# init_session()
```

```python
# 例　行列の例　ｐ３０
A = Matrix([[1, 2, 1], [2, 7, -1]])

print("A=")
display(A)
```

    A=

$$\left[\begin{matrix}1 & 2 & 1\\2 & 7 & -1\end{matrix}\right]$$

```python
# 行列Aの形を求める Aは２ｘ３行列
A.shape
```

```python
# 行列の1行目を求める(最初の行のときは０)
A.row(0)
```

$$\left[\begin{matrix}1 & 2 & 1\end{matrix}\right]$$

```python
 # 行列の1行目を求める(最後からの列のときは－１)
A.col(-1)
```

$$\left[\begin{matrix}1\\-1\end{matrix}\right]$$

```python
# 例 行列の積 p31

A = Matrix([[1, 2], [0, 1]])
B = Matrix([[1, 0], [3, 1]])

print("A=")
display(A)
print("B=")
display(B)

print("A*B=")
display(A*B)

```

    A=

$$\left[\begin{matrix}1 & 2\\0 & 1\end{matrix}\right]$$

    B=

$$\left[\begin{matrix}1 & 0\\3 & 1\end{matrix}\right]$$

    A*B=

$$\left[\begin{matrix}7 & 2\\3 & 1\end{matrix}\right]$$

```python
# 例 行列AとBの和 p33
A = Matrix([[1, 2, 3], [4, 5, 6]])
B = Matrix([[2, 3, 4], [5, 6, 7]])
print("A=")
display(A)
print("B=")
display(B)
print("A+B=")
display(A+B)
```

    A=

$$\left[\begin{matrix}1 & 2 & 3\\4 & 5 & 6\end{matrix}\right]$$

    B=

$$\left[\begin{matrix}2 & 3 & 4\\5 & 6 & 7\end{matrix}\right]$$

    A+B=

$$\left[\begin{matrix}3 & 5 & 7\\9 & 11 & 13\end{matrix}\right]$$

```python
# 例 行列のスカラー倍　p34  Bを3倍したもの
B = Matrix([[2, 3, 4], [5, 6, 7]])
print("B=")
display(B)
print("3B=")
display(3*B)
```

    B=

$$\left[\begin{matrix}2 & 3 & 4\\5 & 6 & 7\end{matrix}\right]$$

    3B=

$$\left[\begin{matrix}6 & 9 & 12\\15 & 18 & 21\end{matrix}\right]$$

```python
# 転置行列を求める
A = Matrix([[1, 2, 3], [4, 5, 6]])

print("A=")
display(A)

print("A.T=")
display(A.T)
```

    A=

$$\left[\begin{matrix}1 & 2 & 3\\4 & 5 & 6\end{matrix}\right]$$

    A.T=

$$\left[\begin{matrix}1 & 4\\2 & 5\\3 & 6\end{matrix}\right]$$

```python
# (AB)の転置を計算する
A = Matrix([[1, 2], [3, 4]])
B = Matrix([[2, 1], [4, 3]])
print("A=")
display(A)
print("B=")
display(B)

print("(AB).T=")
display((A*B).T)
```

    A=

$$\left[\begin{matrix}1 & 2\\3 & 4\end{matrix}\right]$$

    B=

$$\left[\begin{matrix}2 & 1\\4 & 3\end{matrix}\right]$$

    (AB).T=

$$\left[\begin{matrix}10 & 22\\7 & 15\end{matrix}\right]$$

```python
# Bの転置行列とAの転置行列の積を求める

print("(B.T)*(A.T)=")
display((B.T)*(A.T))
```

    (B.T)*(A.T)=

$$\left[\begin{matrix}10 & 22\\7 & 15\end{matrix}\right]$$

```python
#   問２．１０　　転置行列の計算
A = Matrix([[1, 1, -2], [2, 2, 3]])

print("A=")
display(A)
print("A.T=")
display(A.T)

```

    A=

$$\left[\begin{matrix}1 & 1 & -2\\2 & 2 & 3\end{matrix}\right]$$

    A.T=

$$\left[\begin{matrix}1 & 2\\1 & 2\\-2 & 3\end{matrix}\right]$$

```python
# Aの転置行列とAの積
A = Matrix([[1, 1, -2], [2, 2, 3]])
print("A=")
display(A)
print("A.T*A=")
display(A.T*A)
```

    A=

$$\left[\begin{matrix}1 & 1 & -2\\2 & 2 & 3\end{matrix}\right]$$

    A.T*A=

$$\left[\begin{matrix}5 & 5 & 4\\5 & 5 & 4\\4 & 4 & 13\end{matrix}\right]$$

```python
# A行列とAの転置行列
A = Matrix([[1, 1, -2], [2, 2, 3]])
print("A=")
display(A)
print("A*A.T=")
display(A*A.T)

```

    A=

$$\left[\begin{matrix}1 & 1 & -2\\2 & 2 & 3\end{matrix}\right]$$

    A*A.T=

$$\left[\begin{matrix}6 & -2\\-2 & 17\end{matrix}\right]$$

```python
# ゼロ行列　（2X3）のとき
zeros(2, 3)
print("O=")
display(zeros(2, 3))
```

    O=

$$\left[\begin{matrix}0 & 0 & 0\\0 & 0 & 0\end{matrix}\right]$$

##  正方行列

行の数と列の数が等しい$n$x$n$行列をn次正方行列という

$
A = \left(
\begin{array}{ccc}
a_{11} & a_{12} & \ldots & a_{1n} \\
a_{21} & a_{22} & \ldots & a_{2n} \\
\vdots & \vdots & \ddots & \vdots \\
a_{n1} & a_{n2} & \ldots & a_{nn}
\end{array}
\right)
$

$a_{ii}(1=1,2,\cdots,n)$を対角成分という

$A$の$n$回の積は

$\underbrace{AAA \cdots A }_{ n個 } =A^{n}$と書く

#### 対称行列

正方行列Aとして，その転置行列を${}^t \!A$とすると

$ A =  {}^t \!A $のとき　$A$を対称行列という

```python
# 例 正方行列の３乗する場合
A = Matrix([[1, 2], [3, 4]])

print("A=")
display(A)

print("A**3=")
display(A**3)
```

    A=

$$\left[\begin{matrix}1 & 2\\3 & 4\end{matrix}\right]$$

    A**3=

$$\left[\begin{matrix}37 & 54\\81 & 118\end{matrix}\right]$$

### 単位行列

対角成分が１で他の成分が０のｎ次正方行列を単位行列といい，$E$で表す

$
E = \left(
\begin{array}{ccc}
1 & 0 & \ldots & 0 \\
0 & 1 & \ldots & 0 \\
\vdots & \vdots & \ddots & \vdots \\
0 & 0 & \ldots & 1 \\
\end{array}
\right)
$

単位行列の性質

$EA=A$，　$AE=A$

```python
# ３次の単位行列
print("0=")
display(eye(3))
```

    0=

$$\left[\begin{matrix}1 & 0 & 0\\0 & 1 & 0\\0 & 0 & 1\end{matrix}\right]$$

## 逆行列

n次の正方行列Aに対して，

$ AB= E $ かつ　$BA=E$
をみたす$B$を$A$の逆行列をいい，$A^{-1}$と書く，また，逆行列の存在する正方行列を正則行列という

したがって

 ### $AA^{-1}=E $，　$A^{-1}A=E$

 #### 逆行列のルール

また，A,Bがn次の正則行列なら

〇 $(AB)^{-1}= B^{-1}A^{-1}$

〇 $(A^{-1})^{-1}= A $

### 定理２．１　正方行列の逆行列が存在すれば，ただ１つである．

###  定理２．２　2次の正方行列の逆行列

$A=\begin{pmatrix} a & b \\ c & d \end{pmatrix}$は$ad-bc \neq 0$のとき，$A$の逆行列が存在し

$ A^{-1}= \displaystyle \frac{ 1 }{ ad-bc }$ $\begin{pmatrix} d & -b \\ -c & a \end{pmatrix}$

未知数の個数と式の個数が等しい連立方程式

$A\boldsymbol{x}=\boldsymbol{b}$

係数行列Aの逆行列$A^{-1}$が存在するなら，両辺に左から$A^{-1}$をかけて

$A^{-1}A\boldsymbol{x}= A^{-1}\boldsymbol{b}$

$\therefore$　$ \boldsymbol{x}= A^{-1}\boldsymbol{b}$　として求めることができる．

```python
# 　例　逆行列の計算
A = Matrix([[2, 3], [4, 5]])

print("A=")
display(A)
print("A**(-1)=")
display(A**(-1))
```

    A=

$$\left[\begin{matrix}2 & 3\\4 & 5\end{matrix}\right]$$

    A**(-1)=

$$\left[\begin{matrix}- \frac{5}{2} & \frac{3}{2}\\2 & -1\end{matrix}\right]$$

```python
# 　例　逆行列の計算
a, b,c = symbols('a b,c')
A = Matrix([[1, 2, 3], [1, 2,4],[1,3,7]])
B = Matrix([[-2, a, b], [3, -4,c],[-1,1,0]])
E =eye(3)

print("A=")
display(A)
print("B=")
display(B)

print("A*B =E")
display(solve(A*B -E))
```

```python
A = Matrix([[1, 2, 3], [1, 2,4],[1,3,7]])

print("A=")
display(A)
print("A**(-1)=")
display(A**(-1))

```

    A=

$$\left[\begin{matrix}1 & 2 & 3\\1 & 2 & 4\\1 & 3 & 7\end{matrix}\right]$$

    A**(-1)=

$$\left[\begin{matrix}-2 & 5 & -2\\3 & -4 & 1\\-1 & 1 & 0\end{matrix}\right]$$

```python
# 例 連立方程式を求める solveを利用したとき　P40

x,y = symbols('x,y')
equation1 = 2*x + 3*y - 4
equation2 = 4*x + 5*y - 6

print("2*x + 3*y = 4")
print("4*x + 5*y = 6")

display(solve([equation1, equation2]))
```

```python
# 例 連立方程式を求める　逆行列を利用した時　P40

A = Matrix([[2, 3], [4, 5]])
b = Matrix([[4], [6]])

print("A = ")
display(A)
print("b = ")
display(b)
print("x = A**(-1)*b ")
print("x = ")
display(A**(-1)*b)
```

    A =

$$\left[\begin{matrix}2 & 3\\4 & 5\end{matrix}\right]$$

    b =

$$\left[\begin{matrix}4\\6\end{matrix}\right]$$

    x = A**(-1)*b
    x =

$$\left[\begin{matrix}-1\\2\end{matrix}\right]$$

```python
# 拡大係数行列を作り,連立方程式を解く方法
x, y,z = symbols('x, y,z')
A = Matrix([[1, 3, 5], [3, 7,11],[-1,4,7]])
b = Matrix([6,16,-3])

print("A = ")
display(A)
print("b = ")
display(b)

print("x = ")
display(linsolve((A, b), x, y, z))

```

```python

```