```julia
using FileIO, MeshIO, Meshes, MeshViz
using GLMakie
```

    ┌ Info: Precompiling MeshIO [7269a6da-0436-5bbc-96c2-40638cbb6118]
    └ @ Base loading.jl:1313
    ┌ Info: Precompiling Meshes [eacbb407-ea5a-433e-ab97-5258b1ca43fa]
    └ @ Base loading.jl:1313
    ┌ Info: Precompiling MeshViz [9ecf9c4f-6e5a-4b5e-83ae-06f2c7a661d8]
    └ @ Base loading.jl:1313
    ┌ Info: Precompiling GLMakie [e9467ef8-e4e7-5192-8a1a-b1aee30e663a]
    └ @ Base loading.jl:1313

```julia
cat_mesh_ = load(joinpath("..", "data", "cat-00.off"))
lion_mesh_ = load(joinpath("..", "data", "lion-00.off"));
```

```julia
import GeometryBasics
function Base.convert(::Type{<:SimpleMesh}, m::GeometryBasics.Mesh)
    points = [Meshes.Point.(p...) for p in m.position]
    connec = [connect(Tuple(Int(offi.i)+1 for offi in c)) for c in GeometryBasics.faces(m)]
    SimpleMesh(points, connec)
end
```

```julia
cat_mesh = convert(SimpleMesh, cat_mesh_)
lion_mesh = convert(SimpleMesh, lion_mesh_);
```

```julia
# viz(cat_mesh, showfacets =true) |> display
```

```julia
function double_plot(mesh1, mesh2; color1=:yellow, color2=:yellow, showfacets=false, landmarks=undef)
    f = Figure()
    ax1 = Axis3(f[1, 1], aspect = :data)
    viz!(mesh1,vertexcolor=color1,showfacets=showfacets, showvertices=true)
    ax2 = Axis3(f[1, 2], aspect = :data)
    viz!(mesh2, vertexcolor=color2, showfacets=showfacets, showvertices=true)
    return f, [ax1, ax2]
end
```

    double_plot (generic function with 1 method)

```julia
fig, fig_axes = double_plot(cat_mesh, lion_mesh; color1=1:length(cat_mesh.points), color2=1:length(lion_mesh.points), showfacets=false)
fig |> display
```

    GLMakie.Screen(...)

```julia
# load landmark
all_landmarks = open(joinpath("..", "data", "landmarks.txt"),"r") do datafile
    [parse.(Int, split(line)) for line in eachline(datafile)]
end
landmarks = [p .+ 1 for p in all_landmarks[1:5]]
```

    5-element Vector{Vector{Int64}}:
     [3178, 1686]
     [7179, 4911]
     [6566, 4647]
     [5473, 4276]
     [2384, 1384]

```julia
# visualize landmarks
meshes = [cat_mesh, lion_mesh]
for i in [1,2]
    cat_landmarks_x = [meshes[i].points[pp[i]].coords[1] for pp in landmarks]
    cat_landmarks_y = [meshes[i].points[pp[i]].coords[2] for pp in landmarks]
    cat_landmarks_z = [meshes[i].points[pp[i]].coords[3] for pp in landmarks]
    meshscatter!(fig_axes[i], cat_landmarks_x,cat_landmarks_y,cat_landmarks_z, markersize = 0.01, color = 1:length(landmarks))
end

fig |> display
```

    GLMakie.Screen(...)

```julia
using PyCall
```

```julia
py"""
import numpy as np

from pyFM.mesh import TriMesh
from pyFM.functional import FunctionalMapping

def get_fmmodel():
    mesh1 = TriMesh('../data/cat-00.off')
    mesh2 = TriMesh('../data/lion-00.off')

    process_params = {
        'n_ev': (35,35),  # Number of eigenvalues on source and Target
        'landmarks': np.loadtxt('../data/landmarks.txt',dtype=int)[:5],  # loading 5 landmarks
        'subsample_step': 5,  # In order not to use too many descriptors
        'descr_type': 'WKS',  # WKS or HKS
    }

    model = FunctionalMapping(mesh1,mesh2)
    model.preprocess(**process_params,verbose=True)
    return model
"""
```

```julia
py_FM_model = py"get_fmmodel"()
```

    Computing Laplacian spectrum
    Computing 200 eigenvectors
    	Done in 1.71 s
    Computing 200 eigenvectors
    	Done in 1.25 s

    Computing descriptors
    	Normalizing descriptors

    	120 out of 600 possible descriptors kept

    PyObject <pyFM.functional.FunctionalMapping object at 0x00000000651F06A0>

```julia
using SparseArrays
const scipy_sparse_find = pyimport("scipy.sparse")["find"]
# convert py sparse matrix to a Julia sparse array
function mysparse(Apy::PyObject)
    IA, JA, SA = scipy_sparse_find(Apy)
    return sparse(Int[i+1 for i in IA], Int[i+1 for i in JA], SA)
end
```

    WARNING: redefinition of constant scipy_sparse_find. This may fail, cause incorrect answers, or produce other errors.

    mysparse (generic function with 1 method)

```julia
pymesh1 = py_FM_model.mesh1
pymesh2 = py_FM_model.mesh2
const k1 = py_FM_model.k1
const k2 =  py_FM_model.k2
```

    35

```julia
py"""
import meshplot as mp
mp.offline()

def plot_mesh(myMesh,cmap=None,file_name="test.html"):
    p = mp.plot(myMesh.vertlist, myMesh.facelist,c=cmap)
    p.save(file_name)

def double_plot(myMesh1,myMesh2,cmap1=None,cmap2=None):
    d = mp.subplot(myMesh1.vertlist, myMesh1.facelist, c=cmap1, s=[2, 2, 0])
    mp.subplot(myMesh2.vertlist, myMesh2.facelist, c=cmap2, s=[2, 2, 1], data=d)
    d.save('test.html')

def visu(vertices):
    min_coord,max_coord = np.min(vertices,axis=0,keepdims=True),np.max(vertices,axis=0,keepdims=True)
    cmap = (vertices-min_coord)/(max_coord-min_coord)
    return cmap
"""
```

$\newcommand{\RR}{\mathbb{R}}$
$\newcommand{\uargmin}[1]{\underset{#1}{\text{argmin}}\;}$
$\newcommand{\uargmax}[1]{\underset{#1}{\text{argmax}}\;}$
$\def\*#1{\mathbf{#1}}$

The fitting optimization problem is
\begin{equation}
\uargmin{\*C\in\RR^{k_2\times k_1}} \mu_{descr}\|\*C\*A - \*B\|^2 + \mu_{lap}\|\*C\Delta_1 - \Delta_2\*C\|^2 + \mu_{\text{descr comm}}\sum_i \|\*C\Gamma_1^i - \Gamma_2^i\*C\|^2 + \mu_{\text{orient}}\sum_i \|\*C\Lambda_1^i - \Lambda_2^i\*C\|^2
\end{equation}

with $\Gamma_1^i$ and $\Gamma_2^i$ [multipliative operators](http://www.lix.polytechnique.fr/~maks/papers/fundescEG17.pdf) associated to the $i$-th descriptors, $\Lambda_1^i$ and $\Lambda_2^i$ [orientation preserving operators](https://arxiv.org/abs/1806.04455) associated to the $i$-th descriptors

```julia
descr_μ = 1e0 # descr_preservation
lap_μ = 1e-2 # LB_commutation
descr_comm_μ = 1e-1 # oplist_commutation

orient_μ = 0
```

    0

```julia
# Compute multiplicative operators associated to each descriptor
"""
Compute the multiplication operators associated with the descriptors

Output
======
operators : n_descr long list of ((k1,k1),(k2,k2)) operators.
"""
function compute_descr_op(pymesh1, pymesh2, descr1, descr2, k1, k2)
    pinv1 = pymesh1.eigenvectors[:, 1:k1]' * mysparse(pymesh1.A)  # (k1,n)
    pinv2 = pymesh2.eigenvectors[:, 1:k2]' * mysparse(pymesh2.A)  # (k2,n)
    list_descr = [
                  (pinv1 * (descr1[:, i] .* pymesh1.eigenvectors[:, 1:k1]),
                   pinv2 * (descr2[:, i] .* pymesh2.eigenvectors[:, 1:k2])
                   )
                  for i in 1:size(descr1, 2)
                  ]
    return list_descr
end
```

    compute_descr_op

```julia
list_descr = compute_descr_op(pymesh1, pymesh2, py_FM_model.descr1, py_FM_model.descr2, k1, k2);
# (n_descr, ((k1,k1), (k2,k2)) )
```

```julia
# Compute multiplicative operators associated to each descriptor
# skip for now
```

```julia
# Compute the squared differences between eigenvalues for LB commutativity
using LinearAlgebra

ev_sqdiff = (pymesh1.eigenvalues[1:k1]' .- pymesh1.eigenvalues[1:k2]).^2
ev_sqdiff /= LinearAlgebra.norm(ev_sqdiff);
```

```julia
viz_fm_matrix(ev_sqdiff) |> display
```

    GLMakie.Screen(...)

```julia
# initial map guess
# Sets the equivalence between the constant functions
x0 = zeros(Float64, k2, k1)

ev_sign = sign.(pymesh1.eigenvectors[1,1]*pymesh2.eigenvectors[1,1])
area_ratio = sqrt.(pymesh2.area/pymesh1.area)

x0[:,1] = zeros(k2)
x0[1,1] = ev_sign * area_ratio
```

    1.2425867898388572

```julia
function viz_fm_matrix(FM)
    hfig = Figure()
    ax, hm = heatmap(hfig[1, 1], FM)
    Colorbar(hfig[1, 2], hm)
    hfig
end
```

    viz_fm_matrix (generic function with 1 method)

```julia
viz_fm_matrix(x0) |> display
```

    GLMakie.Screen(...)

## optimization objectives

```julia
"""
Compute the descriptor preservation constraint

Parameters
=========
C      : (K2,K1) Functional map
descr1 : (K1,p) descriptors on first basis
descr2 : (K2,p) descriptros on second basis

Output
========
energy : descriptor preservation squared norm
"""
function descr_preservation(C, descr1, descr2)
    0.5 * sum((C * descr1 - descr2).^2)
end

"""
Compute the gradient of the descriptor preservation constraint

Parameters
---------------------
C      : (K2,K1) Functional map
descr1 : (K1,p) descriptors on first basis
descr2 : (K2,p) descriptros on second basis

Output
---------------------
gradient : gradient of the descriptor preservation squared norm
"""
function descr_preservation_grad(C, descr1, descr2)
    return (C * descr1 - descr2) * descr1'
end
```

    descr_preservation_grad

```julia
"""
Compute the LB commutativity constraint

Parameters
===========
C      : (K2,K1) Functional map
ev_sqdiff : (K2,K1) [normalized] matrix of squared eigenvalue differences

Output
======
energy : (float) LB commutativity squared norm
"""
function LB_commutation(C, ev_sqdiff)
    0.5 * sum(C.^2 * ev_sqdiff)
end

"""
Compute the gradient of the LB commutativity constraint

Parameters
---------------------
C         : (K2,K1) Functional map
ev_sqdiff : (K2,K1) [normalized] matrix of squared eigenvalue differences

Output
---------------------
gradient : (K2,K1) gradient of the LB commutativity squared norm
"""
function LB_commutation_grad(C, ev_sqdiff)
    return C * ev_sqdiff
end
```

    LB_commutation_grad

```julia
"""
Compute the operator commutativity constraint.
Can be used with descriptor multiplication operator

Parameters
---------------------
C   : (K2,K1) Functional map
op1 : (K1,K1) operator on first basis
op2 : (K2,K2) descriptros on second basis

Output
---------------------
energy : (float) operator commutativity squared norm
"""
function op_commutation(C, op1, op2)
    return 0.5 * sum((C * op1 - op2 * C).^2)
end

"""
Compute the gradient of the operator commutativity constraint.
Can be used with descriptor multiplication operator

Parameters
---------------------
C   : (K2,K1) Functional map
op1 : (K1,K1) operator on first basis
op2 : (K2,K2) descriptros on second basis

Output
---------------------
gardient : (K2,K1) gradient of the operator commutativity squared norm
"""
function op_commutation_grad(C, op1, op2)
    return op2' * (op2 * C - C * op1) - (op2 * C - C * op1) * op1'
end

"""
Compute the operator commutativity constraint for a list of pairs of operators
Can be used with a list of descriptor multiplication operator

Parameters
---------------------
C   : (K2,K1) Functional map
op_list : list of tuple( (K1,K1), (K2,K2) ) operators on first and second basis

Output
---------------------
energy : (float) sum of operators commutativity squared norm
"""
function oplist_commutation(C, op_list)
    energy = 0
    for (op1, op2) in op_list
        energy += op_commutation(C, op1, op2)
    end
    return energy
end

"""
Compute the gradient of the operator commutativity constraint for a list of pairs of operators
Can be used with a list of descriptor multiplication operator

Parameters
---------------------
C   : (K2,K1) Functional map
op_list : list of tuple( (K1,K1), (K2,K2) ) operators on first and second basis

Output
---------------------
gradient : (K2,K1) gradient of the sum of operators commutativity squared norm
"""
function oplist_commutation_grad(C, op_list)
    gradient = similar(C)
    fill!(gradient, 0.0)
    for (op1, op2) in op_list
        gradient += op_commutation_grad(C, op1, op2)
    end
    return gradient
end
```

    oplist_commutation_grad

```julia
function energy_fn(C_vec)
    C = reshape(C_vec, (k2,k1))
    energy = 0.0
    if descr_μ > 0
        energy += descr_μ * descr_preservation(C, descr1_red, descr2_red)
    end
    if lap_μ > 0
        energy += lap_μ * LB_commutation(C, ev_sqdiff)
    end
    if descr_comm_μ > 0
        energy += descr_comm_μ * oplist_commutation(C, list_descr)
    end
#     if orient_mu > 0:
#         energy += orient_mu * oplist_commutation(C, orient_op)

    return energy
end

"""
Evaluation of the gradient of the energy for standard FM computation

Parameters:
----------------------
C               : (K2*K1) or (K2,K1) Functional map
descr_mu        : scaling of the descriptor preservation term
lap_mu          : scaling of the laplacian commutativity term
descr_comm_mu   : scaling of the descriptor commutativity term
orient_mu       : scaling of the orientation preservation term
descr1          : (K1,p) descriptors on first basis
descr2          : (K2,p) descriptros on second basis
list_descr      : p-uple( (K1,K1), (K2,K2) ) operators on first and second basis
                  related to descriptors.
orient_op       : p-uple( (K1,K1), (K2,K2) ) operators on first and second basis
                  related to orientation preservation operators.
ev_sqdiff       : (K2,K1) [normalized] matrix of squared eigenvalue differences

Output
------------------------
gradient : (K2*K1) - value of the energy
"""
function grad_energy!(g, C)
    C = reshape(C, (k2,k1))
    gradient = similar(C)
    fill!(gradient, 0.0)

    if descr_μ > 0
        gradient += descr_μ * descr_preservation_grad(C, descr1_red, descr2_red)
    end

    if lap_μ > 0
        gradient += lap_μ * LB_commutation_grad(C, ev_sqdiff)
    end

    if descr_comm_μ > 0
        gradient += descr_comm_μ * oplist_commutation_grad(C, list_descr)
    end

#     if orient_mu > 0:
#         gradient += orient_mu * oplist_commutation_grad(C, orient_op)

    gradient[:,1] .= 0
    g .= reshape(gradient, k2*k1)
end
```

    grad_energy!

```julia
@time energy_fn(x0)
```

      0.013195 seconds (9.95 k allocations: 5.222 MiB, 73.97% compilation time)

    93.6305232133181

```julia
@time grad_energy(x0);
```

      0.006916 seconds (1.22 k allocations: 11.585 MiB)

```julia
using Optim
```

    ┌ Info: Precompiling Optim [429524aa-4258-5aef-a3af-852621145aeb]
    └ @ Base loading.jl:1313

```julia
res = optimize(energy_fn, reshape(x0, k2*k1), LBFGS(),
    Optim.Options(show_trace=true, iterations = 15, f_tol=1e-4); autodiff=:forward)
```

    Iter     Function value   Gradient norm
         0     9.363052e+01     6.578420e+01
     * time: 0.012000083923339844
         1     6.010118e+01     2.402624e+01
     * time: 8.934999942779541
         2     4.833240e+01     8.847396e+00
     * time: 22.09599995613098
         3     4.483751e+01     3.880380e+00
     * time: 35.16300010681152
         4     4.293005e+01     3.407950e+00
     * time: 48.914000034332275
         5     4.162311e+01     2.784900e+00
     * time: 61.986000061035156
         6     4.066034e+01     2.649547e+00
     * time: 75.10800004005432
         7     4.007708e+01     1.484537e+00
     * time: 87.99300003051758
         8     3.972626e+01     1.394655e+00
     * time: 100.48399996757507
         9     3.952218e+01     8.998665e-01
     * time: 113.04800009727478
        10     3.937219e+01     7.197819e-01
     * time: 125.85999989509583
        11     3.927948e+01     4.239449e-01
     * time: 138.49699997901917
        12     3.922350e+01     5.746855e-01
     * time: 150.7739999294281
        13     3.918927e+01     2.804708e-01
     * time: 163.95000004768372
        14     3.917008e+01     3.364822e-01
     * time: 176.79800009727478
        15     3.915634e+01     2.217830e-01
     * time: 189.50500011444092

     * Status: failure (reached maximum number of iterations)

     * Candidate solution
        Final objective value:     3.915634e+01

     * Found with
        Algorithm:     L-BFGS

     * Convergence measures
        |x - x'|               = 4.80e-03 ≰ 0.0e+00
        |x - x'|/|x'|          = 9.55e-03 ≰ 0.0e+00
        |f(x) - f(x')|         = 1.37e-02 ≰ 0.0e+00
        |f(x) - f(x')|/|f(x')| = 3.51e-04 ≰ 1.0e-04
        |g(x)|                 = 2.22e-01 ≰ 1.0e-08

     * Work counters
        Seconds run:   190  (vs limit Inf)
        Iterations:    15
        f(x) calls:    45
        ∇f(x) calls:   45

```julia
res = optimize(energy_fn, grad_energy!, reshape(x0, k2*k1), LBFGS(),
    Optim.Options(show_trace=true, iterations = 15, f_tol=1e-4))
```

    Iter     Function value   Gradient norm
         0     9.363052e+01     3.184949e+01
     * time: 0.0010001659393310547
         1     7.683701e+01     1.345098e+01
     * time: 0.878000020980835
         2     6.510130e+01     9.666214e+00
     * time: 0.9360001087188721
         3     6.004698e+01     4.484454e+00
     * time: 0.9810001850128174
         4     5.709395e+01     4.370470e+00
     * time: 1.0160000324249268
         5     5.550407e+01     2.511215e+00
     * time: 1.0510001182556152
         6     5.459393e+01     1.985245e+00
     * time: 1.0840001106262207
         7     5.388069e+01     1.735180e+00
     * time: 1.1190001964569092
         8     5.347314e+01     1.487310e+00
     * time: 1.1540000438690186
         9     5.323387e+01     9.660572e-01
     * time: 1.188000202178955
        10     5.307076e+01     7.539892e-01
     * time: 1.2220001220703125
        11     5.296925e+01     5.275951e-01
     * time: 1.2599999904632568
        12     5.290955e+01     4.267190e-01
     * time: 1.2920000553131104
        13     5.287311e+01     2.942810e-01
     * time: 1.3270001411437988
        14     5.285201e+01     2.783965e-01
     * time: 1.3619999885559082
        15     5.283778e+01     2.693314e-01
     * time: 1.3980000019073486

     * Status: failure (reached maximum number of iterations)

     * Candidate solution
        Final objective value:     5.283778e+01

     * Found with
        Algorithm:     L-BFGS

     * Convergence measures
        |x - x'|               = 4.82e-03 ≰ 0.0e+00
        |x - x'|/|x'|          = 3.88e-03 ≰ 0.0e+00
        |f(x) - f(x')|         = 1.42e-02 ≰ 0.0e+00
        |f(x) - f(x')|/|f(x')| = 2.69e-04 ≰ 1.0e-04
        |g(x)|                 = 2.69e-01 ≰ 1.0e-08

     * Work counters
        Seconds run:   1  (vs limit Inf)
        Iterations:    15
        f(x) calls:    45
        ∇f(x) calls:   45

```julia
FM_opt = reshape(res.minimizer, (k2,k1));
viz_fm_matrix(FM_opt) |> display
```

    GLMakie.Screen(...)

```julia
@show minimum(FM_opt)
@show maximum(FM_opt)
```

    minimum(FM_opt) = -0.5502659816557515
    maximum(FM_opt) = 1.2425867898388572

    1.2425867898388572

```julia
using JLD
```

```julia
save("FM_opt_w_grad.jld", "FM_opt", FM_opt)
```

```julia
FM_opt = load("FM_opt_w_grad.jld", "FM_opt");
```

```julia
py"""
import pyFM.spectral as spectral

def FM_to_p2p(FM_opt, pymesh1, pymesh2):
    return spectral.FM_to_p2p(FM_opt, pymesh1.eigenvectors, pymesh2.eigenvectors)
"""
```

```julia
p2p_map = py"FM_to_p2p"(FM_opt, pymesh1, pymesh2);
```

```julia
py"np.save"("np_FM_w_grad", FM_opt)
py"np.save"("np_p2p_w_grad", p2p_map)
```

```julia
using Statistics

function accuracy(p2p, gt_p2p, D1_geod; return_all=false, sqrt_area=nothing)
    dists = D1_geod[p2p.+1,gt_p2p.+1]
    if sqrt_area != nothing
        dists ./= sqrt_area
    end
    if return_all
        return mean(dists), dists
    end
    return mean(dists)
end
```

    accuracy (generic function with 3 methods)

```julia
A_geod = pymesh1.get_geodesic(verbose=true)
```

    100%|██████████| 15/15 [00:41<00:00,  2.73s/it]

    7207×7207 Matrix{Float64}:
     0.0         0.0110308   0.0180556   …  0.32673     0.32747     0.0110056
     0.00940081  0.0         0.00951037     0.315326    0.316066    0.0141724
     0.0153161   0.00903885  0.0            0.313852    0.314592    0.0108462
     0.0665902   0.0684021   0.0591114      0.350048    0.350788    0.0587313
     0.0706805   0.0704937   0.0607846      0.346174    0.346914    0.0628848
     0.0860992   0.0865389   0.0773665   …  0.360224    0.360964    0.0786823
     0.0835474   0.0854069   0.0765111      0.364102    0.364841    0.0761916
     0.0164435   0.0213075   0.0143829      0.326108    0.326848    0.00870236
     0.0204266   0.0162888   0.00602857     0.3148      0.31554     0.0142782
     0.0321239   0.029821    0.0189438      0.319083    0.319823    0.0244221
     0.0287384   0.0316397   0.0230293   …  0.32826     0.329       0.0203433
     0.117135    0.119484    0.111651       0.395253    0.395993    0.110956
     0.117423    0.118561    0.110448       0.390022    0.390762    0.11101
     ⋮                                   ⋱              ⋮
     0.317134    0.30314     0.301636    …  0.0163906   0.0133965   0.31587
     0.28968     0.275686    0.274182       0.0323327   0.0342968   0.288416
     0.268556    0.254562    0.253058       0.0552643   0.0564513   0.267292
     0.289046    0.275052    0.273548       0.0321165   0.034469    0.287782
     0.267437    0.253442    0.251939       0.0562567   0.0575475   0.266173
     0.310135    0.29614     0.294636    …  0.00992694  0.0117227   0.30887
     0.310818    0.296823    0.295319       0.00544693  0.0101552   0.309553
     0.319113    0.305118    0.303614       0.00596752  0.00250275  0.317849
     0.314546    0.300552    0.299048       0.00657349  0.00706773  0.313282
     0.315357    0.301363    0.299859       0.0         0.00488519  0.314093
     0.319793    0.305799    0.304295    …  0.00382669  0.0         0.318529
     0.00650318  0.0123818   0.00908445     0.324124    0.324864    0.0

```julia
# Load an approximate ground truth map
ground_truth_p2p = Int.(py"np.loadtxt"("../data/lion2cat"));

accuracy(p2p_map, ground_truth_p2p, A_geod, sqrt_area=√(pymesh1.area))
```

    0.5960676531445763

## Misc

```julia
# Project the descriptors on the LB basis
function project(py_mesh, func; k=undef)
    eigen_vecs = py_mesh.eigenvectors
    area_matrix = mysparse(py_mesh.A)
    if k==undef
        return eigen_vecs' * area_matrix * func
    elseif k <= size(py_mesh.eigenvectors, 1)
        return eigen_vecs[:,1:k]' * area_matrix * func
    else
         throw(DomainError(k, "At least $k eigenvectors should be computed before projecting"))
    end
end
```

    project (generic function with 1 method)

```julia
# Project the descriptors on the LB basis
descr1_red = project(pymesh1, py_FM_model.descr1; k=k1)
descr2_red = project(pymesh2, py_FM_model.descr2; k=k2);
```


# The $K$-means clustering algorithm

This notebook is a modified version of the one created by [Jake Vanderplas](https://github.com/jakevdp) for PyCon 2015.

Source and license info of the original notebook are on [GitHub](https://github.com/jakevdp/sklearn_pycon2015/).

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from scipy.spatial.distance import cdist
from fig_code import plot_kmeans_interactive
from sklearn.datasets import make_blobs, load_digits, load_sample_image
from sklearn.decomposition import PCA
from sklearn.metrics import confusion_matrix
from sklearn.cluster import KMeans

# use seaborn plotting defaults
import seaborn as sns; sns.set()
```

## 1. Clustering algorithms

Clustering algorithms try to split a set of data points $\mathcal{S} = \{{\bf x}_0,\ldots,{\bf x}_{L-1}\}$, into mutually exclusive clusters or groups, $\mathcal{G}_0,\ldots, \mathcal{G}_{K-1}$, such that every sample in $\mathcal{S}$ is assigned to one and only one group.

Clustering algorithms belong to the more general family of **unsupervised methods**: clusters are constructed using the data attributes alone. No labels or target values are used. This makes the difference between a clustering algorithm and a *supervised* classification algorithm.

There is not a unique formal definition of the clustering problem. Different algorithms group data into clusters following different criteria. The appropriate choice of the clustering algorithm may depend on the particular application scenario.

The image below, taken from the scikit-learn site, shows that different algorithms follow different grouping criteria, clustering the same datasets in different forms.

In any case, all clustering algorithms share a set of common characteristics. A clustering algorithm makes use of some distance or similarity measure between data points to group data in such a way that:

* Points in some cluster should lie close to each other
* Points in different clusters should be far away
* Clusters should be separated by regions of low density of points
* Clusters may preserve some kind of *connectivity*
* Clusters may get represented by a representative or centroid

## 2. The $K$-means algorithm

$K$-Means is a proximity-based clustering algorithm. It searches for cluster centers or **centroids** which are representative of all points in a cluster. Representativenes is measured by proximity: "good" clusters are those such that all data points are close to its centroid.

Given a dataset $\mathcal{S} = \{{\bf x}_0,\ldots,{\bf x}_{L-1}\}$, $K$-means tries to minimize the following **distortion function**:

$$D = \sum_{k=0}^{K-1} \sum_{{\bf x} \in {\cal{G}}_k}\|{\bf x}-\boldsymbol{\mu}_k\|_2^2$$

where $\boldsymbol{\mu}_k$ is the centroid of cluster $\mathcal{G}_k$.

Note that, in this notebook, we will used $k$ as the index to count groups and centroids, and $K$ for the number of centroides. To avoid any confusion, we will index data samples as ${\bf x}_\ell$ when needed, and the number of samples will be denoted as $L$.

The minimization should be carried out over both the partition $\{{\cal G}_0,\ldots, {\cal G}_{K-1}\}$ of ${\cal S}$ (i.e., the assignment problem) and their respective centroids $\{\boldsymbol{\mu}_0,\ldots,\boldsymbol{\mu}_{K-1}\}$ (i.e. the estimation problem). This joint assignment-estimation problem is what makes optimization difficult (it is an NP-hard problem).

The $K$-means algorithm is based on the fact that, given that one of both problems is solved, the solution to the other is straightworward:

* **Assignment**: For fixed centroids $\boldsymbol{\mu}_0,\ldots,\boldsymbol{\mu}_{K-1}$, the optimal partition is given by the following
$${\cal G}_k^* = \left\{{\bf x} \, \left| \, k \in \arg\min_{k'} \|{\bf x}-\boldsymbol{\mu}_{k'}\|^2\right. \right\}$$
(i.e. each sample is assigned to the group with the closest centroid).
* **Estimation**: For a fixed partition $\{{\cal G}_0,\ldots, {\cal G}_{K-1}\}$, the optimal centroids can be computed easily by differentiation
\begin{equation}
\boldsymbol{\mu}_k^* = \frac{1}{\left|{\cal G}_k\right|} \sum_{{\bf x} \in {\cal{G}}_k} {\bf x}
\end{equation}
where $\left|{\cal G}_k\right|$ is the cardinality of ${\cal G}_k$.

$K$ means is a kind of coordinate descent algorithm that applies ciclycally and iteratively the estimation and assigning steps, fixing the solution of the previous optimization at each time.

**Exercise**: Derive the equation for the optimal centroids.

#### Solution:

[comment]: # (<SOL>)
[comment]: # (</SOL>)

### 2.1. Steps of the algorithm

After initialization of centroids:
1. **Assignment**: Assign each data point to closest centroid
2. **Estimation**: Recalculate centroids positions
3. Go back to 1 until no further changes or max iterations achieved

#### 2.1.1. Initializations

$K$-means convergence is guaranteed ... but just to a local minimum of $D$.

Different initialization possibilities:
1. $K$-means$++$: To maximize inter-centroid distance
2. Random among training points
3. User-selected

Typically, different runs are executed, and the best one is kept.

Check out  the Scikit-Learn site for parameters, attributes, and methods.

#### 2.1.2. Stopping.

Since (1) the total number of possible assignments is finite, and (2) each step of the $K$-means algorithm reduces (or, at least, does not increase) the value of the distortion function, the algorithm will eventually converge to a fixed distortion value.

#### 2.1.3. Local convergence

Unfortunatelly, there is no guarantee that the final distortion is minimum. The quality of the solution obtained by the algorithm may critically depend on the initialization.

### 2.2. Example

Let's look at how KMeans operates on a synthetic example. To emphasize that this is unsupervised, we do not plot the colors of the clusters:

```python
X, y = make_blobs(n_samples=300, centers=4,
                  random_state=0, cluster_std=0.60)

plt.scatter(X[:, 0], X[:, 1], s=50);
plt.axis('equal')
plt.show()
```

By eye, it is relatively easy to pick out the four clusters. If you were to perform an exhaustive search for the different segmentations of the data, however, the search space would be exponential in the number of points. Fortunately, the $K$-Means algorithm implemented in Scikit-learn provides a much more convenient solution.

Exercise:

The following frament of code runs the $K$-means method on the toy example you just created. Modify it, so that you can try other settings for the parameter options implemented by the method. In particular:

   - Reduce the number of runs to check the consequences of a bad initialization
   - Test different kinds of initializations (k-means++ vs random)
   - Provide a user-generated initialization that you consider can result in very suboptimal performance
   - Test other selections of the number of parameters
   - Include in the plot the location of the cluster of each class

```python
est = KMeans(5)  # 4 clusters
est.fit(X)
y_kmeans = est.predict(X)
plt.scatter(X[:, 0], X[:, 1], c=y_kmeans, s=50, cmap='rainbow');
plt.axis('equal')
plt.show()
```

### 2.3. The K-Means Algorithm: Interactive visualization

The following fragment of code allows you to study the evolution of cluster centroids on one run of the algorithm, and to modify also the number of centroids.

```python
# WARNING: This command may fail (interactivity not working properly) depending on the python version.
plot_kmeans_interactive(min_clusters=2, max_clusters=6)
plt.show()
```

### 2.4. Determining the number of clusters

If the number of clusters, $K$, is not known, selecting the appropriate value becomes a major issue. Since the overal distortion $D$ decreases with $K$, the selection of the number o clusters cannot be based on the overal distorsion.

The best value of $K$ may be application dependent. Though we will not discuss specific algorithms in detail, we point out some possible solutions:

* **Penalization functions**: instead of minimizing $D$, we can train the clustering algorithm in order to minimize the functional $D' = D + \lambda f(K)$, where $f$ is an increasing function penalizing large values of $K$, and $\lambda$ is an hyperparameter. For instance, we can take
$$f(K)=\log(K)$$
$$f(K)=K$$
$$f(K)=K^2,$$
etc.

* **Cluster-based metrics**, like
    * Average silohuete coefficient. The Silhouette Coefficient is calculated using the mean intra-cluster distance (a) and the mean nearest-cluster distance (b) for each sample. The Silhouette Coefficient for a sample is (b - a) / max(a, b).
    *  Calinski-Harabaz score . It is defined as the ratio of the between-clusters dispersion mean and the within-cluster dispersion:
\begin{align}
s(K) = \frac{\mathrm{Trace}({\bf B}_K)}{\mathrm{Trace}({\bf W}_K)} \times \frac{L - K}{K - 1}
\end{align}
where ${\bf W}_K$ is the within-cluster dispersion matrix defined by
$$
{\bf W}_K = \sum_{k=0}^{K-1} \sum_{{\bf x} \in {\cal G}_k} (x - \boldsymbol{\mu}_k) ({\bf x} - \boldsymbol{\mu}_k)^T
$$
and ${\bf B}_K$ is the between group dispersion matrix, definded by
$$
{\bf B}_K = \sum_{k=0}^{K-1} \left|{\cal G}_k\right| (\boldsymbol{\mu}_k - \boldsymbol{\mu}) (\boldsymbol{\mu}_k - \boldsymbol{\mu})^T
$$
with $L$ be the number of points in our data and $\boldsymbol{\mu}$ be the average of all data points.

Exercise: Select the number of samples using any of the above metrics for the dataset in the previous examples.

## 3. Application of KMeans to Digits

For a closer-to-real-world example, let us take a look at a digit recognition dataset. Here we'll use KMeans to automatically cluster the data in 64 dimensions, and then look at the cluster centers to see what the algorithm has found.

```python
digits = load_digits()

print('Input data and label number are provided in the following two variables:')
print("digits['images']: {0}".format(digits['images'].shape))
print("digits['target']: {0}".format(digits['target'].shape))
```

Next, we cluster the data into 10 groups, and plot the representatives (centroids of each group).  As with the toy example, you could modify the initialization settings to study the impact of initialization in the performance of the method

```python
est = KMeans(n_clusters=10)
clusters = est.fit_predict(digits.data)
est.cluster_centers_.shape
```

```python
fig = plt.figure(figsize=(8, 3))
for i in range(10):
    ax = fig.add_subplot(2, 5, 1 + i, xticks=[], yticks=[])
    ax.imshow(est.cluster_centers_[i].reshape((8, 8)), cmap=plt.cm.binary)
```

We see that *even without the labels*, KMeans is able to find clusters whose means are recognizable digits (with apologies to the number 8)!

### 3.1. Visualization via Dimensionality Reduction

The following fragment of code projects the data into the two "most representative" dimensions, so that we can somehow visualize the result of the clustering (note that we can not visualize the data in the original 64 dimensions). In order to do so, we use a method known as Principal Component Analysis (PCA). This is a method that allows you to obtain a 2-D representation of multidimensional data: we extract the two most relevant features (using PCA) and look at the true cluster labels and $K$-means cluster labels:

```python
X = PCA(2).fit_transform(digits.data)

kwargs = dict(cmap = plt.cm.get_cmap('rainbow', 10),
              edgecolor='none', alpha=0.6)
fig, ax = plt.subplots(1, 2, figsize=(8, 4))
ax[0].scatter(X[:, 0], X[:, 1], c=est.labels_, **kwargs)
ax[0].set_title('learned cluster labels')

ax[1].scatter(X[:, 0], X[:, 1], c=digits.target, **kwargs)
ax[1].set_title('true labels');
```

### 3.2. Classification performance

Just for kicks, let us see how accurate our $K$-means classifier is **with no label information**. In order to do so, we can work on the confussion matrix:

```python
conf = confusion_matrix(digits.target, est.labels_)
print(conf)

plt.imshow(conf, cmap='Blues', interpolation='nearest')
plt.colorbar()
plt.grid(False)
plt.ylabel('true')
plt.xlabel('Group index');

#And compute the number of right guesses if each identified group were assigned to the right class
print('Percentage of patterns that would be correctly classified: {0}'.format(
      np.sum(np.max(conf,axis=1)) * 100. / np.sum(conf)))
```

This is above 80% classification accuracy for an **entirely unsupervised estimator** which knew nothing about the labels.

## 4. Example: KMeans for Color Compression

One interesting application of clustering is in color image compression. For example, imagine you have an image with millions of colors. In most images, a large number of the colors will be unused, and conversely a large number of pixels will have similar or identical colors.

Scikit-learn has a number of images that you can play with, accessed through the datasets module. For example:

```python
china = load_sample_image("china.jpg")
plt.imshow(china)
plt.grid(False);
```

The image itself is stored in a 3-dimensional array, of size ``(height, width, RGB)``. For each pixel three values are necessary, each in the range 0 to 255. This means that each pixel is stored using 24 bits.

```python
print('The image dimensions are {0}'.format(china.shape))
print('The RGB values of pixel 2 x 2 are '.format(china[2,2,:]))
```

We can envision this image as a cloud of points in a 3-dimensional color space. We'll rescale the colors so they lie between 0 and 1, then reshape the array to be a typical scikit-learn input:

```python
X = (china / 255.0).reshape(-1, 3)
print(X.shape)
```

We now have 273,280 points in 3 dimensions.

Our task is to use KMeans to compress the $256^3$ colors into a smaller number (say, 64 colors). Basically, we want to find $N_{color}$ clusters in the data, and create a new image where the true input color is replaced by the color of the closest cluster. Compressing data in this way, each pixel will be represented using only 6 bits (**25 % of the original image size**)

```python
# reduce the size of the image for speed. Only for the K-means algorithm
image = china[::3, ::3]
n_colors = 128

X = (image / 255.0).reshape(-1, 3)

model = KMeans(n_colors)
model.fit(X)
labels = model.predict((china / 255.0).reshape(-1, 3))
#print labels.shape
colors = model.cluster_centers_
new_image = colors[labels].reshape(china.shape)
new_image = (255 * new_image).astype(np.uint8)

#For comparison purposes, we pick 64 colors at random
perm = np.random.permutation(range(X.shape[0]))[:n_colors]
colors = X[perm,:]

labels = np.argmin(cdist((china / 255.0).reshape(-1, 3),colors),axis=1)
new_image_rnd = colors[labels].reshape(china.shape)
new_image_rnd = (255 * new_image_rnd).astype(np.uint8)

# create and plot the new image
with sns.axes_style('white'):
    plt.figure()
    plt.imshow(china)
    plt.title('Original image')

    plt.figure()
    plt.imshow(new_image)
    plt.title('{0} colors'.format(n_colors))

    plt.figure()
    plt.imshow(new_image_rnd)
    plt.title('{0} colors'.format(n_colors) + ' (random selection)')

```

Compare the input and output image: we've reduced the $256^3$ colors to just 64. An additional image is created by selecting 64 colors at random from the original image. Try reducing the number of colors to 32, 16, 8, and compare the images in these cases.


taudata Analytics

PDE 01: Pendahuluan Fungsi dan Turunannya

(C) Taufik Sutanto

https://taudataid.github.io/pde01/

* Maksud pemetaan satu dan hanya satu, 1-1, & pada
* Di kalkuluas fungsi biasanya fungsi real, di machine learning domain bisa berupa angka (numerik) atau bukan angka kategorik.
image source: Yuliani Wibowo - https://slideplayer.info/slide/12403959/

```python
def fungsi(x):
    return 2*x+1

fungsi(5)
```

Berbagai macam fungsi

```python
import numpy as np

np.cos, dsb
```

Limit Paradox

limit-paradox.png

Limit

```python
from sympy import *

x = symbols('x')
expr = sin(3 * x)/x;

print("Expression : {}".format(expr))

# Use sympy.limit() method
limit_expr = limit(expr, x, 0)

print("Limit of the expression tends to 0 : {}".format(limit_expr))
```

Limit dan Kontinu

- Animasi dari swf

```python
from sympy import Symbol, S
from sympy.calculus.util import continuous_domain

x = Symbol("x")
f = sin(x)/x
continuous_domain(f, x, S.Reals)
Union(Interval.open(-oo, 0), Interval.open(0, oo))
```

Limit dan Turunan

```python
from sympy import *
x, y, z = symbols('x y z')
diff(cos(x), x)
```

Makna turunan dan optimasi

Solusi akar dan nilai optimal

 - refer ke link di taudata

```python
# optimasi di Python
from scipy.optimize import minimize
from numpy.random import rand

# objective function
def objective(x):
    return x[0]**2.0 + x[1]**2.0

# Plot Fungsinya

# define range for input
r_min, r_max = -5.0, 5.0

```

```python
# define the starting point as a random sample from the domain
pt = r_min + rand(2) * (r_max - r_min)
# perform the l-bfgs-b algorithm search
result = minimize(objective, pt, method='L-BFGS-B')
# summarize the result
print('Status : %s' % result['message'])
print('Total Evaluations: %d' % result['nfev'])
# evaluate solution
solution = result['x']
evaluation = objective(solution)
print('Solution: f(%s) = %.5f' % (solution, evaluation))
```

PDB dan PDP

Sifat dasar turunan

 - chainrule, dsb

 ...

#  Akhir Modul PDE 01, Next: " _Pemodelan dan Persamaan Differensial_ "

PDE/01 - Pengenalan Persamaan Differensial.ipynb


## Neural Language Model

Given some content $C$ as a sequence of words, and some words $x_1, \ldots, x_n$, a language model (LM) is modeling the conditional probability

\begin{equation}
  P \left( x_{n+1} \mid x_1, \ldots, x_n; C \right).
\end{equation}

The original nerual language model, a language model by neural netwok, is implemented by RNN. In the case of nerual machine translation (as a LM), it is a RNN auto-encoder. The encoder encodes the sentence from language A, as a sequence of words in A, to a vector; and then the decoder decodes the vector to language B, as another sequence of words in B.

## Backward of RNN Encoder

The information contained the final encoding vector is limited by the dimension of the vector. A `float32` value in range $(-1, 1)$ has $24$ bits. If the encoding dimension is $1000$ (large), then the total information is $1000 \times 24 = 24000$ bits. Can these bits encode all the poems of Shakespeare? No way!

The solution to this obstacle is, naturally, to use the full encoded sequence, instead of the last element of the sequence. This retains almost all the information.

However, at every step of decoding, not all the elements in the sequence are relavent. So, we, or say the machine, has to figure out the relavence. This is attention.

## Focus your Attention!

A detailed architecture:

```python
import abc
import tensorflow as tf

class BaseAttention(abc.ABC):
  """Abstract base class of attention."""

  # Abbreviations for shapes:
  #   batch_shape -> B (list of `int`s)
  #   seqlen -> L (`int`)
  #   query_dim -> Q (`int`)
  #   key_dim -> K (`int`)
  #   value_dim -> V (`int`)

  def __call__(self, query, keys, values, name='Attention'):
    """Returns the context, with attention-score.

    Args:
      query: Tensor with shape `batch_shape + [query_dim]`.
      keys: Tensor with shape `batch_shape + [seqlen, key_dim]`.
      values: Tensor with shape `batch_shape + [seqlen, value_dim]`.
      name: String.

    Returns:
      Tuple of two tensors. The first with shape `batch_shape + [value_dim]`
      as the context; and the second with shape `batch_shape + [seqlen]` as
      the attention-score.
    """
    with tf.name_scope(name):
      score = self._score(query, keys)  # B + [L, 1]
      # score * values: B + [L, V]
      context = tf.reduce_sum(score * values, axis=-2)  # B + [V]
      score = tf.squeeze(score, axis=-1)  # B + [L]
      return score, context

  def _score(self, query, keys, name='attention_score'):
    """Returns the attention-score.

    Args:
      query: Tensor with shape `batch_shape + [query_dim]`.
      keys: Tensor with shape `batch_shape + [seqlen, key_dim]`.
      name: String.

    Returns:
      Tensor with shape `batch_shape + [seqlen, 1]`. The additional `1`
      is made for convienence for being contracted by a "values" tensor
      with shape `batch_shape + [seqlen, value_dim]` along the `seqlen`
      axis.
    """
    with tf.name_scope(name):
      with tf.name_scope('repeat'):  # along `seqlen`-axis.
        query = tf.expand_dims(query, axis=-2)  # B + [1, Q]
        query = tf.tile(query, self._get_repeats(keys))  # B + [L, Q]

      concated = tf.concat([query, keys], axis=-1)  # B + [L, Q+K]
      energy = self.energy(concated)  # B + [L, 1]
      # Softmax along the `L`-axis
      attention_score = tf.nn.softmax(energy, axis=-2)  # B + [L, 1]
      return attention_score

  def _get_repeats(self, keys):
    """Returns the `repeats` argument of the `tf.tile()` in `self.__call__()`."""
    with tf.name_scope('repeats'):
      shape = keys.get_shape().as_list()
      rank = len(shape)
      seqlen = shape[-2]
      return [1] * (rank - 2) + [seqlen, 1]

  @abc.abstractmethod
  def energy(self, x):
    """
    Args:
      x: Tensor with shape `batch_shape + [query_dim + key_dim]`.

    Returns:
      Tensor with shape `batch_shape + [1]`.
    """
    pass
```

## A Simple Example

We show up a simple enough example found in [Ng's course](https://github.com/GSimas/Deep-LearningAI/tree/master/Course%205/Week%203/Neural%20Machine%20Translation). This is a simple neural machine translation model that converts date-format from the natural language format for human to the standard format for machine.

### Date-Format Data

```python
import numpy as np
from nmt_utils import load_dataset, preprocess_data

dataset, human_vocab, machine_vocab, inv_machine_vocab = load_dataset(10000)
```

    100%|██████████| 10000/10000 [00:00<00:00, 17223.09it/s]

Some instances:

```python
dataset[:10]
```

    [('9 may 1998', '1998-05-09'),
     ('10.09.70', '1970-09-10'),
     ('4/28/90', '1990-04-28'),
     ('thursday january 26 1995', '1995-01-26'),
     ('monday march 7 1983', '1983-03-07'),
     ('sunday may 22 1988', '1988-05-22'),
     ('tuesday july 8 2008', '2008-07-08'),
     ('08 sep 1999', '1999-09-08'),
     ('1 jan 1981', '1981-01-01'),
     ('monday may 22 1995', '1995-05-22')]

The vocabularies for both human and machine are character-level:

```python
print('Human vocabulary:', human_vocab, '\n')
print('Machine vocabulary:', machine_vocab)
```

    Human vocabulary: {' ': 0, '.': 1, '/': 2, '0': 3, '1': 4, '2': 5, '3': 6, '4': 7, '5': 8, '6': 9, '7': 10, '8': 11, '9': 12, 'a': 13, 'b': 14, 'c': 15, 'd': 16, 'e': 17, 'f': 18, 'g': 19, 'h': 20, 'i': 21, 'j': 22, 'l': 23, 'm': 24, 'n': 25, 'o': 26, 'p': 27, 'r': 28, 's': 29, 't': 30, 'u': 31, 'v': 32, 'w': 33, 'y': 34, '<unk>': 35, '<pad>': 36}

    Machine vocabulary: {'-': 0, '0': 1, '1': 2, '2': 3, '3': 4, '4': 5, '5': 6, '6': 7, '7': 8, '8': 9, '9': 10}

Preprocessing to model inputs:
  1. replacing unknown with "<unk>";
  1. post-padding;
  1. the inputs and outputs (targets) are all in one-hot format.

```python
input_seqlen = 30
output_seqlen = 10
X, y, X_oh, y_oh = preprocess_data(
    dataset, human_vocab, machine_vocab, input_seqlen, output_seqlen)
```

```python
index = 0
print("Source date:", dataset[index][0])
print("Target date:", dataset[index][1])
print()
print("Source after preprocessing (indices):", X[index])
print("Target after preprocessing (indices):", y[index])
print()
print("Source after preprocessing (one-hot):", X_oh[index])
print("Target after preprocessing (one-hot):", y_oh[index])
```

    Source date: 9 may 1998
    Target date: 1998-05-09

    Source after preprocessing (indices): [12  0 24 13 34  0  4 12 12 11 36 36 36 36 36 36 36 36 36 36 36 36 36 36
     36 36 36 36 36 36]
    Target after preprocessing (indices): [ 2 10 10  9  0  1  6  0  1 10]

    Source after preprocessing (one-hot): [[0. 0. 0. ... 0. 0. 0.]
     [1. 0. 0. ... 0. 0. 0.]
     [0. 0. 0. ... 0. 0. 0.]
     ...
     [0. 0. 0. ... 0. 0. 1.]
     [0. 0. 0. ... 0. 0. 1.]
     [0. 0. 0. ... 0. 0. 1.]]
    Target after preprocessing (one-hot): [[0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]
     [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]
     [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]
     [0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0.]
     [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
     [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
     [0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]
     [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
     [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
     [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]]

### RNN Auto-Encoder with Attention

The model is illustrated as follow:

And now let's implement it.

```python
from tensorflow.keras.layers import Bidirectional, Dense, Dropout
from tfutils.initializer import GlorotInitializer

if tf.test.is_gpu_available():
    from tensorflow.keras.layers import CuDNNLSTM as LSTM
else:
    from tensorflow.keras.layers import LSTM

class Attention(BaseAttention):

  def __init__(self, energy_units, **kwargs):
    super().__init__(**kwargs)
    self._layers = []
    for n in energy_units:
      self._layers += [tf.layers.Dense(n, activation=tf.nn.relu),
                       tf.layers.Dropout()]
    self._layers += [tf.layers.Dense(1)]

  def energy(self, x):
    with tf.name_scope('energy'):
      for layer in self._layers:
        x = layer(x)
      return x

def model_fn(features, labels, mode, params):
  """The `labels` is one-hot."""
  pre_rnn = Bidirectional(LSTM(params['pre_lstm_units'],
                               return_sequences=True))
  attention = Attention(params['energy_units'])
  post_rnn = LSTM(params['post_lstm_units'],
                  return_state=True)
  # Returns the logits
  output_layer = Dense(params['output_vocab_size'])

  # [batch_size, post_lstm_units]
  s = features['init_s']
  c = features['init_c']
  # [batch_size, input_seqlen, input_vocab_size]
  onehot_inputs = features['onehot_inputs']
  # [batch_size, input_seqlen, pre_lstm_units * 2]
  a_seq = pre_rnn(onehot_inputs)

  # Implements the loop
  output_seqlen = tf.constant(params['output_seqlen'])
  logits = tf.TensorArray(dtype=s.dtype, size=output_seqlen)
  attention_scores = tf.TensorArray(dtype=s.dtype, size=output_seqlen)

  def cond(logits, attention_scores, s, c, step):
    return tf.less(step, output_seqlen)

  def body(logits, attention_scores, s, c, step):
    # [batch_size, input_seqlen] and
    # [batch_size, pre_lstm_units * 2]
    attention_score, context = attention(s, a_seq, a_seq)

    with tf.name_scope('logit'):
      # Prepare for inputting `post_rnn`
      # [batch_size, 1, pre_lstm_units * 2]
      context = tf.expand_dims(context, axis=-2)
      # [batch_size, post_lstm_units]
      s, _, c = post_rnn(context, initial_state=[s, c])
      # [batch_size, output_vocab_size]
      logit = output_layer(s)

    logits = logits.write(step, logit)
    attention_scores = attention_scores.write(step, attention_score)
    return logits, attention_scores, s, c, (step + 1)

  logits, attention_scores, *_ = tf.while_loop(
      cond, body, loop_vars=[logits, attention_scores, s, c, 0])

  # Postprocess the TensorArray
  # [output_seqlen, batch_size, output_vocab_size]
  logits = logits.stack()
  # [batch_size, output_seqlen, output_vocab_size]
  logits = tf.transpose(logits, [1, 0, 2])
  # [output_seqlen, batch_size, input_seqlen]
  attention_scores = attention_scores.stack()
  # [batch_size, output_seqlen, input_seqlen]
  attention_scores = tf.transpose(attention_scores, [1, 0, 2])

  # [batch_size, output_seqlen, output_vocab_size]
  predict_probs = tf.nn.softmax(logits, axis=-1)
  # [batch_size, output_seqlen]
  predict_classes = tf.argmax(logits, axis=-1)
  if mode == tf.estimator.ModeKeys.PREDICT:
    predictions = {
        'logits': logits,
        'probs': predict_probs,
        'class_ids': predict_classes,
        'attention_scores': attention_scores,
    }
    return tf.estimator.EstimatorSpec(mode, predictions=predictions)

  loss = tf.losses.softmax_cross_entropy(labels, logits)

  # [batch_size, output_seqlen]
  label_classes = tf.argmax(labels, axis=-1)
  accuracy = tf.metrics.accuracy(
      label_classes, predict_classes, name='acc_op')
  metrics = {'accuracy': accuracy}
  if mode == tf.estimator.ModeKeys.EVAL:
    return tf.estimator.EstimatorSpec(
        mode, loss=loss, eval_metric_ops=metrics)

  assert mode == tf.estimator.ModeKeys.TRAIN
  optimizer = tf.train.AdamOptimizer(params['lr'])
  train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
  return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
```

```python
tf.reset_default_graph()

params={
    'lr': 0.05,
    'output_vocab_size': y_oh.shape[-1],
    'pre_lstm_units': 32,
    'energy_units': [128],
    'post_lstm_units': 64,
    'output_seqlen': y_oh.shape[-2],
}

estimator = tf.estimator.Estimator(model_fn=model_fn,
                                   params=params)

print('\n', params)
```

    INFO:tensorflow:Using default config.
    WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmp4n0pb66v
    INFO:tensorflow:Using config: {'_model_dir': '/tmp/tmp4n0pb66v', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true
    graph_options {
      rewrite_options {
        meta_optimizer_iterations: ONE
      }
    }
    , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fc45ae1ecc0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}

     {'lr': 0.05, 'output_vocab_size': 11, 'pre_lstm_units': 32, 'energy_units': [128], 'post_lstm_units': 64, 'output_seqlen': 10}

### Train and Evaluate the Model

```python
def train_input_fn(X_oh, y_oh, init_s, init_c, batch_size):
    """An input function for training"""
    features = {'onehot_inputs': X_oh, 'init_s': init_s, 'init_c': init_c}
    dataset = tf.data.Dataset.from_tensor_slices((features, y_oh))
    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
    return dataset

def eval_input_fn(X_oh, y_oh, init_s, init_c, batch_size):
    """An input function for evaluating and predicting."""
    features = {'onehot_inputs': X_oh, 'init_s': init_s, 'init_c': init_c}
    if y_oh is None:
        dataset = tf.data.Dataset.from_tensor_slices(features)
    else:
        dataset = tf.data.Dataset.from_tensor_slices((features, y_oh))
    dataset = dataset.batch(batch_size)  # shall NOT shuffle and repeat!
    return dataset

def get_inits(n_data, params):
    init_s = np.zeros([n_data, params['post_lstm_units']], dtype='float32')
    init_c = np.zeros([n_data, params['post_lstm_units']], dtype='float32')
    return init_s, init_c
```

```python
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X_oh, y_oh)
```

```python
init_s, init_c = get_inits(X_train.shape[0], params)
estimator.train(
    input_fn=lambda: train_input_fn(X_train, y_train, init_s, init_c, 100),
    steps=2000)
```

    INFO:tensorflow:Calling model_fn.
    INFO:tensorflow:Done calling model_fn.
    INFO:tensorflow:Create CheckpointSaverHook.
    INFO:tensorflow:Graph was finalized.
    INFO:tensorflow:Running local_init_op.
    INFO:tensorflow:Done running local_init_op.
    INFO:tensorflow:Saving checkpoints for 0 into /tmp/tmp4n0pb66v/model.ckpt.
    INFO:tensorflow:loss = 2.4025064, step = 0
    INFO:tensorflow:global_step/sec: 29.869
    INFO:tensorflow:loss = 0.8244829, step = 100 (3.349 sec)
    INFO:tensorflow:global_step/sec: 37.2547
    INFO:tensorflow:loss = 0.14884116, step = 200 (2.684 sec)
    INFO:tensorflow:global_step/sec: 36.8435
    INFO:tensorflow:loss = 0.007106445, step = 300 (2.714 sec)
    INFO:tensorflow:global_step/sec: 36.9819
    INFO:tensorflow:loss = 0.0022480497, step = 400 (2.704 sec)
    INFO:tensorflow:global_step/sec: 37.013
    INFO:tensorflow:loss = 0.01661434, step = 500 (2.702 sec)
    INFO:tensorflow:global_step/sec: 36.7233
    INFO:tensorflow:loss = 0.010409214, step = 600 (2.723 sec)
    INFO:tensorflow:global_step/sec: 37.1102
    INFO:tensorflow:loss = 0.0004800958, step = 700 (2.695 sec)
    INFO:tensorflow:global_step/sec: 37.3363
    INFO:tensorflow:loss = 0.0002672929, step = 800 (2.678 sec)
    INFO:tensorflow:global_step/sec: 37.0686
    INFO:tensorflow:loss = 0.00021186155, step = 900 (2.697 sec)
    INFO:tensorflow:global_step/sec: 37.1654
    INFO:tensorflow:loss = 0.00016926658, step = 1000 (2.691 sec)
    INFO:tensorflow:global_step/sec: 37.1934
    INFO:tensorflow:loss = 0.00012913694, step = 1100 (2.689 sec)
    INFO:tensorflow:global_step/sec: 37.3581
    INFO:tensorflow:loss = 0.00011296556, step = 1200 (2.677 sec)
    INFO:tensorflow:global_step/sec: 37.5765
    INFO:tensorflow:loss = 9.1343e-05, step = 1300 (2.661 sec)
    INFO:tensorflow:global_step/sec: 37.1447
    INFO:tensorflow:loss = 9.197992e-05, step = 1400 (2.692 sec)
    INFO:tensorflow:global_step/sec: 37.2682
    INFO:tensorflow:loss = 7.716995e-05, step = 1500 (2.683 sec)
    INFO:tensorflow:global_step/sec: 37.1961
    INFO:tensorflow:loss = 6.858538e-05, step = 1600 (2.689 sec)
    INFO:tensorflow:global_step/sec: 37.334
    INFO:tensorflow:loss = 6.3208936e-05, step = 1700 (2.679 sec)
    INFO:tensorflow:global_step/sec: 37.3852
    INFO:tensorflow:loss = 5.7119392e-05, step = 1800 (2.675 sec)
    INFO:tensorflow:global_step/sec: 37.4743
    INFO:tensorflow:loss = 5.7584162e-05, step = 1900 (2.668 sec)
    INFO:tensorflow:Saving checkpoints for 2000 into /tmp/tmp4n0pb66v/model.ckpt.
    INFO:tensorflow:Loss for final step: 4.364942e-05.

    <tensorflow.python.estimator.estimator.Estimator at 0x7fc45ae1ecf8>

```python
init_s, init_c = get_inits(X_test.shape[0], params)
estimator.evaluate(input_fn=lambda: eval_input_fn(
    X_test, y_test, init_s, init_c, batch_size=128))
```

    INFO:tensorflow:Calling model_fn.
    INFO:tensorflow:Done calling model_fn.
    INFO:tensorflow:Starting evaluation at 2019-04-12-09:31:08
    INFO:tensorflow:Graph was finalized.
    INFO:tensorflow:Restoring parameters from /tmp/tmp4n0pb66v/model.ckpt-2000
    INFO:tensorflow:Running local_init_op.
    INFO:tensorflow:Done running local_init_op.
    INFO:tensorflow:Finished evaluation at 2019-04-12-09:31:09
    INFO:tensorflow:Saving dict for global step 2000: accuracy = 1.0, global_step = 2000, loss = 4.37774e-05
    INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2000: /tmp/tmp4n0pb66v/model.ckpt-2000

    {'accuracy': 1.0, 'loss': 4.37774e-05, 'global_step': 2000}

### It Works

```python
from typing import List, Dict

def pad(input_seqlen: int,
        chars: List[str],
        ) -> List[str]:
    chars = chars[:input_seqlen]
    chars += ['<pad>'] * (input_seqlen - len(chars))
    return chars

def unk(input_vocab: Dict[str, int],
        chars: List[str],
        ) -> List[str]:
    chars = [c if c in input_vocab else '<unk>' for c in chars]
    return chars

def onehot(input_vocab: Dict[str, int],
           chars: List[str],
           ) -> List[List[int]]:
    # To indices
    indices = [input_vocab[c] for c in chars]
    # To onehot, shape `[input_seqlen, input_vocab_size]`
    return [[1 if i == index else 0 for i, _ in enumerate(input_vocab)]
            for index in indices]

def preprocess_sources(sources: List[str],
                       input_seqlen: int,
                       input_vocab: Dict[str, int]
                       ) -> np.array:

    def preprocess_source(source: str) -> List[List[int]]:
        chars = [c for c in source.lower()]
        chars = unk(input_vocab, chars)
        chars = pad(input_seqlen, chars)
        return onehot(input_vocab, chars)

    # [len(sources), input_seqlen, input_vocab_size]
    return np.array([preprocess_source(_) for _ in sources],
                    dtype='float32')

def postprocess(sources: List[str], predictions: dict) -> List[dict]:
    all_processed = []
    for src, pred in zip(sources, predictions):
        prediction = ''.join(inv_machine_vocab[_] for _ in pred['class_ids'])
        processed = {
            'source': src,
            'prediction': prediction,
            'attention_scores': pred['attention_scores'],
            'logits': pred['logits']
        }
        all_processed.append(processed)
    return all_processed
```

```python
sources = [
  '3 May 2079', '5 April 09', '21th of August 2016', 'Tue 10 Jul 2007',
  'Saturday May 9 2018', 'March 3 2001', 'March 3rd 2001', '1 March 2001'
]

X_oh_pred = preprocess_sources(sources, X_oh.shape[-2], human_vocab)
n_data_pred = X_oh_pred.shape[0]
input_seqlen = X_oh_pred.shape[1]
init_s_pred, init_c_pred = get_inits(len(sources), params)
predicted = estimator.predict(input_fn=lambda: eval_input_fn(
    X_oh_pred, None, init_s_pred, init_c_pred, batch_size=128))
predictions = postprocess(sources, predicted)

for src, pred in zip(sources, predictions):
  print(src)
  print(pred['prediction'])
  print()
```

    INFO:tensorflow:Calling model_fn.
    INFO:tensorflow:Done calling model_fn.
    INFO:tensorflow:Graph was finalized.
    INFO:tensorflow:Restoring parameters from /tmp/tmp4n0pb66v/model.ckpt-2000
    INFO:tensorflow:Running local_init_op.
    INFO:tensorflow:Done running local_init_op.
    3 May 2079
    2979-05-03

    5 April 09
    2009-04-05

    21th of August 2016
    2016-08-21

    Tue 10 Jul 2007
    2007-07-10

    Saturday May 9 2018
    2018-05-09

    March 3 2001
    2001-03-03

    March 3rd 2001
    2001-03-03

    1 March 2001
    2001-03-01

## Why Attention

The attention has one advantage that almost any other NN architecture absents, that is, the explainability!

```python
import matplotlib.pyplot as plt
from typing import List

def plot_attention(attention: np.array,
                   source: str,
                   prediction: str,
                   figsize=(7, 7)):

    def preprocess(s: str) -> List[str]:
      return ['*' if c == ' ' else c for c in s.lower()]

    source = preprocess(source)
    prediction = preprocess(prediction)
    attention = attention[:len(prediction), :len(source)]
    fig = plt.figure(figsize=figsize)
    ax = fig.add_subplot(1, 1, 1)
    ax.matshow(attention, cmap='viridis')

    # Set axes
    fontdict = {'fontsize': 12}
    ax.set_xticks(range(0, len(source)))
    ax.set_xticklabels(source, fontdict=fontdict)
    ax.set_yticks(range(0, len(prediction)))
    ax.set_yticklabels(prediction, fontdict=fontdict)

    plt.show()
```

```python
for pred in predictions:
    plot_attention(pred['attention_scores'], pred['source'], pred['prediction'])
```

The bi-LSTM mixes the characters in neighbour together because of its bi-directional memory, so in these plots, shifting exists.

```python

```


# ***Introduction to Radar Using Python and MATLAB***
## Andy Harrison - Copyright (C) 2019 Artech House
<br/>

# Power Aperture Product
***

Referring to Equation 4.54

\begin{equation}\label{eq:radar_equation_search_final}
    {SNR}_o = \frac{P_{av}\, A_e\, \sigma }{(4\pi)^3\, k\, T_0\, F\, L\, r^4} \, \frac{T_{scan}}{\Omega},
\end{equation}

$P_{av}\, A_e$ is the power aperture product and is a useful measure of performance for search and surveillance type radar systems.
***

Begin by getting the library path

```python
import lib_path
```

Set the minimum and maximum target range (m)

```python
target_min_range = 10e3

target_max_range = 100e3
```

Import the `linspace` routine from `scipy` and set up the target range array

```python
from numpy import linspace

target_range = linspace(target_min_range, target_max_range, 2000)
```

Set the noise figure (dB), the radar losses (dB), the target RCS (dBsm), the signal to noise ration (dB), the system temperature (K), the search volume (sr), and the scan time (s)

```python
from scipy.constants import pi

noise_figure = 4.0

losses = 9.0

target_rcs = -5.0

signal_to_noise = 23.0

system_temperature = 305

search_volume = 2.0 * pi

scan_time = 0.1
```

Set up the keyword args

```python
kwargs = {'target_range': target_range,

           'system_temperature': system_temperature,

                  'search_volume': search_volume,

                  'noise_factor': 10 ** (noise_figure / 10.0),

                  'losses': 10 ** (losses / 10.0),

                  'signal_to_noise': 10 ** (signal_to_noise / 10.0),

                  'scan_time': scan_time,

                  'target_rcs': 10 ** (target_rcs / 10.0)}
```

Import the `power_aperture` routine from `search_radar_range`

```python
from Libs.radar_range.search_radar_range import power_aperture
```

Calculate the power aperture product

```python
 power_aperture = power_aperture(**kwargs)
```

Import the `matplotlib` routines and the `log10` routine for plotting the power aperture product

```python
from matplotlib import pyplot as plt

from numpy import log10
```

Display the results

```python
# Set the figure size

plt.rcParams["figure.figsize"] = (15, 10)

# Display the results

plt.plot(target_range / 1.0e3, 10.0 * log10(power_aperture), '')

# Set the plot title and labels

plt.title('Power Aperture Product', size=14)

plt.xlabel('Target Range (km)', size=14)

plt.ylabel('Power Aperture (dB)', size=14)

# Set the tick label size

plt.tick_params(labelsize=12)

# Turn on the grid

plt.grid(linestyle=':', linewidth=0.5)
```


# ¿Cómo funciona la suspensión de un auto?

> Una primer aproximación al modelo de la suspensión de un automovil es considerar el *oscilador armónico amortiguado*.

Referencia:
- https://es.wikipedia.org/wiki/Oscilador_arm%C3%B3nico#Oscilador_arm.C3.B3nico_amortiguado

Un **modelo** que describe el comportamiento del sistema mecánico anterior es

\begin{equation}
m\frac{d^2 x}{dt^2}=-c\frac{dx}{dt}-kx
\end{equation}

donde $c$ es la constante de amortiguamiento y $k$ es la constante de elasticidad.  Revisar modelado

Documentación de los paquetes que utilizaremos hoy.
- https://docs.scipy.org/doc/scipy/reference/generated/scipy.integrate.odeint.html
- https://docs.scipy.org/doc/scipy/reference/index.html

___
En `python` existe una función llamada _odeint_ del paquete _integrate_ de la libreria _scipy_, que permite integrar sistemas vectoriales de primer orden, del tipo

\begin{equation}
\frac{d\boldsymbol{y}}{dt} = \boldsymbol{f}(\boldsymbol{y},t); \qquad \text{ con }\quad \boldsymbol{y}\in\mathbb{R}^n,\quad \boldsymbol{f}:\mathbb{R}^n\times\mathbb{R}_{+}\to\mathbb{R}^n
\end{equation}

con condiciones iniciales $\boldsymbol{y}(0) = \boldsymbol{y}_{0}$. Notar que  $\boldsymbol{y}$ representa un vector de $n$ componentes.

Ahora, si nos fijamos bien, el modelo del *oscilador armónico amortiguado* que obtuvimos es una ecuación diferencial ordinaria (EDO) de segundo orden. No hay problema. La podemos convertir en un sistema de ecuaciones de primer orden de la siguiente manera:

1. Seleccionamos el vector $\boldsymbol{y}=\left[y_1\quad y_2\right]^T$, con $y_1=x$ y $y_2=\frac{dx}{dt}$.
2. Notamos que $\frac{dy_1}{dt}=\frac{dx}{dt}=y_2$ y $\frac{dy_2}{dt}=\frac{d^2x}{dt^2}=-\frac{c}{m}\frac{dx}{dt}-\frac{k}{m}x=-\frac{c}{m}y_2-\frac{k}{m}y_1$.
3. Entonces, el modelo de segundo orden lo podemos representar como el siguiente sistema vectorial de primer orden:

\begin{equation}
\frac{d\boldsymbol{y}}{dt}=\left[\begin{array}{c}\frac{dy_1}{dt} \\ \frac{dy_2}{dt}\end{array}\right]=\left[\begin{array}{c}y_2 \\ -\frac{k}{m}y_1-\frac{c}{m}y_2\end{array}\right]=\left[\begin{array}{cc}0 & 1 \\-\frac{k}{m} & -\frac{c}{m}\end{array}\right]\boldsymbol{y}.
\end{equation}

```python
# Primero importamos todas las librerias, paquetes y/o funciones que vamos a utlizar

```

```python
# Ayuda función odeint

```

```python
# Función f(y,t) que vamos a integrar
def amortiguado(y, t, k, m, c):
    pass
```

```python
# Definimos los parámetros k, m y c

# Condiciones iniciales

# Especificamos los puntos de tiempo donde queremos la solución

```

```python
# Solución numérica

```

¿Cómo entrega odeint las soluciones?

```python
# Averiguar la forma de solución

```

```python
# Mostrar la solución

```

- $y$ es una matriz de n filas y 2 columnas.
- La primer columna de $y$ corresponde a $y_1$.
- La segunda columna de $y$ corresponde a $y_2$.

¿Cómo extraemos los resultados $y_1$ y $y_2$ independientemente?

```python
# Extraer y1 y y2

```

### Para hacer participativamente...
- Graficar en una misma ventana $y_1$ vs. $t$ y $y_2$ vs. $t$... ¿qué pueden observar?

```python
# Gráfica

```

- Graficar $y_2/\omega_0$ vs. $y_1$... ¿cómo se complementan estos gráficos? ¿conclusiones?

```python
# Gráfica

```

## Dependiendo de los parámetros, 3 tipos de soluciones

Teníamos
\begin{equation}
m\frac{d^2 x}{dt^2} + c\frac{dx}{dt} + kx = 0
\end{equation}

si recordamos que $\omega_0 ^2 = \frac{k}{m}$ y definimos $\frac{c}{m}\equiv 2\Gamma$, tendremos

\begin{equation}
\frac{d^2 x}{dt^2} + 2\Gamma \frac{dx}{dt}+ \omega_0^2 x = 0
\end{equation}

El comportamiento viene determinado por las raices de la ecuación característica. Ver en el tablero...

### Subamortiguado

Si $\omega_0^2 > \Gamma^2$ se tiene movimiento oscilatorio *subamortiguado*.

```python

```

```python

```

```python

```

Entonces, el primer caso que ya habíamos presentado corresponde a movimiento amortiguado.

```python
# Gráfica, de nuevo

```

### Sobreamortiguado

Si $\omega_0^2 < \Gamma^2$ se tiene movimiento oscilatorio *sobreamortiguado*.

```python
# Nuevas constantes
k = .1 # Constante del muelle
m = 1.0   # Masa
c = 1 # Constante de amortiguación
```

Simular y graficar...

```python

```

```python

```

```python

```

```python
# Simular

```

```python
# Graficar

```

### Amortiguamiento crítico

Si $\omega_0^2 = \Gamma^2$ se tiene movimiento *críticamente amortiguado*.

```python
# Nuevas constantes
k = .0625 # Constante del muelle
m = 1.0   # Masa
c = .5 # Constante de amortiguación
```

Simular y graficar...

```python

```

```python

```

```python

```

```python
# Simular

```

```python
# Graficar

```

En resumen, se tiene entonces:

```python
tt = t
fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(2, 3, sharex='col',
                                                       sharey='row',figsize =(10,6))
ax1.plot(tt, y1, c = 'k')
ax1.set_title('Amortiguado', fontsize = 14)
ax1.set_ylabel('Posición', fontsize = 14)
ax2.plot(tt, y1s, c = 'b')
ax2.set_title('Sobreamortiguado', fontsize = 14)
ax3.plot(tt, y1c, c = 'r')
ax3.set_title('Crítico', fontsize = 16)
ax4.plot(tt, y2, c = 'k')
ax4.set_ylabel('Velocidad', fontsize = 14)
ax4.set_xlabel('tiempo', fontsize = 14)
ax5.plot(tt, y2s, c = 'b')
ax5.set_xlabel('tiempo', fontsize = 14)
ax6.plot(tt, y2c, c = 'r')
ax6.set_xlabel('tiempo', fontsize = 14)
plt.show()
```

> **Tarea**. ¿Cómo se ve el espacio fase para los diferentes casos así como para diferentes condiciones iniciales?

> En un gráfico como el anterior, realizar gráficas del plano fase para los distintos movimientos y para cuatro conjuntos de condiciones iniciales distintas
- y0 = [1, 1]
- y0 = [1, -1]
- y0 = [-1, 1]
- y0 = [-1, -1]

Hacer lo anterior en un nuevo notebook de jupyter llamado Tarea7_ApellidoNombre.ipynb y subir en el espacio habilitado.

Created with Jupyter by Lázaro Alonso. Modified by Esteban Jiménez Rodríguez.


# International Macroeconomics

### Done by Iakov Grigoryev.

The numbers near the code cells are in a row since we restarted kernel and ran all the cells from the beginning to make sure that everything is working fine.

Import necessary libraries:

```python
import numpy as np
import matplotlib.pyplot as plt

from scipy.optimize import root
```

Let us firstly replicate Figure 9 from the page 50 (it will ensure us that we understood everything correctly). The formula (43) is the following:
$$
\theta=\dfrac{\mu_{\$} - \mu_h}{\gamma W\left(\sigma_{\$}^2+\sigma_h^2-2\sigma_{h\$}\right)}+\dfrac{\sigma_h^2-\sigma_{h\$}}{\left(\sigma_{\$}^2+\sigma_h^2-2\sigma_{h\$}\right)}=\dfrac{\mu_{\$} - \mu_h + \gamma W \left(\sigma_h^2-\sigma_{h\$}\right)}{\gamma W\left(\sigma_{\$}^2+\sigma_h^2-2\sigma_{h\$}\right)}.
$$

Here is a code, which implements $ \theta $ as a function:

```python
theta_base_fun = lambda mu_S, mu_h, gamma, W, sigma2_S, sigma2_h, sigma_hS: \
    (mu_S - mu_h + gamma * W * (sigma2_h - sigma_hS)) / (gamma * W * (sigma2_S + sigma2_h - 2 * sigma_hS))
```

On the same page we have specific formulas for the variables which are used in $ \theta $ calculation:

$$
\mu_h=\dfrac{\mathbb{E}_0\mathcal{E}_1^{-\alpha}}{Q_h\nu},\ \mu_{\$}=\dfrac{\mathbb{E}_0\mathcal{E}_1^{1-\alpha}}{Q_{\$}\nu},\ \sigma_h^2=\dfrac{\mathbb{V}_0\mathcal{E}_1^{-\alpha}}{Q_h^2\nu^2},\ \sigma_{\$}^2=\dfrac{\mathbb{V}_0\mathcal{E}_1^{1-\alpha}}{Q_{\$}^2\nu^2},\ \sigma_{h\$}=\dfrac{\mathbb{C}_0\left(\mathcal{E}_1^{-\alpha}, \mathcal{E}_1^{1-\alpha}\right)}{Q_hQ_{\$}\nu^2}.
$$

In our particular case $ \mathcal{E}_1 $ will be a random variable having three possible values with equal probabilities. Thus, mean, variance of this random variable in some power, as well as a covariance of this random variable in different powers, coincide with a sample mean, variance and covariance of a vector of this random variable values (in correnponding power). Particularly, we will have a vector $ [0.7, 1, 1.3] $ as a set of values $ \mathcal{E}_1 $ can have with equal probabilities, and the mean of $ \mathcal{E}_1^t $ will be $ \dfrac{1}{3} (0.7^t + 1^t + 1.3^t)$, which coincides with a sample mean of a vector $ [0.7^t, 1^t, 1.3^t] $ (and the same is for variance and covariance). Thus, we can say that those variables defined with formulas above are dependend on this vector of values. The code implementing those formulas is the following:

```python
# to make vectorized calculations possible (via broadcasting), we should make sure that
# eps_vec is column vector with a shape (n, 1) (but not a simple vector with a shape (-1, ))
# and alpha_vec, Q_h_vec, Q_S_vec and nu_vec are row vectors with a shape (1, m). In our case n = 3, m = 100

# here we calculate means
mu_h_fun = lambda eps_vec, Q_h_vec, nu_vec, alpha_vec: \
    np.mean(eps_vec**(-alpha_vec), axis=0) / (Q_h_vec * nu_vec)     # result shape: (1, m)
mu_S_fun = lambda eps_vec, Q_S_vec, nu_vec, alpha_vec: \
    np.mean(eps_vec**(1 - alpha_vec), axis=0) / (Q_S_vec * nu_vec)  # result shape: (1, m)

# here we wrote a function to calculate covariance because np.cov() function calculates much more than we need
# in mat_1 and mat_2 rows are variables (for each alpha they are different), and columns are values of a variable
# input shapes: mat_1 (m, n), mat_2 (m, n)
covariance_fun = lambda mat_1, mat_2: \
    np.mean((mat_1 - np.reshape(np.mean(mat_1, axis=1), newshape=(-1, 1))) *
            (mat_2 - np.reshape(np.mean(mat_2, axis=1), newshape=(-1, 1))), axis=1)  # result shape: (m, )

# here we calculate variances, and ddof=0 because we want to divide by n=3 but not by (n-1)
sigma2_h_fun = lambda eps_vec, Q_h_vec, nu_vec, alpha_vec: \
    np.var(eps_vec**(-alpha_vec), ddof=0, axis=0) / ((Q_h_vec * nu_vec)**2)     # result shape: (1, m)
sigma2_S_fun = lambda eps_vec, Q_S_vec, nu_vec, alpha_vec: \
    np.var(eps_vec**(1 - alpha_vec), ddof=0, axis=0) / ((Q_S_vec * nu_vec)**2)  # result shape: (1, m)

# here we calculate sigma_hS using covariance function we wrote above
sigma_hS_fun = lambda eps_vec, Q_h_vec, Q_S_vec, nu_vec, alpha_vec: \
    np.reshape(covariance_fun((eps_vec**(-alpha_vec)).T, (eps_vec**(1 - alpha_vec)).T), newshape=(1, -1)) / \
    (Q_h_vec * Q_S_vec * nu_vec**2)  # result shape: (1, m)
```

We also know (see page 49) that $ \nu=\dfrac{1}{\alpha^\alpha(1-\alpha)^{1-\alpha}} $:

```python
nu_fun = lambda alpha_vec: 1 / (alpha_vec**alpha_vec * (1 - alpha_vec)**(1 - alpha_vec))  # result shape: (1, m)
```

Now we are ready to write $ \theta $ as a function of $ \mathcal{E}_1 $ (more accurately, its values), $ \gamma, W, \alpha, Q_h $ and $ Q_{\$} $:

```python
theta_tmp1_fun = lambda eps_vec, gamma, W, alpha_vec, Q_h_vec, Q_S_vec: theta_base_fun(
    mu_S_fun(eps_vec, Q_S_vec, nu_fun(alpha_vec), alpha_vec),
    mu_h_fun(eps_vec, Q_h_vec, nu_fun(alpha_vec), alpha_vec),
    gamma, W,
    sigma2_S_fun(eps_vec, Q_S_vec, nu_fun(alpha_vec), alpha_vec),
    sigma2_h_fun(eps_vec, Q_h_vec, nu_fun(alpha_vec), alpha_vec),
    sigma_hS_fun(eps_vec, Q_h_vec, Q_S_vec, nu_fun(alpha_vec), alpha_vec)
).ravel()  # result shape: (m, )
```

Now, to replicate Figure 9, we need to find $ D_{\$} / D_h $. What we know from the page 49:
$$
\theta=\dfrac{Q_{\$}D_{\$}}{W},\ W=Q_hD_h+\mathcal{E}_0Q_{\$}D_{\$}\text{ (there is an obvious typo in an article, "+" sign is correct here), where } \mathcal{E}_0=1.
$$
Thus, $ 1-\theta=1-\dfrac{Q_{\$}D_{\$}}{W}=\dfrac{W-Q_{\$}D_{\$}}{W}=\dfrac{Q_hD_h}{W} $. Now we can express $ D_{\$} $, $ D_h $ and their quotient:
$$
D_{\$}=\dfrac{W\theta}{Q_{\$}},\ D_h=\dfrac{W(1-\theta)}{Q_h}\Longrightarrow \dfrac{D_{\$}}{D_h}=\dfrac{W\theta\cdot Q_h}{Q_{\$}\cdot W(1-\theta)}=\dfrac{\theta}{1-\theta}\cdot\dfrac{Q_h}{Q_{\$}}.
$$

The code for this function is the following:

```python
def D_S_div_D_h_fun(eps_vec, gamma, W, alpha_vec, Q_h_vec, Q_S_vec):
    theta_vec = theta_tmp1_fun(eps_vec, gamma, W, alpha_vec, Q_h_vec, Q_S_vec)
    return (theta_vec / (1 - theta_vec) * Q_h_vec / Q_S_vec).ravel()      # result shape: (m, )
```

Now we are ready to replicate Figure 9. Start with parameter initialization (values are taken from the page 51):

```python
# as we mentioned above, eps_vec should be column vector with a shape (n, 1)
eps_vec = np.reshape(np.array([0.7, 1, 1.3]), newshape=(-1, 1))

gamma = 15
W = 3

# given Q_h = 0.8 and Q_S = 0.9 we will plot D_S / D_h as a function of alpha / (1 - alpha)
Q_h = 0.8
Q_S = 0.9

# given alpha = 0.5 we will plot D_S / D_h as a function of Q_h / Q_S
alpha = 0.5
```

We will firstly plot $ D_{\$} / D_h $ as a function of $ \alpha/(1-\alpha) $. Note that we can express $ \alpha $ from $ \alpha/(1-\alpha) $:
$$
\dfrac{\alpha}{1-\alpha}=t\Leftrightarrow \alpha=t - t\alpha\Leftrightarrow \alpha=\dfrac{t}{1+t}.
$$

So, we will generate splitting for $ \alpha/(1-\alpha) $, after that we will find a correspoding splitting for $ \alpha $ which we can then substitute to the function `D_S_div_D_h_fun()`:

```python
# number of points to be connected on the plots
m = 100

# 0.2 and 2.3 are the borders of alpha / (1 - alpha) on Figure 9
a_min = 0.2
a_max = 2.3

# 0 and 2 are borders of D_S / D_h on Figure 9 (on both plots)
d_min = 0
d_max = 2

# as we mentioned above, alpha_vec should be row vector with a shape (1, m)
alpha_div_1_minus_alpha_vec = np.reshape(np.linspace(a_min, a_max, m), newshape=(1, -1))
alpha_vec = alpha_div_1_minus_alpha_vec / (1 + alpha_div_1_minus_alpha_vec)  # this is t / (1 + t)

D_S_div_D_h_1_vec = D_S_div_D_h_fun(eps_vec, gamma, W, alpha_vec, Q_h, Q_S)

plt.plot(alpha_div_1_minus_alpha_vec.ravel(), D_S_div_D_h_1_vec, linewidth=4)

plt.xlabel(r"Relative Dollar Preference: $ \alpha/(1-\alpha) $")
plt.title(r"Relative Demand for Dollar Deposits: $ D_\$/D_h $")
plt.xlim(a_min, a_max)
plt.ylim(d_min, d_max)
plt.xticks([0.5, 1, 1.5, 2])
plt.yticks([0, 0.5, 1, 1.5, 2])

plt.show()
```

Awesome, our plot is exactly the same as in Figure 9! We are on the right way.  Now we will replicate the second plot of Figure 9. Note that $ D_{\$}/D_h $ is not exactly the function of $ Q_h/Q_{\$} $ (in a typical sense), but for a constant $ Q_{\$} $ (for example) $ D_{\$}/D_h $ is increasing in $ Q_h $. Thus, let us fix $ Q_{\$}=0.9 $ (as before) and make a splitting for $ Q_h $ (and after that we will repeat the same process for $ Q_{\$}=3\cdot0.9=2.7 $ to make sure that the function has changed, but monotonicity has not).

```python
# 0.84 and 1.065 are the borders of Q_h / Q_S on Figure 9
q_min = 0.84
q_max = 1.065

# as we mentioned above, Q_h_vec should be row vector with a shape (1, m)
Q_h_vec = np.reshape(np.linspace(q_min * Q_S, q_max * Q_S, m), newshape=(1, -1))

# note that here we use a fixed alpha = 0.5
D_S_div_D_h_2_vec = D_S_div_D_h_fun(eps_vec, gamma, W, alpha, Q_h_vec, Q_S)

plt.plot((Q_h_vec / Q_S).ravel(), D_S_div_D_h_2_vec, linewidth=4)

plt.xlabel(r"Relative Price: $ Q_h/Q_{\$} $")
plt.title(r"Relative Demand for Dollar Deposits: $ D_\$/D_h $")
plt.xlim(q_min, q_max)
plt.ylim(d_min, d_max)
plt.xticks([0.85, 0.9, 0.95, 1, 1.05])
plt.yticks([0, 0.5, 1, 1.5, 2])

plt.show()
```

Looks exactly like on Figure 9! But if we change $ Q_{\$} $ to $ 2.7 $, the plot will be different, but most importantly, it will still be increasing function:

```python
Q_S_new = 2.7
Q_h_new_vec = np.reshape(np.linspace(q_min * Q_S_new, q_max * Q_S_new, m), newshape=(1, -1))

D_S_div_D_h_3_vec = D_S_div_D_h_fun(eps_vec, gamma, W, alpha, Q_h_new_vec, Q_S_new)

plt.plot((Q_h_new_vec / Q_S_new).ravel(), D_S_div_D_h_3_vec, linewidth=4)

plt.xlabel(r"Relative Price: $ Q_h/Q_{\$} $")
plt.title(r"Relative Demand for Dollar Deposits: $ D_\$/D_h $")
plt.xlim(q_min, q_max)
plt.ylim(d_min, d_max)
plt.xticks([0.85, 0.9, 0.95, 1, 1.05])
plt.yticks([0, 0.5, 1, 1.5, 2])

plt.show()
```

OK, enough for Figure 9. Let us move to the replication of Figure 10. The banking sector in our problem is the same as in Section 2.2:
$$
\max\limits_{B_h,B_{\$},B_R}\mathbb{E}_0\left[\gamma N - B_h - B_{\$}-\xi B_R\right],
$$
subject to,
$$
Q_hB_h+Q_{\$}B_{\$}+Q_RB_R\ge N,
$$
$$
\bar{\mathcal{E}}B_{\$}+B_h\le \gamma_L N.
$$

We will not write here the meaning of the parameters as they are perfectly described in the paper. Let us concentrate on the mathematical calculations.

Let $ \lambda $ and $ \mu $ be the Lagrange multipliers on the constraints above, respectively. The first-order conditions for the problem imply:
$$
Q_{\$}=\dfrac{\mu \bar{\mathcal{E}} + 1}{\lambda},
$$
$$
Q_h = \dfrac{\mu + 1}{\lambda},
$$
$$
Q_R=\dfrac{1}{\lambda}.
$$

Remember that in our problem there are risk-neutral investors (pages 48-49) and from their problem it follows that $ Q_R = \beta $. Thus, $ \dfrac{1}{\lambda}=\beta $ implying $ Q_{\$}-\beta=\beta\mu\bar{\mathcal{E}} $ and $ Q_{h}-\beta=\beta\mu $. Firstly, consider the internal solution case (when $ B_{\$}>0 $, $ \bar{\mathcal{E}}B_{\$}+B_h= \gamma_L N $ and $ \mu\ne0 $). In this case we can divide $ Q_{\$}-\beta $ by $ Q_h-\beta $ and derive the following equation:
\begin{equation} \label{eqn::internal_sol} \tag{1}
\dfrac{Q_{\$}-\beta}{Q_h-\beta}=\bar{\mathcal{E}}.
\end{equation}

But note that in the border case, when $ B_{\$}=0 $, the equation (1) is not correct (because $ \mu=0 $).

Let us move to the market clearing conditions: $ D_{\$}=B_{\$}+X $, $ D_h=B_h $. Thus, assuming that the safe asset constraint binds,
\begin{equation}
\bar{\mathcal{E}}B_{\$}+B_h= \gamma_L N\Leftrightarrow \bar{\mathcal{E}}\left(D_{\$}-X\right)+B_h= \gamma_L N\Leftrightarrow D_h=\gamma_L N + \bar{\mathcal{E}}\left(X-D_{\$}\right).
\label{eqn::safe_asset} \tag{2}
\end{equation}

We also derived above that
\begin{equation}
D_{\$}=\dfrac{W\theta}{Q_{\$}}
\label{eqn::D_S_eq} \tag{3}
\end{equation}
and
\begin{equation}
D_h=\dfrac{W(1-\theta)}{Q_h}.
\label{eqn::D_h_eq} \tag{4}
\end{equation}

So, we have two cases:
- internal solution, in which there is a system of four equations (1)-(4);
- border solution, in which $ B_{\$}=D_{\$}-X=0 $ implying $ D_{\$}=X $, which (with the equation (2)) entails that $ D_h=\gamma_L N $, and these in sum imply the system of two equations (5)-(6):
\begin{equation}
X=\dfrac{W\theta}{Q_{\$}}
\label{eqn::border_1} \tag{5}
\end{equation}
and
\begin{equation}
\gamma_L N=\dfrac{W(1-\theta)}{Q_h}.
\label{eqn::border_2} \tag{6}
\end{equation}

Let us first find the internal solution by solving the first system for the variables $ D_{\$}, D_h, Q_{\$}, Q_h $ as functions of $ \alpha $. Note that $ \theta $ is already a function of $ \alpha, Q_h, Q_{\$} $ (given $ \gamma, W $ and the vector of values of $ \mathcal{E}_1 $ fixed). Let us implement it in code:

```python
# from now on all calculations won't be vectorized since
# we will use function root finders (solvers) which cannot handle vector input,
# so we won't note output shapes anymore

# note that we won't use W, gamma and eps_vec as function parameters anymore and we will consider them
# as global variables defined above (it is reasonable since we won't change them and we also have a lot of variables)

theta_tmp2_fun = lambda alpha, Q_h, Q_S: theta_tmp1_fun(eps_vec, gamma, W, alpha, Q_h, Q_S)
```

From (1) it follows that $ Q_{\$}=Q_{\$}(Q_h)=\beta+\bar{\mathcal{E}}\left(Q_h-\beta\right) $. Let us implement this in code as well:

```python
Q_S_fun = lambda Q_h, beta, eps_max: beta + eps_max * (Q_h - beta)
```

Now we can rewrite $ \theta $ as a function of only $ \alpha $ and $ Q_h $ (and some parameters which will later be initialized):

```python
theta_fun = lambda alpha, Q_h, beta, eps_max: theta_tmp2_fun(alpha, Q_h, Q_S_fun(Q_h, beta, eps_max))
```

In equations (2) and (4) the left side is the same and equals $ D_h $. Let us equate the right sides:
$$
\dfrac{W(1-\theta)}{Q_h}=\gamma_L N + \bar{\mathcal{E}}(X-D_{\$}).
$$

Now substitute $ D_{\$} $ from equation (3) into the previous formula and move everything to the left:
\begin{equation}
\dfrac{W(1-\theta)}{Q_h}-\gamma_L N-\bar{\mathcal{E}}\left(X-\dfrac{W\theta}{Q_{\$}}\right)=0.
\end{equation}

So, what did we get? We got an equation in the form $ F(\alpha, Q_h)=0 $ (where $ \theta $ and $ Q_{\$} $ are expressed as a functions of $ \alpha $ and $ Q_h $ above; more accurately, $ Q_{\$} $ is a function of only $ Q_h $ so far), given the parameters $ W, \gamma, \gamma_L, N, X $ and the values of $ \mathcal{E}_1 $ including the maximum value $ \bar{\mathcal{E}} $ fixed. Thus, for a given $ \alpha $ we can find a corresponding value of $ Q_h $ solving the equation $ F(\alpha, Q_h)=0 $. The function $ F $ which we are going to find a root for is implemented in the following code:

```python
def F_opt_fun(alpha, Q_h, beta, eps_max, gamma_L, N, X):
    theta = theta_fun(alpha, Q_h, beta, eps_max)
    return W * (1 - theta) / Q_h - gamma_L * N - eps_max * (X - W * theta / Q_S_fun(Q_h, beta, eps_max))
```

Now we can find $ Q_h $ as a function of $ \alpha $ (using a method described in the previous cell). We start our search in $ Q_h=1 $ and use Levenberg-Marquardt method (`lm` in code):

```python
Q_h_fun = lambda alpha, beta, eps_max, gamma_L, N, X: \
    root(lambda Q_h: F_opt_fun(alpha, Q_h, beta, eps_max, gamma_L, N, X), [1,], method='lm').x[0]
```

We got $ Q_h $ as a function of $ \alpha $. Thus, using $ Q_{\$}=Q_{\$}(Q_h)=\beta+\bar{\mathcal{E}}\left(Q_h-\beta\right) $, we can derive $ Q_{\$} $ for the same $ \alpha $. After that we can find $ \theta $ for this $ \alpha $. Finally, using equations (3) and (4) we can find $ D_{\$} $ and $ D_h $. Thus, the internal solution is found.

Secondly, let us find the value of $ \alpha $ which is a border value for the internal solution. This value is such that $ B_{\$}=D_{\$}-X=0 $ (being $ >0 $ for the values to the right of this value of $ \alpha $). We already found $ D_{\$} $ as a function of $ \alpha $ in the internal solution, and the border value $ \bar{\alpha} $ is such that $ D_{\$}(\bar{\alpha})-X=0 $. Let us find this value (and now it will be a function of parameters which will be initialized a bit later):

```python
def alpha_opt_fun(alpha, beta, eps_max, gamma_L, N, X):
    # find Q_h for a given alpha
    Q_h = Q_h_fun(alpha, beta, eps_max, gamma_L, N, X)
    # find Q_S for the same alpha
    Q_S = Q_S_fun(Q_h, beta, eps_max)

    # find theta for the same alpha
    theta = theta_fun(alpha, Q_h, beta, eps_max)

    # find D_S for the same alpha
    D_S = W * theta / Q_S
    # the function to find a root of is D_S(alpha) - X
    return D_S - X

# here we simply find roots of the function written above (start with alpha=0.2 and use the same method as before)
alpha_border_fun = lambda beta, eps_max, gamma_L, N, X: \
    root(lambda alpha: alpha_opt_fun(alpha, beta, eps_max, gamma_L, N, X), [0.2,], method='lm').x[0]
```

Finally, let us find the border solution (solving the system (5)-(6)). We will do that directly:

For a given $ \alpha<\bar{\alpha} $
- find $ \theta $ as a function of $ Q_h, Q_{\$} $;
- substitute it to the system (5)-(6) getting a system of two equations and two variables $ Q_h, Q_{\$} $;
- solve that system.

In the following code we will do that for any fixed values of $ D_h $ and $ D_{\$} $ (thus, solving (3) and (4)), but later we will substitute the values $ D_{\$}=X $ and $ D_h=\gamma_L N $ to the found functions $ Q_h(D_h, D_{\$}, \alpha) $ and $ Q_{\$}(D_h, D_{\$}, \alpha) $.

```python
def Q_h_Q_S_ext_opt_fun(alpha, Q_h, Q_S, D_h, D_S):
    # find theta for a given alpha, Q_h and Q_S
    theta = theta_tmp2_fun(alpha, Q_h, Q_S)[0]

    # to solve the system, we move everything to the left and equate it to zero
    return [W * theta / Q_S - D_S, W * (1 - theta) / Q_h - D_h]

# note that in the root() the function should be dependend of one vectorized variable,
# so we moved Q_h and Q_S into Q_h_Q_S_vec
# initial values for a search are 1 and 1, and the method is the same as before
Q_h_Q_S_ext_vec_fun = lambda alpha, D_h, D_S: \
    root(lambda Q_h_Q_S_vec: Q_h_Q_S_ext_opt_fun(alpha, Q_h_Q_S_vec[0], Q_h_Q_S_vec[1], D_h, D_S),
         [1, 1], method='lm').x
```

Now let us initialize our parameters (values are taken from the page 51). Remember that $ \bar{\mathcal{E}} $ is simply a maximum possible value of $ \mathcal{E}_1 $ (in our casse it is $ 1.3 $).

```python
N = 4.5
X = 0.8
beta = 0.75
gamma_L = 0.6

eps_max = np.max(eps_vec)
```

Next let us find the exact border value of $ \alpha $:

```python
alpha_border = alpha_border_fun(beta, eps_max, gamma_L, N, X)
print(f"Border value of alpha is {alpha_border:.3f}.")
```

    Border value of alpha is 0.248.

It seems that we found the right value (comparing to Figure 10)! Now everything is prepared to find $ Q_{\$}-Q_h $ and $ B_{\$} $ as functions of $ \alpha $!

```python
# we will use this function in the loop with scalar alpha, so we needn't worry about shapes
# we will find both Q_S - Q_h and B_S in the same function
def Q_S_minus_Q_h_and_B_S_fun(alpha):
    if alpha >= alpha_border:
        # the internal solution
        Q_h = Q_h_fun(alpha, beta, eps_max, gamma_L, N, X)
        Q_S = Q_S_fun(Q_h, beta, eps_max)

        theta = theta_fun(alpha, Q_h, beta, eps_max)

        D_S = W * theta / Q_S
        D_h = W * (1 - theta) / Q_h

        B_S = D_S - X

    else:
        # the border solution
        D_S = X
        D_h = gamma_L * N

        B_S = 0

        Q_h, Q_S = Q_h_Q_S_ext_vec_fun(alpha, D_h, D_S)
    return Q_S - Q_h, B_S
```

Finally, to replicate Figure 10, we should create a splitting for $ \alpha $ and find the values for $ Q_{\$}-Q_h $ and $ B_{\$} $ in the loop:

```python
# 0.16 and 0.7 are the borders of alpha on Figure 10
a_min = 0.16
a_max = 0.7

alpha_vec = np.linspace(a_min, a_max, m)

Q_S_minus_Q_h_vec = np.empty(shape=m)
B_S_vec = np.empty(shape=m)

for (i, alpha) in enumerate(alpha_vec):
    Q_S_minus_Q_h_vec[i], B_S_vec[i] = Q_S_minus_Q_h_and_B_S_fun(alpha)
```

Now we are ready to plot!

```python
# -0.15 and 0.1 are the borders of Q_S - Q_h on Figure 10
q_min = -0.15
q_max = 0.1

plt.plot(alpha_vec, Q_S_minus_Q_h_vec, linewidth=4)

plt.xlabel(r"Dollar Share in Consumption: $ \alpha $")
plt.title(r"UIP Deviations: $ Q_{\$}-Q_h $")
plt.xlim(a_min, a_max)
plt.ylim(q_min, q_max)
plt.xticks([0.2, 0.3, 0.4, 0.5, 0.6, 0.7])
plt.yticks([-0.15, -0.1, -0.05, 0, 0.05, 0.1])

# here we fill the area of border solution (alpha < alpha_border)
plt.fill_between([a_min, alpha_border], q_min, q_max, facecolor='blue', alpha=0.2)

plt.show()
```

```python
# 0 and 3.2 are the borders of B_S on Figure 10
b_min = 0
b_max = 3.2

# we use b_min - 2e-2 below to be able to see the zero line (and a part of the curve) in border solution area
plt.plot(alpha_vec, B_S_vec, linewidth=4)

plt.xlabel(r"Dollar Share in Consumption: $ \alpha $")
plt.title(r"Dollar Borrowing: $ B_{\$} $")
plt.xlim(a_min, a_max)
plt.ylim(b_min - 2e-2, b_max)
plt.xticks([0.2, 0.3, 0.4, 0.5, 0.6, 0.7])
plt.yticks([0, 0.5, 1, 1.5, 2, 2.5, 3])

# here we fill the area of border solution (alpha < alpha_border)
plt.fill_between([a_min, alpha_border], b_min - 2e-2, b_max, facecolor='blue', alpha=0.2)

plt.show()
```

Hooray, our plots are exactly the same as in Figure 10 from the paper! I feel relieved.