# Fornberg-Whitham equation

```julia
include("setup.jl")
```

# Periodic boundary conditions

```julia
function relaxation!(integrator)
    told = integrator.tprev
    uold = integrator.uprev
    tnew = integrator.t
    unew = integrator.u

    @unpack D, tmp1 = integrator.p

    @. tmp1 = (unew - uold)^2
    c = integrate(tmp1, D)
    @. tmp1 = (unew - uold) * uold
    γ = one(c)
    if !iszero(c)
        γ = -2 * integrate(tmp1, D) / c
    end
    @. unew = uold + γ * (unew - uold)
    DiffEqBase.set_u!(integrator, unew)
    if !(tnew ≈ top(integrator.opts.tstops))
        tγ = told + γ * (tnew - told)
        DiffEqBase.set_t!(integrator, tγ)
    end

    nothing
end

function save_func_fw_periodic(u, t, integrator)
    @unpack D, invImD2, tmp1, usol = integrator.p

    mass = integrate(u, D)

    ldiv!(tmp1, invImD2, u)
    linear = integrate(tmp1, D)

    @. tmp1 = u^2
    quadratic = integrate(tmp1, D)

    x = grid(D)
    tmp1 .= ( usol.(t, x, x[1], -x[1]) .- u ).^2
    error_u = integrate(tmp1, D) |> sqrt

    SVector(mass, linear, quadratic)
end

function fw_periodic!(du, u, param, t)
    @unpack D, invImD2, tmp1, tmp2 = param
    one_third = one(t) / 3

    # energy and mass conservative split form semidiscretization
    mul!(tmp1, D, u)
    ldiv!(du, invImD2, tmp1)
    @. du += one_third * (u * tmp1)
    @. tmp1 = u * u
    mul!(tmp2, D, tmp1)
    @. du = -du - one_third * tmp2

    # standard central method without splitting: not energy conservative
#     mul!(tmp1, D, u)
#     ldiv!(du, invImD2, tmp1)
#     @. tmp1 = u * u
#     mul!(du, D, tmp1, -one(eltype(D))/2, -one(eltype(D)))

    nothing
end

function solve_ode_fw_periodic(usol, D, D2, tspan, alg, tol, dt, adaptive)
    invImD2 = isa(D2, AbstractMatrix) ? factorize(I - D2) : I - D2

    x = grid(D)
    u0 = usol.(tspan[1], x, x[1], -x[1])
    tmp1 = similar(u0); tmp2 = similar(tmp1)
    param = (D=D, D2=D2, invImD2=invImD2, tmp1=tmp1, tmp2=tmp2, usol=usol)

    ode = ODEProblem(fw_periodic!, u0, tspan, param)

    saveat = range(tspan..., length=100)
    saved_values_baseline = SavedValues(eltype(D), SVector{3,eltype(D)})
    saving_baseline = SavingCallback(save_func_fw_periodic, saved_values_baseline, saveat=saveat)
    saved_values_relaxation = SavedValues(eltype(D), SVector{3,eltype(D)})
    saving_relaxation = SavingCallback(save_func_fw_periodic, saved_values_relaxation, saveat=saveat)
    relaxation = DiscreteCallback((u,t,integrator) -> true, relaxation!, save_positions=(false,true))
    cb_baseline = CallbackSet(saving_baseline)
    cb_relaxation = CallbackSet(relaxation, saving_relaxation)

    sol_relaxation = solve(ode, alg, abstol=tol, reltol=tol, dt=dt, adaptive=adaptive, save_everystep=false,
        callback=cb_relaxation, tstops=saveat)
    sol_baseline = solve(ode, alg, abstol=tol, reltol=tol, dt=dt, adaptive=adaptive, save_everystep=false,
        callback=cb_baseline, tstops=saveat)

    unum_baseline   = sol_baseline[end]
    unum_relaxation = sol_relaxation[end]
    uana = usol.(tspan[end], x, x[1], -x[1])
    @printf("Error in u (baseline):   %.3e\n", integrate(u->u^2, unum_baseline - uana, D) |> sqrt)
    @printf("Error in u (relaxation): %.3e\n", integrate(u->u^2, unum_relaxation - uana, D) |> sqrt)
    @printf("Difference of baseline and relaxation in u: %.3e\n",
        integrate(u->u^2, unum_baseline - unum_relaxation, D) |> sqrt)

    sleep(0.1)
    fig_u, ax = plt.subplots(1, 1)
    plt.plot(x, u0, label=L"u^0")
    plt.plot(x, uana, label=L"$u^\mathrm{ana}$")
    plt.plot(x, unum_baseline, label=L"$u^\mathrm{num}$ (baseline)")
    plt.plot(x, unum_relaxation, label=L"$u^\mathrm{num}$ (relaxation)")
    plt.xlabel(L"x"); plt.ylabel(L"u")
    plt.legend(loc="center left", bbox_to_anchor=(1.0, 0.5));

    t_baseline = saved_values_baseline.t
    t_relaxation = saved_values_relaxation.t
    mass_baseline        = map(x->x[1], saved_values_baseline.saveval)
    mass_relaxation      = map(x->x[1], saved_values_relaxation.saveval)
    linear_baseline      = map(x->x[2], saved_values_baseline.saveval)
    linear_relaxation    = map(x->x[2], saved_values_relaxation.saveval)
    quadratic_baseline   = map(x->x[3], saved_values_baseline.saveval)
    quadratic_relaxation = map(x->x[3], saved_values_relaxation.saveval)

    fig_invariants, ax = plt.subplots(1, 1)
    ax.set_yscale("symlog", linthreshy=1.0e-14)
    plt.plot(t_baseline,   mass_baseline   .- mass_baseline[1], label=L"$\int u$ (baseline)")
    plt.plot(t_relaxation, mass_relaxation .- mass_relaxation[1], label=L"$\int u$ (relaxation)")
    plt.plot(t_baseline,   linear_baseline   .- linear_baseline[1], label=L"$\int (u - \partial_x^2 u)$ (baseline)")
    plt.plot(t_relaxation, linear_relaxation .- linear_relaxation[1], label=L"$\int (u - \partial_x^2 u)$ (relaxation)")
    plt.plot(t_baseline,   quadratic_baseline    .- quadratic_baseline[1], label=L"$\int u^2$ (baseline)")
    plt.plot(t_relaxation, quadratic_relaxation  .- quadratic_relaxation[1], label=L"$\int u^2$ (relaxation)")
    plt.xlabel(L"t"); plt.ylabel("Change of Invariants")
    plt.legend(loc="center left", bbox_to_anchor=(1.0, 0.5))

    fig_u, fig_invariants
end

```

```julia
# # single peakon solution
# xmin = -60.
# xmax = -xmin
# get_c() = 1.2
# function usol(t, x, xmin, xmax)
#     c = get_c()
#     x_t = mod(x - c*t - xmin, xmax - xmin) + xmin

#     4/3 * exp(-0.5 * abs(x_t)) + c - 4/3
# end

# traveling wave solution obtained numerically
c, data = open("fw_traveling_wave_init_c12_l80_N65536.txt", "r") do io
    line = readline(io)
    line = readline(io)
    line = readline(io)
    c = parse(Float64, line[8:end])
    data = readdlm(io, comments=true)
    c, data
end
x = range(data[1,1], data[end,1], length=size(data,1))
xmin = x[1]; xmax = x[end]
u0 = data[:, 2]
u0itp = CubicSplineInterpolation((x,), u0, extrapolation_bc=Periodic())
get_c() = c
function usol(t, x, xmin, xmax)
    c = get_c()
    x_t = mod(x - c*t - xmin, xmax - xmin) + xmin
    u0itp(x_t)
end

println("c = ", get_c())
println("xmin = ", xmin)
println("xmax = ", xmax)
@show usol(0., xmin, xmin, xmax)
@show usol(0., xmax, xmin, xmax)
@show N = 2^8
@show dt = 1.0 * (xmax - xmin) / (N * get_c())

# @show tspan = (0., (xmax-xmin)/(3*get_c()))
# @show tspan = (0.0, (xmax-xmin)/(3*get_c()) + 1*(xmax-xmin)/get_c())
@show tspan = (0.0, (xmax-xmin)/(3*get_c()) + 10*(xmax-xmin)/get_c())
# @show tspan = (0.0, (xmax-xmin)/(3*get_c()) + 100*(xmax-xmin)/get_c())
# @show tspan = (0.0, (xmax-xmin)/(3*get_c()) + 1000*(xmax-xmin)/get_c())
flush(stdout)

tol = 1.0e-7
adaptive = false
D = fourier_derivative_operator(xmin, xmax, N)
# D = periodic_derivative_operator(1, 6, xmin, xmax, N+1)
# D = periodic_derivative_operator(Holoborodko2008(), 1, 4, xmin, xmax, N+1)
D2 = D^2
# D2 = periodic_derivative_operator(2, 6, xmin, xmax, N+1)
# D2 = periodic_derivative_operator(Holoborodko2008(), 1, 4, xmin, xmax, N+1)^2
# p = 8; mesh = UniformPeriodicMesh1D(xmin, xmax, N÷p)
# D = couple_continuosly(legendre_derivative_operator(-1., 1., p+1), mesh)
# D2 = sparse(couple_continuosly(legendre_second_derivative_operator(-1., 1., p+1), mesh))
# D2 = sparse(D)^2
# p = 5; Dop = legendre_derivative_operator(-1., 1., p+1); mesh = UniformPeriodicMesh1D(xmin, xmax, N÷(p+1))
# D = couple_discontinuosly(Dop, mesh); D₊ = couple_discontinuosly(Dop, mesh, Val(:plus)); D₋ = couple_discontinuosly(Dop, mesh, Val(:minus))
# D2 = sparse(D₊) * sparse(D₋)
# D2 = sparse(D₋) * sparse(D₊)
# D2 = sparse(D)^2

fig_u, fig_invariants = solve_ode_fw_periodic(usol, D, D2, tspan, RK4(), tol, dt, adaptive)
```

```julia

```

# Convergence study with manufactured solutions

```julia
import SymPy; sp = SymPy

function math_replacements(s)
    s = replace(s, "cos(pi*" => "cospi(")
    s = replace(s, "sin(pi*" => "sinpi(")
end

function usol(t, x)
    exp(t/2) * sinpi(2*(x-t/2))
end

let (t, x) = sp.symbols("t, x", real=true)
    u = usol(t, x)

    println("u:")
    sp.diff(u, t) - sp.diff(u, x, 2, t, 1) + sp.diff(u, x) + sp.diff(u^2/2, x) - sp.diff(u^2/2, x, 3) |>
        sp.simplify |> sp.string |> math_replacements |> println
end
```

```julia
function save_func_fw_manufactured(u, t, integrator)
    @unpack D, tmp1, usol = integrator.p

    x = grid(D)
    tmp1 .= ( usol.(t, x) .- u ).^2
    error_u = integrate(tmp1, D) |> sqrt

    error_u
end

function fw_periodic_manufactured!(du, u, param, t)
    @unpack D, invImD2, tmp1, tmp2 = param
    x = grid(D)
    one_third = one(t) / 3

    mul!(tmp1, D, u)
    @. tmp2 = -tmp1 - 2*pi^2*(sinpi((t - 2*x)) + 2*pi*cospi((t - 2*x)))*exp(t/2) - exp(t/2)*sinpi((t - 2*x))/2 + pi*exp(t/2)*cospi((t - 2*x)) - 16*pi^3*exp(t)*sinpi((2*t - 4*x)) - pi*exp(t)*sinpi((2*t - 4*x))
    ldiv!(du, invImD2, tmp2)
    @. du -= one_third * (u * tmp1)
    @. tmp1 = u * u
    mul!(tmp2, D, tmp1)
    @. du -= one_third * tmp2

    nothing
end

function errors_fw_periodic_manufactured(usol, D, D2, tspan, alg, tol, dt, adaptive)
    invImD2 = isa(D2, AbstractMatrix) ? factorize(I - D2) : I - D2

    x = grid(D)
    u0 = usol.(tspan[1], x)
    tmp1 = similar(u0); tmp2 = similar(tmp1)
    param = (D=D, D2=D2, invImD2=invImD2, tmp1=tmp1, tmp2=tmp2, usol=usol)

    ode = ODEProblem(fw_periodic_manufactured!, u0, tspan, param)

    saveat = range(tspan..., length=2)
    saved_values_baseline = SavedValues(eltype(D), eltype(D))
    saving_baseline = SavingCallback(save_func_fw_manufactured, saved_values_baseline, saveat=saveat)
    cb_baseline = CallbackSet(saving_baseline)

    sol_baseline = solve(ode, alg, abstol=tol, reltol=tol, dt=dt, adaptive=adaptive, save_everystep=false,
        callback=cb_baseline, tstops=saveat)

    error_u = saved_values_baseline.saveval[end]
end

xmin = 0.0
xmax = 1.0
tspan = (0.0, 1.0)

tol = 1.0e-13

# val_N = round.(Int, 2 .^ range(4, 5, length=2))
# val_N = round.(Int, 2 .^ range(4, 8, length=7)) |> evenodd_values
val_N = round.(Int, 2 .^ range(3, 4.5, length=7)) |> evenodd_values
# val_N = 2 .^ (2:7) .+ 1
val_error = Float64[]
for N in val_N
#     D = fourier_derivative_operator(xmin, xmax, N)
#     p = 6; D = periodic_derivative_operator(1, p, xmin, xmax, N+1)
#     p = 4; D = periodic_derivative_operator(Holoborodko2008(), 1, p, xmin, xmax, N+1)
#     D2 = D^2
#     D2 = periodic_derivative_operator(2, p, xmin, xmax, N+1)
#     D2 = periodic_derivative_operator(Holoborodko2008(), 1, p, xmin, xmax, N+1)^2
    p = 3; mesh = UniformPeriodicMesh1D(xmin, xmax, N)
    D = couple_continuosly(legendre_derivative_operator(-1., 1., p+1), mesh)
    D2 = sparse(couple_continuosly(legendre_second_derivative_operator(-1., 1., p+1), mesh))
#     D2 = sparse(D)^2
    p = 6; Dop = legendre_derivative_operator(-1., 1., p+1); mesh = UniformPeriodicMesh1D(xmin, xmax, N)
    D = couple_discontinuosly(Dop, mesh); D₊ = couple_discontinuosly(Dop, mesh, Val(:plus)); D₋ = couple_discontinuosly(Dop, mesh, Val(:minus))
    D2 = sparse(D₊) * sparse(D₋)
#     D2 = sparse(D₋) * sparse(D₊)
#     D2 = sparse(D)^2

    error_u = errors_fw_periodic_manufactured(usol, D, D2, tspan, Tsit5(), tol, 1/N, true)
    push!(val_error, error_u)
end

fig, ax = plt.subplots(1, 1)
ax.set_prop_cycle(marker_cycler)
plt.plot(val_N, val_error, label=L"$\| u - u_{\mathrm{ana}} \|$")
c0, c1 = linear_regression(log.(val_N), log.(val_error))
plt.plot(val_N, exp(c0) .* val_N.^c1, marker="", linestyle=":", color="gray", label=@sprintf("Order %.2f", -c1))
plt.xscale("log", basex=2)
plt.yscale("symlog", linthreshy=1.0e-12)
plt.legend(loc="center left", bbox_to_anchor=(1.0, 0.5))
ax.locator_params(axis="y", numticks=9)
```

## Convergence study: Plots FD

```julia
function do_stuff(p, val_N)
    val_error = Float64[]
    for N in val_N
        D = periodic_derivative_operator(1, p, xmin, xmax, N+1)
#         D2 = D^2
        D2 = periodic_derivative_operator(2, p, xmin, xmax, N+1)
        error_u = errors_fw_periodic_manufactured(usol, D, D2, tspan, Tsit5(), tol, 1/N, true)
        push!(val_error, error_u)
    end
    Dict("val_p$(p)_N" => val_N, "val_p$(p)_error" => val_error)
end

filename = "fw_periodic_manufactured_convergence_FD_narrow"
even_odd_values = evenodd_values
res = Dict{String,Any}()

p = 2
val_N = round.(Int, 2 .^ range(4, 10, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 4
val_N = round.(Int, 2 .^ range(4, 10, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 6
val_N = round.(Int, 2 .^ range(4, 8.5, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 8
val_N = round.(Int, 2 .^ range(4, 7.5, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

open("../data/" * filename * ".json", "w") do io
    JSON.print(io, res, 2)
end
```

```julia
fig, ax = plt.subplots(1, 1)
ax.set_prop_cycle(marker_cycler)

filename = "fw_periodic_manufactured_convergence_FD_narrow"
res = JSON.parsefile("../data/" * filename * ".json")

let p = 2
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("\$ p = %d \$, EOC %.2f", p, -c1), (val_N[4], 2.0*val_error_u[4]), color="gray")
end

let p = 4
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("\$ p = %d \$, EOC %.2f", p, -c1), (val_N[6], 2.0*val_error_u[6]), color="gray")
end

let p = 6
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("\$ p = %d \$, EOC %.2f", p, -c1), (0.6*val_N[end], 0.1*val_error_u[end]), color="gray")
end

let p = 8
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("\$ p = %d \$, EOC %.2f", p, -c1), (0.15*val_N[end], val_error_u[end]), color="gray")
end

ax.set_ylim(1.0e-13, 1.5e0)
ax.set_xscale("log", basex=2)
ax.set_yscale("symlog", linthreshy=1.0e-13)
ax.set_xlabel(L"N")
ax.set_ylabel(L"\| u - u_{\mathrm{ana}} \|")
fig.savefig("../figures/" * filename * ".pdf", bbox_inches="tight")
```

```julia
function do_stuff(p, val_N)
    val_error = Float64[]
    for N in val_N
        D = periodic_derivative_operator(1, p, xmin, xmax, N+1)
        D2 = D^2
#         D2 = periodic_derivative_operator(2, p, xmin, xmax, N+1)
        error_u = errors_fw_periodic_manufactured(usol, D, D2, tspan, Tsit5(), tol, 1/N, true)
        push!(val_error, error_u)
    end
    Dict("val_p$(p)_N" => val_N, "val_p$(p)_error" => val_error)
end

filename = "fw_periodic_manufactured_convergence_FD_wide"
even_odd_values = evenodd_values
res = Dict{String,Any}()

p = 2
val_N = round.(Int, 2 .^ range(4, 10, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 4
val_N = round.(Int, 2 .^ range(4, 10, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 6
val_N = round.(Int, 2 .^ range(4, 8.5, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 8
val_N = round.(Int, 2 .^ range(4, 7.5, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

open("../data/" * filename * ".json", "w") do io
    JSON.print(io, res, 2)
end
```

```julia
fig, ax = plt.subplots(1, 1)
ax.set_prop_cycle(marker_cycler)

filename = "fw_periodic_manufactured_convergence_FD_wide"
res = JSON.parsefile("../data/" * filename * ".json")

let p = 2
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("\$ p = %d \$, EOC %.2f", p, -c1), (val_N[4], 2.0*val_error_u[4]), color="gray")
end

let p = 4
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("\$ p = %d \$, EOC %.2f", p, -c1), (val_N[6], 2.0*val_error_u[6]), color="gray")
end

let p = 6
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("\$ p = %d \$, EOC %.2f", p, -c1), (0.6*val_N[end], 0.1*val_error_u[end]), color="gray")
end

let p = 8
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("\$ p = %d \$, EOC %.2f", p, -c1), (0.15*val_N[end], val_error_u[end]), color="gray")
end

ax.set_ylim(1.0e-13, 1.5e0)
ax.set_xscale("log", basex=2)
ax.set_yscale("symlog", linthreshy=1.0e-13)
ax.set_xlabel(L"N")
ax.set_ylabel(L"\| u - u_{\mathrm{ana}} \|")
fig.savefig("../figures/" * filename * ".pdf", bbox_inches="tight")
```

## Convergence study: Plots CG

```julia
function do_stuff(p, val_N)
    val_error = Float64[]
    for N in val_N
        mesh = UniformPeriodicMesh1D(xmin, xmax, N)
        D = couple_continuosly(legendre_derivative_operator(-1., 1., p+1), mesh)
        D2 = sparse(D)^2
#         D2 = sparse(couple_continuosly(legendre_second_derivative_operator(-1., 1., p+1), mesh))
        error_u = errors_fw_periodic_manufactured(usol, D, D2, tspan, Tsit5(), tol, 1/N, true)
        push!(val_error, error_u)
    end
    Dict("val_p$(p)_N" => val_N, "val_p$(p)_error" => val_error)
end

filename = "fw_periodic_manufactured_convergence_CG_D1c2"
even_odd_values = evenodd_values
res = Dict{String,Any}()

p = 1
val_N = round.(Int, 2 .^ range(6, 12, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 2
val_N = round.(Int, 2 .^ range(5, 10, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 3
val_N = round.(Int, 2 .^ range(4, 8, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 4
val_N = round.(Int, 2 .^ range(4, 7, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 5
val_N = round.(Int, 2 .^ range(3, 6, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 6
val_N = round.(Int, 2 .^ range(3, 5.25, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

open("../data/" * filename * ".json", "w") do io
    JSON.print(io, res, 2)
end
```

```julia
fig, ax = plt.subplots(1, 1)
ax.set_prop_cycle(marker_cycler)

filename = "fw_periodic_manufactured_convergence_CG_D1c2"
res = JSON.parsefile("../data/" * filename * ".json")

let p = 1
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[2], 2.0*val_error_u[2]), color="gray")
end

let p = 2
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[end], 0.1*val_error_u[end]), color="gray")
end

let p = 3
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[end], 1.5*val_error_u[end]), color="gray")
end

let p = 4
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (1.2*val_N[end], 0.5*val_error_u[end]), color="gray")
end

let p = 5
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (1.2*val_N[end], 0.2*val_error_u[end]), color="gray")
end

let p = 6
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (0.2*val_N[end], 0.1*val_error_u[end]), color="gray")
end

ax.set_xlim(2^2, 2^13-1)
ax.set_ylim(1.0e-13, 1.5e0)
ax.set_xscale("log", basex=2)
ax.set_yscale("symlog", linthreshy=1.0e-13)
ax.set_xlabel(L"N")
ax.set_ylabel(L"\| u - u_{\mathrm{ana}} \|")
fig.savefig("../figures/" * filename * ".pdf", bbox_inches="tight")
```

```julia
function do_stuff(p, val_N)
    val_error = Float64[]
    for N in val_N
        mesh = UniformPeriodicMesh1D(xmin, xmax, N)
        D = couple_continuosly(legendre_derivative_operator(-1., 1., p+1), mesh)
#         D2 = sparse(D)^2
        D2 = sparse(couple_continuosly(legendre_second_derivative_operator(-1., 1., p+1), mesh))
        error_u = errors_fw_periodic_manufactured(usol, D, D2, tspan, Tsit5(), tol, 1/N, true)
        push!(val_error, error_u)
    end
    Dict("val_p$(p)_N" => val_N, "val_p$(p)_error" => val_error)
end

filename = "fw_periodic_manufactured_convergence_CG_narrow"
even_odd_values = evenodd_values
res = Dict{String,Any}()

p = 1
val_N = round.(Int, 2 .^ range(6, 12, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 2
val_N = round.(Int, 2 .^ range(5, 10, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 3
val_N = round.(Int, 2 .^ range(4, 8, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 4
val_N = round.(Int, 2 .^ range(4, 7, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 5
val_N = round.(Int, 2 .^ range(3, 6, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 6
val_N = round.(Int, 2 .^ range(3, 5.25, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

open("../data/" * filename * ".json", "w") do io
    JSON.print(io, res, 2)
end
```

```julia
fig, ax = plt.subplots(1, 1)
ax.set_prop_cycle(marker_cycler)

filename = "fw_periodic_manufactured_convergence_CG_narrow"
res = JSON.parsefile("../data/" * filename * ".json")

let p = 1
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[2], 2.0*val_error_u[2]), color="gray")
end

let p = 2
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[end], 0.1*val_error_u[end]), color="gray")
end

let p = 3
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[end], 1.5*val_error_u[end]), color="gray")
end

let p = 4
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (1.2*val_N[end], 0.5*val_error_u[end]), color="gray")
end

let p = 5
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (1.2*val_N[end], 0.2*val_error_u[end]), color="gray")
end

let p = 6
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (0.2*val_N[end], 0.1*val_error_u[end]), color="gray")
end

ax.set_xlim(2^2, 2^13-1)
ax.set_ylim(1.0e-13, 1.5e0)
ax.set_xscale("log", basex=2)
ax.set_yscale("symlog", linthreshy=1.0e-13)
ax.set_xlabel(L"N")
ax.set_ylabel(L"\| u - u_{\mathrm{ana}} \|")
fig.savefig("../figures/" * filename * ".pdf", bbox_inches="tight")
```

## Convergence study: Plots DG

```julia
function do_stuff(p, val_N)
    val_error = Float64[]
    for N in val_N
        Dop = legendre_derivative_operator(-1., 1., p+1); mesh = UniformPeriodicMesh1D(xmin, xmax, N)
        D = couple_discontinuosly(Dop, mesh); D₊ = couple_discontinuosly(Dop, mesh, Val(:plus)); D₋ = couple_discontinuosly(Dop, mesh, Val(:minus))
        D2 = sparse(D)^2
#         D2 = sparse(D₊) * sparse(D₋)
#         D2 = sparse(D₋) * sparse(D₊)
        error_u = errors_fw_periodic_manufactured(usol, D, D2, tspan, Tsit5(), tol, 1/N, true)
        push!(val_error, error_u)
    end
    Dict("val_p$(p)_N" => val_N, "val_p$(p)_error" => val_error)
end

filename = "fw_periodic_manufactured_convergence_DG_D1c2"
even_odd_values = evenodd_values
res = Dict{String,Any}()

p = 1
val_N = round.(Int, 2 .^ range(6, 12, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 2
val_N = round.(Int, 2 .^ range(5, 10, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 3
val_N = round.(Int, 2 .^ range(4, 8, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 4
val_N = round.(Int, 2 .^ range(4, 7.15, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 5
val_N = round.(Int, 2 .^ range(3, 5.75, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 6
val_N = round.(Int, 2 .^ range(3, 5.25, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

open("../data/" * filename * ".json", "w") do io
    JSON.print(io, res, 2)
end
```

```julia
fig, ax = plt.subplots(1, 1)
ax.set_prop_cycle(marker_cycler)

filename = "fw_periodic_manufactured_convergence_DG_D1c2"
res = JSON.parsefile("../data/" * filename * ".json")

let p = 1
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[2], 2.0*val_error_u[2]), color="gray")
end

let p = 2
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[end], 2.0*val_error_u[end]), color="gray")
end

let p = 3
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (0.4*val_N[end], 0.1*val_error_u[end]), color="gray")
end

let p = 4
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (1.2*val_N[end], 0.2*val_error_u[end]), color="gray")
end

let p = 5
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (0.6*val_N[1], 1.5*val_error_u[1]), color="gray")
end

let p = 6
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (0.5*val_N[end], 0.1*val_error_u[end]), color="gray")
end

ax.set_xlim(2^2, 2^13-1)
ax.set_ylim(1.0e-13, 1.5e0)
ax.set_xscale("log", basex=2)
ax.set_yscale("symlog", linthreshy=1.0e-13)
ax.set_xlabel(L"N")
ax.set_ylabel(L"\| u - u_{\mathrm{ana}} \|")
fig.savefig("../figures/" * filename * ".pdf", bbox_inches="tight")
```

```julia
function do_stuff(p, val_N)
    val_error = Float64[]
    for N in val_N
        Dop = legendre_derivative_operator(-1., 1., p+1); mesh = UniformPeriodicMesh1D(xmin, xmax, N)
        D = couple_discontinuosly(Dop, mesh); D₊ = couple_discontinuosly(Dop, mesh, Val(:plus)); D₋ = couple_discontinuosly(Dop, mesh, Val(:minus))
#         D2 = sparse(D)^2
        D2 = sparse(D₊) * sparse(D₋)
#         D2 = sparse(D₋) * sparse(D₊)
        error_u = errors_fw_periodic_manufactured(usol, D, D2, tspan, Tsit5(), tol, 1/N, true)
        push!(val_error, error_u)
    end
    Dict("val_p$(p)_N" => val_N, "val_p$(p)_error" => val_error)
end

filename = "fw_periodic_manufactured_convergence_DG_D1pD1m"
even_odd_values = evenodd_values
res = Dict{String,Any}()

p = 1
val_N = round.(Int, 2 .^ range(6, 12, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 2
val_N = round.(Int, 2 .^ range(5, 10, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 3
val_N = round.(Int, 2 .^ range(4, 8, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 4
val_N = round.(Int, 2 .^ range(4, 7.15, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 5
val_N = round.(Int, 2 .^ range(3, 5.75, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

p = 6
val_N = round.(Int, 2 .^ range(3, 5.25, length=7)) |> even_odd_values
res = merge(res, do_stuff(p, val_N))

open("../data/" * filename * ".json", "w") do io
    JSON.print(io, res, 2)
end
```

```julia
fig, ax = plt.subplots(1, 1)
ax.set_prop_cycle(marker_cycler)

filename = "fw_periodic_manufactured_convergence_DG_D1pD1m"
res = JSON.parsefile("../data/" * filename * ".json")

let p = 1
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[2], 2.0*val_error_u[2]), color="gray")
end

let p = 2
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (val_N[end], 2.0*val_error_u[end]), color="gray")
end

let p = 3
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (0.4*val_N[end], 0.1*val_error_u[end]), color="gray")
end

let p = 4
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (1.2*val_N[end], 0.2*val_error_u[end]), color="gray")
end

let p = 5
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (0.6*val_N[1], 1.5*val_error_u[1]), color="gray")
end

let p = 6
    val_N, val_error_u = res["val_p$(p)_N"], res["val_p$(p)_error"]
    ax.plot(val_N, val_error_u, label="\$ p = $p \$")
    c0, c1 = linear_regression(log.(val_N), log.(val_error_u))
    plt.plot(val_N, exp(c0).*val_N.^c1, marker="", linestyle=":", color="gray")
    plt.annotate(@sprintf("EOC %.2f", -c1), (0.5*val_N[end], 0.1*val_error_u[end]), color="gray")
end

ax.set_xlim(2^2, 2^13-1)
ax.set_ylim(1.0e-13, 1.5e0)
ax.set_xscale("log", basex=2)
ax.set_yscale("symlog", linthreshy=1.0e-13)
ax.set_xlabel(L"N")
ax.set_ylabel(L"\| u - u_{\mathrm{ana}} \|")
fig.savefig("../figures/" * filename * ".pdf", bbox_inches="tight")
```

```julia

```


$\newcommand{\pr}{\textrm{Pr}}$
$\newcommand{\l}{\left}$
$\newcommand{\r}{\right}$
$\newcommand\given[1][]{\:#1\vert\:}$
$\newcommand{\var}{\textrm{Var}}$
$\newcommand{\mc}{\mathcal}$
$\newcommand{\lp}{\left(}$
$\newcommand{\rp}{\right)}$
$\newcommand{\lb}{\left\{}$
$\newcommand{\rb}{\right\}}$
$\newcommand{\iid}{\textrm{i.i.d. }}$

# 2.1 Belief functions and probabilities

Probabilities are a way to numerically express rational beliefs.
This first section shows that probabilities satisfy some general
features that we would expect a measure of "belief" to have, so
it seems reasonable to use probabilities to represent our belief
in something.

# 2.2 Events, partitions, and Bayes' rule

**Bayes' rule**:
\begin{align}
\pr\left(H_j \given E\right) &= \frac{\pr\l(E \given H_j\r)\pr\l(H_j\r)}{\pr\l(E\r)} \\
&= \frac{\pr\l(E \given H_k\r)\pr\l(H_j\r)}{\sum_{k=1}^K \pr\l(E \given H_k\r)\pr\l(H_k\r)}
\end{align}

**Bayes factors**: $\l\{H_1, \cdots, H_k\r\}$ often refer to disjoint hypotheses and $E$ refers
to data. To compare hypotheses post-experimentally, we often calculate the ratio:
\begin{align}
\frac{\pr\l(H_i \given E\r)}{\pr\l(H_j \given E\r)} &=
\frac{\pr\l(E \given H_i\r)}{\pr\l(E \given H_j\r)} \times \frac{\pr\l(H_i\r)}{\pr\l(H_j\r)} \\
&= \textrm{"Bayes factor"}\times \textrm{"prior beliefs"}
\end{align}

This quantity reminds us that Bayes' rule does not determine what our beliefs should be after we
see data, it only tells us how they should change.

# 2.4 Random variables

A random variable is an unknown numerical quantity about which we make probability statements.

## 2.4.3 Descriptions of distributions

### Mean, mode, median

The mean is a good quantity to look at because

1. The mean is a scaled value of the total of $\l\{Y_1, \cdots, Y_n\r\}$, and the total
is often a quantity of interest.
2. If you were forced to guess the value of $Y$, guessing the mean would minimize your error
if it was measured as $\l(Y - y_{guess}\r)^2$.
3. In some simple models, the mean contains all of the information about the population that
can be obtained from the data.

### Variance
 * The variance is the average squared distance that a sample value $Y$ will be from
the population mean $E\l[Y\r]$.
 * The standard deviation is the square root of the variance
and is on the same scale as the mean.

## 2.5 Joint distributions

### Discrete distributions

The **joint pdf** or **joint density** of discrete random variables $Y_1$ and $Y_2$ is defined as
$$p\l(y_1, y_2\r) = \pr\l(y_1 \cap y_2 \r)$$
for $y_1 \in \mc{Y}_1$, $y_2 \in \mc{Y}_2$.

The **marginal density** of $Y_1$ can be computed from the joint density of $Y_1$ and $Y_2$:
\begin{align}
p\l(y_1\r) &= \sum_{y_2 \in \mc{Y}_2}p\l(y_1, y_2\r)
\end{align}

The **conditional density** of $Y_2$ given $\l\{Y_1 = y_1\r\}$ can be computed from the
joint density and the marginal density:
\begin{align}
p\l(y_2 \given y_1\r) &= \frac{p\l(y_1, y_2\r)}{p\l(y_1\r)}
\end{align}

### Continuous joint distributions

If $Y_1$ and $Y_2$ are continuous, the marginal density of $Y_1$ is given by
\begin{align}
p\l(y_1\r) = \int_{-\infty}^\infty p\l(y_1,y_2\r)dy_2
\end{align}
and the conditional density is given by
\begin{align}
p\l(y_2 \given y_1\r) = p\l(y_1, y_2\r) / p\l(y_1\r)
\end{align}

### Mixed continuous and discrete variables

Let $Y_1$ be discrete and $Y_2$ be continuous. Suppose we know $p\left(y_1\right)$ and
$p\left(y_2 \given y_1\right)$

The joint density of $Y_1$ and $Y_2$ is then
$$p\l(y_1, y_2\r) = p\l(y_1\r) \times p\l(y_2 \given y_1\r)$$

and has the property that
$$\pr\l(Y_1 \in A, Y_2 \in B\r) = \int_{y_2 \in B} \left\{\sum_{y_1 \in A} p_{Y_1Y_2}\l(y_1, y_2\r)\r\}dy_2$$

In other words, we can summation and integration to calculate the joint density.

### Bayes rule and parameter estimation

Let $\theta$ be a continuous parameter we want to estimate and let $Y$ be a discrete
data measurement. Having observed $\l\{Y = y\r\}$, we need to compute our updated beliefs about $\theta$:
$$p\l(\theta | y\r) = \frac{p\l(\theta, y\r)}{p\lp y\rp} = \frac{p\lp \theta \rp p\lp y \given \theta \rp}{p\lp y \rp}$$

**This conditional density is called the posterior density of $\theta$.** If $\theta_a$ and $\theta_b$
are two estimates of $\theta$, the posterior probability (density) of $\theta_a$ relative to $\theta_b$,
conditional on $Y=y$, is
\begin{align}
\frac{p\lp \theta_a \given y \rp}{p \lp \theta_b \given y \rp} &=
\frac{p\lp \theta_a \rp p\lp y | \theta_a \rp}{p\lp \theta_b \rp p\lp y | \theta_b \rp}
\end{align}

To evaluate the **relative** posterior odds of $\theta_a$ and $\theta_b$, we do not need to evaluate
the marginal density $p\lp y \rp$. We see that
$$p\lp \theta | y \rp \propto p\lp \theta \rp p\lp y \given \theta \rp$$

The constant of proportionality is $1 / p\lp y \rp$ which we *could* calculate using
$$p\lp y \rp = \int_\Theta p\lp y, \theta \rp d\theta = \int_\Theta p\lp y \given \theta \rp p\lp \theta \rp d\theta$$

Later we will see that the numerator is more important than this denominator.

# 2.7 Exchangeability

The observations in a data set $Y_1, \cdots, Y_N$ are exchangeable if the subscript labels convey no information about the outcomes. In other words, you could repeatedly permute the data and calculate the joint density and it wouldn't change.

Therom: If $Y_1, \cdots, Y_N$ are conditionally i.i.d. given $\theta$ and $\theta \sim p\lp \theta \rp$,
then $Y_1, \cdots, Y_N$ are exchangeable.

# 2.8 de Finetti's thereom

We know that if $Y_1, \cdots, Y_n \given \theta$ are i.i.d. and $\theta \sim p\lp \theta \rp$, then
$Y_1, \cdots, Y_N$ are exchangeable. de Finetti's theorem says that this statement is if and only if:

$$Y_1, \cdots, Y_n \given \theta \textrm{ are i.i.d. and }\theta \sim p\lp \theta \rp \iff
Y_1, \cdots, Y_N \textrm{ are exchangeable for all } n$$

The question is when are $Y_1, \cdots, Y_n$ exchangeable for all $n$? For this to be true,
we need both exchangeability and repeatability. Exchangeability is true when the labels
have no meaning. Repeatability is true when

* $Y_1, \cdots, Y_n$ are outcomes of a repeatable experiment
* $Y_1, \cdots, Y_n$ are sampled from a finite population *with* replacement
* $Y_1, \cdots, Y_n$ are sampled from an infinite population *without* replacement

If $Y_1, \cdots, Y_n$ are exchangeable and sampled from a finite population *without* replacement
of size $N >> n$, then they can be modeled as approximately conditionally i.i.d.

```python

```

readings/A First Course in Bayesian Statistical Methods/2. Belief, probability, and exchangeability (short).ipynb


# Markov Chain Monte Carlo (MCMC)
> Recipes for MCMC

```
#hide
from IPython.display import display, Math
```

## Overview
### Goal
- Sample from some distribution $p(x) = \frac{f(x)}{NC}$, but we don't know the equation for $p(x)$, only $f(x)$.

### Why not use Accept-Reject?
- In Accept-Reject, we:
  - Choose some function $g(x)$ that is close to the distribution we want to sample $p(x)$
  - Sample from $g(x)$ and accept with probability $\frac{f(x)}{Mg(x)}$
  - Intuitively, the ratio $\frac{f(x)}{g(x)}$ tells us that we should prefer samples with high densities (big $f(x)$, likely to come from $p(x)$) or are rare candidates (low $g(x)$)
- For high dimensional data, it is hard to efficiently sample from $Mg(x) > f(x)$ because $M$ tends to be very large.
  - Large M results in low acceptance probabilties (because $\frac{1}{M} \propto \text{acceptance}$)
  - This means we need to sample a lot to get accepted samples.

### Aside (math)

```
#collapse
display(Math(r'''
\begin{align}
\\NC = \int_{-\infty}^{\infty}f(x)dx
\\
\\\text{Bayes Theorem: } P(S|A) &= \frac{P(A|S)D(S)}{P(A)}
\\&=\frac{\frac{f(x)}{Mg(x)} \times g(x)}{P(A)}
\\
\\\text{rearrange: } P(A) &= \int_{-\infty}^{\infty}g(x)\frac{f(x)}{Mg(x)}dx
\\&=\frac{1}{M}\int_{-\infty}^{\infty}f(x)dx
\\&=\frac{NC}{M}
\\\text{rearrange: } P(S|A) &= \frac{f(x)/M}{NC/M}
\\&=\frac{f(x)}{NC}
\\&=p(x)
\end{align}
'''))
```

$\displaystyle
\begin{align}
\\NC = \int_{-\infty}^{\infty}f(x)dx
\\
\\\text{Bayes Theorem: } P(S|A) &= \frac{P(A|S)D(S)}{P(A)}
\\&=\frac{\frac{f(x)}{Mg(x)} \times g(x)}{P(A)}
\\
\\\text{rearrange: } P(A) &= \int_{-\infty}^{\infty}g(x)\frac{f(x)}{Mg(x)}dx
\\&=\frac{1}{M}\int_{-\infty}^{\infty}f(x)dx
\\&=\frac{NC}{M}
\\\text{rearrange: } P(S|A) &= \frac{f(x)/M}{NC/M}
\\&=\frac{f(x)}{NC}
\\&=p(x)
\end{align}
$

### Solution
- Build a markov chain that has a stationary distribution that represents $p(x)$, then sample from it.
  - Each draw from the distribution is dependent on the previous draw (Markov Chain)
  - Simulate draws from the markov chain (Monte Carlo)
  - Requires drawing from the markov chain for some "burn-in" period before converging to $p(x)$.

[](https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoiZ3JhcGggTFJcblxuc3ViZ3JhcGggYnVybi1pblxuICAgIFhfMCAtLT4gWF8xIC0tPiBYXy4uLlxuZW5kXG5cbnN1YmdyYXBoIFwicCh4KVwiXG4gICAgWF8uLi4gLS0-IFhfQiAtLT4gWF9CKzEgLS0-IFhfQisuLi5cbmVuZCIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ)

- To guarantee that the markov chain converges to $p(x)$, you need to satisfy the detailed balance condition:

1. $y \rarr x$.  $\forall{x,y} p(x)T(y|x) = p(y)T(x|y)$  --  The probability of going $x \rarr y$ is the same as the probability of going

2. $\sum{p(x)T(y|x)} = \sum{p(y)T(x|y)}$  --  Therefore the sums of those probabilities is the same

3. $pT = p$  --  Therefore we reach steady state

## Metropolis-Hastings

The transition probability is equal to your likelihood of going from A to B times your likelihood of going from B to A.
$$
\\ \forall{a,b}
\\ p(a)T(a\rarr b) = p(b)T(b\rarr a)
\\ \frac{f(a)}{NC}g(b|a)A(a\rarr b) = \frac{f(b)}{NC}g(a|b)A(b\rarr a)
\\ \frac{A(a\rarr b)}{A(b\rarr a)} = \frac{f(b)}{f(a)} \times \frac{g(a|b)}{g(b|a)}
$$

Here, we note that $r_f = \frac{f(b)}{f(a)}$ and $r_g = \frac{g(a|b)}{g(b|a)}$

For $r_fr_g <1$, $A(a\rarr b) = r_fr_g$, and $A(b\rarr a) = 1$
For $r_fr_g \ge1$, $A(a\rarr b) = 1$, and $A(b\rarr a) = \frac{1}{r_fr_g}$

Therefore, $A(a\rarr b) = \min{(1, r_fr_g)}$

At each step, we will sample some function $g(x_{t+1}|x_t)$.  This function can be any distribution, e.g. $\mathcal{N}(x_t, \sigma^2)$

## Gibbs Sampling
- Useful for multidimensional sampling


# Chapter 2 - Using Normals Instead of Weights

We've seen how we can create a simple SoftMax distribtion by specifying each class's parameters (weights), but we also saw that the log-odds between two classes generates a hyperplane when one class equals the other. If we wanted to chop up the state space, couldn't we simply specify those hyperplanes instead? This could let us use some simple pre-defined templates or groundings to subdivide the space, so we don't need to specify all weights for each class, which might not be obvious.

## Specification from Hyperplanes

First, we need to investigate the relationships between the weights and the hyperplanes. Take our first attempt at the intercardinal bearing Pac-Man problem: we have four classes, and each class shares a boundary with each other class over an equiprobable region. In the case of *NE* and *SE*, for instance, we have a line dividing north from south as our boundary. In the case of *NE* and *SW*, we have a diagonal line through the cardinal center as our boundary. Let's see if we can calculate these from the weights.

We're going to walk through two examples: first, we take the weights and biases we used in our previous examples and turn them into hyperplanes; then, we specify hyperplanes and turn them into weights and biases. This will show a technique for specifying SoftMax distributions from hyperplanes, which we can build off of in the future.

### Example: Pac-Man weights to normals
Recall the shifted Pac-Man example:

```python
import numpy as np
from cops_and_robots.robo_tools.fusion.softmax import SoftMax
%matplotlib inline

labels = ['SW', 'NW', 'SE',' NE']
weights = np.array([[-1, -1],
                    [-1, 1],
                    [1, -1],
                    [1, 1],
                   ])

biases = np.array([1, -5, 5, -1,])
pacman = SoftMax(weights, biases, class_labels=labels)
pacman.plot(title='Shifted Pac-Man Bearing Model')
```

Remember the definition of our class boundaries:

\begin{align}
0 &= (\mathbf{w}_i - \mathbf{w}_j)^T\mathbf{x} + (b_i - b_j) \\
  &= (w_{i,x_1} - w_{j,x_1})x_1 + (w_{i,x_2} - w_{j,x_2})x_2 + \dots + (w_{i,x_n} - w_{j,x_n})x_n + (b_i - b_j)
\end{align}

From the equation of a plane, recall that the **normal vector** can be taken from the coefficients for the non-constant terms. Since the normal vector to a bounding hyperplane completely describes that hyperplane, we'll use the two terms interchangeably. In $\mathbb{R}^2$, each hyperplane is of the form $(w_{i,x} - w_{j,x})x + (w_{i,y} - w_{j,y})y + (b_i - b_j) = 0$.

We can define a normal vector for each class boundary:

$$
\begin{align}
\mathbf{n}_{SE,SW} &= (\mathbf{w}_{SW} - \mathbf{w}_{SE}) \\
&=\begin{bmatrix}-1 & -1\end{bmatrix}^T - \begin{bmatrix}1 & -1\end{bmatrix}^T = \begin{bmatrix}-2 & 0\end{bmatrix}^T \\
\mathbf{n}_{SW,NW} &= \begin{bmatrix}-1 & 1\end{bmatrix}^T - \begin{bmatrix}-1 & -1\end{bmatrix}^T = \begin{bmatrix}0 & 2\end{bmatrix}^T \\
\mathbf{n}_{NE,SE} &= \begin{bmatrix}1 & -1\end{bmatrix}^T - \begin{bmatrix}1 & 1\end{bmatrix}^T = \begin{bmatrix}0 & -2\end{bmatrix}^T \\
\mathbf{n}_{NW,NE} &= \begin{bmatrix}1 & 1\end{bmatrix}^T - \begin{bmatrix}-1 & 1\end{bmatrix}^T = \begin{bmatrix}2 & 0\end{bmatrix}^T \\
\end{align}
$$

But these aren't the only class boundaries -- diagonally positioned classes also have normals:

$$
\begin{align}
\mathbf{n}_{NW,SE} &= \begin{bmatrix}1 & -1\end{bmatrix}^T - \begin{bmatrix}-1 & 1\end{bmatrix}^T = \begin{bmatrix}2 & -2\end{bmatrix} \\
\mathbf{n}_{NE,SW} &= \begin{bmatrix}-1 & -1\end{bmatrix}^T - \begin{bmatrix}1 & 1\end{bmatrix}^T = \begin{bmatrix}-2 & -2\end{bmatrix} \\
\end{align}
$$

And we can define six more boundaries, but they're all mirror images of the ones we already have (i.e. $\mathbf{n}_{NW,SE} = - \mathbf{n}_{SE,NW}$). We'll ignore those for simplicity, but remember that we have 12 total normal vectors (and bias terms).

Let's figure out the **offset term** $d$ for each normal vector:

$$
\begin{align}
d_{SE,SW} &= b_{SW} - b_{SE} = 1 - 5 = -4 \\
d_{SW,NW} &=  -5 - 1 = -6\\
d_{NE,SE} &= 5 - (-1) = 6\\
d_{NW,NE} &= -1 - (-5) = 4\\
d_{NW,SE} &= 5 - (-5) = 10\\
d_{NE,SW} &= 1 - (-1) = 2\\
\end{align}
$$

We can represent each normal line in slope-intercept form. In [standard form](http://en.wikipedia.org/wiki/Linear_equation#General_.28or_standard.29_form), we have

$$
0 = \mathbf{n}\mathbf{x} + d
= n_xx + n_yy + d\\
$$

Which, in [slope-intercept form](http://en.wikipedia.org/wiki/Linear_equation#Slope.E2.80.93intercept_form), is

$$
y = \frac{-n_x}{n_y}x + \frac{-d}{n_y} = mx + d'
$$

So, for each class:

$$
\begin{align}
y_{SE,SW} &= m_{SE,SW}x + d'_{SE,SW} = \frac{-2}{0}x + \frac{4}{0} \\
y_{SW,NW} &= \frac{0}{2}x + \frac{6}{2} = 3 \\
y_{NE,SE} &= \frac{0}{-2}x + \frac{-6}{-2} = 3 \\
y_{NW,NE} &= \frac{-2}{0}x + \frac{-4}{0}\\
y_{NW,SE} &= \frac{-2}{-2}x + \frac{-10}{-2} = x + 5 \\
y_{NE,SW} &= \frac{2}{-2}x + \frac{-2}{-2} = -x + 1 \\
\end{align}
$$

Note that, while $y_{SE,SW}$ and $y_{NW,NE}$ are undefined, we recognize them as vertical lines (as can be seen in standard form).

So we have a map of the normal vectors:

```python
import numpy as np
from cops_and_robots.robo_tools.fusion.softmax import SoftMax
import matplotlib.pyplot as plt
%matplotlib inline

x = np.arange(-5,5,0.1)
y_SESW = float('inf') * x + 2  # Vertical Line
y_SWNW = 0 * x + 3  # Horizontal Line
y_NESE = 0 * x + 3  # Horizontal Line
y_NWNE = float('inf') * x + 2  # Vertical Line
y_NWSE = x + 5
y_NESW = -x + 1

plt.axvline(x = -2, color='pink', ls='-', label="SESW", lw=3)
plt.plot(x, y_SWNW, 'y-', label="SWNW", lw=3)
plt.plot(x, y_NESE, 'g--', label="NESE", lw=3)
plt.axvline(x = -2, color='blue', ls='--' , label="NWNE", lw=3)
plt.plot(x, y_NWSE, 'k-', label="NWSE", lw=3)
plt.plot(x, y_NESW, 'r-', label="NESW", lw=3)

plt.grid()
plt.xlabel('x [m]')
plt.ylabel('y [m]')
plt.xlim([-5, 5])
plt.ylim([-5, 5])
plt.title('Boundaries Between Intercardinal Spaces')
plt.legend(loc='lower center', bbox_to_anchor=(-0.2, -0.275, 1.4, -0.175),
            mode='expand', ncol=6)
```

Using this visualization as a guide, we notice that the sum of all normal vectors is zero in each dimension (that is, they are symmetric about $x$ and $y$ in the shifted coordinate frame). Will this always be true? (Hint: we 'noticed' it because it's *important*.)

First, let's generalize to classes $i$,$j$, and $k$. Even non-critical classes share equiprobable zones (since the distribution for each class spans the entire state space), so each class will share some boundary with all other classes. This allows us to constrain the equations of the normal vectors:

$$
\begin{align}
\mathbf{n}_{i,j} &= \mathbf{w_j} - \mathbf{w_i} \\
\mathbf{n}_{j,k} &= \mathbf{w_k} - \mathbf{w_j} \\
\mathbf{n}_{k,i} &= \mathbf{w_i} - \mathbf{w_k} = -\mathbf{n}_{i,j} - \mathbf{n}_{j,k}\\
\end{align}
$$

From this, we can construct a transformation matrix $A$, where:

$$
\mathbf{n} = \mathbf{A}\mathbf{w}
$$

Where $a_{i,j} \in \{-1,0,1\}, \; \forall \: a_{i,j} \in A$. In defining our transformation matrix, we can translate between a weight specification and a normal vector specification in generating SoftMax models.

How do we find $A$? First, we know that $A$ is not full rank, because we can  reconstruct $\mathbf{n}_{k,i}$ from a linear combination of $\mathbf{n}_{i,j}$ and  $\mathbf{n}_{j,k}$. This means we can get an infinite number of mappings between weights and normal vectors (which makes sense, considering our earlier point that normals are generated from *relative* weights). We have a [consistent](http://en.wikipedia.org/wiki/System_of_linear_equations#Consistency) and [underdetermined](http://en.wikipedia.org/wiki/Underdetermined_system) linear system, which means we can impose arbitrary weights and normals, then accept any other values that lead to a solution.

Given a strict ordering of normals and weights, $A$ has a deterministic form. With classes $i$, $j$ and $k$, we have:

$$
\mathbf{n} =
\begin{bmatrix}
\mathbf{n}_{i,j} \\
\mathbf{n}_{i,k} \\
\mathbf{n}_{j,k} \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 \\
-1 & 0 & 1 \\
0 & -1 & 1 \\
\end{bmatrix}
\begin{bmatrix}
\mathbf{w}_i \\
\mathbf{w}_j \\
\mathbf{w}_k \\
\end{bmatrix}
= \mathbf{A}\mathbf{w}
$$

If we included a new class $l$, we would have:

$$
\mathbf{n} =
\begin{bmatrix}
\mathbf{n}_{l,i} \\
\mathbf{n}_{l,j} \\
\mathbf{n}_{l,k} \\
\mathbf{n}_{i,j} \\
\mathbf{n}_{i,k} \\
\mathbf{n}_{j,k} \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 & 0 \\
-1 & 0 & 1 & 0 \\
-1 & 0 & 0 & 1 \\
0 & -1 & 1 & 0 \\
0 & -1 & 0 & 1 \\
0 & 0 & -1 & 1 \\
\end{bmatrix}
\begin{bmatrix}
\mathbf{w}_l \\
\mathbf{w}_i \\
\mathbf{w}_j \\
\mathbf{w}_k \\
\end{bmatrix}
= \mathbf{A}\mathbf{w}
$$

You may have noticed the pattern developing in $\mathbf{A}$ already: $\mathbf{A}$ is an $\frac{M(M-1)}{2}$ by $M$ matrix where, in iterating through each class' not-yet-defined normals to all other classes, it takes the form of a column of $-1$ appended to an $\mathbf{I}$ matrix for each iteration. With a state-space of dimensionality $n$, this gives us $\frac{nM(M-1)}{2}$ equations and $nM$ total unknowns (contained in $M$ weight vectors of size $n$ by $1$).

But, since our system is overdetermined, we can specify only $M$ normals and still achieve $M$ weight vectors. To select our normals, we use [Rouché-Capelli's theorem](http://en.wikipedia.org/wiki/Rouch%C3%A9%E2%80%93Capelli_theorem) to assure ourselves that we have a unique solution. Using a subset of normals, called $\mathbf{n}_{min}$, we have at least one solution if:

$$
rank\left(\mathbf{A}_{min}\right) = rank\left(\left[\begin{array}{r|r} \mathbf{A}_{min} & \mathbf{n}_{min}\end{array}\right]\right)
$$

Where $\mathbf{A}_{min}$ is the transformation matrix between our minimal set of normals our weights. Our solution is unique if $rank\left(\mathbf{A_{min}}\right) = M$.

When doing a rank test, we can check for, say,

$$
\mathbf{n}_{min} =
\begin{bmatrix}
\mathbf{n}_{l,i} \\
\mathbf{n}_{l,j} \\
\mathbf{n}_{l,k} \\
\mathbf{n}_{i,j} \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 & 0 \\
-1 & 0 & 1 & 0 \\
-1 & 0 & 0 & 1 \\
0 & -1 & 1 & 0 \\
\end{bmatrix}
\begin{bmatrix}
\mathbf{w}_l \\
\mathbf{w}_i \\
\mathbf{w}_j \\
\mathbf{w}_k \\
\end{bmatrix}
= \mathbf{A}_{min}\mathbf{w}
$$

Similarly, we can find the bias terms for each class by knowing the offsets of the normal vectors, that is:

$$
\begin{align}
d_{i,j} &= b_j - b_i \\
d_{j,k} &= b_k - b_j \\
d_{k,i} &= b_i - b_k = -d_{i,j} - d_{j,k}\\
\end{align}
$$

Which gives us a similar form to figure out all our bias terms. Let $\mathbf{b} = \begin{bmatrix}b_i & b_j & b_k\end{bmatrix}^T$ and $\mathbf{d} = \begin{bmatrix}d_{i,j} & d_{j,k} & d_{k,i}\end{bmatrix}^T$ (both generalizable for any number of classes). Following the previous logic, we have:

$$
\mathbf{d} = \mathbf{A}\mathbf{b}
$$

and

$$
\mathbf{d}_{min} = \mathbf{A}_{min}\mathbf{b}
$$

Where $A$ is the same transformation matrix as before.

### Example: T-shape Pac-Man and weights from normals

Let's say we want to partition the space in a T-shape, with the North-East, North-West and South spaces. We can define our normal vectors by inspection (i.e. $\mathbf{n}_{NE,NW}^T  = \begin{bmatrix}1 & 0 \end{bmatrix}$ because we need a positive $x$ component to move from NW to NE).

We should simply be able to plug these normals into our previous transformation equation:

$$
\mathbf{n}
= \begin{bmatrix}
\mathbf{n}_{NW,NE}^T \\
\mathbf{n}_{NW,S}^T \\
\mathbf{n}_{NE,S}^T \\
\mathbf{n}_{S,NE}^T \\
\mathbf{n}_{S,NW}^T \\
\mathbf{n}_{NE,NW}^T \\
\end{bmatrix}
= \begin{bmatrix}
1 & 0 \\
0 & -1 \\
0 & -1 \\
0 & 1 \\
0 & 1 \\
-1 & 0 \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 \\
-1 & 0 & 1 \\
0 & -1 & 1 \\
0 & 1 & -1 \\
1 & 0 & -1 \\
1 & -1 & 0 \\
\end{bmatrix}
\begin{bmatrix}
\mathbf{w}_{NW}^T \\
\mathbf{w}_{NE}^T \\
\mathbf{w}_{S}^T
\end{bmatrix}
= \mathbf{A}\mathbf{w}
$$

But we don't need to specify all the normal vectors in order to get our weights, so let's try a minimal representation:

$$
\mathbf{n}_{min}
= \begin{bmatrix}
\mathbf{n}_{NW,NE}^T \\
\mathbf{n}_{NW,S}^T \\
\mathbf{n}_{NE,S}^T \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 0 \\
0 & 1 \\
0 & 1 \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 \\
-1 & 0 & 1 \\
0 & -1 & 1
\end{bmatrix}
\begin{bmatrix}
\mathbf{w}_{NW}^T \\
\mathbf{w}_{NE}^T \\
\mathbf{w}_{S}^T
\end{bmatrix}
= \mathbf{A}_{min}\mathbf{w}
$$

Have you spotted the problem yet? We set up the following system of equations:

$$
\begin{bmatrix}
-1 & 0 \\
0 & 1 \\
0 & 1 \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 \\
-1 & 0 & 1 \\
0 & -1 & 1
\end{bmatrix}
\begin{bmatrix}
w_{NW,x} & w_{NW,y} \\
w_{NE,x} & w_{NE,y} \\
w_{S,x} & w_{S,y} \\
\end{bmatrix}
$$

But this system is **inconsistent**, as it gives equations such as:

$$
\begin{align}
w_{NE,x} - w_{NW,x} = -1 \\
w_{S,x} - w_{NW,x} = 0 \\
w_{S,x} - w_{NE,x} = 0
\end{align}
$$

for which there is no solution – that is, we cannot specify weights to appropriately partition the state space. This is further validated by our rank test, but, to perform the rank test, we need to vectorize the values we're solving for:

$$
\begin{bmatrix}
-1 \\ 0 \\
0 \\ 1 \\
0 \\ 1 \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 \\
-1 & 1 & 0 \\
-1 & 0 & 1 \\
-1 & 0 & 1 \\
0 & -1 & 1 \\
0 & -1 & 1
\end{bmatrix}
\begin{bmatrix}
w_{NW,x} \\ w_{NW,y} \\
w_{NE,x} \\ w_{NE,y} \\
w_{S,x} \\ w_{S,y} \\
\end{bmatrix}
$$

$$
rank\left(\begin{array}{rrr|r}
-1 & 1 & 0 & -1 \\
-1 & 0 & 1 & 0 \\
0 & -1 & 1 & 0 \\
-1 & 1 & 0 & 0 \\
-1 & 0 & 1 & 1 \\
0 & -1 & 1 & 1
\end{array}\right)
$$

How do we fix it? We can split the southern class up into two subclasses: SE and SW. This gives us the normal vectors we'd solved for in the previous case. Let's plug these in and find some weights:

$$
\mathbf{N} = \begin{bmatrix}
\mathbf{n}_{NW,NE}^T \\
\mathbf{n}_{NW,SE}^T \\
\mathbf{n}_{NW,SW}^T \\
\mathbf{n}_{NE,SE}^T \\
\mathbf{n}_{NE,SW}^T \\
\mathbf{n}_{SE,SW}^T \\
\end{bmatrix}
= \begin{bmatrix}
2 & 0 \\
2 & -2 \\
0 & -2 \\
0 & -2 \\
-2 & -2 \\
-2 & 0 \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 & 0 \\
-1 & 0 & 1 & 0 \\
-1 & 0 & 0 & 1 \\
0 & -1 & 1 & 0 \\
0 & -1 & 0 & 1 \\
0 & 0 & -1 & 1
\end{bmatrix}
\begin{bmatrix}
\mathbf{w}_{NW}^T \\
\mathbf{w}_{NE}^T \\
\mathbf{w}_{SE}^T \\
\mathbf{w}_{SW}^T \\
\end{bmatrix}
= \mathbf{A}\mathbf{W}
$$

Using our favourite [matrix decomposition technique](http://en.wikipedia.org/wiki/Matrix_decomposition) we can solve for our weights. Note that, for a strict ordering, the form of matrix $A$ should be generalizable -- that is, we do not need to re-solve $A$ for each number of weights or normals, nor the dimensionality of the state vector. We'll get to how to generalize $A$ after we've finished solving this example.

We can use the same technique to solve for our biases from our offsets:

$$
\mathbf{d}
= \begin{bmatrix}
d_{NW,NE} \\
d_{NW,SE} \\
d_{NW,SW} \\
d_{NE,SE} \\
d_{NE,SW} \\
d_{SE,SW} \\
\end{bmatrix}
= \begin{bmatrix}
4 \\
10 \\
6 \\
6 \\
2 \\
-4 \\
\end{bmatrix}
= \begin{bmatrix}
-1 & 1 & 0 & 0 \\
-1 & 0 & 1 & 0 \\
-1 & 0 & 0 & 1 \\
0 & -1 & 1 & 0 \\
0 & -1 & 0 & 1 \\
0 & 0 & -1 & 1
\end{bmatrix}
\begin{bmatrix}
b_{NW} \\
b_{NE} \\
b_{SE} \\
b_{SW}
\end{bmatrix}
= \mathbf{A}\mathbf{b}
$$

Let's see if we can solve for the weights and biases of our Pac-Man example by just specifying the normals and offsets:

```python
np.set_printoptions(precision=2, suppress=True)
A = np.array([[-1, 1, 0, 0],
              [-1, 0, 1, 0],
              [-1, 0, 0, 1],
              [0, -1, 1, 0],
             ])

# Solve for weights
n = np.array([[2, 0],
              [2, -2],
              [0, -2],
              [0, -2],])

w = np.dot(np.linalg.pinv(A),n)  # Using the Moore-Penrose pseudo-inverse
print('Weights:\n{}'.format(w))

# Solve for biases
d = np.array((4, 10, 6, 6))
biases = np.dot(np.linalg.pinv(A), d)
print('Biases:{}'.format(biases))

labels = ['NE','NW','SW','SE']
pacman = SoftMax(weights=w, biases=biases, class_labels=labels)
pacman.plot(title='Pac-Man Bearing Model')
```

That looks about right (save for the color scheme). Except, what we wanted was one class to represent the South, and one class to represent North-East and one to represnt North-West. We can't do that with our current SoftMax model, because the normals wouldn't sum to zero, but we'll investigate a technique called Multimodal SoftMax in [Chapter 4](04_mms.ipynb) to resolve this problem.

**NOT TRUE** Note the weights generated by our decomposition are unit vectors, as opposed to the vectors of magnitude $\sqrt{2}$ that we'd originally used. The slopes for the probability distributions from weights are less steep than our original case, even if the class boundaries are identical. Thankfully, our math works out such that we can control the slope of the SoftMax distributions by changing the magnitude of the normals, like so:

```python
n = np.array([[0,5,0],
              [0,0,-5],
              [0,-5,0],
              [0,0,5]])
A = np.roll(np.eye(4,4),1,axis=1) - np.eye(4,4)
w = np.dot(np.linalg.pinv(A),n)  # Using the Moore-Penrose pseudo-inverse
print(w)

labels = ['SW','NW','SE','NE']
pacman = SoftMax(weights=w, class_labels=labels)
pacman.plot(title='Pac-Man Bearing Model')
```

```python
from IPython.core.display import HTML

# Borrowed style from Probabilistic Programming and Bayesian Methods for Hackers
def css_styling():
    styles = open("../styles/custom.css", "r").read()
    return HTML(styles)
css_styling()
```

<link href='http://fonts.googleapis.com/css?family=Roboto:100,100italic,500,300,300italic,400' rel='stylesheet' type='text/css'>

```python

```


# Creating custom plots

Sometimes, the functions exposed by Sympy's plotting module are not enough to accomplish our visualization objectives. If that's the case, we can either:
1. `lambdify` the symbolic expressions and evaluate it numerically. However, this process is manually intensive.
2. If the expressions can be plotted by the common plotting functions (`plot`, `plot3d`, `plot_parametric`, ...), then we can use the `get_plot_data` function, which automate the _lambdifying_ process. This function accepts the same arguments of the aforementioned plotting functions, therefore it is really easy to get the numerical data we are interested in.

Once we have the numerical data, we can use our preferred plotting library. If we are lucky enough, we can also:
1. use one of the plotting functions as a starting point;
2. extract the numerical data with the `get_plot_data` function;
3. extract the plot object associated to the plotting library;
4. use the appropriate command of the plotting library to add new data to the plot.

Let's see a few examples.

## Example 1 - Editing and Adding Data

The current backends are able to plot lines, gradient lines, contours, quivers, streamlines. However, they are not able to plot things like _curve fills_, bars, ...

In this example we are going to illustrate a procedure that can be used to further customize the plot. Note that the procedure depends on the backend we are going to use, because we are going to use backend-specific commands. In the following, we are going to use ``PlotlyBackend``. For other backends, the procedure might need to be adjusted.

```python
from sympy import *
from spb import *
from spb.backends.matplotlib import MB
from spb.backends.plotly import PB
```

Let's say we would like to plot on the same figure:
* a normal distribution filled to the horizontal axis.
* a dampened oscillation.
* bars following an exponential decay at integer locations.

```python
x, mu, sigma = symbols("x, mu, sigma")
expr1 = 1 / (sigma * sqrt(2 * pi)) * exp(-((x - mu) / sigma)**2 / 2)
expr2 = cos(x) * exp(-x / 6)
expr3 = exp(-x / 5)
display(expr1, expr2, expr3)
```

We start by plotting the first two expressions, as the third one requires a different approach:

```python
p = plot(
    (expr1.subs({sigma: 0.8, mu: 5}), "normal"),
    (expr2, "oscillation"),
    (x, 0, 10), backend=PB)
```

Now, we'd like to fill the first curve. First, we extract the figure object; then we set the necessary attribute to get the job done:

```python
f = p.fig
f.data[0]["fill"]="tozerox"
f
```

At this point we have to convert ``expr3`` to numerical data. We can do it with ``get_plot_data``, which requires the same arguments as the ``plot`` function, namely ``(expr, range, label [optional], **kwargs)``:

```python
xx, yy = get_plot_data(exp(-x / 5), (x, 0, 10), only_integers=True)
print(xx)
print(yy)
```

Now that we have generated the numerical values at integer locations, we can add the bars with the appropriate command:

```python
import plotly.graph_objects as go
import numpy as np

f.add_trace(go.Bar(x=xx, y=yy, width=np.ones_like(xx) / 2, name="bars"))
f
```

That's it, job done.

## Example 2

The backends are unable to mix 2D and 3D data series. But what if we would like to plot a contour into a 3D plot?

Let's say we'd like to explore the following vector field, $\vec{F}(x, y, z) = (\cos{(z)}, y, x)$, in the rectangular volume limited by $-5 \le x \le 5, \, -5 \le y \le 5, \, -5 \le z \le 5$. We are going to plot the contours of the magnitude of the vector field over 3 orthogonal planes, as well as quivers over a plane normal to the y-direction.

```python
x, y, z = symbols("x:z")
v = Matrix([cos(z), y, x])

# magnitudes of the vector field over 3 orthogonal planes
mag_func = lambda vec: sqrt(sum(t**2 for t in vec))
mag = mag_func(v)
m1 = mag.subs(x, 5)
m2 = mag.subs(y, 5)
m3 = mag.subs(z, 5)
display(mag, m1, m2, m3)
```

Let's extract the data of the magnitudes:

```python
# ranges
rx = (x, -5, 5)
ry = (y, -5, 5)
rz = (z, -5, 5)
# contour data: similarly to plot3d/plot_contour the parameters
# to get_plot_data follows (expr, range_x, range_y)
xx1, yy1, zz1 = get_plot_data(m1, ry, rz)
xx2, yy2, zz2 = get_plot_data(m2, rx, rz)
xx3, yy3, zz3 = get_plot_data(m3, rx, ry)
```

Now, let's extract the data of the sliced-vector field:

```python
xq1, yq1, zq1, uu, vv, ww = get_plot_data(v, rx, ry, rz, slice=Plane((0, 0, 0), (0, 1, 0)))
```

Finally, we create the custom plot:

```python
%matplotlib widget
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
from matplotlib import cm

ax = plt.figure().add_subplot(projection='3d')

# Plot projections of the contours for each dimension.  By choosing offsets
# that match the appropriate axes limits, the projected contours will sit on
# the 'walls' of the graph
ax.contourf(xx1, zz1, yy1, zdir='y', offset=5, cmap=cm.coolwarm)
ax.contourf(zz2, xx2, yy2, zdir='x', offset=-5, cmap=cm.coolwarm)
ax.contourf(xx3, yy3, zz3, zdir='z', offset=-5, cmap=cm.coolwarm)
ax.quiver(xq1, yq1, zq1, uu, vv, ww, color="g", length=0.5, normalize=True)

ax.set_xlim(-5, 5)
ax.set_ylim(-5, 5)
ax.set_zlim(-5, 5)

ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')

plt.show()
```

```python

```


# Seminar: SVM, Kernel trick
Machine Learning by prof. Evgeny Burnaev

Author: Andrey Lange

Given training data $x_i \in \mathbb{R}^p$, $y_i\in\{-1, 1\}$, $i=1,\ldots, n$, we have the following problem:

Primal problem:

\begin{equation}
    \frac{1}{2} ||w||^2 + C \sum_{i=1}^{n} \xi_i \longrightarrow \min_{w, b, \xi},\\
      \begin{split}
      \textrm {subject to }\ & y_i (w^T \phi(x_i) + b) \geq 1 - \xi_i,\\
                             & \xi_i \geq 0.
      \end{split}
\end{equation}

Dual problem:

\begin{equation}
    \frac{1}{2} \sum_{i=1}^n\sum_{j=1}^n \alpha_i\alpha_j y_i y_j \underbrace{\phi(x_i)^{T} \phi(x_j)}_{K(x_i, x_j)} - \sum_{i=1}^n \alpha_i \longrightarrow  \min_{\alpha}, \\
    \begin{split}
    \textrm {subject to }\ & \sum_{i=1}^n y_i \alpha_i = 0,\\
                           & 0 \leq \alpha_i \leq C.
    \end{split}
\end{equation}

$\alpha_i = 0$ for non-support vectors,

$\alpha_i < C$ if $x_i$ lies on the margin,

$\alpha_i = C$ if $x_i$ lies inside a margin or misclassified.

The decision function is a sum over only SUPPORT data points $S$:

\begin{equation}
y(x)= w^T\phi(x) + b = \sum_{x_i \in S} \alpha_i y_i K(x_i, x) + b \qquad \qquad (1)
\end{equation}

The sum is over only the support vectors!

Less support vectors (sparsity) - faster classifications!

```python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
```

## Topic 1: Separation by a hyperplane

```python
from sklearn import svm, datasets, metrics

X, y = datasets.make_blobs(n_samples=100, centers=2, cluster_std=1.0, center_box = (-15, 15), random_state=0)
X = np.concatenate((X, [[4, 8], [0, 0]]))
y = np.concatenate((y, [1, 0]))

plt.scatter(X[:, 0], X[:, 1], c=y, s=30, cmap=plt.cm.Paired)
plt.xlim(-1, 6)
plt.ylim(-1, 8)

clf = svm.SVC(kernel='linear')
clf.fit(X, y)

# create grid to evaluate model
xx = np.linspace(X[:, 0].min(), X[:, 0].max(), 30)
yy = np.linspace(X[:, 1].min(), X[:, 1].max(), 30)
YY, XX = np.meshgrid(yy, xx)
xy = np.vstack([XX.ravel(), YY.ravel()]).T
Z = clf.decision_function(xy).reshape(XX.shape)

# plot decision boundary and margins
plt.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5, linestyles=['--', '-', '--'])
plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=100,
           linewidth=1, facecolors='none', edgecolors='k')

print('accuracy on the training set:', metrics.accuracy_score(y, clf.predict(X)))
```

### Question 1.1: Change the code above to make accuracy of 1. Do not tune any hyperparameters of SVM. What prevents ideal classification in this case?

### Question 1.2: How to calculate the value of accuracy without any function call, just looking on the code and on the picture?

## Topic 2: Primal SVM and explicit transforms

```python
df = pd.read_csv('https://raw.githubusercontent.com/adasegroup/ML2021_seminars/master/seminar4/data/dataset0.csv')

X = df[['X1', 'X2']]
y = df['y']
plt.scatter(X['X1'], X['X2'], c=['brown' if c == 1 else 'b' for c in y])
plt.title('labeled dataset')
plt.show()

clf = svm.SVC(kernel='linear')
clf.fit(X, y)

plt.scatter(X['X1'], X['X2'], c=['brown' if c == 1 else 'b' for c in clf.predict(X)])
plt.title('predicted class labels: all blue class was missclassified!')
plt.show()

print('Linear SVM is not good if neither basic nor kernel is used:')
print('accuracy on the training set:', metrics.accuracy_score(y, clf.predict(X)))
```

### Question 2.1: One of the ideas of SVM is that to transform the initial features to some other space where the classes are better separated by a hyperplane. What about using the polar coordinates in the example above? Apply it and show how SVM works with such a new basis using the following template.

```python
df = pd.read_csv('https://raw.githubusercontent.com/adasegroup/ML2021_seminars/master/seminar4/data/dataset0.csv')

X = df[['X1', 'X2']]
y = df['y']

def to_polar(x, y):
    r = np.sqrt(x**2 + y**2)
    phi = np.arctan2(y, x)
    return phi, r

# --- HERE IS YOUR CODE!!!
# change this block so that it gives new pandas DataFrame called X_transform
# with columns 'phi' and 'r' and X and X_transform have the corresponding rows of data.
X_transform = pd.DataFrame()
X_transform['phi'] = X['X1'] # its wrong
X_transform['r'] = X['X2']   # its wrong

# --- HERE IS YOUR CODE!!!
clf = svm.SVC(kernel='linear')
clf.fit(X_transform, y)
print('accuracy on the training set:', metrics.accuracy_score(y, clf.predict(X_transform)))

plt.scatter(X_transform['phi'], X_transform['r'], c=['brown' if c == 1 else 'b' for c in y])
plt.xlabel('phi')
plt.ylabel('r')

pp = np.linspace(X_transform['phi'].min(), X_transform['phi'].max(), 30)
rr = np.linspace(X_transform['r'].min(), X_transform['r'].max(), 30)
RR, PP = np.meshgrid(rr, pp)
pr = np.vstack([PP.ravel(), RR.ravel()]).T
Z = clf.decision_function(pr).reshape(PP.shape)
plt.contour(PP, RR, Z, colors='k', levels=[-1, 0, 1], linestyles=['--', '-', '--'])
plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=100,
            linewidth=1, facecolors='none', edgecolors='k')
plt.show()
```

### Question 2.2: The separating hyperplane (line in 2D) in polar basis above looks strange: why shouldn't it be a little higher because many blue points are missclassified - they are on the other side of the line! We need to tune the hyperparameters of the model - constant $C$ in SVM. Try different values of $C$, using the code below and then find the optimal value of $C$, which provides the maximum accuracy.

```python
df = pd.read_csv('https://raw.githubusercontent.com/adasegroup/ML2021_seminars/master/seminar4/data/dataset0.csv')

X = df[['X1', 'X2']]
y = df['y']

phi, r = to_polar(X['X1'], X['X2'])
X_transform = pd.DataFrame({'phi': phi, 'r': r})
# # --- YOUR CODE HERE!!!
# select the optimal C among some set of values (say from 10**-5 to 10**5)
# providing the best model 'clf' and print its accuracy metric on the training dataset
clf = svm.SVC(kernel='linear', C=1).fit(X_transform, y)
print('accuracy on the training set:', metrics.accuracy_score(y, clf.predict(X_transform)))
# # --- YOUR CODE HERE!!!

plt.scatter(X_transform['phi'], X_transform['r'], c=['brown' if c == 1 else 'b' for c in y])
plt.xlabel('phi')
plt.ylabel('r')

pp = np.linspace(X_transform['phi'].min(), X_transform['phi'].max(), 30)
rr = np.linspace(X_transform['r'].min(), X_transform['r'].max(), 30)
RR, PP = np.meshgrid(rr, pp)
pr = np.vstack([PP.ravel(), RR.ravel()]).T
Z = clf.decision_function(pr).reshape(PP.shape)
plt.contour(PP, RR, Z, colors='k', levels=[-1, 0, 1], linestyles=['--', '-', '--'])
plt.show()
```

### Question 2.3: Usually, SVM transforms into a space with the dimension higher than number of original features because it makes easier to separate classes by a hyperplane there. Unlike to polar coordinates, one can try the transform
### $$y_1=x_1, \quad y_2=x_2, \quad y_3=|x_1|+|x_2|.$$
### Why classes are better separable in that 3D space? Imagine the picture in 3D. Since we have previously tuned the model even reaching the accuracy of 1, improve this transform and the range for hyperparameter $C$ for better fitting.

```python
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score, make_scorer

df = pd.read_csv('https://raw.githubusercontent.com/adasegroup/ML2021_seminars/master/seminar4/data/dataset0.csv')

X = df[['X1', 'X2']]
y = df['y']

# --- YOUR CODE HERE !!! improve the transform
def to_y1y2y3(x1, x2):
    y1 = x1
    y2 = x2
    y3 = abs(x1) + abs(x2)
    return y1, y2, y3
# --- YOUR CODE HERE !!! improve the transform

# improve the range for hyperparameter C
y1, y2, y3 = to_y1y2y3(X['X1'], X['X2'])
X_transform = pd.DataFrame({'Y1': y1, 'Y2': y2, 'Y3': y3})
clf = GridSearchCV(svm.SVC(kernel='linear'),
                   {'C': [1]},         # YOUR CODE HERE, IMPROVE THE RANGE  !!!
                   scoring=make_scorer(accuracy_score),  # we can skip this line because accuracy_score is by default
                   cv = [(np.array(range(len(y))), np.array(range(len(y))))] # only 1 split and training and testing sets are the same
                   )
clf.fit(X_transform, y)
print('best C:', clf.best_params_['C'])
print('accuracy on the training set:', clf.best_score_)

xx1 = np.linspace(X['X1'].min(), X['X1'].max(), 50)
xx2 = np.linspace(X['X2'].min(), X['X2'].max(), 50)
XX1, XX2 = np.meshgrid(xx1, xx2)
YY1, YY2, YY3 = to_y1y2y3(XX1, XX2)
xy = np.vstack([YY1.ravel(), YY2.ravel(), YY3.ravel()]).T
Z = clf.decision_function(xy).reshape(XX1.shape)
plt.scatter(X['X1'], X['X2'], c=['brown' if c == 1 else 'b' for c in y])
plt.contour(XX1, XX2, Z, colors='k', levels=[-1, 0, 1], linestyles=['--', '-', '--'])
plt.show()
```

### Question 2.4: Execute the code below. There is a solution for SVM classification by some decision curve that looks like a circle and the ROC-curve that have 1 point between $(0, 0)$ and $(1, 1)$. Why this point is only one? Obtain its coordinates by another way - using the confusion matrix. What part of blue points are missclassified?

### Question 2.5: Unlike to explicit transform (like polar coordinates $(x_1, x_2) \to (\phi, r)$ or $(x_1, x_2) \to (y_1, y_2, y_3)$) in primal SVM problem it can be implied implicitly by the corresponding kernel in the equivalent dual SVM problem. Apply RBF (Radial Basis Function) kernel to this example with initial features $x_1$ and $x_2$ and tune the hyperparameters $C$ and $\gamma$.

```python
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score, make_scorer

df = pd.read_csv('https://raw.githubusercontent.com/adasegroup/ML2021_seminars/master/seminar4/data/dataset0.csv')

X = df[['X1', 'X2']]
y = df['y']

# improve the ranges for hyperparameters C and gamma
clf = GridSearchCV(
                   svm.SVC(kernel='rbf'),                 # kernel is changed to non-linear 'rbf'
                   {'C': [1], 'gamma': [1]},              # YOUR IMPROVEMENT HERE !!!
                   scoring=make_scorer(accuracy_score),   # we can skip this line because accuracy_score is by default
                   cv = [(np.array(range(len(y))), np.array(range(len(y))))]  # only 1 split and training and testing sets are the same
                  )
clf.fit(X, y) # note that we use the initial X, not X_transform !
print('best params:', clf.best_params_)
print('accuracy on the training set:', clf.best_score_)

# ROC - curve
plt.figure()
from sklearn.metrics import accuracy_score, make_scorer, roc_curve

fpr, tpr, thresh = roc_curve(y, clf.decision_function(X))
plt.title('ROC curve')
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.plot(fpr, tpr, '--o')

fpr, tpr, _ = roc_curve(y, clf.predict(X))
plt.title('ROC curve')
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.plot(fpr, tpr, '--o')

plt.show()
print('points: ', list(zip(fpr, tpr, thresh)))

# Plot classification
xx1 = np.linspace(X['X1'].min(), X['X1'].max(), 30)
xx2 = np.linspace(X['X2'].min(), X['X2'].max(), 30)
XX1, XX2 = np.meshgrid(xx1, xx2)
xy = np.vstack([XX1.ravel(), XX2.ravel()]).T
Z = clf.decision_function(xy).reshape(XX1.shape)
plt.scatter(X['X1'], X['X2'], c=['brown' if c == 1 else 'b' for c in y])
plt.contour(XX1, XX2, Z, colors='k', levels=[0], linestyles=['-'])
plt.title('many blue points were missclassified!')       # YOUR IMPROVEMENT HERE :)
plt.show()
```

### Question 2.6: Here is the example with polar coordinates from above shown in initial features:

```python
df = pd.read_csv('https://raw.githubusercontent.com/adasegroup/ML2021_seminars/master/seminar4/data/dataset0.csv')

X = df[['X1', 'X2']]
y = df['y']

def to_polar(x, y):
    r = np.sqrt(x**2 + y**2)
    phi = np.arctan2(y, x)
    return phi, r

phi, r = to_polar(X['X1'], X['X2'])
X_transform = pd.DataFrame({'phi': phi, 'r': r})

clf = svm.SVC(kernel='linear', C=1)
clf.fit(X_transform, y)
print('accuracy on the training set:', metrics.accuracy_score(y, clf.predict(X_transform)))

plt.scatter(X['X1'], X['X2'], c=['brown' if c == 1 else 'b' for c in y])
plt.xlabel('X1')
plt.ylabel('X2')

xx1 = np.linspace(X['X1'].min(), X['X1'].max(), 30)
xx2 = np.linspace(X['X2'].min(), X['X2'].max(), 30)
XX2, XX1 = np.meshgrid(xx2, xx1)
X1X2 = np.vstack([XX1.ravel(), XX2.ravel()]).T

phi_rho = np.array([to_polar(x1, x2) for x1, x2 in X1X2])
Z = clf.decision_function(phi_rho).reshape(PP.shape)
plt.contour(XX1, XX2, Z, colors='k', levels=[-1, 0, 1], linestyles=['--', '-', '--'])

plt.scatter(X['X1'][clf.support_], X['X2'][clf.support_], s=100, linewidth=1, facecolors='none', edgecolors='k')
plt.show()
```

### Find the expression for the equivalent kernel analytically. Train a model and show the same picture of classification but using the kernel instead of explicit transform (modify the code below):

```python
df = pd.read_csv('https://raw.githubusercontent.com/adasegroup/ML2021_seminars/master/seminar4/data/dataset0.csv')

X = df[['X1', 'X2']]
y = df['y']

def to_polar(x1, x2):
    r = np.sqrt(x1**2 + x2**2)
    varphi = np.arctan2(x2, x1)
    return varphi, r

# modify this function of kernel calculation
def my_kernel(X, Y):
    # here phi(x1, x2) is the basis function of the transformation (varphi, r) = phi(x1, x2)
    # now it is just a linear kernel, make it to be 'kernel of polar coordinates':
    phi_x = np.array([(x1, x2) for x1, x2 in X])
    phi_y = np.array([(x1, x2) for x1, x2 in Y])
    return np.matmul(phi_x, phi_y.T) # all values of inner product phi(x) * phi^T(y)

clf = svm.SVC(kernel=my_kernel, C=1) # kernel is used instead of explicit transform (varphi, r) = phi(x1, x2), dual problem
clf.fit(X, y)                        # initial features X are used, but no explicit polar coordinates!
print('accuracy on the training set:', metrics.accuracy_score(y, clf.predict(X)))

plt.scatter(X['X1'], X['X2'], c=['brown' if c == 1 else 'b' for c in y])
plt.xlabel('X1')
plt.ylabel('X2')

xx1 = np.linspace(X['X1'].min(), X['X1'].max(), 30)
xx2 = np.linspace(X['X2'].min(), X['X2'].max(), 30)
XX2, XX1 = np.meshgrid(xx2, xx1)
X1X2 = np.vstack([XX1.ravel(), XX2.ravel()]).T
Z = clf.decision_function(X1X2).reshape(PP.shape)
plt.contour(XX1, XX2, Z, colors='k', levels=[-1, 0, 1], linestyles=['--', '-', '--'])

plt.scatter(X['X1'][clf.support_], X['X2'][clf.support_], s=100, linewidth=1, facecolors='none', edgecolors='k')
plt.show()
```

# Topic 3: Training and testing sets

```python
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score, make_scorer
from sklearn.model_selection import train_test_split

df = pd.read_csv('https://raw.githubusercontent.com/adasegroup/ML2021_seminars/master/seminar4/data/dataset1.csv')

X = df[['X1', 'X2']]
y = df['y']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/3, random_state=0)

clf = GridSearchCV(svm.SVC(kernel='rbf'),                                     # kernel is 'rbf'!
                            {'C': [1], 'gamma': [1.5]},                       # Underfitting
#                           {'C': [1000], 'gamma': [100]},                    # Overfitting
#                      ... YOUR IMPROVEMENT HERE !!!
                   scoring=make_scorer(accuracy_score),
                   cv=3)    # we can skip this line because accuracy is by default
clf.fit(X_train, y_train) # note that we use X_train, no explicit X_transform !
print('best params:', clf.best_params_)
print('accuracy on the training set:', clf.score(X_train, y_train))
print('accuracy on the testing set:', clf.score(X_test, y_test))

# plots results
xx1 = np.linspace(X['X1'].min(), X['X1'].max(), 50)
xx2 = np.linspace(X['X2'].min(), X['X2'].max(), 50)
XX1, XX2 = np.meshgrid(xx1, xx2)
xy = np.vstack([XX1.ravel(), XX2.ravel()]).T
Z = clf.decision_function(xy).reshape(XX1.shape)

def flter(vv, mask):
    return [v for v, m in zip(vv, mask) if m]

plt.scatter(flter(X_train['X1'], y_train), flter(X_train['X2'], y_train), c='brown', alpha = 0.5, label='y_train == 1')
plt.scatter(flter(X_train['X1'], 1-y_train), flter(X_train['X2'], 1-y_train), c='b', alpha = 0.5, label='y_train == 0')
plt.legend(bbox_to_anchor=(0.999, 1))
plt.contour(XX1, XX2, Z, colors='k', levels=[0], linestyles=['-'])
plt.title('$\mathbf{Training}$ set and $\mathbf{trained}$ decision curve')
plt.show()

plt.scatter(flter(X_test['X1'], y_test), flter(X_test['X2'], y_test), c='brown', label='y_test == 1')
plt.scatter(flter(X_test['X1'], 1-y_test), flter(X_test['X2'], 1-y_test), c='b', label='y_test == 0')
plt.legend(bbox_to_anchor=(0.999, 1))
plt.contour(XX1, XX2, Z, colors='k', levels=[0], linestyles=['-'])
plt.title('$\mathbf{Testing}$ set and $\mathbf{trained}$ decision curve')
plt.show()
```

### Question 3.1: Try different sets of hyperparameters including:
#### Underfitting:
```
accuracy on the training set: 0.707317073171 - poor :(
accuracy on the testing set: 0.780487804878  - poor :(
```
#### Overfitting:
```
accuracy on the training set: 1.0            - even 1.0! but it is useless :|
accuracy on the testing set: 0.841463414634  - poor :(
```
### See the decision curve in both cases (it is the hyperplane in the space of RBF kernel!). Explain them in terms of bias-variance tradeoff.  Tune hyperparameters $C$ and $\gamma$ aiming to increase the score on the testing set.

# Topic 4: Dual SVM

### Question 4.1: In the following example with RBF kernel 4 points are given by their coordinates. Explain why $y([0, 0])=b$. Use formula (1) in the problem formulation in the beginning.

```python
from sklearn import svm, datasets, metrics

# data of 4 points:
X = np.array([[-1, 0], [0, 1], [0, -1], [0, -1.5]])
y = np.array([1, 1, -1, -1])

plt.scatter(X[:, 0], X[:, 1], c=y, s=30, cmap=plt.cm.Paired)

clf = svm.SVC(kernel='rbf', C=2, gamma='auto')
clf.fit(X, y)

# create grid to evaluate model
xx = np.linspace(X[:, 0].min(), X[:, 0].max(), 30)
yy = np.linspace(X[:, 1].min(), X[:, 1].max(), 30)
YY, XX = np.meshgrid(yy, xx)
xy = np.vstack([XX.ravel(), YY.ravel()]).T
Z = clf.decision_function(xy).reshape(XX.shape)

# plot decision boundary and margins
plt.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5, linestyles=['--', '-', '--'])
plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=100,
           linewidth=1, facecolors='none', edgecolors='k')
plt.plot([0], [0], 'dr');

# Why y(x_0) == b, x_0=[0, 0]?
print('why are they the same:', clf.decision_function(np.array([[0, 0]])), '==', clf.intercept_, '?')
```

### Question 4.2: Let in the example above  $x_1$ and $x_2$ - brown points, $x_3$  - cyan support vector, $x_4$ - cyan non-support vector. Check what's true:
\begin{align*}
1. \qquad & \alpha_2 = С, \\
2. \qquad & \alpha_1 > \alpha_3,\\
3. \qquad & \alpha_3 - \alpha_1 = \alpha_2,\\
4. \qquad & \alpha_2 - \alpha_3 < 2C?
\end{align*}


# SEIRHVD model example
## Work in progress
\begin{align}
\dot{S} & = S_f - \alpha\beta\frac{SI}{N+k_I I+k_R R} + r_{R\_S} R\\
\dot{E} & = E_f + \alpha\beta\frac{SI}{N+k_I I+k_R R} - E\frac{1}{t_{E\_I}} \\
\dot{I} & = I_f + E\frac{1}{t_{E\_I}} - I\frac{1}{t_{I\_R}} \\
\dot{R} & = R_f + I\frac{1}{t_{I\_R}} - r_{I\_R} R\\
\end{align}

Where:
* $S:$ Susceptible
* $E:$ Exposed
* $I:$ Infectious
* $R:$ Removed
* $\alpha:$ Mobilty
* $\beta:$ Infection rate
* $N:$ Total population
* $t_{E\_I}:$ # Transition time between exposed and infectious
* $t_{I\_R}:$ # Transition time between infectious and recovered
* $r_{R\_S}:$ Immunity loss rate ($\frac{1}{t_{R\_S}}$)
* $S_f,E_f,I_f,R_f:$ External flux
* $k_I:$ Infected saturation
* $k_R:$ Immunity shield

```python
# Util libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime

# cv19 libraries
from cv19gm.models.seirhvd import SEIRHVD
from cv19gm.models.seir import SEIR
from cv19gm.utils import cv19functions
from cv19gm.data.cv19data import ImportData
```

```python
# For pop-up plots execute this code (optional)
import platform
OS = platform.system()

if OS == 'Linux':
    %matplotlib tk
    print('Linux')
elif OS == 'Windows':
    %matplotlib qt
    print('Windows')
elif OS == 'Darwin':
    %matplotlib tk
    print('Mac (Funciona?)')
```

    Linux

```python
cd ../cv19gm/models/
```

    /home/samuel/covid19geomodeller/cv19gm/models

```python
from seirhvd import SEIRHVD
```

```python
cd ../../Examples/
```

    /home/samuel/covid19geomodeller/Examples

# Build a basic SEIR with SEIRHVD lib

import importlib
importlib.reload(cv19functions)
import SEIRHVD
importlib.reload(SEIRHVD)
from SEIRHVD import SEIRHVD

```python
# Input configuration file
config = 'cfg/SEIRHVD_SEIR.toml'
# Build simulation object
model = SEIRHVD(config = config)
```

```python
%%capture
# Simulate (solve ODE)
model.solve()
```

```python
# Show results
#model.results
```

```python
# Plot matplotlib
plt.plot(model.t,model.S,label='S')
plt.plot(model.t,model.E,label='E')
plt.plot(model.t,model.I,label='I')
plt.plot(model.t,model.R,label='R')
plt.plot(model.t,model.H,label='H')
plt.plot(model.t,model.Iv,label='Iv')
plt.plot(model.t,model.D,label='D')
plt.xlim(0,300)
plt.legend(loc=0)
plt.title('SEIR from SEIRHVD lib')
plt.show()
```

```python
%%capture
# Plot with pandas
seirconfig = 'cfg/SEIR.toml'
seirmodel = SEIR(config = seirconfig)# Input configuration file
seirmodel.solve()
```

```python
# Plot matplotlib
plt.plot(model.t,model.I,label='I_seirhvd')
plt.plot(seirmodel.t,seirmodel.I,label='I_seir')
plt.xlim(0,300)
plt.legend(loc=0)
plt.title('SEIR and SEIRHVD Comparison')
plt.show()
```

# Vaccines

### Simple SEIR with vaccines

```python
# Vaccine campaign
vac_d = cv19functions.events(values=[0,20000,0],days=[[0,40],[40,60],[60,500]])
```

```python
%%capture
# Input configuration file
config = 'cfg/SEIRHVD_SEIR.toml'
# Build simulation object
model_vac = SEIRHVD(config = config,vac_d=vac_d,v = 1.0)
# Simulate (solve ODE)
model_vac.solve()
```

```python
# Plot matplotlib

plt.plot(model_vac.t,model_vac.I,label='Itotal (vaccines scenario)')
plt.plot(model.t,model.I,label='Itotal (no vaccines scenario)')
plt.plot(model_vac.t,model_vac.Iv,label='Vaccinated infected')
plt.plot(np.arange(model_vac.tsim),[vac_d(i) for i in np.arange(model_vac.tsim)],label='Daily Vaccination')
plt.xlim(0,300)
plt.legend(loc=0)
plt.title('SEIRV')
plt.show()
```

# Vaccines effectivity

```python
vac_eff = list(np.arange(0,1.1,0.1))
model_vac2 = [SEIRHVD(config = config,vac_d=vac_d,vac_eff = i) for i in vac_eff]
for i in model_vac2:
    i.solve()
```

```python
colors = plt.cm.rainbow_r(np.linspace(0,1,len(vac_eff)))
```

```python
i = 0
for model_vac in model_vac2:
    #plt.plot(model_vac.t,model_vac.I,label='I with vaccines')
    "{:#.2g}".format(i)
    plt.plot(model_vac.t,model_vac.I,label='I eff:'+"{:#.1g}".format(model_vac.vac_eff(0)),color=colors[i])
    #plt.plot(model_vac.t,model_vac.Iv,label='Iv eff:'+"{:#.1g}".format(model_vac.v(0)),color=colors[i],linestyle='dashed')
    i+=1
#plt.plot(np.arange(model_vac.tsim),[Psi(i) for i in np.arange(model_vac.tsim)],color='red',label='Daily Vaccination')
plt.legend(loc=0)

plt.xlim(0,300)
plt.title('Total Infected due to vaccines effectivity')
plt.show()
```

# Hospitalization

```python
%%capture
# Input configuration file
config = 'cfg/SEIRHVD.toml'
# Build simulation object
model1 = SEIRHVD(config = config, H_cap=4000)
model2 = SEIRHVD(config = config, H_cap=3500,vac_d=vac_d)
# Simulate (solve ODE)
model1.solve()
model2.solve()
```

```python
# Plot matplotlib
#plt.plot(model.t,model.S,label='S')
#plt.plot(model.t,model.E,label='E')
plt.plot(model1.t,model1.I,label='I')
plt.plot(model1.t,model1.Im,label='Im')
plt.plot(model1.t,model1.Icr,label='Icr')
#plt.plot(model.t,model.R,label='R')
plt.xlim(0,300)
plt.legend(loc=0)
plt.title('SEIR from SEIRHVD lib')
plt.show()
```

```python
# Plot matplotlib
plt.plot(model1.t,model1.H,color='tab:red',label='H 1')
plt.plot(model1.t,model1.D,color='tab:red',linestyle='dotted',label='D 1')
plt.plot(model1.t,[model1.H_cap]*len(model.t),color='tab:red',linestyle='dashed',label='H_cap 1')

plt.plot(model2.t,model2.H,color='tab:blue',label='H 2')
plt.plot(model2.t,model2.D,color='tab:blue',linestyle='dotted',label='D 2')
plt.plot(model2.t,[model2.H_cap]*len(model.t),color='tab:blue',linestyle='dashed',label='H_cap 2')
#plt.xlim(0,300)
#plt.plot(np.arange(model_vac.tsim),[Psi(i) for i in np.arange(model_vac.tsim)])
plt.legend(loc=0)
plt.title('ICU capacity and deaths')
plt.show()
```

# Effectivity and Iv_ac/Itot_ac proportion

```python
from cv19gm.cv19sim import CV19SIM
```

```python
vac_eff = list(np.arange(0,1.1,0.1))
#Psi = cv19functions.events(values=[0,20000,0],days=[[0,5],[5,25],[25,500]])
population = 1000000
Sv0 = population*0.5
```

```python
colors = plt.cm.rainbow_r(np.linspace(0,1,len(vac_eff)))
```

```python
model3 = CV19SIM(config = config,model='SEIRHVD',vac_eff=vac_eff,population=population,Sv=Sv0)
```

```python
model3.solve()
```

```python
sims = model3.sims
```

```python
fig, axs = plt.subplots(figsize=(10,6),linewidth=5,edgecolor='black',facecolor="white")

axs.set_ylabel('Infectados Activos totales')

for i in range(len(sims)):
    axs.plot(sims[i].t,sims[i].I,linestyle='dashed',color=colors[i])

axs2 = axs.twinx()
for i in range(len(sims)):
    axs2.plot(sims[i].t,sims[i].Iv_ac/sims[i].I_ac,label='Veff: '+"{:#.1g}".format(sims[i].vac_eff(0)),color=colors[i])

axs2.set_ylabel('Iv/I')

#axs2.tick_params(axis='y', labelcolor='tab:blue')
#axs2.set_xlim(0,200)
#lines, labels = axs.get_legend_handles_labels()
#lines2, labels2 = axs2.get_legend_handles_labels()
#axs.legend(lines + lines2, labels + labels2, loc=[0.05,0.1])
#axs.legend(loc=[0.05,0.05])
#axs.legend(loc=[0.05,0.17])

fig.suptitle('Iv/Itot proportion depending on effectivity')
#axs.legend(loc=1)
axs2.legend(loc=0)
fig.show()
```

## Including beta

```python
beta = list(np.arange(0.1,0.3,0.1))
model4 = CV19SIM(config = config,model='SEIRHVD',beta=beta,betav='beta',vac_eff=vac_eff,population=population,Sv=Sv0)
```

```python
model4.solve()
```

# Hospitalization CFR variable

```python
pH_R = cv19functions.events(values=[0.7,0.6],days=[[0,30],[30,500]])
pH_D = cv19functions.events(values=[0.3,0.4],days=[[0,30],[30,500]])
```

```python
%%capture
# Input configuration file
config = 'cfg/SEIRHVD.toml'
# Build simulation object

model1 = SEIRHVD(config = config, H_cap=4000, pH_R = pH_R,pH_D = pH_D)
# Simulate (solve ODE)
model1.integrate()
```

```python
# Plot matplotlib
#plt.plot(model.t,model.S,label='S')
#plt.plot(model.t,model.E,label='E')
plt.plot(model1.t,model1.I,label='I')
plt.plot(model1.t,model1.Im,label='Im')
plt.plot(model1.t,model1.Icr,label='Icr')
#plt.plot(model.t,model.R,label='R')
plt.xlim(0,300)
plt.legend(loc=0)
plt.title('SEIR from SEIRHVD lib')
plt.show()
```

```python
# Plot matplotlib
#plt.plot(model1.t,model1.H,color='tab:red',label='H 1')
plt.plot(model1.t,model1.D_d,color='tab:red',linestyle='dotted',label='D 1')
#plt.plot(model1.t,[model1.H_cap]*len(model.t),color='tab:red',linestyle='dashed',label='H_cap 1')
plt.plot(model1.t,[pH_D(i) for i in model.t],color='tab:red',linestyle='dashed',label='pH_D')

#plt.plot(np.arange(model_vac.tsim),[Psi(i) for i in np.arange(model_vac.tsim)])
plt.legend(loc=0)
plt.title('ICU capacity and deaths')
plt.show()
```

# Vaccines amount, effectivity  and deaths
(work in progress)

```python
eff = list(np.arange(0,1.1,0.1))
amount = np.linspace(0,30000,31)
```

```python
# Vaccine campaign
vac_d = [cv19functions.events(values=[0,i,0],days=[[0,40],[40,60],[60,500]]) for i in amount]
```

```python
%%capture
# Input configuration file
config = 'cfg/SEIRHVD_SEIR.toml'
# Build simulation object
models = []
for i in eff:
    aux = []
    for j in vac_d:
        a = SEIRHVD(config = config,vac_d=j,vac_eff = i)
        a.integrate()
        aux.append(a)
    models.append(aux)
        # Simulate (solve ODE)
#model_vac.integrate()
```

```python
# Plot matplotlib
plt.plot(model_vac.t,model_vac.I,label='I with vaccines')
plt.plot(model.t,model.I,label='I no vaccines')
plt.plot(model_vac.t,model_vac.Iv,label='Vaccinated infected')
plt.plot(np.arange(model_vac.tsim),[vac_d(i) for i in np.arange(model_vac.tsim)])
plt.xlim(0,300)
plt.legend(loc=0)
plt.title('SEIRV')
plt.show()
```

```python
type(vac_d)
```

```python
model_vac2 = [SEIRHVD(config = config,vac_d=0.7,vac_eff = i) for i in vac_eff]
for i in model_vac2:
    i.solve()
```

```python
colors = plt.cm.rainbow_r(np.linspace(0,1,len(vac_eff)))
```

```python
i = 0
for model_vac in model_vac2:
    #plt.plot(model_vac.t,model_vac.I,label='I with vaccines')
    "{:#.2g}".format(i)
    plt.plot(model_vac.t,model_vac.I,label='I eff:'+"{:#.1g}".format(model_vac.vac_eff(0)),color=colors[i])
    #plt.plot(model_vac.t,model_vac.Iv,label='Iv eff:'+"{:#.1g}".format(model_vac.v(0)),color=colors[i],linestyle='dashed')
    i+=1
plt.plot(np.arange(model_vac.tsim),[vac_d(i) for i in np.arange(model_vac.tsim)])
plt.legend(loc=0)

plt.xlim(0,300)
plt.title('Vaccines effectivity')
plt.show()
```

```python

```