#### Jupyter notebooks

This is a [Jupyter](http://jupyter.org/) notebook using Python.  You can install Jupyter locally to edit and interact with this notebook.

# Finite Difference methods in 2 dimensions

Let's start by generalizing the 1D Laplacian,

\begin{align} - u''(x) &= f(x) \text{ on } \Omega = (a,b) & u(a) &= g_0(a) & u'(b) = g_1(b) \end{align}

to two dimensions

\begin{align} -\nabla\cdot \big( \nabla u(x,y) \big) &= f(x,y) \text{ on } \Omega \subset \mathbb R^2
& u|_{\Gamma_D} &= g_0(x,y) & \nabla u \cdot \hat n|_{\Gamma_N} &= g_1(x,y)
\end{align}

where $\Omega$ is some well-connected open set (we will assume simply connected) and the Dirichlet boundary $\Gamma_D \subset \partial \Omega$ is nonempty.

We need to choose a system for specifying the domain $\Omega$ and ordering degrees of freedom.  Perhaps the most significant limitation of finite difference methods is that this specification is messy for complicated domains.  We will choose
$$ \Omega = (0, 1) \times (0, 1) $$
and
\begin{align} (x, y)_{im+j} &= (i h, j h) & h &= 1/(m-1) & i,j \in \{0, 1, \dotsc, m-1 \} .
\end{align}

```python
%matplotlib inline
import numpy
from matplotlib import pyplot
pyplot.style.use('ggplot')

def laplacian2d_dense(h, f, g0):
    m = int(1/h + 1)
    c = numpy.linspace(0, 1, m)
    y, x = numpy.meshgrid(c, c)
    u0 = g0(x, y).flatten()
    rhs = f(x, y).flatten()
    A = numpy.zeros((m*m, m*m))
    def idx(i, j):
        return i*m + j
    for i in range(m):
        for j in range(m):
            row = idx(i, j)
            if i in (0, m-1) or j in (0, m-1):
                A[row, row] = 1
                rhs[row] = u0[row]
            else:
                cols = [idx(*pair) for pair in
                        [(i-1, j), (i, j-1), (i, j), (i, j+1), (i+1, j)]]
                stencil = 1/h**2 * numpy.array([-1, -1, 4, -1, -1])
                A[row, cols] = stencil
    return x, y, A, rhs

x, y, A, rhs = laplacian2d_dense(.1, lambda x,y: 0*x+1, lambda x,y: 0*x)

pyplot.spy(A);
```

```python
u = numpy.linalg.solve(A, rhs).reshape(x.shape)

pyplot.contourf(x, y, u)
pyplot.colorbar();
```

```python
import cProfile
prof = cProfile.Profile()
prof.enable()
x, y, A, rhs = laplacian2d_dense(.0125, lambda x,y: 0*x+1, lambda x,y: 0*x)
u = numpy.linalg.solve(A, rhs).reshape(x.shape)
prof.disable()
prof.print_stats(sort='tottime')
```

             50365 function calls in 2.706 seconds

       Ordered by: internal time

       ncalls  tottime  percall  cumtime  percall filename:lineno(function)
            1    2.623    2.623    2.623    2.623 linalg.py:300(solve)
            1    0.063    0.063    0.083    0.083 <ipython-input-1-bc43fafc383c>:6(laplacian2d_dense)
         6251    0.009    0.000    0.009    0.000 {built-in method numpy.core.multiarray.array}
         6241    0.007    0.000    0.010    0.000 <ipython-input-1-bc43fafc383c>:22(<listcomp>)
        37766    0.005    0.000    0.005    0.000 <ipython-input-1-bc43fafc383c>:13(idx)
            3    0.000    0.000    0.000    0.000 {built-in method builtins.compile}
            2    0.000    0.000    0.000    0.000 {method 'copy' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 function_base.py:25(linspace)
            3    0.000    0.000    2.706    0.902 interactiveshell.py:2880(run_code)
            3    0.000    0.000    0.000    0.000 codeop.py:132(__call__)
            2    0.000    0.000    0.000    0.000 stride_tricks.py:115(_broadcast_to)
            2    0.000    0.000    0.000    0.000 {method 'astype' of 'numpy.ndarray' objects}
            2    0.000    0.000    0.000    0.000 linalg.py:106(_makearray)
            2    0.000    0.000    0.000    0.000 {method 'flatten' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 function_base.py:4554(meshgrid)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.zeros}
            1    0.000    0.000    0.000    0.000 <ipython-input-3-e477be7765b2>:4(<lambda>)
            1    0.000    0.000    0.083    0.083 <ipython-input-3-e477be7765b2>:4(<module>)
            3    0.000    0.000    0.000    0.000 hooks.py:142(__call__)
            1    0.000    0.000    2.623    2.623 <ipython-input-3-e477be7765b2>:5(<module>)
            1    0.000    0.000    0.000    0.000 linalg.py:139(_commonType)
            1    0.000    0.000    0.000    0.000 linalg.py:209(_assertNdSquareness)
            3    0.000    0.000    0.000    0.000 ipstruct.py:125(__getattr__)
            3    0.000    0.000    0.000    0.000 {method 'reshape' of 'numpy.ndarray' objects}
            3    0.000    0.000    2.706    0.902 {built-in method builtins.exec}
            1    0.000    0.000    0.000    0.000 stride_tricks.py:195(broadcast_arrays)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.arange}
            1    0.000    0.000    0.000    0.000 linalg.py:198(_assertRankAtLeast2)
            1    0.000    0.000    0.000    0.000 function_base.py:4671(<listcomp>)
            2    0.000    0.000    0.000    0.000 linalg.py:124(_realType)
            1    0.000    0.000    0.000    0.000 stride_tricks.py:176(_broadcast_shape)
            1    0.000    0.000    0.000    0.000 linalg.py:101(get_linalg_error_extobj)
            4    0.000    0.000    0.000    0.000 numeric.py:534(asanyarray)
            1    0.000    0.000    0.000    0.000 <ipython-input-3-e477be7765b2>:6(<module>)
            2    0.000    0.000    0.000    0.000 function_base.py:213(iterable)
            1    0.000    0.000    0.000    0.000 function_base.py:13(_index_deprecate)
            3    0.000    0.000    0.000    0.000 interactiveshell.py:1069(user_global_ns)
            1    0.000    0.000    0.000    0.000 function_base.py:4684(<listcomp>)
            2    0.000    0.000    0.000    0.000 stride_tricks.py:251(<genexpr>)
            3    0.000    0.000    0.000    0.000 linalg.py:111(isComplexType)
            2    0.000    0.000    0.000    0.000 {built-in method builtins.any}
            2    0.000    0.000    0.000    0.000 {built-in method builtins.getattr}
            1    0.000    0.000    0.000    0.000 {built-in method builtins.max}
            6    0.000    0.000    0.000    0.000 stride_tricks.py:120(<genexpr>)
            1    0.000    0.000    0.000    0.000 stride_tricks.py:257(<listcomp>)
            2    0.000    0.000    0.000    0.000 numeric.py:463(asarray)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.result_type}
            1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
            3    0.000    0.000    0.000    0.000 hooks.py:207(pre_run_code_hook)
            1    0.000    0.000    0.000    0.000 {built-in method builtins.all}
            5    0.000    0.000    0.000    0.000 {built-in method builtins.issubclass}
            2    0.000    0.000    0.000    0.000 {built-in method builtins.len}
            2    0.000    0.000    0.000    0.000 {method 'get' of 'dict' objects}
            4    0.000    0.000    0.000    0.000 {method 'pop' of 'dict' objects}
            2    0.000    0.000    0.000    0.000 stride_tricks.py:25(_maybe_view_as_subclass)
            1    0.000    0.000    0.000    0.000 stride_tricks.py:247(<listcomp>)
            1    0.000    0.000    0.000    0.000 {method '__array_prepare__' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 {built-in method _operator.index}
            2    0.000    0.000    0.000    0.000 {built-in method builtins.iter}
            1    0.000    0.000    0.000    0.000 {built-in method builtins.min}

```python
import scipy.sparse as sp
import scipy.sparse.linalg

def laplacian2d(h, f, g0):
    m = int(1/h + 1)  # Number of elements in terms of nominal grid spacing h
    h = 1 / (m-1)     # Actual grid spacing
    c = numpy.linspace(0, 1, m)
    y, x = numpy.meshgrid(c, c)
    u0 = g0(x, y).flatten()
    rhs = f(x, y).flatten()
    A = sp.lil_matrix((m*m, m*m))
    def idx(i, j):
        return i*m + j
    mask = numpy.zeros_like(x, dtype=int)
    mask[1:-1,1:-1] = 1
    mask = mask.flatten()
    for i in range(m):
        for j in range(m):
            row = idx(i, j)
            stencili = numpy.array([idx(*pair) for pair in
                                    [(i-1, j), (i, j-1),
                                     (i, j),
                                     (i, j+1), (i+1, j)]])
            stencilw = 1/h**2 * numpy.array([-1, -1, 4, -1, -1])
            if mask[row] == 0: # Dirichlet boundary
                A[row, row] = 1
                rhs[row] = u0[row]
            else:
                smask = mask[stencili]
                cols = stencili[smask == 1]
                A[row, cols] = stencilw[smask == 1]
                bdycols = stencili[smask == 0]
                rhs[row] -= stencilw[smask == 0] @ u0[bdycols]
    return x, y, A.tocsr(), rhs

x, y, A, rhs = laplacian2d(.15, lambda x,y: 0*x+1, lambda x,y: 0*x)

pyplot.spy(A);
sp.linalg.norm(A - A.T)
```

```python
prof = cProfile.Profile()
prof.enable()
x, y, A, rhs = laplacian2d(.005, lambda x,y: 0*x+1, lambda x,y: 0*x)
u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
prof.disable()
prof.print_stats(sort='tottime')
```

             5362376 function calls in 4.868 seconds

       Ordered by: internal time

       ncalls  tottime  percall  cumtime  percall filename:lineno(function)
       158406    0.753    0.000    1.025    0.000 stride_tricks.py:115(_broadcast_to)
            1    0.752    0.752    4.644    4.644 <ipython-input-4-66fa325aa3de>:4(laplacian2d)
       516438    0.321    0.000    0.321    0.000 {built-in method numpy.core.multiarray.array}
        39601    0.283    0.000    0.283    0.000 {scipy.sparse._csparsetools.lil_fancy_set}
        40401    0.252    0.000    3.562    0.000 lil.py:333(__setitem__)
        79203    0.220    0.000    1.645    0.000 stride_tricks.py:195(broadcast_arrays)
            1    0.218    0.218    0.218    0.218 {built-in method scipy.sparse.linalg.dsolve._superlu.gssv}
        79202    0.167    0.000    0.349    0.000 sputils.py:331(_check_boolean)
        39601    0.160    0.000    1.592    0.000 sputils.py:351(_index_to_arrays)
        79203    0.158    0.000    0.167    0.000 stride_tricks.py:176(_broadcast_shape)
        39601    0.137    0.000    0.486    0.000 sputils.py:265(_unpack_index)
       834034    0.117    0.000    0.117    0.000 {built-in method builtins.isinstance}
        79202    0.101    0.000    0.273    0.000 shape_base.py:11(atleast_1d)
       118805    0.076    0.000    0.076    0.000 {built-in method builtins.hasattr}
       158406    0.070    0.000    0.121    0.000 {built-in method builtins.any}
        79203    0.069    0.000    1.094    0.000 stride_tricks.py:257(<listcomp>)
            1    0.067    0.067    0.067    0.067 lil.py:84(__init__)
       118804    0.063    0.000    0.089    0.000 <frozen importlib._bootstrap>:402(parent)
        39601    0.062    0.000    0.062    0.000 lil.py:486(_prepare_index_for_memoryview)
        39604    0.057    0.000    0.057    0.000 {method 'reshape' of 'numpy.ndarray' objects}
        40401    0.054    0.000    0.080    0.000 <ipython-input-4-66fa325aa3de>:20(<listcomp>)
       237609    0.052    0.000    0.085    0.000 base.py:1111(isspmatrix)
        39601    0.052    0.000    0.056    0.000 sputils.py:293(_check_ellipsis)
       396016    0.051    0.000    0.051    0.000 stride_tricks.py:120(<genexpr>)
       158406    0.051    0.000    0.051    0.000 stride_tricks.py:251(<genexpr>)
       158406    0.048    0.000    0.048    0.000 stride_tricks.py:25(_maybe_view_as_subclass)
        39603    0.047    0.000    0.052    0.000 numeric.py:2135(isscalar)
       158406    0.046    0.000    0.069    0.000 function_base.py:213(iterable)
        79203    0.045    0.000    0.079    0.000 stride_tricks.py:247(<listcomp>)
       242406    0.039    0.000    0.039    0.000 <ipython-input-4-66fa325aa3de>:12(idx)
        39603    0.036    0.000    0.102    0.000 sputils.py:183(isscalarlike)
       118804    0.033    0.000    0.110    0.000 <frozen importlib._bootstrap>:989(_handle_fromlist)
       278821    0.031    0.000    0.031    0.000 {built-in method builtins.len}
       118804    0.026    0.000    0.026    0.000 {method 'rpartition' of 'str' objects}
       158406    0.024    0.000    0.024    0.000 {built-in method builtins.iter}
        79206    0.024    0.000    0.096    0.000 numeric.py:534(asanyarray)
        79203    0.021    0.000    0.049    0.000 {built-in method builtins.all}
        80807    0.013    0.000    0.013    0.000 base.py:100(get_shape)
        79206    0.012    0.000    0.012    0.000 {method 'pop' of 'dict' objects}
        39614    0.011    0.000    0.067    0.000 numeric.py:463(asarray)
        79202    0.011    0.000    0.011    0.000 {method 'append' of 'list' objects}
        39601    0.011    0.000    0.015    0.000 sputils.py:227(isdense)
            1    0.010    0.010    0.043    0.043 lil.py:463(tocsr)
            1    0.006    0.006    4.650    4.650 <ipython-input-5-8cdfec67b31c>:3(<module>)
        80802    0.006    0.000    0.006    0.000 {method 'extend' of 'list' objects}
            1    0.004    0.004    0.006    0.006 lil.py:464(<listcomp>)
          800    0.002    0.000    0.002    0.000 {built-in method scipy.sparse._csparsetools.lil_insert}
            2    0.002    0.001    0.002    0.001 {method 'copy' of 'numpy.ndarray' objects}
            3    0.000    0.000    0.000    0.000 {method 'flatten' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 <ipython-input-5-8cdfec67b31c>:3(<lambda>)
            1    0.000    0.000    0.000    0.000 {built-in method builtins.sum}
            2    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.empty}
            3    0.000    0.000    0.000    0.000 {built-in method builtins.compile}
            1    0.000    0.000    0.000    0.000 {built-in method scipy.sparse._sparsetools.csr_has_sorted_indices}
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.empty_like}
            1    0.000    0.000    0.000    0.000 {method 'cumsum' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 function_base.py:25(linspace)
            1    0.000    0.000    0.218    0.218 linsolve.py:62(spsolve)
            3    0.000    0.000    4.868    1.623 interactiveshell.py:2880(run_code)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.copyto}
            1    0.000    0.000    0.000    0.000 compressed.py:128(check_format)
            3    0.000    0.000    0.000    0.000 getlimits.py:507(__init__)
            3    0.000    0.000    0.000    0.000 sputils.py:119(get_index_dtype)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.concatenate}
            1    0.000    0.000    0.000    0.000 compressed.py:25(__init__)
            1    0.000    0.000    0.002    0.002 function_base.py:4554(meshgrid)
            3    0.000    0.000    0.000    0.000 codeop.py:132(__call__)
            2    0.000    0.000    0.000    0.000 sputils.py:188(isintlike)
            1    0.000    0.000    0.000    0.000 base.py:142(asfptype)
            1    0.000    0.000    0.000    0.000 numeric.py:87(zeros_like)
            2    0.000    0.000    0.000    0.000 base.py:70(__init__)
            2    0.000    0.000    0.000    0.000 sputils.py:200(isshape)
            1    0.000    0.000    0.218    0.218 <ipython-input-5-8cdfec67b31c>:4(<module>)
            3    0.000    0.000    0.000    0.000 ipstruct.py:125(__getattr__)
            1    0.000    0.000    0.000    0.000 sputils.py:95(getdtype)
            3    0.000    0.000    0.000    0.000 hooks.py:142(__call__)
            5    0.000    0.000    0.000    0.000 compressed.py:100(getnnz)
            1    0.000    0.000    0.000    0.000 function_base.py:4671(<listcomp>)
            1    0.000    0.000    0.000    0.000 fromnumeric.py:55(_wrapfunc)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.arange}
            1    0.000    0.000    0.000    0.000 compressed.py:1065(prune)
            8    0.000    0.000    0.000    0.000 cycler.py:227(<genexpr>)
            1    0.000    0.000    0.002    0.002 function_base.py:4684(<listcomp>)
            3    0.000    0.000    4.868    1.623 {built-in method builtins.exec}
            1    0.000    0.000    0.000    0.000 compressed.py:1025(__get_sorted)
            2    0.000    0.000    0.000    0.000 fromnumeric.py:2584(ndim)
            1    0.000    0.000    0.000    0.000 sputils.py:91(to_native)
            1    0.000    0.000    0.000    0.000 data.py:22(__init__)
            1    0.000    0.000    0.000    0.000 lil.py:130(set_shape)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.zeros}
            2    0.000    0.000    0.000    0.000 _util.py:128(_prune_array)
            1    0.000    0.000    0.000    0.000 base.py:77(set_shape)
            3    0.000    0.000    0.000    0.000 getlimits.py:532(max)
            1    0.000    0.000    0.000    0.000 fromnumeric.py:2053(cumsum)
            1    0.000    0.000    0.000    0.000 {method 'ravel' of 'numpy.ndarray' objects}
            5    0.000    0.000    0.000    0.000 base.py:193(nnz)
            1    0.000    0.000    0.000    0.000 <ipython-input-5-8cdfec67b31c>:5(<module>)
            1    0.000    0.000    0.000    0.000 {method 'newbyteorder' of 'numpy.dtype' objects}
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.result_type}
            3    0.000    0.000    0.000    0.000 interactiveshell.py:1069(user_global_ns)
            1    0.000    0.000    0.000    0.000 {built-in method builtins.getattr}
            2    0.000    0.000    0.000    0.000 csc.py:220(isspmatrix_csc)
            1    0.000    0.000    0.000    0.000 base.py:562(__getattr__)
            1    0.000    0.000    0.000    0.000 function_base.py:13(_index_deprecate)
            1    0.000    0.000    0.000    0.000 {method 'astype' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.promote_types}
            2    0.000    0.000    0.000    0.000 {built-in method builtins.max}
            3    0.000    0.000    0.000    0.000 csr.py:231(_swap)
            1    0.000    0.000    0.000    0.000 csr.py:458(isspmatrix_csr)
            1    0.000    0.000    0.000    0.000 compressed.py:1056(sort_indices)
            3    0.000    0.000    0.000    0.000 data.py:25(_get_dtype)
            3    0.000    0.000    0.000    0.000 hooks.py:207(pre_run_code_hook)
            1    0.000    0.000    0.000    0.000 {built-in method _operator.index}
            1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}

## A manufactured solution

```python
class mms0:
    def u(x, y):
        return x*numpy.exp(-x)*numpy.tanh(y)
    def grad_u(x, y):
        return numpy.array([(1 - x)*numpy.exp(-x)*numpy.tanh(y),
                            x*numpy.exp(-x)*(1 - numpy.tanh(y)**2)])
    def laplacian_u(x, y):
        return ((2 - x)*numpy.exp(-x)*numpy.tanh(y)
                - 2*x*numpy.exp(-x)*(numpy.tanh(y)**2 - 1)*numpy.tanh(y))
    def grad_u_dot_normal(x, y, n):
        return grad_u(x, y) @ n

x, y, A, rhs = laplacian2d(.02, mms0.laplacian_u, mms0.u)
u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
print(u.shape, numpy.linalg.norm((u - mms0.u(x,y)).flatten(), numpy.inf))
pyplot.contourf(x, y, u)
pyplot.colorbar()
pyplot.title('Numeric solution')
pyplot.figure()
pyplot.contourf(x, y, u - mms0.u(x, y))
pyplot.colorbar()
pyplot.title('Error');
```

```python
hs = numpy.logspace(-2, -.5, 12)
def mms_error(h):
    x, y, A, rhs = laplacian2d(h, mms0.laplacian_u, mms0.u)
    u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
    return numpy.linalg.norm((u - mms0.u(x, y)).flatten(), numpy.inf)

pyplot.loglog(hs, [mms_error(h) for h in hs], 'o', label='numeric error')
pyplot.loglog(hs, hs**1/100, label='$h^1/100$')
pyplot.loglog(hs, hs**2/100, label='$h^2/100$')
pyplot.legend();
```

# Neumann boundary conditions

Recall that in 1D, we would reflect the solution into ghost points according to

$$ u_{-i} = u_i - (x_i - x_{-i}) g_1(x_0, y) $$

and similarly for the right boundary and in the $y$ direction.  After this, we (optionally) scale the row in the matrix for symmetry and shift the known parts to the right hand side.  Below, we implement the reflected symmetry, but not the inhomogeneous contribution or rescaling of the matrix row.

```python
def laplacian2d_bc(h, f, g0, dirichlet=((),())):
    m = int(1/h + 1)  # Number of elements in terms of nominal grid spacing h
    h = 1 / (m-1)     # Actual grid spacing
    c = numpy.linspace(0, 1, m)
    y, x = numpy.meshgrid(c, c)
    u0 = g0(x, y).flatten()
    rhs = f(x, y).flatten()
    ai = []
    aj = []
    av = []
    def idx(i, j):
        i = (m-1) - abs(m-1 - abs(i))
        j = (m-1) - abs(m-1 - abs(j))
        return i*m + j
    mask = numpy.ones_like(x, dtype=int)
    mask[dirichlet[0],:] = 0
    mask[:,dirichlet[1]] = 0
    mask = mask.flatten()
    for i in range(m):
        for j in range(m):
            row = idx(i, j)
            stencili = numpy.array([idx(*pair) for pair in [(i-1, j), (i, j-1), (i, j), (i, j+1), (i+1, j)]])
            stencilw = 1/h**2 * numpy.array([-1, -1, 4, -1, -1])
            if mask[row] == 0: # Dirichlet boundary
                ai.append(row)
                aj.append(row)
                av.append(1)
                rhs[row] = u0[row]
            else:
                smask = mask[stencili]
                ai += [row]*sum(smask)
                aj += stencili[smask == 1].tolist()
                av += stencilw[smask == 1].tolist()
                bdycols = stencili[smask == 0]
                rhs[row] -= stencilw[smask == 0] @ u0[bdycols]
    A = sp.csr_matrix((av, (ai, aj)), shape=(m*m,m*m))
    return x, y, A, rhs

x, y, A, rhs = laplacian2d_bc(.05, lambda x,y: 0*x+1,
                              lambda x,y: 0*x, dirichlet=((0,),()))
u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
print(sp.linalg.eigs(A, which='SM')[0])
pyplot.contourf(x, y, u)
pyplot.colorbar();
```

```python
# We used a different technique for assembling the sparse matrix.
# This is faster with scipy.sparse, but may be worse for other sparse matrix packages, such as PETSc.

prof = cProfile.Profile()
prof.enable()
x, y, A, rhs = laplacian2d_bc(.005, lambda x,y: 0*x+1, lambda x,y: 0*x)
u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
prof.disable()
prof.print_stats(sort='tottime')
```

             1454850 function calls (1454848 primitive calls) in 1.504 seconds

       Ordered by: internal time

       ncalls  tottime  percall  cumtime  percall filename:lineno(function)
            1    0.660    0.660    1.233    1.233 <ipython-input-8-7866e0ac5ad1>:1(laplacian2d_bc)
            1    0.264    0.264    0.264    0.264 {built-in method scipy.sparse.linalg.dsolve._superlu.gssv}
       242406    0.168    0.000    0.231    0.000 <ipython-input-8-7866e0ac5ad1>:11(idx)
        80841    0.152    0.000    0.152    0.000 {built-in method numpy.core.multiarray.array}
        40401    0.097    0.000    0.097    0.000 {built-in method builtins.sum}
       969624    0.063    0.000    0.063    0.000 {built-in method builtins.abs}
        40401    0.056    0.000    0.236    0.000 <ipython-input-8-7866e0ac5ad1>:22(<listcomp>)
        80802    0.024    0.000    0.024    0.000 {method 'tolist' of 'numpy.ndarray' objects}
            1    0.009    0.009    0.009    0.009 {built-in method numpy.core.multiarray.lexsort}
            1    0.006    0.006    1.239    1.239 <ipython-input-9-b5abe0e23855>:6(<module>)
            1    0.002    0.002    0.012    0.012 coo.py:460(_sum_duplicates)
            1    0.001    0.001    0.001    0.001 {method 'reduceat' of 'numpy.ufunc' objects}
            1    0.001    0.001    0.001    0.001 {built-in method scipy.sparse._sparsetools.coo_tocsr}
            2    0.001    0.000    0.001    0.000 {method 'copy' of 'numpy.ndarray' objects}
            4    0.001    0.000    0.001    0.000 {method 'reduce' of 'numpy.ufunc' objects}
            3    0.000    0.000    0.000    0.000 {method 'flatten' of 'numpy.ndarray' objects}
            3    0.000    0.000    0.000    0.000 {built-in method builtins.compile}
            1    0.000    0.000    0.000    0.000 {method 'nonzero' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 {built-in method scipy.sparse._sparsetools.csr_has_sorted_indices}
            1    0.000    0.000    0.000    0.000 <ipython-input-9-b5abe0e23855>:6(<lambda>)
            1    0.000    0.000    0.001    0.001 coo.py:212(_check)
            1    0.000    0.000    0.000    0.000 function_base.py:25(linspace)
            3    0.000    0.000    0.000    0.000 compressed.py:128(check_format)
            1    0.000    0.000    0.264    0.264 linsolve.py:62(spsolve)
            7    0.000    0.000    0.000    0.000 getlimits.py:507(__init__)
          3/1    0.000    0.000    0.040    0.040 compressed.py:25(__init__)
            7    0.000    0.000    0.000    0.000 sputils.py:119(get_index_dtype)
            1    0.000    0.000    0.027    0.027 coo.py:118(__init__)
            1    0.000    0.000    0.013    0.013 coo.py:301(tocsr)
            3    0.000    0.000    1.504    0.501 interactiveshell.py:2880(run_code)
            1    0.000    0.000    0.001    0.001 function_base.py:4554(meshgrid)
            2    0.000    0.000    0.000    0.000 stride_tricks.py:115(_broadcast_to)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.copyto}
            3    0.000    0.000    0.000    0.000 codeop.py:132(__call__)
            3    0.000    0.000    0.000    0.000 compressed.py:1065(prune)
            5    0.000    0.000    0.000    0.000 base.py:77(set_shape)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.concatenate}
            4    0.000    0.000    0.000    0.000 coo.py:186(getnnz)
            1    0.000    0.000    0.000    0.000 function_base.py:5100(append)
            3    0.000    0.000    0.000    0.000 sputils.py:200(isshape)
           17    0.000    0.000    0.000    0.000 base.py:193(nnz)
           23    0.000    0.000    0.000    0.000 numeric.py:463(asarray)
            1    0.000    0.000    0.000    0.000 stride_tricks.py:195(broadcast_arrays)
           45    0.000    0.000    0.000    0.000 {built-in method builtins.len}
            4    0.000    0.000    0.000    0.000 sputils.py:91(to_native)
            1    0.000    0.000    0.264    0.264 <ipython-input-9-b5abe0e23855>:7(<module>)
            4    0.000    0.000    0.000    0.000 base.py:70(__init__)
            3    0.000    0.000    0.000    0.000 {method 'reshape' of 'numpy.ndarray' objects}
            3    0.000    0.000    1.503    0.501 {built-in method builtins.exec}
           22    0.000    0.000    0.000    0.000 {built-in method builtins.isinstance}
            1    0.000    0.000    0.012    0.012 coo.py:449(sum_duplicates)
           13    0.000    0.000    0.000    0.000 compressed.py:100(getnnz)
            7    0.000    0.000    0.000    0.000 getlimits.py:532(max)
            3    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.empty_like}
            3    0.000    0.000    0.000    0.000 hooks.py:142(__call__)
            4    0.000    0.000    0.000    0.000 {built-in method builtins.max}
            1    0.000    0.000    0.000    0.000 numeric.py:197(ones_like)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.arange}
            3    0.000    0.000    0.000    0.000 ipstruct.py:125(__getattr__)
            3    0.000    0.000    0.000    0.000 {built-in method builtins.hasattr}
            4    0.000    0.000    0.000    0.000 base.py:1111(isspmatrix)
            4    0.000    0.000    0.000    0.000 data.py:22(__init__)
            1    0.000    0.000    0.000    0.000 stride_tricks.py:176(_broadcast_shape)
            3    0.000    0.000    0.000    0.000 {method 'ravel' of 'numpy.ndarray' objects}
            2    0.000    0.000    0.000    0.000 <frozen importlib._bootstrap>:402(parent)
           16    0.000    0.000    0.000    0.000 base.py:100(get_shape)
            1    0.000    0.000    0.000    0.000 function_base.py:4671(<listcomp>)
            1    0.000    0.000    0.000    0.000 fromnumeric.py:55(_wrapfunc)
            4    0.000    0.000    0.000    0.000 {method 'newbyteorder' of 'numpy.dtype' objects}
            1    0.000    0.000    0.000    0.000 compressed.py:1025(__get_sorted)
            1    0.000    0.000    0.000    0.000 base.py:142(asfptype)
            1    0.000    0.000    0.013    0.013 base.py:249(asformat)
            1    0.000    0.000    0.000    0.000 <ipython-input-9-b5abe0e23855>:8(<module>)
            2    0.000    0.000    0.000    0.000 numeric.py:2135(isscalar)
            1    0.000    0.000    0.000    0.000 sputils.py:20(upcast)
            6    0.000    0.000    0.000    0.000 _util.py:128(_prune_array)
            6    0.000    0.000    0.000    0.000 numeric.py:534(asanyarray)
            3    0.000    0.000    0.000    0.000 {method 'astype' of 'numpy.ndarray' objects}
            2    0.000    0.000    0.000    0.000 {method 'min' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.empty}
            2    0.000    0.000    0.000    0.000 compressed.py:117(_set_self)
            2    0.000    0.000    0.000    0.000 sputils.py:183(isscalarlike)
            5    0.000    0.000    0.000    0.000 data.py:25(_get_dtype)
            1    0.000    0.000    0.000    0.000 function_base.py:13(_index_deprecate)
            2    0.000    0.000    0.000    0.000 _methods.py:28(_amin)
            2    0.000    0.000    0.000    0.000 {method 'max' of 'numpy.ndarray' objects}
            1    0.000    0.000    0.000    0.000 compressed.py:1056(sort_indices)
            1    0.000    0.000    0.000    0.000 base.py:562(__getattr__)
            1    0.000    0.000    0.001    0.001 function_base.py:4684(<listcomp>)
            6    0.000    0.000    0.000    0.000 stride_tricks.py:120(<genexpr>)
            1    0.000    0.000    0.000    0.000 stride_tricks.py:247(<listcomp>)
            1    0.000    0.000    0.000    0.000 stride_tricks.py:257(<listcomp>)
            1    0.000    0.000    0.000    0.000 fromnumeric.py:1487(nonzero)
            2    0.000    0.000    0.000    0.000 _methods.py:25(_amax)
            3    0.000    0.000    0.000    0.000 interactiveshell.py:1069(user_global_ns)
            2    0.000    0.000    0.000    0.000 <frozen importlib._bootstrap>:989(_handle_fromlist)
            2    0.000    0.000    0.000    0.000 sputils.py:188(isintlike)
            2    0.000    0.000    0.000    0.000 sputils.py:227(isdense)
            2    0.000    0.000    0.000    0.000 csc.py:220(isspmatrix_csc)
            2    0.000    0.000    0.000    0.000 stride_tricks.py:251(<genexpr>)
            2    0.000    0.000    0.000    0.000 function_base.py:213(iterable)
            1    0.000    0.000    0.000    0.000 fromnumeric.py:1380(ravel)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.result_type}
            2    0.000    0.000    0.000    0.000 {built-in method builtins.getattr}
            2    0.000    0.000    0.000    0.000 {method 'rpartition' of 'str' objects}
            1    0.000    0.000    0.000    0.000 csr.py:458(isspmatrix_csr)
            2    0.000    0.000    0.000    0.000 stride_tricks.py:25(_maybe_view_as_subclass)
            1    0.000    0.000    0.000    0.000 {built-in method numpy.core.multiarray.promote_types}
            1    0.000    0.000    0.000    0.000 {built-in method builtins.all}
            2    0.000    0.000    0.000    0.000 {built-in method builtins.any}
            2    0.000    0.000    0.000    0.000 {built-in method builtins.iter}
            1    0.000    0.000    0.000    0.000 {method 'get' of 'dict' objects}
            4    0.000    0.000    0.000    0.000 {method 'pop' of 'dict' objects}
            9    0.000    0.000    0.000    0.000 csr.py:231(_swap)
            1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
            3    0.000    0.000    0.000    0.000 hooks.py:207(pre_run_code_hook)
            1    0.000    0.000    0.000    0.000 {built-in method _operator.index}
            1    0.000    0.000    0.000    0.000 {built-in method builtins.hash}

# Variable coefficients

In physical systems, it is common for equations to be given in **divergence form** (sometimes called **conservative form**),
$$ -\nabla\cdot \Big( \kappa(x,y) \nabla u \Big) = f(x,y) . $$
This can be converted to **non-divergence form**,
$$ - \kappa(x,y) \nabla\cdot \nabla u - \nabla \kappa(x,y) \cdot \nabla u = f(x,y) . $$

* What assumptions did we just make on $\kappa(x,y)$?

```python
def laplacian2d_nondiv(h, f, kappa, grad_kappa, g0, dirichlet=((),())):
    m = int(1/h + 1)  # Number of elements in terms of nominal grid spacing h
    h = 1 / (m-1)     # Actual grid spacing
    c = numpy.linspace(0, 1, m)
    y, x = numpy.meshgrid(c, c)
    u0 = g0(x, y).flatten()
    rhs = f(x, y).flatten()
    ai = []
    aj = []
    av = []
    def idx(i, j):
        i = (m-1) - abs(m-1 - abs(i))
        j = (m-1) - abs(m-1 - abs(j))
        return i*m + j
    mask = numpy.ones_like(x, dtype=int)
    mask[dirichlet[0],:] = 0
    mask[:,dirichlet[1]] = 0
    mask = mask.flatten()
    for i in range(m):
        for j in range(m):
            row = idx(i, j)
            stencili = numpy.array([idx(*pair) for pair in [(i-1, j), (i, j-1), (i, j), (i, j+1), (i+1, j)]])
            stencilw = kappa(i*h, j*h)/h**2 * numpy.array([-1, -1, 4, -1, -1])
            if grad_kappa is None:
                gk = 1/h * numpy.array([kappa((i+.5)*h,j*h) - kappa((i-.5)*h,j*h),
                                        kappa(i*h,(j+.5)*h) - kappa(i*h,(j-.5)*h)])
            else:
                gk = grad_kappa(i*h, j*h)
            stencilw -= gk[0] / (2*h) * numpy.array([-1, 0, 0, 0, 1])
            stencilw -= gk[1] / (2*h) * numpy.array([0, -1, 0, 1, 0])
            if mask[row] == 0: # Dirichlet boundary
                ai.append(row)
                aj.append(row)
                av.append(1)
                rhs[row] = u0[row]
            else:
                smask = mask[stencili]
                ai += [row]*sum(smask)
                aj += stencili[smask == 1].tolist()
                av += stencilw[smask == 1].tolist()
                bdycols = stencili[smask == 0]
                rhs[row] -= stencilw[smask == 0] @ u0[bdycols]
    A = sp.csr_matrix((av, (ai, aj)), shape=(m*m,m*m))
    return x, y, A, rhs

def kappa(x, y):
    #return 1 - 2*(x-.5)**2 - 2*(y-.5)**2
    return 1e-2 + 2*(x-.5)**2 + 2*(y-.5)**2
def grad_kappa(x, y):
    #return -4*(x-.5), -4*(y-.5)
    return 4*(x-.5), 4*(y-.5)

pyplot.contourf(x, y, kappa(x,y))
pyplot.colorbar();
```

```python
x, y, A, rhs = laplacian2d_nondiv(.05, lambda x,y: 0*x+1,
                                  kappa, grad_kappa,
                                  lambda x,y: 0*x, dirichlet=((0,-1),()))
u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, u)
pyplot.colorbar();
```

```python
x, y, A, rhs = laplacian2d_nondiv(.05, lambda x,y: 0*x,
                                  kappa, grad_kappa,
                                  lambda x,y: x, dirichlet=((0,-1),()))
u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, u)
pyplot.colorbar();
```

```python
def laplacian2d_div(h, f, kappa, g0, dirichlet=((),())):
    m = int(1/h + 1)  # Number of elements in terms of nominal grid spacing h
    h = 1 / (m-1)     # Actual grid spacing
    c = numpy.linspace(0, 1, m)
    y, x = numpy.meshgrid(c, c)
    u0 = g0(x, y).flatten()
    rhs = f(x, y).flatten()
    ai = []
    aj = []
    av = []
    def idx(i, j):
        i = (m-1) - abs(m-1 - abs(i))
        j = (m-1) - abs(m-1 - abs(j))
        return i*m + j
    mask = numpy.ones_like(x, dtype=int)
    mask[dirichlet[0],:] = 0
    mask[:,dirichlet[1]] = 0
    mask = mask.flatten()
    for i in range(m):
        for j in range(m):
            row = idx(i, j)
            stencili = numpy.array([idx(*pair) for pair in [(i-1, j), (i, j-1), (i, j), (i, j+1), (i+1, j)]])
            stencilw = 1/h**2 * (  kappa((i-.5)*h, j*h) * numpy.array([-1, 0, 1, 0, 0])
                                 + kappa(i*h, (j-.5)*h) * numpy.array([0, -1, 1, 0, 0])
                                 + kappa(i*h, (j+.5)*h) * numpy.array([0, 0, 1, -1, 0])
                                 + kappa((i+.5)*h, j*h) * numpy.array([0, 0, 1, 0, -1]))
            if mask[row] == 0: # Dirichlet boundary
                ai.append(row)
                aj.append(row)
                av.append(1)
                rhs[row] = u0[row]
            else:
                smask = mask[stencili]
                ai += [row]*sum(smask)
                aj += stencili[smask == 1].tolist()
                av += stencilw[smask == 1].tolist()
                bdycols = stencili[smask == 0]
                rhs[row] -= stencilw[smask == 0] @ u0[bdycols]
    A = sp.csr_matrix((av, (ai, aj)), shape=(m*m,m*m))
    return x, y, A, rhs

x, y, A, rhs = laplacian2d_div(.05, lambda x,y: 0*x+1,
                                  kappa,
                                  lambda x,y: 0*x, dirichlet=((0,-1),()))
u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, u)
pyplot.colorbar();
```

```python
x, y, A, rhs = laplacian2d_div(.05, lambda x,y: 0*x,
                                  kappa,
                                  lambda x,y: x, dirichlet=((0,-1),()))
u = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, u)
pyplot.colorbar();
```

```python
x, y, A, rhs = laplacian2d_nondiv(.05, lambda x,y: 0*x+1,
                                  kappa, grad_kappa,
                                  lambda x,y: 0*x, dirichlet=((0,-1),()))
u_nondiv = sp.linalg.spsolve(A, rhs).reshape(x.shape)

x, y, A, rhs = laplacian2d_div(.05, lambda x,y: 0*x+1,
                                  kappa,
                                  lambda x,y: 0*x, dirichlet=((0,-1),()))
u_div = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, u_nondiv - u_div)
pyplot.colorbar();
```

```python
class mms1:
    def __init__(self):
        import sympy
        x, y = sympy.symbols('x y')
        uexpr = x*sympy.exp(-2*x) * sympy.tanh(1.2*y+.1)
        kexpr = 1e-2 + 2*(x-.42)**2 + 2*(y-.51)**2
        self.u = sympy.lambdify((x,y), uexpr)
        self.kappa = sympy.lambdify((x,y), kexpr)
        def grad_kappa(xx, yy):
            kx = sympy.lambdify((x,y), sympy.diff(kexpr, x))
            ky = sympy.lambdify((x,y), sympy.diff(kexpr, y))
            return kx(xx, yy), ky(xx, yy)
        self.grad_kappa = grad_kappa
        self.div_kappa_grad_u = sympy.lambdify((x,y),
                                              -(  sympy.diff(kexpr * sympy.diff(uexpr, x), x)
                                                + sympy.diff(kexpr * sympy.diff(uexpr, y), y)))

mms = mms1()
x, y, A, rhs = laplacian2d_nondiv(.05, mms.div_kappa_grad_u,
                                  mms.kappa, mms.grad_kappa,
                                  mms.u, dirichlet=((0,-1),(0,-1)))
u_nondiv = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, u_nondiv)
pyplot.colorbar()
numpy.linalg.norm((u_nondiv - mms.u(x, y)).flatten(), numpy.inf)
```

```python
x, y, A, rhs = laplacian2d_div(.05, mms.div_kappa_grad_u,
                                  mms.kappa,
                                  mms.u, dirichlet=((0,-1),(0,-1)))
u_div = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, u_div)
pyplot.colorbar()
numpy.linalg.norm((u_div - mms.u(x, y)).flatten(), numpy.inf)
```

```python
def mms_error(h):
    x, y, A, rhs = laplacian2d_nondiv(h, mms.div_kappa_grad_u,
                                  mms.kappa, mms.grad_kappa,
                                  mms.u, dirichlet=((0,-1),(0,-1)))
    u_nondiv = sp.linalg.spsolve(A, rhs).flatten()
    x, y, A, rhs = laplacian2d_div(h, mms.div_kappa_grad_u,
                                  mms.kappa, mms.u, dirichlet=((0,-1),(0,-1)))
    u_div = sp.linalg.spsolve(A, rhs).flatten()
    u_exact = mms.u(x, y).flatten()
    return numpy.linalg.norm(u_nondiv - u_exact, numpy.inf), numpy.linalg.norm(u_div - u_exact, numpy.inf)

hs = numpy.logspace(-1.5, -.5, 10)
errors = numpy.array([mms_error(h) for h in hs])
pyplot.loglog(hs, errors[:,0], 'o', label='nondiv')
pyplot.loglog(hs, errors[:,1], 's', label='div')
pyplot.plot(hs, hs**2, label='$h^2$')
pyplot.legend();
```

```python
#kappablob = lambda x,y: .01 + ((x-.5)**2 + (y-.5)**2 < .125)
def kappablob(x, y):
    #return .01 + ((x-.5)**2 + (y-.5)**2 < .125)
    return .01 + (numpy.abs(x-.505) < .25) # + (numpy.abs(y-.5) < .25)
x, y, A, rhs = laplacian2d_div(.02, lambda x,y: 0*x, kappablob,
                                  lambda x,y:x, dirichlet=((0,-1),()))
u_div = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, kappablob(x, y))
pyplot.colorbar();
pyplot.figure()
pyplot.contourf(x, y, u_div, 10)
pyplot.colorbar();
```

```python
x, y, A, rhs = laplacian2d_nondiv(.01, lambda x,y: 0*x, kappablob, None,
                                  lambda x,y:x, dirichlet=((0,-1),()))
u_nondiv = sp.linalg.spsolve(A, rhs).reshape(x.shape)
pyplot.contourf(x, y, u_nondiv, 10)
pyplot.colorbar();
```

## Weak forms

When we write

$$ {\huge "} - \nabla\cdot \big( \kappa \nabla u \big) = 0 {\huge "} \text{ on } \Omega $$

where $\kappa$ is a discontinuous function, that's not exactly what we mean the derivative of that discontinuous function doesn't exist.  Formally, however, let us multiply by a "test function" $v$ and integrate,

\begin{split}
  - \int_\Omega v \nabla\cdot \big( \kappa \nabla u \big) = 0 & \text{ for all } v \\
  \int_\Omega \nabla v \cdot \kappa \nabla u = \int_{\partial \Omega} v \kappa \nabla u \cdot \hat n & \text{ for all } v
\end{split}

where we have used integration by parts.  This is called the **weak form** of the PDE and will be what we actually discretize using finite element methods.  All the terms make sense when $\kappa$ is discontinuous.  Now suppose our domain is decomposed into two disjoint sub domains $$\overline{\Omega_1 \cup \Omega_2} = \overline\Omega $$
with interface $$\Gamma = \overline\Omega_1 \cap \overline\Omega_2$$ and $\kappa_1$ is continuous on $\overline\Omega_1$ and $\kappa_2$ is continuous on $\overline\Omega_2$, but possibly $\kappa_1(x) \ne \kappa_2(x)$ for $x \in \Gamma$,

\begin{split}
\int_\Omega \nabla v \cdot \kappa \nabla u &= \int_{\Omega_1} \nabla v \cdot \kappa_1\nabla u + \int_{\Omega_2} \nabla v \cdot \kappa_2 \nabla u \\
&= -\int_{\Omega_1} v \nabla\cdot \big(\kappa_1 \nabla u \big) + \int_{\partial \Omega_1} v \kappa_1 \nabla u \cdot \hat n \\
&\qquad -\int_{\Omega_2} v \nabla\cdot \big(\kappa_2 \nabla u \big) + \int_{\partial \Omega_2} v \kappa_2 \nabla u \cdot \hat n \\
&= -\int_{\Omega} v \nabla\cdot \big(\kappa \nabla u \big) + \int_{\partial \Omega} v \kappa \nabla u \cdot \hat n + \int_{\Gamma} v (\kappa_1 - \kappa_2) \nabla u\cdot \hat n .
\end{split}

* Which direction is $\hat n$ for the integral over $\Gamma$?
* Does it matter what we choose for the value of $\kappa$ on $\Gamma$ in the volume integral?

When $\kappa$ is continuous, the jump term vanishes and we recover the **strong form**
$$ - \nabla\cdot \big( \kappa \nabla u \big) = 0 \text{ on } \Omega . $$
But if $\kappa$ is discontinuous, we would need to augment this with a jump condition ensuring that the flux $-\kappa \nabla u$ is continuous.  We could go add this condition to our FD code to recover convergence in case of discontinuous $\kappa$, but it is messy.

## Nonlinear problems

Let's consider the nonlinear problem
$$ -\nabla \cdot \big(\underbrace{(1 + u^2)}_{\kappa(u)} \nabla u \big) = f \text{ on } (0,1)^2 $$
subject to Dirichlet boundary conditions.  We will discretize the divergence form and thus will need
$\kappa(u)$ evaluated at staggered points $(i-1/2,j)$, $(i,j-1/2)$, etc.  We will calculate these by averaging
$$ u_{i-1/2,j} = \frac{u_{i-1,j} + u_{i,j}}{2} $$
and similarly for the other staggered directions.
To use a Newton method, we also need the derivatives
$$ \frac{\partial \kappa_{i-1/2,j}}{\partial u_{i,j}} = 2 u_{i-1/2,j} \frac{\partial u_{i-1/2,j}}{\partial u_{i,j}} = u_{i-1/2,j} . $$

In the function below, we compute both the residual
$$F(u) = -\nabla\cdot \kappa(u) \nabla u - f(x,y)$$
and its Jacobian
$$J(u) = \frac{\partial F}{\partial u} . $$

```python
def hgrid(h):
    m = int(1/h + 1)  # Number of elements in terms of nominal grid spacing h
    h = 1 / (m-1)     # Actual grid spacing
    c = numpy.linspace(0, 1, m)
    y, x = numpy.meshgrid(c, c)
    return x, y

def nonlinear2d_div(h, x, y, u, forcing, g0, dirichlet=((),())):
    m = x.shape[0]
    u0 = g0(x, y).flatten()
    F = -forcing(x, y).flatten()
    ai = []
    aj = []
    av = []
    def idx(i, j):
        i = (m-1) - abs(m-1 - abs(i))
        j = (m-1) - abs(m-1 - abs(j))
        return i*m + j
    mask = numpy.ones_like(x, dtype=bool)
    mask[dirichlet[0],:] = False
    mask[:,dirichlet[1]] = False
    mask = mask.flatten()
    u = u.flatten()
    F[mask == False] = u[mask == False] - u0[mask == False]
    u[mask == False] = u0[mask == False]
    for i in range(m):
        for j in range(m):
            row = idx(i, j)
            stencili = numpy.array([idx(*pair) for pair in [(i-1, j), (i, j-1), (i, j), (i, j+1), (i+1, j)]])
            # Stencil to evaluate gradient at four staggered points
            grad = numpy.array([[-1, 0, 1, 0, 0],
                                [0, -1, 1, 0, 0],
                                [0, 0, -1, 1, 0],
                                [0, 0, -1, 0, 1]]) / h
            # Stencil to average at four staggered points
            avg  = numpy.array([[1, 0, 1, 0, 0],
                                [0, 1, 1, 0, 0],
                                [0, 0, 1, 1, 0],
                                [0, 0, 1, 0, 1]]) / 2
            # Stencil to compute divergence at cell centers from fluxes at four staggered points
            div = numpy.array([-1, -1, 1, 1]) / h
            ustencil = u[stencili]
            ustag = avg @ ustencil
            kappa = 1 + ustag**2
            if mask[row] == 0: # Dirichlet boundary
                ai.append(row)
                aj.append(row)
                av.append(1)
            else:
                F[row] -= div @ (kappa[:,None] * grad @ ustencil)
                Jstencil = -div @ (kappa[:,None] * grad
                    + 2*(ustag*(grad @ ustencil))[:,None] * avg)
                smask = mask[stencili]
                ai += [row]*sum(smask)
                aj += stencili[smask].tolist()
                av += Jstencil[smask].tolist()
    J = sp.csr_matrix((av, (ai, aj)), shape=(m*m,m*m))
    return F, J

h = .1
x, y = hgrid(h)
u = 0*x
F, J = nonlinear2d_div(h, x, y, u, lambda x,y: 0*x+1,
                       lambda x,y: 0*x, dirichlet=((0,-1),(0,-1)))
deltau = sp.linalg.spsolve(J, -F).reshape(x.shape)
pyplot.contourf(x, y, deltau)
pyplot.colorbar();
```

```python
def solve_nonlinear(h, g0, dirichlet, atol=1e-8, verbose=False):
    x, y = hgrid(h)
    u = 0*x
    for i in range(50):
        F, J = nonlinear2d_div(h, x, y, u, lambda x,y: 0*x+1,
                               lambda x,y: 0*x, dirichlet=((0,-1),(0,-1)))
        anorm = numpy.linalg.norm(F, numpy.inf)
        if verbose:
            print('{:2d}: anorm {:8e}'.format(i,anorm))
        if anorm < atol:
            break
        deltau = sp.linalg.spsolve(J, -F)
        u += deltau.reshape(x.shape)
    return x, y, u, i

x, y, u, i = solve_nonlinear(.1, lambda x,y: 0*x, dirichlet=((0,-1),(0,-1)), verbose=True)
pyplot.contourf(x, y, u)
pyplot.colorbar();
```

## Homework 3: Due 2017-11-03

Write a solver for the regularized $p$-Laplacian,
$$ -\nabla\cdot\big( \kappa(\nabla u) \nabla u \big) = 0 $$
where
$$ \kappa(\nabla u) = \big(\frac 1 2 \epsilon^2 + \frac 1 2 \nabla u \cdot \nabla u \big)^{\frac{p-2}{2}}, $$
$ \epsilon > 0$, and $1 < p < \infty$.  The case $p=2$ is the conventional Laplacian. This problem gets more strongly nonlinear when $p$ is far from 2 and when $\epsilon$ approaches zero. The $p \to 1$ limit is related to plasticity and has applications in non-Newtonion flows and structural mechanics.

1. Implement a "Picard" solver, which is like a Newton solver except that the Jacobian is replaced by the linear system
$$ J_{\text{Picard}}(u) \delta u \sim -\nabla\cdot\big( \kappa(\nabla u) \nabla \delta u \big) . $$
This is much easier to implement than the full Newton linearization.  How fast does this method converge for values of $p < 2$ and $p > 2$?

* Use the linearization above as a preconditioner to a Newton-Krylov method.  That is, use [`scipy.sparse.linalg.LinearOperator`](https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.LinearOperator.html) to apply the Jacobian to a vector
$$ \tilde J(u) v = \frac{F(u + h v) - F(u)}{h} . $$
Then for each linear solve, use [`scipy.sparse.linalg.gmres`](https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.gmres.html) and pass as a preconditioner, a direct solve with the Picard linearization above. (You might find [`scipy.sparse.linalg.factorized`](https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.factorized.html#scipy.sparse.linalg.factorized) to be useful. Compare algebraic convergence to that of the Picard method.

* Can you directly implement a Newton linearization?  Either do it or explain what is involved.  How will its nonlinear convergence compare to that of the Newton-Krylov method?

# Wave equations and multi-component systems

The acoustic wave equation with constant wave speed $c$ can be written
$$ \ddot u - c^2 \nabla\cdot \nabla u = 0 $$
where $u$ is typically a pressure.
We can convert to a first order system
$$ \begin{bmatrix} \dot u \\ \dot v \end{bmatrix} = \begin{bmatrix} 0 & I \\ c^2 \nabla\cdot \nabla & 0 \end{bmatrix} \begin{bmatrix} u \\ v \end{bmatrix} . $$
We will choose a zero-penetration boundary condition $\nabla u \cdot \hat n = 0$, which will cause waves to reflect.

```python
%run fdtools.py

x, y, L, _ = laplacian2d_bc(.1, lambda x,y: 0*x,
                              lambda x,y: 0*x, dirichlet=((),()))
A = sp.bmat([[None, sp.eye(*L.shape)],
             [-L, None]])
eigs = sp.linalg.eigs(A, 10, which='LM')[0]
print(eigs)
maxeig = max(eigs.imag)
u0 = numpy.concatenate([numpy.exp(-8*(x**2 + y**2)), 0*x], axis=None)
hist = ode_rkexplicit(lambda t, u: A @ u, u0, tfinal=2, h=2/maxeig)

def plot_wave(x, y, time, U):
    u = U[:x.size].reshape(x.shape)
    pyplot.contourf(x, y, u)
    pyplot.colorbar()
    pyplot.title('Wave solution t={:f}'.format(time));

for step in numpy.linspace(0, len(hist)-1, 6, dtype=int):
    pyplot.figure()
    plot_wave(x, y, *hist[step])
```

* This was a second order discretization, but we could extend it to higher order.
* The largest eigenvalues of this operator are proportional to $c/h$.
* Formally, we can write this equation in conservative form
$$ \begin{bmatrix} \dot u \\ \dot{\mathbf v} \end{bmatrix} = \begin{bmatrix} 0 & c\nabla\cdot \\ c \nabla & 0 \end{bmatrix} \begin{bmatrix} u \\ \mathbf v \end{bmatrix} $$
where $\mathbf{v}$ is now a momentum vector and $\nabla u = \nabla\cdot (u I)$.  This formulation could produce an anti-symmetric ($A^T = -A$) discretization.  Discretizations with this property are sometimes called "mimetic".
* A conservative form is often preferred when studiying waves traveling through materials with different wave speeds $c$.
* This is a Hamiltonian system.  While high order Runge-Kutta methods can be quite accurate, "symplectic" time integrators are needed to preserve the structure of the Hamiltonian (related to energy conservation) over long periods of time. The midpoint method (aka $\theta=1/2$) is one such method.  There are also explicit symplectic methods such as [Verlet methods](https://en.wikipedia.org/wiki/Verlet_integration), though these can be fragile.


```python
%matplotlib notebook
# %matplotlib inline

from IPython.display import display, HTML

import matplotlib.pyplot as plt
import numpy as np

plt.style.use('ggplot')
```

## The (College Student) Diet Problem

Consider the canonical college student. After a hard afternoon's work of solving way too many partial differential equations, she emerges from her room to obtain sustenance for the day.

She has a choice between getting chicken over rice (\$5) from the halal cart on her street ($r$), or subs (\$7) from the deli ($s$). She's a poor college student, so she will obviously want to get her money's worth. This is obviously an optimisation problem: she wants to find the amount of chicken over rice and subs she has to buy in order to minimise the total cost she spends on food.

$$
\text{minimise} \quad 5r + 7s
$$

In optimisation, we like to call this expression the **objective function**.

Well, it's not as simple as that. A girl's got to get her fill of daily nutrients. Fibre, protein, and carbohydrates are all important, and however far away food pyramids are from the quotidien thoughts of college students, a girl can still dream of a pseudo-healthy diet with at least 4 servings of fibre, 3 servings of protein, and 6 servings of carbohydrates.

A chicken over rice has 2 servings of fibre, 3 servings of protein, and 3 servings of carbohydrates, while a sub has 1 serving of fibre, 3 servings of protein, and 4 servings of carbohydrates. To find the combination of meals that satisfies the daily nutritional requirements, we impose the following **constraints**:

\begin{align}
\text{Fibre: } &2r +  s \geq 4 \\
\text{Protein: } &3r + 3s \geq 3 \\
\text{Carbohydrates: } &3r + 4s \geq 6
\end{align}

### Visualising the Problem

```python
fig = plt.figure()
axes = fig.add_subplot(1,1,1)

# define view
r_min = 0.0
r_max = 3.0
s_min = 0.0
s_max = 5.0
res = 50

r = numpy.linspace(r_min, r_max, res)

# plot axes
axes.axhline(0, color='#B3B3B3', linewidth=5)
axes.axvline(0, color='#B3B3B3', linewidth=5)

# plot constraints
c_1 = lambda x: 4 - 2*x
c_2 = lambda x: 1 - x
c_3 = lambda x: 0.25 * ( 6 - 3*x )
c_1_line = axes.plot( r, c_1(r), label='Fibre' ) # 2r +  s \geq 4
c_2_line = axes.plot( r, c_2(r), label='Protein' ) # 3r + 3s \geq 3
c_3_line = axes.plot( r, c_3(r), label='Carbohydrate' ) # 3r + 4s \geq 6

# plot objective
s = numpy.linspace(s_min, s_max, res)
c = numpy.empty([r.size, s.size])
for i, r_i in enumerate(r):
    c[:,i] = 5 * r_i + 12 * s

axes.contourf(r, s, c, res, cmap='Oranges', alpha=0.5)
r_cut = numpy.linspace(0.0, 2.0, 100)
axes.fill_between(r_cut, c_1(r_cut), color='w')

# plot cost minimising point
axes.plot(2.0, 0, 'o')

# label graph
axes.set_title('Visualising the Diet Problem')
axes.set_xlabel('Chicken Over Rice')
axes.set_ylabel('Sub')
axes.legend()

plt.show()
```

    <IPython.core.display.Javascript object>

We can visualise our diet problem on a graph of "Number of Subs vs. Number of Chicken or Rice", where lines each represent a constraint, and our cost function can be represented in shades of blue: the deeper the blue, the more we will spend on meals.

The regions where we will satisfy our constraints will be the regions _above_ our constraint lines, since we want more than or equal to the number of minimum servings. Obviously, we can't buy a negative number of subs or chicken over rice, so we have the implicit constraints that $r>0$ and $s>0$.

The intersection of all the regions that satisfy each of our constraints is what we call the **feasible region**, or the **feasible set**, the region where solutions that satisfy all constraints. In the graph, this is the region with the blue gradient fill.

So our problem of deciding how much of what food to buy has been essentially reduced to finding the point in the feasible set with the minimum cost (i.e. the lightest shade of blue.) With one glance, we can tell that this point is $(0, 2)$, so we should buy 2 chicken over rice, and 0 subs. Interestingly, our feasible region is determined largely by the fibre constraint—read from this what you want.

Well, you think to yourself, that was easy; I can stop reading now!

That's true, if you only have 2 foods to choose between. But in general, life isn't as simple as this; if, say, you're a functioning adult and actually cook, you'll want to choose between the 1000's of grocery items available to you at the local supermarket. In that case, you'll have to draw out one axis for each food item (how you'll do that, I don't know), and then compare the colors across this unvisualisable space. This shit gets real, and fast.

## Linear Programming

Well, luckily for us, a clever guy by the name of **George Dantzig** managed to solve exactly this type of problem for us while he was working for the U.S. Air Force in WWII, when computers were just starting to come out of the realm of science fiction. They faced a similar problem then, as many do now: they only had a set amount of men and resources, and wanted to maximise the amount of work they could do in winning the war.

In other areas, you could also imagine say, a furniture manufacturer wanting to find the most efficient way of using the manpower and planks, screws, tools, and whatever they use to build furniture these days, to produce the combination of furniture that will maximise their profits. Or, on Wall Street, a trader wanting to find the best combination of differently priced assets that maximises projected profits, or minimises risk (or something along those lines; I know nuts about finance).

We call these sorts of problems, wherein we want to maximise (or minimise!) some linear objective function subject to a set of linear constraints **linear optimisation problems**, and the methods we use to solve these problems **linear programming**.

### Standard Form and Duality
Linear optimisation problems can always be expressed as

\begin{align}
\text{maximise}   \quad & b_1 x_1 + b_2 x_2 + \ldots + b_m x_m \\
\text{subject to} \quad & a_{11} x_{1} + a_{21} x_{2} + \ldots + a_{m1} x_{m} \leq c_1 \\
                        & a_{12} x_{1} + a_{22} x_{2} + \ldots + a_{m2} x_{m} \leq c_2 \\
                        & \vdots \\
                        & a_{1n} x_{1} + a_{2n} x_{2} + \ldots + a_{mn} x_{m} \leq c_n
\end{align}

In less symbols, this is

\begin{align}
\text{maximise}    \quad & b^T x     \\
\text{subject to}  \quad & Ax \leq c
\end{align}

This is what is commonly known as the **dual form** of the problem. Well, so if there is a dual, then there must actually be 2 problems, right? So what was the first?

Turns out, we call the "first" problem the **primal problem**, and surprisingly (or not), the solution of the primal problem will give us an upper bound on the corresponding solution of the dual problem. It looks like this:

\begin{align}
\text{minimise}   \quad & c_1 y_1 + c_2 y_2 + \ldots + c_m y_ n\\
\text{subject to} \quad & a_{11} y_{1} + a_{12} y_{2} + \ldots + a_{1n} y_{n} = b_1 \\
                        & a_{21} y_{1} + a_{22} y_{2} + \ldots + a_{m2} y_{n} = b_2 \\
                        & \vdots \\
                        & a_{m1} y_{1} + a_{m2} y_{2} + \ldots + a_{nm} y_{n} = b_m \\
\text{and}        \quad & \{ y_i \geq 0 \}_{i=1}^m
\end{align}

aka

\begin{align}
\text{minimise}    \quad & c^T y     \\
\text{subject to}  \quad & A^T y = b \\
\text{and}         \quad & y \geq 0
\end{align}

We basically interchange the constraints' constants and the coefficients in our objective function, and turn the inequalities into equalities. The nice thing about the dual problem and its primal, is that the primal problem has an optimal solution $x^*$, then the dual also has an optimal solution $y^*$ related by $b^Tx^*=c^Ty^*$, i.e. the two problems have the same optimum value!

The dual problem for linear optimisation problems was first conjectured by von Neumann, who was then working on game theory. We can think of the fact that any linear programme has a dual problem as 2 players are playing a zero-sum game; any gains on the part of one player must necessarily result in losses for the other player. When you maximise utility for one player, you are at the same time minimising utility for the other.

So what does our college student diet problem look like in the standard form (and its primal?)

Since maximising a function is just minimising the negative of the function, the problem becomes

\begin{align}
\text{maximise}   \quad & - 5r - 7s          \\
\text{subject to} \quad & - 2r -  s \leq - 4 \\
                        & - 3r - 3s \leq - 3 \\
                        & - 3r - 4s \leq - 6
\end{align}

```python
fig = plt.figure()
axes = fig.add_subplot(1,1,1)

# plot axes
axes.axhline(0, color='k')
axes.axvline(0, color='k')

# plot constraints
c_1 = lambda x: 4 - 2*x
c_2 = lambda x: 1 - x
c_3 = lambda x: - 0.25 * ( - 6 + 3*x )
c_1_line = axes.plot( r, c_1(r), label='Fibre' ) # 2r +  s \geq 4
c_2_line = axes.plot( r, c_2(r), label='Protein' ) # 3r + 3s \geq 3
c_3_line = axes.plot( r, c_3(r), label='Carbohydrate' ) # 3r + 4s \geq 6

# plot objective
s = numpy.linspace(s_min, s_max, res)
c = numpy.empty([r.size, s.size])
for i, r_i in enumerate(r):
    c[:,i] = - 5 * r_i - 12 * s

axes.contourf(r, s, c, res, cmap='Oranges', alpha=0.5)
r_cut = numpy.linspace(0.0, 2.0, 100)
axes.fill_between(r_cut, c_1(r_cut), color='w')

# plot cost minimising point
axes.plot(2.0, 0, 'o')

# label graph
axes.set_title('Visualising the Diet Problem, Standard Form')
axes.set_xlabel('Chicken Over Rice')
axes.set_ylabel('Sub')
axes.legend(loc=1)

plt.show()
```

    <IPython.core.display.Javascript object>

In dual form, this is

\begin{align}
\text{minimise}   \quad & - 4y_1 - 3y_2 - 6y_3       \\
\text{subject to} \quad &   2y_1 + 3y_2 + 3y_3 = 5 \\
                        &    y_1 + 3y_2 + 4y_3 = 7 \\
\text{and}        \quad & \{ y_i \geq 0 \}_{i=1}^3 \\
\end{align}

Which can be seen as minimising the objective function on the line segment  formed by intersecting the 2 constraint planes. We can also interpret this as wanting to **maximise** the nutritional value of our meals, given that trying to increase the quantity of one nutrient will necessarily mean that we have to give up some amount of another nutrient.

## The Simplex Method

### Standard Form (for the Simplex Method)

\begin{align}
\text{maximise}   \quad & c_1 x_1 + c_2 x_2 + \ldots + c_m x_m \\
\text{subject to} \quad & a_{11} x_1 + a_{12} x_2 + \ldots + a_{1m} x_m \leq b_1 \\
                        & a_{21} x_2 + a_{22} x_2 + \ldots + a_{2m} x_m \leq b_2 \\
                        & \vdots \\
                        & a_{n1} x_n + a_{n2} x_n + \ldots + a_{nm} x_m \leq b_n \\
\text{and}        \quad & \{ x_i \geq 0 \}_{i=1}^m \text{ and } \{ b_j \geq 0 \}_{j=1}^n
\end{align}

1. If you are currently trying to minimise the objective function, turn it into a maximisation problem by taking the negative of the expression
2. Turn all the inequality constraints into equality constraints by adding **slack variables**
3. If these transformation still don't allow your system of equations to fit the form, solve the **dual form** of the problem!

#### System of Constraint Equations

\begin{align}
\text{maximise}   \quad & c_1 x_1 + c_2 x_2 + \ldots + c_m x_m = z \\
\text{subject to} \quad & a_11 x_1 + a_12 x_2 + \ldots + a_1m x_m + s_1 = b_1 \\
                        & a_21 x_2 + a_22 x_2 + \ldots + a_2m x_m + s_2 =  b_2 \\
                        & \vdots \\
                        & a_n1 x_n + a_n2 x_n + \ldots + a_nm x_m + s_n =  b_n \\
\text{and}        \quad & \{ x_i \geq 0 \}_{i=1}^{m}, ~ \{ s_i \geq 0 \}_{j=1}^{n}, ~ \text{ and } \{ b_j \geq 0 \}_{j=1}^n
\end{align}

Taking another look at our diet problem, we can put this problem

\begin{align}
\text{maximise}   \quad & - 5r - 7s          \\
\text{subject to} \quad & - 2r -  s \leq - 4 \\
                        & - 3r - 3s \leq - 3 \\
                        & - 3r - 4s \leq - 6 \\
\text{and}        \quad & r, s \geq 0
\end{align}

into standard form for the simplex method by putting it into its dual form:

\begin{align}
\text{maximise}   \quad &   6y_1 + 3y_2 + 4y_3       \\
\text{subject to} \quad &   3y_1 + 3y_2 + 2y_3 \leq 5 \\
                        &   4y_1 + 3y_2 +  y_3 \leq 7 \\
\text{and}        \quad & \{ y_i \geq 0 \}_{i=1}^3 \\
\end{align}

Hence, the constraint equations are

\begin{align}
\text{maximise}   \quad &   6y_1 + 3y_2 + 4y_3 = z      \\
\text{subject to} \quad &   3y_1 + 3y_2 + 2y_3 + s_1 = 5 \\
                        &   4y_1 + 3y_2 +  y_3 + s_2 = 7 \\
\text{and}        \quad & \{ y_i \geq 0 \}_{i=1}^3 \text{ and } \{ s_i \geq 0 \}_{i=1}^2 \\
\end{align}

### The Algorithm

```python
import pandas as pd
pd.set_option('display.notebook_repr_html', True)
```

```python
def pivot(departing, entering, tab):
    dpi = tab[tab['basic_variable']==departing].index[0] # index of the departing row

    # update basic variable
    tab['basic_variable'][dpi] = entering

    # normalise departing_row
    tab.ix[dpi,0:-1] = tab.ix[dpi,0:-1] / tab[entering][dpi]

    departing_row = tab.ix[dpi,0:-1]

    # do gauss-jordan on entering variable column
    for row in tab.index[tab.index!=dpi]:
        tab.ix[row, 0:-1] = tab.ix[row, 0:-1] - tab[entering][row] * departing_row
```

```python
# Bland's rule
def calculate_ratio(entering, tab):
    ratios = tab.ix[0:-1, 'value'] * 0 - 1

    for index, is_valid in enumerate(tab.ix[0:-1, entering] > 0):
        if is_valid==True:
            ratios[index] = tab.ix[index, 'value']/tab.ix[index, entering]
    return ratios
```

```python
def find_entering(tab):
    return tab.ix['z',0:-2].idxmin()
```

```python
def find_departing(ratios, tab):
    return tab.ix[ratios[ratios>=0].idxmin(),'basic_variable']
```

```python
def update_stats(tab):

    print "Basic variables: "
    basic_variables = tab.ix[0:-1, 'basic_variable'].values
    print basic_variables

    print "Non-basic variables: "
    non_basic_variables = numpy.setdiff1d(tab.columns[0:-2], basic_variables)
    print non_basic_variables

    print "Entering variable: "
    entering_variable = find_entering(tab)
    print entering_variable

    print "Ratios: "
    ratios = calculate_ratio(entering_variable, tab)
    print ratios

    print "Departing variable: "
    departing_variable = find_departing(ratios, tab)
    print departing_variable

    return departing_variable, entering_variable
```

```python
def is_optimum(tab):
    return (tab.ix['z',0:-2] >= 0).all()
```

```python
def run_simplex(tableau_dict, tableau_orig, max_iterations=10, force_iterations=0):
    if force_iterations == 0:
        for i in xrange(max_iterations):
            tableau_dict[i] = tableau_orig.copy()
            display(tableau_orig)
            if is_optimum(tableau_orig):
                break
            departing_variable, entering_variable = update_stats(tableau_orig)
            pivot(departing_variable, entering_variable, tableau_orig)
    else:
        for i in xrange(force_iterations):
            tableau_dict[i] = tableau_orig.copy()
            display(tableau_orig)
            departing_variable, entering_variable = update_stats(tableau_orig)
            pivot(departing_variable, entering_variable, tableau_orig)
```

```python
c_1 = numpy.array([[ 3,  3,  2, 1, 0, 5, 's_1']])
c_2 = numpy.array([[ 4,  3,  1, 0, 1, 7, 's_2']])
z   = numpy.array([[-6, -3, -4, 0, 0, 0,    '']])
rows= numpy.concatenate((c_1, c_2, z), axis=0)

tableau = pd.DataFrame(rows, columns=['y_1','y_2','y_3','s_1','s_2','value', 'basic_variable'], index=['c_1','c_2','z'])
tableau.ix[:,0:-1] = tableau.ix[:,0:-1].astype('float')
```

```python
tableaux = dict()
run_simplex(tableaux, tableau)
```

      y_1
      y_2
      y_3
      s_1
      s_2
      value
      basic_variable

      c_1
      3
      3
      2
      1
      0
      5
      s_1

      c_2
      4
      3
      1
      0
      1
      7
      s_2

      z
      -6
      -3
      -4
      0
      0
      0

    Basic variables:
    ['s_1' 's_2']
    Non-basic variables:
    ['y_1' 'y_2' 'y_3']
    Entering variable:
    y_1
    Ratios:
    c_1    1.66667
    c_2       1.75
    Name: value, dtype: object
    Departing variable:
    s_1

      y_1
      y_2
      y_3
      s_1
      s_2
      value
      basic_variable

      c_1
      1
      1
      0.666667
      0.333333
      0
      1.66667
      y_1

      c_2
      0
      -1
      -1.66667
      -1.33333
      1
      0.333333
      s_2

      z
      0
      3
      0
      2
      0
      10

```python
from ipywidgets import interact

def diet_problem(step):
    fig = plt.figure()
    axes = fig.add_subplot(1,1,1)

    # plot axes
    axes.axhline(0, color='k')
    axes.axvline(0, color='k')

    # plot constraints
    c_1 = lambda x: 4 - 2*x
    c_2 = lambda x: 1 - x
    c_3 = lambda x: - 0.25 * ( - 6 + 3*x )
    c_1_line = axes.plot( r, c_1(r), label='Fibre' ) # 2r +  s \geq 4
    c_2_line = axes.plot( r, c_2(r), label='Protein' ) # 3r + 3s \geq 3
    c_3_line = axes.plot( r, c_3(r), label='Carbohydrate' ) # 3r + 4s \geq 6

    # plot objective
    for i, r_i in enumerate(r):
        c[:,i] = - 5 * r_i - 12 * s

    axes.contourf(r, s, c, res, cmap='Oranges', alpha=0.5)
    axes.fill_between(r_cut, c_1(r_cut), color='w')

    step_coords = numpy.array([[0.0, 0.0], [2.0, 0.0]])

    # plot point
    axes.plot(step_coords[step][0], step_coords[step][1], 'ro', markersize=10)

    # label graph
    axes.set_title('Simplex Method on the College Diet Problem, Iteration ' + str(step))
    axes.set_xlabel('Chicken Over Rice')
    axes.set_ylabel('Sub')
    axes.legend(loc=1)

    plt.show()
    display(tableaux[step])
```

```python
interact(diet_problem, step=(0,1));
```

    <IPython.core.display.Javascript object>

      y_1
      y_2
      y_3
      s_1
      s_2
      value
      basic_variable

      c_1
      3
      3
      2
      1
      0
      5
      s_1

      c_2
      4
      3
      1
      0
      1
      7
      s_2

      z
      -6
      -3
      -4
      0
      0
      0

### Bland's Rule

This seemingly arbitrary rule will seem less arbitrary in just a while.

### Multiple Optimal Solutions

So, given the graphical intuition we now have for how the simplex method works, do we know if there ever a time when we would encounter more than 1 optimal solution for a given problem?

\begin{align}
\text{maximise}   \quad &  5x_1 +  7x_2          \\
\text{subject to} \quad &  2x_1 +   x_2 \leq 4 \\
                        & 10x_1 + 14x_2 \leq 30 \\
\text{and}        \quad & x_1, x_2 \geq 0
\end{align}

```python
fig = plt.figure()
axes = fig.add_subplot(1,1,1)

# define view
x_1_min = 0.0
x_1_max = 3.0
x_2_min = 0.0
x_2_max = 5.0
res = 50

# plot axes
axes.axhline(0, color='k')
axes.axvline(0, color='k')

# plot constraints
x_1 = numpy.linspace(x_1_min, x_1_max, res)
c_1 = lambda x: 4.0 - 2.0*x
c_2 = lambda x: (30.0 - 10.0*x)/14.0
c_1_line = axes.plot( x_1, c_1(x_1), label='Constraint 1' ) # 2x_1 +   x_2 \leq 4
c_2_line = axes.plot( x_1, c_2(x_1), label='Constraint 2' ) # 10x_1 + 14x_2 \leq 30

# plot objective
x_2 = numpy.linspace(x_2_min, x_2_max, res)
c = numpy.empty([x_1.size, x_2.size])
for i, x_1_i in enumerate(x_1):
    c[:,i] = 5 * x_1_i + 7 * x_2

axes.contourf(x_1, x_2, c, res, cmap='Oranges', alpha=0.5)
# shade feasible region
c_1_bottom = numpy.linspace(0.0, 2.0, res)
c_2_bottom = numpy.linspace(0.0, 3.0, res)
axes.fill_between(c_1_bottom, c_1(c_1_bottom), color=plt.rcParams['axes.color_cycle'][0], alpha=0.5)
axes.fill_between(c_2_bottom, c_2(c_2_bottom), color=plt.rcParams['axes.color_cycle'][1], alpha=0.5)

# label graph
axes.set_title('How many solutions?')
axes.set_xlabel(r'x_1')
axes.set_ylabel(r'x_2')
axes.legend(loc=1)

plt.show()
```

    <IPython.core.display.Javascript object>

\begin{align}
\text{maximise}   \quad &  5x_1 +  7x_2          \\
\text{subject to} \quad &  2x_1 +   x_2 + s_1 = 4 \\
                        & 10x_1 + 14x_2 + s_2 = 30 \\
\text{and}        \quad & x_1, x_2, s_1, s_2 \geq 0
\end{align}

```python
c_1 = numpy.array([[ 2,  1, 1, 0,  4, 's_1']])
c_2 = numpy.array([[10, 14, 0, 1, 30, 's_2']])
z   = numpy.array([[-5, -7, 0, 0,  0,    '']])
rows= numpy.concatenate((c_1, c_2, z), axis=0)

tableau_multiple = pd.DataFrame(rows, columns=['x_1','x_2','s_1','s_2','value', 'basic_variable'], index=['c_1','c_2','z'])
tableau_multiple.ix[:,0:-1] = tableau_multiple.ix[:,0:-1].astype('float')
```

```python
tableaux_multiple = dict()
run_simplex(tableaux_multiple, tableau_multiple, force_iterations=3)
```

      x_1
      x_2
      s_1
      s_2
      value
      basic_variable

      c_1
      2
      1
      1
      0
      4
      s_1

      c_2
      10
      14
      0
      1
      30
      s_2

      z
      -5
      -7
      0
      0
      0

    Basic variables:
    ['s_1' 's_2']
    Non-basic variables:
    ['x_1' 'x_2']
    Entering variable:
    x_2
    Ratios:
    c_1          4
    c_2    2.14286
    Name: value, dtype: object
    Departing variable:
    s_2

      x_1
      x_2
      s_1
      s_2
      value
      basic_variable

      c_1
      1.28571
      0
      1
      -0.0714286
      1.85714
      s_1

      c_2
      0.714286
      1
      0
      0.0714286
      2.14286
      x_2

      z
      0
      0
      0
      0.5
      15

    Basic variables:
    ['s_1' 'x_2']
    Non-basic variables:
    ['s_2' 'x_1']
    Entering variable:
    x_1
    Ratios:
    c_1    1.44444
    c_2          3
    Name: value, dtype: object
    Departing variable:
    s_1

      x_1
      x_2
      s_1
      s_2
      value
      basic_variable

      c_1
      1
      0
      0.777778
      -0.0555556
      1.44444
      x_1

      c_2
      0
      1
      -0.555556
      0.111111
      1.11111
      x_2

      z
      0
      0
      0
      0.5
      15

    Basic variables:
    ['x_1' 'x_2']
    Non-basic variables:
    ['s_1' 's_2']
    Entering variable:
    x_1
    Ratios:
    c_1    1.44444
    c_2         -1
    Name: value, dtype: object
    Departing variable:
    x_1

```python
step_coords = numpy.array([[0.0, 0.0], [0.0, 2.14286], [tableaux_multiple[2].ix['c_1','value'], tableaux_multiple[2].ix['c_2','value']]])
step_value = numpy.array([tableaux_multiple[0].ix['z','value'], tableaux_multiple[1].ix['z','value'], tableaux_multiple[2].ix['z','value']])

def multiple_problem(step):
    fig = plt.figure()
    axes = fig.add_subplot(1,1,1)

    # define view
    x_1_min = 0.0
    x_1_max = 3.0
    x_2_min = 0.0
    x_2_max = 5.0

    # plot axes
    axes.axhline(0, color='k')
    axes.axvline(0, color='k')

    # plot constraints
    x_1 = numpy.linspace(x_1_min, x_1_max, res)
    c_1 = lambda x: 4.0 - 2.0*x
    c_2 = lambda x: (30.0 - 10.0*x)/14.0
    c_1_line = axes.plot( r, c_1(r), label='Constraint 1' ) # 2x_1 +   x_2 \leq 4
    c_2_line = axes.plot( r, c_2(r), label='Constraint 2' ) # 10x_1 + 14x_2 \leq 30

    # plot objective
    x_2 = numpy.linspace(x_2_min, x_2_max, res)
    c = numpy.empty([x_1.size, x_2.size])
    for i, x_1_i in enumerate(x_1):
        c[:,i] = 5 * x_1_i + 7 * x_2

    # color map of objective function values
    axes.contourf(x_1, x_2, c, res, cmap='Oranges', alpha=0.5)

    # shade feasible region
    c_1_bottom = numpy.linspace(0.0, 2.0, res)
    c_2_bottom = numpy.linspace(0.0, 3.0, res)
    axes.fill_between(c_1_bottom, c_1(c_1_bottom), color=plt.rcParams['axes.color_cycle'][0], alpha=0.5)
    axes.fill_between(c_2_bottom, c_2(c_2_bottom), color=plt.rcParams['axes.color_cycle'][1], alpha=0.5)

    # plot point
    axes.plot(step_coords[step][0], step_coords[step][1], 'ro', markersize=10)
    axes.text(step_coords[step][0]+0.1, step_coords[step][1], step_value[step])

    # label graph
    axes.set_title('How many solutions?')
    axes.set_xlabel('x_1')
    axes.set_ylabel('x_2')
    axes.legend(loc=1)

    plt.show()
    display(tableaux_multiple[step])
```

```python
interact(multiple_problem, step=(0,2));
```

    <IPython.core.display.Javascript object>

      x_1
      x_2
      s_1
      s_2
      value
      basic_variable

      c_1
      1.28571
      0
      1
      -0.0714286
      1.85714
      s_1

      c_2
      0.714286
      1
      0
      0.0714286
      2.14286
      x_2

      z
      0
      0
      0
      0.5
      15

### Unbounded Optima

\begin{align}
\text{maximise}   \quad &  5x_1 +  7x_2          \\
\text{subject to} \quad &  -x_1 +   x_2 \leq 5 \\
                        & -\frac{1}{2}x_1 + x_2 \leq 7 \\
\text{and}        \quad & x_1, x_2 \geq 0
\end{align}

```python
fig = plt.figure()
axes = fig.add_subplot(1,1,1)

# define view
x_1_min = 0.0
x_1_max = 10.0
x_2_min = 0.0
x_2_max = 15.0
# res = 100

# plot axes
axes.axhline(0, color='k')
axes.axvline(0, color='k')

# plot constraints
x_1 = numpy.linspace(x_1_min, x_1_max, res)
c_1 = lambda x: 5.0 + x
c_2 = lambda x: 7 + 0.5*x
c_1_line = axes.plot( x_1, c_1(x_1), label='Constraint 1' ) # -x_1 +   x_2 \leq 5
c_2_line = axes.plot( x_1, c_2(x_1), label='Constraint 2' ) # -\frac{1}{2}x_1 + x_2 \leq 7

# plot objective
x_2 = numpy.linspace(x_2_min, x_2_max, res)
c = numpy.empty([x_1.size, x_2.size])
for i, x_1_i in enumerate(x_1):
    c[:,i] = 5 * x_1_i + 7 * x_2

axes.contourf(x_1, x_2, c, res, cmap='Oranges', alpha=0.5)
# shade feasible region
# c_1_bottom = numpy.linspace(0.0, 2.0, res)
# c_2_bottom = numpy.linspace(0.0, 3.0, res)
axes.fill_between(x_1, c_1(x_1), color=plt.rcParams['axes.color_cycle'][0], alpha=0.5)
axes.fill_between(x_1, c_2(x_1), color=plt.rcParams['axes.color_cycle'][1], alpha=0.5)

# label graph
axes.set_title('Unbounded Optima')
axes.set_xlabel(r'$x_1$')
axes.set_ylabel(r'$x_2$')
axes.legend(loc=2)

plt.show()
```

    <IPython.core.display.Javascript object>

\begin{align}
\text{maximise}   \quad &  5x_1 +  7x_2          \\
\text{subject to} \quad &  -x_1 +   x_2 + s_1 = 5 \\
                        & -\frac{1}{2}x_1 + x_2 + s_2 = 7 \\
\text{and}        \quad & x_1, x_2, s_1, s_2 \geq 0
\end{align}

```python
c_1 = numpy.array([[  -1,  1, 1, 0,  5, 's_1']])
c_2 = numpy.array([[-0.5,  1, 0, 1,  7, 's_2']])
z   = numpy.array([[  -5, -7, 0, 0,  0,    '']])
rows= numpy.concatenate((c_1, c_2, z), axis=0)

tableau_unbounded = pd.DataFrame(rows, columns=['x_1','x_2','s_1','s_2','value', 'basic_variable'], index=['c_1','c_2','z'])
tableau_unbounded.ix[:,0:-1] = tableau_unbounded.ix[:,0:-1].astype('float')
```

```python
tableaux_unbounded = dict()
run_simplex(tableaux_unbounded, tableau_unbounded)
```

We got an error!

`ValueError: attempt to get argmin of an empty sequence`

Usually, errors are bad things, but in this case, the error is trying to tell us something

In the code:

`return tab.ix[ratios[ratios>=0].idxmin(),'basic_variable']`

Which is telling us that no non-negative ratio was found! Why is this a problem for us? Let's take a look at the equations our tableau, and our equations at this point in time:

```python
display(tableau_unbounded)
```

      x_1
      x_2
      s_1
      s_2
      value
      basic_variable

      c_1
      0
      1
      -1
      2
      9
      x_2

      c_2
      1
      0
      -2
      2
      4
      x_1

      z
      0
      0
      -17
      24
      83

\begin{gather}
  z = 83 + 17 s_1 - 24 s_2 \\
x_1 = 4 + 2 s_1 - 2 s_2 \\
x_2 = 9 + s_1 - 2 s_2
\end{gather}

At this point, we want to pick $s_1$ as our entering variable because it is has most negative coefficient, and increasing the value of $s_1$ would most increase the value of $z$.

Usually, increasing the value of $s_1$ would also mean that we have to decrease the value of one of the basic variables to 0 (so that we stay within our feasible region).

Here, what we have is that **increasing the value of $s_1$ would also increase the value of both our basic variables**, which means that our objective function will be able to increase without bound.

So the simplex method is able to tell us when our problem is unbounded, by virtue of the fact that the negative coefficient in the tableau indicates that we have not attained the optimum, but we are also unable to find a positive ratio to choose our departing variable.

### Degeneracy and Cycling

Disclaimer: this example was stolen from [here](http://mat.gsia.cmu.edu/classes/QUANT/NOTES/chap7.pdf).

\begin{align}
\text{maximise}   \quad &  2x_1 +  7x_2         \\
\text{subject to} \quad &  -x_1 +   x_2 \leq 3  \\
                        &   x_1 -   x_2 \leq 3 \\
                        &           x_2 \leq 2  \\
\text{and}        \quad & x_1, x_2 \geq 0
\end{align}

```python
fig = plt.figure()
axes = fig.add_subplot(1,1,1)

# define view
x_1_min = 0.0
x_1_max = 3.0
x_2_min = 0.0
x_2_max = 5.0
# res = 100

# plot axes
axes.axhline(0, color='k')
axes.axvline(0, color='k')

# plot constraints
x_1 = numpy.linspace(x_1_min, x_1_max, res)
c_1 = lambda x: 3.0 - x
c_2 = lambda x: -3.0 + x
c_3 = lambda x: 2.0 * numpy.ones(x.size)
c_1_line = axes.plot( x_1, c_1(x_1), label='Constraint 1' ) # 2x_1 +   x_2 \leq 4
c_2_line = axes.plot( x_1, c_2(x_1), label='Constraint 2' ) # 10x_1 + 14x_2 \leq 30
c_3_line = axes.plot( x_1, c_3(x_1), label='Constraint 3' ) # -2x_1 +  x_2  \leq 0

# plot objective
x_2 = numpy.linspace(x_2_min, x_2_max, res)
c = numpy.empty([x_1.size, x_2.size])
for i, x_1_i in enumerate(x_1):
    c[:,i] = 2.0 * x_1_i + x_2

axes.contourf(x_1, x_2, c, res, cmap='Oranges', alpha=0.5)
# shade feasible region
c_1_bottom = numpy.linspace(0.0, 3.0, res)
c_2_bottom = numpy.linspace(0.0, 3.0, res)
c_3_bottom = numpy.linspace(0.0, 3.0, res)
axes.fill_between(c_1_bottom, c_1(c_1_bottom), color=plt.rcParams['axes.color_cycle'][0], alpha=0.5)
axes.fill_between(c_2_bottom, c_2(c_2_bottom), color=plt.rcParams['axes.color_cycle'][1], alpha=0.5)
axes.fill_between(c_3_bottom, c_3(c_3_bottom), color=plt.rcParams['axes.color_cycle'][2], alpha=0.5)

# label graph
axes.set_title('Degeneracy and Cycling')
axes.set_xlabel(r'x_1')
axes.set_ylabel(r'x_2')
axes.legend(loc=1)

plt.show()
```

    <IPython.core.display.Javascript object>

\begin{align}
\text{maximise}   \quad &  2x_1 +  7x_2          \\
\text{subject to} \quad &  -x_1 +   x_2 + s_1 = 3 \\
                        &   x_1 -   x_2 + s_2 = 3 \\
                        &           x_2 + s_3 = 2 \\
\text{and}        \quad & \{x_i\}_{i=1}^2, \{s_j\}_{j=1}^3 \geq 0
\end{align}

```python
c_1 = numpy.array([[ 3,  1, 1, 0, 0,  6, 's_1']])
c_2 = numpy.array([[ 1, -1, 0, 1, 0,  2, 's_2']])
c_3 = numpy.array([[ 0,  1, 0, 0, 1,  3, 's_3']])
z   = numpy.array([[-2, -1, 0, 0, 0,  0,    '']])
rows= numpy.concatenate((c_1, c_2, c_3, z), axis=0)

tableau_degenerate = pd.DataFrame(rows, columns=['x_1','x_2','s_1','s_2','s_3','value', 'basic_variable'], index=['c_1','c_2','c_3','z'])
tableau_degenerate.ix[:,0:-1] = tableau_degenerate.ix[:,0:-1].astype('float')
```

```python
tableaux_degenerate = dict()
run_simplex(tableaux_degenerate, tableau_degenerate)
```

      x_1
      x_2
      s_1
      s_2
      s_3
      value
      basic_variable

      c_1
      3
      1
      1
      0
      0
      6
      s_1

      c_2
      1
      -1
      0
      1
      0
      2
      s_2

      c_3
      0
      1
      0
      0
      1
      3
      s_3

      z
      -2
      -1
      0
      0
      0
      0

    Basic variables:
    ['s_1' 's_2' 's_3']
    Non-basic variables:
    ['x_1' 'x_2']
    Entering variable:
    x_1
    Ratios:
    c_1    2
    c_2    2
    c_3   -1
    Name: value, dtype: object
    Departing variable:
    s_1

      x_1
      x_2
      s_1
      s_2
      s_3
      value
      basic_variable

      c_1
      1
      0.333333
      0.333333
      0
      0
      2
      x_1

      c_2
      0
      -1.33333
      -0.333333
      1
      0
      0
      s_2

      c_3
      0
      1
      0
      0
      1
      3
      s_3

      z
      0
      -0.333333
      0.666667
      0
      0
      4

    Basic variables:
    ['x_1' 's_2' 's_3']
    Non-basic variables:
    ['s_1' 'x_2']
    Entering variable:
    x_2
    Ratios:
    c_1    6
    c_2   -1
    c_3    3
    Name: value, dtype: object
    Departing variable:
    s_3

      x_1
      x_2
      s_1
      s_2
      s_3
      value
      basic_variable

      c_1
      1
      0
      0.333333
      0
      -0.333333
      1
      x_1

      c_2
      0
      0
      -0.333333
      1
      1.33333
      4
      s_2

      c_3
      0
      1
      0
      0
      1
      3
      x_2

      z
      0
      0
      0.666667
      0
      0.333333
      5

```python
step_coords = numpy.transpose([numpy.zeros(len(tableaux_degenerate)), 2.0*numpy.ones(len(tableaux_degenerate))])
step_coords[0][1] = 0.0

def degeneracy_plot(step):
    fig = plt.figure()
    axes = fig.add_subplot(1,1,1)

    # define view
    x_1_min = 0.0
    x_1_max = 3.0
    x_2_min = 0.0
    x_2_max = 5.0
#     res = 100

    # plot axes
    axes.axhline(0, color='k')
    axes.axvline(0, color='k')

    # plot constraints
    x_1 = numpy.linspace(x_1_min, x_1_max, res)
    c_1 = lambda x: 3.0 - x
    c_2 = lambda x: -3.0 + x
    c_3 = lambda x: 2.0 * numpy.ones(x.size)
    c_1_line = axes.plot( x_1, c_1(x_1), label='Constraint 1' ) # 2x_1 +   x_2 \leq 4
    c_2_line = axes.plot( x_1, c_2(x_1), label='Constraint 2' ) # 10x_1 + 14x_2 \leq 30
    c_3_line = axes.plot( x_1, c_3(x_1), label='Constraint 3' ) # -2x_1 +  x_2  \leq 0

    # plot objective
    x_2 = numpy.linspace(x_2_min, x_2_max, res)
    c = numpy.empty([x_1.size, x_2.size])
    for i, x_1_i in enumerate(x_1):
        c[:,i] = 2.0 * x_1_i + x_2

    axes.contourf(x_1, x_2, c, res, cmap='Oranges', alpha=0.5)
    # shade feasible region
    c_1_bottom = numpy.linspace(0.0, 3.0, res)
    c_2_bottom = numpy.linspace(0.0, 3.0, res)
    c_3_bottom = numpy.linspace(0.0, 3.0, res)
    axes.fill_between(c_1_bottom, c_1(c_1_bottom), color=plt.rcParams['axes.color_cycle'][0], alpha=0.5)
    axes.fill_between(c_2_bottom, c_2(c_2_bottom), color=plt.rcParams['axes.color_cycle'][1], alpha=0.5)
    axes.fill_between(c_3_bottom, c_3(c_3_bottom), color=plt.rcParams['axes.color_cycle'][2], alpha=0.5)

    # plot point
    axes.plot(step_coords[step][0], step_coords[step][1], 'ro', markersize=10)

    # label graph
    axes.set_title('Degeneracy and Cycling, Iteration ' + str(step))
    axes.set_xlabel(r'x_1')
    axes.set_ylabel(r'x_2')
    axes.legend(loc=1)

    plt.show()
    display(tableaux_degenerate[step])
```

```python
interact(degeneracy_plot, step=(0,len(tableaux_degenerate)-1))
```

    <IPython.core.display.Javascript object>

      x_1
      x_2
      s_1
      s_2
      s_3
      value
      basic_variable

      c_1
      1
      0.333333
      0.333333
      0
      0
      2
      x_1

      c_2
      0
      -1.33333
      -0.333333
      1
      0
      0
      s_2

      c_3
      0
      1
      0
      0
      1
      3
      s_3

      z
      0
      -0.333333
      0.666667
      0
      0
      4

> You think you're moving, but you get nowhere. — _Stop and Stare_, OneRepublic

As its name suggests, degeneracy is when you get a basic variable (that's supposed to have a non-zero value) with a value of 0, and you are able to modify the value of the objective function without moving on the simplex.

In general, predicting when degeneracy will occur is non-trivial; one [source](http://theory.stanford.edu/~megiddo/pdf/degen.pdf) claims that it is NP-complete. You can read more about it [here](http://www.amazon.com/Degeneracy-Simplex-Cycling-Economics-Mathematical/dp/354054593X).

#### Bland's Rule

* Choose the non-basic variable with the most negative coefficient as the entering variable
* Choose the basic variable producing the smallest value/pivot ratio as the departing variable

Using Bland's Rule, the Simplex Method will never cycle if it encounters degeneracy (i.e. it halts on all inputs)

#### So what is cycling?

```python
tableaux_degenerate[1]
```

      x_1
      x_2
      s_1
      s_2
      s_3
      value
      basic_variable

      c_1
      1
      0.333333
      0.333333
      0
      0
      2
      x_1

      c_2
      0
      -1.33333
      -0.333333
      1
      0
      0
      s_2

      c_3
      0
      1
      0
      0
      1
      3
      s_3

      z
      0
      -0.333333
      0.666667
      0
      0
      4

Without Bland's Rule, one could potentially choose to pivot on $s_2$, which will give us

```python
pivot('s_2', 'x_2', tableaux_degenerate[1])
tableaux_degenerate[1]
```

      x_1
      x_2
      s_1
      s_2
      s_3
      value
      basic_variable

      c_1
      1
      0
      0.25
      0.25
      0
      2
      x_1

      c_2
      -0
      1
      0.25
      -0.75
      -0
      -0
      x_2

      c_3
      0
      0
      -0.25
      0.75
      1
      3
      s_3

      z
      0
      0
      0.75
      -0.25
      0
      4

Choosing $x_2$ to pivot back to seems like a good idea, right? Nope.

```python
pivot('x_2', 's_2', tableaux_degenerate[1])
tableaux_degenerate[1]
```

Cycling, ladies and gentlemen, aka a slow spiral into insanity.

##### $\epsilon-$perturbations

Another earlier (and nowadays less popular) method for avoiding degeneracy is by introducing $\epsilon$-perturbations into the problem. Recall that the standard system goes like

\begin{align}
\text{maximise}   \quad & c^T x \\
\text{subject to} \quad & Ax = b \\
\text{and}        \quad & x \geq 0
\end{align}

With $\epsilon$-perturbations, we will instead solve

\begin{align}
\text{maximise}   \quad & c^T x \\
\text{subject to} \quad & Ax = b + \epsilon \\
\text{and}        \quad & x \geq 0
\end{align}

which will give us a close enough answer to the original problem, and help us avoid the problem with the 0's. This kind of happens automatically as a bonus if you're running the simplex algorithm on a computer; as the program runs, errors from truncation, etc. build up, and you eventually get out of the cycle because your computer is doing floating point arithmetic.

Which is just about the one good thing about floating point arithmetic, I guess.

### Time Complexity of the Simplex Method
#### The Klee-Minty Cube

\begin{align}
\text{maximise}   \quad &  100x_1 +  10x_2 + x_3          \\
\text{subject to} \quad &     x_1                \leq 1   \\
                        &   20x_1 +    x_2       \leq 100 \\
                        &  200x_1 +  20x_2 + x_3 \leq 10000\\
\text{and}        \quad & x_1, x_2, x_3 \geq 0
\end{align}

```python
c_1 = numpy.array([[   1.0,   0.0,  0.0, 1.0, 0.0, 0.0,     1.0, 's_1']])
c_2 = numpy.array([[  20.0,   1.0,  0.0, 0.0, 1.0, 0.0,   100.0, 's_2']])
c_3 = numpy.array([[ 200.0,  20.0,  1.0, 0.0, 0.0, 1.0, 10000.0, 's_3']])
z   = numpy.array([[-100.0, -10.0, -1.0, 0.0, 0.0, 0.0,     0.0,    '']])
rows= numpy.concatenate((c_1, c_2, c_3, z), axis=0)

tableau_klee_minty = pd.DataFrame(rows, columns=['x_1','x_2', 'x_3','s_1','s_2','s_3','value', 'basic_variable'], index=['c_1','c_2','c_3','z'])
tableau_klee_minty.ix[:,0:-1] = tableau_klee_minty.ix[:,0:-1].astype('float')
```

```python
tableaux_klee_minty = dict()
run_simplex(tableaux_klee_minty, tableau_klee_minty)
```

Notice that our ending basic variables are $s_1$, $s_2$, and $x_3$, which means that if we had chosen $x_3$ to enter instead of $x_1$, we could have been done in one step! The greedy algoritm of the simplex method backfires on us here, since choosing the non-basic variable with the **least negative** coefficient would have gotten us to our solution much faster!

Instead, using the simplex method, we are forced to visit all 8 corners of the constraint cube in our quest to find the optimum.

#### Defeating the Klee-Minty Cube

The Klee-Minty Cube is the de-facto standard nowadays for determining how well a linear programming algorithm would perform. Algorithms like the Simplex Method and the Criss-Cross algorithm have exponential worst-case time complexities.

Other methods that choose pivots randomly have been shown to have polynomial worst-case time bounds, but perform worst than the Simplex method on average.

## Open Problems in Linear Programming

1. Integer solutions?
2. "Strongly polynomial" solutions

## References

George B. Dantzig, Mukund N. Thapa _Linear Programming 1- Introduction_ (Springer Series in Operations Research and Financial Engineering)
http://mat.gsia.cmu.edu/classes/QUANT/NOTES/chap7.pdf
http://college.cengage.com/mathematics/larson/elementary_linear/4e/shared/downloads/c09s3.pdf
http://college.cengage.com/mathematics/larson/elementary_linear/4e/shared/downloads/c09s4.pdf
http://www.iip.ist.i.kyoto-u.ac.jp/member/cuturi/Teaching/ORF522/lec8v2.pdf
http://math.stackexchange.com/questions/82006/in-simplex-method-if-the-leaving-variable-fails-for-all-candidates-of-mrt-what
https://www.math.unl.edu/~sdunbar1/ProbabilityTheory/Lessons/Applications/LinearOptimization/linearoptimization.pdf

```python

```

```python

```

```python

```

```python

```

```python

```


```python
from numpy import roots, random, real, imag
from math import atan, fabs, pi, tan, cos, sin, sqrt
import sympy
from sympy import Symbol

def cubic (Qx, Qy, Ux, Uy):

    a = Uy
    b = (Ux + 2*Qy)
    c = (2*Qx - Uy)
    d = -Ux

    det = -4*b*b*b*d + b*b*c*c -4*a*c*c*c + 18*a*b*c*d - 27*a*a*d*d

    if (det < 0):
        return 'c'

    if (det > 0):

        a = roots([a, b, c, d])
        a = a.real
        a = [atan(a[0]), atan(a[1]), atan(a[2])]

        U = [Ux*cos(a[0]) + Uy*sin(a[0]), Ux*cos(a[1]) + Uy*sin(a[1]), Ux*cos(a[2]) + Uy*sin(a[2])]
        rightU = [2*sin(a[0])*cos(a[0]), 2*sin(a[1])*cos(a[1]), 2*sin(a[2])*cos(a[2])]

        for i in range(0, 3):
            if (U[i] * rightU[i] < 0):
                a[i] = a[i] + pi

        a = sorted(a)
        a = [a[0] - a[0], a[1] - a[0], a[2] - a[0]]

        #print a

        if (a[2] > pi):
            return 'a'
        else:
            return 'b'
```

```python
mu, sigma = 0, 1
```

```python
Ux = random.normal(mu, sigma, 1000000)
Uy = random.normal(mu, sigma, 1000000)
Qx = random.normal(mu, sigma, 1000000)
Qy = random.normal(mu, sigma, 1000000)
```

```python
a, b, c = 0, 0, 0

for i in range(0, 100000):
    ch = ''
    ch = cubic(Qx[i], Qy[i], Ux[i], Uy[i])
    if (ch == 'a'):
        a = a + 1
    if (ch == 'b'):
        b = b + 1
    if (ch == 'c'):
        c = c + 1

print a/100000.0, b/100000.0, c/100000.0
```

    0.527 0.09225 0.38075

```python
def cubic_solver (Qx, Qy, Ux, Uy):

    a = (Uy)
    b = (Ux + 2*Qy)
    c = (2*Qx - Uy)
    d = (-Ux)

    return (roots([a, b, c, d]))
```

```python
# Another test
saddles, beaks, comets = 0, 0, 0

for i in range(0, 1000):

    root1, root2, root3 = cubic_solver(Qx[i], Qy[i]+0*i, Ux[i]+0*i, Uy[i]+0*i)

    if (Qx[i]*Uy[i] - Qy[i]*Ux[i] > 0):
        saddles = saddles + 1
    elif (imag(root1) == 0 and imag(root2) == 0 and imag(root3) == 0):
        beaks = beaks + 1

print saddles/1000.0, beaks/1000.0
```

    0.5 0.5

```python

```