In this notebook several methods for proving inequalities have been compared.

First of all we need a dataset. 35 inequalities were selected from https://www.imomath.com/index.php?options=592&lmm=0. Each inequality is represented by a tuple: inequality (in LaTeX), inequality constraints (for example: $a\in [0,1]$, equality constraints (for example: $abc=1$) and a function which converts inequality to a formula which we want to prove it's nonnegativity. In most cases when this function is simply a difference between left and right-hand side, but sometimes it's better to use difference between squares of sides (to get rid of square roots). These tuples are used as parameters in `parser` function which converts them to equivalent inequalities with default constraints (i.e. all variables are positive).

Some tuples have less than 4 elements. In this case `parser` use default arguments.

```python
from sympy import *
import shiroin
from shiroin import *
from sympy.parsing.latex import parse_latex
shiro.seed=1
from IPython.display import Latex
shiro.display=lambda x:display(Latex(x))
```

`sympy` has a `parse_latex` function which converts LaTeX formula to a `sympy` one. Unfortunately it doesn't deal with missing braces in the way that LaTeX do. For example `\frac12` generates $\frac12$ which is the same as `\frac{1}{2}`, but `parse_latex` accepts only the second version. So here there is a boring function which adds missing braces.

```python
def addbraces(s):
    arg={r'\frac':2,r'\sqrt':1}
    s2=''
    p=0
    while 1:
        m=re.search(r"\\[a-zA-Z]+", s[p:])
        if not m:
            break
        s2+=s[p:p+m.end()]
        p+=m.end()
        if m.group() in arg:
            for i in range(arg[m.group()]):
                sp=re.search('^ *',s[p:])
                s2+=sp.group()
                p+=sp.end()
                if s[p]=='{':
                    cb=re.search(r'^\{.*?\}',s[p:])
                    ab=addbraces(cb.group())
                    s2+=ab
                    p+=cb.end()
                else:
                    s2+='{'+s[p]+'}'
                    p+=1
    s2+=s[p:]
    return s2
print(addbraces(r'\frac{ \sqrt 3}2'))
print(addbraces(r'a^2+b^2+c^2\geq ab+bc+ca'))
```

    \frac{ \sqrt {3}}{2}
    a^2+b^2+c^2\geq ab+bc+ca

```python
#(formula,intervals,subs,function)
dif=lambda b,s:b-s
dif2=lambda b,s:b*b-s*s
ineqs=[
    (r'a^2+b^2+c^2\geq ab+bc+ca',),
    (r'a^2+b^2+c^2+d^2\geq a(b+c+d)',),
    (r'1\leq\frac{a^2b^2}{c^2}+\frac{b^2c^2}{a^2}+\frac{c^2a^2}{b^2}',
     '[0,1-f],[0,1]','[a,sqrt(1-e-f)],[b,sqrt(e)],[c,sqrt(f)]',),
    (r'\frac1{1-x^2}+\frac1{1-y^2}\geq \frac2{1-xy}','[0,1],[0,1]'),
    (r'a^3+b^3\geq a^2b+ab^2',),
    (r'\frac a{b+c}+\frac b{c+a}+\frac c{a+b}\geq \frac32',),
    (r'2a^3+b^3\geq 3a^2b',),
    (r'a^3+b^3+c^3\geq a^2b+b^2c+c^2a',),
    (r'\frac a{b+c}+\frac b{c+d}+ \frac c{d+a}+ \frac d{a+b}\geq 2',),
    (r'\frac{a^3}{a^2+ab+b^2}+ \frac{b^3}{b^2+bc+c^2}+ \frac{c^3}{c^2+ca+a^2} \geq \frac{a+b+c}3',),
    (r'\sqrt{ab}+\sqrt{cd}+\sqrt{ef}\leq\sqrt{(a+c+e)(b+d+f)}','','',dif2),
    (r'\frac{5a^3-ab^2}{a+b}+\frac{5b^3-bc^2}{b+c}+\frac{5c^3-ca^2}{c+a}\geq 2(a^2+b^2+c^2)',),
    #(r'\frac{a^2}{(1+b)(1-c)}+\frac{b^3}{(1+c)(1-d)}+\frac{c^3}{(1+d)(1-a)}+\frac{d^3}{(1+a)(1-b)}\geq\frac1{15}',
    # '[0,1-c-d],[0,1-d],[0,1]','[a,1-b-c-d]'),
    (r'\frac{x^3}{(1+y)(1+z)}+\frac{y^3}{(1+z)(1+x)}+\frac{z^3}{(1+x)(1+y)}\geq\frac{3}{4}','','[z,1/(x*y)]'),
    (r'\frac ab+\frac bc+\frac ca\geq \frac{(a+b+c)^2}{ab+bc+ca}',),
    (r'\frac{a^2}b+\frac{b^2}c+\frac{c^2}a\geq \frac{a^2+b^2+c^2}{a+b+c}',),
    (r'\frac{a^2}b+\frac{b^2}c+\frac{c^2}a\geq a+b+c+\frac{4(a-b)^2}{a+b+c}',),
    (r'\frac1{a^3+b^3+abc}+ \frac1{b^3+c^3+abc} +\frac1{c^3+a^3+ abc} \leq \frac1{abc}',),
    (r'\frac1{a^3(b+c)}+ \frac1{b^3(c+a)}+ \frac1{c^3(a+b)} \geq \frac32','','[c,1/(a*b)]'),
    (r'\frac{a^3}{b^2-bc+c^2}+\frac{b^3}{c^2-ca+a^2} + \frac{c^3}{a^2-ab+b^2} \geq 3 \cdot\frac{ab+bc+ca}{a+b+c}',),
    (r'\frac{x^5-x^2}{x^5+y^2+z^2}+\frac{y^5-y^2}{y^5+z^2+x^2}+\frac{z^5-z^2}{z^5+x^2+y^2}\geq0','','[x,1/(y*z)]'),
    (r'(a+b-c)(b+c-a)(c+a-b)\leq abc',),
    (r'\frac1{1+xy}+\frac1{1+yz}+\frac1{1+zx}\leq \frac34','','[x,(y+z)/(y*z-1)]'),
    (r'\frac{x\sqrt x}{y+z}+\frac{y\sqrt y}{z+x}+\frac{z\sqrt z}{x+y}\geq\frac{ \sqrt 3}2',
     '[0,1-z],[0,1]','[x,1-y-z]'),
    (r'a^4+b^4+c^4+d^4\geq 4abcd',),
    (r'\frac{ab}{a+b}+\frac{bc}{b+c}+\frac{ca}{c+a}\leq\frac{3(ab+bc+ca)}{2(a+b+c)}',),
    (r'\sqrt{a-1}+\sqrt{b-1}+ \sqrt{c-1} \leq \sqrt{c(ab+1)}','[1,oo],[1,oo],[1,oo]','',dif2),
    (r'(x-1)(y-1)(z-1)\geq 8','[0,1-b-c],[0,1-c],[0,1]','[x,1/a],[y,1/b],[z,1/c]'),
    (r'ay+bz+cx\leq s^2','[0,s],[0,s],[0,s]','[x,s-a],[y,s-b],[z,s-c]'),
    (r'x_1^2+x_2^2+x_3^2+x_4^2+x_5^2\geq 2 (x_1x_2+x_2x_3+x_3x_4+x_4x_5)/\sqrt{3}',),
    (r' xy+yz+zx - 2xyz \leq \frac7{27}', '[0,1-z],[0,1]','[x,1-y-z]'),
    (r'0 \leq xy+yz+zx - 2xyz', '[0,1-z],[0,1]','[x,1-y-z]'),
    (r'\sqrt{3+a+b+c}\geq\sqrt a+\sqrt b+\sqrt c','[1-z,1],[0,1]','[a,1/x-1],[b,1/y-1],[c,1/z-1],[x,2-y-z]',
     dif2),
    (r'\frac{2a^3}{a^2+b^2}+\frac{2b^3}{b^2+c^2}+\frac{2c^3}{c^2+a^2}\geq a+b+c',),
    (r'\frac{a^2}{b+c}+\frac{b^2}{c+a}+\frac{c^2}{a+b}\geq \frac12','[0,1-c],[0,1]','[a,1-b-c]'),
    (r'\frac{a+b}{2b+c}+\frac{b+c}{2c+a}+\frac{c+a}{2a+b}\geq 2',),
    #(r'\frac x{5-y^2}+\frac y{5-z^2}+\frac z{5-x^2}\geq \frac34','[0,sqrt(5)],[0,sqrt(5)]','[x,1/(y*z)]')
]

```

```python
def parser(formula,intervals='[]',subs='[]',func=dif):
    newproof()
    shiro.display=lambda x:None
    if intervals=='':
        intervals='[]'
    if subs=='':
        subs='[]'
    formula=addbraces(formula)
    formula=Sm(str(parse_latex(formula)))
    #formula,_=fractioncancel(formula)
    b,s=formula.lhs,formula.rhs
    if type(formula)==LessThan:
        b,s=s,b
    formula=func(b,s)
    formula=formula.subs(shiroin._smakeiterable2(subs))
    formula=makesubs(formula,intervals)
    formula=simplify(formula)
    num,den=fractioncancel(formula)
    return num
```

```python
from tqdm import tqdm
ineqs2=[]
for ineq in tqdm(ineqs):
    ineqs2+=[parser(*ineq)]
```

    100%|██████████| 35/35 [00:09<00:00,  3.63it/s]

Now let's look at the formulas when converted to polynomials.

```python
for i,ineq in zip(range(len(ineqs2)),ineqs2):
    print(i)
    display(reducegens(assumeall(ineq,positive=True)))
```

    0

$\displaystyle \operatorname{Poly}{\left( a^{2} - ab - ac + b^{2} - bc + c^{2}, a, b, c, domain=\mathbb{Z} \right)}$

    1

$\displaystyle \operatorname{Poly}{\left( a^{2} - ab - ac - ad + b^{2} + c^{2} + d^{2}, a, b, c, d, domain=\mathbb{Z} \right)}$

    2

$\displaystyle \operatorname{Poly}{\left( a^{4}b^{2} + a^{3}b^{2} - a^{3}b - 2 a^{2}b + a^{2} + ab^{2} - ab + b^{2}, a, b, domain=\mathbb{Z} \right)}$

    3

$\displaystyle \operatorname{Poly}{\left( 2 a^{3}b + a^{3} - 4 a^{2}b^{2} - a^{2}b + a^{2} + 2 ab^{3} - ab^{2} - 2 ab + b^{3} + b^{2}, a, b, domain=\mathbb{Z} \right)}$

    4

$\displaystyle \operatorname{Poly}{\left( a^{3} - a^{2}b - ab^{2} + b^{3}, a, b, domain=\mathbb{Z} \right)}$

    5

$\displaystyle \operatorname{Poly}{\left( 2 a^{3} - a^{2}b - a^{2}c - ab^{2} - ac^{2} + 2 b^{3} - b^{2}c - bc^{2} + 2 c^{3}, a, b, c, domain=\mathbb{Z} \right)}$

    6

$\displaystyle \operatorname{Poly}{\left( 2 a^{3} - 3 a^{2}b + b^{3}, a, b, domain=\mathbb{Z} \right)}$

    7

$\displaystyle \operatorname{Poly}{\left( a^{3} - a^{2}b - ac^{2} + b^{3} - b^{2}c + c^{3}, a, b, c, domain=\mathbb{Z} \right)}$

    8

$\displaystyle \operatorname{Poly}{\left( a^{3}c + a^{3}d + a^{2}b^{2} - a^{2}bd - 2 a^{2}c^{2} - a^{2}cd + a^{2}d^{2} + ab^{3} - ab^{2}c - ab^{2}d - abc^{2} + ac^{3} - acd^{2} + b^{3}d + b^{2}c^{2} - 2 b^{2}d^{2} + bc^{3} - bc^{2}d - bcd^{2} + bd^{3} + c^{2}d^{2} + cd^{3}, a, b, c, d, domain=\mathbb{Z} \right)}$

    9

$\displaystyle \operatorname{Poly}{\left( 2 a^{5}b^{2} + 2 a^{5}bc + 2 a^{5}c^{2} + a^{4}b^{3} - a^{4}b^{2}c - a^{4}bc^{2} + a^{4}c^{3} + a^{3}b^{4} - 2 a^{3}b^{3}c - 4 a^{3}b^{2}c^{2} - 2 a^{3}bc^{3} + a^{3}c^{4} + 2 a^{2}b^{5} - a^{2}b^{4}c - 4 a^{2}b^{3}c^{2} - 4 a^{2}b^{2}c^{3} - a^{2}bc^{4} + 2 a^{2}c^{5} + 2 ab^{5}c - ab^{4}c^{2} - 2 ab^{3}c^{3} - ab^{2}c^{4} + 2 abc^{5} + 2 b^{5}c^{2} + b^{4}c^{3} + b^{3}c^{4} + 2 b^{2}c^{5}, a, b, c, domain=\mathbb{Z} \right)}$

    10

$\displaystyle \operatorname{Poly}{\left( ad + af - 2 \sqrt{a}\sqrt{b}\sqrt{c}\sqrt{d} - 2 \sqrt{a}\sqrt{b}\sqrt{e}\sqrt{f} + bc + be + cf - 2 \sqrt{c}\sqrt{d}\sqrt{e}\sqrt{f} + de, \sqrt{a}, \sqrt{b}, \sqrt{c}, \sqrt{d}, \sqrt{e}, \sqrt{f}, domain=\mathbb{Z} \right)}$

    11

$\displaystyle \operatorname{Poly}{\left( 3 a^{4}b + 3 a^{4}c - 2 a^{3}b^{2} + 2 a^{3}c^{2} + 2 a^{2}b^{3} - 6 a^{2}b^{2}c - 6 a^{2}bc^{2} - 2 a^{2}c^{3} + 3 ab^{4} - 6 ab^{2}c^{2} + 3 ac^{4} + 3 b^{4}c - 2 b^{3}c^{2} + 2 b^{2}c^{3} + 3 bc^{4}, a, b, c, domain=\mathbb{Z} \right)}$

    12

$\displaystyle \operatorname{Poly}{\left( 4 x^{8}y^{4} + 4 x^{7}y^{4} - 3 x^{5}y^{5} - 3 x^{5}y^{4} + 4 x^{4}y^{8} + 4 x^{4}y^{7} - 3 x^{4}y^{5} - 6 x^{4}y^{4} - 3 x^{4}y^{3} - 3 x^{3}y^{4} - 3 x^{3}y^{3} + 4 xy + 4, x, y, domain=\mathbb{Z} \right)}$

    13

$\displaystyle \operatorname{Poly}{\left( a^{3}c^{2} + a^{2}b^{3} - a^{2}b^{2}c - a^{2}bc^{2} - ab^{2}c^{2} + b^{2}c^{3}, a, b, c, domain=\mathbb{Z} \right)}$

    14

$\displaystyle \operatorname{Poly}{\left( a^{4}c + a^{3}c^{2} + a^{2}b^{3} + ab^{4} + b^{2}c^{3} + bc^{4}, a, b, c, domain=\mathbb{Z} \right)}$

    15

$\displaystyle \operatorname{Poly}{\left( a^{4}c - 4 a^{3}bc + a^{3}c^{2} + a^{2}b^{3} + 6 a^{2}b^{2}c - 2 a^{2}bc^{2} + ab^{4} - 4 ab^{3}c - 2 ab^{2}c^{2} + b^{2}c^{3} + bc^{4}, a, b, c, domain=\mathbb{Z} \right)}$

    16

$\displaystyle \operatorname{Poly}{\left( a^{6}b^{3} + a^{6}c^{3} - 2 a^{5}b^{2}c^{2} + a^{3}b^{6} + a^{3}c^{6} - 2 a^{2}b^{5}c^{2} - 2 a^{2}b^{2}c^{5} + b^{6}c^{3} + b^{3}c^{6}, a, b, c, domain=\mathbb{Z} \right)}$

    17

$\displaystyle \operatorname{Poly}{\left( 2 a^{8}b^{8} + 2 a^{7}b^{6} + 2 a^{6}b^{7} - 3 a^{6}b^{5} - 3 a^{5}b^{6} + 2 a^{5}b^{5} - 3 a^{5}b^{3} + 2 a^{5}b^{2} - 6 a^{4}b^{4} + 2 a^{4}b^{3} + 2 a^{4} - 3 a^{3}b^{5} + 2 a^{3}b^{4} - 3 a^{3}b^{2} + 2 a^{3}b + 2 a^{2}b^{5} - 3 a^{2}b^{3} + 2 ab^{3} + 2 b^{4}, a, b, domain=\mathbb{Z} \right)}$

    18

$\displaystyle \operatorname{Poly}{\left( a^{8} - a^{6}bc - 2 a^{5}b^{3} + a^{5}b^{2}c + a^{5}bc^{2} - 2 a^{5}c^{3} + 3 a^{4}b^{4} - a^{4}b^{3}c + 2 a^{4}b^{2}c^{2} - a^{4}bc^{3} + 3 a^{4}c^{4} - 2 a^{3}b^{5} - a^{3}b^{4}c - a^{3}b^{3}c^{2} - a^{3}b^{2}c^{3} - a^{3}bc^{4} - 2 a^{3}c^{5} + a^{2}b^{5}c + 2 a^{2}b^{4}c^{2} - a^{2}b^{3}c^{3} + 2 a^{2}b^{2}c^{4} + a^{2}bc^{5} - ab^{6}c + ab^{5}c^{2} - ab^{4}c^{3} - ab^{3}c^{4} + ab^{2}c^{5} - abc^{6} + b^{8} - 2 b^{5}c^{3} + 3 b^{4}c^{4} - 2 b^{3}c^{5} + c^{8}, a, b, c, domain=\mathbb{Z} \right)}$

    19

$\displaystyle \operatorname{Poly}{\left( y^{18}z^{9} + 2 y^{16}z^{14} - y^{15}z^{9} + 2 y^{14}z^{16} - y^{14}z^{13} - y^{13}z^{14} - y^{13}z^{11} - y^{12}z^{12} + y^{12}z^{9} - y^{11}z^{13} - y^{11}z^{7} + 2 y^{11}z^{4} - y^{10}z^{5} + y^{9}z^{18} - y^{9}z^{15} + y^{9}z^{12} - y^{9}z^{6} - y^{8}z^{4} - y^{7}z^{11} - y^{7}z^{5} + 2 y^{7}z^{2} - y^{6}z^{9} + y^{6}z^{6} - y^{5}z^{10} - y^{5}z^{7} + 2 y^{4}z^{11} - y^{4}z^{8} - y^{3}z^{3} + 2 y^{2}z^{7} + 1, y, z, domain=\mathbb{Z} \right)}$

    20

$\displaystyle \operatorname{Poly}{\left( a^{3} - a^{2}b - a^{2}c - ab^{2} + 3 abc - ac^{2} + b^{3} - b^{2}c - bc^{2} + c^{3}, a, b, c, domain=\mathbb{Z} \right)}$

    21

$\displaystyle \operatorname{Poly}{\left( 2 y^{4}z^{2} - y^{3}z^{3} - 5 y^{3}z + 2 y^{2}z^{4} - 9 y^{2}z^{2} + 5 y^{2} - 5 yz^{3} + 23 yz + 5 z^{2} - 9, y, z, domain=\mathbb{Z} \right)}$

    22

$\displaystyle \operatorname{Poly}{\left( 2 a^{\frac{5}{2}}b\frac{1}{\sqrt{a b + a + b + 1}} + 2 a^{\frac{5}{2}}\frac{1}{\sqrt{a b + a + b + 1}} + 2 a^{2}b^{\frac{7}{2}}\frac{1}{\sqrt{b + 1}} + 2 a^{2}b^{\frac{5}{2}}\frac{1}{\sqrt{b + 1}} + 2 a^{2}b^{2}\frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}} - a^{2}b^{2}\sqrt{3} + 2 a^{2}b\frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}} - a^{2}b\sqrt{3} + 2 a^{\frac{3}{2}}b\frac{1}{\sqrt{a b + a + b + 1}} + 4 ab^{\frac{7}{2}}\frac{1}{\sqrt{b + 1}} + 4 ab^{\frac{5}{2}}\frac{1}{\sqrt{b + 1}} + 4 ab^{2}\frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}} - 2 ab^{2}\sqrt{3} + 2 ab^{\frac{3}{2}}\frac{1}{\sqrt{b + 1}} + 6 ab\frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}} - 2 ab\sqrt{3} + 2 a\frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}} - a\sqrt{3} + 2 b^{\frac{7}{2}}\frac{1}{\sqrt{b + 1}} + 2 b^{\frac{5}{2}}\frac{1}{\sqrt{b + 1}} + 2 b^{2}\frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}} - b^{2}\sqrt{3} + 4 b\frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}} - b\sqrt{3} + 2 \frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}}, \sqrt{a}, \sqrt{b}, \frac{1}{a b \sqrt{a b + a + b + 1} + a \sqrt{a b + a + b + 1} + b \sqrt{a b + a + b + 1} + \sqrt{a b + a + b + 1}}, \frac{1}{\sqrt{a b + a + b + 1}}, \frac{1}{\sqrt{b + 1}}, \sqrt{3}, domain=\mathbb{Z} \right)}$

    23

$\displaystyle \operatorname{Poly}{\left( a^{4} - 4 abcd + b^{4} + c^{4} + d^{4}, a, b, c, d, domain=\mathbb{Z} \right)}$

    24

$\displaystyle \operatorname{Poly}{\left( a^{3}b^{2} + a^{3}c^{2} + a^{2}b^{3} - 2 a^{2}b^{2}c - 2 a^{2}bc^{2} + a^{2}c^{3} - 2 ab^{2}c^{2} + b^{3}c^{2} + b^{2}c^{3}, a, b, c, domain=\mathbb{Z} \right)}$

    25

$\displaystyle \operatorname{Poly}{\left( def + de + df - 2 \sqrt{d}\sqrt{e} - 2 \sqrt{d}\sqrt{f} + ef - 2 \sqrt{e}\sqrt{f} + f + 2, \sqrt{d}, \sqrt{e}, \sqrt{f}, domain=\mathbb{Z} \right)}$

    26

$\displaystyle \operatorname{Poly}{\left( de^{2}f^{2} + de^{2}f + 2 def^{2} - 6 def + de + df^{2} + df + e^{2}f^{2} + e^{2}f + 2 ef^{2} + 3 ef + e + f^{2} + 2 f + 1, d, e, f, domain=\mathbb{Z} \right)}$

    27

$\displaystyle \operatorname{Poly}{\left( s^{2}def + s^{2}, s, d, e, f, domain=\mathbb{Z} \right)}$

    28

$\displaystyle \operatorname{Poly}{\left( -80 \sqrt{3}x_{}^{2} + 165 x_{}, \sqrt{3}, x_{}, domain=\mathbb{Z} \right)}$

    29

$\displaystyle \operatorname{Poly}{\left( 7 a^{2}b^{3} - 6 a^{2}b^{2} - 6 a^{2}b + 7 a^{2} + 14 ab^{3} - 12 ab^{2} + 15 ab - 13 a + 7 b^{3} - 6 b^{2} - 6 b + 7, a, b, domain=\mathbb{Z} \right)}$

    30

$\displaystyle \operatorname{Poly}{\left( a^{2}b^{2} + a^{2}b + 2 ab^{2} + ab + a + b^{2} + b, a, b, domain=\mathbb{Z} \right)}$

    31

$\displaystyle \operatorname{Poly}{\left( -2 \frac{1}{\sqrt{b}}\sqrt{\frac{a^{2} b^{2}}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a^{2} b}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a b^{2}}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a b}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1}} - 2 \frac{1}{\sqrt{b}}\sqrt{\frac{a b}{a^{2} b + a^{2} + a b + 2 a + 1} + \frac{b}{a^{2} b + a^{2} + a b + 2 a + 1}} - 2 \sqrt{\frac{a^{2} b^{2}}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a^{2} b}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a b^{2}}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a b}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1}}\sqrt{\frac{a b}{a^{2} b + a^{2} + a b + 2 a + 1} + \frac{b}{a^{2} b + a^{2} + a b + 2 a + 1}} + 3, \frac{1}{\sqrt{b}}, \sqrt{\frac{a^{2} b^{2}}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a^{2} b}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a b^{2}}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1} + \frac{a b}{a^{2} b + a^{2} + a b^{2} + 3 a b + 2 a + b^{2} + 2 b + 1}}, \sqrt{\frac{a b}{a^{2} b + a^{2} + a b + 2 a + 1} + \frac{b}{a^{2} b + a^{2} + a b + 2 a + 1}}, domain=\mathbb{Z} \right)}$

    32

$\displaystyle \operatorname{Poly}{\left( a^{5}b^{2} + a^{5}c^{2} + a^{4}b^{3} - a^{4}b^{2}c - a^{4}bc^{2} - a^{4}c^{3} - a^{3}b^{4} + a^{3}c^{4} + a^{2}b^{5} - a^{2}b^{4}c - a^{2}bc^{4} + a^{2}c^{5} - ab^{4}c^{2} - ab^{2}c^{4} + b^{5}c^{2} + b^{4}c^{3} - b^{3}c^{4} + b^{2}c^{5}, a, b, c, domain=\mathbb{Z} \right)}$

    33

$\displaystyle \operatorname{Poly}{\left( 2 a^{2}d^{3} - a^{2}d^{2} - a^{2}d + 2 a^{2} + 4 ad^{3} - 2 ad^{2} - 3 a + 2 d^{3} - d^{2} - d + 2, a, d, domain=\mathbb{Z} \right)}$

    34

$\displaystyle \operatorname{Poly}{\left( 2 a^{3} - 3 a^{2}b + a^{2}c + ab^{2} - 3 ac^{2} + 2 b^{3} - 3 b^{2}c + bc^{2} + 2 c^{3}, a, b, c, domain=\mathbb{Z} \right)}$

```python
tm=[0]*4
```

```python
from collections import Counter
from timeit import default_timer as timer
start=timer()
t=[]
for ineq in ineqs2:
    t+=[prove(ineq)]
    print(t[-1],end=',')
tm[0]=timer()-start
print('\n',tm[0],sep='')
Counter(t)

```

    -1,2,2,-1,-1,-1,0,-1,2,-1,0,0,-1,-1,0,2,-1,0,2,-1,2,2,2,0,-1,2,2,0,2,2,0,2,-1,2,-1,
    19.77632514099969

    Counter({-1: 13, 2: 14, 0: 8})

Codes 0 or -1 mean that the proof was found, code 2 means that proof wasn't found. So this method has proved 21 inequalities.

The second method uses `findvalues` function, rationalizes the result numbers and gives them as additional parameter to `prove` function.

```python
def cut(a):
    if a<=0 or a>=100 or (a is None):
        return 1
    return a
start=timer()
t2=[]
for ineq in ineqs2:
    numvalues=findvalues(ineq,disp=0)
    values=nsimplify(numvalues,tolerance=0.1,rational=True)
    values=list(map(cut,values))
    t2+=[prove(ineq,values=values)]
    print(t2[-1],end=',')
tm[1]=timer()-start
print('\n',tm[1],sep='')
Counter(t2)

```

    -1,0,0,-1,-1,-1,0,-1,2,-1,0,0,-1,-1,0,2,-1,0,2,-1,2,2,2,0,-1,2,0,0,2,2,0,2,-1,2,-1,
    20.529997757001183

    Counter({-1: 13, 0: 11, 2: 11})

The third method is similar to the second one, but instead of rationalize values it squares, rationalizes and makes square roots of these values.

```python
def cut(a):
    if a<=0 or a>=1000 or (a is None):
        return S(1)
    return a

start=timer()
t3=[]
for ineq in ineqs2:
    numvalues=findvalues(ineq,disp=0)
    numvalues=tuple([x**2 for x in numvalues])
    values=nsimplify(numvalues,tolerance=0.1,rational=True)
    values=[sqrt(x) for x in values]
    values=list(map(cut,values))
    t3+=[prove(ineq,values=values)]
    print(t3[-1],end=',')
tm[2]=timer()-start
print('\n',tm[2],sep='')
Counter(t3)

```

    -1,2,0,-1,-1,-1,0,-1,2,-1,0,0,-1,-1,0,2,-1,0,2,-1,2,2,2,0,-1,2,0,0,2,2,0,2,-1,2,-1,
    20.430799151001338

    Counter({-1: 13, 2: 12, 0: 10})

Finally, the fourth method is a slight modification to the third method. It does the same "findvalues, square, rationalize and make square roots" thing, but then it scales the values and runs it again. It can sometimes help with uniform formulas.

```python
def betw(a):
    return a>0.001 and a<1000 and a!=None
def cut(a):
    if betw(a):
        return a
    return S(1)

start=timer()
t4=[]
for ineq in ineqs2:
    numvalues=findvalues(ineq,disp=0)
    n=1
    numvalues2=[]
    for i in numvalues:
        if betw(i):
            n=1/i
            break
    for i in numvalues:
        if betw(i):
            numvalues2+=[i*n]
        else:
            numvalues2+=[1]
    numvalues3=findvalues(ineq,values=numvalues2,disp=0)
    numvalues4=tuple([x**2 for x in numvalues3])
    values=nsimplify(numvalues4,tolerance=0.1,rational=True)
    values=[sqrt(x) for x in values]
    values=list(map(cut,values))
    t4+=[prove(ineq,values=values)]
    print(t4[-1],end=',')
tm[3]=timer()-start
print('\n',tm[3],sep='')
Counter(t4)
```

    -1,2,0,-1,-1,-1,0,-1,2,-1,0,0,-1,-1,0,2,-1,0,2,-1,2,2,2,0,-1,2,0,0,2,2,0,2,-1,2,-1,
    21.87015606299974

    Counter({-1: 13, 2: 12, 0: 10})

```python
import matplotlib.pyplot as plt
plt.bar(['1','2','3','4'],tm)
plt.ylabel('time [seconds]')
plt.xlabel('method')
plt.show()
```

```python
import pandas as pd
u=pd.DataFrame(zip(['']*len(ineqs),t,t2,t3,t4))
```

```python
plt.bar(['1','2','3','4'],[sum(u[i]<=0)/len(ineqs2)*100 for i in range(1,5)])
plt.ylabel('inequalities proven [%]')
plt.xlabel('method')
plt.show()
```

Down below there are contingency tables and McNemar test for every pair of methods.

```python
for i in range(4):
    for j in range(i+1,4):
        display(pd.crosstab(u[i+1]<=0, u[j+1]<=0))
```

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th>2</th>
      <th>False</th>
      <th>True</th>
    </tr>
    <tr>
      <th>1</th>
      <th></th>
      <th></th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>False</th>
      <td>11</td>
      <td>3</td>
    </tr>
    <tr>
      <th>True</th>
      <td>0</td>
      <td>21</td>
    </tr>
  </tbody>
</table>
</div>

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th>3</th>
      <th>False</th>
      <th>True</th>
    </tr>
    <tr>
      <th>1</th>
      <th></th>
      <th></th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>False</th>
      <td>12</td>
      <td>2</td>
    </tr>
    <tr>
      <th>True</th>
      <td>0</td>
      <td>21</td>
    </tr>
  </tbody>
</table>
</div>

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th>4</th>
      <th>False</th>
      <th>True</th>
    </tr>
    <tr>
      <th>1</th>
      <th></th>
      <th></th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>False</th>
      <td>12</td>
      <td>2</td>
    </tr>
    <tr>
      <th>True</th>
      <td>0</td>
      <td>21</td>
    </tr>
  </tbody>
</table>
</div>

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th>3</th>
      <th>False</th>
      <th>True</th>
    </tr>
    <tr>
      <th>2</th>
      <th></th>
      <th></th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>False</th>
      <td>11</td>
      <td>0</td>
    </tr>
    <tr>
      <th>True</th>
      <td>1</td>
      <td>23</td>
    </tr>
  </tbody>
</table>
</div>

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th>4</th>
      <th>False</th>
      <th>True</th>
    </tr>
    <tr>
      <th>2</th>
      <th></th>
      <th></th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>False</th>
      <td>11</td>
      <td>0</td>
    </tr>
    <tr>
      <th>True</th>
      <td>1</td>
      <td>23</td>
    </tr>
  </tbody>
</table>
</div>

<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th>4</th>
      <th>False</th>
      <th>True</th>
    </tr>
    <tr>
      <th>3</th>
      <th></th>
      <th></th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>False</th>
      <td>12</td>
      <td>0</td>
    </tr>
    <tr>
      <th>True</th>
      <td>0</td>
      <td>23</td>
    </tr>
  </tbody>
</table>
</div>

```python
from statsmodels.stats.contingency_tables import mcnemar
for i in range(4):
    for j in range(i+1,4):
        print(mcnemar(pd.crosstab(u[i+1]<=0, u[j+1]<=0)))
```

    pvalue      0.25
    statistic   0.0
    pvalue      0.5
    statistic   0.0
    pvalue      0.5
    statistic   0.0
    pvalue      1.0
    statistic   0.0
    pvalue      1.0
    statistic   0.0
    pvalue      1.0
    statistic   0.0

In all cases p-value is greater than 0.05, so there is no statistical difference between the methods.


# **Thermodynamics Observables Calculation with Quantum Computer**

```python
import os
import sys
sys.path.insert(0, os.path.abspath('thermodynamics_utils'))
```

```python
import partition_function
```

```python
import thermodynamics
```

```python
import vibrational_structure_fd
```

```python
# imports
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from functools import partial

from qiskit.utils import QuantumInstance
from qiskit import Aer
from qiskit.algorithms import NumPyMinimumEigensolver, VQE

from qiskit_nature.drivers import UnitsType, Molecule
from qiskit_nature.drivers.second_quantization import (
    ElectronicStructureDriverType,
    ElectronicStructureMoleculeDriver,
)
from qiskit_nature.problems.second_quantization import ElectronicStructureProblem
from qiskit_nature.converters.second_quantization import QubitConverter
from qiskit_nature.mappers.second_quantization import JordanWignerMapper
from qiskit_nature.algorithms import GroundStateEigensolver
import qiskit_nature.constants as const
from qiskit_nature.algorithms.pes_samplers import BOPESSampler, EnergySurface1DSpline
from thermodynamics import constant_volume_heat_capacity
from vibrational_structure_fd import VibrationalStructure1DFD
from partition_function import DiatomicPartitionFunction
from thermodynamics import Thermodynamics

import warnings

warnings.simplefilter("ignore", np.RankWarning)
```

A preliminary draft with more information related to this tutorial can be found in preprint: Stober et al, arXiv 2003.02303 (2020)

## **Calculation of the Born Oppenheimer Potential Energy Surface (BOPES)**

To compute thermodynamic observables we begin with single point energy calculation which calculates the wavefunction and charge density and therefore  the energy of a particular arrangement of nuclei. Here we  compute the Born-Oppenheimer potential energy surface of a hydrogen molecule, as an example, which is simply the electronic energy as a function of bond length.

```python
qubit_converter = QubitConverter(mapper=JordanWignerMapper())
quantum_instance = QuantumInstance(backend=Aer.get_backend("aer_simulator_statevector"))
solver = VQE(quantum_instance=quantum_instance)

me_gss = GroundStateEigensolver(qubit_converter, solver)
```

```python
stretch1 = partial(Molecule.absolute_distance, atom_pair=(1, 0))
mol = Molecule(
    geometry=[("H", [0.0, 0.0, 0.0]), ("H", [0.0, 0.0, 0.2])],
    degrees_of_freedom=[stretch1],
    masses=[1.6735328e-27, 1.6735328e-27],
)

# pass molecule to PSYCF driver
driver = ElectronicStructureMoleculeDriver(mol, driver_type=ElectronicStructureDriverType.PYSCF)

es_problem = ElectronicStructureProblem(driver)
```

```python
# BOPES sampler testing
bs = BOPESSampler(gss=me_gss, bootstrap=True)
points = np.linspace(0.45, 5, 50)
res = bs.sample(es_problem, points)
```

```python
energies = []
bs_res_full = res.raw_results
for point in points:
    energy = bs_res_full[point].computed_energies + bs_res_full[point].nuclear_repulsion_energy
    energies.append(energy)
```

```python
fig = plt.figure()
plt.plot(points, energies)
plt.title("Dissociation profile")
plt.xlabel("Interatomic distance")
plt.ylabel("Energy")
```

```python
energy_surface = EnergySurface1DSpline()

xdata = res.points
ydata = res.energies
energy_surface.fit(xdata=xdata, ydata=ydata)
```

```python
plt.plot(xdata, ydata, "kx")
x = np.arange(min(xdata) - 0.25, max(xdata) + 0.25, 0.05)
plt.plot(x, energy_surface.eval(x), "r-")
plt.xlabel(r"distance, $\AA$")
plt.ylabel("energy, Hartree")
dist = max(ydata) - min(ydata)
plt.ylim(min(ydata) - 0.1 * dist, max(ydata) + 0.1 * dist)
```

## **Calculation of the Molecular Vibrational Energy Levels**

The Born-Oppeheimer approximation removes internuclear vibrations from the molecular Hamiltonian and the energy computed from quantum mechanical ground-state energy calculations using this approximation contain only the electronic energy. Since even at absolute zero internuclear vibrations still occur, a correction is required to obtain the true zero-temperature energy of a molecule. This correction is called the zero-point vibrational energy (ZPE), which is computed by summing the contribution from internuclear vibrational modes. Therefore, the next step in computing thermodynamic observables is determining the vibrational energy levels. This can be done by constructing the Hessian matrix based on computed single point energies close to the equilibrium bond length. The eigenvalues of the Hessian matrix can then be used to determine the vibrational energy levels and the zero-point vibrational energy
\begin{equation}
{\rm ZPE} = \frac{1}{2}\, \sum_i ^M \nu_i \, ,
\end{equation}
with $\nu_i$ being the vibrational frequencies, $M = 3N − 6$ or $M = 3N − 5$ for non-linear or linear molecules, respectively, and $N$ is number of the particles.

Here we fit a "full" energy surface using a 1D spline potential and use it to evaluate molecular vibrational energy levels.

```python
vibrational_structure = VibrationalStructure1DFD(mol, energy_surface)

plt.plot(xdata, ydata, "kx")
x = np.arange(min(xdata) - 0.25, max(xdata) + 0.25, 0.05)
plt.plot(x, energy_surface.eval(x), "r-")
plt.xlabel(r"distance, $\AA$")
plt.ylabel("energy, Hartree")
dist = max(ydata) - min(ydata)
plt.ylim(min(ydata) - 0.1 * dist, max(ydata) + 0.1 * dist)
for N in range(15):
    on = np.ones(x.shape)
    on *= energy_surface.eval(
        energy_surface.get_equilibrium_geometry()
    ) + vibrational_structure.vibrational_energy_level(N)
    plt.plot(x, on, "g:")
on = np.ones(x.shape)

plt.show()
```

## **Create a Partition Function for the Calculation of Heat Capacity**

The partition function for a molecule is the product of contributions from translational, rotational, vibrational, electronic, and nuclear degrees of freedom. Having the vibrational frequencies, now we can obtain the vibrational partition function  $q_{\rm vibration}$ to compute the whole molecular partition function
\begin{equation}
q_{\rm vibration} = \prod_{i=1} ^M \frac{\exp\,(-\Theta_{\nu_i}/2T)}{1-\exp\,(-\Theta_{\nu_i}/2T} \, .
\end{equation}
Here $\Theta_{\nu_i}= h\nu_i/k_B$, $T$ is the temperature and $k_B$ is the Boltzmann constant.

The single-point energy calculations and the resulting partition function can be used to calculate the (constant volume or constant pressure) heat capacity of the molecules. The constant volume heat capacity, for example, is given by

\begin{equation}
C_v = \left.\frac{\partial U}{\partial T}\right|_{N,V}\, ,
\qquad
{\rm with} \quad
U=k_B T^2 \left.\frac{\partial {\rm ln} Q}{\partial T}\right|_{N,V} .
\end{equation}

$U$ is the internal energy, $V$ is the volume and $Q$ is the partition function.

Here we illustrate the simplest usage of the partition function, namely creating a Thermodynamics object to compute properties like the constant pressure heat capacity defined above.

```python
Q = DiatomicPartitionFunction(mol, energy_surface, vibrational_structure)

P = 101350  # Pa
temps = np.arange(10, 1050, 5)  # K

mol.spins = [1 / 2, 1 / 2]

td = Thermodynamics(Q, pressure=101350)
td.set_pressure(101350)
temps = np.arange(10, 1500, 5)
ymin = 5
ymax = 11

plt.plot(temps, td.constant_pressure_heat_capacity(temps) / const.CAL_TO_J)
plt.xlim(0, 1025)
plt.ylim(ymin, ymax)
plt.xlabel("Temperature, K")
plt.ylabel("Cp, cal mol$^{-1}$ K$^{-1}$")

plt.show()
```

Here we demonstrate how to access particular components (the rotational part) of the partition function, which in the H2 case we can further split to para-hydrogen and ortho-hydrogen components.

```python
eq = Q.get_partition(part="rot", split="eq")
para = Q.get_partition(part="rot", split="para")
ortho = Q.get_partition(part="rot", split="ortho")
```

We will now plot the constant volume heat capacity (of the rotational part) demonstrating how we can call directly the functions in the 'thermodynamics' module, providing a callable object for the partition function (or in this case its rotational component). Note that in the plot we normalize the plot dividing by the universal gas constant R (Avogadro's number times Boltzmann's constant) and we use crossed to compare with experimental data found in literature.

```python
# REFERENCE DATA from literature
df_brink_T = [80.913535, 135.240157, 176.633783, 219.808499, 246.226899]
df_brink_Cv = [0.118605, 0.469925, 0.711510, 0.833597, 0.895701]

df_eucken_T = [
    25.120525,
    30.162485,
    36.048121,
    41.920364,
    56.195875,
    62.484934,
    72.148692,
    73.805910,
    73.804236,
    92.214423,
    180.031917,
    230.300866,
]
df_eucken_Cv = [
    0.012287,
    0.012354,
    0.008448,
    0.020478,
    0.032620,
    0.048640,
    0.048768,
    0.076678,
    0.078670,
    0.170548,
    0.667731,
    0.847681,
]

df_gia_T = [
    190.919338,
    195.951254,
    202.652107,
    204.292585,
    209.322828,
    225.300754,
    234.514217,
    243.747768,
]
df_gia_Cv = [0.711700, 0.723719, 0.749704, 0.797535, 0.811546, 0.797814, 0.833793, 0.845868]

df_parting_T = [80.101665, 86.358919, 185.914204, 239.927797]
df_parting_Cv = [0.084730, 0.138598, 0.667809, 0.891634]

df_ce_T = [
    80.669344,
    135.550569,
    145.464190,
    165.301153,
    182.144856,
    203.372528,
    237.993108,
    268.696642,
    294.095771,
    308.872014,
]
df_ce_Cv = [
    0.103048,
    0.467344,
    0.541364,
    0.647315,
    0.714078,
    0.798258,
    0.891147,
    0.944848,
    0.966618,
    0.985486,
]
```

```python
HeatCapacity = constant_volume_heat_capacity

R = const.N_A * const.KB_J_PER_K
plt.plot(temps, HeatCapacity(eq, temps) / R, "-k", label="Cv_rot Equilibrium")
plt.plot(temps, HeatCapacity(para, temps) / R, "-b", label="Cv_rot Para")
plt.plot(temps, HeatCapacity(ortho, temps) / R, "-r", label="Cv_rot Ortho")
plt.plot(
    temps,
    0.25 * HeatCapacity(para, temps) / R + 0.75 * HeatCapacity(ortho, temps) / R,
    "-g",
    label="Cv_rot 1:3 para:ortho",
)
plt.plot(df_brink_T, df_brink_Cv, "+g")
plt.plot(df_eucken_T, df_eucken_Cv, "+g")
plt.plot(df_gia_T, df_gia_Cv, "+g")
plt.plot(df_parting_T, df_parting_Cv, "+g")
plt.plot(df_ce_T, df_ce_Cv, "+g", label="experimental data")
plt.legend(loc="upper right", frameon=False)
plt.xlim(10, 400)
plt.ylim(-0.1, 2.8)
plt.xlabel("Temperature, K")
plt.ylabel("Cv (rotational)/R")
plt.tight_layout()
plt.show()
```

```python
import qiskit.tools.jupyter

%qiskit_version_table
```

Version Information

Qiskit Software	Version

qiskit-terra	0.19.1

qiskit-aer	0.10.2

qiskit-ignis	0.7.0

qiskit-ibmq-provider	0.18.3

qiskit	0.34.1

qiskit-nature	0.3.0

qiskit-finance	0.3.0

qiskit-optimization	0.3.0

qiskit-machine-learning	0.3.0

System information

Python version	3.8.12

Python compiler	GCC 9.4.0

Python build	default, Oct 12 2021 21:59:51

OS	Linux

CPUs	8

Memory (Gb)	31.400043487548828

Thu Jan 20 14:39:52 2022 UTC

Project Files/Thermodynamics Observables Calculation with Quantum Computer.ipynb


```python
%%HTML
<!-- Mejorar visualización en proyector -->
<style>
.rendered_html {font-size: 1.2em; line-height: 150%;}
div.prompt {min-width: 0ex; padding: 0px;}
.container {width:95% !important;}
</style>
```

```python
%autosave 0
%matplotlib notebook
import numpy as np
import matplotlib.pyplot as plt
```

# *Deep Reinforced Learning*

Hoy en día el estado del arte en el reconocimiento de patrones está dominado por las **redes neuronales profundas**

- Visión Computacional: Redes Neuronales Convolucionales, Adversarios generativos
- Reconocimiento de habla: Redes Recurrentes, WaveNet
- Procesamiento de lenguaje Natural: Transformers

Las redes neuronales son excelentes para representar el mundo: modelos

> Podemos aprovechar esta capacidad para diseñar mejores algoritmos de aprendizaje reforzado

# Recuerdo: Value Learning

En este tipo de algoritmos usamos un política de máxima utilidad para escoger acciones

$$
\pi^*(s) = \text{arg} \max_{a\in \mathcal{A}} Q(s, a)
$$

Y el problema entonces se reduce a aprender **Q**, *e.g* Q-Learning

Sin embargo Q-learning tiene limitaciones:
- requiere de heurísticas para explorar
- espacio de acciones debe ser discreto
- espacio de estados debe ser discreto

De hecho el estado no puede ser demasiado grande, como veremos a continuación

#### Ambiente: [Space invaders](https://es.wikipedia.org/wiki/Space_Invaders)

Originalmente un juego de Arcade lanzando en 1978, en 1980 tuvo una versión para ATARI 2600

> El objetivo es derrivar a los extraterrestres usando un cañon antes de que desciendan a la Tierra

- El cañon puede moverse a la izquierda, a la derecha y disparar
- Hay cuatro escudos con los que el cañon puede protegerse de los disparos enemigos
- Mientras menos enemigos más rápido se mueven

```python
import gym
from time import sleep

env = gym.make("SpaceInvaders-v0")
env.reset()
end = False

while not end:
    a = env.action_space.sample()
    s, r, end, info = env.step(a)
    env.render()
    sleep(.02)
```

```python
env.close()
```

```python
# Espacio de estados
display(env.observation_space)
# Espacio de acciones
display(env.action_space)
display(env.action_space.n)
display(env.env.get_action_meanings())
```

Por ejemplo si usamos como estado un stack de 4 imágenes consecutivas y asumimos que cada pixel tiene 255 niveles:

$$
255^{210\cdot160\cdot3\cdot4}
$$

# Aproximación de funciones

Claramente, el espacio de estados del ejemplo anterior es imposible de mantener en una tabla Q

Ojo: este espacio de estados está aun lejos de un problema del mundo real

> Nos va a faltar memoría para guardar la tabla y datos para poder entrenar nuestro agente

¿Qué podemos hacer?

> Usar una representación más compacta para Q

En lugar de tener una tabla con todos las combinaciones estado/acción podemos

> Aproximar **Q** usando un **modelo paramétrico**

Esta es la idea principal tras *Value function approximation* (VFA) y *Q function approximation*

El caso más sencillo es usar un **modelo lineal en sus parámetros**

$$
\begin{align}
\hat Q_\theta(s,a) &= \theta_0 \phi_0(s, a) + \theta_1 \phi_1(s, a) + \theta_2 \phi_2(s, a) + \ldots + \theta_M \phi_M (s,a) \nonumber \\
&= \sum_{j=0}^M \theta_j \phi_j (s,a) \nonumber
\end{align}
$$

donde
- $\{ \theta\}$ es un vector de parámetros con $M+1$ componentes
- $\{\phi\}$ es un conjunto de funciones base, *e.g.* polinomios, Fourier, árbol de decisión, kernels

En lugar de aprender $Q$ explicitamente el objetivo es aprender $\theta$

> La cantidad de parámetros es ahora independiente de la dimensionalidad del estado

Al igual que antes nuestro objetivo es acercanos a la solución de la Ecuación de Bellman

Podemos escribir esto como el siguiente problema de optimización

$$
\min_\theta \| R(s,a) + \gamma \max_{a' \in \mathcal{A}} \hat Q_\theta( s',a') - \hat Q_\theta(s,a)\|^2
$$

de donde podemos aprender $\theta$ iterativamente usando usando gradiente descendente

$$
\theta_j \leftarrow \theta_j + 2 \alpha \left(R(s,a) + \gamma \max_{a' \in \mathcal{A}} \hat Q_\theta (s',a') - \hat Q_\theta(s,a) \right) \phi_j(s,a)
$$

Sin embargo, un modelo lineal podría ser muy limitado

En la unidad 1 estudiamos el estado del arte en aproximación de funciones: **redes neuronales artificiales**

> A continuación veremos como usar redes profundas para aproximar la función Q

# Deep Q-Network (DQN)

> En [(Minh et al. 2013)](https://arxiv.org/abs/1312.5602) se usaron redes neuronales profundas de tipo convolucional para resolver una serie de juegos de ATARI con Aprendizaje Reforzardo obteniendo [desempeño sobre-humano en muchos de las pruebas](https://deepmind.com/blog/article/deep-reinforcement-learning). El modelo, llamado *Deep Q-network*, utiliza como estado el valor de todos los píxeles de cuatro cuadros consecutivos.

La idea clave es

> Aprovechar la capacidad de las redes neuronales profundas para representar datos complejos, e.g. imágenes

o más en concreto

> Aproximar la función Q usando una red convolucional entrenada directamente sobre los píxeles

Veremos primero una formulación general y luego su aplicación al caso de imágenes

### Modelo

- La entrada de la red es el estado $s$.
    - El vector de estado puede tener valores continuos o discretos
- La salida de la red neuronal son los valores $Q(s, a_1), Q(s, a_2), \ldots, Q(s, a_N)$
    - Se considera un espacio de acciones discreto
    - Esto es más eficiente que considerar $a'$ como una entrada y retornar $Q(s,a')$
- La cantidad y tipo de las capas intermedias es decisión del usuario
    - Si tenemos datos continuos (atributos) usamos capas completamente conectadas
    - Si usamos píxeles es natural usar capas convolucionales
- La función de perdida que se ocupa en DQN es el error cuadrático medio entre la ecuación de Bellman y la predicción de la red

$$
\mathcal{L}(\theta) = \mathbb{E}\left[\left \| R(s,a) + \gamma \max_{a' \in \mathcal{A}} Q_\theta(s', a') - Q_\theta(s, a)\right \|^2\right]
$$

Recordemos: $s'$ es el estado al que llegamos luego de ejecutar $a$ sobre $s$

# DQN con estados continuos: El retorno del carro con péndulo

Previamente fue necesario discretizar el estado para construir la matriz Q

En DQN podemos obviar este paso y sus complicaciones

Vamos a usar una red neuronal con 3 capas completamente conectadas, 4 entradas y 2 salidas

```python
import torch

class MultilayerPerceptron(torch.nn.Module):
    def __init__(self, n_input, n_output, n_hidden=10):
        super(type(self), self).__init__()
        self.hidden1 = torch.nn.Linear(n_input, n_hidden)
        self.hidden2 = torch.nn.Linear(n_hidden, n_hidden)
        self.output = torch.nn.Linear(n_hidden, n_output)
        self.activation = torch.nn.ReLU()

    def forward(self, x):
        h = self.activation(self.hidden1(x))
        h = self.activation(self.hidden2(h))
        return  self.output(h)
```

# Experience Replay y Memory replay

El entrenamiento usando sólo una instancia es bastante ruidoso (y un poco lento)

Consideremos también que estamos entrenando con muestras muy correlacionadas (no iid)

Sabemos que esto puede introducir sesgos el entrenamiento de una red neuronal

> Para entrenar la DQN adecuadamente  (Minh et al 2013) propone un astuto "truco" llamado *Experience replay*

Esto consiste en almacenar la historia del agente en una memoria: *replay memory*

Cada elemento en la memoria es una tupla $(s_t, a_t, r_{t+1}, s_{t+1})$

Con esto se crean mini-batches en orden aleatorio para entrenar

```python
class ReplayMemory:

    def __init__(self, state_dim, memory_length=2000):
        self.length = memory_length
        self.pointer = 0
        self.filled = False
        # Tensores vacíos para la historia
        self.s_current = torch.zeros((memory_length,) + state_dim)
        self.s_future = torch.zeros((memory_length,) + state_dim)
        self.a = torch.zeros(memory_length, 1, dtype=int)
        self.r = torch.zeros(memory_length, 1)
        # Adicionalmente guardaremos la condición de término
        self.end = torch.zeros(memory_length, 1, dtype=bool)

    def push(self, s_current, s_future, a, r, end):
        # Agregamos una tupla en la memoria
        self.s_current[self.pointer] = s_current
        self.s_future[self.pointer] = s_future
        self.a[self.pointer] = a
        self.r[self.pointer] = r
        self.end[self.pointer] = end
        if self.pointer + 1 == self.length:
            self.filled = True
        self.pointer =  (self.pointer + 1) % self.length

    def sample(self, size=64):
        # Extraemos una muestra aleatoria de la memoria
        if self.filled:
            idx = np.random.choice(self.length, size)
        elif self.pointer > size:
            idx = np.random.choice(self.pointer, size)
        else:
            return None
        return self.s_current[idx], self.s_future[idx], self.a[idx], self.r[idx], self.end[idx]

```

# *Double DQN*

En [(van Hasselt, Guer y Silver, 2015)](https://arxiv.org/pdf/1509.06461.pdf) los autores notaron un problema importante en DQN.

> Cuando calculamos $Q(s, a)$ y $\max Q(s', a')$ usando la misma red es muy posible que sobre-estimemos la calidad de nuestro objetivo. Adicionalmente si el objetivo cambia constantemente el entrenamiento será inestable

La solución propuesta consiste en usar redes neuronales distintas para la escoger la acción y para calcular el objetivo (ecuación de Bellman)

$$
\mathcal{L}(\theta, \phi) = \mathbb{E}\left[\left \| R(s,a) + \gamma Q_\phi\left(s', \text{arg}\max_{a' \in \mathcal{A}} Q_\theta(s', a')\right) - Q_\theta(s, a)\right\|^2\right]
$$

Usamos $Q_\theta$ con parámetros $\theta$ para escoger la acción: *policy network*

Usamos $Q_\phi$ con parámetros $\phi$ para construir el objetivo: *target network*

- Ambas redes comparten arquitectura y cantidad de neuronas
- Sólo se optimiza la *policy network*
- Después de un cierto número de épocas los parametros de la policy network "se copian" en la *target network*

¿Cada cuantas épocas actualizo la *target network*? Otro hyperparámetro para el algoritmo...

## Otras consideraciones

- El optimizador a usar es ADAM: Gradiente descendente con tasa de aprendizaje adaptiva
- Se usa la heurística $\epsilon$ greedy para favorecer la exploración

```python
import torch
import copy

class DeepQNetwork:
    def __init__(self, q_model, gamma=0.99, double_dqn=False, learning_rate=1e-3,
                 target_update_freq=500, clip_grads=True, clip_error=True):
        self.double_dqn = double_dqn
        self.gamma = gamma
        self.q_policy = q_model
        self.n_output = q_model.output.out_features
        self.clip_error = clip_error
        self.clip_grads = clip_grads
        self.update_counter = 0
        self.target_update_freq = target_update_freq

        self.criterion = torch.nn.MSELoss()
        self.optimizer = torch.optim.RMSprop(self.q_policy.parameters(), lr=learning_rate)

        if double_dqn:
            self.q_target = copy.deepcopy(self.q_policy)
            self.q_target.eval()

    def select_action(self, state, epsilon=0.):
        # Estrategia epsilon greedy para seleccionar acción
        if torch.rand(1).item() < 1. - epsilon:
            self.q_policy.eval()
            with torch.no_grad():
                q = self.q_policy(state)[0]
                a = q.argmax().item()
                q = q[a]
            self.q_policy.train()
        else:
            q = None
            a = torch.randint(high=self.n_output, size=(1,)).item()

        return a, q

    def update(self, mini_batch):
        self.update_counter += 1
        state, state_next, action, reward, end = mini_batch
        # Calcular Q
        q_current = self.q_policy(state).gather(1, action)
        with torch.no_grad():
            if not self.double_dqn:
                q_next_best = self.q_policy(state_next).max(1, keepdim=True)[0]
            else:
                action_next = self.q_policy(state_next).argmax(dim=1, keepdim=True)
                q_next_best = self.q_target(state_next).gather(1, action_next)
        # Construir el target: r + gamma*max Q(s', a')
        td_target = reward
        td_target[~end] += self.gamma*q_next_best[~end]
        td_target[end] = -1.
        # Calcular pérdido y sus gradientes
        self.optimizer.zero_grad()
        loss = self.criterion(q_current, td_target)
        if self.clip_error:
            loss.clamp_(-1., 1.)
        loss.backward()
        # Cortar gradientes grandes (mejora la estabilidad)
        if self.clip_grads:
            for param in self.q_policy.parameters():
                param.grad.data.clamp_(-1., 1.)
            # torch.nn.utils.clip_grad.clip_grad_norm_(self.model.parameters(), 10)
        # Actualizar
        self.optimizer.step()
        # Transfer policy to target
        self.transfer_policy2target()
        # Retornar el valor de la loss
        return loss.item()

    def transfer_policy2target(self):
        if self.double_dqn:
            if self.update_counter % self.target_update_freq == 0:
                self.q_target.load_state_dict(self.q_policy.state_dict())
                return True
        return False
```

```python
%matplotlib notebook
import matplotlib.pyplot as plt
from scipy.signal import convolve

fig, ax = plt.subplots(4, figsize=(6, 5), sharex=True, tight_layout=True)

def smooth_data(x, window_length=10):
    return convolve(x, np.ones(window_length)/window_length, mode='valid')

def update_plot(step, episode, smooth_window=10, target=195, target_update=500):
    for ax_ in ax:
        ax_.cla()
    episodes = np.arange((episode))
    ax[0].scatter(episodes, diagnostics['rewards'], s=1)
    if episode > smooth_window:
        ax[0].plot(episodes[:-smooth_window+1],
                   smooth_data(diagnostics['rewards']), alpha=0.5, lw=2)
    ax[1].plot(episodes, diagnostics['loss'])
    ax[2].plot(episodes, np.array(diagnostics['q_sum'])/(np.array(diagnostics['q_N'])+1e-4))
    #for update_time in diagnostics['target_updates']:
    #    ax[1].plot([update_time, update_time],
    #               [diagnostics['loss'].min(), diagnostics['loss'].max()], 'k--', alpha=0.1)

    ax[0].plot(episodes, [target]*len(episodes), 'k--')
    ax[0].set_ylabel('Recompensa');
    ax[1].set_ylabel('Loss')
    ax[2].set_ylabel('Q promedio')
    ax[3].plot(episodes, epsilon(episodes))
    ax[3].set_ylabel('Epsilon')
    ax[3].set_xlabel('Episodios')
    ax[0].set_title("Paso %d" % (step))
    fig.canvas.draw()
```

## Experimentos

Usemos la clase DeepQNetwork para entrenar una agente que resuelva el problema del carro con péndulo

- Si uso `memory_length=1` es equivalente a no usar *replay memory*. Mientras más corta es la memoria más ruidoso es el entrenamiento.
- Con el parámetro `double_dqn` podemos activar la red objetivo que se actualizará cada `target_update_freq` pasos
- Otros parámetros a considerar son `gamma`, el tamaño del batch, la tasa de aprendizaje y la arquitectura de la red

```python
import numpy as np
import gym
from tqdm.notebook import tqdm

torch.manual_seed(123)

env = gym.make("CartPole-v0")
n_state = (env.observation_space.shape[0],) # Número de estados
n_action = env.action_space.n # Número de acciones

dqn_model = DeepQNetwork(q_model=MultilayerPerceptron(n_state[0], n_action, n_hidden=16),
                         gamma = 0.99,
                         double_dqn=True,
                         target_update_freq=100,
                         learning_rate=1e-4)

def epsilon(episode, epsilon_init=1., epsilon_end=0.1, epsilon_rate=1e-2):
    return epsilon_end + (epsilon_init - epsilon_end) * np.exp(-epsilon_rate*episode)

memory = ReplayMemory(n_state, memory_length=100000)

diagnostics = {'rewards': [0], 'loss': [0],
               'q_sum': [0], 'q_N': [0]}

episode = 1
end = False
state = env.reset()
for step in tqdm(range(30000)):
    # Escoger acción
    state = torch.tensor(state).float()
    a, q = dqn_model.select_action(state.unsqueeze(0), epsilon(episode))
    if q is not None:
        diagnostics['q_sum'][-1] += q
        diagnostics['q_N'][-1] += 1
    # Ejecutar acción
    s, r, end, info = env.step(a)
    # Guardar en memoria
    memory.push(state, torch.tensor(s).float(), a, torch.tensor(r/200.), end)
    # Actualizar modelo
    mini_batch = memory.sample(32)
    if not mini_batch is None:
        diagnostics['loss'][-1] += dqn_model.update(mini_batch)
    # Guardar nuevo estado y continuar
    diagnostics['rewards'][-1] += r
    state = s
    # Preparar siguiente episodio
    if end:
        if episode % 25 == 0:
            update_plot(step, episode)
        episode += 1
        end = False
        state = env.reset()
        diagnostics['rewards'].append(0)
        diagnostics['loss'].append(0)
        diagnostics['q_sum'].append(0)
        diagnostics['q_N'].append(0)
```

## Intuición

> La *loss* está subiendo, mi algoritmo está mal

No necesariamente, recordemos que el "training set" está cambiando con cada paso

Es normal ver que los *loss* tenga subidas abruptas

En cierto modo estas subidas son deseables pues significa que nuestro algoritmo está "sorprendiendose" con nuevos estados

> La recompensa promedio se fue en picada, mi algoritmo está mal

Lo que está ocurriendo es un problema común en deep RL que se llama: **"olvido catastrófico"**

El modelo es cada vez más bueno resolviendo el problema y la memoria se llena de "buenas transiciones"

Los *minibatches* ya no tienen ejemplos erroneos: El modelo ya no sabe lo que está mal

Se puede combatir con *prioritized replay memory* o con otros heurísticas

- Reservar un espacio en la memoria para las transiciones (de exploración) iniciales
- **Early stopping:** Guardar el modelo de mayor recompensa y detener el aprendizaje si la recompensa no sube en un cierto número de episodios

##### Probando el agente DQN

```python
import numpy as np
import gym

env = gym.make("CartPole-v0")
```

```python
env.reset()

for k in range(500):
    env.render()
    state = torch.from_numpy(np.array(env.state).astype('float32')).unsqueeze(0)
    a, q = dqn_model.select_action(state)
    s_future, r, end, info = env.step(a)
    if r == 0:
        display(k)
        break
display(end, k)
```

```python
env.close()
```

# [Rainbow DQN](https://arxiv.org/pdf/1710.02298.pdf)

Rainbow DQN es un trabajo que reune todos los avances relacionados a DQN

1. *Double DQN*: Usar un modelo separado para calcular los targets
1. *Replay memory*: Almacenar experiencias pasadas y muestrear aleatoriamente de la memoria para disminuir la correlación
1. [*Prioritized experience replay*](https://arxiv.org/abs/1511.05952): El minibatch se construye ponderando las experiencias anteriores en función del error. Experiencias con más error tienen más probabilidad de ser escogidas. "Se puede aprender más de los ejemplos más difíciles"
1. [*Dueling networks*](https://arxiv.org/abs/1511.06581): Se separa la función $Q(s,a) = V(s) + A(s,a)$, donde cada función es una red neuronal que comparten capas. Con esto se puede aprender el valor de cada estado de forma desacoplada a la importancia relativa de las acciones
1. [*Noisy nets*](https://arxiv.org/abs/1706.10295): Capas de redes neuronales que incluyen un ruido paramétrico en sus parámetros lo que permite explorar sin necesidad de la heurística $\epsilon$-greddy
1. entre otros

```python

```


```python
from IPython.display import Image
Image('../../Python_probability_statistics_machine_learning_2E.png',width=200)
```

Logistic regression is one example of a wider class of Generalized Linear
Models (GLMs). These GLMs have the following three key features

* A target $Y$ variable distributed according to one of the exponential family of distributions (e.g., Normal, binomial, Poisson)

* An equation that links the expected value of $Y$ with a linear combination
  of the observed variables (i.e., $\left\{ x_1,x_2,\ldots,x_n \right\}$).

* A smooth invertible *link* function $g(x)$ such that $g(\mathbb{E}(Y)) = \sum_k \beta_k x_k$

### Exponential Family

Here is the one-parameter exponential family,

$$
f(y;\lambda) = e^{\lambda y - \gamma(\lambda)}
$$

 The *natural parameter* is $\lambda$ and $y$ is the sufficient
statistic. For example, for logistic regression, we have
$\gamma(\lambda)=-\log(1+e^{\lambda})$ and $\lambda=\log{\frac{p}{1-p}}$.

An important property of this exponential family is that

$$
\begin{equation}
\mathbb{E}_{\lambda}(y) = \frac{d\gamma(\lambda)}{d\lambda}=\gamma'(\lambda)
\end{equation}
\label{eq:dgamma} \tag{1}
$$

 To see this, we compute the following,

$$
\begin{align*}
1 &=  \int f(y;\lambda) dy  =  \int e^{\lambda y - \gamma(\lambda)} dy \\
0 &=  \int \frac{df(y;\lambda)}{d\lambda} dy =\int e^{\lambda y-\gamma (\lambda)} \left(y-\gamma'(\lambda)\right) dy  \\
\int y e^{\lambda  y-\gamma (\lambda )} dy &= \mathbb{E}_{\lambda}(y)=\gamma'(\lambda )
\end{align*}
$$

 Using the same technique, we also have,

$$
\mathbb{V}_{\lambda}(Y) = \gamma''(\lambda)
$$

 which explains the usefulness of this generalized notation for the
exponential family.

### Deviance

The scaled Kullback-Leibler divergence is called the *deviance* as
defined below,

$$
D(f_1,f_2) = 2 \int f_1(y) \log{\frac{f_1(y)}{f_2(y)}}dy
$$

**Hoeffding's Lemma.**

Using our exponential family notation, we can write out the deviance as
the following,

$$
\begin{align*}
\frac{1}{2} D(f(y;\lambda_1), f(y;\lambda_2)) & = \int f(y;\lambda_1)\log \frac{f(y;\lambda_1)}{f(y;\lambda_2)}  dy   \\
                                              & = \int f(y;\lambda_1) ((\lambda_1-\lambda_2) y -(\gamma(\lambda_1)-\gamma(\lambda_2))) dy \\
                                              & = \mathbb{E}_{\lambda_1}  [ (\lambda_1-\lambda_2) y -(\gamma(\lambda_1)-\gamma(\lambda_2)) ] \\
                                              & = (\lambda_1-\lambda_2) \mathbb{E}_{\lambda_1}(y) -(\gamma(\lambda_1)-\gamma(\lambda_2)) \\
                                              & = (\lambda_1-\lambda_2) \mu_1 -(\gamma(\lambda_1)-\gamma(\lambda_2))
\end{align*}
$$

 where $\mu_1:=\mathbb{E}_{\lambda_1}(y)$. For the maximum likelihood
estimate $\hat{\lambda}_1$, we have $\mu_1=y$. Plugging this
into the above equation gives the following,

$$
\begin{align*}
\frac{1}{2} D(f(y;\hat{\lambda}_1),f(y;\lambda_2))&=(\hat{\lambda}_1-\lambda_2) y -(\gamma(\hat{\lambda}_1)-\gamma(\lambda_2))  \\
&= \log{f(y;\hat{\lambda}_1)} - \log{f(y;\lambda_2)} \\
&= \log\frac{f(y;\hat{\lambda}_1)}{f(y;\lambda_2)}
\end{align*}
$$

 Taking the negative exponential of both sides gives,

$$
f(y;\lambda_2) = f(y;\hat{\lambda}_1) e^{-\frac{1}{2} D(f(y;\hat{\lambda}_1),f(y;\lambda_2)) }
$$

 Because $D$ is always non-negative, the likelihood
is maximized when the the deviance is zero. In particular,
for the scalar case, it means that $y$ itself is the
best maximum likelihood estimate for the mean. Also,
$f(y;\hat{\lambda}_1)$ is called the *saturated* model.
We write Hoeffding's Lemma as the following,

$$
\begin{equation}
f(y;\mu) = f(y;y)e^{-\frac{1}{2} D(f(y;y),f(y;\mu))}
\end{equation}
\label{eq:lemma} \tag{2}
$$

  to emphasize that $f(y;y)$ is the likelihood function when the
mean is replaced by the sample itself and $f(y;\mu)$ is the likelihood function
when the mean is replaced by $\mu$.

Vectorizing Equation ([2](#eq:lemma)) using mutual independence gives the
following,

$$
f(\mathbf{y};\boldsymbol{\mu}) = e^{-\sum_i D(y_i,\mu_i)} \prod f(y_i;y_i)
$$

 The idea now is to minimize the deviance by deriving,

$$
\boldsymbol{\mu}(\boldsymbol{\beta}) = g^{-1}(\mathbf{M}^T\boldsymbol{\beta})
$$

 This means the the MLE $\hat{\boldsymbol{\beta}}$ is the
best $p\times 1$ vector $\boldsymbol{\beta}$ that minimizes the
total deviance where $g$ is the *link* function and $\mathbf{M}$ is
the $p\times n$ *structure* matrix. This is the key step with GLM
estimation because it reduces the number of parameters from $n$ to
$p$. The structure matrix is where the associated $x_i$ data enters
into the problem.  Thus, GLM maximum likelihood fitting minimizes the
total deviance like plain linear regression minimizes the sum of
squares.

With the following,

$$
\boldsymbol{\lambda} = \mathbf{M}^T \boldsymbol{\beta}
$$

 with $2\times n$ dimensional $\mathbf{M}$.  The corresponding joint
density function is the following,

$$
f(\mathbf{y};\beta)=e^{\boldsymbol{\beta}^T\boldsymbol{\xi}-\psi(\boldsymbol{\beta})} f_0(\mathbf{y})
$$

 where

$$
\boldsymbol{\xi} = \mathbf{M} \mathbf{y}
$$

 and

$$
\psi(\boldsymbol{\beta}) = \sum \gamma(\mathbf{m}_i^T \boldsymbol{\beta})
$$

 where now the sufficient statistic is $\boldsymbol{\xi}$ and the
parameter vector is $\boldsymbol{\beta}$, which fits into our exponential
family format, and $\mathbf{m}_i$  is the $i^{th}$ column of $\mathbf{M}$.

Given this joint density, we can compute the log likelihood as the following,

$$
\ell = \boldsymbol{\beta}^T\boldsymbol{\xi}-\psi(\boldsymbol{\beta})
$$

 To maximize this likelihood, we take the derivative of this with
respect to $\boldsymbol{\beta}$ to obtain the following,

$$
\frac{d\ell}{d\boldsymbol{\beta}}=\mathbf{M}\mathbf{y}-\mathbf{M}\boldsymbol{\mu}(\mathbf{M}^T\boldsymbol{\beta})
$$

 since $\gamma'(\mathbf{m}_i^T \boldsymbol{\beta}) =
\mathbf{m}_i^T \mu_i(\boldsymbol{\beta})$ and (c.f. Equation ([1](#eq:dgamma))),
$\gamma' = \mu_{\lambda}$.
Setting this derivative equal to zero gives the conditions
for the maximum likelihood solution,

$$
\begin{equation}
\mathbf{M}(\mathbf{y}- \boldsymbol{\mu}(\mathbf{M}^T\boldsymbol{\beta})) = \mathbf{0}
\end{equation}
\label{eq:conditions} \tag{3}
$$

 where $\boldsymbol{\mu}$ is the element-wise inverse of the link function. This
leads us to exactly the same place we started: trying to regress $\mathbf{y}$ against
$\boldsymbol{\mu}(\mathbf{M}^T\boldsymbol{\beta})$.

### Example

The structure matrix $\mathbf{M}$ is where the $x_i$ data
associated with the corresponding $y_i$ enters the problem. If we choose

$$
\mathbf{M}^T = [\mathbf{1}, \mathbf{x}]
$$

 where $\mathbf{1}$ is an $n$-length vector and

$$
\boldsymbol{\beta} = [\beta_0, \beta_1]^T
$$

 with $\mu(x) = 1/(1+e^{-x})$, we have the original logistic regression problem.

Generally, $\boldsymbol{\mu}(\boldsymbol{\beta})$ is a nonlinear
function and thus we regress against our transformed variable $\mathbf{z}$

$$
\mathbf{z} = \mathbf{M}^T\boldsymbol{\beta} + \diag(g'(\boldsymbol{\mu}))(\mathbf{y}-\boldsymbol{\mu}(\mathbf{M}^T\boldsymbol{\beta}))
$$

 This fits the format of the Gauss Markov
(see [ch:stats:sec:gauss](#ch:stats:sec:gauss)) problem and has the

following solution,

$$
\begin{equation}
\hat{\boldsymbol{\beta}}=(\mathbf{M} \mathbf{R}_z^{-1}\mathbf{M}^T)^{-1}\mathbf{M} \mathbf{R}_z^{-1}\mathbf{z}
\end{equation}
\label{eq:bhat} \tag{4}
$$

 where

$$
\mathbf{R}_z:=\mathbb{V}(\mathbf{z})=\diag(g'(\boldsymbol{\mu}))^2\mathbf{R}=\mathbf{v}(\mu)\diag(g'(\boldsymbol{\mu}))^2\mathbf{I}
$$

 where $g$ is the link function and $\mathbf{v}$ is the variance
function on the designated distribution of the $y_i$.
Thus, $\hat{\boldsymbol{\beta}}$ has the following covariance matrix,

$$
\mathbb{V}(\hat{\boldsymbol{\beta}}) = (\mathbf{M}\mathbf{R}_z^{-1}\mathbf{M}^T)^{-1}
$$

 These results allow inferences about the estimated parameters
$\hat{\boldsymbol{\beta}}$.  We can easily write Equation ([4](#eq:bhat))
as an iteration as follow,

$$
\hat{\boldsymbol{\beta}}_{k+1}=(\mathbf{M} \mathbf{R}_{z_k}^{-1}\mathbf{M}^T)^{-1}\mathbf{M} \mathbf{R}_{z_k}^{-1}\mathbf{z}_k
$$

### Example

Consider the data shown in [Figure](#fig:glm_003). Note that the variance of
the data increases for each $x$ and the data increases as a power of $x$ along
$x$. This makes this data a good candidate for a Poisson GLM with $g(\mu) =
\log(\mu)$.

```python
%matplotlib inline

from scipy.stats.distributions import poisson
import statsmodels.api as sm
from matplotlib.pylab import subplots
import numpy as np

def gen_data(n,ns=10):
    param = n**2
    return poisson(param).rvs(ns)

fig,ax=subplots()
xi = []
yi = []
for i in [2,3,4,5,6]:
    xi.append([i]*10)
    yi.append(gen_data(i))
    _=ax.plot(xi[-1],yi[-1],'ko',alpha=.3)

_=ax.axis(xmin=1,xmax=7)
xi = np.array(xi)
yi = np.array(yi)
x = xi.flatten()
y = yi.flatten()
_=ax.set_xlabel('x');
fig.savefig('fig-machine_learning/glm_003.png')
```

Some data for Poisson example.

We can use our iterative matrix-based approach. The following code
initializes the iteration.

```python
M   = np.c_[x*0+1,x].T
gi  = np.exp             # inverse g link function
bk  = np.array([.9,0.5]) # initial point
muk = gi(M.T @ bk).flatten()
Rz  = np.diag(1/muk)
zk  = M.T @ bk + Rz @ (y-muk)
```

 and this next block establishes the main iteration

```python
while abs(sum(M @ (y-muk))) > .01: # orthogonality condition as threshold
   Rzi = np.linalg.inv(Rz)
   bk = (np.linalg.inv(M @ Rzi @ M.T)) @ M @ Rzi @ zk
   muk = gi(M.T @ bk).flatten()
   Rz =np.diag(1/muk)
   zk = M.T @ bk + Rz @ (y-muk)
```

 with corresponding final $\boldsymbol{\beta}$ computed as the
following,

```python
print(bk)
```

    [0.72758176 0.48763741]

 with corresponding estimated $\mathbb{V}(\hat{\boldsymbol{\beta}})$ as

```python
print(np.linalg.inv(M @ Rzi @ M.T))
```

    [[ 0.01850392 -0.0035621 ]
     [-0.0035621   0.00072885]]

 The orthogonality condition Equation ([3](#eq:conditions)) is the following,

```python
print(M @ (y-muk))
```

    [-4.90472131e-05 -2.62970365e-04]

For comparison, the `statsmodels` module provides the Poisson GLM object.
Note that the reported standard error is the square root of the
diagonal elements of $\mathbb{V}(\hat{\boldsymbol{\beta}})$. A plot of
the data and the fitted model is shown below in Figure ([fig:glm_004](#fig:glm_004)).

```python
pm=sm.GLM(y, sm.tools.add_constant(x),
                        family=sm.families.Poisson())
pm_results=pm.fit()
pm_results.summary()
```

Generalized Linear Model Regression Results

  Dep. Variable:	           y	          No. Observations:  	      50

  Model:	                  GLM	         Df Residuals:      	      48

  Model Family:	         Poisson	       Df Model:          	       1

  Link Function:	          log	         Scale:             	   1.0000

  Method:	                IRLS	         Log-Likelihood:    	  -143.03

  Date:	            Tue, 27 Aug 2019	   Deviance:          	   63.998

  Time:	                13:19:09	       Pearson chi2:      	    62.0

  No. Iterations:	          5

  Covariance Type:	     nonrobust

    	       coef	     std err	      z	      P>|z|	  [0.025	    0.975]

  const	     0.7276	     0.136	     5.348	  0.000	     0.461	     0.994

  x1	        0.4876	     0.027	    18.060	  0.000	     0.435	     0.541

```python
b0,b1=pm_results.params
fig,ax=subplots()
_=ax.plot(xi[:,0],np.exp(xi[:,0]*b1+b0),'-o');
_=ax.plot(xi,yi,'ok',alpha=.3);
fig.savefig('fig-machine_learning/glm_004.png')
_=ax.set_xlabel('x');
```

Fitted using the Poisson GLM.