chebyshev.ipynb was getting too big, so we're starting a new one

```python
import sys
from sympy import init_session
init_session()
from mpmath import mpf
%matplotlib inline
```

    IPython console for SymPy 1.0.1.dev (Python 3.5.2-64-bit) (ground types: gmpy)

    These commands were executed:
    >>> from __future__ import division
    >>> from sympy import *
    >>> x, y, z, t = symbols('x y z t')
    >>> k, m, n = symbols('k m n', integer=True)
    >>> f, g, h = symbols('f g h', cls=Function)
    >>> init_printing()

    Documentation can be found at http://docs.sympy.org/dev

Now we try to mechanise the CRAM process

```python
def bisect_roots(expr, bounds, eps=1e-7, division_cutoff=30, solver='bisect', **kwargs):
    """
    Numerically find all the roots of expr in bounds via bisection

    division_cutoff is the fraction of bounds to check for roots.
    One should expect the roots to be no more than |bounds|/division_cutoff apart.
    """
    roots = []
    intervals = [bounds]
    var = expr.free_symbols.pop()
    N = 0
    while intervals:
        N += 1
        #print(intervals)
        b = intervals.pop()
        try:
            root = nsolve(expr, b, solver=solver, **kwargs)
        except ValueError:
            # No root found
            if (b[1] - b[0]) > (bounds[1] - bounds[0])/division_cutoff and sign(expr.evalf(subs={var: b[0]})) == sign(expr.evalf(subs={var: b[1]})):
                # The bisect method will only work if the interval has alternate signs.
                # If not, it will miss 2 roots, so try again with split intervals.
                intervals.append([b[0], (b[0] + b[1])/2])
                intervals.append([(b[0] + b[1])/2, b[1]])
                continue
        else:
            if root not in roots:
                roots.append(root)
                intervals.append([b[0], root - eps])
                intervals.append([root + eps, b[1]])

    print(N)
    return sorted(roots)

bisect_roots(sin(x), [-10, 13])
```

```python
def nsolve_intervals(expr, bounds, division=30, warn=False, verbose=False, solver='bisect', **kwargs):
    """
    Divide bounds into division intervals and nsolve in each one
    """
    if verbose:
        warn = True
    roots = []
    L = bounds[1] - bounds[0]
    for i in range(division):
        interval = [bounds[0] + i*L/division, bounds[0] + (i + 1)*L/division]
        try:
            if verbose:
                print("Solving in interval", interval)
            root = nsolve(expr, interval, solver=solver, **kwargs)
        except ValueError:
            if verbose:
                print("No solution found")
            continue
        else:
            if interval[0] < root < interval[1]:
                if verbose:
                    print("Solution found:", root)
                roots.append(root)
            else:
                if warn:
                    print(root, "is not in", interval, 'discarding', file=sys.stderr)

    return roots

#nsolve_intervals(D, [-1, 0.99])
```

```python
def general_rat_func(d, x, chebyshev=False):
    """
    Return a general rational function with numerator and denominator degree d

    Returns a tuple, (rational function, numerator coefficients, denominator coefficients)

    The constant coefficient in the denominator is always 1.
    """
    num_coeffs = symbols('p:%s' % (d+1))
    den_coeffs = symbols('q:%s' % (d+1))[1:]
    if chebyshev:
        num = Add(*(Mul(c, chebyshevt(i, x)) for i, c in enumerate(num_coeffs)))
        den = Add(*(Mul(c, chebyshevt(i, x)) for i, c in enumerate([1, *den_coeffs])))
        rat_func = num/den
    else:
        rat_func = Poly(reversed(num_coeffs), x)/Poly([*reversed(den_coeffs), 1], x)
    return rat_func, num_coeffs, den_coeffs

general_rat_func(8, x), general_rat_func(8, x, chebyshev=True)
```

```python
from sympy.utilities.decorator import conserve_mpmath_dps

@conserve_mpmath_dps
def CRAM_exp2(degree, prec=128, *, max_loops=10):
    #mpmath.mp.dps = prec

    epsilon = symbols("epsilon")
    i = symbols("i")
    #c = S(1)/2
    #c = 10
    c = degree*0.6

    r, num_coeffs, den_coeffs = general_rat_func(degree, t, chebyshev=True)
    E = exp(c*(t + 1)/(t - 1)) - r
    expr = E + (-1)**i*epsilon
    expr = expr*r.as_numer_denom()[1]
    expr = simplify(expr)

    points = [chebyshevt_root(2*(degree + 1) + 1, 2*(degree + 1) - j) for j in range(1, 2*(degree + 1) + 1)]
    points = [i.evalf(prec) for i in points]
    for iteration in range(max_loops):
        print('-'*80)
        print("Iteration %s:" % iteration)
        system = Tuple(*[expr.subs({i: j, t: points[j]}) for j in range(2*degree + 1)])
        system = system + Tuple(expr.replace(exp, lambda i: 0).subs({i: 2*degree + 1, t: 1}))
        #print(system)
        #print([*num_coeffs, *den_coeffs, epsilon])
        sol = dict(zip([*num_coeffs, *den_coeffs, epsilon], nsolve(system, [*num_coeffs, *den_coeffs, epsilon], [*[1]*(2*(degree + 1) - 1), 0], prec=prec)))
        print('sol', sol)
        print('system.subs(sol)', [i.evalf() for i in system.subs(sol)])
        D = diff(E.subs(sol), t)
        plot(E.subs(sol), (t, -1, 0.999), adaptive=False, nb_of_points=1000)
        plot(E.subs(sol), (t, -0.5, 0.5), adaptive=False, nb_of_points=1000)
        #plot(E.subs(sol), (t, 0.9, 1))
        # we can't use 1 because of the singularity
        print(E.subs(sol))
        points = [-1, *nsolve_intervals(D, [-1, 0.999999], maxsteps=10000, prec=prec, division=200, warn=True, tol=10**-(2*prec)), 1]
        #points2 = [-1, *nsolve_intervals(D, [-1, 0.999999], maxsteps=300, prec=prec, solver='illinois', verify=True), 1]
        #print('points', points)
        #print('points2', points2)
        print('D', D)
        print('[(i, D.subs(t, i)) for i in points]', [(i, D.subs(t, i)) for i in points])
        assert len(points) == 2*(degree + 1), len(points)
        Evals = [E.evalf(prec, subs={**sol, t: point}) for point in points[:-1]] + [-r.evalf(prec, subs={**sol, t: 1})]
        print('Evals', Evals)
        maxmin = N(max(map(abs, Evals)) - min(map(abs, Evals)))
        print('max - min', maxmin)
        print('epsilon', N(sol[epsilon]))
        if maxmin < 10**-prec:
            print("Converged in", iteration + 1, "iterations.")
            break
    else:
        print("!!!WARNING: DID NOT CONVERGE AFTER", max_loops, "ITERATIONS!!!", file=sys.stderr)

    #print(sol)
    #sol = {i: Rational(str(sol[i])) for i in sol}
    #print(sol)
    inv = solve(-c*(t + 1)/(t - 1) - y, t)[0].subs(y, t)
    n, d = together(r.subs(sol).subs(t, inv)).as_numer_denom() # simplify/cancel here will add degree to the numerator and denominator
    rat_func = (Poly(n)/Poly(d).TC())/(Poly(d)/Poly(d).TC())
    return rat_func.evalf(prec)

#D = CRAM_exp2()

rat_func = CRAM_exp2(15, 1000)
print(rat_func)
plot(rat_func - exp(-t), (t, 0, 100))
```

```python
!say done
```

```python
correct_rat_func2 = (9.92641e-1 + -1.88332e-1*t + 4.21096e-3*t**2)/(1 + 6.69295e-1*t + 5.72258e-1*t**2)
plot(correct_rat_func2 - exp(-t), (t, 0, 100))
```

```python
#plot(rat_func - correct_rat_func2, (t, 0, 100))
```

```python
plot((correct_rat_func2 - exp(-t))/correct_rat_func2, (t, 20, 100))
```

```python
nsolve(D, (-1, -0.39506511254484362 - 1e-20), solver='bisect', tol=1e-20)
```

```python
#import mpmath
#mpmath.findroot?
```

```python
plot(sin(x), (x, -10, 13))
```

```python
nsolve(sin(x), [-10, -0.1], solver='bisect')
```

```python
bisect_roots(D, [-1, 0.9], solver='bisect')
```

```python
plot(D, (t, -1, 1))
```

```python
nsolve(D, [mpf('0.2069106899549174') + 1e-4, 0.999], solver='ridder')
```

```python
intervals = [[i/30, (i+1)/30] for i in range(-30, 29)]
```

```python
def catch_ValueError(f):
    try:
        return f()
    except ValueError:
        pass
```

```python
[catch_ValueError(lambda: nsolve(D, b, solver='bisect')) for b in intervals]
```

```python
sin(x).evalf(subs={x: mpf('9.9999999999999995e-8')})
```

```python
nsolve(D, (-1, 0), solver='bisect')
```

```python
D
```

```python
import mpmath
try:
    mpmath.mp.dps = 128
    print(nsolve(D, (-1, 0), solver='bisect', maxsteps=300))
finally:
    mpmath.mp.dps = 15
```

```python
chebyshevt(2, x)
```

```python
0.9999999999999999
```

```python
0.99999999999999999
```

```python
exp(-(-t - 1)/(2*t - 2))
```

```python
solve((0.57224957904836676579778131095824616229041363386436155632065723326613208320645433529158968043480368460271370103317310738469165646*t**2 + 0.66930154271086975944415360267931286108869186534800442869820775970713699218292282609079303557228276229468949430966691265928683428*t + 1.0))
```

```python
[1, 2, 3][:-1]
```

```python
plot((t + 1)/(t - 1)*(-1/2), (t, -1, 1))
```

```python
solve(y - -(-t - 1)/(2*t - 2), t)
```

```python
exp(-(-t - 1)/(2*t - 2)).evalf(subs={t: 0})
```

```python
len('42109959068982')
```

```python
@conserve_mpmath_dps
def test_increased_dps():
    # Issue 8564
    import mpmath
    mpmath.mp.dps = 128
    x = Symbol('x')
    e1 = x**2 - pi
    q = nsolve(e1, x, 3.0)

    assert abs(sqrt(pi).evalf(128) - q) < 1e-128

test_increased_dps()
```

```python
@conserve_mpmath_dps
def test_mpf_subs():
    import mpmath
    mpmath.mp.dps = 128
    a = mpf('1') + mpf('1e-100')
    print(a)
    print(x.subs(x, a))
    print(Float(a))

test_mpf_subs()
```

```python
@conserve_mpmath_dps
def test_increased_dps2():
    # Issue 8564
    import mpmath
    mpmath.mp.dps = 128
    x = Symbol('x')
    e1 = x**2 - pi
    q = nsolve(e1, x, 3.0)
    print((x**2).subs(x, q))
    print(pi.evalf(128))

test_increased_dps2()
```

```python
p0, p1, p2, q1, q2, epsilon = symbols("p0, p1, p2, q1, q2, epsilon")
r = (p0 + p1*t + p2*t**2)/(1 + q1*t + q2*t**2)
mpf = lambda i: Float(i, 128)
sol = {q1: mpf('-1.1598158896958144389688907541448290768017556418690225475316113676987123000368036156835800708110657203943382445749046292411027838124'), epsilon: mpf('-0.0073586701695804955713909885597251540212160797660122172060482127633180200310633941837926800633336626154349676612971775593725674790502'), p1: mpf('-1.3420582608178928401767377898250826050312440510776128093723998679786607122879166676244905769597498693757444204486264349423147412592'), q2: mpf('0.54706937851182403348832181267224373295821044378627516587652817178767445252898045643900602804552389487580236216086250141313671452195'), p2: mpf('0.73617861351264467252445532341623038921509966014009367227031009417645097851179620934054146565716699654528814524660644163951335402577'), p0: mpf('0.6087293180014645115203537929174427433501475052521283394857491045045741371996335477254671722188240766010031663167379415577767292914')}
other_rat_func ="(0.0042109959068981902319228224147959051043334784348885130731594832554304543920692286150512328715813585751529478908147308247510818304*t**2 - 0.1883335019892743246297189560004799477752569428797900790843481507588522143198473107264222704579544281038255097909548061885535647*t + 0.99264132983041950442860901144027484597878392023398778279395178723668197996893660581620731993666633738456503233870282244062743252)/(0.57224957904836377040707255710162720198078512465710225748045909842276672878045476079146532782980255617241659394936652703249328588*t**2 + 0.66930154271087159949217750010925437712785181665997133501826390508750116474339450131113420220623360917322211168993495402213680723*t + 1.0)"

n, d = together(r.subs(sol).subs(t, (2*t - 1)/(2*t + 1))).as_numer_denom() # simplify/cancel here will add degree to the numerator and denominator
rat_func = (Poly(n)/Poly(d).TC())/(Poly(d)/Poly(d).TC())
print(rat_func)
print(other_rat_func)
del mpf
```

```python
import sys
print('test', file=sys.stderr)
```

```python
10**-10
```

```python
D = (-2*(t + 1)/(2*t - 2)**2 + 1/(2*t - 2))*exp((t + 1)/(2*t - 2)) - (-0.46319625019273218908*t**13 + 11.076239035694458023*t**12 - 75.38527286719552296*t**11 + 242.00660160608425014*t**10 - 389.01043643289017999*t**9 + 165.53549638768474282*t**8 + 538.42392697795239075*t**7 - 1160.0181368287938552*t**6 + 1064.6133061016735644*t**5 - 435.60064981285300771*t**4 - 47.836952574354747137*t**3 + 134.72356135348252232*t**2 - 56.370239030181725748*t + 8.3057523340557002803)*(0.054548679300092563818*t**14 - 1.3501921589697942721*t**13 + 9.0345240868201266233*t**12 - 27.90433797500592764*t**11 + 40.52259728739627843*t**10 - 2.2008514365998888364*t**9 - 97.728246570230795738*t**8 + 181.14960511639832727*t**7 - 159.50642335868838789*t**6 + 56.243320662736403761*t**5 + 26.808952900880779959*t**4 - 41.915371037748377236*t**3 + 21.829567246295711762*t**2 - 5.6442241022971821787*t + 0.6065306597126334236)/(0.033085446442338013506*t**14 - 0.85201838736111215565*t**13 + 6.28210607226629358*t**12 - 22.000600146007659103*t**11 + 38.901043643289017999*t**10 - 18.392832931964971424*t**9 - 67.302990872244048844*t**8 + 165.71687668982769361*t**7 - 177.43555101694559406*t**6 + 87.120129962570601541*t**5 + 11.959238143588686784*t**4 - 44.907853784494174105*t**3 + 28.185119515090862874*t**2 - 8.3057523340557002803*t + 1)**2 - (0.76368151020129589345*t**13 - 17.552498066607325537*t**12 + 108.41428904184151948*t**11 - 306.94771772506520404*t**10 + 405.2259728739627843*t**9 - 19.807662929398999527*t**8 - 781.82597256184636591*t**7 + 1268.0472358147882909*t**6 - 957.03854015213032735*t**5 + 281.2166033136820188*t**4 + 107.23581160352311984*t**3 - 125.74611311324513171*t**2 + 43.659134492591423524*t - 5.6442241022971821787)/(0.033085446442338013506*t**14 - 0.85201838736111215565*t**13 + 6.28210607226629358*t**12 - 22.000600146007659103*t**11 + 38.901043643289017999*t**10 - 18.392832931964971424*t**9 - 67.302990872244048844*t**8 + 165.71687668982769361*t**7 - 177.43555101694559406*t**6 + 87.120129962570601541*t**5 + 11.959238143588686784*t**4 - 44.907853784494174105*t**3 + 28.185119515090862874*t**2 - 8.3057523340557002803*t + 1)
```

```python
sols = nsolve_intervals(D, [-1, 0.999999], division=100)
print(sols)
print(len(sols))
```

```python
!say done
```

```python
[D.subs(t, s) for s in sols]
```

```python
E = S("exp((t + 1)/(2*t - 2)) - (0.041232283944696212177*t**14 + 1.3765734756274427394*t**13 - 11.423548266294669551*t**12 + 34.897781468305479785*t**11 - 48.170553853698286848*t**10 + 13.655351758955469199*t**9 + 46.6129883570929947*t**8 - 56.479130360958931409*t**7 + 5.525098125593180923*t**6 + 31.467377104584776578*t**5 - 18.246029053439227394*t**4 - 5.9943769501854388096*t**3 + 10.340721799222326113*t**2 - 4.2100165484624456618*t + 0.6065306597126334236)/(0.025008644382436209356*t**14 + 0.80992537393279613982*t**13 - 7.7761706067212429946*t**12 + 27.673671566138794968*t**11 - 47.057239733847679512*t**10 + 28.109557440827452275*t**9 + 30.919259146443448213*t**8 - 61.83483011093278413*t**7 + 22.957457767970616141*t**6 + 26.975517392632053618*t**5 - 26.389699317234955361*t**4 - 1.0791378811026364212*t**3 + 11.607824151320777869*t**2 - 5.9411438334495710432*t + 1)")
```

```python
plot(E, (t, 0.7, 0.8), adaptive=False, nb_of_points=10000)
```

```python
chebyshevt(1, 1)
```

```python
[*enumerate(['a', 'b', 'c'])]
```

```python
system = S("Tuple(-p0 + 0.98846832432811139916*p1 - 0.050649168838712711749*p10 - 0.10116832198743217456*p11 + 0.25065253225872053539*p12 - 0.39435585511331859613*p13 + 0.5289640103269624416*p14 - 0.95413925640004885148*p2 + 0.89780453957074165714*p3 - 0.82076344120727632636*p4 + 0.72479278722911995887*p5 - 0.61210598254766284416*p6 + 0.48530196253108102515*p7 - 0.34730525284482028541*p8 + 0.20129852008866007873*p9 + (epsilon + 0.99710456213012649629)*(-0.98846832432811139916*q1 + 0.050649168838712711749*q10 + 0.10116832198743217456*q11 - 0.25065253225872053539*q12 + 0.39435585511331859613*q13 - 0.5289640103269624416*q14 + 0.95413925640004885148*q2 - 0.89780453957074165714*q3 + 0.82076344120727632636*q4 - 0.72479278722911995887*q5 + 0.61210598254766284416*q6 - 0.48530196253108102515*q7 + 0.34730525284482028541*q8 - 0.20129852008866007873*q9 + 1), -p0 + 0.96807711886620430515*p1 + 0.82076344120727632615*p10 - 0.93775213214708045842*p11 + 0.99486932339189514118*p12 - 0.98846832432811142281*p13 + 0.91895781162023063016*p14 - 0.87434661614458211883*p2 + 0.72479278722911995886*p3 - 0.52896401032696245737*p4 + 0.29936312297335795401*p5 - 0.050649168838712712237*p6 - 0.20129852008866007919*p7 + 0.44039415155763430936*p8 - 0.65137248272222220705*p9 + (-epsilon + 0.99192262841900652865)*(-0.96807711886620430515*q1 - 0.82076344120727632615*q10 + 0.93775213214708045842*q11 - 0.99486932339189514118*q12 + 0.98846832432811142281*q13 - 0.91895781162023063016*q14 + 0.87434661614458211883*q2 - 0.72479278722911995886*q3 + 0.52896401032696245737*q4 - 0.29936312297335795401*q5 + 0.050649168838712712237*q6 + 0.20129852008866007919*q7 - 0.44039415155763430936*q8 + 0.65137248272222220705*q9 + 1), -p0 + 0.93775213214708045843*p1 + 0.91895781162023062755*p10 - 0.72479278722911995726*p11 + 0.44039415155763430437*p12 - 0.10116832198743218276*p13 - 0.2506525322587205562*p14 - 0.75875812269279090192*p2 + 0.48530196253108102522*p3 - 0.15142777750457666366*p4 - 0.20129852008866007914*p5 + 0.52896401032696245735*p6 - 0.79077573693769858208*p7 + 0.95413925640004885141*p8 - 0.99871650717105280743*p9 + (epsilon + 0.98406642835075986461)*(-0.93775213214708045843*q1 - 0.91895781162023062755*q10 + 0.72479278722911995726*q11 - 0.44039415155763430437*q12 + 0.10116832198743218276*q13 + 0.2506525322587205562*q14 + 0.75875812269279090192*q2 - 0.48530196253108102522*q3 + 0.15142777750457666366*q4 + 0.20129852008866007914*q5 - 0.52896401032696245735*q6 + 0.79077573693769858208*q7 - 0.95413925640004885141*q8 + 0.99871650717105280743*q9 + 1), -p0 + 0.89780453957074165714*p1 + 0.15142777750457666382*p10 + 0.29936312297335795208*p11 - 0.68896691907568656868*p12 + 0.93775213214708046506*p13 - 0.99486932339189512731*p14 - 0.61210598254766284415*p2 + 0.20129852008866007914*p3 + 0.25065253225872053932*p4 - 0.65137248272222220746*p5 + 0.91895781162023062913*p6 - 0.99871650717105280713*p7 + 0.87434661614458211861*p8 - 0.57126821509479227908*p9 + (-epsilon + 0.97343458335469763991)*(-0.89780453957074165714*q1 - 0.15142777750457666382*q10 - 0.29936312297335795208*q11 + 0.68896691907568656868*q12 - 0.93775213214708046506*q13 + 0.99486932339189512731*q14 + 0.61210598254766284415*q2 - 0.20129852008866007914*q3 - 0.25065253225872053932*q4 + 0.65137248272222220746*q5 - 0.91895781162023062913*q6 + 0.99871650717105280713*q7 - 0.87434661614458211861*q8 + 0.57126821509479227908*q9 + 1), -p0 + 0.84864425749475095046*p1 - 0.7587581226927909016*p10 + 0.98846832432811139721*p11 - 0.91895781162023063016*p12 + 0.57126821509479228581*p13 - 0.050649168838712711749*p14 - 0.44039415155763430952*p2 - 0.10116832198743217778*p3 + 0.61210598254766284414*p4 - 0.93775213214708045842*p5 + 0.97952994125249449394*p6 - 0.72479278722911995886*p7 + 0.25065253225872053918*p8 + 0.29936312297335795414*p9 + (epsilon + 0.9598896419605118508)*(-0.84864425749475095046*q1 + 0.7587581226927909016*q10 - 0.98846832432811139721*q11 + 0.91895781162023063016*q12 - 0.57126821509479228581*q13 + 0.050649168838712711749*q14 + 0.44039415155763430952*q2 + 0.10116832198743217778*q3 - 0.61210598254766284414*q4 + 0.93775213214708045842*q5 - 0.97952994125249449394*q6 + 0.72479278722911995886*q7 - 0.25065253225872053918*q8 - 0.29936312297335795414*q9 + 1), -p0 + 0.79077573693769858208*p1 - 0.95413925640004885152*p10 + 0.57126821509479227959*p11 + 0.050649168838712713483*p12 - 0.65137248272222220499*p13 + 0.97952994125249449528*p14 - 0.25065253225872053932*p2 - 0.3943558551133185801*p3 + 0.87434661614458211883*p4 - 0.98846832432811139917*p5 + 0.6889669190756865678*p6 - 0.10116832198743217778*p7 - 0.52896401032696245732*p8 + 0.93775213214708045842*p9 + (-epsilon + 0.94325632448977341962)*(-0.79077573693769858208*q1 + 0.95413925640004885152*q10 - 0.57126821509479227959*q11 - 0.050649168838712713483*q12 + 0.65137248272222220499*q13 - 0.97952994125249449528*q14 + 0.25065253225872053932*q2 + 0.3943558551133185801*q3 - 0.87434661614458211883*q4 + 0.98846832432811139917*q5 - 0.6889669190756865678*q6 + 0.10116832198743217778*q7 + 0.52896401032696245732*q8 - 0.93775213214708045842*q9 + 1), -p0 + 0.72479278722911995887*p1 - 0.25065253225872053951*p10 - 0.48530196253108102528*p11 + 0.95413925640004885152*p12 - 0.8978045395707416562*p13 + 0.34730525284482029018*p14 - 0.050649168838712712279*p2 - 0.65137248272222220745*p3 + 0.99486932339189514632*p4 - 0.79077573693769858208*p5 + 0.15142777750457666365*p6 + 0.57126821509479227918*p7 - 0.97952994125249449387*p8 + 0.84864425749475095033*p9 + (epsilon + 0.92331964820666129003)*(-0.72479278722911995887*q1 + 0.25065253225872053951*q10 + 0.48530196253108102528*q11 - 0.95413925640004885152*q12 + 0.8978045395707416562*q13 - 0.34730525284482029018*q14 + 0.050649168838712712279*q2 + 0.65137248272222220745*q3 - 0.99486932339189514632*q4 + 0.79077573693769858208*q5 - 0.15142777750457666365*q6 - 0.57126821509479227918*q7 + 0.97952994125249449387*q8 - 0.84864425749475095033*q9 + 1), -p0 + 0.65137248272222220745*p1 + 0.68896691907568656781*p10 - 0.9987165071710528071*p11 + 0.61210598254766284418*p12 + 0.20129852008866007835*p13 - 0.87434661614458211807*p14 + 0.15142777750457666366*p2 - 0.84864425749475095047*p3 + 0.95413925640004885148*p4 - 0.3943558551133185801*p5 - 0.44039415155763430951*p6 + 0.96807711886620430516*p7 - 0.82076344120727632634*p8 + 0.1011683219874321778*p9 + (-epsilon + 0.89982327219932776296)*(-0.65137248272222220745*q1 - 0.68896691907568656781*q10 + 0.9987165071710528071*q11 - 0.61210598254766284418*q12 - 0.20129852008866007835*q13 + 0.87434661614458211807*q14 - 0.15142777750457666366*q2 + 0.84864425749475095047*q3 - 0.95413925640004885148*q4 + 0.3943558551133185801*q5 + 0.44039415155763430951*q6 - 0.96807711886620430516*q7 + 0.82076344120727632634*q8 - 0.1011683219874321778*q9 + 1), -p0 + 0.57126821509479227916*p1 + 0.97952994125249449398*p10 - 0.39435585511331858023*p11 - 0.52896401032696245754*p12 + 0.99871650717105280716*p13 - 0.61210598254766284397*p14 + 0.34730525284482028554*p2 - 0.96807711886620430515*p3 + 0.75875812269279090191*p4 + 0.10116832198743217779*p5 - 0.87434661614458211882*p6 + 0.89780453957074165714*p7 - 0.15142777750457666369*p8 - 0.72479278722911995884*p9 + (epsilon + 0.8724686291509901109)*(-0.57126821509479227916*q1 - 0.97952994125249449398*q10 + 0.39435585511331858023*q11 + 0.52896401032696245754*q12 - 0.99871650717105280716*q13 + 0.61210598254766284397*q14 - 0.34730525284482028554*q2 + 0.96807711886620430515*q3 - 0.75875812269279090191*q4 - 0.10116832198743217779*q5 + 0.87434661614458211882*q6 - 0.89780453957074165714*q7 + 0.15142777750457666369*q8 + 0.72479278722911995884*q9 + 1), -p0 + 0.48530196253108102521*p1 + 0.34730525284482028554*p10 + 0.65137248272222220744*p11 - 0.97952994125249449395*p12 + 0.29936312297335795411*p13 + 0.68896691907568656803*p14 + 0.52896401032696245737*p2 - 0.99871650717105280715*p3 + 0.44039415155763430952*p4 + 0.57126821509479227916*p5 - 0.99486932339189514632*p6 + 0.3943558551133185801*p7 + 0.61210598254766284416*p8 - 0.98846832432811139917*p9 + (-epsilon + 0.84091577660137237175)*(-0.48530196253108102521*q1 - 0.34730525284482028554*q10 - 0.65137248272222220744*q11 + 0.97952994125249449395*q12 - 0.29936312297335795411*q13 - 0.68896691907568656803*q14 - 0.52896401032696245737*q2 + 0.99871650717105280715*q3 - 0.44039415155763430952*q4 - 0.57126821509479227916*q5 + 0.99486932339189514632*q6 - 0.3943558551133185801*q7 - 0.61210598254766284416*q8 + 0.98846832432811139917*q9 + 1), -p0 + 0.3943558551133185801*p1 - 0.61210598254766284414*p10 + 0.96807711886620430514*p11 - 0.15142777750457666367*p12 - 0.84864425749475095045*p13 + 0.82076344120727632639*p14 + 0.6889669190756865678*p2 - 0.93775213214708045843*p3 + 0.050649168838712712278*p4 + 0.89780453957074165714*p5 - 0.75875812269279090191*p6 - 0.29936312297335795401*p7 + 0.99486932339189514632*p8 - 0.48530196253108102521*p9 + (epsilon + 0.80478747993180621938)*(-0.3943558551133185801*q1 + 0.61210598254766284414*q10 - 0.96807711886620430514*q11 + 0.15142777750457666367*q12 + 0.84864425749475095045*q13 - 0.82076344120727632639*q14 - 0.6889669190756865678*q2 + 0.93775213214708045843*q3 - 0.050649168838712712278*q4 - 0.89780453957074165714*q5 + 0.75875812269279090191*q6 + 0.29936312297335795401*q7 - 0.99486932339189514632*q8 + 0.48530196253108102521*q9 + 1), -p0 + 0.29936312297335795401*p1 - 0.99486932339189514632*p10 + 0.20129852008866007913*p11 + 0.87434661614458211883*p12 - 0.72479278722911995887*p13 - 0.44039415155763430949*p14 + 0.82076344120727632636*p2 - 0.79077573693769858208*p3 - 0.34730525284482028554*p4 + 0.99871650717105280715*p5 - 0.25065253225872053931*p6 - 0.84864425749475095047*p7 + 0.75875812269279090191*p8 + 0.39435585511331858011*p9 + (-epsilon + 0.76367894724544165258)*(-0.29936312297335795401*q1 + 0.99486932339189514632*q10 - 0.20129852008866007913*q11 - 0.87434661614458211883*q12 + 0.72479278722911995887*q13 + 0.44039415155763430949*q14 - 0.82076344120727632636*q2 + 0.79077573693769858208*q3 + 0.34730525284482028554*q4 - 0.99871650717105280715*q5 + 0.25065253225872053931*q6 + 0.84864425749475095047*q7 - 0.75875812269279090191*q8 - 0.39435585511331858011*q9 + 1), -p0 + 0.20129852008866007914*p1 - 0.44039415155763430952*p10 - 0.79077573693769858208*p11 + 0.75875812269279090191*p12 + 0.48530196253108102521*p13 - 0.95413925640004885148*p14 + 0.91895781162023062913*p2 - 0.57126821509479227916*p3 - 0.6889669190756865678*p4 + 0.84864425749475095046*p5 + 0.34730525284482028554*p6 - 0.98846832432811139916*p7 + 0.050649168838712712279*p8 + 0.96807711886620430515*p9 + (epsilon + 0.71717703508602379371)*(-0.20129852008866007914*q1 + 0.44039415155763430952*q10 + 0.79077573693769858208*q11 - 0.75875812269279090191*q12 - 0.48530196253108102521*q13 + 0.95413925640004885148*q14 - 0.91895781162023062913*q2 + 0.57126821509479227916*q3 + 0.6889669190756865678*q4 - 0.84864425749475095046*q5 - 0.34730525284482028554*q6 + 0.98846832432811139916*q7 - 0.050649168838712712279*q8 - 0.96807711886620430515*q9 + 1), -p0 + 0.10116832198743217779*p1 + 0.52896401032696245737*p10 - 0.89780453957074165714*p11 - 0.34730525284482028554*p12 + 0.96807711886620430515*p13 + 0.15142777750457666366*p14 + 0.97952994125249449394*p2 - 0.29936312297335795401*p3 - 0.91895781162023062913*p4 + 0.48530196253108102521*p5 + 0.82076344120727632636*p6 - 0.65137248272222220745*p7 - 0.6889669190756865678*p8 + 0.79077573693769858208*p9 + (-epsilon + 0.66489484531991716168)*(-0.10116832198743217779*q1 - 0.52896401032696245737*q10 + 0.89780453957074165714*q11 + 0.34730525284482028554*q12 - 0.96807711886620430515*q13 - 0.15142777750457666366*q14 - 0.97952994125249449394*q2 + 0.29936312297335795401*q3 + 0.91895781162023062913*q4 - 0.48530196253108102521*q5 - 0.82076344120727632636*q6 + 0.65137248272222220745*q7 + 0.6889669190756865678*q8 - 0.79077573693769858208*q9 + 1), -p0 + p10 - p12 + p14 + p2 - p4 + p6 - p8 + (epsilon + exp(-1/2))*(-q10 + q12 - q14 - q2 + q4 - q6 + q8 + 1), -p0 - 0.10116832198743217779*p1 + 0.52896401032696245737*p10 + 0.89780453957074165714*p11 - 0.34730525284482028554*p12 - 0.96807711886620430515*p13 + 0.15142777750457666366*p14 + 0.97952994125249449394*p2 + 0.29936312297335795401*p3 - 0.91895781162023062913*p4 - 0.48530196253108102521*p5 + 0.82076344120727632636*p6 + 0.65137248272222220745*p7 - 0.6889669190756865678*p8 - 0.79077573693769858208*p9 + (-epsilon + 0.54196418516113069989)*(0.10116832198743217779*q1 - 0.52896401032696245737*q10 - 0.89780453957074165714*q11 + 0.34730525284482028554*q12 + 0.96807711886620430515*q13 - 0.15142777750457666366*q14 - 0.97952994125249449394*q2 - 0.29936312297335795401*q3 + 0.91895781162023062913*q4 + 0.48530196253108102521*q5 - 0.82076344120727632636*q6 - 0.65137248272222220745*q7 + 0.6889669190756865678*q8 + 0.79077573693769858208*q9 + 1), -p0 - 0.20129852008866007914*p1 - 0.44039415155763430952*p10 + 0.79077573693769858208*p11 + 0.75875812269279090191*p12 - 0.48530196253108102521*p13 - 0.95413925640004885148*p14 + 0.91895781162023062913*p2 + 0.57126821509479227916*p3 - 0.6889669190756865678*p4 - 0.84864425749475095046*p5 + 0.34730525284482028554*p6 + 0.98846832432811139916*p7 + 0.050649168838712712279*p8 - 0.96807711886620430515*p9 + (epsilon + 0.47140756707585578791)*(0.20129852008866007914*q1 + 0.44039415155763430952*q10 - 0.79077573693769858208*q11 - 0.75875812269279090191*q12 + 0.48530196253108102521*q13 + 0.95413925640004885148*q14 - 0.91895781162023062913*q2 - 0.57126821509479227916*q3 + 0.6889669190756865678*q4 + 0.84864425749475095046*q5 - 0.34730525284482028554*q6 - 0.98846832432811139916*q7 - 0.050649168838712712279*q8 + 0.96807711886620430515*q9 + 1), -p0 - 0.29936312297335795401*p1 - 0.99486932339189514632*p10 - 0.20129852008866007913*p11 + 0.87434661614458211883*p12 + 0.72479278722911995887*p13 - 0.44039415155763430949*p14 + 0.82076344120727632636*p2 + 0.79077573693769858208*p3 - 0.34730525284482028554*p4 - 0.99871650717105280715*p5 - 0.25065253225872053931*p6 + 0.84864425749475095047*p7 + 0.75875812269279090191*p8 - 0.39435585511331858011*p9 + (-epsilon + 0.39563118171972868313)*(0.29936312297335795401*q1 + 0.99486932339189514632*q10 + 0.20129852008866007913*q11 - 0.87434661614458211883*q12 - 0.72479278722911995887*q13 + 0.44039415155763430949*q14 - 0.82076344120727632636*q2 - 0.79077573693769858208*q3 + 0.34730525284482028554*q4 + 0.99871650717105280715*q5 + 0.25065253225872053931*q6 - 0.84864425749475095047*q7 - 0.75875812269279090191*q8 + 0.39435585511331858011*q9 + 1), -p0 - 0.3943558551133185801*p1 - 0.61210598254766284414*p10 - 0.96807711886620430514*p11 - 0.15142777750457666367*p12 + 0.84864425749475095045*p13 + 0.82076344120727632639*p14 + 0.6889669190756865678*p2 + 0.93775213214708045843*p3 + 0.050649168838712712278*p4 - 0.89780453957074165714*p5 - 0.75875812269279090191*p6 + 0.29936312297335795401*p7 + 0.99486932339189514632*p8 + 0.48530196253108102521*p9 + (epsilon + 0.3162777185489854716)*(0.3943558551133185801*q1 + 0.61210598254766284414*q10 + 0.96807711886620430514*q11 + 0.15142777750457666367*q12 - 0.84864425749475095045*q13 - 0.82076344120727632639*q14 - 0.6889669190756865678*q2 - 0.93775213214708045843*q3 - 0.050649168838712712278*q4 + 0.89780453957074165714*q5 + 0.75875812269279090191*q6 - 0.29936312297335795401*q7 - 0.99486932339189514632*q8 - 0.48530196253108102521*q9 + 1), -p0 - 0.48530196253108102521*p1 + 0.34730525284482028554*p10 - 0.65137248272222220744*p11 - 0.97952994125249449395*p12 - 0.29936312297335795411*p13 + 0.68896691907568656803*p14 + 0.52896401032696245737*p2 + 0.99871650717105280715*p3 + 0.44039415155763430952*p4 - 0.57126821509479227916*p5 - 0.99486932339189514632*p6 - 0.3943558551133185801*p7 + 0.61210598254766284416*p8 + 0.98846832432811139917*p9 + (-epsilon + 0.23624479246104157387)*(0.48530196253108102521*q1 - 0.34730525284482028554*q10 + 0.65137248272222220744*q11 + 0.97952994125249449395*q12 + 0.29936312297335795411*q13 - 0.68896691907568656803*q14 - 0.52896401032696245737*q2 - 0.99871650717105280715*q3 - 0.44039415155763430952*q4 + 0.57126821509479227916*q5 + 0.99486932339189514632*q6 + 0.3943558551133185801*q7 - 0.61210598254766284416*q8 - 0.98846832432811139917*q9 + 1), -p0 - 0.57126821509479227916*p1 + 0.97952994125249449398*p10 + 0.39435585511331858023*p11 - 0.52896401032696245754*p12 - 0.99871650717105280716*p13 - 0.61210598254766284397*p14 + 0.34730525284482028554*p2 + 0.96807711886620430515*p3 + 0.75875812269279090191*p4 - 0.10116832198743217779*p5 - 0.87434661614458211882*p6 - 0.89780453957074165714*p7 - 0.15142777750457666369*p8 + 0.72479278722911995884*p9 + (epsilon + 0.1600193379721607684)*(0.57126821509479227916*q1 - 0.97952994125249449398*q10 - 0.39435585511331858023*q11 + 0.52896401032696245754*q12 + 0.99871650717105280716*q13 + 0.61210598254766284397*q14 - 0.34730525284482028554*q2 - 0.96807711886620430515*q3 - 0.75875812269279090191*q4 + 0.10116832198743217779*q5 + 0.87434661614458211882*q6 + 0.89780453957074165714*q7 + 0.15142777750457666369*q8 - 0.72479278722911995884*q9 + 1), -p0 - 0.65137248272222220745*p1 + 0.68896691907568656781*p10 + 0.9987165071710528071*p11 + 0.61210598254766284418*p12 - 0.20129852008866007835*p13 - 0.87434661614458211807*p14 + 0.15142777750457666366*p2 + 0.84864425749475095047*p3 + 0.95413925640004885148*p4 + 0.3943558551133185801*p5 - 0.44039415155763430951*p6 - 0.96807711886620430516*p7 - 0.82076344120727632634*p8 - 0.1011683219874321778*p9 + (-epsilon + 0.093631266675993090694)*(0.65137248272222220745*q1 - 0.68896691907568656781*q10 - 0.9987165071710528071*q11 - 0.61210598254766284418*q12 + 0.20129852008866007835*q13 + 0.87434661614458211807*q14 - 0.15142777750457666366*q2 - 0.84864425749475095047*q3 - 0.95413925640004885148*q4 - 0.3943558551133185801*q5 + 0.44039415155763430951*q6 + 0.96807711886620430516*q7 + 0.82076344120727632634*q8 + 0.1011683219874321778*q9 + 1), -p0 - 0.72479278722911995887*p1 - 0.25065253225872053951*p10 + 0.48530196253108102528*p11 + 0.95413925640004885152*p12 + 0.8978045395707416562*p13 + 0.34730525284482029018*p14 - 0.050649168838712712279*p2 + 0.65137248272222220745*p3 + 0.99486932339189514632*p4 + 0.79077573693769858208*p5 + 0.15142777750457666365*p6 - 0.57126821509479227918*p7 - 0.97952994125249449387*p8 - 0.84864425749475095033*p9 + (epsilon + 0.0435595768207820696)*(0.72479278722911995887*q1 + 0.25065253225872053951*q10 - 0.48530196253108102528*q11 - 0.95413925640004885152*q12 - 0.8978045395707416562*q13 - 0.34730525284482029018*q14 + 0.050649168838712712279*q2 - 0.65137248272222220745*q3 - 0.99486932339189514632*q4 - 0.79077573693769858208*q5 - 0.15142777750457666365*q6 + 0.57126821509479227918*q7 + 0.97952994125249449387*q8 + 0.84864425749475095033*q9 + 1), -p0 - 0.79077573693769858208*p1 - 0.95413925640004885152*p10 - 0.57126821509479227959*p11 + 0.050649168838712713483*p12 + 0.65137248272222220499*p13 + 0.97952994125249449528*p14 - 0.25065253225872053932*p2 + 0.3943558551133185801*p3 + 0.87434661614458211883*p4 + 0.98846832432811139917*p5 + 0.6889669190756865678*p6 + 0.10116832198743217778*p7 - 0.52896401032696245732*p8 - 0.93775213214708045842*p9 + (-epsilon + 0.013848748820747408312)*(0.79077573693769858208*q1 + 0.95413925640004885152*q10 + 0.57126821509479227959*q11 - 0.050649168838712713483*q12 - 0.65137248272222220499*q13 - 0.97952994125249449528*q14 + 0.25065253225872053932*q2 - 0.3943558551133185801*q3 - 0.87434661614458211883*q4 - 0.98846832432811139917*q5 - 0.6889669190756865678*q6 - 0.10116832198743217778*q7 + 0.52896401032696245732*q8 + 0.93775213214708045842*q9 + 1), -p0 - 0.84864425749475095046*p1 - 0.7587581226927909016*p10 - 0.98846832432811139721*p11 - 0.91895781162023063016*p12 - 0.57126821509479228581*p13 - 0.050649168838712711749*p14 - 0.44039415155763430952*p2 + 0.10116832198743217778*p3 + 0.61210598254766284414*p4 + 0.93775213214708045842*p5 + 0.97952994125249449394*p6 + 0.72479278722911995886*p7 + 0.25065253225872053918*p8 - 0.29936312297335795414*p9 + (epsilon + 0.0022273312261039493487)*(0.84864425749475095046*q1 + 0.7587581226927909016*q10 + 0.98846832432811139721*q11 + 0.91895781162023063016*q12 + 0.57126821509479228581*q13 + 0.050649168838712711749*q14 + 0.44039415155763430952*q2 - 0.10116832198743217778*q3 - 0.61210598254766284414*q4 - 0.93775213214708045842*q5 - 0.97952994125249449394*q6 - 0.72479278722911995886*q7 - 0.25065253225872053918*q8 + 0.29936312297335795414*q9 + 1), -p0 - 0.89780453957074165714*p1 + 0.15142777750457666382*p10 - 0.29936312297335795208*p11 - 0.68896691907568656868*p12 - 0.93775213214708046506*p13 - 0.99486932339189512731*p14 - 0.61210598254766284415*p2 - 0.20129852008866007914*p3 + 0.25065253225872053932*p4 + 0.65137248272222220746*p5 + 0.91895781162023062913*p6 + 0.99871650717105280713*p7 + 0.87434661614458211861*p8 + 0.57126821509479227908*p9 + (-epsilon + 0.000092790113933967817733)*(0.89780453957074165714*q1 - 0.15142777750457666382*q10 + 0.29936312297335795208*q11 + 0.68896691907568656868*q12 + 0.93775213214708046506*q13 + 0.99486932339189512731*q14 + 0.61210598254766284415*q2 + 0.20129852008866007914*q3 - 0.25065253225872053932*q4 - 0.65137248272222220746*q5 - 0.91895781162023062913*q6 - 0.99871650717105280713*q7 - 0.87434661614458211861*q8 - 0.57126821509479227908*q9 + 1), -p0 - 0.93775213214708045843*p1 + 0.91895781162023062755*p10 + 0.72479278722911995726*p11 + 0.44039415155763430437*p12 + 0.10116832198743218276*p13 - 0.2506525322587205562*p14 - 0.75875812269279090192*p2 - 0.48530196253108102522*p3 - 0.15142777750457666366*p4 + 0.20129852008866007914*p5 + 0.52896401032696245735*p6 + 0.79077573693769858208*p7 + 0.95413925640004885141*p8 + 0.99871650717105280743*p9 + (epsilon + 1.7389619963042768938e-7)*(0.93775213214708045843*q1 - 0.91895781162023062755*q10 - 0.72479278722911995726*q11 - 0.44039415155763430437*q12 - 0.10116832198743218276*q13 + 0.2506525322587205562*q14 + 0.75875812269279090192*q2 + 0.48530196253108102522*q3 + 0.15142777750457666366*q4 - 0.20129852008866007914*q5 - 0.52896401032696245735*q6 - 0.79077573693769858208*q7 - 0.95413925640004885141*q8 - 0.99871650717105280743*q9 + 1), -p0 - 0.96807711886620430515*p1 + 0.82076344120727632615*p10 + 0.93775213214708045842*p11 + 0.99486932339189514118*p12 + 0.98846832432811142281*p13 + 0.91895781162023063016*p14 - 0.87434661614458211883*p2 - 0.72479278722911995886*p3 - 0.52896401032696245737*p4 - 0.29936312297335795401*p5 - 0.050649168838712712237*p6 + 0.20129852008866007919*p7 + 0.44039415155763430936*p8 + 0.65137248272222220705*p9 + (-epsilon + 4.0988152678429733251e-14)*(0.96807711886620430515*q1 - 0.82076344120727632615*q10 - 0.93775213214708045842*q11 - 0.99486932339189514118*q12 - 0.98846832432811142281*q13 - 0.91895781162023063016*q14 + 0.87434661614458211883*q2 + 0.72479278722911995886*q3 + 0.52896401032696245737*q4 + 0.29936312297335795401*q5 + 0.050649168838712712237*q6 - 0.20129852008866007919*q7 - 0.44039415155763430936*q8 - 0.65137248272222220705*q9 + 1), -p0 - 0.98846832432811139916*p1 - 0.050649168838712711749*p10 + 0.10116832198743217456*p11 + 0.25065253225872053539*p12 + 0.39435585511331859613*p13 + 0.5289640103269624416*p14 - 0.95413925640004885148*p2 - 0.89780453957074165714*p3 - 0.82076344120727632636*p4 - 0.72479278722911995887*p5 - 0.61210598254766284416*p6 - 0.48530196253108102515*p7 - 0.34730525284482028541*p8 - 0.20129852008866007873*p9 + (epsilon + 3.5986813119484602573e-38)*(0.98846832432811139916*q1 + 0.050649168838712711749*q10 - 0.10116832198743217456*q11 - 0.25065253225872053539*q12 - 0.39435585511331859613*q13 - 0.5289640103269624416*q14 + 0.95413925640004885148*q2 + 0.89780453957074165714*q3 + 0.82076344120727632636*q4 + 0.72479278722911995887*q5 + 0.61210598254766284416*q6 + 0.48530196253108102515*q7 + 0.34730525284482028541*q8 + 0.20129852008866007873*q9 + 1), -epsilon*(q1 + q10 + q11 + q12 + q13 + q14 + q2 + q3 + q4 + q5 + q6 + q7 + q8 + q9 + 1) - p0 - p1 - p10 - p11 - p12 - p13 - p14 - p2 - p3 - p4 - p5 - p6 - p7 - p8 - p9)")
vars = S("[p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, q12, q13, q14, epsilon]")
```

```python
sol = dict(zip(vars, nsolve(system, vars, [1]*len(vars), solver='secant')))
sol
```

```python
print(system.subs(sol))
```

```python
plot(exp((t + 1)/(t - 1)), (t, -1, 1))
```

```python
plot(exp(10*(t + 1)/(t - 1)), (t, -1, 1))
```

```python
r = S("(0.0011875196369021855177*t**14 - 0.011420090831167348621*t**13 + 0.050741761684873561064*t**12 - 0.13809510690997622664*t**11 + 0.25723786667078296754*t**10 - 0.34703482945300861262*t**9 + 0.34976856562058063458*t**8 - 0.26762758451401203127*t**7 + 0.15627145956034238656*t**6 - 0.069319897634892213306*t**5 + 0.023002511830763159929*t**4 - 0.0055388208662975187897*t**3 + 0.00091518140765965519386*t**2 - 0.00009291001238164753822*t + 4.3738099008807505585e-6)/(0.00089586291474760584255*t**14 + 0.0089336931214819976007*t**13 + 0.042095909802441250189*t**12 + 0.12482069131842848916*t**11 + 0.26604814866002116766*t**10 + 0.42347046743527135243*t**9 + 0.57833691404471361527*t**8 + 0.54769996073689544827*t**7 + 0.68765070665076017993*t**6 + 0.2278059784921081907*t**5 + 0.6501516075867275978*t**4 - 0.14644874880736715755*t**3 + 0.42333407306668857903*t**2 - 0.11968772580151045925*t + 0.096339574212779644172)").subs(t, solve(10*(t + 1)/(t - 1) - y, t)[0].subs(y, t))
n, d = together(r).as_numer_denom() # simplify/cancel here will add degree to the numerator and denominator
rat_func = (Poly(n)/Poly(d).TC())/(Poly(d)/Poly(d).TC())
rat_func
```

```python
D = S("(10/(t - 1) - (10*t + 10)/(t - 1)**2)*exp((10*t + 10)/(t - 1)) - (-0.011109949473020816773*t**13 - 0.10498117392496721903*t**12 - 0.4662693207692269047*t**11 - 1.2944463271987387573*t**10 - 2.560791476062548363*t**9 - 3.7445986907826045347*t**8 - 4.6175581668981299287*t**7 - 3.9202376055011740667*t**6 - 4.1853692405031723697*t**5 - 1.2730163279308121306*t**4 - 2.6093036740226881464*t**3 + 0.37572825889837415047*t**2 - 0.84260381473372911342*t + 0.11395040244406088625)*(0.0011012754893812472661*t**14 - 0.010681747521049671358*t**13 + 0.047852308718456039572*t**12 - 0.13125412233037760452*t**11 + 0.24631294936212648611*t**10 - 0.33462134384239723802*t**9 + 0.3394635962011490975*t**8 - 0.2613214567551016616*t**7 + 0.15344584617039959246*t**6 - 0.068417469219647235265*t**5 + 0.022810045726230519721*t**4 - 0.0055160290938826628221*t**3 + 0.00091495897212439435512*t**2 - 0.00009321390565315082022*t + 4.402028268659469097e-6)/(0.00079356781950148691235*t**14 + 0.0080754749173051706947*t**13 + 0.038855776730768908725*t**12 + 0.11767693883624897794*t**11 + 0.2560791476062548363*t**10 + 0.41606652119806717052*t**9 + 0.57719477086226624108*t**8 + 0.56003394364302486668*t**7 + 0.69756154008386206162*t**6 + 0.25460326558616242613*t**5 + 0.65232591850567203661*t**4 - 0.12524275296612471682*t**3 + 0.42130190736686455671*t**2 - 0.11395040244406088625*t + 0.096961125131689425859)**2 - (0.015417856851337461725*t**13 - 0.13886271777364572765*t**12 + 0.57422770462147247487*t**11 - 1.4437953456341536497*t**10 + 2.4631294936212648611*t**9 - 3.0115920945815751422*t**8 + 2.71570876960919278*t**7 - 1.8292501972857116312*t**6 + 0.92067507702239755476*t**5 - 0.34208734609823617633*t**4 + 0.091240182904922078882*t**3 - 0.016548087281647988466*t**2 + 0.0018299179442487887102*t - 0.00009321390565315082022)/(0.00079356781950148691235*t**14 + 0.0080754749173051706947*t**13 + 0.038855776730768908725*t**12 + 0.11767693883624897794*t**11 + 0.2560791476062548363*t**10 + 0.41606652119806717052*t**9 + 0.57719477086226624108*t**8 + 0.56003394364302486668*t**7 + 0.69756154008386206162*t**6 + 0.25460326558616242613*t**5 + 0.65232591850567203661*t**4 - 0.12524275296612471682*t**3 + 0.42130190736686455671*t**2 - 0.11395040244406088625*t + 0.096961125131689425859)")
```

```python
D.subs(t, Float("-5.0000000006988898296e-7"))
```

```python
interval = [-5.00000000069889e-07, 0.019999489999999787]
sol = nsolve(D, interval, solver='bisect', prec=20)
sol
```

```python
D.subs(t, sol)
```

```python
interval[0] < sol < interval[1]
```

```python
interval[0]
```

```python
import mpmath
mpmath.mp.dps = 20
f = lambdify(t, D, 'mpmath')
sol = mpmath.findroot(f, [0.57999921, 0.5999991999999998] , solver='bisect')
sol
```

```python
f(sol)
```

```python
plot(-exp(-t) + S("(0.00421099590689821779984639140604*t**2 + 0.188333501989274148910007547496*t + 0.992641329830419396033230547887)/(0.572249579048364881552280260776*t**2 - 0.669301542710871251180032333275*t + 1.0)"), (t, 0, 100))
```

```python
rat_func
```

```python
plot(rat_func - exp(-t), (t, 0, 100))
```

```python
plot(rat_func - S("(0.00421099590689821779984639140604*t**2 + 0.188333501989274148910007547496*t + 0.992641329830419396033230547887)/(0.572249579048364881552280260776*t**2 - 0.669301542710871251180032333275*t + 1.0)"), (t, 0, 100))
```

```python
rat_func
```

```python
S("(0.00421099590689821779984639140604*t**2 + 0.188333501989274148910007547496*t + 0.992641329830419396033230547887)/(0.572249579048364881552280260776*t**2 - 0.669301542710871251180032333275*t + 1.0)")
```

```python
rat_func == S("(0.00421099590689821779984639140604*t**2 + 0.188333501989274148910007547496*t + 0.992641329830419396033230547887)/(0.572249579048364881552280260776*t**2 - 0.669301542710871251180032333275*t + 1.0)")
```

```python
str(rat_func) == "(0.00421099590689821779984639140604*t**2 + 0.188333501989274148910007547496*t + 0.992641329830419396033230547887)/(0.572249579048364881552280260776*t**2 - 0.669301542710871251180032333275*t + 1.0)"
```

```python
plot(rat_func)
```

```python
plot(S("(0.00421099590689821779984639140604*t**2 + 0.188333501989274148910007547496*t + 0.992641329830419396033230547887)/(0.572249579048364881552280260776*t**2 - 0.669301542710871251180032333275*t + 1.0)"))
```

```python
good_rat_func = S("(0.00421099590689821779984639140604*t**2 + 0.188333501989274148910007547496*t + 0.992641329830419396033230547887)/(0.572249579048364881552280260776*t**2 - 0.669301542710871251180032333275*t + 1.0)")
```

```python
plot(rat_func - exp(-t), (t, 0, 100))
```

```python
plot(good_rat_func - exp(-t), (t, 0, 100))
```

```python
plot(-exp(-t) + good_rat_func, (t, 0, 100))
```

```python
plot(-exp(-t) - S("(0.00421099590689821779984639140604*t**2 + 0.188333501989274148910007547496*t + 0.992641329830419396033230547887)/(0.572249579048364881552280260776*t**2 - 0.669301542710871251180032333275*t + 1.0)"), (t, 0, 100))
```

```python
plot(-exp(-t) + S(srepr(rat_func)), (t, 0, 100))
```

```python
plot(-exp(-t) + S(str(rat_func)), (t, 0, 100))
```

```python
plot(rat_func - exp(-t), (t, 0, 100), adaptive=False, nb_of_points=1000)
```

```python
import numpy
import matplotlib.pyplot as plt
vals = numpy.linspace(0, 100, 1000)
plt.plot(vals, lambdify(t, rat_func.subs(t, -t) - exp(-t), 'numpy')(vals))

```

```python
degree = 14
points = [chebyshevt_root(2*(degree + 1) + 1, 2*(degree + 1) - j) for j in range(1, 2*(degree + 1) + 1)]
[i.evalf() for i in points]
c = symbols('c', positive=True)
len(points)
solve(sqrt(c*(1 + points[degree])/(1 - points[degree])*c*(1 + points[degree+1])/(1 - points[degree+1])) - c, c)

```

```python
sqrt(c*(1 + points[degree])/(1 - points[degree])*c*(1 + points[degree+1])/(1 - points[degree+1])) - c
```

```python
E = S("exp((10.8*t + 10.8)/(t - 1)) - (0.000292923440172061376307712067378*t**18 - 0.00382369712669093978523377528192*t**17 + 0.0235117968300681682735465482247*t**16 - 0.0904961683900329437749507666096*t**15 + 0.244347315405413563336773282433*t**14 - 0.491533156343580993021894164578*t**13 + 0.763512364001241488427664101087*t**12 - 0.936481460181764203042245859202*t**11 + 0.919553635779126092586969716435*t**10 - 0.728338033009921941415427470583*t**9 + 0.466494241057100272722298949598*t**8 - 0.241066390548087834899533957827*t**7 + 0.0997744318509568132657709953788*t**6 - 0.0326299433259599107384290896079*t**5 + 0.00824748059197329933751330243673*t**4 - 0.00155480878299504771864337728231*t**3 + 0.000205891550849325966892424491264*t**2 - 0.0000170923433737289258328111955236*t + 0.000000669545506459572399276839032838)/(0.0000256271908755749476501555060643*t**18 + 0.00029976157640698698493412804827*t**17 + 0.0016792592359715410041296380456*t**16 + 0.00593759917905712411027230635685*t**15 + 0.0156202804404542365771258126305*t**14 + 0.0289577506063902781990988819345*t**13 + 0.0560418346810373552008650596658*t**12 + 0.0418316309497562848580964550569*t**11 + 0.151260111256952231777215316531*t**10 - 0.0963027347150387933031292538397*t**9 + 0.434868911821397452965639691116*t**8 - 0.535112011457860838676364996918*t**7 + 0.885545060524724480506124658926*t**6 - 0.910515237881500396140647664512*t**5 + 0.901052413999683103211771128713*t**4 - 0.618793687227890880788897623539*t**3 + 0.360336843257148540488631327245*t**2 - 0.128932571601980421778511686416*t + 0.0328216571239099274214356723826)")
```

```python
plot(E, (t, -1, 1))
```

```python
E.replace(exp, lambda x: 0)
```

```python
c = 10.8
prec = 30
r = -E.replace(exp, lambda x: 0)
inv = solve(-c*(t + 1)/(t - 1) - y, t)[0].subs(y, t)
n, d = together(r.subs(t, inv)).as_numer_denom() # simplify/cancel here will add degree to the numerator and denominator
rat_func = (Poly(n)/Poly(d).TC())/(Poly(d)/Poly(d).TC())
rat_func
```

```python
inv = solve(-c*(t + 1)/(t - 1) - y, t)[0].subs(y, t)
n, d = together(r.subs(t, inv.evalf(prec))).as_numer_denom() # simplify/cancel here will add degree to the numerator and denominator
rat_func = expand(n/d.subs(t, 0))/expand(d/d.subs(t, 0))
rat_func
```

```python
r.subs(t, inv)
```

```python
master_rat_func = S("(0.000000000000000000000000000000000000076480564016753411391758926810678059166246681879403496152487645239211280144217888827074664937622241433057155411729238059005620362*t**18 - 0.00000000000000000000000000000000144349696189967720280773581225860550659908207558078615762681841828055896712976164458203824215880891842340327527192908591610821*t**17 + 0.0000000000000000000000000000044910880325416024060232183011837489287970058235258370172770028208341880031097425649152413600814540715147042236183237457793362906*t**16 - 0.0000000000000000000000000055904194675876842224749371778659158365295791811734393172655086431051636828145602210004215923885798770655051649836756001927074635*t**15 + 0.0000000000000000000000036534617094118107737307644008627750110509773097939005142805165889454907972711014257711631906543393485730765556180421132890852392*t**14 - 0.000000000000000000001452005752480375025477783595297687855428318333130316532815795944869337169252966327504293176967865458909941895728922083075100905*t**13 + 0.00000000000000000038258486902246032408006621010963716586950687991236057815854665336890002346614805976419029288917692940037303895895123632103288712*t**12 - 0.000000000000000070617682439427615900771877450592882868606546484110524383009238796767626116906296729855457707191810570035340443853799274578659569*t**11 + 0.0000000000000094758123346723567170908152172735939652157451575834910841031916583217101183394528014444063348758109776739928297123459805390104553*t**10 - 0.00000000000094773868840452502508269303119154845985934338599523295887294825053761377691930897288612488861758600642472630646115958140782655741*t**9 + 0.000000000071794537029977599630049677051557682902685713470170819790111653928229512768189817446726011375388897055644123749103346310180957703*t**8 - 0.0000000041548085790436606814151882639426540923873459207739075135022708664862920165014122761594444148444999444676937908166192319430948739*t**7 + 0.00000018396482187290156607582394998664929895412165041697075842037042924223322317897465358352879794796218422276896933375527141853756368*t**6 - 0.0000061940471597289213538311760411627015047318480396138859471763151560132556207793110081461448458843840327989497213331241784911809188*t**5 + 0.00015609017337127813834188030193516319436098147682156485822323423924211374867515035340573616843723433765654338888756861271039524759*t**4 - 0.0028562580858878094323585065185801663096872149540312530171997790387127408262872701820963659684205263711498876442924046259043818741*t**3 + 0.035908020274519704107157835421234360862435489836434974542107322036865342386568144753587976535805549361943459578492579177565909169*t**2 - 0.27789748837786198960284734868450774184090800132807427206186974905648882760573987089502539075626928473970338728172474858726571638*t + 1.0)/(0.000000000000000000031040251435758559612107016726409522805133890008185285868034602477549411378597769320246238650254246564453320307940208743427452132*t**18 + 0.00000000000000000033278408185195783331139523007623347013904881323415039608612228753896462298159222572181471639107826698580483779471762583452979632*t**17 + 0.000000000000000036787450097487433810641016956456458321996969855733085703659877884336499129663780877438684132168500286553243353222376638553979316*t**16 + 0.00000000000000086471483697461572289931466729671519305573126608639342123901368411795553895792792034205116888738244515155185481746831557263871812*t**15 + 0.000000000000028542945098191436824578047622161061818620141667875379414297168053758179023344405243390741611910581146082198383107320705069583298*t**14 + 0.00000000000069925714572325319966132548491379676104543251464498742295512527923847731544663020484057497681632084412599937152698907579711598455*t**13 + 0.000000000016023022340045153918478559722974792676358615762326800442501765171455792601242697969576723610509994457322049182740923202818976629*t**12 + 0.00000000032288400362818264475862911376930287544499616030122169108718711417899107361671375705975465433486011408164287643509933657504241192*t**11 + 0.0000000058031481469953025925067093526070113494249880473036061932865265507723402865048713484792246133690205685646178264769891316933338782*t**10 + 0.000000092334736432550055237788885523966859751018477031256929995376082333113413768260370819988748007438778907195512402675485670731415463*t**9 + 0.0000012926132501100766989333427587529904458908220243641974136275253050146198705426564792856481298765046890202380566214250561052101974*t**8 + 0.000015764526871377686388685138109739710158659275647616228857451800259851474387813943355076456887413075988332171386231137024453368698*t**7 + 0.00016523598870223734044973391186293737035539948649029834978809577442020543781821653294313415106131238369918759029115415865449394346*t**6 + 0.0014607084457312565069881359757159810508484208317079191023373636723638109071771404868435318582711990519471832277113738367820431404*t**5 + 0.010604260828765659024101011711311726965939177316289266086487560938209634069089784251192671810661317429692738097439733970455088636*t**4 + 0.060769684666293428066362095643979446983198257906551701462685569300225217899448253566777035915490895538454676999338813585125899174*t**3 + 0.25801053189666603350610238873105407999382491457750647353754793173734492220703368984743001028426676940884044469045936407185268167*t**2 + 0.72210251162213748406312012324555238365117862900849998495383644968647977930578316078230461402023282743548534135417525983920899434*t + 1.0)")
```

```python
smichr_nsolve_rat_func = S("(0.000000000000000000000000000000000000076480564016753411391758926810678059166246681879403496152487645239211280144217888827074664937622241433057155411729238059005620362*t**18 - 0.00000000000000000000000000000000144349696189967720280773581225860550659908207558078615762681841828055896712976164458203824215880891842340327527192908591610821*t**17 + 0.0000000000000000000000000000044910880325416024060232183011837489287970058235258370172770028208341880031097425649152413600814540715147042236183237457793362906*t**16 - 0.0000000000000000000000000055904194675876842224749371778659158365295791811734393172655086431051636828145602210004215923885798770655051649836756001927074635*t**15 + 0.0000000000000000000000036534617094118107737307644008627750110509773097939005142805165889454907972711014257711631906543393485730765556180421132890852392*t**14 - 0.000000000000000000001452005752480375025477783595297687855428318333130316532815795944869337169252966327504293176967865458909941895728922083075100905*t**13 + 0.00000000000000000038258486902246032408006621010963716586950687991236057815854665336890002346614805976419029288917692940037303895895123632103288712*t**12 - 0.000000000000000070617682439427615900771877450592882868606546484110524383009238796767626116906296729855457707191810570035340443853799274578659569*t**11 + 0.0000000000000094758123346723567170908152172735939652157451575834910841031916583217101183394528014444063348758109776739928297123459805390104553*t**10 - 0.00000000000094773868840452502508269303119154845985934338599523295887294825053761377691930897288612488861758600642472630646115958140782655741*t**9 + 0.000000000071794537029977599630049677051557682902685713470170819790111653928229512768189817446726011375388897055644123749103346310180957703*t**8 - 0.0000000041548085790436606814151882639426540923873459207739075135022708664862920165014122761594444148444999444676937908166192319430948739*t**7 + 0.00000018396482187290156607582394998664929895412165041697075842037042924223322317897465358352879794796218422276896933375527141853756368*t**6 - 0.0000061940471597289213538311760411627015047318480396138859471763151560132556207793110081461448458843840327989497213331241784911809188*t**5 + 0.00015609017337127813834188030193516319436098147682156485822323423924211374867515035340573616843723433765654338888756861271039524759*t**4 - 0.0028562580858878094323585065185801663096872149540312530171997790387127408262872701820963659684205263711498876442924046259043818741*t**3 + 0.035908020274519704107157835421234360862435489836434974542107322036865342386568144753587976535805549361943459578492579177565909169*t**2 - 0.27789748837786198960284734868450774184090800132807427206186974905648882760573987089502539075626928473970338728172474858726571638*t + 1.0)/(0.000000000000000000031040251435758559612107016726409522805133890008185285868034602477549411378597769320246238650254246564453320307940208743427452132*t**18 + 0.00000000000000000033278408185195783331139523007623347013904881323415039608612228753896462298159222572181471639107826698580483779471762583452979632*t**17 + 0.000000000000000036787450097487433810641016956456458321996969855733085703659877884336499129663780877438684132168500286553243353222376638553979316*t**16 + 0.00000000000000086471483697461572289931466729671519305573126608639342123901368411795553895792792034205116888738244515155185481746831557263871812*t**15 + 0.000000000000028542945098191436824578047622161061818620141667875379414297168053758179023344405243390741611910581146082198383107320705069583298*t**14 + 0.00000000000069925714572325319966132548491379676104543251464498742295512527923847731544663020484057497681632084412599937152698907579711598455*t**13 + 0.000000000016023022340045153918478559722974792676358615762326800442501765171455792601242697969576723610509994457322049182740923202818976629*t**12 + 0.00000000032288400362818264475862911376930287544499616030122169108718711417899107361671375705975465433486011408164287643509933657504241192*t**11 + 0.0000000058031481469953025925067093526070113494249880473036061932865265507723402865048713484792246133690205685646178264769891316933338782*t**10 + 0.000000092334736432550055237788885523966859751018477031256929995376082333113413768260370819988748007438778907195512402675485670731415463*t**9 + 0.0000012926132501100766989333427587529904458908220243641974136275253050146198705426564792856481298765046890202380566214250561052101974*t**8 + 0.000015764526871377686388685138109739710158659275647616228857451800259851474387813943355076456887413075988332171386231137024453368698*t**7 + 0.00016523598870223734044973391186293737035539948649029834978809577442020543781821653294313415106131238369918759029115415865449394346*t**6 + 0.0014607084457312565069881359757159810508484208317079191023373636723638109071771404868435318582711990519471832277113738367820431404*t**5 + 0.010604260828765659024101011711311726965939177316289266086487560938209634069089784251192671810661317429692738097439733970455088636*t**4 + 0.060769684666293428066362095643979446983198257906551701462685569300225217899448253566777035915490895538454676999338813585125899174*t**3 + 0.25801053189666603350610238873105407999382491457750647353754793173734492220703368984743001028426676940884044469045936407185268167*t**2 + 0.72210251162213748406312012324555238365117862900849998495383644968647977930578316078230461402023282743548534135417525983920899434*t + 1.0)")
```

```python
master_rat_func == smichr_nsolve_rat_func
```

```python

```


# Optimizer tweaks

```
%load_ext autoreload
%autoreload 2

%matplotlib inline
```

```
#export
from exp.nb_08 import *
```

## Imagenette data

We grab the data from the previous notebook.

```
path = datasets.untar_data(datasets.URLs.IMAGENETTE_160)
```

```
tfms = [make_rgb, ResizeFixed(128), to_byte_tensor, to_float_tensor]
bs=128

il = ImageList.from_files(path, tfms=tfms)
sd = SplitData.split_by_func(il, partial(grandparent_splitter, valid_name='val'))
ll = label_by_func(sd, parent_labeler)
data = ll.to_databunch(bs, c_in=3, c_out=10, num_workers=4)
```

Then a model

```
nfs = [32,64,128,256]
```

```
cbfs = [partial(AvgStatsCallback,accuracy), CudaCallback,
        partial(BatchTransformXCallback, norm_imagenette)]
```

This is the baseline of training with vanilla SGD.

```
learn,run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs)
```

```
run.fit(1, learn)
```

    train: [1.7456596212385607, tensor(0.3921, device='cuda:0')]
    valid: [1.64411865234375, tensor(0.4220, device='cuda:0')]

## Refining the optimizer

In PyTorch, the base optimizer in `torch.optim` is just a dictionary that stores the hyper-parameters and references to the parameters of the model we want to train in parameter groups (different groups can have different learning rates/momentum/weight decay... which is what lets us do discriminative learning rates).

It contains a method `step` that will update our parameters with the gradients and a method `zero_grad` to detach and zero the gradients of all our parameters.

We build the equivalent from scratch. In our implementation, the step function loops over all the parameters to execute the step using stepper functions that we have to provide when initializing the optimizer.

```
class Optimizer():
    def __init__(self, params, steppers, **defaults):
        # might be a generator
        self.param_groups = list(params)
        # ensure params is a list of lists
        if not isinstance(self.param_groups[0], list): self.param_groups = [self.param_groups]
        self.hypers = [{**defaults} for p in self.param_groups]
        self.steppers = listify(steppers)

    def grad_params(self):
        return [(p,hyper) for pg,hyper in zip(self.param_groups,self.hypers)
            for p in pg if p.grad is not None]

    def zero_grad(self):
        for p,hyper in self.grad_params():
            p.grad.detach_()
            p.grad.zero_()

    def step(self):
        for p,hyper in self.grad_params(): compose(p, self.steppers, **hyper)
```

Now that we have changed the optimizer, we will need to adjust the callbacks that were using properties from the PyTorch optimizer: in particular the hyper-parameters are in the list of dictionaries `opt.hypers` (PyTorch has everything in the the list of param groups).

```
#export
class Recorder(Callback):
    def begin_fit(self): self.lrs,self.losses = [],[]

    def after_batch(self):
        if not self.in_train: return
        self.lrs.append(self.opt.hypers[-1]['lr'])
        self.losses.append(self.loss.detach().cpu())

    def plot_lr  (self): plt.plot(self.lrs)
    def plot_loss(self): plt.plot(self.losses)

class ParamScheduler(Callback):
    _order=1
    def __init__(self, pname, sched_func):
        self.pname,self.sched_func = pname,sched_func

    def set_param(self):
        for h in self.opt.hypers:
            h[self.pname] = self.sched_func(self.n_epochs/self.epochs)

    def begin_batch(self):
        if self.in_train: self.set_param()
```

To do basic SGD, this what a step looks like:

```
#export
def sgd_step(p, lr, **kwargs):
    p.data.add_(-lr, p.grad.data)
    return p
```

So let's check we didn't break anything and that recorder and param scheduler work properly.

```
sched = combine_scheds([0.3, 0.7], [sched_cos(0.3, 0.6), sched_cos(0.6, 0.2)])
```

```
cbfs = [partial(AvgStatsCallback,accuracy),
        CudaCallback, Recorder,
        partial(ParamScheduler, 'lr', sched)]
```

```
opt_func = partial(Optimizer, steppers=[sgd_step])
```

```
learn,run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs, opt_func=opt_func)
```

```
%time run.fit(1, learn)
```

    train: [1.8056946605494804, tensor(0.3653, device='cuda:0')]
    valid: [1.511763916015625, tensor(0.4840, device='cuda:0')]
    CPU times: user 4.93 s, sys: 2.32 s, total: 7.25 s
    Wall time: 11.1 s

```
run.recorder.plot_loss()
```

```
run.recorder.plot_lr()
```

## Weight decay

By letting our model learn high parameters, it might fit all the data points in the training set with an over-complex function that has very sharp changes, which will lead to overfitting.

Weight decay comes from the idea of L2 regularization, which consists in adding to your loss function the sum of all the weights squared. Why do that? Because when we compute the gradients, it will add a contribution to them that will encourage the weights to be as small as possible.

Limiting our weights from growing too much is going to hinder the training of the model, but it will yield to a state where it generalizes better. Going back to the theory a little bit, weight decay (or just `wd`) is a parameter that controls that sum of squares we add to our loss:
``` python
loss_with_wd = loss + (wd/2) * (weights**2).sum()
```

In practice though, it would be very inefficient (and maybe numerically unstable) to compute that big sum and add it to the loss. If you remember a little bit of high schoool math, you should now that the derivative of `p**2` with respect to `p` is simple `2*p`, so adding that big sum to our loss is exactly the same as doing
``` python
weight.grad += wd * weight
```

for every weight in our model, which is equivalent to (in the case of vanilla SGD) updating the parameters
with
``` python
weight = weight - lr*(weight.grad - wd*weight)
```

This last formula explains why the name of this technique is weight decay, as each weight is decayed by a factor `lr * wd`.

This only works for standard SGD, as we have seen that with momentum, RMSProp or in Adam, the update has some additional formulas around the gradient. In those cases, the formula that comes from L2 regularization:
``` python
weight.grad += wd * weight
```
is different than weight decay
``` python
new_weight = weight - lr * weight.grad - lr * wd * weight
```

Most libraries use the first one, but as it was pointed out in [Decoupled Weight Regularization](https://arxiv.org/pdf/1711.05101.pdf) by Ilya Loshchilov and Frank Hutter, it is better to use the second one with the Adam optimizer, which is why fastai made it its default.

Let's allow steppers to add to our `defaults` (which are the default values of all the hyper-parameters). This helper function adds in `dest` the key/values it finds while going through `os` and applying `f` when they was no `key` of the same name.

```
#export
def maybe_update(os, dest, f):
    for o in os:
        for k,v in f(o).items():
            if k not in dest: dest[k] = v

def get_defaults(d): return getattr(d,'_defaults',{})
```

This is the same as before, we just take the default values of the steppers when none are provided in the kwargs.

```
#export
class Optimizer():
    def __init__(self, params, steppers, **defaults):
        self.steppers = listify(steppers)
        maybe_update(self.steppers, defaults, get_defaults)
        # might be a generator
        self.param_groups = list(params)
        # ensure params is a list of lists
        if not isinstance(self.param_groups[0], list): self.param_groups = [self.param_groups]
        self.hypers = [{**defaults} for p in self.param_groups]

    def grad_params(self):
        return [(p,hyper) for pg,hyper in zip(self.param_groups,self.hypers)
            for p in pg if p.grad is not None]

    def zero_grad(self):
        for p,hyper in self.grad_params():
            p.grad.detach_()
            p.grad.zero_()

    def step(self):
        for p,hyper in self.grad_params(): compose(p, self.steppers, **hyper)
```

Weight decay is subtracting `lr*wd*weight` from the weights. We need this function to have an attribute `_defaults` so that we are sure there is an hyper-parameter of the same name in our `Optimizer`.

```
#export
def weight_decay(p, lr, wd, **kwargs):
    p.data.mul_(1 - lr*wd)
    return p
weight_decay._defaults = dict(wd=0.)
```

L2 regularization is adding `wd*weight` to the gradients.

```
#export
def l2_reg(p, lr, wd, **kwargs):
    p.grad.data.add_(wd, p.data)
    return p
l2_reg._defaults = dict(wd=0.)
```

```
#export
sgd_opt = partial(Optimizer, steppers=[weight_decay, sgd_step])
```

```
learn,run = get_learn_run(nfs, data, 0.4, conv_layer, cbs=cbfs, opt_func=sgd_opt)
```

Before trying to train, let's check the behavior works as intended: when we don't provide a value for `wd`, we pull the corresponding default from `weight_decay`.

```
model = learn.model
```

```
opt = sgd_opt(model.parameters(), lr=0.1)
test_eq(opt.hypers[0]['wd'], 0.)
test_eq(opt.hypers[0]['lr'], 0.1)
```

But if we provide a value, it overrides the default.

```
opt = sgd_opt(model.parameters(), lr=0.1, wd=1e-4)
test_eq(opt.hypers[0]['wd'], 1e-4)
test_eq(opt.hypers[0]['lr'], 0.1)
```

Now let's fit.

```
cbfs = [partial(AvgStatsCallback,accuracy), CudaCallback]
```

```
learn,run = get_learn_run(nfs, data, 0.3, conv_layer, cbs=cbfs, opt_func=partial(sgd_opt, wd=0.01))
```

```
run.fit(1, learn)
```

    train: [1.8429603582577168, tensor(0.3637, device='cuda:0')]
    valid: [1.543950439453125, tensor(0.4700, device='cuda:0')]

This is already better than the baseline!

## With momentum

Momentum requires to add some state. We need to save the moving average of the gradients to be able to do the step and store this inside the optimizer state. To do this, we introduce statistics. Statistics are object with two methods:
- `init_state`, that returns the initial state (a tensor of 0. for the moving average of gradients)
- `update`, that updates the state with the new gradient value

We also read the `_defaults` values of those objects, to allow them to provide default values to hyper-parameters.

```
#export
class StatefulOptimizer(Optimizer):
    def __init__(self, params, steppers, stats=None, **defaults):
        self.stats = listify(stats)
        maybe_update(self.stats, defaults, get_defaults)
        super().__init__(params, steppers, **defaults)
        self.state = {}

    def step(self):
        for p,hyper in self.grad_params():
            if p not in self.state:
                #Create a state for p and call all the statistics to initialize it.
                self.state[p] = {}
                maybe_update(self.stats, self.state[p], lambda o: o.init_state(p))
            state = self.state[p]
            for stat in self.stats: state = stat.update(p, state, **hyper)
            compose(p, self.steppers, **state, **hyper)
            self.state[p] = state
```

```
#export
class Stat():
    _defaults = {}
    def init_state(self, p): raise NotImplementedError
    def update(self, p, state, **kwargs): raise NotImplementedError
```

Here is an example of `Stat`:

```
class AverageGrad(Stat):
    _defaults = dict(mom=0.9)

    def init_state(self, p): return {'grad_avg': torch.zeros_like(p.grad.data)}
    def update(self, p, state, mom, **kwargs):
        state['grad_avg'].mul_(mom).add_(p.grad.data)
        return state
```

Then we add the momentum step (instead of using the gradients to perform the step, we use the average).

```
#export
def momentum_step(p, lr, grad_avg, **kwargs):
    p.data.add_(-lr, grad_avg)
    return p
```

```
sgd_mom_opt = partial(StatefulOptimizer, steppers=[momentum_step,weight_decay],
                  stats=AverageGrad(), wd=0.01)
```

```
learn,run = get_learn_run(nfs, data, 0.3, conv_layer, cbs=cbfs, opt_func=sgd_mom_opt)
```

```
run.fit(1, learn)
```

    train: [1.7483108967833876, tensor(0.3988, device='cuda:0')]
    valid: [1.58801171875, tensor(0.4600, device='cuda:0')]

### Momentum experiments

What does momentum do to the gradients exactly? Let's do some plots to find out!

```
x = torch.linspace(-4, 4, 200)
y = torch.randn(200) + 0.3
betas = [0.5,0.7,0.9,0.99]
```

```
def plot_mom(f):
    _,axs = plt.subplots(2,2, figsize=(12,8))
    for beta,ax in zip(betas, axs.flatten()):
        ax.plot(y, linestyle='None', marker='.')
        avg,res = None,[]
        for i,yi in enumerate(y):
            avg,p = f(avg, beta, yi, i)
            res.append(p)
        ax.plot(res, color='red')
        ax.set_title(f'beta={beta}')
```

This is the regular momentum.

```
def mom1(avg, beta, yi, i):
    if avg is None: avg=yi
    res = beta*avg + yi
    return res,res
plot_mom(mom1)
```

As we can see, with a too high value, it may go way too high with no way to change its course.

Another way to smooth noisy data is to do an exponentially weighted moving average. In this case, there is a dampening of (1-beta) in front of the new value, which is less trusted than the current average.

```
def ewma(v1, v2, beta): return beta*v1 + (1-beta)*v2
```

```
def mom2(avg, beta, yi, i):
    if avg is None: avg=yi
    avg = ewma(avg, yi, beta)
    return avg, avg
plot_mom(mom2)
```

We can see it gets to a zero-constant when the data is purely random. If the data has a certain shape, it will get that shape (with some delay for high beta).

```
y = 1 - (x/3) ** 2 + torch.randn(200) * 0.1
```

```
y[0]=0.5
```

```
plot_mom(mom2)
```

Debiasing is here to correct the wrong information we may have in the very first batch. The debias term corresponds to the sum of the coefficient in our moving average. At the time step i, our average is:

$\begin{align*}
avg_{i} &= \beta\ avg_{i-1} + (1-\beta)\ v_{i} = \beta\ (\beta\ avg_{i-2} + (1-\beta)\ v_{i-1}) + (1-\beta)\ v_{i} \\
&= \beta^{2}\ avg_{i-2} + (1-\beta)\ \beta\ v_{i-1} + (1-\beta)\ v_{i} \\
&= \beta^{3}\ avg_{i-3} + (1-\beta)\ \beta^{2}\ v_{i-2} + (1-\beta)\ \beta\ v_{i-1} + (1-\beta)\ v_{i} \\
&\vdots \\
&= (1-\beta)\ \beta^{i}\ v_{0} + (1-\beta)\ \beta^{i-1}\ v_{1} + \cdots + (1-\beta)\ \beta^{2}\ v_{i-2} + (1-\beta)\ \beta\  v_{i-1} + (1-\beta)\ v_{i}
\end{align*}$

and so the sum of the coefficients is

$\begin{align*}
S &=(1-\beta)\ \beta^{i} + (1-\beta)\ \beta^{i-1} + \cdots + (1-\beta)\ \beta^{2} + (1-\beta)\ \beta + (1-\beta) \\
&= (\beta^{i} - \beta^{i+1}) + (\beta^{i-1} - \beta^{i}) + \cdots + (\beta^{2} - \beta^{3}) + (\beta - \beta^{2}) + (1-\beta) \\
&= 1 - \beta^{i+1}
\end{align*}$

since all the other terms cancel out each other.

By dividing by this term, we make our moving average a true average (in the sense that all the coefficients we used for the average sum up to 1).

```
def mom3(avg, beta, yi, i):
    if avg is None: avg=0
    avg = ewma(avg, yi, beta)
    return avg, avg/(1-beta**(i+1))
plot_mom(mom3)
```

## Adam and friends

In Adam, we use the gradient averages but with dampening (not like in SGD with momentum), so let's add this to the `AverageGrad` class.

```
#export
class AverageGrad(Stat):
    _defaults = dict(mom=0.9)

    def __init__(self, dampening:bool=False): self.dampening=dampening
    def init_state(self, p): return {'grad_avg': torch.zeros_like(p.grad.data)}
    def update(self, p, state, mom, **kwargs):
        state['mom_damp'] = 1-mom if self.dampening else 1.
        state['grad_avg'].mul_(mom).add_(state['mom_damp'], p.grad.data)
        return state
```

We also need to track the moving average of the gradients squared.

```
#export
class AverageSqrGrad(Stat):
    _defaults = dict(sqr_mom=0.99)

    def __init__(self, dampening:bool=True): self.dampening=dampening
    def init_state(self, p): return {'sqr_avg': torch.zeros_like(p.grad.data)}
    def update(self, p, state, sqr_mom, **kwargs):
        state['sqr_damp'] = 1 - sqr_mom if self.dampening else 1.
        state['sqr_avg'].mul_(sqr_mom).addcmul_(state['sqr_damp'], p.grad.data, p.grad.data)
        return state
```

We will also need the number of steps done during training for the debiasing.

```
#export
class StepCount(Stat):
    def init_state(self, p): return {'step': 0}
    def update(self, p, state, **kwargs):
        state['step'] += 1
        return state
```

This helper function computes the debias term. If we dampening, `damp = 1 - mom` and we get the same result as before. If we don't use dampening, (`damp = 1`) we will need to divide by `1 - mom` because that term is missing everywhere.

```
#export
def debias(mom, damp, step): return damp * (1 - mom**step) / (1-mom)
```

Then the Adam step is just the following:

```
#export
def adam_step(p, lr, mom, mom_damp, step, sqr_mom, sqr_damp, grad_avg, sqr_avg, eps, **kwargs):
    debias1 = debias(mom,     mom_damp, step)
    debias2 = debias(sqr_mom, sqr_damp, step)
    p.data.addcdiv_(-lr / debias1, grad_avg, (sqr_avg/debias2).sqrt() + eps)
    return p
adam_step._defaults = dict(eps=1e-5)
```

```
#Export
adam_opt = partial(StatefulOptimizer, steppers=adam_step, stats=[AverageGrad(dampening=True), AverageSqrGrad(), StepCount()])
```

```
learn,run = get_learn_run(nfs, data, 0.001, conv_layer, cbs=cbfs, opt_func=adam_opt)
```

```
run.fit(3, learn)
```

    train: [1.7502999214751047, tensor(0.3965, device='cuda:0')]
    valid: [1.395741943359375, tensor(0.5280, device='cuda:0')]
    train: [1.2368107259190322, tensor(0.5956, device='cuda:0')]
    valid: [1.130247314453125, tensor(0.6280, device='cuda:0')]
    train: [0.9639695091951683, tensor(0.6891, device='cuda:0')]
    valid: [1.077808837890625, tensor(0.6520, device='cuda:0')]

## LAMB

It's then super easy to implement a new optimizer. This is LAMB from a [very recent paper](https://arxiv.org/pdf/1904.00962.pdf):

$\begin{align}
g_{t}^{l} &= \nabla L(w_{t-1}^{l}, x_{t}) \\
m_{t}^{l} &= \beta_{1} m_{t-1}^{l} + (1-\beta_{1}) g_{t}^{l} \\
v_{t}^{l} &= \beta_{2} v_{t-1}^{l} + (1-\beta_{2}) g_{t}^{l} \odot g_{t}^{l} \\
m_{t}^{l} &= m_{t}^{l} / (1 - \beta_{1}^{t}) \\
v_{t}^{l} &= v_{t}^{l} / (1 - \beta_{2}^{t}) \\
r_{1} &= \|w_{t-1}^{l}\|_{2} \\
s_{t}^{l} &= \frac{m_{t}^{l}}{\sqrt{v_{t}^{l} + \epsilon}} + \lambda w_{t-1}^{l} \\
r_{2} &= \| s_{t}^{l} \|_{2} \\
\eta^{l} &= \eta * r_{1}/r_{2} \\
w_{t}^{l} &= w_{t}^{l-1} - \eta_{l} * s_{t}^{l} \\
\end{align}$

```
def lamb_step(p, lr, mom, mom_damp, step, sqr_mom, sqr_damp, grad_avg, sqr_avg, eps, wd, **kwargs):
    debias1 = debias(mom,     mom_damp, step)
    debias2 = debias(sqr_mom, sqr_damp, step)
    r1 = p.data.pow(2).mean().sqrt()
    step = (grad_avg/debias1) / ((sqr_avg/debias2).sqrt()+eps) + wd*p.data
    r2 = step.pow(2).mean().sqrt()
    p.data.add_(-lr * min(r1/r2,10), step)
    return p
lamb_step._defaults = dict(eps=1e-6, wd=0.)
```

```
lamb = partial(StatefulOptimizer, steppers=lamb_step, stats=[AverageGrad(dampening=True), AverageSqrGrad(), StepCount()])
```

```
learn,run = get_learn_run(nfs, data, 0.003, conv_layer, cbs=cbfs, opt_func=lamb)
```

```
run.fit(3, learn)
```

    train: [1.8331064230456027, tensor(0.3623, device='cuda:0')]
    valid: [1.45334814453125, tensor(0.5120, device='cuda:0')]
    train: [1.344959228856445, tensor(0.5551, device='cuda:0')]
    valid: [1.387230712890625, tensor(0.5440, device='cuda:0')]
    train: [1.0950260295486274, tensor(0.6420, device='cuda:0')]
    valid: [1.0847763671875, tensor(0.6380, device='cuda:0')]

Other recent variants of optimizers:
- [Large Batch Training of Convolutional Networks](https://arxiv.org/abs/1708.03888) (LARS also uses weight statistics, not just gradient statistics. Can you add that to this class?)
- [Adafactor: Adaptive Learning Rates with Sublinear Memory Cost](https://arxiv.org/abs/1804.04235) (Adafactor combines stats over multiple sets of axes)
- [Adaptive Gradient Methods with Dynamic Bound of Learning Rate](https://arxiv.org/abs/1902.09843)

## Export

```
#export
sgd_mom_opt = partial(StatefulOptimizer, steppers=[momentum_step,weight_decay],
                  stats=AverageGrad(), wd=0.01)
```

```
!python notebook2script.py 09_optimizers.ipynb
```

    Converted 09_optimizers.ipynb to exp/nb_09.py

```

```


[](https://colab.research.google.com/github/pronobis/libspn-keras/blob/master/examples/notebooks/DGC-SPN%20Image%20Completion.ipynb)
# Training a Deep Generalized Convolutional Sum-Product Network (DGC-SPN) for image completion.
Let's go through an example of building complex SPNs with [`libspn-keras`](https://github.com/pronobis/libspn-keras). The layer-based API of the library makes it straightforward to build advanced SPN architectures.

First let's set up the dependencies:

```python
!pip install libspn-keras matplotlib
```

## The Data
We'll use the Olivetti faces dataset. Note that this dataset is small by today's standards. Nevertheless, we'll be able to produce pretty good completions.

```python
!wget https://raw.githubusercontent.com/pronobis/libspn-keras/master/examples/notebooks/olivetti.raw
```

```python
import numpy as np
import tensorflow as tf

def load_olivetti(test_size=50):
    x = np.loadtxt("olivetti.raw").transpose().reshape(
        400, 64, 64, 1).transpose((0, 2, 1, 3)).astype(np.float32)
    train_x = x[:-test_size]
    test_x = x[-test_size:]
    return train_x, test_x

train_x, test_x = load_olivetti()

train_x_ds = tf.data.Dataset.from_tensor_slices((train_x,)).batch(32)
test_x_ds = tf.data.Dataset.from_tensor_slices((test_x,)).batch(32)

```

## A Reasonable Default for Accumulator Initializers
Since we're going to be using hard EM-based learning that will automatically break symmetries for equally weighted sums in the SPN,
we'll use a default accumulator initializer that initializes all weights with
$$\boldsymbol w_{\text{sum}} \sim Dir(\alpha)$$

```python
import libspn_keras as spnk

spnk.set_default_accumulator_initializer(
    spnk.initializers.Dirichlet(alpha=0.1)
)
```

## Leaf Variables
For the leaf variable, we'll use `spnk.layers.NormalLeaf`. We initialize the locations of the normal distribution components through an initializer that was based on the seminal work by [Poon and Domingos (2011)](https://arxiv.org/abs/1202.3732). This splits up train data into $n$ quantiles per pixel, after which
the mean of the $i$-th quantile is used as the location of the $i$-th component.

```python
normalize = spnk.layers.NormalizeStandardScore(axes=spnk.layers.NormalizeAxes.GLOBAL, input_shape=(64, 64, 1))

normalize.adapt(train_x_ds)

train_x_normalized = train_x_ds.map(normalize)

location_initializer = spnk.initializers.PoonDomingosMeanOfQuantileSplit(
    data=train_x_normalized
)
```

## DGC-SPN
A DGC-SPN consists of convolutional product and sum nodes.

We begin by using non-overlapping convolution patches for our first two product layers,
since this way we make sure that no scopes overlap between products.

For all convolutional product layers, we'll use *exponentially increasing dilation rates*. By doing so, we have 'overlapping' patches that still yield a valid SPN. These exponentially increasing dilation rates for convolutional SPNs were first introduced in [this paper](https://arxiv.org/abs/1902.06155).

The layer SPN construction is wrapped in a function so that we can redefine it later for other purposes. Don't worry, we'll get to the details.

```python
def define_spn(sum_op, infer_no_evidence=False):
    spnk.set_default_sum_op(sum_op)
    spnk.set_default_linear_accumulators_constraint(spnk.constraints.GreaterEqualEpsilon())
    stack = [
        spnk.layers.NormalizeStandardScore(input_shape=(64, 64, 1)),
        # Non-overlapping products
        spnk.layers.NormalLeaf(
            num_components=8,
            location_trainable=True,
            location_initializer=location_initializer,
            use_accumulators=True,
            scale_trainable=False
        ),
        tf.keras.layers.Dropout(rate=0.1),
        spnk.layers.Conv2DProduct(
            depthwise=True,
            strides=[1, 1],
            dilations=[1, 1],
            kernel_size=[2, 2],
            padding='full'
        ),
        spnk.layers.Local2DSum(num_sums=2),
        # Non-overlapping products
        spnk.layers.Conv2DProduct(
            depthwise=False,
            strides=[1, 1],
            dilations=[2, 2],
            kernel_size=[2, 2],
            padding='full'
        ),
        spnk.layers.Local2DSum(num_sums=2),
        # Overlapping products, starting at dilations [1, 1]
        spnk.layers.Conv2DProduct(
            depthwise=False,
            strides=[1, 1],
            dilations=[4, 4],
            kernel_size=[2, 2],
            padding='full'
        ),
        spnk.layers.Local2DSum(num_sums=2),
        # Overlapping products, with dilations [2, 2] and full padding
        spnk.layers.Conv2DProduct(
            depthwise=False,
            strides=[1, 1],
            dilations=[8, 8],
            kernel_size=[2, 2],
            padding='full'
        ),
        spnk.layers.Local2DSum(num_sums=2),
        # Overlapping products, with dilations [2, 2] and full padding
        spnk.layers.Conv2DProduct(
            depthwise=False,
            strides=[1, 1],
            dilations=[16, 16],
            kernel_size=[2, 2],
            padding='full'
        ),
        spnk.layers.Local2DSum(num_sums=2),
        spnk.layers.Conv2DProduct(
            depthwise=False,
            strides=[1, 1],
            dilations=[32, 32],
            kernel_size=[2, 2],
            padding='full'
        ),
        spnk.layers.Local2DSum(num_sums=2),
        # Overlapping products, with dilations [2, 2] and 'final' padding to combine
        # all scopes
        spnk.layers.Conv2DProduct(
            depthwise=False,
            strides=[1, 1],
            dilations=[64, 64],
            kernel_size=[2, 2],
            padding='final'
        ),
        spnk.layers.LogDropout(rate=0.1),
        spnk.layers.SpatialToRegions(),
        spnk.layers.RootSum(
            return_weighted_child_logits=False
        )
    ]
    sum_product_network = spnk.models.SequentialSumProductNetwork(
      stack, infer_no_evidence=infer_no_evidence)
    return sum_product_network

sum_product_network = define_spn(sum_op=spnk.SumOpEMBackprop())
sum_product_network.summary()
```

### Training the DGC-SPN with hard EM
We use the `libspn_keras.optimizers.OnlineExpectationMaximization` optimizer. Note that this optimizer works for generative training setups only and should be
combined with one of the EM-based sum ops. In the snippet above, we have set this to ``SumOpUnweightedEMBackprop``, so we're good to go!

We'll configure a train set and a test set using `tf.data.Dataset`. As first suggested in the work by [Poon and Domingos](https://arxiv.org/abs/1202.3732), we normalize each sample by subtracting the mean and dividing by the
standard devation.

```python
import tensorflow as tf
from libspn_keras.optimizers import OnlineExpectationMaximization
from libspn_keras import losses
from libspn_keras import metrics

batch_size = 100

train_data = (
    tf.data.Dataset.from_tensor_slices((train_x,))
    .shuffle(350)
    .batch(batch_size)
)

test_data = (
    tf.data.Dataset.from_tensor_slices((test_x,))
    .batch(batch_size)
)

sum_product_network.compile(
    optimizer=spnk.optimizers.OnlineExpectationMaximization(learning_rate=0.05),
    loss=spnk.losses.NegativeLogLikelihood(),
    metrics=[spnk.metrics.LogLikelihood()]
)
sum_product_network.fit(train_data, epochs=100)
sum_product_network.evaluate(test_data)
```

Now comes the tricky part. To do image completion, we need to propagate some kind of signal to the leafs that we occlude. Probabilistically, we exclude the variables corresponding to the occluded pixels from the evidence $\mathbf E$.

In the seminal work by [Poon and Domingos (2011)](https://arxiv.org/abs/1202.3732), in which Sum-Product Networks were proposed as a new type of deep probabilistic models, the image completion was accomplished by backpropagating the **gradients** and using the these to obtain the 'posterior marginal' of the missing variables.

The paper itself, however, does not mention this explicitly. Only after we inspect the code from the paper that comes with it we find:
```java
// compute marginal by differentiation; see Darwiche-03 for details
public void cmpMAPBottomHalfMarginal(Instance inst) {
  setInputOccludeBottomHalf(inst);
  eval();
  cmpDerivative();

  for (int i=0; i<Parameter.inputDim1_/2; i++) {
    for (int j=0; j<Parameter.inputDim2_; j++)
      MyMPI.buf_int_[MyMPI.buf_idx_++]=Utils.getIntVal(inst, inst.vals_[i][j]);
  }
  for (int i=Parameter.inputDim1_/2; i<Parameter.inputDim1_; i++) {
    for (int j=0; j<Parameter.inputDim2_; j++) {
      int ri=Region.getRegionId(i, i+1, j, j+1);
      Region r=Region.getRegion(ri);
      double p=cmpMarginal(r);
      MyMPI.buf_int_[MyMPI.buf_idx_++]=Utils.getIntVal(inst,p);//(int)(p*255);
    }
  }
}
```
Note the commpent on top! [The paper by Darwiche (2003)](http://reasoning.cs.ucla.edu/fetch.php?id=22&type=pdf) indeed mentions a way of determining the _posterior marginal_ of any variable when the network computes a _network polynomial_. SPNs happen to compute a network polynomial, so we can indeed use gradients to compute posterior marginals!

From the paper, we have:

_For every variable $X$ and evidence $\mathbf e \notin \mathbf E$_:
$$
P(x_i \mid \mathbf e) = \frac{1}{\text{Root}(\mathbf e)} \frac{\partial \text{Root}}{\partial \text{Leaf}_{x_i}}(\mathbf e)
$$

Note that in the last equation we are using lower case variable with an index: $x_i$. The index corresponds to the _component_ on top of the variable. In other words, $P(x_i \mid \mathbf e)$ is the posterior marginal of the $i$-th component attached to variable $X$. The variable $X$ is excluded from the evidence $\mathbf E$. Lower case $\mathbf e$ is the actual assignment of pixel values in the part of the image that is _included_ (not occluded). Note that $P(x_i \mid \mathbf e)$ is **not** the output probability of the leaf. Rather, it is the probability that the pixel value at $X$ was _generated_ by the component at $x_i$.

In the code by Poon and Domingos we mentioned earlier, the posterior marginals are used as follows:
$$
\text{InferredPixelValue} = \sum_i \mu_i P(x_i \mid \mathbf e)
$$
Where $\mu_i$ is the mean of the $i$-th Gaussian component.

So how would this work for the SPN we have now? The layer implementations in LibSPN Keras are propagating log probabilities. So at the root of the network we'll find $\log(\text{Root}(\mathbf e))$ instead of just $\text{Root}(\mathbf e)$. Simply wrapping the value with $\exp(\cdot)$ results in numerical difficulties, so that's a dead end.

Let's reverse our approach: we'll start propagating gradients from the roots to the leafs and see how far we are from solving the problem!

If we just use TensorFlow's autograd engine we can obtain something like:
```python
leaf_log_prob = leaf(img)
leaf_log_prob = tf.where(evidence_mask, leaf_log_prob, tf.zeros_like(leaf_log_prob))
with tf.GradientTape() as g:
    g.watch(leaf_log_prob)
    root = apply_sum_product_stack(leaf_log_prob)
droot_log_prob_dleaf_log_prob = g.gradient(root_log_prob, leaf_log_prob)
```

This gives us:

$$
\frac{\partial \log(\text{Root})}{\partial \log(\text{Leaf}_{x_i})}
$$

Which we can rewrite a bit:

\begin{align}
\frac{\partial \log(\text{Root})}{\partial \log(\text{Leaf}_{x_i})}
&=\frac{\partial \log(\text{Root})}{\partial \text{Root}}
\frac{\partial \text{Root}}{\partial \log(\text{Leaf}_{x_i})} \\&= \frac{1}{Root} \frac{\partial \text{Root}}{\partial \log(\text{Leaf}_{x_i})} \\ &= \frac{1}{Root} \frac{\partial \text{Root}}{\partial \log(\text{Leaf}_{x_i})} \frac{1}{\text{Leaf}_{x_i}} \text{Leaf}_{x_i} \\&=  \frac{1}{Root} \frac{\partial \text{Root}}{\partial \log(\text{Leaf}_{x_i})} \frac{\partial \log (\text{Leaf}_{x_i})}{\partial \text{Leaf}_{x_i}} \text{Leaf}_{x_i} \\&=  \frac{1}{Root} \frac{\partial \text{Root}}{\partial \text{Leaf}_{x_i}}  \text{Leaf}_{x_i} \\
&=\frac{1}{Root} \frac{\partial \text{Root}}{\partial \text{Leaf}_{x_i}} & (x_i \notin \mathbf E\text{, so } \text{Leaf}_{x_i} = 1)\\
&= P(x_i \mid \mathbf e)
\end{align}

So in fact, we already have what we want in `droot_log_prob_dleaf_log_prob`!

Below, we use those observations to compute the image completions.

```python
import matplotlib.pyplot as plt
from tensorflow import keras

%matplotlib inline

completion_model = define_spn(
    sum_op=spnk.SumOpGradBackprop(logspace_accumulators=False),
    infer_no_evidence=True
)
completion_model.set_weights(sum_product_network.get_weights())
completion_model.compile(metrics=[keras.metrics.MeanSquaredError()])

def eval(model, x, omit_side):
    print("omitting ", omit_side)
    evidence_mask = get_evidence_mask(omit_side).astype(np.bool)
    model.evaluate([x, evidence_mask], x, verbose=2)
    completion_out = model.predict([x, evidence_mask])
    image_grid = make_image_grid(completion_out, num_rows=5)

    plt.figure(figsize=(14, 14))
    plt.imshow(image_grid.squeeze(), cmap='gray')
    plt.show()

def make_image_grid(images, num_rows):
    images_per_row = np.split(images, axis=0, indices_or_sections=num_rows)
    rows = [np.concatenate(imgs, axis=1) for imgs in images_per_row]
    full_grid = np.concatenate(rows, axis=0)
    return full_grid

def get_evidence_mask(omit_side):
  if omit_side == "top":
    return np.concatenate(
        [np.zeros([50, 32, 64, 1]), np.ones([50, 32, 64, 1])], axis=1)
  elif omit_side == 'bottom':
    return np.concatenate(
        [np.ones([50, 32, 64, 1]), np.zeros([50, 32, 64, 1])], axis=1)
  elif omit_side == 'right':
    return np.concatenate(
        [np.ones([50, 64, 32, 1]), np.zeros([50, 64, 32, 1])], axis=2)
  elif omit_side == 'left':
    return np.concatenate(
        [np.zeros([50, 64, 32, 1]), np.ones([50, 64, 32, 1])], axis=2)
  else:
    raise ValueError("We have a problem")

eval(completion_model, test_x, 'top')
eval(completion_model, test_x, 'bottom')
eval(completion_model, test_x, 'left')
eval(completion_model, test_x, 'right')
```


# Characterization of Systems in the Time Domain

*This Jupyter notebook is part of a [collection of notebooks](../index.ipynb) in the bachelors module Signals and Systems, Communications Engineering, Universität Rostock. Please direct questions and suggestions to [Sascha.Spors@uni-rostock.de](mailto:Sascha.Spors@uni-rostock.de).*

## Step Response

The response of an LTI system to the [Heaviside signal](../continuous_signals/standard_signals.ipynb#Heaviside-Signal) $\epsilon(t)$ at its input is known as [*step response*](https://en.wikipedia.org/wiki/Step_response). It is defined as

\begin{equation}
h_\epsilon(t) = \mathcal{H} \{ \epsilon(t) \}
\end{equation}

The step response characterizes the properties of a system when the input signal is 'switched on' at $t=0$. It can be related to the impulse response $h(t)$ of the system by

\begin{equation}
h_\epsilon(t) = \epsilon(t) * h(t) = \int_{-\infty}^{t} h(\tau) \; d\tau
\end{equation}

The last equality can be deduced for instance by [graphical interpretation of the convolution](convolution.ipynb#Graphical-Interpretation). As the inverse operation to above integral is the derivative, this implies that the impulse response is the derivative of the step response

\begin{equation}
h(t) = \frac{d h_\epsilon(t)}{dt}
\end{equation}

Using this result together with the properties of the convolution for derivatives, the output signal $y(t) = \mathcal{H} \{ x(t) \}$ in terms of the step response reads

\begin{equation}
y(t) = x(t) * \frac{d h_\epsilon(t)}{dt} = \frac{d x(t)}{dt} * h_\epsilon(t)
\end{equation}

The output signal of an LTI system is given by convolving the derivative of the input signal with the step response.

The step response is an alternative to characterize the properties of an LTI system since a Dirac impulse cannot be realized in practice. It plays an important role in the theory of [control systems](https://en.wikipedia.org/wiki/Control_system).

**Copyright**

This notebook is provided as [Open Educational Resource](https://en.wikipedia.org/wiki/Open_educational_resources). Feel free to use the notebook for your own purposes. The text is licensed under [Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/), the code of the IPython examples under the [MIT license](https://opensource.org/licenses/MIT). Please attribute the work as follows: *Sascha Spors, Continuous- and Discrete-Time Signals and Systems - Theory and Computational Examples*.