#  Winpython Default checker

```python
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", category=FutureWarning)
# warnings.filterwarnings("ignore")  # would silence all warnings
```

```python
%matplotlib inline
```

## Compilers: Numba and Cython

##### Requirement
To get Cython working, Winpython 3.5 users should install "Microsoft Visual C++ Build Tools 2015" (visualcppbuildtools_full.exe, a 4 Go installation) at https://beta.visualstudio.com/download-visual-studio-vs/

To get Numba working, not-windows10 users may have to install "Microsoft Visual C++ 2015 Redistributable" (vc_redist) at  <https://beta.visualstudio.com/download-visual-studio-vs/>

#### Compiler toolchains

```python
# checking Numba JIT toolchain
import numpy as np
image = np.zeros((1024, 1536), dtype = np.uint8)

from pylab import imshow, show
from timeit import default_timer as timer

def create_fractal(min_x, max_x, min_y, max_y, image, iters , mandelx):
  height = image.shape[0]
  width = image.shape[1]
  pixel_size_x = (max_x - min_x) / width
  pixel_size_y = (max_y - min_y) / height

  for x in range(width):
    real = min_x + x * pixel_size_x
    for y in range(height):
      imag = min_y + y * pixel_size_y
      color = mandelx(real, imag, iters)
      image[y, x] = color
```

##### Numba (a JIT Compiler)

```python
from numba import autojit

@autojit
def mandel(x, y, max_iters):
  c = complex(x, y)
  z = 0.0j
  for i in range(max_iters):
    z = z*z + c
    if (z.real*z.real + z.imag*z.imag) >= 4:
      return i
  return max_iters

start = timer()
create_fractal(-2.0, 1.0, -1.0, 1.0, image, 20 , mandel)
dt = timer() - start

print ("Mandelbrot created by numba in %f s" % dt)
imshow(image)
show()
```

##### Cython (a compiler for writing C extensions for the Python language)
WinPython 3.5 and 3.6 users may not have mingwpy available, and so need "VisualStudio C++ Community Edition 2015" https://www.visualstudio.com/downloads/download-visual-studio-vs#d-visual-c

```python
# Cython + Mingwpy compiler toolchain test
%load_ext Cython
```

```cython
%%cython -a
# with %%cython -a , full C-speed lines are shown in white, slowest python-speed lines are shown in dark yellow lines
# ==> put your cython rewrite effort on dark yellow lines
def mandel_cython(x, y, max_iters):
  cdef int   i
  cdef double   cx, cy , zx, zy
  cx , cy = x, y
  zx , zy =0 ,0
  for i in range(max_iters):
    zx , zy = zx*zx - zy*zy + cx , zx*zy*2 + cy
    if (zx*zx + zy*zy) >= 4:
      return i
  return max_iters
```

```python
start = timer()
create_fractal(-2.0, 1.0, -1.0, 1.0, image, 20 , mandel_cython)
dt = timer() - start

print ("Mandelbrot created by cython in %f s" % dt)
imshow(image)
```

##  Graphics: Matplotlib, Pandas, Seaborn, Holoviews, Bokeh, bqplot, ipyleaflet, plotnine

```python
# Matplotlib
# for more examples, see: http://matplotlib.org/gallery.html
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
from matplotlib import cm

fig = plt.figure()
ax = fig.gca(projection='3d')
X, Y, Z = axes3d.get_test_data(0.05)
ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3)
cset = ax.contourf(X, Y, Z, zdir='z', offset=-100, cmap=cm.coolwarm)
cset = ax.contourf(X, Y, Z, zdir='x', offset=-40, cmap=cm.coolwarm)
cset = ax.contourf(X, Y, Z, zdir='y', offset=40, cmap=cm.coolwarm)

ax.set_xlabel('X')
ax.set_xlim(-40, 40)
ax.set_ylabel('Y')
ax.set_ylim(-40, 40)
ax.set_zlabel('Z')
ax.set_zlim(-100, 100)

plt.show()
```

```python
# Seaborn
# for more examples, see http://stanford.edu/~mwaskom/software/seaborn/examples/index.html
import seaborn as sns
sns.set()
df = sns.load_dataset("iris")
sns.pairplot(df, hue="species", height=1.5)
```

```python
# altair-2.0.0 example
import altair as alt

# Uncomment/run this line to enable Altair in JupyterLab/nteract:
#alt.renderers.enable('default') # api_v2
#alt.renderers.enable('notebook') # api_v2,if in Notebook
alt.Chart(df).mark_bar().encode(
    x=alt.X('sepal_length', bin=alt.Bin(maxbins=50)),
    y='count(*):Q',
    color='species:N',
    #column='species',
).interactive() # api_v1 .configure_cell(width=200)

```

```python
# temporary warning removal
import warnings
import matplotlib as mpl
warnings.filterwarnings("ignore", category=mpl.cbook.MatplotlibDeprecationWarning)
# Holoviews
# for more example, see http://holoviews.org/Tutorials/index.html
import numpy as np
import holoviews as hv
hv.extension('matplotlib')
dots = np.linspace(-0.45, 0.45, 11)
fractal = hv.Image(image)

layouts = {y: (fractal * hv.Points(fractal.sample([(i,y) for i in dots])) +
               fractal.sample(y=y) )
            for y in np.linspace(0, 0.45,11)}

hv.HoloMap(layouts, kdims=['Y']).collate().cols(2)
```

```python
# Bokeh 0.12.5
import numpy as np
from six.moves import zip
from bokeh.plotting import figure, show, output_notebook
N = 4000
x = np.random.random(size=N) * 100
y = np.random.random(size=N) * 100
radii = np.random.random(size=N) * 1.5
colors = ["#%02x%02x%02x" % (int(r), int(g), 150) for r, g in zip(50+2*x, 30+2*y)]

output_notebook()
TOOLS="hover,crosshair,pan,wheel_zoom,box_zoom,reset,tap,save,box_select,poly_select,lasso_select"

p = figure(tools=TOOLS)
p.scatter(x,y, radius=radii, fill_color=colors, fill_alpha=0.6, line_color=None)
show(p)
```

```python
# Datashader (holoviews+Bokeh)
import numpy as np
import pandas as pd
import holoviews as hv
import datashader as ds
from holoviews.operation.datashader import aggregate, shade, datashade, dynspread
from bokeh.models import DatetimeTickFormatter
hv.extension('bokeh')

def time_series(T = 1, N = 100, mu = 0.1, sigma = 0.1, S0 = 20):
    """Parameterized noisy time series"""
    dt = float(T)/N
    t = np.linspace(0, T, N)
    W = np.random.standard_normal(size = N)
    W = np.cumsum(W)*np.sqrt(dt) # standard brownian motion
    X = (mu-0.5*sigma**2)*t + sigma*W
    S = S0*np.exp(X) # geometric brownian motion
    return S

def apply_formatter(plot, element):
    plot.handles['xaxis'].formatter = DatetimeTickFormatter()

drange = pd.date_range(start="2014-01-01", end="2016-01-01", freq='1D') # or '1min'
dates = drange.values.astype('int64')/10**6 # Convert dates to ints
curve = hv.Curve((dates, time_series(N=len(dates), sigma = 1)))
```

```python
%%opts RGB [finalize_hooks=[apply_formatter] width=800]
%%opts Overlay [finalize_hooks=[apply_formatter] width=800]
%%opts Scatter [tools=['hover', 'box_select']] (line_color="black" fill_color="red" size=10)

from holoviews.operation.timeseries import rolling, rolling_outlier_std
smoothed = rolling(curve, rolling_window=50)
outliers = rolling_outlier_std(curve, rolling_window=50, sigma=2)
datashade(curve, cmap=["blue"]) * dynspread(datashade(smoothed, cmap=["red"]),max_px=1) * outliers
```

```python
#bqplot
from IPython.display import display
from bqplot import (Figure, Map, Mercator, Orthographic, ColorScale, ColorAxis,
                    AlbersUSA, topo_load, Tooltip)
def_tt = Tooltip(fields=['id', 'name'])
map_mark = Map(scales={'projection': Mercator()}, tooltip=def_tt)
map_mark.interactions = {'click': 'select', 'hover': 'tooltip'}
fig = Figure(marks=[map_mark], title='Interactions Example')
display(fig)
```

```python
# ipyleaflet (javascript library usage)
from ipyleaflet import (
    Map, Marker, TileLayer, ImageOverlay, Polyline, Polygon,
    Rectangle, Circle, CircleMarker, GeoJSON, DrawControl
)
from traitlets import link
center = [34.6252978589571, -77.34580993652344]
m = Map(center=[34.6252978589571, -77.34580993652344], zoom=10)
dc = DrawControl()

def handle_draw(self, action, geo_json):
    print(action)
    print(geo_json)
m
m
```

```python
dc.on_draw(handle_draw)
m.add_control(dc)
```

```python
# plotnine: giving a taste of ggplot of R langage (formerly we were using ggpy)
from plotnine import ggplot, aes, geom_blank, geom_point, stat_smooth, facet_wrap, theme_bw
from plotnine.data import mtcars
ggplot(mtcars, aes(x='hp', y='wt', color='mpg')) + geom_point() +\
facet_wrap("~cyl") + theme_bw()
```

## Ipython Notebook: Interactivity & other

```python
import IPython;IPython.__version__
```

```python
# Audio Example : https://github.com/ipython/ipywidgets/blob/master/examples/Beat%20Frequencies.ipynb
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
from ipywidgets import interactive
from IPython.display import Audio, display
def beat_freq(f1=220.0, f2=224.0):
    max_time = 3
    rate = 8000
    times = np.linspace(0,max_time,rate*max_time)
    signal = np.sin(2*np.pi*f1*times) + np.sin(2*np.pi*f2*times)
    print(f1, f2, abs(f1-f2))
    display(Audio(data=signal, rate=rate))
    try:
         plt.plot(signal); #plt.plot(v.result);
    except:
         pass
    return signal
v = interactive(beat_freq, f1=(200.0,300.0), f2=(200.0,300.0))
display(v)
```

```python
# Networks graph Example : https://github.com/ipython/ipywidgets/blob/master/examples/Exploring%20Graphs.ipynb
%matplotlib inline
from ipywidgets import interact
import matplotlib.pyplot as plt
import networkx as nx
# wrap a few graph generation functions so they have the same signature

def random_lobster(n, m, k, p):
    return nx.random_lobster(n, p, p / m)

def powerlaw_cluster(n, m, k, p):
    return nx.powerlaw_cluster_graph(n, m, p)

def erdos_renyi(n, m, k, p):
    return nx.erdos_renyi_graph(n, p)

def newman_watts_strogatz(n, m, k, p):
    return nx.newman_watts_strogatz_graph(n, k, p)

@interact(n=(2,30), m=(1,10), k=(1,10), p=(0.0, 1.0, 0.001),
        generator={'lobster': random_lobster,
                   'power law': powerlaw_cluster,
                   'Newman-Watts-Strogatz': newman_watts_strogatz,
                   u'Erdős-Rényi': erdos_renyi,
                   })
def plot_random_graph(n, m, k, p, generator):
    g = generator(n, m, k, p)
    nx.draw(g)
    plt.title(generator.__name__)
    plt.show()

```

## Mathematical: statsmodels, lmfit,

```python
# checking statsmodels
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('ggplot')
import statsmodels.api as sm
data = sm.datasets.anes96.load_pandas()
party_ID = np.arange(7)
labels = ["Strong Democrat", "Weak Democrat", "Independent-Democrat",
          "Independent-Independent", "Independent-Republican",
          "Weak Republican", "Strong Republican"]
plt.rcParams['figure.subplot.bottom'] = 0.23  # keep labels visible
plt.rcParams['figure.figsize'] = (6.0, 4.0)  # make plot larger in notebook
age = [data.exog['age'][data.endog == id] for id in party_ID]
fig = plt.figure()
ax = fig.add_subplot(111)
plot_opts={'cutoff_val':5, 'cutoff_type':'abs',
                                'label_fontsize':'small',
                                'label_rotation':30}
sm.graphics.beanplot(age, ax=ax, labels=labels,
                     plot_opts=plot_opts)
ax.set_xlabel("Party identification of respondent")
ax.set_ylabel("Age")
```

```python
# lmfit test (from http://nbviewer.ipython.org/github/lmfit/lmfit-py/blob/master/examples/lmfit-model.ipynb)
import numpy as np
import matplotlib.pyplot as plt
def decay(t, N, tau):
    return N*np.exp(-t/tau)
t = np.linspace(0, 5, num=1000)
data = decay(t, 7, 3) + np.random.randn(*t.shape)

from lmfit import Model

model = Model(decay, independent_vars=['t'])
result = model.fit(data, t=t, N=10, tau=1)
plt.plot(t, data)  # data
plt.plot(t, decay(t=t, **result.values), color='orange', linewidth=5)  # best-fit model
```

## DataFrames: Pandas, Dask

```python
#Pandas
import pandas as pd
import numpy as np

idx = pd.date_range('2000', '2005', freq='d', closed='left')
datas = pd.DataFrame({'Color':  [ 'green' if x> 1 else 'red' for x in np.random.randn(len(idx))],
         'Measure': np.random.randn(len(idx)), 'Year': idx.year},
          index=idx.date)
datas.head()
```

### Split / Apply / Combine
    Split your data into multiple independent groups.
    Apply some function to each group.
    Combine your groups back into a single data object.

```python
datas.query('Measure > 0').groupby(['Color','Year']).size().unstack()
```

## Web Scraping: Beautifulsoup

```python
# checking Web Scraping: beautifulsoup and requests
import requests
from bs4 import BeautifulSoup

URL = 'http://en.wikipedia.org/wiki/Franklin,_Tennessee'

req = requests.get(URL, headers={'User-Agent' : "Mining the Social Web"})
soup = BeautifulSoup(req.text, "lxml")

geoTag = soup.find(True, 'geo')

if geoTag and len(geoTag) > 1:
    lat = geoTag.find(True, 'latitude').string
    lon = geoTag.find(True, 'longitude').string
    print ('Location is at', lat, lon)
elif geoTag and len(geoTag) == 1:
    (lat, lon) = geoTag.string.split(';')
    (lat, lon) = (lat.strip(), lon.strip())
    print ('Location is at', lat, lon)
else:
    print ('No location found')
```

## Operations Research: Pulp

```python
# Pulp example : minimizing the weight to carry 99 pennies
#                (from Philip I Thomas)
# see https://www.youtube.com/watch?v=UmMn-N5w-lI#t=995
# Import PuLP modeler functions
from pulp import *
# The prob variable is created to contain the problem data
prob = LpProblem("99 pennies Problem",LpMinimize)

# Variables represent how many of each coin we want to carry
pennies = LpVariable("Number of pennies",0,None,LpInteger)
nickels = LpVariable("Number of nickels",0,None,LpInteger)
dimes = LpVariable("Number of dimes",0,None,LpInteger)
quarters = LpVariable("Number of quarters",0,None,LpInteger)

# The objective function is added to 'prob' first

# we want to minimize (LpMinimize) this
prob += 2.5 * pennies + 5 * nickels + 2.268 * dimes + 5.670 * quarters, "Total coins Weight"

# We want exactly 99 cents
prob += 1 * pennies + 5 * nickels + 10 * dimes + 25 * quarters == 99, ""

# The problem data is written to an .lp file
prob.writeLP("99cents.lp")
prob.solve()

# print ("status",LpStatus[prob.status] )
print ("Minimal Weight to carry exactly 99 pennies is %s grams" % value(prob.objective))
# Each of the variables is printed with it's resolved optimum value
for v in prob.variables():
    print (v.name, "=", v.varValue)
```

## Deep Learning: see tutorial-first-neural-network-python-keras

## Symbolic Calculation: sympy

```python
# checking sympy
import sympy
a, b =sympy.symbols('a b')
e=(a+b)**5
e.expand()
```

## SQL tools: sqlite, Ipython-sql, sqlite_bro, baresql, db.py

```python
# checking Ipython-sql, sqlparse, SQLalchemy
%load_ext sql
```

```sql
%%sql sqlite:///.baresql.db
DROP TABLE IF EXISTS writer;
CREATE TABLE writer (first_name, last_name, year_of_death);
INSERT INTO writer VALUES ('William', 'Shakespeare', 1616);
INSERT INTO writer VALUES ('Bertold', 'Brecht', 1956);
SELECT  * , sqlite_version() as sqlite_version from Writer order by Year_of_death
```

```python
# checking baresql
from __future__ import print_function, unicode_literals, division  # line needed only if Python2.7
from baresql import baresql
bsql = baresql.baresql(connection="sqlite:///.baresql.db")
bsqldf = lambda q: bsql.df(q, dict(globals(),**locals()))

users = ['Alexander', 'Billy', 'Charles', 'Danielle', 'Esmeralda', 'Franz', 'Greg']
#  We use the python 'users' list like a SQL table
sql = "select 'Welcome ' ||  c0 || ' !' as say_hello, length(c0) as name_length from users$$ where c0 like '%a%' "
bsqldf(sql)
```

```python
# Transfering Datas to sqlite, doing transformation in sql, going back to Pandas and Matplotlib
bsqldf('''
select Color, Year, count(*) as size
from datas$$
where Measure > 0
group by Color, Year'''
          ).set_index(['Year', 'Color']).unstack().plot(kind='bar')
```

```python
# checking db.py
from db import DB
db=DB(dbtype="sqlite", filename=".baresql.db")
db.query("select  sqlite_version() as sqlite_version  ;")
```

```python
db.tables
```

```python
# checking sqlite_bro: this should lanch a separate non-browser window with sqlite_bro's welcome
!cmd start cmd /C sqlite_bro
```

```python
# pyodbc
import pyodbc

# look for pyodbc providers
sources = pyodbc.dataSources()
dsns = list(sources.keys())
sl = ['    %s [%s]' % (dsn, sources[dsn]) for dsn in dsns]
print("pyodbc Providers: (beware 32/64 bit driver and python version must match)\n", '\n'.join(sl))
```

```python
# pythonnet
import clr
clr.AddReference("System.Data")
clr.AddReference('System.Data.Common')
import System.Data.OleDb as ADONET
import System.Data.Odbc as ODBCNET
import System.Data.Common as DATACOM

table = DATACOM.DbProviderFactories.GetFactoryClasses()
print("\n .NET Providers: (beware 32/64 bit driver and python version must match)")
for  row in table.Rows:
    print("   %s" % row[table.Columns[0]])
    print("      ",[row[column] for column in table.Columns if column != table.Columns[0]])
```

## Qt libraries Demo

#### See  [Dedicated Qt Libraries Demo](Qt_libraries_demo.ipynb)

## Wrap-up

```python
# optional scipy full test (takes up to 10 minutes)
#!cmd /C start cmd /k python.exe  -c "import scipy;scipy.test()"
```

```python
!pip list
```

```python

```

```python

```

```python

```


```
#default_exp causal_model
```

# Causal Model
> Implementing instrumental variable and backdoor criteria plus propensity and regression methods

**Notes**

* Causal Bayesian Networkx: [here](http://conference.scipy.org/proceedings/scipy2015/pdfs/mike_pacer.pdf)

dowhy > causal_model.py CausalModel > causal_graph.py CausalGraph

```
%matplotlib inline
%load_ext autoreload
%autoreload 2
```

```
#export
import dowhy as dw
from typing import List, Union
import networkx as nx
import itertools
import sympy as sp
import matplotlib.pyplot as plt
import sklearn
from sklearn import linear_model, neighbors
import numpy as np
from scipy import stats
from bcg.basics import CommonCauses, Instruments, EffectModifiers, Treatments, Outcomes, get_Xy
import pandas as pd
```

```
plt.style.use('bmh')
```

```
treatments = ['V0', 'V1']
outcome = 'Y'
common_causes = ['W0']
effect_modifiers = ['X0']
instruments = []
observed_nodes = treatments + [outcome] + instruments + effect_modifiers + common_causes
add_unobserved_confounder = True
missing_nodes_as_confounders = True
```

```
cg_ref = dw.causal_graph.CausalGraph(treatments,
                                 [outcome], graph=None,
                                 common_cause_names=common_causes,
                                 instrument_names=instruments,
                                 effect_modifier_names=effect_modifiers,
                                 observed_node_names=observed_nodes)
```

```
cg_ref._graph.nodes(data=True)
```

    NodeDataView({'V0': {'observed': 'yes'}, 'V1': {'observed': 'yes'}, 'Y': {'observed': 'yes'}, 'W0': {'observed': 'yes'}, 'X0': {'observed': 'yes'}, 'U': {'label': 'Unobserved Confounders', 'observed': 'no'}})

```
#export
class CausalGraph:
    def __init__(self, treatments:List[str], outcome:str='Y',
                 common_causes:List[str]=None, effect_modifiers:List[str]=None,
                 instruments:List[str]=None, observed_nodes:List[str]=None,
                 missing_nodes_as_confounders:bool=False,
                 add_unobserved_confounder:bool=True):
        if common_causes is None: common_causes = []
        if effect_modifiers is None: effect_modifiers = []
        if instruments is None: instruments = []
        if missing_nodes_as_confounders:
            all_passed_nodes = treatments + [outcome] + \
                common_causes + effect_modifiers + instruments
            missing_nodes = [node for node in all_passed_nodes if node not in observed_nodes]
            common_causes = list(common_causes) + missing_nodes

        self.g = self.create_nx_digraph(treatments, outcome,
                                        common_causes, instruments,
                                        effect_modifiers, add_unobserved_confounder)

    @staticmethod
    def create_nx_digraph(treatments:List[str], outcome:str, common_causes:List[str],
                          instruments:List[str], effect_modifiers:List[str],
                          add_unobserved_confounder:bool=False):
        g = nx.DiGraph()
        g.add_edges_from([(treatment, outcome) for treatment in treatments])
        g.add_edges_from([(common_cause, treatment)
                          for common_cause, treatment in itertools.product(common_causes, treatments)])
        g.add_edges_from([(common_cause, outcome)
                          for common_cause in common_causes])
        g.add_edges_from([(effect_modifier, outcome) for effect_modifier in effect_modifiers])
        g.add_edges_from([(instrument, treatment) for instrument, treatment in itertools.product(instruments, treatments)])
        nx.set_node_attributes(g, True, 'observed')
        if add_unobserved_confounder:
            g.add_node('U', observed=False)
            g.add_edges_from([('U', treatment) for treatment in treatments])
            g.add_edge('U', outcome)

        return g

```

```
cg = CausalGraph(treatments=treatments, outcome=outcome, common_causes=common_causes,
                 effect_modifiers=effect_modifiers, observed_nodes=observed_nodes,
                 missing_nodes_as_confounders=missing_nodes_as_confounders,
                 add_unobserved_confounder=add_unobserved_confounder)
```

```
cg.g.nodes['U']['observed']
```

    False

```
#export
def show_graph(g:nx.Graph, kind:str='spectral'):
    try:
        layout = getattr(nx, f'{kind}_layout')(g)
    except AttributeError as ae:
        raise AttributeError(f'No nx.{kind}_layout found')
    nx.draw(g, layout=layout, with_labels=True)

def view_graph(self, kind:str='spectral'):
    show_graph(self.g, kind=kind)

CausalGraph.view_graph = view_graph
```

```
cg.view_graph()
```

```
#export
def get_ancestors(self, node:str, g:nx.DiGraph=None,
                  parents_only:bool=False):
    if parents_only:
        f = self.g if g is None else g
        return f.predecessors(node)
    return nx.ancestors(self.g if g is None else g, node)

CausalGraph.get_ancestors = get_ancestors
```

```
cg.get_ancestors('V0')
```

    {'U', 'W0'}

```
#hide
assert cg_ref.get_ancestors('V0') == cg.get_ancestors('V0')
```

```
#export
def cut_edges(self, edges_to_cut:List[tuple]=None):
    if edges_to_cut is None: return None
    g_cut = self.g.copy()
    g_cut.remove_edges_from(edges_to_cut)
    return g_cut

CausalGraph.cut_edges = cut_edges
```

```
g_cut = cg.cut_edges([('U','Y'), ('W0', 'V1')])
show_graph(g_cut)
```

```
#hide
assert cg.cut_edges() is None
```

```
#export
def get_causes(self, nodes:List[str],
               edges_to_cut:List[tuple]=None):
    g_cut = self.cut_edges(edges_to_cut)
    causes = set()
    for node in nodes:
        causes.update(self.get_ancestors(node, g_cut))
    return causes

CausalGraph.get_causes = get_causes
```

```
#hide
assert {'X0', 'U', 'W0', 'V0', 'V1'} == cg.get_causes(['Y'])
assert {'U', 'W0'} == cg.get_causes(['V0'])
assert cg_ref.get_causes(['Y']) == cg.get_causes(['Y'])
assert cg_ref.get_causes(['V0']) == cg.get_causes(['V0'])
```

```
cg.get_causes(['V0'])
```

    {'U', 'W0'}

```
#export
def get_instruments(self, treatments:List[str], outcome:str):
    treatment_parents_edges = set()
    treatment_parents = set()
    for treatment in treatments:
        parents = self.get_ancestors(treatment, parents_only=True)
        treatment_parents.update(parents)
        treatment_parents_edges.update([(parent, treatment)
                               for parent in parents])

    g_cut = self.cut_edges(treatment_parents_edges)

    outcome_ancestors = self.get_ancestors(outcome, g_cut)

    instruments_candidates = treatment_parents.difference(outcome_ancestors)

    descendants = set()
    for parent in outcome_ancestors:
        descendants.update(nx.descendants(g_cut, parent))
    instruments = instruments_candidates.difference(descendants)
    return instruments

CausalGraph.get_instruments = get_instruments
```

```
cg.get_instruments(treatments, outcome)
```

```
#hide
assert set(cg_ref.get_instruments(treatments, [outcome])) == set(cg.get_instruments(treatments, outcome))
```

```
#export
def get_effect_modifiers(self, treatments:List[str], outcomes:List[str]):
    modifiers = set()
    for outcome in outcomes:
        modifiers.update(self.get_ancestors(outcome))
    modifiers = modifiers.difference(treatments)
    for treatment in treatments:
        modifiers = modifiers.difference(self.get_ancestors(treatment))
    return list(modifiers)

CausalGraph.get_effect_modifiers = get_effect_modifiers
```

```
cg.get_effect_modifiers(treatments, [outcome])
```

    ['X0']

```
#hide
assert set(cg_ref.get_effect_modifiers(treatments, [outcome])) == set(cg.get_effect_modifiers(treatments, [outcome])), f'{effect_modifiers} != {cg.get_effect_modifiers(treatments, [outcome])}'
```

```
#export
class CausalModel:

    def __init__(self, treatments:List[str], outcome:str='Y',
                 common_causes:List[str]=None, effect_modifiers:List[str]=None,
                 instruments:List[str]=None, causal_graph_kwargs=None):
        if not causal_graph_kwargs: causal_graph_kwargs = dict()
        self.cg = CausalGraph(treatments, outcome,
                              common_causes=common_causes,
                              effect_modifiers=effect_modifiers,
                              instruments=instruments,
                              **causal_graph_kwargs)
        self.treatments = treatments
        self.outcome = outcome
        self.common_causes = common_causes
        self.effect_modifiers = effect_modifiers
        self.instruments = instruments

    def identify_effect(self):
        pass
    def estimate_effect(self):
        pass
    def refute_estimate(self):
        pass
```

```
treatments = ['V0',]  # 'V1']
outcome = 'Y'
common_causes = ['W0']
effect_modifiers = ['X0']
instruments = []
observed_nodes = treatments + [outcome] + instruments + effect_modifiers
add_unobserved_confounder = True
missing_nodes_as_confounders = True

cg_kwargs = dict(
    missing_nodes_as_confounders=missing_nodes_as_confounders,
    add_unobserved_confounder=add_unobserved_confounder,
    observed_nodes=observed_nodes
)
```

```
cm = CausalModel(treatments=treatments, outcome=outcome, common_causes=common_causes,
                 effect_modifiers=effect_modifiers,
                 causal_graph_kwargs=cg_kwargs)
```

```
#export
def identify_effect(self, estimand_type:str='nonparametric-ate'):
    causes = {
        'treatments': self.cg.get_causes(self.treatments),
        'effects': self.cg.get_causes([self.outcome],
                                      edges_to_cut=[(t, self.outcome) for t in self.treatments])
    }
    print(f'causes: {causes}')
    common_causes = causes['treatments'].intersection(causes['effects'])
    print(f'common causes: {common_causes}')

    instruments = self.cg.get_instruments(self.treatments, self.outcome)

    # constructing backdoor estimand
    backdoor = self.construct_backdoor(self.treatments, self.outcome,
                                       common_causes, estimand_type=estimand_type)
    print('Backdoor:', backdoor)

    # constructing instrumental variable estimand
    instrumental_variable = None
    if len(instruments) > 0:
        instrumental_variable = self.construct_instrumental_variable(treatments, outcome, instruments,
                                                                 estimand_type=estimand_type)

    print('Instrumental variable:', instrumental_variable)
    return {
        'observed_common_causes': common_causes,
        'backdoor': backdoor,
        'instrumental_variable': instrumental_variable
    }

def construct_backdoor(self, treatments:List[str], outcome:str,
                       common_causes:List[str],
                       estimand_type:str='nonparametric-ate'):

    if estimand_type != 'nonparametric-ate': raise NotImplementedError

    # treatment variables
    sym_treatments = sp.Array([sp.Symbol(treatment) for treatment in treatments])

    # outcome given common causes
    expr = f'{outcome} | {",".join(common_causes)}' \
        if len(common_causes) > 0 else outcome

    # assigning a normal distribution to the outcome given common causes
    sym_mu = sp.Symbol("mu")
    sym_sigma = sp.Symbol("sigma", positive=True)
    sym_outcome = sp.stats.Normal(expr, sym_mu, sym_sigma)
    # expected outcome given common causes
    sym_conditional_outcome = sp.stats.Expectation(sym_outcome)

    # effect of treatment on outcome given common causes
    sym_effect = sp.Derivative(sym_conditional_outcome, sym_treatments)
    return sym_effect

def construct_instrumental_variable(self, treatments:List[str], outcome:str,
                                    instruments:List[str], estimand_type:str='nonparametric-ate'):

    if estimand_type != 'nonparametric-ate': raise NotImplementedError

    sym_mu, sym_sigma = 0, 1
    sym_outcome = sp.stats.Normal(outcome, sym_mu, sym_sigma)
    sym_treatments = sp.Array([sp.Symbol(sp.stats.Normal(treatment, sym_mu, sym_sigma))
                              for treatment in treatments])
    sym_instruments = sp.Array([sp.Symbol(instrument)
                                for instrument in instruments])
    sym_effect = sp.stats.Expectation(
        sp.Derivative(sym_outcome, sym_instruments)
        /
        sp.Derivative(sym_treatments, sym_instruments)
    )

    return sym_effect

CausalModel.construct_backdoor = construct_backdoor
CausalModel.construct_instrumental_variable = construct_instrumental_variable
CausalModel.identify_effect = identify_effect
```

```
estimands = cm.identify_effect(); estimands
```

    causes: {'treatments': {'U', 'W0'}, 'effects': {'U', 'X0', 'W0'}}
    common causes: {'U', 'W0'}
    Backdoor: Derivative(Expectation(Y | U,W0), [V0])
    Instrumental variable: None

    {'observed_common_causes': {'U', 'W0'},
     'backdoor': Derivative(Expectation(Y | U,W0), [V0]),
     'instrumental_variable': None}

Regression estimators based on sklearn regression classes

```
isinstance(linear_model.LinearRegression(), sklearn.base.RegressorMixin)
```

    True

```
#export
class RegressionEstimator:

    def __init__(self, model:sklearn.base.RegressorMixin):
        assert isinstance(model, sklearn.base.RegressorMixin)
        self.m = model

    def fit(self, X:np.ndarray, y:np.ndarray, ix:int, ix_confounders:List[int], reset:bool=True):
        if not isinstance(ix_confounders, list):
            ix_confounders = list(ix_confounders)
        self.ix = ix
        self.ix_confounders = ix_confounders
        _ix = [ix] + ix_confounders
        self._ix = _ix
        if reset:
            self.m.fit(X[:,self._ix],y)

    def estimate_effect(self, X:np.ndarray, treatment:Union[int, float], control:Union[int, float],
                        y:np.ndarray=None):
        n, _ = X.shape
        _X = X.copy()
        _X[:, self.ix] = treatment

        treatment_outcomes = self.m.predict(_X[:, self._ix])
        _X[:, self.ix] = control
        control_outcomes = self.m.predict(_X[:, self._ix])

        treatment_mean = treatment_outcomes.mean()
        control_mean = control_outcomes.mean()
        ate = treatment_mean - control_mean

        return ate
```

Sanity checking on a quadratic polynomial toy dataset

```
X = np.linspace(-1, 1, 200)
X = np.array([X**2, X, np.ones(len(X))*.5]).T
w = np.array([2, 0, .5])
y = X @ w

fig, ax = plt.subplots(figsize=(8,4), constrained_layout=True)
ax.scatter(X[:,-2], y)
ax.set(xlabel='x', ylabel='y', title='dataset')
plt.show()
```

```
regression_model = linear_model.LinearRegression()
estimator = RegressionEstimator(regression_model)
estimator.fit(X, y, ix=0, ix_confounders=[1])
ate = estimator.estimate_effect(X=X, treatment=1, control=0)
print(f'ate = {ate:.3f} coefs {estimator.m.coef_}')
```

    ate = 2.000 coefs [ 2.00000000e+00 -1.27928927e-17]

```
#hide
assert np.isclose(ate, w[0], atol=.1)
```

Classification estimator

propensity score: common causes -> prediction of treatment (class) -> grouping by score to select pairs of most similar treatment and control group samples to compute the difference in outcome

grouping is done using some nearest neighbour search:
- ATC if nearest neighbor is set up with the **treated** group and for each **control** group sample a match is looked up and then the difference of the outcome is computed
- ATT if nearest neighbor is set up with the **control** group and for each **treated** group sample a match is looked up and then the difference of the outcome is computed

**TODO**: test `PropensityScoreMatcher` on data generated using `bcg.basics` classes

```
n = 200
x_treatment = np.random.choice([True, False], p=[.5, .5], size=n)
x_common_causes = np.array([
    [np.random.normal(loc=v, scale=.1) for v in x_treatment],
    [np.random.normal(loc=10-v, scale=.1) for v in x_treatment],
])

y_outcome = np.array([np.random.normal(loc=v, scale=.1) for v in x_treatment])

fig, axs = plt.subplots(figsize=(8,6), nrows=4, constrained_layout=True)
axs[0].hist(x_treatment.astype(float))
axs[0].set(xlabel='treatment')
axs[1].hist(x_common_causes[0])
axs[1].set(xlabel='cc0')
axs[2].hist(x_common_causes[1])
axs[2].set(xlabel='cc1')
axs[3].hist(y_outcome)
axs[3].set(xlabel='outcome')
plt.show()

fig, ax = plt.subplots(figsize=(8,4), constrained_layout=True)
ax.scatter(x_treatment, y_outcome)
ax.set(xlabel='treatment', ylabel='outcome', title='dataset')
plt.show()

X, y = np.concatenate((x_treatment[:,None], x_common_causes.T), axis=1), y_outcome
X.shape, y.shape
```

```
class PropensityScoreMatcher:

    def __init__(self, propensity_model:sklearn.base.ClassifierMixin):
        assert isinstance(propensity_model, sklearn.base.ClassifierMixin)
        self.pm = propensity_model

    def fit(self, X:np.ndarray, y:np.ndarray, ix:int, ix_confounders:List[int], reset:bool=True):
        '''building the classifier model & nearest neigbhor search thingy

        ix: needs to point to a binary variable
        '''
        if not isinstance(ix_confounders, list):
            ix_confounders = list(ix_confounders)
        self.ix = ix
        self.ix_confounders = ix_confounders

        _ix = [ix] + ix_confounders
        self._ix = _ix
        if reset:
            self.pm.fit(X[:, self.ix_confounders], X[:,self.ix])

    def estimate_effect(self, X:np.ndarray, treatment:Union[int, bool], control:Union[int, bool],
                        y:np.ndarray=None, kind:str='ate'):
        assert y is not None, 'Cannot be None. That\'s just the default to have consistent method parameters.'
        assert kind in ['ate', 'att', 'atc']
        propensity_score = self.pm.predict(X[:, self.ix_confounders])
        ix_treat, ix_control = X[:,self.ix] == treatment, X[:,self.ix] == control

        X_treat, X_cont = X[ix_treat,:], X[ix_control,:]
        y_treat, y_cont = y[ix_treat], y[ix_control]

        searcher = neighbors.NearestNeighbors(n_neighbors=1)

        def get_att():
            searcher.fit(propensity_score[ix_control][:,None])
            distances, indices = searcher.kneighbors(propensity_score[ix_treat][:,None])

            att = 0
            n_treat = ix_treat.sum()
            for i in range(n_treat):
                out_treat = y_treat[i]
                out_cont = y_cont[indices[i][0]]
                att += out_treat - out_cont
            return att / n_treat

        def get_atc():
            searcher.fit(propensity_score[ix_treat][:,None])
            distances, indices = searcher.kneighbors(propensity_score[ix_control][:,None])

            atc = 0
            n_cont = ix_control.sum()
            for i in range(n_cont):
                out_treat = y_treat[indices[i][0]]
                out_cont = y_cont[i]
                atc += out_treat - out_cont
            return atc / n_cont

        def get_ate():
            n_treat = ix_treat.sum()
            n_cont = ix_control.sum()
            att = get_att()
            atc = get_atc()
            return (att*n_treat + atc*n_cont) / (n_treat + n_cont)

        if kind == 'ate':
            return get_ate()
        elif kind == 'att':
            return get_att()
        elif kind == 'atc':
            return get_atc()
        else:
            raise NotImplementedError
```

```
propensity_model = linear_model.LogisticRegression(solver='lbfgs')
estimator = PropensityScoreMatcher(propensity_model)
estimator.fit(X, y, ix=0, ix_confounders=[1, 2])
ate = estimator.estimate_effect(X=X, treatment=True, control=False, y=y)
print(f'ate = {ate:.3f}')
```

    ate = 0.934

Generating data for the graphical model using `bcg.basics` functions

```
outcome_is_binary = True
treatment_is_binary = True

n = 333
n_common_causes = len(common_causes)
n_instruments = len(instruments)
n_eff_mods = len(effect_modifiers)
n_treatments = len(treatments)
beta = 1  # max random value

cc = CommonCauses.get_obs(n, n_common_causes)
ins = Instruments.get_obs(n, n_instruments)
em = EffectModifiers.get_obs(n, n_eff_mods)
treat = Treatments.get_obs(n, n_treatments, cc, ins, beta, treatment_is_binary=treatment_is_binary)
out = Outcomes.get_obs(treat, cc, em, outcome_is_binary=outcome_is_binary)
obs = pd.concat((treat.obs, cc.obs, em.obs, ins.obs, out.obs), axis=1)
X, y, not_target = get_Xy(obs, target=outcome)
```

```
obs.head(), obs.tail()
```

    (   V0        W0        X0  Y
     0   1  1.176682  0.468362  0
     1   0 -0.581795 -0.010592  1
     2   0  0.413044  0.275547  1
     3   0 -0.741007 -2.044762  1
     4   1 -0.167759 -2.053056  0,
          V0        W0        X0  Y
     328   0  0.937082  0.408209  0
     329   0 -0.020501  0.657049  0
     330   1  1.063346 -0.548377  0
     331   1  1.241344 -0.537261  1
     332   0  0.667529  0.745335  0)

```
not_target.index('V0')
```

Adding effect estimate functionality to `CausalModel`

Changing the implementation of `get_Xy`, incorporating products with effect modifiers, based on lns 59-71 in `causal_estimators/linear_regression_estimator.py` with the new argument`feature_product_groups`. The variable is supposed to consist of two lists, each containing features in `obs`, of which all products will be computed.

```
#export
def get_Xy_with_products(obs:pd.DataFrame, target:str='Y', feature_product_groups:List[list]=None):
    'feaure_product_groups (e.g. [["V0", "V1", "W0"], ["X0", "X1"]]) to compute products between each var in the first and second list (not within each list)'
    not_target = [c for c in obs.columns if c != target and c not in feature_product_groups[1]]
#     out_cols = [col for col in obs.columns if col != target ]

    X, y = obs.loc[:, not_target].values, \
           obs.loc[:, target].values.ravel()
    if feature_product_groups:
        assert isinstance(feature_product_groups, list)
        assert len(feature_product_groups) == 2
        assert all([isinstance(f, list) for f in feature_product_groups])
        product_cols = [(t,e) for t,e in itertools.product(*feature_product_groups)]
        cols = list(obs.columns.values)
        for t, e in product_cols:
            ix_t = cols.index(t)
            ix_e = cols.index(e)
            x = (obs[t] * obs[e]).values
            X = np.concatenate((X, x[:,None]), axis=1)
            not_target.append(f'{t}_{e}')
    return X, y, not_target
```

```
get_Xy_with_products(obs, target=outcome, feature_product_groups=[treatments, effect_modifiers])
```

    (array([[ 1.        ,  1.17668216,  0.46836244],
            [ 0.        , -0.58179495, -0.        ],
            [ 0.        ,  0.41304439,  0.        ],
            [ 0.        , -0.7410068 , -0.        ],
            [ 1.        , -0.1677586 , -2.05305636],
            [ 0.        ,  0.01860169,  0.        ],
            [ 0.        ,  0.43848953,  0.        ],
            [ 1.        , -0.87806002, -0.03161624],
            [ 0.        ,  0.69264442,  0.        ],
            [ 0.        , -0.19594743, -0.        ],
            [ 1.        ,  1.00767243,  0.62229535],
            [ 0.        ,  2.50225413, -0.        ],
            [ 1.        , -1.5025282 , -1.04862138],
            [ 0.        ,  0.59197037, -0.        ],
            [ 0.        ,  0.81509076, -0.        ],
            [ 1.        ,  0.82414504,  0.60770411],
            [ 0.        ,  0.47207575, -0.        ],
            [ 1.        ,  0.22302966, -1.34418616],
            [ 0.        ,  0.84251715, -0.        ],
            [ 1.        , -0.88151327,  0.24835875],
            [ 1.        , -0.30735475, -0.04794243],
            [ 0.        , -0.58668586,  0.        ],
            [ 1.        ,  0.27842421, -1.26786009],
            [ 1.        , -1.38543832, -0.21981882],
            [ 0.        ,  0.04636712,  0.        ],
            [ 1.        , -0.27225662, -1.49817642],
            [ 0.        ,  0.69317737, -0.        ],
            [ 0.        ,  0.55967209,  0.        ],
            [ 0.        , -0.28012082,  0.        ],
            [ 0.        ,  0.83954801, -0.        ],
            [ 1.        ,  1.17486959, -0.19711913],
            [ 0.        , -0.42327682, -0.        ],
            [ 1.        ,  1.19543944, -0.63228272],
            [ 1.        , -0.5933412 , -1.49673812],
            [ 0.        , -0.33224539,  0.        ],
            [ 1.        , -0.48398116, -0.14450943],
            [ 1.        ,  0.35354921,  0.03223502],
            [ 0.        ,  0.78950319,  0.        ],
            [ 1.        ,  0.27726173, -0.90122988],
            [ 0.        , -1.02464195,  0.        ],
            [ 1.        , -0.11925939,  0.1559452 ],
            [ 1.        ,  2.6178189 , -0.05506116],
            [ 0.        ,  1.0751829 , -0.        ],
            [ 1.        ,  1.74349409, -1.47659248],
            [ 0.        , -0.11530499,  0.        ],
            [ 1.        ,  1.15353089, -2.97294368],
            [ 1.        , -0.20033576, -1.74212625],
            [ 0.        , -1.38846114,  0.        ],
            [ 1.        , -0.36144888,  0.67152466],
            [ 1.        ,  0.66968925,  0.61413237],
            [ 0.        ,  0.16281224, -0.        ],
            [ 0.        ,  0.21976562, -0.        ],
            [ 1.        ,  1.53845996,  0.54539873],
            [ 0.        , -0.91126766, -0.        ],
            [ 1.        , -0.68694697, -0.67405784],
            [ 1.        , -1.18724641,  0.96144145],
            [ 1.        ,  0.10311262,  0.75524058],
            [ 0.        ,  0.29532879, -0.        ],
            [ 0.        ,  0.540325  ,  0.        ],
            [ 0.        , -2.04097777, -0.        ],
            [ 0.        ,  1.82561622, -0.        ],
            [ 0.        ,  0.09735642,  0.        ],
            [ 0.        , -0.6646908 , -0.        ],
            [ 1.        ,  1.12239077,  0.09404086],
            [ 1.        , -0.12017347,  0.35420544],
            [ 1.        , -0.65994112, -0.62236337],
            [ 0.        ,  1.27624662, -0.        ],
            [ 1.        , -1.51983953, -0.37870379],
            [ 1.        , -0.40253108,  1.22736594],
            [ 0.        ,  1.97445598,  0.        ],
            [ 0.        ,  0.19682415, -0.        ],
            [ 0.        ,  0.2096079 , -0.        ],
            [ 1.        , -0.82997588, -0.30663071],
            [ 0.        , -1.28976469, -0.        ],
            [ 1.        ,  1.42382724, -0.40233017],
            [ 1.        , -1.14401892,  0.55393806],
            [ 0.        , -0.93563609, -0.        ],
            [ 1.        , -0.66266796,  0.01546187],
            [ 1.        , -2.18297248,  0.8571488 ],
            [ 1.        ,  0.61298124,  0.3419954 ],
            [ 0.        ,  0.69951473, -0.        ],
            [ 1.        , -0.49642078, -0.61186666],
            [ 1.        ,  0.58792823, -0.72918475],
            [ 1.        ,  0.33747147, -0.68207888],
            [ 0.        ,  0.7830838 , -0.        ],
            [ 1.        , -0.19651434,  0.41325202],
            [ 0.        , -0.82630505, -0.        ],
            [ 0.        , -1.08308539, -0.        ],
            [ 1.        ,  1.51949011, -0.5132064 ],
            [ 0.        , -0.55609254, -0.        ],
            [ 0.        , -0.55020446, -0.        ],
            [ 1.        , -0.23046882, -1.18978961],
            [ 1.        , -0.83335204, -0.51365536],
            [ 0.        , -0.75398382, -0.        ],
            [ 0.        ,  1.0331082 , -0.        ],
            [ 0.        , -1.90836353, -0.        ],
            [ 1.        , -0.24204649, -1.16788193],
            [ 0.        , -0.48366006, -0.        ],
            [ 1.        ,  1.9609003 , -0.10092163],
            [ 1.        , -0.10930367,  0.19715841],
            [ 1.        , -0.91681464, -1.6097447 ],
            [ 0.        , -1.96662707, -0.        ],
            [ 0.        ,  1.56986699,  0.        ],
            [ 1.        , -0.01387704,  0.65497536],
            [ 0.        , -0.68650505, -0.        ],
            [ 0.        ,  0.21987463, -0.        ],
            [ 0.        , -1.30508432, -0.        ],
            [ 1.        ,  1.61626745, -1.02732012],
            [ 1.        , -0.31633112, -1.61368864],
            [ 1.        , -0.83786904,  1.04923957],
            [ 0.        ,  2.02425014, -0.        ],
            [ 0.        , -1.18013087,  0.        ],
            [ 1.        , -0.53768099,  0.35227688],
            [ 0.        , -0.73424031,  0.        ],
            [ 1.        ,  0.87101132,  0.22893354],
            [ 0.        ,  0.37984918, -0.        ],
            [ 0.        ,  0.66534242, -0.        ],
            [ 1.        , -1.64420658, -1.95857656],
            [ 1.        ,  0.1383424 , -1.55802608],
            [ 0.        , -0.74683721, -0.        ],
            [ 1.        , -0.43560403,  1.94214118],
            [ 1.        , -1.06956346, -0.5789204 ],
            [ 1.        ,  1.99088147, -0.14986403],
            [ 0.        , -0.12728962, -0.        ],
            [ 0.        ,  1.99533371, -0.        ],
            [ 1.        ,  0.1353928 , -1.37197832],
            [ 0.        ,  1.37546801,  0.        ],
            [ 0.        , -1.08458817,  0.        ],
            [ 0.        ,  0.25768835,  0.        ],
            [ 1.        , -1.97007802, -0.17201895],
            [ 0.        ,  0.88053984,  0.        ],
            [ 1.        ,  0.96000679, -0.33383015],
            [ 0.        , -0.02423215,  0.        ],
            [ 0.        ,  1.62918072, -0.        ],
            [ 0.        , -0.35953656, -0.        ],
            [ 1.        , -3.33968888, -0.1151471 ],
            [ 1.        , -1.22286802, -1.08098558],
            [ 1.        , -0.27731606, -1.75435442],
            [ 1.        , -0.14299429, -1.4113362 ],
            [ 0.        , -0.79238418,  0.        ],
            [ 0.        ,  0.15492188, -0.        ],
            [ 1.        ,  0.24196779, -1.23409943],
            [ 1.        ,  0.50064661, -0.11293217],
            [ 1.        , -0.37903674, -1.85947872],
            [ 0.        , -2.12054598, -0.        ],
            [ 0.        , -2.69900803, -0.        ],
            [ 1.        , -1.04118005, -0.87282059],
            [ 1.        , -1.26530384,  0.03916189],
            [ 1.        , -0.08328762, -0.93456518],
            [ 1.        , -0.57943718,  0.64581655],
            [ 1.        ,  0.45377051, -1.2342177 ],
            [ 1.        ,  0.77285238,  0.21019265],
            [ 1.        ,  1.99209498, -1.75214351],
            [ 0.        , -1.5709821 , -0.        ],
            [ 1.        ,  0.57874377, -0.49886028],
            [ 1.        ,  0.52104172, -1.6131055 ],
            [ 0.        , -1.29674892, -0.        ],
            [ 1.        ,  0.25799875, -0.28173773],
            [ 1.        ,  0.09164494, -0.93814027],
            [ 0.        , -0.19604978,  0.        ],
            [ 1.        ,  2.40007948,  0.13682563],
            [ 0.        , -0.31576543, -0.        ],
            [ 1.        ,  1.27486599,  1.58551915],
            [ 0.        , -0.0988648 , -0.        ],
            [ 0.        ,  0.57158242, -0.        ],
            [ 0.        , -0.46094401,  0.        ],
            [ 0.        ,  0.2513693 ,  0.        ],
            [ 0.        , -1.19324532, -0.        ],
            [ 0.        ,  0.16327776, -0.        ],
            [ 0.        , -0.55345049, -0.        ],
            [ 0.        ,  0.88697158,  0.        ],
            [ 1.        ,  0.31997452, -1.81553901],
            [ 1.        ,  1.20021945, -1.44244412],
            [ 1.        ,  1.47430806, -0.29657584],
            [ 1.        ,  2.2577756 , -0.47501888],
            [ 0.        , -1.95906451, -0.        ],
            [ 1.        , -0.33739542,  1.47269935],
            [ 1.        ,  0.11656465,  0.54634721],
            [ 1.        ,  0.56925007,  2.19265007],
            [ 0.        , -0.84636977,  0.        ],
            [ 1.        , -0.20755873,  0.52894867],
            [ 1.        ,  0.86748963,  0.25236621],
            [ 1.        ,  0.77730636,  0.88968126],
            [ 0.        ,  0.83362934, -0.        ],
            [ 0.        ,  1.2010197 , -0.        ],
            [ 0.        , -1.78993312,  0.        ],
            [ 0.        , -0.88537341, -0.        ],
            [ 1.        ,  0.6281433 ,  0.52400025],
            [ 1.        ,  0.1306515 , -1.51775631],
            [ 0.        , -1.11019716, -0.        ],
            [ 1.        , -0.1040678 ,  0.81633243],
            [ 0.        , -2.2783055 ,  0.        ],
            [ 0.        ,  0.32917672, -0.        ],
            [ 0.        ,  0.30402605, -0.        ],
            [ 1.        ,  0.65239543, -0.03703682],
            [ 0.        , -0.70969102, -0.        ],
            [ 1.        ,  0.76836166,  1.09724708],
            [ 0.        , -0.95307736, -0.        ],
            [ 1.        , -0.60126687, -0.68932624],
            [ 1.        ,  0.2611847 , -0.51854676],
            [ 1.        ,  0.86086798, -0.99311398],
            [ 0.        , -0.60212345, -0.        ],
            [ 0.        , -0.21489864, -0.        ],
            [ 1.        ,  0.16784098, -0.96451859],
            [ 1.        ,  0.99933976, -0.92318495],
            [ 1.        , -0.77836613, -0.27311638],
            [ 1.        , -0.58136406,  1.129748  ],
            [ 0.        , -0.99482575, -0.        ],
            [ 0.        , -2.26833038,  0.        ],
            [ 0.        , -0.49140647,  0.        ],
            [ 0.        ,  0.3417349 ,  0.        ],
            [ 0.        ,  0.81352755,  0.        ],
            [ 1.        ,  1.04808466, -1.95391987],
            [ 0.        ,  0.03739659, -0.        ],
            [ 1.        ,  1.90839777, -0.97396579],
            [ 1.        ,  0.35613674, -1.19637825],
            [ 1.        , -1.54520658, -0.47753458],
            [ 1.        ,  0.38647342,  0.31642644],
            [ 1.        , -0.26497912,  1.07614905],
            [ 0.        , -1.18843513, -0.        ],
            [ 0.        , -0.18649098, -0.        ],
            [ 1.        , -0.95641689, -0.53847434],
            [ 0.        , -0.62785403, -0.        ],
            [ 0.        ,  0.54499987,  0.        ],
            [ 1.        , -0.3661312 , -1.00765035],
            [ 0.        , -1.84270122, -0.        ],
            [ 1.        , -1.33943218, -0.6999864 ],
            [ 1.        , -0.1482295 , -0.50383975],
            [ 1.        , -0.30679679, -1.6769    ],
            [ 0.        ,  0.6370456 , -0.        ],
            [ 1.        ,  1.31814802, -0.06917642],
            [ 1.        ,  1.94500489,  0.707354  ],
            [ 0.        , -1.82858479, -0.        ],
            [ 0.        ,  1.91970659, -0.        ],
            [ 0.        ,  0.19284009,  0.        ],
            [ 1.        ,  0.60358981, -1.69540105],
            [ 0.        , -0.65389405,  0.        ],
            [ 0.        , -0.99792959,  0.        ],
            [ 0.        , -0.61591353,  0.        ],
            [ 0.        ,  0.9736039 ,  0.        ],
            [ 0.        , -2.2061269 , -0.        ],
            [ 0.        , -1.18717205, -0.        ],
            [ 0.        , -2.96283253,  0.        ],
            [ 0.        , -0.16640058,  0.        ],
            [ 1.        , -0.80142357,  0.80451183],
            [ 0.        ,  0.7034448 ,  0.        ],
            [ 1.        , -0.65441058, -1.11367468],
            [ 0.        ,  1.9866488 ,  0.        ],
            [ 0.        , -0.12074271,  0.        ],
            [ 0.        ,  0.85874995,  0.        ],
            [ 0.        , -0.23471034, -0.        ],
            [ 1.        , -0.69421745, -0.37865228],
            [ 0.        , -1.29753069, -0.        ],
            [ 1.        , -0.12132465, -2.64655954],
            [ 0.        ,  0.5908582 , -0.        ],
            [ 0.        , -0.13259458, -0.        ],
            [ 1.        ,  0.12074633, -2.19508328],
            [ 0.        ,  0.98194261, -0.        ],
            [ 0.        ,  0.76501008,  0.        ],
            [ 0.        ,  0.47128088,  0.        ],
            [ 0.        , -1.47328253, -0.        ],
            [ 1.        , -1.91590797, -0.96676211],
            [ 0.        , -0.46992243, -0.        ],
            [ 1.        , -0.54882431,  0.21779183],
            [ 1.        , -0.58274857, -2.89552988],
            [ 1.        ,  0.50685665, -1.53525251],
            [ 0.        , -0.80810694, -0.        ],
            [ 1.        , -0.22109691, -0.85674545],
            [ 0.        ,  0.57368129,  0.        ],
            [ 0.        , -0.86655538,  0.        ],
            [ 0.        ,  1.28184488,  0.        ],
            [ 0.        , -0.40320565,  0.        ],
            [ 0.        ,  0.12229881,  0.        ],
            [ 0.        ,  0.53200215, -0.        ],
            [ 0.        , -1.39066443, -0.        ],
            [ 0.        , -0.08015956, -0.        ],
            [ 1.        , -0.18230888, -0.2610354 ],
            [ 0.        ,  0.13218753, -0.        ],
            [ 1.        , -0.48240024,  0.28261661],
            [ 0.        ,  0.58114232, -0.        ],
            [ 1.        ,  0.24950259,  0.34883739],
            [ 0.        ,  0.23214457,  0.        ],
            [ 1.        ,  0.4265521 , -0.8793497 ],
            [ 1.        , -0.09835993, -0.23177168],
            [ 1.        ,  0.31605596, -0.27188055],
            [ 0.        ,  0.49403607, -0.        ],
            [ 1.        ,  0.03023592, -0.56611124],
            [ 0.        , -2.16333854,  0.        ],
            [ 1.        , -0.56309567,  1.3852228 ],
            [ 1.        , -0.11057424, -0.7816743 ],
            [ 0.        ,  0.92326045,  0.        ],
            [ 1.        , -1.57325458, -2.17917003],
            [ 0.        ,  0.51388745, -0.        ],
            [ 1.        , -1.33023702, -0.98718309],
            [ 1.        , -0.00646919, -1.45698329],
            [ 1.        , -0.68117076,  0.11792957],
            [ 0.        , -0.50013649, -0.        ],
            [ 1.        , -2.29327951, -1.69190102],
            [ 0.        , -0.10177061, -0.        ],
            [ 1.        ,  0.33030482, -0.99170174],
            [ 1.        , -1.20297097, -0.40180826],
            [ 0.        , -0.42249472, -0.        ],
            [ 1.        ,  0.74796245, -0.40187473],
            [ 0.        ,  0.29643631,  0.        ],
            [ 1.        , -0.9387942 ,  0.0142777 ],
            [ 0.        , -1.39634321,  0.        ],
            [ 1.        ,  0.74506433, -1.77284876],
            [ 1.        , -0.22198416,  1.61809186],
            [ 0.        , -0.60407254, -0.        ],
            [ 1.        , -0.43769244, -0.98667038],
            [ 1.        , -0.65880241, -0.65922251],
            [ 1.        ,  1.19145053, -1.51247836],
            [ 1.        , -2.42059679, -0.83186204],
            [ 1.        , -0.89557387, -0.096262  ],
            [ 0.        , -0.17293205,  0.        ],
            [ 0.        , -0.32774982,  0.        ],
            [ 0.        , -0.28746924,  0.        ],
            [ 1.        , -0.67139085,  1.60276736],
            [ 0.        ,  0.0357559 , -0.        ],
            [ 0.        ,  0.07941142,  0.        ],
            [ 0.        ,  0.24710014, -0.        ],
            [ 1.        , -0.79006552, -0.27965665],
            [ 0.        ,  0.21546596, -0.        ],
            [ 0.        , -1.11276003, -0.        ],
            [ 1.        ,  0.20444365, -1.07183991],
            [ 1.        , -0.15059638, -2.32557143],
            [ 0.        ,  0.45714317, -0.        ],
            [ 0.        ,  1.27504927, -0.        ],
            [ 0.        ,  0.93708227,  0.        ],
            [ 0.        , -0.02050105,  0.        ],
            [ 1.        ,  1.06334631, -0.54837674],
            [ 1.        ,  1.24134447, -0.53726089],
            [ 0.        ,  0.66752857,  0.        ]]),
     array([0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1,
            1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1,
            1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1,
            0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0,
            1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1,
            0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1,
            1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0,
            1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0,
            1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1,
            0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1,
            0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1,
            1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1,
            1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1,
            1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1,
            1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0,
            0, 1, 0]),
     ['V0', 'W0', 'V0_X0'])

```
#hide
X, y, not_target = get_Xy_with_products(obs, target=outcome, feature_product_groups=[treatments, effect_modifiers])
assert y.shape[0] == X.shape[0]
assert X.shape[1] == len(not_target)
n_not_target = len(not_target)
rest_cols = [v for v in obs.columns.values if v not in effect_modifiers and v != outcome]
n_not_target_ideal = len(treatments) * len(effect_modifiers) + len(rest_cols)
assert len(not_target) == n_not_target_ideal, f'{n_not_target} != {n_not_target_ideal}: not_target = {not_target}'
```

```
#export
def estimate_effect(self, estimands:dict, control_value:float,
                    treatment_name:str, treatment_value:float,
                    obs:pd.DataFrame, outcome:str='Y',
                    causal_method:str='backdoor',
                    model:Union[sklearn.base.RegressorMixin,sklearn.base.ClassifierMixin]=None, target_unit:str='ate',
                    effect_modifiers:List[str]=None,
                    supervised_type_is_regression:bool=True):
    assert causal_method in {'backdoor', 'instrumental_variable'}
    assert target_unit == 'ate'
    print('model', model)
    if model is None:
        if supervised_type_is_regression:
            model = linear_model.LinearRegression()
        else:
            model = linear_model.LogisticRegression(solver='lbfgs')

    if effect_modifiers is None:
        effect_modifiers = self.effect_modifiers

    # decide on approach given causal_method and model_type

    # estimate the effect using the arrived on approach
    X, y, not_outcome = get_Xy_with_products(obs, target=outcome, feature_product_groups=[treatments, effect_modifiers])
    if supervised_type_is_regression:
        estimator = RegressionEstimator(model)
    else:
        estimator = PropensityScoreMatcher(model)

    ix = [v.lower() for v in not_outcome].index(treatment_name)
    confounders = self.treatments + list(estimands['observed_common_causes']) + effect_modifiers
    print('confounders', confounders)
    ix_confounders = [_i for _i,_v in enumerate(obs.columns.values) if _v in confounders]
    estimator.fit(X, y, ix, ix_confounders)
    effect = estimator.estimate_effect(X=X, treatment=treatment_value, control=control_value, y=y)
    return effect

CausalModel.estimate_effect = estimate_effect

# propensity_model = linear_model.LogisticRegression(solver='lbfgs')
# estimator = PropensityScoreMatcher(propensity_model)
# estimator.fit(X, y, ix=0, ix_confounders=[1, 2])
# ate = estimator.estimate_effect(X=X, y=y, treatment=True, control=False)
# print(f'ate = {ate:.3f}')
```

```
causal_method = 'backdoor'
control_value = 0
treatment_name = 'v0'
treatment_value = 2

effect_modifiers = effect_modifiers
target_unit = 'ate'

# model = linear_model.LinearRegression()
# model = linear_model.LogisticRegression()
model = None
supervised_type_is_regression = False

cm.estimate_effect(estimands, control_value, treatment_name, treatment_value,
                   obs, outcome=outcome, causal_method=causal_method, model=model,
                   target_unit=target_unit, effect_modifiers=effect_modifiers,
                   supervised_type_is_regression=supervised_type_is_regression)
```

```
a = np.linspace(1, 4, 5)
b = a[:, np.newaxis]; b
```

```

```


# Scenario D - Peakshape Variation (results evaluation)

This file is used to evaluate the inference (numerical) results.

The model used in the inference of the parameters is formulated as follows:

\begin{equation}
\large y = f(x) = \sum\limits_{m=1}^M \big[A_m \cdot f_{pseudo-Voigt}(x)\big] + \epsilon
\end{equation}

where:

\begin{equation}
\large f_{pseudo-Voigt}(x) = \eta \cdot \frac{\sigma_m^2}{(x-\mu_m)^2 + \sigma_m^2} + (1 - \eta) \cdot e^{-\frac{(x-\mu_m)^2}{2\cdot\sigma_m^2}}
\end{equation}

```python
%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pymc3 as pm
import arviz as az
import seaborn as sns

#az.style.use('arviz-darkgrid')

print('Running on PyMC3 v{}'.format(pm.__version__))
```

    WARNING (theano.tensor.blas): Using NumPy C-API based implementation for BLAS functions.

    Running on PyMC3 v3.8

## Import local modules

```python
import sys
sys.path.append('../../modules')
import results as res
import figures as fig
```

## Load results and extract convergence information

```python
# list of result files to load
filelst = ['./scenario_peakshape_mruns.csv']

ldf = res.load_results(filelst)
```

    reading file: ./scenario_peakshape_mruns.csv

```python
# extract the convergence results per model
labellist = [0.0, 0.25, 0.5, 0.75, 1.0]
dres = res.get_model_summary(ldf, labellist)
```

```python
# figure size and color mapping
figs=(8,8)

col = "Blues"
col_r = col + "_r"
```

## Heatmaps of n-peak model vs. n-peak number in dataset

### WAIC

```python
fig.plot_heatmap(dres['waic'], labellist, title="", color=col, fsize=figs, fname="hmap_waic", precision=".0f")
```

### Rhat

```python
fig.plot_heatmap(dres['rhat'], labellist, title="", color=col, fsize=figs, fname="hmap_rhat", precision=".2f")
```

### R2

```python
fig.plot_heatmap(dres['r2'], labellist, title="", color=col_r, fsize=figs, fname="hmap_r2", precision=".2f")
```

### BFMI

```python
fig.plot_heatmap(dres['bfmi'], labellist, title="", color=col_r, fsize=figs,
                     fname="hmap_bfmi", precision=".2f")
```

### MCSE

```python
fig.plot_heatmap(dres['mcse'], labellist, title="", color=col, fsize=figs, fname="hmap_mcse", precision=".2f")
```

### Noise

```python
fig.plot_heatmap(dres['noise'], labellist, title="", color=col, fsize=figs,
                     fname="hmap_noise", precision=".2f")
```

### ESS

```python
fig.plot_heatmap(dres['ess'], labellist, title="", color=col_r, fsize=figs, fname="hmap_ess", precision=".0f")
```

```python

```


```python
import sympy
from phasor.utilities.ipynb.displays import *
from phasor.utilities.ipynb.ipy_sympy import *
import scipy.linalg

import numpy.testing as np_test
import declarative as decl
from declarative.bunch import (
    DeepBunch,
)

#import numpy as np

from phasor import system
from phasor import readouts
from phasor import optics
from phasor.optics.nonlinear_crystal import NonlinearCrystal
from phasor.utilities.print import pprint

from phasor.optics.models.KTP_test_stand import SHGTestStandResonant

asavefig.formats.jpg.use = True
```

    Populating the interactive namespace from numpy and matplotlib
    Sympy version:  1.0

```python
db = DeepBunch()
db.test.mDC1.mirror_H1.T_hr = .1
db.test.ktp.length.val = np.linspace(0, .4, 30)
db.test.ktp.N_ode = 100
db.test.PSLR.power.val = .4
db.test.ktp.solution_order = 4
db.test.include_AC = False
sys = system.BGSystem(
    ctree = db,
)
sys.own.test = SHGTestStandResonant()
db = sys.ctree_shadow()
#print(sys.test.ktp.ctree_as_yaml())
```

```python
#print(sys.test.DC_R.DC_readout)
#print(sys.test.DC_G.DC_readout)
```

```python
axB = mplfigB(Nrows=1)
test = sys.test
axB.ax0.plot(test.ktp.length_mm.val, test.DC_R.DC_readout, color = 'red')
axB.ax0.plot(test.ktp.length_mm.val, test.DC_G.DC_readout, color = 'green')
axB.ax0.plot(test.ktp.length_mm.val, test.DC_R.DC_readout + test.DC_G.DC_readout, color = 'black')
#axB.ax0.plot(test.ktp.length_mm.val, 1 * np.tanh(.200 * test.ktp.length_mm.val)**2, ls = '--', color = 'blue')
#axB.ax0.set_xscale('log')
#axB.ax0.set_ylim(0, 1.1)
```

```python
db = DeepBunch()
db.test.ktp.length.val = .12 * 2
db.test.ktp.N_ode = 10
db.test.mDC1.mirror_H1.T_hr = .03
db.test.mDC2.mirror_H1.L_hr = .01
db.test.PSLR.power.val = np.linspace(0, .5, 30)
db.test.ktp.solution_order = 4
db.test.include_AC = True
sys = system.BGSystem(
    ctree = db,
    solver_name = 'scisparse'
)
sys.own.test = SHGTestStandResonant()
db = sys.ctree_shadow()
#print(sys.test.ktp.ctree_as_yaml())
print(db.test.mDC1.mirror_H1.T_hr)
```

```python
def plot_power_study(test):
    axB = mplfigB(Nrows=4)
    X_NLG = test.PSLR.power_W.val
    axB.ax0.plot(
        X_NLG,
        test.DC_R.DC_readout,
        color = 'red',
        label = '1064 power',
    )
    axB.ax0.plot(
        X_NLG,
        test.DC_G.DC_readout,
        color = 'green',
        label = '532 power',
    )
    axB.ax0.plot(
        X_NLG,
        test.DC_R.DC_readout + test.DC_G.DC_readout,
        color = 'black',
        label = 'total power [W]',
    )
    #axB.ax0.set_ylim(0, .5)
    axB.ax0.set_ylabel('Readout Power [W]')
    axB.ax0.legend(
        fontsize = 8,
        loc = 'center left'
    )
    axB.ax1.plot(
        X_NLG,
        100 * test.DC_G.DC_readout / test.PSLR.power_W.val,
        color = 'black',
        label = 'efficiency',
    )
    axB.ax1.plot(
        X_NLG,
        100 - 100 * (test.DC_R.DC_readout + test.DC_G.DC_readout) / test.PSLR.power_W.val,
        color = 'purple',
        label = 'loss',
    )
    axB.ax1.set_ylabel('Conversion\nEfficiency')
    axB.ax1.legend(
        fontsize = 8,
        loc = 'center left'
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_IQ[0, 0],
        color = 'red',
        label = 'amplitude quadrature',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_IQ[1, 1],
        color = 'orange',
        label = 'phase quadrature',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_ellipse.max,
        color = 'blue',
        label = 'ellipse max',
        ls = '--'
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_ellipse.min,
        color = 'purple',
        label = 'ellipse min',
        ls = '--',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_ellipse.min**.25 * test.AC_R.AC_CSD_ellipse.max**.25,
        color = 'black',
        ls = '--',
        label = 'geometric mean',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_RGI.AC_CSD_ellipse.Imin,
        color = 'magenta',
        ls = '-',
        label = 'green corr min',
    )
    axB.ax2.set_ylabel('1064 PSD/ShotN\n[quanta/Hz]')
    #axB.ax2.set_yscale('log')
    axB.ax2.legend(
        fontsize = 8,
        loc = 'center left'
    )
    axB.ax3.set_ylabel('532  PSD/ShotN\n[quanta/Hz]')
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_IQ[0, 0],
        color = 'green',
        label = 'amplitude quadrature',
    )
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_IQ[1, 1],
        color = 'blue',
        label = 'phase quadrature',
    )
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_ellipse.max,
        color = 'cyan',
        label = 'ellipse max',
        ls = '--',
    )
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_ellipse.min,
        color = 'purple',
        label = 'ellipse min',
        ls = '--',
    )
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_ellipse.min**.5 * test.AC_G.AC_CSD_ellipse.max**.5,
        color = 'black',
        label = 'geometric mean',
        ls = '--',
    )
    axB.ax3.set_xlabel('Input 1064 Power')
    axB.ax3.legend(
        fontsize = 8,
        loc = 'center left'
    )
    tkwarg = dict(
        inperc = 100 * test.mDC1.mirror_H1.T_hr,
        Lperc = 100 * test.mDC2.mirror_H1.L_hr,
        NLG = test.ktp.nlg * test.ktp.length_mm.val,
    )
    axB.ax0.set_title(
        'SHG design with {inperc}% input coupler and {Lperc}% loss (NLG {NLG} rtW/W)'.format(**tkwarg)
    )
    axB.save(
        'SHGpwr-{inperc}%IT-{Lperc}%IL-{NLG}NLG'.format(**tkwarg)
    )
    return axB
axB = plot_power_study(sys.test)
```

```python
def generate_study(
    T_hr = .03,
    L_hr = .01,
):
    db = DeepBunch()
    db.test.ktp.length.val = .12 * 2
    db.test.ktp.N_ode = 10
    db.test.mDC1.mirror_H1.T_hr = T_hr
    db.test.mDC2.mirror_H1.L_hr = L_hr
    db.test.PSLR.power.val = np.linspace(0, .5, 30)
    db.test.ktp.solution_order = 4
    db.test.include_AC = True
    sys = system.BGSystem(
        ctree = db,
    )
    sys.own.test = SHGTestStandResonant()
    print("study!")
    axB = plot_power_study(sys.test)
    return axB
generate_study(
    T_hr = .1,
    L_hr = .001,
)
generate_study(
    T_hr = .1,
    L_hr = .01,
)
generate_study(
    T_hr = .1,
    L_hr = .03,
)
generate_study(
    T_hr = .03,
    L_hr = .001,
)
generate_study(
    T_hr = .03,
    L_hr = .01,
)
generate_study(
    T_hr = .03,
    L_hr = .03,
)
```

```python

```

```python
def plot_loss_study(test, save = True):
    axB = mplfigB(Nrows=4)
    X_NLG = test.mDC2.mirror_H1.L_hr
    axB.ax0.plot(
        X_NLG,
        test.DC_R.DC_readout,
        color = 'red',
        label = '1064 power',
    )
    axB.ax0.plot(
        X_NLG,
        test.DC_G.DC_readout,
        color = 'green',
        label = '532 power',
    )
    axB.ax0.plot(
        X_NLG,
        test.DC_R.DC_readout + test.DC_G.DC_readout,
        color = 'black',
        label = 'total power [W]',
    )
    #axB.ax0.set_ylim(0, .5)
    axB.ax0.set_ylabel('Readout Power [W]')
    axB.ax0.legend(
        fontsize = 8,
        loc = 'upper right'
    )
    axB.ax1.plot(
        X_NLG,
        100 * test.DC_G.DC_readout / test.PSLR.power_W.val,
        color = 'black',
        label = 'efficiency',
    )
    axB.ax1.plot(
        X_NLG,
        100 - 100 * (test.DC_R.DC_readout + test.DC_G.DC_readout) / test.PSLR.power_W.val,
        color = 'purple',
        label = 'loss',
    )
    axB.ax1.set_ylabel('Conversion\nEfficiency')
    axB.ax1.legend(
        fontsize = 8,
        loc = 'upper right'
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_IQ[0, 0],
        color = 'red',
        label = 'amplitude quadrature',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_IQ[1, 1],
        color = 'orange',
        label = 'phase quadrature',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_ellipse.max,
        color = 'blue',
        label = 'ellipse max',
        ls = '--'
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_ellipse.min,
        color = 'purple',
        label = 'ellipse min',
        ls = '--',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_R.AC_CSD_ellipse.min**.25 * test.AC_R.AC_CSD_ellipse.max**.25,
        color = 'black',
        ls = '--',
        label = 'geometric mean',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_RGI.AC_CSD_ellipse.Imin,
        color = 'magenta',
        ls = '-',
        label = 'green corr min',
    )
    axB.ax2.plot(
        X_NLG,
        test.AC_RGQ.AC_CSD_ellipse.Imin,
        color = 'magenta',
        ls = '-',
        label = 'green corrQ min',
    )
    axB.ax2.set_ylabel('1064 PSD/ShotN\n[quanta/Hz]')
    #axB.ax2.set_yscale('log')
    axB.ax2.legend(
        fontsize = 8,
        loc = 'upper right'
    )
    axB.ax3.set_ylabel('532  PSD/ShotN\n[quanta/Hz]')
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_IQ[0, 0],
        color = 'green',
        label = 'amplitude quadrature',
    )
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_IQ[1, 1],
        color = 'blue',
        label = 'phase quadrature',
    )
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_ellipse.max,
        color = 'cyan',
        label = 'ellipse max',
        ls = '--',
    )
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_ellipse.min,
        color = 'purple',
        label = 'ellipse min',
        ls = '--',
    )
    axB.ax3.plot(
        X_NLG,
        test.AC_G.AC_CSD_ellipse.min**.5 * test.AC_G.AC_CSD_ellipse.max**.5,
        color = 'black',
        label = 'geometric mean',
        ls = '--',
    )
    axB.ax3.set_xlabel('Red Cavity Loss')
    axB.ax3.legend(
        fontsize = 8,
        loc = 'upper right'
    )
    tkwarg = dict(
        inperc = 100 * test.mDC1.mirror_H1.T_hr,
        mW = 1000 * test.PSLR.power_W.val,
        NLG = test.ktp.nlg * test.ktp.length_mm.val,
    )
    axB.ax0.set_title(
        'SHG design with {inperc}% input coupler {mW}mW (NLG {NLG} rtW/W)'.format(**tkwarg)
    )
    axB.ax0.set_xscale('log_zoom')
    axB.ax1.set_xscale('log_zoom')
    axB.ax2.set_xscale('log_zoom')
    axB.ax3.set_xscale('log_zoom')
    if save:
        axB.save(
            'SHGloss-{inperc}%IT-{mW}%mW-{NLG}NLG'.format(**tkwarg)
        )
    return axB
```

```python
from phasor.optics.models.KTP_test_stand import SHGTestStandResonant
db = DeepBunch()
db.test.ktp.length.val = .12 * 2
db.test.ktp.N_ode = 10
db.test.mDC1.mirror_H1.T_hr = .1
db.test.mDC2.mirror_H1.L_hr = logspaced(.003, .1, 30)
db.test.PSLR.power.val = .4
db.test.ktp.solution_order = 4
db.test.include_AC = True
sys = system.BGSystem(
    ctree = db,
)
sys.own.test = SHGTestStandResonant()
db = sys.ctree_shadow()
#print(sys.test.ktp.ctree_as_yaml())
print(db.test.mDC1.mirror_H1.T_hr)
axB = plot_loss_study(sys.test)
```

```python
from phasor.optics.models.KTP_test_stand import SHGTestStandResonant
db = DeepBunch()
db.test.ktp.length.val = .12 * 2
db.test.ktp.N_ode = 10
db.test.mDC1.mirror_H1.T_hr = .03
db.test.mDC2.mirror_H1.L_hr = logspaced(.003, .1, 30)
db.test.PSLR.power.val = .4
db.test.ktp.solution_order = 4
db.test.include_AC = True
sys = system.BGSystem(
    ctree = db,
)
sys.own.test = SHGTestStandResonant()
db = sys.ctree_shadow()
#print(sys.test.ktp.ctree_as_yaml())
print(db.test.mDC1.mirror_H1.T_hr)
axB = plot_loss_study(sys.test)
```

```python
from phasor.optics.models.KTP_test_stand import SHGTestStandResonant
db = DeepBunch()
db.test.ktp.length.val = .12
db.test.ktp.N_ode = 10
db.test.mDC1.mirror_H1.T_hr = .1
db.test.mDC2.mirror_H1.L_hr = logspaced(.003, .1, 30)
db.test.PSLR.power.val = .4
db.test.ktp.solution_order = 4
db.test.include_AC = True
db.test.mirror_gres.T_hr = .5
sys = system.BGSystem(
    ctree = db,
)
sys.own.test = SHGTestStandResonant()
db = sys.ctree_shadow()
#print(sys.test.ktp.ctree_as_yaml())
print(db.test.mDC1.mirror_H1.T_hr)
axB = plot_loss_study(sys.test, save = False)
axB.save('SHGlossResGrn-10ic-50gc-400mW')
```

```python
from phasor.optics.models.KTP_test_stand import SHGTestStandResonant
db = DeepBunch()
db.test.ktp.length.val = .12 * 2
db.test.ktp.N_ode = 10
db.test.mDC1.mirror_H1.T_hr = .1
db.test.mDC2.mirror_H1.L_hr = logspaced(.003, .1, 30)
db.test.PSLR.power.val = .1
db.test.ktp.solution_order = 4
db.test.include_AC = True
db.test.mirror_gres.T_hr = .1
sys = system.BGSystem(
    ctree = db,
)
sys.own.test = SHGTestStandResonant()
db = sys.ctree_shadow()
#print(sys.test.ktp.ctree_as_yaml())
print(db.test.mDC1.mirror_H1.T_hr)
axB = plot_loss_study(sys.test, save = False)
axB.save('SHGlossResGrn-10ic-10gc-100mW')
```

```python
sys.test.AC_RGQ.AC_CSD_ellipse.Imin * sys.test.AC_RGI.AC_CSD_ellipse.Imin
```

```python
axB = mplfigB()
for src, CSD in sys.test.AC_R.noise.CSD_by_source.items():
    if not np.all(CSD['ps_In', 'ps_In'] == 0):
        axB.ax0.plot(
            sys.test.mDC2.mirror_H1.L_hr,
            np.ones_like(sys.test.mDC2.mirror_H1.L_hr) * CSD['ps_In', 'ps_In'].real,
            label = str(src),
        )
axB.ax0.legend(
    fontsize = 8,
)

```

```python
from phasor.optics.models.KTP_test_stand import SHGTestStandResonant
db = DeepBunch()
db.test.ktp.length.val = .12 * 2
db.test.ktp.N_ode = 10
db.test.mDC1.mirror_H1.T_hr = .1
db.test.mDC2.mirror_H1.L_hr = .001
db.test.PSLR.power.val = .1
db.test.ktp.solution_order = 4
db.test.include_AC = True
db.test.mirror_gres.T_hr = .1
db.test.mirror_gres.L_hr = .01
sys = system.BGSystem(
    ctree = db,
)
sys.own.test = SHGTestStandResonant()
db = sys.ctree_shadow()
#print(sys.test.ktp.ctree_as_yaml())
print(db.test.mDC1.mirror_H1.T_hr)
sys.test.full_noise_matrix()
```

```python
sys.test.AC_N.CSD
```

```python
sys.test.AC_RGI.AC_CSD_ellipse
```

```python
from YALL.utilities.modpack import generate_modpack
generate_modpack()
```

```python
10 * np.log10(.39)
```

```python
sys.test.AC_RGQ.AC_CSD_IQ
```

```python

```

```python

```


```python
# import all relevant libraries dependencies and set up the backend
import qsharp
import time
import progressbar
import numpy as np
import matplotlib.pyplot as plt
from qsharp import Result
import exact_diagonalisation.ed_ising as ED
from Quantum.Isinglongrange import Evolve, EvolveSingleTimestepDummy
# from Quantum.Ising2dimport import Evolve, EvolveSingleTimestepDummy

c1_a = '#003AF0'
c2_a = '#008F24'
c3_a = '#FE7B22'
c4_a = '#FE000C'
c5_a = '#FB30F2'
c6_a = '#82E0AA'
c7_a = '#9966CC'
c8_a = '#7FFFD4'
c9_a = '#66FF00'
c10_a = '#8B008B'
c11_a = '#000000'
colors = [c1_a,c2_a,c3_a,c4_a,c5_a,c6_a,c7_a,c8_a,c9_a,c10_a,c11_a]
```

    Preparing Q# environment...
    .

# Time Evolution

Assumes an L x W lattice of sites and uses lists length LW to specify hx and hz for each site and a matrix J (LW x LW) specified for the couplings. Evolves under Hamiltonian
\begin{equation}\label{eq:}
H = -\sum_{i, j>i} J_{ij} \sigma_z^i\sigma_z^j - \sum_{i}hx_i \sigma_x^i - \sum_{i}hz_i\sigma_z^i
\end{equation}

Expects initial state of correct shape in 0s and 1s (Z basis) to be specified. An initial state in the X basis can be used instead by specifying the x_rot parameter

```python
# define system parameters
attempt_ED = True
xinit = False
xmeas = False
nnonly = False
parallel = True
p = {}
p['L'] = L = 3 # system length
p['W'] = W  = 1 # system width
p['N'] = p['L'] * p['W'] # number of system in the system
p['hx'] = hx = np.array([1. for _ in range(L * W)]) # transverse Ising field for each site
p['hz'] = hz = np.array([0. for _ in range(L * W)]) # longitudinal Ising field for each site
J = [[0., 1., 0.],
     [1., 0., 1.],
     [0., 1., 0.]]
# J = [[0., 0., 0., 0.],
#      [0., 0., 0., 0.],
#      [0., 0., 0., 0.],
#      [0., 0., 0., 0.]]

# couplings = [0., 1., 0.5, 0.1, 0.]
# J = [[0. for _ in range(L * W)] for _ in range(L * W)]
# for i in range(L * W):
#     for j in range(L * W):
#         s_r, s_c = i // W, i % W
#         t_r, t_c = j // W, j % W
#         if s_r == t_r:
#             h_dist = min([abs(s_c - t_c), W - abs(s_c - t_c)])
#             J[i][j] = couplings[h_dist]
#         elif s_c == t_c:
#             v_dist = min([abs(s_r - t_r), L - abs(s_r - t_r)])
#             J[i][j] = couplings[v_dist]
p['J'] = np.asarray(J) # Ising interaction for all sites (i,j)
p['t_initial'] = 0 # begin of time evolution
p['dt'] = dt = 0.01 # time-step
p['t_final'] = total_time = 1.0 # end of time evolution
times = np.linspace(0, total_time, num=11)
ave = 500 # measurement count for qsharp meas
initial_state = [[0, 1, 0],]
flat_initial_state = [item for sub in initial_state for item in sub]
state_table = ED.generate_state_table(p)
if L * W > 9:
    attempt_ED = False
```

```python
# do exact evolution
if attempt_ED:
    tic = time.time()
    sim, state_table = ED.evolve(p, state_table, flat_initial_state, kind="list", trotterised=True, xinit=xinit)
    toc = time.time()
    print("Exact evolution took ", toc-tic, "s")
    basis = 'X' if xmeas else 'Z'
```

    Exact evolution took  0.047789812088012695 s

```python
# do qsharp evolution
tic = time.time()
result = np.zeros((len(times), *np.array(initial_state).shape))
progress_bar = progressbar.ProgressBar(max_value=ave)
points_taken = 0
for _ in range(ave):
    for i, t in enumerate(times):
        res = Evolve.simulate(
                initialState=flat_initial_state, time=t, dt=dt, g=list(hx), h=list(hz), J=J,
                nnonly=nnonly, parallel=parallel, xinit=xinit, xmeas=xmeas)
        result[i] += np.array(res).reshape(np.array(initial_state).shape) / ave
    points_taken += 1
    progress_bar.update(points_taken)
toc = time.time()
print("Q# Evolution took {} mins {} s".format(int((toc-tic) // 60), int((toc-tic) % 60)))
```

```python

```

```python
def EvolveCouplingsAllToAll(J, mapping, qubits):
    nSites = len(qubits)
    idxTarget = 0
    for step in np.arange(nSites):
        print('step: {},  mapping: {}'.format(step, mapping))
        if (step % 2 == 0):
            for idxSource in np.arange(nSites//2):
                idxTarget = nSites - idxSource - 1
                print('source: {}, target: {}, J{}{}'.format(qubits[mapping[idxSource]],
                                                             qubits[mapping[idxTarget]],
                                                             mapping[idxSource],
                                                             mapping[idxTarget]))
            for idx in np.arange((nSites + 1)//2):
                mapping[nSites - idx], mapping[nSites - idx - 1] = mapping[nSites - idx - 1], mapping[nSites - idx]
        else:
            for idxSource in np.arange((nSites - 1)//2):
                idxTarget = nSites - idxSource - 1
                print('source: {}, target: {}, J{}{}'.format(qubits[mapping[idxSource]],
                                                             qubits[mapping[idxTarget]],
                                                             mapping[idxSource],
                                                             mapping[idxTarget]))
            for ind in np.arange(nSites//2):
                mapping[nSites//2 - ind], mapping[nSites//2 - ind - 1] = mapping[nSites//2 - ind - 1], mapping[nSites//2 - ind]
            mapping[0], mapping[nSites] = mapping[nSites], mapping[0]
    print('mapping: {}'.format(mapping))

```

```python
mapping = np.array([0, 1, 2, 3, 4, -1])
qubits = np.array([0, 1, 2, 3, 4])
EvolveCouplingsAllToAll(J, mapping,qubits)
EvolveCouplingsAllToAll(J, mapping,qubits )
```

    step: 0,  mapping: [ 0  1  2  3  4 -1]
    source: 0, target: 4, J04
    source: 1, target: 3, J13
    step: 1,  mapping: [ 0  1 -1  2  3  4]
    source: 0, target: 3, J03
    source: 1, target: 2, J12
    step: 2,  mapping: [ 4  0  1  2  3 -1]
    source: 4, target: 3, J43
    source: 0, target: 2, J02
    step: 3,  mapping: [ 4  0 -1  1  2  3]
    source: 4, target: 2, J42
    source: 0, target: 1, J01
    step: 4,  mapping: [ 3  4  0  1  2 -1]
    source: 3, target: 2, J32
    source: 4, target: 1, J41
    mapping: [ 3  4 -1  0  1  2]
    step: 0,  mapping: [ 3  4 -1  0  1  2]
    source: 3, target: 1, J31
    source: 4, target: 0, J40
    step: 1,  mapping: [ 3  4  2 -1  0  1]
    source: 3, target: 0, J30
    source: 4, target: 4, J4-1
    step: 2,  mapping: [ 1  3  4 -1  0  2]
    source: 1, target: 0, J10
    source: 3, target: 4, J3-1
    step: 3,  mapping: [ 1  3  2  4 -1  0]
    source: 1, target: 4, J1-1
    source: 3, target: 4, J34
    step: 4,  mapping: [ 0  1  3  4 -1  2]
    source: 0, target: 4, J0-1
    source: 1, target: 4, J14
    mapping: [ 0  1  2  3  4 -1]

```python

```

```python

```

```python
# plot
fig = plt.figure()
qubits = [(0, 0), (0, 1), (0, 2)]
for i, q in enumerate(qubits):
    site = p['W']*q[0]+q[1]
    if attempt_ED:
        site_res = 1 - (sim['{}i Site {}'.format(basis, site)] + 1) / 2
        plt.plot(sim['Time'], site_res, label='{}'.format(q), marker='', linestyle='-', color=colors[i])
    plt.plot(times, result[:, q[0], q[1]], color=colors[i], marker='o', linestyle='--')
plt.ylim(0, 1)
handles, labels = fig.axes[0].get_legend_handles_labels()
handles = list(handles) + [plt.Line2D((0, 1), (0, 0), color='k', linestyle='-'),
                           plt.Line2D((0, 1), (0, 0), color='k',linestyle='--')]
labels = list(labels) + ['ED', 'Q#']
plt.xlabel('time')
plt.ylabel('time')
plt.legend(handles, labels)
```

# Resource Estimation

Estimate the resources for the full evolution (1st order) and for a single timestep. Currently the depth only measures T gates of which there are none.

```python
# estimate resources
full_gatecount = Evolve.estimate_resources(
                    initialState=flat_initial_state, time=t, dt=dt, g=list(hx), h=list(hz), J=J,
                    nested=nested, nnonly=nnonly, xinit=xinit, xmeas=xmeas)
timestep_gatecount = EvolveSingleTimestepDummy.estimate_resources(
                        nSites=len(flat_initial_state), dt=dt, g=list(hx), h=list(hz), J=J,
                        nested=nested, nnonly=nnonly)

print('Full Evolution')
print(*['\t{}: {}\n'.format(k, v) for k, v in full_gatecount.items()])
print('Single Timestep')
print(*['\t{}: {}\n'.format(k, v) for k, v in timestep_gatecount.items()])
```

    Full Evolution
    	CNOT: 1200
     	QubitClifford: 1
     	R: 1400
     	Measure: 4
     	T: 0
     	Depth: 0
     	Width: 4
     	BorrowedWidth: 0

    Single Timestep
    	CNOT: 12
     	QubitClifford: 0
     	R: 14
     	Measure: 0
     	T: 0
     	Depth: 0
     	Width: 4
     	BorrowedWidth: 0

```python

```