# Example Data Project: the Train Illusion

Marius 't Hart, Megan Peters, Paul Schrater, Jean Laurens, Gunnar Blohm

**Disclaimer**:  this is a "toy" data neuroscience project used to demonstrate the [10 step procedure of how-to-model](https://doi.org/10.1523/ENEURO.0352-19.2019). It is not meant to be state of the art research.

**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

##  Setup

```python
# @title Setup

# for matrices and plotting:
import numpy as np
import matplotlib.pyplot as plt

# for random distributions:
from scipy.stats import norm, poisson

# for logistic regression:
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
!pip install tqdm --quiet
```

##  Generate Data

```python
# @title Generate Data

def generateSpikeTrains():

  gain        = 2
  neurons     = 50
  movements   = [0,1,2]
  repetitions = 800

  np.random.seed(37)

  # set up the basic parameters:
  dt = 1/100
  start, stop = -1.5, 1.5
  t = np.arange(start, stop+dt, dt)                                            # a time interval
  Velocity_sigma = 0.5                                                         # std dev of the velocity profile
  Velocity_Profile = norm.pdf(t,0,Velocity_sigma)/norm.pdf(0,0,Velocity_sigma) # The Gaussian velocity profile, normalized to a peak of 1

  # set up the neuron properties:
  Gains = np.random.rand(neurons) * gain          # random sensitivity between 0 and `gain`
  FRs   = (np.random.rand(neurons) * 60 ) - 10    # random base firing rate between -10 and 50

  # output matrix will have this shape:
  target_shape = [len(movements), repetitions, neurons, len(Velocity_Profile)]

  # build matrix for spikes, first, they depend on the velocity profile:
  Spikes = np.repeat(Velocity_Profile.reshape([1,1,1,len(Velocity_Profile)]),len(movements)*repetitions*neurons,axis=2).reshape(target_shape)

  # multiplied by gains:
  S_gains = np.repeat(np.repeat(Gains.reshape([1,1,neurons]), len(movements)*repetitions, axis=1).reshape(target_shape[:3]), len(Velocity_Profile)).reshape(target_shape)
  Spikes = Spikes * S_gains

  # and multiplied by the movement:
  S_moves = np.repeat( np.array(movements).reshape([len(movements),1,1,1]), repetitions*neurons*len(Velocity_Profile), axis=3 ).reshape(target_shape)
  Spikes = Spikes * S_moves

  # on top of a baseline firing rate:
  S_FR = np.repeat(np.repeat(FRs.reshape([1,1,neurons]), len(movements)*repetitions, axis=1).reshape(target_shape[:3]), len(Velocity_Profile)).reshape(target_shape)
  Spikes = Spikes + S_FR

  # can not run the poisson random number generator on input lower than 0:
  Spikes = np.where(Spikes < 0, 0, Spikes)

  # so far, these were expected firing rates per second, correct for dt:
  Spikes = poisson.rvs(Spikes * dt)

  return(Spikes)

def subsetPerception(spikes):

  movements = [0,1,2]
  split     = 400
  subset    = 40
  hwin      = 3

  [num_movements, repetitions, neurons, timepoints] = np.shape(spikes)

  decision = np.zeros([num_movements, repetitions])

  # ground truth for logistic regression:
  y_train = np.repeat([0,1,1],split)
  y_test  = np.repeat([0,1,1],repetitions-split)

  m_train = np.repeat(movements, split)
  m_test  = np.repeat(movements, split)

  # reproduce the time points:
  dt = 1/100
  start, stop = -1.5, 1.5
  t = np.arange(start, stop+dt, dt)

  w_idx = list( (abs(t) < (hwin*dt)).nonzero()[0] )
  w_0 = min(w_idx)
  w_1 = max(w_idx)+1  # python...

  # get the total spike counts from stationary and movement trials:
  spikes_stat = np.sum( spikes[0,:,:,:], axis=2)
  spikes_move = np.sum( spikes[1:,:,:,:], axis=3)

  train_spikes_stat = spikes_stat[:split,:]
  train_spikes_move = spikes_move[:,:split,:].reshape([-1,neurons])

  test_spikes_stat = spikes_stat[split:,:]
  test_spikes_move = spikes_move[:,split:,:].reshape([-1,neurons])

  # data to use to predict y:
  x_train = np.concatenate((train_spikes_stat, train_spikes_move))
  x_test  = np.concatenate(( test_spikes_stat,  test_spikes_move))

  # this line creates a logistics regression model object, and immediately fits it:
  population_model = LogisticRegression(solver='liblinear', random_state=0).fit(x_train, y_train)

  # solver, one of: 'liblinear', 'newton-cg', 'lbfgs', 'sag', and 'saga'
  # some of those require certain other options
  #print(population_model.coef_)       # slope
  #print(population_model.intercept_)  # intercept

  ground_truth = np.array(population_model.predict(x_test))
  ground_truth = ground_truth.reshape([3,-1])

  output = {}
  output['perception'] = ground_truth
  output['spikes']     = spikes[:,split:,:subset,:]

  return(output)

def getData():

  spikes = generateSpikeTrains()

  dataset = subsetPerception(spikes=spikes)

  return(dataset)

dataset = getData()
perception = dataset['perception']
spikes = dataset['spikes']
```

##  Plot Functions

```python
# @title Plot Functions

def rasterplot(spikes,movement,trial):

  [movements, trials, neurons, timepoints] = np.shape(spikes)

  trial_spikes = spikes[movement,trial,:,:]

  trial_events = [((trial_spikes[x,:] > 0).nonzero()[0]-150)/100 for x in range(neurons)]

  plt.figure()
  dt=1/100
  plt.eventplot(trial_events, linewidths=1);
  plt.title('movement: %d - trial: %d'%(movement, trial))
  plt.ylabel('neuron')
  plt.xlabel('time [s]')

def plotCrossValAccuracies(accuracies):
  f, ax = plt.subplots(figsize=(8, 3))
  ax.boxplot(accuracies, vert=False, widths=.7)
  ax.scatter(accuracies, np.ones(8))
  ax.set(
    xlabel="Accuracy",
    yticks=[],
    title=f"Average test accuracy: {accuracies.mean():.2%}"
  )
  ax.spines["left"].set_visible(False)
```

----
# Phenomenon
*Part of Steps 1-2*

The train illusion occurs when sitting on a train and viewing another train outside the window. Suddenly, the other train *seems* to move, i.e. you experience visual motion of the other train relative to your train. But which train is actually moving?

Often people mix this up. In particular, they think their own train might be moving when it's the other train that moves; or vice versa. The illusion is usually resolved once you gain vision of the surroundings that lets you disambiguate the relative motion; or if you experience strong vibrations indicating that it is indeed your own train that is in motion.

----
# Question

*Part of Step 1*

We assume that we have build the train illusion model (see the other example project colab). That model predicts that accumulated sensory evidence from vestibular signals determines the decision of whether self-motion is experienced or not. We now have vestibular neuron data (simulated in our case, but let's pretend) and would like to see if that prediction holds true.

The data contains $N$ neurons and $M$ trials for each of 3 motion conditions: no self-motion, slowly accelerating self-motion and faster accelerating self-motion.

\begin{align}
N &= 40\\
M &= 400\\
\end{align}

**So we can ask the following question**: "Does accumulated vestibular neuron activity correlate with self-motion judgements?"

# Background

_Part of step 2_

While it seems a well-known fact that vestibular signals are noisy, we should check if we can also find this in the literature.

Let's also see what's in our data, there should be a 4d array called `spikes` that has spike counts (positive integers), a 2d array called `perception` with self-motion judgements (0=no motion or 1=motion). Let's see what this data looks like:

```python
print(np.shape(spikes))
print(np.shape(perception))
print(np.mean(perception, axis=1))
```

In the `spikes` array, we see our 3 acceleration conditions (first dimension), with 400 trials each (second dimensions) and simultaneous recordings from 40 neurons (third dimension), across 3 seconds in 10 ms bins (fourth dimension). The first two dimensions are also there in the `perception` array.

Perfect perception would have looked like [0, 1, 1]. The average judgements are far from correct (lots of self-motion illusions) but they do make some sense: it's closer to 0 in the no-motion condition and closer to 1 in both of the real-motion conditions.

The idea of our project is that the vestibular signals are noisy so that they might be mis-interpreted by the brain. Let's see if we can reproduce the stimuli from the data:

```python
for move_no in range(3):
  plt.plot(np.arange(-1.5,1.5+(1/100),(1/100)),np.mean(np.mean(spikes[move_no,:,:,:], axis=0), axis=0), label=['no motion', '$1 m/s^2$', '$2 m/s^2$'][move_no])
plt.xlabel('time [s]');
plt.ylabel('averaged spike counts');
plt.legend()
```

Blue is the no-motion condition, and produces flat average spike counts across the 3 s time interval. The orange and green line do show a bell-shaped curve that corresponds to the acceleration profile. But there also seems to be considerable noise: exactly what we need. Let's see what the spike trains for a single trial look like:

```python
for move in range(3):
  rasterplot(spikes = spikes, movement = move, trial = 0)
```

You can change the trial number in the bit of code above to compare what the rasterplots look like in different trials. You'll notice that they all look kind of the same: the 3 conditions are very hard (impossible?) to distinguish by eye-balling.

Now that we have seen the data, let's see if we can extract self-motion judgements from the spike counts.

----
# Ingredients

*Part of step 3*

In order to address our question we need to design an appropriate computational data analysis pipeline. We did some brainstorming and think that we need to somehow extract the self-motion judgements from the spike counts of our neurons. Based on that, our algorithm needs to make a decision: was there self motion or not? This is a classical 2-choice classification problem. We will have to transform the raw spike data into the right input for the algorithm (spike pre-processing).

So we determined that we probably needed the following ingredients:

* spike trains $S$ of 3-second trials (10ms spike bins)
* ground truth movement $m_r$ (real) and perceived movement $m_p$
* some form of classifier $C$ giving us a classification $c$
* spike pre-processing

----
# Hypotheses

*Part of step 4*

We think that noise in the signal drives whether or not people perceive self motion. Maybe the brain uses the strongest signal at peak acceleration to decide on self motion, but we actually think it is better to accumulate evidence over some period of time. We want to test this. The noise idea also means that when the signal-to-noise ratio is higher, the brain does better, and this would be in the faster acceleration condition. We want to test this too.

We came up with the following hypotheses focussing on specific details of our overall research question:

* Hyp 1: Accumulated vestibular spike rates explain self-motion judgements better than average spike rates around peak acceleration.
* Hyp 2: Classification performance should be better for faster vs slower self-motion.

> There are many other hypotheses you could come up with, but for simplicity, let's go with those.

Mathematically, we can write our hypotheses as follows (using our above ingredients):
* Hyp 1: $\mathbf{E}(c_{accum})>\mathbf{E}(c_{win})$
* Hyp 2: $\mathbf{E}(c_{fast})>\mathbf{E}(c_{slow})$

Where $\mathbf{E}$ denotes taking the expected value (in this case the mean) of its argument: classification outcome in a given trial type.

----
# Selected toolkit

*Part of step 5*

We want to build some form of classification pipeline. There are many options here, but a simple decoder would be a good start. Since we will learn all about Generalized Linear Models soon, we decided to use that! From W1D4: Generalized Linear Models there were two models that looked relevant:

* Linear-Nonlinear-Poisson GLM: predict spike counts from stimulus
* Logistic Regression: predict stimulus from spike counts

Here we want to predict self-motion from spike counts, so we pick Logistic Regression.

----
# Model draft

*Part of step 6*

Brainstorming on the whiteboard, we came up with this set of steps:

1. convert spike count data into design matrix: trials x neurons (with accumulated or windowed data)
2. get the classes each trial belongs to: stimulus (stationary / moving) or self-motion judgements (moving or not?) as a vector array
3. split data into train and test sets (200 trials each for the 3 conditions?) with an equal number of each class
4. fit logistic regression model on training set, and get predictions for test set

----
# Model implementation

*Part of step 7*

Below is a function that gets us the design matrix of observations X features (in this case: trials X neurons). It sums all spike counts, either across the whole time interval, or a window, specified in seconds before and after the peak acceleration at time=0 (it is the half-width of the window: `halfwin`).

```python
m_r = np.repeat([0,1,1], 400)
m_p = perception.reshape([-1])
a_r = np.repeat([0,1,2], 400) # for grouping in cross validation? maybe not

def getDesignMatrix(spikes, motions, classes, halfwin=None):
  """
    Get the design matrix and vector of classes for a logistic regression.

    Args:
        spikes (numpy.ndarray): a 4d matrix with spike counts for 1) three motion
        conditions, 2) 400 trials, 3) 40 neurons, 4) 301 time bins
        motions (list): motion conditions to include, any subset of [0,1,2]
        classes (list): list of length 3 * 400 with classes to use for the
        logistic regression
        halfwin (None or float): None to use the whole 3 s time interval, or a
        value between 0 and 1.5 seconds as the extent of the window from 0 (when
        acceleration peaks) in both directions.

    Returns:
        (list of numpy.ndarray): first element of the list is the design matrix
        of shape [trials] X [neurons], second elements is a 1d matrix to use as
        a vector of classes to predict from the design matrix

    This function can be used to get the right design matrix and set of class
    labels for a variety of logistic regressions we might want to do.
  """

  # select motion conditions:
  spikes = spikes[motions,:,:,:]
  y = (np.array(classes).reshape([3,-1])[motions,:]).reshape([-1])

  [movstims, trials, neurons, timepoints] = np.shape(spikes)

  # first we get the window (if any):
  if halfwin is None:
    return( [ np.sum(spikes, axis=3).reshape([movstims*trials,neurons]), y ] )
  else:
    dt = 1/100
    t = np.arange(-1.5,1.5+dt,dt)
    win_idx = (abs(t) < halfwin).nonzero()[0]
    w_0, w_1 = min(win_idx), max(win_idx)+1
    return( [ np.sum(spikes[:,:,:,w_0:w_1], axis=3).reshape([movstims*trials,neurons]), y ] )

# test the function:
[desmat, y] = getDesignMatrix(spikes, motions=[0,1], classes=m_r)
# let's check the shape:
print(np.shape(desmat))
# and let's check if the mean sum of spikes checks out:
print(np.mean(np.sum(spikes,axis=3),axis=(1,2)))
for mov in range(2):
  print(np.mean(desmat[(mov*400):((mov+1)*400),:]))
```

First, the design matrix: `desmat` has the right shape: 800 trials (for two conditions), by 40 neurons.

We also get the same average spike counts from our function (76.3 and 77.5) as when calculating it 'by hand' ([76.33475 77.53275 78.61975]).

This means our function `getDesignMatrix()` works correctly (unit test).

It also gets us the classes for each trial. We can use both judgements, $m_p$, and stimuli, $m_r$, as one vector, and `getDesignMatrix()` returns the correct subset (depending on which motion conditions are used) as the second variable.

We wanted to split the data into a training and test set, but we can have scikit-learn do this for us with the `cross_val_score()` function, as we saw in the GLM day:

```python
accuracies = cross_val_score(LogisticRegression(solver='liblinear'), X=desmat, y=y, cv=8)
plotCrossValAccuracies(accuracies) # this plotting function is copied from W1D4: Generalized Linear Models
```

We asked for 8 cross validations, which show up as the blue dots in the graph (two have the same accuracy). Prediction accuracy ranges from 56% to 72%, with the average at 65%, and the orange line is the median. Given the noisy data, that is not too bad actually.

But maybe it's better to split it according to the conditions where there is motion but of different magnitude. It should work better to classify higher acceleration motion from no motion as compared to classifying the lower acceleration motion.

Maybe it also works better to ignore some of the noise at the beginning and end of each trial by focussing on the spikes around the maximum acceleration, using our window option. The average spike count plot above, seems to best discriminate between the three motion conditions around peak acceleration (at 0 s).

We also want to test if it's possible to predict the self-motion judgements rather than the actual motion.

So we write a function that brings it all together:

```python
m_r = np.repeat([0,1,1], 400)
m_p = perception.reshape([-1])

def classifyMotionFromSpikes(spikes, classes, halfwin=None, motions=[0,1,2], cv=8):
  """
    Runs one logistic regression using the specified parameters.

    Args:
        spikes (numpy.ndarray): a 4d matrix with spike counts for 1) three motion
        conditions, 2) 400 trials, 3) 40 neurons, 4) 301 time bins
        classes (list): list of length 3 * 400 with classes to use for the
        logistic regression
        halfwin: None to use the whole 3 s time interval, or a value between 0
        and 0.750 seconds as the extent of the window from 0 (when acceleration
        peaks)
        motions (list): motion conditions to include, any subset of [0,1,2]
        cv (int): number of cross validations to do in logistic regression

    Returns:
        (numpy.ndarray): 1d array of shape (cv,) with prediction accuracies for
        each cross validation

    This function uses our getDesignMatrix() function according to specification
    and then uses cross-validated logistic regression and returns the accuracy
    for each run of the model.
  """

  # get the right design matrix:
  X, y = getDesignMatrix(spikes, motions, classes=classes, halfwin=halfwin)

  # right now, we are not using regularization:
  return( cross_val_score(LogisticRegression(solver='liblinear'), X=X, y=y, cv=cv) )

accuracies = classifyMotionFromSpikes(spikes, m_r, motions=[0,1])
plotCrossValAccuracies(accuracies)
```

This is the exact same figure as before, so our function `classifyMotionFromSpikes()` also works as intended.

----
# Model completion

*Part of step 8*

**Can we answer our question?** Question: "Does accumulated vestibular neuron activity correlate with self-motion judgements?" Yes, we can get an answer to the question by applying our analysis to predict self-motion judgements.

**Can we speak to our hypothesis?** We had two hypotheses: 1) prediction is better with total accumulated spike counts than from a window around peak acceleration, and 2) prediction is better with higher acceleration as the signal-to-noise ratio increases.

**Does the model reach our goals?** We want to understand if motion judgement and not true motion correlates better with vestibular neural activity.

Note: We have somewhat refined our goal here by explicitly contrasting true motion with motion judgements.

----
# Model evaluation & testing

*Part of step 9*

We can now evaluate the performance of our analysis pipeline.

To do so, we will run the logistic regression quite a few times: on all data or split by the velocities, and then we use the data to predict the presence of real motion, or the judgements of self-motion. We repeat this for total accumulated spikes and for a window around the peak acceleration. And then we'll plot the average classification performance in those 12 cases.

```python
def runAnalysis(spikes):
  """
    Runs one logistic regression using the specified parameters.

    Args:
        spikes (numpy.ndarray): a 4d matrix with spike counts for 1) three motion
        conditions, 2) 400 trials, 3) 40 neurons, 4) 301 time bins

    Returns:
        (numpy.ndarray): array of shape (2, 2, 3, 12), where the first dimension
        denotes full interval or 100 ms window, the second dimension denotes
        real motion or self-motion judgements, the third dimension denotes slow,
        fast or all data, and for each there 12 prediction accuracies for each
        of the 12 cross validations

    This function uses our classifyMotionFromSpikes()) function according to
    specification for all the different ways we want to classify the data and
    gives us the classification performance.
  """

  # variables we use for testing our analysis pipeline:
  m_r = np.repeat([0,1,1], 400)
  m_p = perception.reshape([-1])
  halfwins    = [None, 0.050]
  motions     = [[0,1],[0,2],[0,1,2]]
  class_sets  = [m_r, m_p]
  cv = 12

  # empty array to collect classification performance:
  accuracies = np.zeros([len(halfwins),len(class_sets),len(motions),cv])

  for halfwin_no in range(len(halfwins)):

    halfwin = halfwins[halfwin_no]
    lty = ['-','--'][halfwin_no]
    leg_hw = ['accumulated', '100 ms window'][halfwin_no]

    for classes_no in range(len(class_sets)):

      classes = class_sets[classes_no]
      leg_class = ['real', 'judgements'][classes_no]
      color = ['orange','purple'][classes_no]

      for motions_no in range(len(motions)):

        motion = motions[motions_no]
        cond_acc = classifyMotionFromSpikes(spikes  = spikes,
                                            classes = classes,
                                            halfwin = halfwin,
                                            motions = motion,
                                            cv=12)
        accuracies[halfwin_no,classes_no,motions_no,:] = cond_acc

      m_acc = np.mean(accuracies[halfwin_no,classes_no,:,:], axis=1)

  return(accuracies)

# here we run the above function and store the output:
accuracies = runAnalysis(spikes)
```

We will need to plot this output, and to simplify the code, we do that in a separate function, which we write here:

```python
def plotAccuracies(accuracies):
  """
    Plot the accuracies from our main fitting function.

    Args:
        accuarcies (numpy.ndarray): a 4d matrix with performance accuracy for
        1) full interval or 100 ms window, 2) real motion or self-motion
        judgements, 3) slow, fast or all conditions, and 4) the 12 cross
        validations

    Returns:

    This function plots the output of our data analysis pipeline and does not
    return anything.
  """
  ax = plt.figure(figsize=(6,4))
  plt.title('classification accuracy')
  plt.xlim([0.5,3.5])
  plt.ylim([0.4,1.0])
  plt.ylabel('proportion correct')
  plt.xlabel('velocity condition vs. stationary')

  for halfwin_no in range(2):

    lty = ['-','--'][halfwin_no]
    leg_hw = ['accumulated', '100 ms window'][halfwin_no]

    for classes_no in range(2):

      leg_class = ['real', 'judgements'][classes_no]
      color = ['orange','purple'][classes_no]

      m_acc = np.mean(accuracies[halfwin_no,classes_no,:,:], axis=1)
      plt.plot([1,2,3], m_acc, lty, color=color, label=leg_class+' '+leg_hw)

  plt.xticks(ticks=[1,2,3], labels=['slow ($1 m/s^2$)','fast ($2 m/s^2$)','both'])
  plt.legend(loc='lower right')
  plt.show()

plotAccuracies(accuracies)
```

Well, that's interesting! The logistic regression doesn't do a perfect job, but there is information in these results.

1. The dashed lines reflect predictions based on a small window of data, and they do worse than the full data set: this could mean that the brain also integrates signals across longer time frames for perception.

2. In the predictions based on accumulated spike counts, the slow movements are harder to separate from no movements than the faster movements. This is clearer when predicting real motion than when predicting self-motion judgements.

Those were our two hypotheses, but we notice something else:

3. Self-motion judgments display higher decoding accuracy than the actual motion.

Actually, if self-motion judgements and our logistic regression use input from the same noisy sensors, it kind of makes sense that they would both give similar output. This is in line the notion that self-motion judgements can be wrong because the underlying sensory signals are noisy. Of course, this only works if we record activity from neuronal populations that contribute to self-motion judgements. On the other hand, we would also see this result if the sensory signal was not noisy and we recorded from one of several populations that contribute to self-motion judgements in a noisy way. So we need to do more research here.

Either way, we learned something today!

----
# Summary
*Part of Step 10*

Let's write a simple abstract following the guidelines.

**A. What is the phenomena?** Here summarize the part of the phenomena which your model addresses.

_When sitting in a stationary train and seeing the train next to them move, people may experience the "train illusion": they feel like they are moving when they are not, or vice versa. Vestibular information can disambiguate self motion from motion of the adjacent train._

**B. What is the key scientific question?** Clearly articulate the question which your model tries to answer.

_However, it is unknown if accumulated, noisy vestibular neuron activity correlates with self-motion judgement._

**C. What was our hypothesis?** Explain the key relationships which we relied on to simulate the phenomena.

_Based on previous modeling efforts, we hypothesized that cumulative spike counts (not instantaneous) recorded from vestibular neurons can better discriminate experienced self motion judgements as opposed to true self motion._

**D. How did your model work?** Give an overview of the model, it's main components, and how the model works. "Here we ..."

_Here we use logistic regression to classify both true self motion and experienced self motion judgements from vestibular neuron activity. This was done both with the vestibular signal accumulated across the full trial and with a 100 ms window ('instantaneous') around peak acceleration, and with 2 different speeds compared to a no-motion condition._

**E. What did we find?** Did the model work? Explain the key outcomes of your model evaluation.

_Decoding performance for accumulated vestibular signals was higher than for instantaneous signals. Further, experienced self motion judgements could be classified better than true self motion._

**F. What can we conclude?** Conclude as much as you can with reference to the hypothesis, within the limits of the model.

_We conclude that accumulated, noisy vestibular signals during slowly-accelerating self motion may drive the train illusion._

**G. What are the limitations and future directions?** What is left to be learned? Briefly argue the plausibility of the approach and/or what you think is essential that may have been left out.

_Future research should examine how visual signals may combine with vestibular signals in real-world scenarios to causally link these sensory signals to self-motion experience and illusions._

If we put this all in one paragraph, we have our final complete abstract. But, first, do not include the letters in _your_ abstract, and second, we did paraphrase the answers a little so they fit together.

# Abstract

(A) When sitting in a stationary train and seeing the train next to them move, people may experience the "train illusion": they feel like they are moving when they are not, or vice versa. Vestibular information can disambiguate self motion from motion of the adjacent train. (B) However, it is unknown if accumulated, noisy vestibular neuron activity correlates with self-motion judgement. (C) Based on previous modeling efforts, we hypothesized that cumulative spike counts (not instantaneous) recorded from vestibular neurons can better discriminate experienced self motion judgements as opposed to true self motion. (D) Here we use logistic regression to classify both true self motion and experienced self motion judgements from vestibular neuron activity. This was done both with the vestibular signal accumulated across the full trial and with a 100 ms window ('instantaneous') around peak acceleration, and with 2 different speeds compared to a no-motion condition. (E) Decoding performance for accumulated vestibular signals was higher than for instantaneous signals. Further, experienced self motion judgements could be classified better than true self motion. (F) We conclude that accumulated, noisy vestibular signals during slowly-accelerating self motion may drive the train illusion. (G) Future research should model possible mechanisms by which visual signals may combine with vestibular signals to causally link these sensory signals to self-motion experience and illusions.

----
# Final thoughts

Note that the analysis pipeline we built here was extremely simple and we used artificial data on purpose. It allowed us to go through all the steps of building a data neuroscience project, and hopefully you noticed that it is not always a linear process, you will go back to different steps if you hit a roadblock somewhere.

There are many issues that we did not address, e.g.:
* we could have tried different decoders
* or find the optimal window
* what if we had more data...
* how do different neurons contribute to the decision? And why?
  * this is something that you want to explore as a neuroscientist
* we could have run a GLM on the neurons*time array and then analyze the weightings to see if all acceleration steps are weighted equally (perfect integration)... or not?
* what is the role of visual motion in this phenomenon?

However, this project is not meant to be complete, and yours doesn't have to be either. The goal of the projects is to go through _the process_ of a modeling or data science project and put into practice one or more of the toolkits you learn at NMA with your group.


# Slenderness $l/d$ vs $\rho$ curves for limiting deflections
This notebook presents slenderness $l/d$ vs $\rho$ curves that are used for limiting deflections according to EC2. Steel curves are reproduced and carbon curves are investigated.

## Background

Based on section 7.4.2 of the EC2 (Nachweis der Begrenzung der Verformungen ohne direkte Berechnung; Cases where calculations may be omitted)

##### eq (7.16.a):

$$\dfrac{l}{d} = K.\left[11+1.5\sqrt{f_{ck}}\dfrac{\rho_0}{\rho} + 3.2\sqrt{f_{ck}}.\sqrt{\left(\dfrac{\rho_0}{\rho} - 1\right)^3}\right] \mathrm{if} \rho \leq \rho_0$$

$$ \dfrac{l}{d} = K.\left[11+1.5\sqrt{f_{ck}}\dfrac{\rho_0}{\rho- \rho^\prime} + \dfrac{1}{12}\sqrt[2]{f_{ck}}.\sqrt{(\dfrac{\rho^\prime}{\rho_0})}\right] \mathrm{if} \rho > \rho_0 $$

| Symbol | Unit | Description
| :-: | :-: | :- |
| $\dfrac{l}{d}$ | - | the limit value of the bending slenderness (ratio of span to effective height) |
| K | - | the coefficient for consideration of the different static systems according to Table 7.4N |
| $\rho_0$ | - | the reference reinforcement ratio, $\rho_0 = 10^{-3}.\sqrt{f_{ck}}$ |
| $\rho$ | - | the required ratio of tensile reinforcement in the center of the span to resist the design moment (at the restraint section for cantilever beams) |
| $\rho^\prime$ | - | the required ratio of compression reinforcement in the center of the span to resist the design moment (at the restraint section for cantilever beams) |
| $f_{ck}$ | Mpa | the Characteristic cylinder compressive strength of the concrete after 28 days |

```python
%matplotlib widget
# %matplotlib qt
import matplotlib.pyplot as plt
import json
import numpy as np
import pandas as pd
import sympy as sp
from sympy import symbols
import bmcs_utils.api as bu
from bmcs_cross_section.api import MKappa, TShape, EC2, BarLayer, ReinfLayer
from bmcs_beam.api import BoundaryConditions, BoundaryConfig, DeflectionProfile, BeamDesign, BeamSLSCurve
from os.path import join
from mpl_toolkits import mplot3d
sp.init_printing()
```

```python
# Concrete C30/37

f_ck = 70

b = 1000
h = 300
d = 0.9 * h

E = EC2.get_E_cm(f_ck)

fctk_0_05 = EC2.get_f_ctk_0_05(f_ck)
f_ctd = fctk_0_05
# f_ctd = 0.85 * fctk_0_05 / 1.5
f_ctd_fl = max((1.6 - h /1000) * f_ctd, f_ctd)

f_ctm = EC2.get_f_ctm(f_ck)
print(f_ctm)
# Info: eps_cr =  0.000170 with default concrete law gives good slenderness curve

mc = MKappa(low_kappa = 0, high_kappa = 0.00003)

# mc.cs_design.matrix='EC2 with plateau'
mc.cs_design.matrix_.trait_set(
        E_cc=E,
        E_ct=E,
        eps_cr=2.5* f_ctd_fl /E, #0.000125 * f_ck/30, # f_ctm /E,
        eps_cy=EC2.get_eps_c2(f_ck),
        eps_cu=EC2.get_eps_cu2(f_ck),
        eps_tu=0.002,
        mu=0.0,
    f_cd = f_ck, # f_ck * 0.85/1.5,
    factor = 0.85/1.5
    )

mc.cross_section_shape_.B = b
mc.cross_section_shape_.H = h

rho = 0.01
A_s= rho * b * d

bl1 = ReinfLayer(name='layer1', z=h-d, A=A_s, matmod='steel')
bl1.matmod_.trait_set(E_s=200000, f_sy=500, factor = 1./1.15)
mc.cross_section_layout.add_layer(bl1)

dp = DeflectionProfile(mc=mc)
dp.beam_design.beam_conf_name = BoundaryConfig.SIMPLE_BEAM_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.FIXED_AND_ROLLER_SUPPORT_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.FIXED_SUPPORT_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.CANTILEVER_DIST_LOAD

dp.F_scale = 1
dp.beam_design.L = 10 * d # l/d = 10
```

    4.610473649546422

```python
# beam_sls = BeamSLSCurve(dp = dp)
# beam_sls.interact()
```

### Load-Deflection profile analysis

| Symbol | Unit | Description
| :-: | :-: | :- |
| $F_{s}$ | - | the load corresponds to L/250 |
| $F_{u}$ | - | the ultimate load |
| $w_{s}$ | - | the deflection L/250 |
| $w_{u}$ | - | the ultimate deflection |

```python
# def get_Fu_and_Fs(dp, n_i, rho_range=(0.0025, 0.025), slenderness_range=(10, 35), upper_reinforcement = False):
#     rho_range = np.linspace(*rho_range, n_i)
#     slenderness_range = np.linspace(*slenderness_range, n_i)

def get_Fu_and_Fs(dp, rho_range=np.linspace(0.0025, 0.025), slenderness_range=np.linspace(10, 35), upper_reinforcement = False):
    if upper_reinforcement:
        d = mc.cross_section_layout.items[0].z
    else:
        d = dp.mc.cross_section_shape_.H -  mc.cross_section_layout.items[0].z
    b = dp.mc.cross_section_shape_.B
    area_g = b * d

    rho_grid, sl_grid = np.meshgrid(rho_range, slenderness_range)
    F_u_grid = np.zeros_like(rho_grid)
    F_s_grid = np.zeros_like(rho_grid)

    _ , ax = plt.subplots()
    ax.set_xlabel(r'$w$ [mm]')
    ax.set_ylabel(r'$F$ [KN]')

    for sl_idx in range(0,len(slenderness_range)):
        for rho_idx in range(0,len(rho_range)):
            rho = rho_grid[rho_idx, sl_idx]
            sl = sl_grid[rho_idx, sl_idx]

            print('parameter combination', rho, sl)

            # assigning the grid area (area_g) to the reinforcement area variable
            A_j_g = rho * area_g
            dp.mc.cross_section_layout.items[0].A = A_j_g

            # assigning the grid length (L_g) to the beam length variable
            L_g = sl * d
            dp.beam_design.L = L_g

            dp.mc.state_changed = True

            # running the deflection analysis
            F_data, w_data = dp.get_Fw()

            # plotting, post-processing & saving the data
            ax.plot(w_data, F_data/1000, label = "rho={}%-sl={} ".format(rho * 100, sl))

            w_s = dp.beam_design.L / 250
            F_u = max(F_data)
            F_s = np.interp(w_s, w_data, F_data, right=F_u*2)

            F_u_grid[rho_idx,sl_idx] = F_u
            F_s_grid[rho_idx,sl_idx] = F_s

    return F_u_grid, F_s_grid, rho_grid, sl_grid
```

```python
def save_data_vars_in_json(f_ck, mc, dp, path):
    rein = mc.cross_section_layout.items
    output_data = {'mc.n_m': mc.n_m,
                  'mc.n_kappa': mc.n_kappa,
                'mc.low_kappa': mc.low_kappa,
                  'mc.high_kappa': mc.high_kappa,
                  'mc.E_cc': mc.cs_design.matrix_.E_cc,
                  'mc.E_ct': mc.cs_design.matrix_.E_ct,
                  'mc.eps_tu': mc.cs_design.matrix_.eps_tu,
                  'mc.eps_cr': mc.cs_design.matrix_.eps_cr,
                  'mc.eps_cy': mc.cs_design.matrix_.eps_cy,
                  'mc.eps_cu': mc.cs_design.matrix_.eps_cu,
                  'mc.mu': mc.cs_design.matrix_.mu,
                  'f_ck': f_ck,
                  'rein[0].E':rein[0].matmod_.E,
                  'rein[0].z': rein[0].z,
                  'rein[0].A': rein[0].A,
                  'dp.beam_design.L': dp.beam_design.L,}
    with open(path, 'w') as outfile:
        json.dump(output_data, outfile, sort_keys=True, indent=4)
```

## Verification with EC2 assumptions

### 1. EC2 - Steel

```python
# Concrete C30/37

f_ck = 70

b = 1000
h = 300
d = 0.9 * h

E = EC2.get_E_cm(f_ck)

fctk_0_05 = EC2.get_f_ctk_0_05(f_ck)
f_ctd = fctk_0_05
# f_ctd = 0.85 * fctk_0_05 / 1.5
f_ctd_fl = max((1.6 - h /1000) * f_ctd, f_ctd)

f_ctm = EC2.get_f_ctm(f_ck)
print(f_ctm)
# Info: eps_cr =  0.000170 with default concrete law gives good slenderness curve

mc = MKappa(low_kappa = 0, high_kappa = 0.00003)

# mc.cs_design.matrix='EC2 with plateau'
mc.cs_design.matrix_.trait_set(
        E_cc=E,
        E_ct=E,
        eps_cr=2.5* f_ctd_fl /E, #0.000125 * f_ck/30, # f_ctm /E,
        eps_cy=EC2.get_eps_c2(f_ck),
        eps_cu=EC2.get_eps_cu2(f_ck),
        eps_tu=0.002,
        mu=0.0,
    f_cd = f_ck, # f_ck * 0.85/1.5,
    factor = 0.85/1.5
    )

mc.cross_section_shape_.B = b
mc.cross_section_shape_.H = h

rho = 0.01
A_s= rho * b * d

bl1 = ReinfLayer(name='layer1', z=h-d, A=A_s, matmod='steel')
bl1.matmod_.trait_set(E_s=200000, f_sy=500, factor = 1./1.15)
mc.cross_section_layout.add_layer(bl1)

dp = DeflectionProfile(mc=mc)
dp.beam_design.beam_conf_name = BoundaryConfig.SIMPLE_BEAM_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.FIXED_AND_ROLLER_SUPPORT_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.FIXED_SUPPORT_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.CANTILEVER_DIST_LOAD

dp.F_scale = 1
dp.beam_design.L = 10 * d # l/d = 10
```

    4.610473649546422

```python
dp.interact()
```

    VBox(children=(HBox(children=(VBox(children=(Tree(layout=Layout(align_items='stretch', border='solid 1px black…

```python
F_u_grid, F_s_grid, rho_grid, sl_grid = get_Fu_and_Fs(dp, 6)
# F_u_grid, F_s_grid, rho_grid, sl_grid = get_Fu_and_Fs(dp, 7, upper_reinforcement = True, rho_range=(0.001, 0.025), slenderness_range=(3, 50))
```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

    parameter combination 0.0025 10.0
    parameter combination 0.0025 15.0
    parameter combination 0.0025 20.0
    parameter combination 0.0025 25.0
    parameter combination 0.0025 30.0
    parameter combination 0.0025 35.0
    parameter combination 0.007000000000000001 10.0
    parameter combination 0.007000000000000001 15.0
    parameter combination 0.007000000000000001 20.0
    parameter combination 0.007000000000000001 25.0
    parameter combination 0.007000000000000001 30.0
    parameter combination 0.007000000000000001 35.0
    parameter combination 0.011500000000000002 10.0
    parameter combination 0.011500000000000002 15.0
    parameter combination 0.011500000000000002 20.0
    parameter combination 0.011500000000000002 25.0
    parameter combination 0.011500000000000002 30.0
    parameter combination 0.011500000000000002 35.0
    parameter combination 0.016 10.0
    parameter combination 0.016 15.0
    parameter combination 0.016 20.0
    parameter combination 0.016 25.0
    parameter combination 0.016 30.0
    parameter combination 0.016 35.0
    parameter combination 0.0205 10.0
    parameter combination 0.0205 15.0
    parameter combination 0.0205 20.0
    parameter combination 0.0205 25.0
    parameter combination 0.0205 30.0
    parameter combination 0.0205 35.0
    parameter combination 0.025 10.0
    parameter combination 0.025 15.0
    parameter combination 0.025 20.0
    parameter combination 0.025 25.0
    parameter combination 0.025 30.0
    parameter combination 0.025 35.0

```python
np.save('F_u_grid_steel_EC2_eq2_tension_c' + str(f_ck) + '.npy', F_u_grid)
np.save('F_s_grid_steel_EC2_eq2_tension_c' + str(f_ck) + '.npy', F_s_grid)
np.save('rho_grid_steel_EC2_eq2_tension_c' + str(f_ck) + '.npy', rho_grid)
np.save('sl_grid_steel_EC2_eq2_tension_c' + str(f_ck) + '.npy', sl_grid)
save_data_vars_in_json(f_ck, mc, dp, 'data_steel_EC2_eq2_tension_c' + str(f_ck) + '.json')
```

```python
fig, ax = plt.subplots()

z = F_u_grid / F_s_grid - 1./0.59
CS = ax.contour(rho_grid, sl_grid, z)
ax.clabel(CS, inline=1, fontsize=10)
ax.set_ylabel('L/d')
ax.set_xlabel(r'$\rho$ %')
ax.set_ylim(0, 35)
ax.set_xlim(0.0, 0.025)

# Draw EC2 curve
slenderness = BeamSLSCurve().plot_steel_sls_curves(ax=ax, f_cks=[70])
```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

### Plotting all curves

```python
f_cks = [30, 50, 70]
F_u_grids = []
F_s_grids = []
rho_grids = []
sl_grids = []
reinforcement = 'steel'
for f_ck in f_cks:
    f_ck = str(f_ck)

    F_u_grid = np.load('F_u_grid_' + reinforcement + '_EC2_eq2_tension_c' + f_ck + '.npy')
    F_s_grid = np.load('F_s_grid_' + reinforcement + '_EC2_eq2_tension_c' + f_ck + '.npy')
    rho_grid = np.load('rho_grid_' + reinforcement + '_EC2_eq2_tension_c' + f_ck + '.npy')
    sl_grid = np.load('sl_grid_' + reinforcement + '_EC2_eq2_tension_c' + f_ck + '.npy')

#     F_u_grid = np.load('F_u_grid_' + reinforcement + '_c' + f_ck + '.npy')
#     F_s_grid = np.load('F_s_grid_' + reinforcement + '_c' + f_ck + '.npy')
#     rho_grid = np.load('rho_grid_' + reinforcement + '_c' + f_ck + '.npy')
#     sl_grid = np.load('sl_grid_' + reinforcement + '_c' + f_ck + '.npy')

    F_u_grids.append(F_u_grid)
    F_s_grids.append(F_s_grid)
    rho_grids.append(rho_grid)
    sl_grids.append(sl_grid)

```

```python
fig, ax = plt.subplots(1,1)

ax.set_ylabel('L/d')
ax.set_xlabel(r'$\rho$ %')
ax.set_ylim(0, 35)
ax.set_xlim(0.0, 0.025)

for f_ck, F_u_grid, F_s_grid, rho_grid, sl_grid in zip(f_cks, F_u_grids, F_s_grids, rho_grids, sl_grids):
    color = np.random.rand(3,)
    z = F_u_grid / F_s_grid - 1./0.59
    CS = ax.contour(rho_grid, sl_grid, z, levels=[0], colors=[color])
    CS.collections[0].set_label('C' + str(f_ck))
#     ax.clabel(CS, inline=1, fontsize=10)

    # Draw EC2 curve
    rho_range = np.arange(0.0025, 0.025, 0.0001)
    slenderness = get_slenderness(rho_range, 0,  1, f_ck)
    ax.plot(rho_range, slenderness, label='C' + str(f_ck) + '_EC2', ls='--', c = color)

ax.legend()
# fig.savefig('test.pdf')
```

### 2. EC2 - Carbon

```python
for E_t in [230]:
    f_ck = 30
    b = 1000
    h = 300
    d = 0.9 * h

    E = EC2.get_E_cm(f_ck)

    fctk_0_05 = EC2.get_f_ctk_0_05(f_ck)
    f_ctd = fctk_0_05
    # f_ctd = 0.85 * fctk_0_05 / 1.5
    f_ctd_fl = max((1.6 - h /1000) * f_ctd, f_ctd)

    eps_cr = f_ctd_fl/E
    print(eps_cr)

    mc = MKappa(low_kappa = 0,
                high_kappa = 0.00004, # high_kappa = 0.00009

                # Cantilever
    #             low_kappa = -0.00005,
    #             high_kappa =  0,
               )
    mc.cs_design.matrix='EC2 with plateau'
    mc.cs_design.matrix_.trait_set(
            E_cc=E,
            E_ct=E,
            eps_cr=eps_cr, #0.000132 * f_ck/30,
            eps_cy=EC2.get_eps_c2(f_ck),
            eps_cu=EC2.get_eps_cu2(f_ck),
            eps_tu=0.002,
            mu=0.0,
        f_cd = f_ck*0.85/1.5,
    #     factor = 0.85/1.5
        )

    mc.cross_section_shape_.B = b
    mc.cross_section_shape_.H = h

    rho = 0.01
    # A_s=6 * np.pi * (32/2)**2
    A_s= 0.5 * rho * b * d

    bl1 = ReinfLayer(name='layer1', z=h-d, A=A_s, matmod='carbon')
    bl1.matmod_.trait_set(E=E_t*1000, f_t=3000, factor = 1./1.5)
    mc.cross_section_layout.add_layer(bl1)

    dp = DeflectionProfile(mc=mc)
    dp.beam_design.beam_conf_name = BoundaryConfig.SIMPLE_BEAM_DIST_LOAD
    # dp.beam_design.beam_conf_name = BoundaryConfig.FIXED_AND_ROLLER_SUPPORT_DIST_LOAD
    # dp.beam_design.beam_conf_name = BoundaryConfig.FIXED_SUPPORT_DIST_LOAD
    # dp.beam_design.beam_conf_name = BoundaryConfig.CANTILEVER_DIST_LOAD

    dp.F_scale = 1
    dp.beam_design.L = 10 * d

#     dp.interact()

    rho1 = np.linspace(0.0005, 0.007, 7)
    rho2 = np.linspace(0.0075, 0.025, 5)
    rho_range=np.concatenate((rho1, rho2))

    sl1 = np.linspace(3, 10, 7)
    sl2 = np.linspace(11, 50, 5)
    slenderness_range=np.concatenate((sl1, sl2))

    F_u_grid, F_s_grid, rho_grid, sl_grid = get_Fu_and_Fs(dp, rho_range=rho_range, slenderness_range=slenderness_range)

    np.save('F_u_grid_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.npy', F_u_grid)
    np.save('F_s_grid_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.npy', F_s_grid)
    np.save('rho_grid_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.npy', rho_grid)
    np.save('sl_grid_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.npy', sl_grid)
    save_data_vars_in_json(f_ck, mc, dp, 'data_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.json')

#F_u_grid, F_s_grid, rho_grid, sl_grid = get_Fu_and_Fs(dp, 12, rho_range=(0.0005, 0.025), slenderness_range=(3, 50))
```

    8.026983871939514e-05

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

    parameter combination 0.0005 3.0
    parameter combination 0.0005 4.166666666666667
    parameter combination 0.0005 5.333333333333334
    parameter combination 0.0005 6.5
    parameter combination 0.0005 7.666666666666667
    parameter combination 0.0005 8.833333333333334
    parameter combination 0.0005 10.0
    parameter combination 0.0005 11.0
    parameter combination 0.0005 20.75
    parameter combination 0.0005 30.5
    parameter combination 0.0005 40.25
    parameter combination 0.0005 50.0
    parameter combination 0.0015833333333333335 3.0
    parameter combination 0.0015833333333333335 4.166666666666667
    parameter combination 0.0015833333333333335 5.333333333333334
    parameter combination 0.0015833333333333335 6.5
    parameter combination 0.0015833333333333335 7.666666666666667
    parameter combination 0.0015833333333333335 8.833333333333334
    parameter combination 0.0015833333333333335 10.0
    parameter combination 0.0015833333333333335 11.0
    parameter combination 0.0015833333333333335 20.75
    parameter combination 0.0015833333333333335 30.5
    parameter combination 0.0015833333333333335 40.25
    parameter combination 0.0015833333333333335 50.0
    parameter combination 0.002666666666666667 3.0
    parameter combination 0.002666666666666667 4.166666666666667
    parameter combination 0.002666666666666667 5.333333333333334
    parameter combination 0.002666666666666667 6.5
    parameter combination 0.002666666666666667 7.666666666666667
    parameter combination 0.002666666666666667 8.833333333333334
    parameter combination 0.002666666666666667 10.0
    parameter combination 0.002666666666666667 11.0
    parameter combination 0.002666666666666667 20.75
    parameter combination 0.002666666666666667 30.5
    parameter combination 0.002666666666666667 40.25
    parameter combination 0.002666666666666667 50.0
    parameter combination 0.0037500000000000003 3.0
    parameter combination 0.0037500000000000003 4.166666666666667
    parameter combination 0.0037500000000000003 5.333333333333334
    parameter combination 0.0037500000000000003 6.5
    parameter combination 0.0037500000000000003 7.666666666666667
    parameter combination 0.0037500000000000003 8.833333333333334
    parameter combination 0.0037500000000000003 10.0
    parameter combination 0.0037500000000000003 11.0
    parameter combination 0.0037500000000000003 20.75
    parameter combination 0.0037500000000000003 30.5
    parameter combination 0.0037500000000000003 40.25
    parameter combination 0.0037500000000000003 50.0
    parameter combination 0.004833333333333334 3.0
    parameter combination 0.004833333333333334 4.166666666666667
    parameter combination 0.004833333333333334 5.333333333333334
    parameter combination 0.004833333333333334 6.5
    parameter combination 0.004833333333333334 7.666666666666667
    parameter combination 0.004833333333333334 8.833333333333334
    parameter combination 0.004833333333333334 10.0
    parameter combination 0.004833333333333334 11.0
    parameter combination 0.004833333333333334 20.75
    parameter combination 0.004833333333333334 30.5
    parameter combination 0.004833333333333334 40.25
    parameter combination 0.004833333333333334 50.0
    parameter combination 0.005916666666666667 3.0
    parameter combination 0.005916666666666667 4.166666666666667
    parameter combination 0.005916666666666667 5.333333333333334
    parameter combination 0.005916666666666667 6.5
    parameter combination 0.005916666666666667 7.666666666666667
    parameter combination 0.005916666666666667 8.833333333333334
    parameter combination 0.005916666666666667 10.0
    parameter combination 0.005916666666666667 11.0
    parameter combination 0.005916666666666667 20.75
    parameter combination 0.005916666666666667 30.5
    parameter combination 0.005916666666666667 40.25
    parameter combination 0.005916666666666667 50.0
    parameter combination 0.007 3.0
    parameter combination 0.007 4.166666666666667
    parameter combination 0.007 5.333333333333334
    parameter combination 0.007 6.5
    parameter combination 0.007 7.666666666666667
    parameter combination 0.007 8.833333333333334
    parameter combination 0.007 10.0
    parameter combination 0.007 11.0
    parameter combination 0.007 20.75
    parameter combination 0.007 30.5
    parameter combination 0.007 40.25
    parameter combination 0.007 50.0
    parameter combination 0.0075 3.0
    parameter combination 0.0075 4.166666666666667
    parameter combination 0.0075 5.333333333333334
    parameter combination 0.0075 6.5
    parameter combination 0.0075 7.666666666666667
    parameter combination 0.0075 8.833333333333334
    parameter combination 0.0075 10.0
    parameter combination 0.0075 11.0
    parameter combination 0.0075 20.75
    parameter combination 0.0075 30.5
    parameter combination 0.0075 40.25
    parameter combination 0.0075 50.0
    parameter combination 0.011875 3.0
    parameter combination 0.011875 4.166666666666667
    parameter combination 0.011875 5.333333333333334
    parameter combination 0.011875 6.5
    parameter combination 0.011875 7.666666666666667
    parameter combination 0.011875 8.833333333333334
    parameter combination 0.011875 10.0
    parameter combination 0.011875 11.0
    parameter combination 0.011875 20.75
    parameter combination 0.011875 30.5
    parameter combination 0.011875 40.25
    parameter combination 0.011875 50.0
    parameter combination 0.01625 3.0
    parameter combination 0.01625 4.166666666666667
    parameter combination 0.01625 5.333333333333334
    parameter combination 0.01625 6.5
    parameter combination 0.01625 7.666666666666667
    parameter combination 0.01625 8.833333333333334
    parameter combination 0.01625 10.0
    parameter combination 0.01625 11.0
    parameter combination 0.01625 20.75
    parameter combination 0.01625 30.5
    parameter combination 0.01625 40.25
    parameter combination 0.01625 50.0
    parameter combination 0.020625 3.0
    parameter combination 0.020625 4.166666666666667
    parameter combination 0.020625 5.333333333333334
    parameter combination 0.020625 6.5
    parameter combination 0.020625 7.666666666666667
    parameter combination 0.020625 8.833333333333334
    parameter combination 0.020625 10.0
    parameter combination 0.020625 11.0
    parameter combination 0.020625 20.75
    parameter combination 0.020625 30.5
    parameter combination 0.020625 40.25
    parameter combination 0.020625 50.0
    parameter combination 0.025 3.0
    parameter combination 0.025 4.166666666666667
    parameter combination 0.025 5.333333333333334
    parameter combination 0.025 6.5
    parameter combination 0.025 7.666666666666667
    parameter combination 0.025 8.833333333333334
    parameter combination 0.025 10.0
    parameter combination 0.025 11.0
    parameter combination 0.025 20.75
    parameter combination 0.025 30.5
    parameter combination 0.025 40.25
    parameter combination 0.025 50.0

```python
np.save('F_u_grid_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.npy', F_u_grid)
np.save('F_s_grid_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.npy', F_s_grid)
np.save('rho_grid_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.npy', rho_grid)
np.save('sl_grid_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.npy', sl_grid)
save_data_vars_in_json(f_ck, mc, dp, 'data_carbon_EC2_eq2_tension_E' + str(E_t) + '_ft_1500_c' + str(f_ck) + '.json')
```

```python
# Concrete C30/37

f_ck = 30

b = 1000
h = 300
d = 0.9 * h

E = EC2.get_E_cm(f_ck)

mc = MKappa(low_kappa = 0,
            high_kappa = 0.00004, # high_kappa = 0.00009

            # Cantilever
#             low_kappa = -0.00005,
#             high_kappa =  0,
           )
mc.cs_design.matrix='EC2 with plateau'
mc.cs_design.matrix_.trait_set(
        E_cc=E,
        E_ct=E,
        eps_cr=0.000132 * f_ck/30,
        eps_cy=EC2.get_eps_c2(f_ck),
        eps_cu=EC2.get_eps_cu2(f_ck),
        eps_tu=0.002,
        mu=0.0,
    f_cd = f_ck*0.85/1.5,
#     factor = 0.85/1.5
    )

mc.cross_section_shape_.B = b
mc.cross_section_shape_.H = h

rho = 0.01
# A_s=6 * np.pi * (32/2)**2
A_s= 0.5 * rho * b * d

bl1 = ReinfLayer(name='layer1', z=h-d, A=A_s, matmod='carbon')
bl1.matmod_.trait_set(E=100000, f_t=1000, factor = 1./1.5)
mc.cross_section_layout.add_layer(bl1)

dp = DeflectionProfile(mc=mc)
dp.beam_design.beam_conf_name = BoundaryConfig.SIMPLE_BEAM_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.FIXED_AND_ROLLER_SUPPORT_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.FIXED_SUPPORT_DIST_LOAD
# dp.beam_design.beam_conf_name = BoundaryConfig.CANTILEVER_DIST_LOAD

dp.F_scale = 1
dp.beam_design.L = 1000
# dp.beam_design.L = 10 * d # l/d = 10
# dp.interact()
```

```python
F_u_grid, F_s_grid, rho_grid, sl_grid = get_Fu_and_Fs(dp, 4, rho_range=(0.0005, 0.025), slenderness_range=(3, 50))
# F_u_grid, F_s_grid, rho_grid, sl_grid = get_Fu_and_Fs(dp, 7, rho_range=(0.001, 0.025), slenderness_range=(3, 50),
#                                                       upper_reinforcement = True)
```

```python
_, ax = plt.subplots(1,1)

z = F_u_grid / F_s_grid - 1./0.59

CS = ax.contour(rho_grid, sl_grid, z, levels=[0])
ax.clabel(CS, inline=1, fontsize=10)
ax.set_ylabel('L/d')
ax.set_xlabel(r'$\rho$ %')
ax.set_ylim(0, 35)
ax.set_xlim(0.0, 0.025)

# Draw C30 curve
rho_range = np.arange(0.0025, 0.025, 0.0001)
f_ck = 30
slenderness = get_slenderness(rho_range, 0,  1, f_ck)
ax.plot(rho_range, slenderness, c=np.random.rand(3,))
```

```python
# Checking stress status for different rho, sl

# Initializing dp
# ---------------------------------------------------------------
# Concrete C30/37

f_ck = 30

b = 1000
h = 300
d = 0.9 * h

E = EC2.get_E_cm(f_ck)

mc = MKappa(low_kappa = 0, high_kappa = 0.00007) # high_kappa = 0.00009
mc.cs_design.matrix='EC2 with plateau'
mc.cs_design.matrix_.trait_set(
        E_cc=E,
        E_ct=E,
        eps_cr=0.000132 * f_ck/30,
        eps_cy=EC2.get_eps_c2(f_ck),
        eps_cu=EC2.get_eps_cu2(f_ck),
        eps_tu=0.002,
        mu=0.0,
    f_cd = f_ck*0.85/1.5,
#     factor = 0.85/1.5
    )

mc.cross_section_shape_.B = b
mc.cross_section_shape_.H = h

# Steel
# bl1 = ReinfLayer(name='layer1', z=h-d, matmod='steel')
# bl1.matmod_.trait_set(E_s=200000, f_sy=500, factor = 1./1.15)

# Carbon
bl1 = ReinfLayer(name='layer1', z=h-d, matmod='carbon')
bl1.matmod_.trait_set(E=100000, f_t=1500, factor = 1./1.5)

mc.cross_section_layout.add_layer(bl1)

dp = DeflectionProfile(mc=mc)
dp.beam_design.beam_conf_name = BoundaryConfig.SIMPLE_BEAM_DIST_LOAD
# ---------------------------------------------------------------
# Setting rho and sl:

rho = 0.0013
sl = 10

dp.mc.cross_section_layout.items[0].A = rho * b * d
dp.beam_design.L = sl * d

dp.mc.state_changed = True

F_data, w_data = dp.get_Fw()
w_s = dp.beam_design.L / 250
F_u = max(F_data)
F_s = np.interp(w_s, w_data, F_data, right=F_u*2)

print('w_limit = ',w_s)
print('q_ULT = ', F_u)
print('q_SLS = ', 0.59 * F_u)
print('q_limit = ',F_s)

dp.interact()
```

```python
f_ck = 200
np.save('F_u_grid_carbon_EC2_eq2_tension_c' + str(f_ck) + '.npy', F_u_grid)
np.save('F_s_grid_carbon_EC2_eq2_tension_c' + str(f_ck) + '.npy', F_s_grid)
np.save('rho_grid_carbon_EC2_eq2_tension_c' + str(f_ck) + '.npy', rho_grid)
np.save('sl_grid_carbon_EC2_eq2_tension_c' + str(f_ck) + '.npy', sl_grid)
save_data_vars_in_json(f_ck, mc, dp, 'data_carbon_EC2_eq2_tension_c' + str(f_ck) + '.json')
```

### Plotting all curves

```python
f_cks = [30]
F_u_grids = []
F_s_grids = []
rho_grids = []
sl_grids = []
reinforcement = 'carbon'
for f_ck in f_cks:
    f_ck = str(f_ck)
#     F_u_grids.append(np.load('F_u_grid_' + reinforcement + '_EC2_eq2_tension_c' + f_ck + '.npy'))
#     F_s_grids.append(np.load('F_s_grid_' + reinforcement + '_EC2_eq2_tension_c' + f_ck + '.npy'))
#     rho_grids.append(np.load('rho_grid_' + reinforcement + '_EC2_eq2_tension_c' + f_ck + '.npy'))
#     sl_grids.append(np.load('sl_grid_' + reinforcement + '_EC2_eq2_tension_c' + f_ck + '.npy'))

    F_u_grids.append(np.load('F_u_grid_' + reinforcement + '_EC2_eq2_tension_E230_ft_1500_c' + f_ck + '.npy'))
    F_s_grids.append(np.load('F_s_grid_' + reinforcement + '_EC2_eq2_tension_E230_ft_1500_c' + f_ck + '.npy'))
    rho_grids.append(np.load('rho_grid_' + reinforcement + '_EC2_eq2_tension_E230_ft_1500_c' + f_ck + '.npy'))
    sl_grids.append(np.load('sl_grid_' + reinforcement + '_EC2_eq2_tension_E230_ft_1500_c' + f_ck + '.npy'))
```

```python
_, ax = plt.subplots(1,1)

ax.set_ylabel('L/d')
ax.set_xlabel(r'$\rho$ %')
ax.set_ylim(0, 35)
ax.set_xlim(0.0, 0.025)

for f_ck, F_u_grid, F_s_grid, rho_grid, sl_grid in zip(f_cks, F_u_grids, F_s_grids, rho_grids, sl_grids):
    z = F_u_grid / F_s_grid - 1./0.59
    CS = ax.contour(rho_grid, sl_grid, z, levels=[0], colors=[np.random.rand(3,)])
    CS.collections[0].set_label('C' + str(f_ck))
#     ax.clabel(CS, inline=1, fontsize=10)

    BeamSLSCurve().plot_steel_sls_curves(ax=ax, f_cks=[30], axes_start_from_zero=True)

ax.legend()

```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

    <matplotlib.legend.Legend at 0x18331ecdd30>

## Testing for Steel
#### 1. Verification with Gribniak et al. (2012), https://doi.org/10.1016/j.engstruct.2012.04.032, Steel reinforcement

```python
mc_params = dict(
            reinforcement_type= ReinforcementType.STEEL,
            apply_material_safety_factors = True,

            n_m=100,
            n_kappa = 100,
            kappa_slider = 0.000001,
            low_kappa = 0,
            high_kappa =  0.00002, # 12e-6,
            eps_tu = 0.002508,
            E_cc = 34000,
            E_ct = 34000,
            eps_cr = 0.000132,
            eps_cy = 0.0015312,
            mu = 0.0,            # I changed this to zero
            eps_cu = 0.003498)

mc = MKappa(**mc_params)

# mc.cross_section_shape.B =284
# mc.cross_section_shape.H =298
# mc.cross_section_layout.reinforcement = [Reinforcement(eps_sy=560/203000, E=203000, z=26.82, A=231.04536)]

b = 300
h = 300
mc.cross_section_shape.B = b
mc.cross_section_shape.H = h
rho = 0.01

mc.cross_section_layout.reinforcement = [Reinforcement(eps_sy=(500/1.15)/203000, E=203000, z=25, A=rho * b * (h - 25))]

dp = DeflectionProfile(mc=mc)
dp.beam_design.beam_conf_name = BoundaryConfig.FOUR_PB
dp.beam_design.L = 3000
dp.beam_design.beam_conf_name.first_load_distance = dp.beam_design.L/3

bu.InteractiveWindow([dp,
                      dp.beam_design,
                      dp.mc,
                      dp.mc.cs_design.cross_section_shape,
                      dp.mc.cs_design.cross_section_layout], tight_layout=True).interact()
```

```python
F_u_grid, F_s_grid = get_Fu_and_Fs(dp, 4)
```

```python
np.save('F_u_grid_steel_with_material_factors_applied_on_material_law.npy', F_u_grid)
np.save('F_s_grid_steel_with_material_factors_applied_on_material_law.npy', F_s_grid)
# np.save('F_u_grid_steel.npy', F_u_grid)
# np.save('F_s_grid_steel.npy', F_s_grid)
```

```python
# This cell is for loading results later if needed
rho_range = np.linspace(0.0025, 0.025, 4)
slenderness_range = np.linspace(10, 35, 4)
rho_grid, sl_grid = np.meshgrid(rho_range, slenderness_range)
F_u_grid = np.load('F_u_grid_steel_with_1_material_param.npy') # ('F_u_grid_steel_with.npy')
F_s_grid = np.load('F_s_grid_steel_with_1_material_param.npy') # ('F_s_grid_steel_with.npy')
```

```python
_, ax = plt.subplots(1,1)
# z = F_u_grid - F_s_grid
# z = F_u_grid / F_s_grid - 1.5

# z = 0.51 *  F_u_grid - F_s_grid

z = F_u_grid / F_s_grid - 1./0.51

CS = ax.contour(rho_grid, sl_grid, z)
ax.clabel(CS, inline=1, fontsize=10)
ax.set_ylabel('L/d')
ax.set_xlabel(r'$\rho$ %')
ax.set_ylim(10, 35)
ax.set_xlim(0.0025, 0.025)

# Draw C50 curve
rho_range = np.arange(0.0025, 0.025, 0.0001)
f_ck = 50
slenderness = []
for rho in rho_range:
    slenderness.append(get_slenderness(rho, 0,  1, f_ck))
ax.plot(rho_range, slenderness)
```

```python
# from mpl_toolkits import mplot3d
# # %matplotlib widget
# # import numpy as np
# import matplotlib.pyplot as plt
# # excel_file = join('./verification_data','assessmentruledata.xlsx')
# # df = data = pd.read_excel(excel_file)

# # fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8 , 8))

# # X, Y = np.meshgrid(df['sl'], df['rho'])
# # FU, FS = np.meshgrid(df['F_u'], df['F_s'])
# # WU, WS = np.meshgrid(df['w_u'], df['w_s'])

# # Z = np.array(FU / FS)

# # fig = plt.figure()
# ax = plt.axes(projection='3d')
# ax.contour3D(sl_grid, rho_grid, F_u_grid/F_s_grid - 1, 500, cmap='rainbow')
# ax.set_xlabel('L/d')
# ax.set_ylabel(r'$\rho$')
# ax.set_zlabel('Fu/Fs')
# ax.set_title('Deflection Assessment Rule')
# plt.show()
```

## Testing for Carbon
#### 1. Verification with Heid et al. (2020), [link](https://www.heringinternational.com/fileadmin/media/archive1/downloads/architekturbeton/Bauingenieur_06_2020_S.210-219.pdf), Carbon reinforcement

```python
# Testing for Carbon
# HB-SU-0
E = 39500

# from bending test from σ_c,Riss
eps_cr = ((6.365 + 6.306 + 5.635 + 5.627 + 5.452 + 5.627)/6)/E

mc = MKappa(reinforcement_type= ReinforcementType.CARBON,
            apply_material_safety_factors = True,

            low_kappa = 0,
            high_kappa = 0.0007,
            E_cc = E,
            E_ct = E,
            eps_cr = eps_cr, # 0.000163
            eps_tu = 9.02/E, # beta_tu = 20
            eps_cy = 72/E, # omega = 10.0
            eps_cu = (72/E)*2, # lambda_cu = 20
            mu = 0.0)

mc.cross_section_shape.B = 45
mc.cross_section_shape.H = 30
mc.cross_section_layout.reinforcement = [Reinforcement(eps_sy=2712/240000, E=240000, z=8, A=3.62),
                                         Reinforcement(eps_sy=2712/240000, E=240000, z=22, A=3.62),]

dp = DeflectionProfile(mc=mc)

dp.beam_design.beam_conf_name = BoundaryConfig.FOUR_PB
dp.beam_design.beam_conf_name.first_load_distance = 320
dp.beam_design.L = 840
```

```python
F_u_grid, F_s_grid, rho_grid, sl_grid = get_Fu_and_Fs(dp, 5)
```

```python
np.save('F_u_grid_carbon.npy', F_u_grid)
np.save('F_s_grid_carbon.npy', F_s_grid)
```

```python
# This cell is for loading results later if needed
rho_range = np.linspace(0.0025, 0.025, 6)
slenderness_range = np.linspace(10, 35, 6)
rho_grid, sl_grid = np.meshgrid(rho_range, slenderness_range)
F_u_grid = np.load('F_u_grid_carbon.npy')
F_s_grid = np.load('F_s_grid_carbon.npy')
```

```python
_, ax = plt.subplots(1,1)
# z = F_u_grid/F_s_grid -1
z = (F_s_grid + F_u_grid) / F_s_grid - 1.35

CS = ax.contour(rho_grid, sl_grid, z )
ax.clabel(CS, inline=1, fontsize=10)
ax.set_ylabel('L/d')
ax.set_xlabel(r'$\rho$ %')
ax.set_ylim(10, 35)
ax.set_xlim(0.0025, 0.025)

# Draw C50 curve
rho_range = np.arange(0, 0.05, 0.0001)
f_ck = 70
slenderness = []
for rho in rho_range:
    slenderness.append(get_slenderness(rho, 0,  1, f_ck))
ax.plot(rho_range, slenderness)
```

```python
# from mpl_toolkits import mplot3d
%matplotlib widget
import numpy as np
import matplotlib.pyplot as plt
excel_file = join('./verification_data','assessmentruledata.xlsx')
df = data = pd.read_excel(excel_file)

fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8 , 8))

X, Y = np.meshgrid(df['sl'], df['rho'])
FU, FS = np.meshgrid(df['F_u'], df['F_s'])
WU, WS = np.meshgrid(df['w_u'], df['w_s'])

Z = np.array(FU / FS)

# fig = plt.figure()
ax = plt.axes(projection='3d')
ax.contour3D(X, Y, Z, 500, cmap='rainbow')
ax.set_xlabel('L/d')
ax.set_ylabel(r'$\rho$')
ax.set_zlabel('Fu/Fs')
ax.set_title('Deflection Assessment Rule')
plt.show()
```


# Solving affine rank minimization problem using `NExOS.jl`
**Shuvomoy Das Gupta**

### Background

The problem in consideration can be written as:

$$
\begin{equation}
\begin{array}{ll}
& \textrm{minimize} & \left\Vert \mathcal{A}(X)-b\right\Vert _{2}^{2}\\
& \textrm{subject to} & \mathop{{\bf rank}}(X)\leq r\\
& & \|X\|_{2}\leq M\\
& & X\in\mathbf{R}^{m\times n},
\end{array}
\end{equation}
$$

 where $X\in\mathbf{R}^{m\times n}$ is the decision variable, $b\in\mathbf{R}^{k}$ is a noisy measurement data, and $\mathcal{A}:\mathbf{R}^{m\times n}\to\mathbf{R}^{k}$ is a linear map. The affine map $\mathcal{A}$ can be determined by $k$ matrices $A_{1},\ldots,A_{k}$ in $\mathbf{R}^{m\times n}$ such that

$$
\mathcal{A}(X)=(\mathbf{tr}(A_{1}^{T}X),\ldots,\mathbf{tr}(A_{k}^{T}X)).
$$

### Data Generation

First we generate our data for this example.

```julia
using Random, NExOS, ProximalOperators, LinearAlgebra

Random.seed!(1234)

m = 10

n = 2*m

M = 1.0

k = convert(Int64, m*n/20)
```

    10

Here, $k$ is the number of components in output of the affine operator $\mathcal{A}$, i.e., for any matrix $X \in \mathbf{X}$, we have $\mathcal{A}(X) \in \mathbf{R}^k$.

```julia
r = convert(Int64,round(m*.35))
```

    4

Here, $r$ corresponds to the rank of the matrix $\mathbf{rank}(X) <= r$.

The affine operator $\mathcal{A}$ is passed as a $k \times mn$ matrix $\bar{A}$, so that when it acts on $\mathbf{vec}(X)$ we have $\bar{A} (\mathbf{vec}(X)) = \mathcal{A}(X)$.

```julia
barA = randn(k, m*n)
```

    10×200 Array{Float64,2}:
      0.867347  -0.560501    1.56417    …   0.384485    0.685573   -0.858928
     -0.901744  -0.0192918  -1.39674        0.315489    1.21533     1.12188
     -0.494479   0.128064    1.1055        -0.382068    1.29772    -2.45236
     -0.902914   1.85278    -1.10673        0.691941   -1.71088    -2.30555
      0.864401  -0.827763   -3.21136        0.0473293  -0.747106    1.54823
      2.21188    0.110096   -0.0740145  …  -0.455901    0.0330671  -0.297855
      0.532813  -0.251176    0.150976       0.100961    2.05177     1.58331
     -0.271735   0.369714    0.769278      -1.12375     1.05237     0.562472
      0.502334   0.0721164  -0.310153      -0.579068    0.430384    0.85342
     -0.516984  -1.50343    -0.602707      -0.493044    0.211279   -0.321671

The vector $b$ is the observed output.

```julia
b = randn(k)
```

    10-element Array{Float64,1}:
     -0.4667377936599136
     -0.17047970356351902
      0.8292034377961193
     -0.4500585937344793
     -1.3038856208344294
      0.5869555339344937
      0.17548586215288262
     -0.2760027307659979
     -0.2631151380019278
     -1.1348769546238908

Let us chose an initial point.

```julia
Z0 = zeros(m,n);
```

Create the objective function.

```julia
f = LeastSquaresOverMatrix(barA, b, 1.0, iterative = true);
```

Let us create the constraint set.

```julia
C = RankSet(M, r)
```

    RankSet{Float64,Int64}(1.0, 4)

Create the `settings` file, note that we are working with the `:safe` rule for updating the proximal parameter $\gamma$. Also, the solver output is turned off here, we could turn it on by settings `verbose=true`.

```julia
settings = Settings(μ_max = 2, μ_min = 1e-8, μ_mult_fact = 0.5, verbose = false, freq = 500, γ_updt_rule = :safe)

problem = Problem(f, C, settings.β, Z0)
```

    Problem{ProximalOperators.LeastSquaresIterative{1,Float64,Float64,Array{Float64,2},Array{Float64,1},ProximalOperators.AAc},RankSet{Float64,Int64},Float64,Array{Float64,2}}(description : Least squares penalty
    domain      : n/a
    expression  : n/a
    parameters  : n/a, RankSet{Float64,Int64}(1.0, 4), 1.0e-10, [0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0])

Solve the problem!

```julia
state_final = solve!(problem, settings)
```

    State{Array{Float64,2},Int64,Float64}([0.009640555740470996 -0.0039049066980724172 … 0.0032744246264381385 -0.011401924226827814; 0.017039021787684776 -0.006486467044692794 … -0.006723985185855889 0.011346398802052203; … ; -0.0031155465177579336 0.012103228512837969 … -0.0030847566736254646 0.006789864137943311; 0.0037643322883237993 0.011064929502908967 … 0.003141509734574783 -0.006391901052935991], [0.009640555310037102 -0.003904906555286084 … 0.0032744247618116023 -0.011401924380723396; 0.017039022426592925 -0.006486466874719959 … -0.006723985197637846 0.011346398967171602; … ; -0.0031155462203965234 0.012103228542114463 … -0.0030847567074370006 0.006789863932342989; 0.0037643323913579916 0.011064929307709616 … 0.0031415097913574316 -0.00639190102063273], [0.009640555709940084 -0.003904906817854433 … 0.003274424252976659 -0.011401923965769611; 0.017039021755258916 -0.006486467029139976 … -0.006723985147301422 0.011346398638919817; … ; -0.0031155461667072546 0.012103228769906214 … -0.003084756747081626 0.006789864408033303; 0.0037643322644109536 0.011064929211436649 … 0.003141509432661576 -0.006391901114060031], 1, 9.267131556491698e-10, 2.192690473634684e-15, 7.450580596923828e-9, 0.002154434690031884)

**Observe the output.** Let us check the objective value first, we want it to be small.

```julia
f(state_final.x)
```

    2.1913058045348726e-14

Finally, we want to check what if the best state found by our algorithm has converged to a locally optimal solution, which is checked by testing whether the best state has reached the desired fixed point gap and feasibility gap.

```julia
log10(state_final.fxd_pnt_gap) <= -4

log10(state_final.fsblt_gap) <= -4
```

    true

tutorials/.ipynb_checkpoints/Affine rank minimization using NExOS.jl-checkpoint.ipynb


Markdown macros

$\newcommand{\I}{\mathrm{i}}
\newcommand{\E}{\mathrm{e}}
\newcommand{\D}{\mathop{}\!\mathrm{d}}
\newcommand{\bra}[1]{\langle{#1}|}
\newcommand{\ket}[1]{|{#1}\rangle}
\newcommand{\braket}[1]{\langle{#1}\rangle}
\newcommand{\bm}[1]{\boldsymbol{#1}}$

# Imports

```python
# numpy
import numpy as np
```

```python
# matplotlib
import matplotlib
import matplotlib.pylab as plt
```

```python
%matplotlib inline
```

```python
# sympy
import sympy as sy
sy.init_printing(use_unicode = True)
```

Bibliographie:

Voir,

* Landau & Lifshitz "Mécanique"
* Goldstein "Classical Mechanics"

# Angles d'Euler

## Matrices de changement de coordonnées

### Rotation d'un repère

La rotation des *axes de coordonnées* $(\bm e_X,\bm e_Y,\bm e_Z)$ d'un angle $\phi$ autour de $\bm n$ est donnée par la matrice de changement de coordonnées:
$$ C_{\bm n}(\phi) = R_{\bm n}(\phi)^{\mathrm{T}} = R_{\bm n}(-\phi)$$
qui est en fait la transposée de la matrice de rotation $R_{\bm n}(\phi)$.

Les matrices $C$ pertmettent de passer du repère fixe au mobile, par exemple
$$\begin{pmatrix} \bm e_x \\ \bm e_y \\ \bm e_z \end{pmatrix} = C_{\bm n}(\phi) \begin{pmatrix} \bm e_X \\ \bm e_Y \\ \bm e_Z \end{pmatrix}.$$

Les rotations autour des axes $X$, $Y$ et $Z$ sont:
\begin{align*} C_X(\phi) &= \begin{pmatrix} 1 & 0 & 0 \\ 0 & \cos \phi & \sin \phi \\ 0 & -\sin \phi & \cos \phi \end{pmatrix}\\
C_Y(\phi) &= \begin{pmatrix} \cos \phi & 0 & -\sin \phi \\ 0 & 1 & 0 \\ \sin \phi & 1 & \cos \phi\end{pmatrix}\\
C_Z(\phi) &= \begin{pmatrix} \cos \phi & \sin \phi & 0 \\ -\sin \phi & \cos \phi & 0 \\ 0 & 0 & 1 \end{pmatrix}
\end{align*}

### Remarque: rotation d'un vecteur

Notez la différence entre $R$ et $C$: la matrice de rotation s'applique à un vecteur pour en obtenir un autre, tandis que la matrice de changement de coordonnées s'applique aux axes d'un repère pour obtenir un nouveau repère.

En effet, pare exemple la rotation d'un *vecteur* $\bm r = (x,y,z)$ d'un angle $\phi$ autour de l'axe $Z$ (disons) est donnée par la matrice:
$$R_{Z}(\phi) = \begin{pmatrix} \cos \phi & -\sin \phi & 0 \\ \sin \phi & \cos \phi & 0 \\ 0 & 0 & 1 \end{pmatrix}$$
On a donc
$$ \bm r' = R_Z(\phi) \bm r$$

Par contre, le vecteur $\bm r = (X,Y,Z)$ dans le repère $(\bm e_X,\bm e_Y,\bm e_Z)$, s'écrit dans les coordonnées d'un autre repère $\bm r' = (\bm e_x,\bm e_y,\bm e_z)$, après une rotation d'axe $\bm e_Z$ et d'angle $\phi$, comme:
$$ \bm r' = C_Z(\phi) \bm r$$
avec
$$C_{Z}(\phi) = \begin{pmatrix} \cos \phi & \sin \phi & 0 \\ -\sin \phi & \cos \phi & 0 \\ 0 & 0 & 1 \end{pmatrix}$$
La différence par rapport à l'équation précédente, c'est qu'ici $\bm r$ et $\bm r'$ sont le *même* vecteur, mais exprimé par des coordonnées différentes, tandis qu'auparavant $\bm r$ et $\bm r'$ étaient différents, mais dont le repère était le même. Notez toutefois que les deux transformations sont données par des matrices transposées l'une de l'autre, $C_Z(\phi) = R_Z(-\phi)$.

## Angles $(\psi,\theta,\phi)$

Ils décrivent le passage du repère fixe $(\bm e_X,\bm e_Y,\bm e_Z)$ au repère mobile $(\bm e_x,\bm e_y,\bm e_z)$
Les angles d'Euler sont

* $\psi$, l'angle de précesion
* $\theta$, l'angle de nutation
* et $\phi$, l'angle de rotation (propre)

$\psi$ est une rotation autour de l'axe $\bm e_Z$ du repère fixe; $\theta$ est un rotation autour de la ligne nodale $\bm N$; enfin, $\phi$ est une rotation autour de l'axe $e_z$ mobile (ou fixé au solide). Dans la littérature physique vous allez souvent trouver l'échange de $\psi$ et $\phi$, dans la notation des angles.

La transformation du reprère fixe au repère mobile, décrites par ces trois angles, s'obtient par une serie de trois rotations:

1. On part $(\bm e_X,\bm e_Y,\bm e_Z)$, et on fait une rotation d'angle $\psi$ autour de $Z$ pour obtenir le repère $(\bm N,\bm e_{y'},\bm e_Z)$ ($N$ et $y'$ sont les images de $X$ et $Y$)

2. On fait une rotation de $\theta$ autour de $\bm e_N$ (la ligne nodale), on obtient un noveau reprère $(\bm e_N,\bm e_{y''},\bm e_z)$

3. Enfin, on obtient le repère mobile $(\bm e_x,\bm e_y,\bm e_z)$, après une rotation d'angle $\phi$ autour de $\bm e_z$

En utilisant les matrices de rotation successivement, on a simplement:
$$ \begin{pmatrix} \bm e_x \\ \bm e_y \\ \bm e_z \end{pmatrix} = C_Z(\phi) C_X(\theta) C_Z(\psi) \begin{pmatrix} \bm e_X \\ \bm e_Y \\ \bm e_Z \end{pmatrix}$$

Ou en forme explicite
$$C(\psi, \theta, \phi) = \begin{pmatrix}- \cos\theta  \sin\psi \sin\phi + \cos\psi \cos\phi &  \cos\theta \cos\psi \sin\phi + \sin\psi \cos\phi & \sin\theta \sin\phi \\ -\cos\theta \sin\psi \cos\phi - \cos\psi \sin\phi & \cos\theta \cos\psi \cos\phi - \sin\psi \sin\phi & \sin\theta \cos\phi \\  \sin\theta \sin\psi  & - \sin\theta \cos\psi & \cos\theta \end{pmatrix}$$

Notez que la transformation inverse est donnée par:
$$C(\psi, \theta, \phi)^\mathrm{T} = R_Z(\psi) R_X(\theta) R_Z(\phi)$$
(on a utilisé $R_n(-\alpha)^\mathrm{T} = R_n(\alpha)$ pour $n$ et $\alpha$ arbitraires), elle permet de passer du repère mobile au repère fixe.

```python
# matrices de changement de repère

phi, theta, psi = sy.symbols("phi, theta, psi", real=True)

def C_x(phi):
    c = sy.cos(phi)
    s = sy.sin(phi)
    return sy.Matrix([[1,  0, 0],
                  [0, c, s],
                  [0, -s,  c]])
def C_y(phi):
    c = sy.cos(phi)
    s = sy.sin(phi)
    return sy.Matrix([[ c, 0, -s],
                  [ 0, 1, 0],
                  [s, 0, c]])

def C_z(phi):
    c = sy.cos(phi)
    s = sy.sin(phi)
    return sy.Matrix([[c, s, 0],
                  [-s,  c, 0],
                  [0,  0, 1]])
```

```python
# Changement de base (repère)
C = C_z(phi)*C_x(theta)*C_z(psi)
C
```

```python
# Rotation: matrice inverse (transposée de C)
CT = C_z(-psi)*C_x(-theta)*C_z(-phi)
CT
```

# Vitesse de rotation

La vitesse angulaire $\bm \Omega$ s'écrit facilement en utilisant les axes de rotation respectifs:
$$\bm \Omega\big|_{(Z,N,z)} = \dot{\psi} \bm e_Z + \dot{\theta} \bm e_N + \dot{\phi} \bm e_z$$
où nous notons les noms de coordonnées (remarquez que ces axes ne forment pas un repère orthogonal).

Pour avoir son expression explicite dans le repère mobile il suffit de faire les rotations nécessaires pour passer d'un repère à un autre. Le vecteur $\bm e_Z = (0,0,1)$ devient dans la base mobile:
$$ C(\psi, \theta, \phi) \bm e_Z = C_Z(\phi)C_X(\theta) \begin{pmatrix}   \sin\theta \sin\phi \\ \sin\theta \cos\phi \\ \cos\theta \end{pmatrix}$$
($Z$ est tranformé par les trois rotations des axes, notez cependant que $C_Z(\psi)$ agit comme l'identité sur l'axe $Z$) et le vecteur $\bm e_N =(1,0,0)$ devient,
$$C_Z(\phi)C_X(\theta) \bm e_N = C_Z(\phi) \bm e_N = \begin{pmatrix} \cos\phi \\ - \sin\phi \\ 0 \end{pmatrix}$$
(il est transformé lors de l'étape 3 dans la définition des angles d'Euler, la rotation d'axe $X$ actue comme l'identité sur $N$).

En remplaçant dans $\bm \Omega$, on transforme ses "coordonnées" $(Z,N,z)$, en coordonnées du repère mobile:
$$\bm \Omega\big|_{(x,y,z)} = \dot{\psi} C_Z(\phi)C_X(\theta) \bm e_Z + \dot{\theta} C_Z(\phi)\bm e_N + \dot{\phi} \bm e_z$$
on obtient finalement l'expression de la vitesse angulaire dans le repère du solide:
$$\bm \Omega =  \begin{pmatrix} \dot{\psi} \sin \theta \sin \phi + \dot{\theta} \cos\phi \\ \dot{\psi} \sin\theta \cos\phi - \dot{\theta} \sin\phi \\ \dot{\psi}\cos\theta + \dot{\phi} \end{pmatrix}$$

```python
 C_z(phi)*C_x(theta)*sy.Matrix([0,0,1])
```

```python
C_z(phi)*sy.Matrix([1,0,0])
```

# Brouillon

```python
print(sy.latex(C))
```

    \left[\begin{matrix}- \sin{\left(\phi \right)} \sin{\left(\psi \right)} \cos{\left(\theta \right)} + \cos{\left(\phi \right)} \cos{\left(\psi \right)} & \sin{\left(\phi \right)} \cos{\left(\psi \right)} \cos{\left(\theta \right)} + \sin{\left(\psi \right)} \cos{\left(\phi \right)} & \sin{\left(\phi \right)} \sin{\left(\theta \right)}\\- \sin{\left(\phi \right)} \cos{\left(\psi \right)} - \sin{\left(\psi \right)} \cos{\left(\phi \right)} \cos{\left(\theta \right)} & - \sin{\left(\phi \right)} \sin{\left(\psi \right)} + \cos{\left(\phi \right)} \cos{\left(\psi \right)} \cos{\left(\theta \right)} & \sin{\left(\theta \right)} \cos{\left(\phi \right)}\\\sin{\left(\psi \right)} \sin{\left(\theta \right)} & - \sin{\left(\theta \right)} \cos{\left(\psi \right)} & \cos{\left(\theta \right)}\end{matrix}\right]

```python

```


## Imports

```python
import numpy as np
import pylab as plt
from matplotlib import cm
import seaborn as sns
from tqdm import tqdm
```

## System

> NLSE differential equation

\begin{equation}
\partial_z A + \frac{\alpha}{2}A+\frac{i}{2}\beta_2 \partial_T^2 A = i \gamma \| A\|^2 A
\end{equation}

For now, let's suppose $\alpha=0$

\begin{equation}
\partial_z A = (\hat{D}+\hat{N})A
\end{equation}

with $\hat{D}=-\frac{i}{2}\beta_2 \partial_T^2$ and $\hat{N}=i \gamma \| A\|^2$

The operator $\hat{D}$ will be computed in the fourier domain such that $\hat{D}(\omega)=\frac{i}{2}\omega^2\beta_2$

Finally we have at first glance :

\begin{equation}
A(z+h, T)=e^{h\hat{N}} F^{-1} \cdot e^{h\hat{D}(\omega)} \cdot F \cdot A(z,t)
\end{equation}

Where $F$ is the Fourier operator, $h$ is the propagation step (of a few meters) and $\beta_2=-D\lambda^2/2\pi c$ wich is related to the optical fiber chromatic dispersion.

> Source : doi:10.1109/ICEE.2007.4287333

```python
# Constants

c=3e8 # light celerity
l0 = 1.55e-6 # wavelength
nm = 1e-9 # nanometer
ns = 1e-9
km = 5e3 #kilometer
ps = 1e-12 #picosecond
D = 17*ps/nm/km # Dispersion
b2 = -D*l0**2/(2*np.pi*c) # group velocity dispersion (2nd order)
w0 = 1*2*np.pi*1e10
# Parameters

alpha=0.0001 # Losses
gamma=0.78e3 # Nonlinear factor https://ieeexplore.ieee.org/document/7764544

L = 5e3 # Fiber length

Nt= 50000 # Time sampling
Nl = 2000 # Length sampling

# Calculated factors and vectors

h = L/Nl # Lengthstep
dT = 0.001*ns # FWHM
T = np.linspace(-2, 2, Nt)*ns # Pulse local time vector
z = np.arange(0, Nl, 1)*h # Propagation distance vector
P0=220e-3 # W
n = 0.02

dt = T[1]-T[0] # timestep
w =  2*np.pi*c/l0 + np.fft.fftshift(np.fft.fftfreq(Nt, d=dt)) # Pulsation vector # ?? 2*np.pi*c/l0 +
Dw = 0.5*1j*w**2*b2 # Calculated dispersion operator

Noise = np.random.randn(1,Nt) # Amplitude noise vector

A =np.asarray(np.zeros((Nl, Nt), dtype=complex)) # System matrix
B =np.asarray(np.zeros((Nl, Nt), dtype=complex))
#A[0,:]= n*np.sqrt(P0)*Noise + np.sqrt(P0)*(np.exp(-T**2/(2*(dT)**2)))+ np.sqrt(P0)*(np.exp(-(T-0.5*ns)**2/(2*(dT)**2))) # Initial state
A[0,:]= n*np.sqrt(P0)*Noise + np.sqrt(P0)*np.sin(w0*T) # Initial state
B[0,:]=np.fft.fftshift(np.fft.fft(A[0,:]))

```

```python
plt.plot(T,np.abs(A[0,:]))
plt.title("Initial pulse")
plt.show()

```

```python
for i in tqdm(range(1,Nl)):
    # Fourier domain
    N = 1j*gamma*np.abs(A[i-1,:])**2-0.5*alpha
    Ai = np.exp(0.5*h*N)*A[i-1,:] # half Nonlinearity 1
    Ai = np.fft.fftshift(np.fft.fft(Ai)) # Fourier
    Ai = np.exp(h*Dw)*Ai # Dispersion
    Ai = np.fft.ifft(np.fft.ifftshift(Ai)) # Temporal
    Ai = np.exp(0.5*h*N)*Ai # half Nonlinearity 2
    A[i,:] = Ai
    B[i,:] = np.fft.fftshift(np.fft.fft(Ai))
```

    100%|██████████████████████████████████████████████████████████████████████████████| 1999/1999 [01:07<00:00, 29.52it/s]

```python
f = w/(2*np.pi)
extent = [f[0], f[-1], z[-1], z[0]]
plt.imshow(np.abs(A), extent=extent, aspect=1e8, cmap='jet')
plt.xlabel(r"f (Hz)")
plt.ylabel(r"L (m)")

plt.tight_layout()
plt.show()
```

```python
plt.plot(T/ns,np.abs(A[0,:]), label='input')
plt.plot(T/ns,np.abs(A[Nl-1,:]), label='output')

plt.grid()
plt.legend()
plt.xlabel("Pulse time (ns)")
plt.ylabel("Pulse Power (W)")
plt.show()

```

```python
w[0]
```

    -2499875000001.19

```python
Dw
```

    array([0.+0.01354091j, 0.+0.0135382j , 0.+0.01353549j, ...,
           0.+0.01353278j, 0.+0.01353549j, 0.+0.0135382j ])

```python
N
```

    array([0.+17.06862284j, 0.+15.83780934j, 0. +6.44657317j, ...,
           0. +5.71969144j, 0.+16.61307259j, 0. +0.36226816j])

```python
h
```

    50.0

```python
c/l0
```

    193548387096774.2

```python

```