```python
import pandas as pd
import matplotlib as mpl
import numpy as np
import scipy.stats
mpl.rcParams['figure.dpi'] = 600
```

```python
from tcga_dicts import *
```

# Part 0: Load Data

## Immune Cell Fractions

CIBERSORT result from [*The Immune Landscape of Cancer*](https://www.sciencedirect.com/science/article/pii/S1074761318301213) downloaded from [NIH PanCanAtlas](https://gdc.cancer.gov/about-data/publications/panimmune) includes are SampleID ([TCGA barcode](https://docs.gdc.cancer.gov/Encyclopedia/pages/TCGA_Barcode/)), Cancer Type ([TCGA Study Abbreviation](https://gdc.cancer.gov/resources-tcga-users/tcga-code-tables/tcga-study-abbreviations)), and the abundance of 22 imuune cell types.

Note that the abundances in each row sum to 1. In other words, the abundance is the proportion of a cell type in the leukocyte compartment (see [below](#lf)), but not all cells (stroma, tumor, etc.).

```python
data = pd.read_csv("TCGA.Kallisto.fullIDs.cibersort.relative.tsv", sep="\t")
data["SampleID"] = data["SampleID"].apply(lambda x: x.replace('.', '-'))
data
```

      SampleID
      CancerType
      B.cells.naive
      B.cells.memory
      Plasma.cells
      T.cells.CD8
      T.cells.CD4.naive
      T.cells.CD4.memory.resting
      T.cells.CD4.memory.activated
      T.cells.follicular.helper
      ...
      Macrophages.M2
      Dendritic.cells.resting
      Dendritic.cells.activated
      Mast.cells.resting
      Mast.cells.activated
      Eosinophils
      Neutrophils
      P.value
      Correlation
      RMSE

      0
      TCGA-OR-A5JG-01A-11R-A29S-07
      ACC
      0.000000
      0.048529
      0.016052
      0.046099
      0.027037
      0.290682
      0.0
      0.000000
      ...
      0.363861
      0.002715
      0.026125
      0.032788
      0.000000
      0.010290
      0.009607
      0.112
      0.095797
      1.047142

      1
      TCGA-OR-A5LG-01A-11R-A29S-07
      ACC
      0.007169
      0.011125
      0.007982
      0.139842
      0.000000
      0.142742
      0.0
      0.001614
      ...
      0.448243
      0.000000
      0.007464
      0.126237
      0.000000
      0.000000
      0.000000
      0.104
      0.103345
      1.046163

      2
      TCGA-OR-A5JD-01A-11R-A29S-07
      ACC
      0.000023
      0.014607
      0.000000
      0.104888
      0.000000
      0.174895
      0.0
      0.017928
      ...
      0.329552
      0.000000
      0.009330
      0.000000
      0.190730
      0.000000
      0.000000
      0.068
      0.143259
      1.039812

      3
      TCGA-OR-A5LH-01A-11R-A29S-07
      ACC
      0.047299
      0.038180
      0.000000
      0.198591
      0.000000
      0.065240
      0.0
      0.043310
      ...
      0.287939
      0.000000
      0.000576
      0.000000
      0.064725
      0.000000
      0.000000
      0.388
      0.028763
      1.080553

      4
      TCGA-OR-A5KY-01A-11R-A29S-07
      ACC
      0.000000
      0.024795
      0.004418
      0.051506
      0.000000
      0.225892
      0.0
      0.000000
      ...
      0.486400
      0.000000
      0.042827
      0.000000
      0.017418
      0.014769
      0.000000
      0.246
      0.055621
      1.073474

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      11368
      TCGA-V4-A9EQ-01A-11R-A405-07
      UVM
      0.000000
      0.028939
      0.008894
      0.000000
      0.023294
      0.015676
      0.0
      0.002279
      ...
      0.521353
      0.009128
      0.000000
      0.208036
      0.000000
      0.000000
      0.000000
      0.090
      0.111538
      1.060529

      11369
      TCGA-V4-A9EV-01A-11R-A405-07
      UVM
      0.000000
      0.013299
      0.025069
      0.315356
      0.000000
      0.017361
      0.0
      0.095233
      ...
      0.163671
      0.015558
      0.000000
      0.061707
      0.000000
      0.000000
      0.000000
      0.002
      0.347125
      0.952021

      11370
      TCGA-V4-A9EY-01A-11R-A405-07
      UVM
      0.000000
      0.032507
      0.076160
      0.061400
      0.034359
      0.000000
      0.0
      0.007778
      ...
      0.431568
      0.004550
      0.000000
      0.227797
      0.000000
      0.000000
      0.000000
      0.086
      0.113888
      1.041876

      11371
      TCGA-V4-A9EU-01A-11R-A405-07
      UVM
      0.000000
      0.021228
      0.010225
      0.067985
      0.003434
      0.006675
      0.0
      0.004884
      ...
      0.627631
      0.000000
      0.000000
      0.147489
      0.000000
      0.000000
      0.000000
      0.096
      0.107852
      1.068155

      11372
      TCGA-V4-A9EM-01A-11R-A405-07
      UVM
      0.000000
      0.089636
      0.054750
      0.059644
      0.000000
      0.039969
      0.0
      0.005501
      ...
      0.205302
      0.000000
      0.000000
      0.271339
      0.000000
      0.000000
      0.000000
      0.318
      0.041508
      1.064184

11373 rows × 27 columns

```python
tcga_study_abbr_dict = {
    "LAML": "Acute Myeloid Leukemia",
    "ACC": "Adrenocortical carcinoma",
    "BLCA": "Bladder Urothelial Carcinoma",
    "LGG": "Brain Lower Grade Glioma",
    "BRCA": "Breast invasive carcinoma",
    "CESC": "Cervical squamous cell carcinoma and endocervical adenocarcinoma",
    "CHOL": "Cholangiocarcinoma",
    "LCML": "Chronic Myelogenous Leukemia",
    "COAD": "Colon adenocarcinoma",
    "CNTL": "Controls",
    "ESCA": "Esophageal carcinoma",
    "FPPP": "FFPE Pilot Phase II",
    "GBM": "Glioblastoma multiforme",
    "HNSC": "Head and Neck squamous cell carcinoma",
    "KICH": "Kidney Chromophobe",
    "KIRC": "Kidney renal clear cell carcinoma",
    "KIRP": "Kidney renal papillary cell carcinoma",
    "LIHC": "Liver hepatocellular carcinoma",
    "LUAD": "Lung adenocarcinoma",
    "LUSC": "Lung squamous cell carcinoma",
    "DLBC": "Lymphoid Neoplasm Diffuse Large B-cell Lymphoma",
    "MESO": "Mesothelioma",
    "MISC": "Miscellaneous",
    "OV": "Ovarian serous cystadenocarcinoma",
    "PAAD": "Pancreatic adenocarcinoma",
    "PCPG": "Pheochromocytoma and Paraganglioma",
    "PRAD": "Prostate adenocarcinoma",
    "READ": "Rectum adenocarcinoma",
    "SARC": "Sarcoma",
    "SKCM": "Skin Cutaneous Melanoma",
    "STAD": "Stomach adenocarcinoma",
    "TGCT": "Testicular Germ Cell Tumors",
    "THYM": "Thymoma",
    "THCA": "Thyroid carcinoma",
    "UCS": "Uterine Carcinosarcoma",
    "UCEC": "Uterine Corpus Endometrial Carcinoma",
    "UVM": "Uveal Melanoma"
}
```

```python
tcga_study_abbr_dict2 = {k: v + ' (' + k + ')' for k, v in tcga_study_abbr_dict.items()}
tcga_study_abbr_dict2
```

    {'LAML': 'Acute Myeloid Leukemia (LAML)',
     'ACC': 'Adrenocortical carcinoma (ACC)',
     'BLCA': 'Bladder Urothelial Carcinoma (BLCA)',
     'LGG': 'Brain Lower Grade Glioma (LGG)',
     'BRCA': 'Breast invasive carcinoma (BRCA)',
     'CESC': 'Cervical squamous cell carcinoma and endocervical adenocarcinoma (CESC)',
     'CHOL': 'Cholangiocarcinoma (CHOL)',
     'LCML': 'Chronic Myelogenous Leukemia (LCML)',
     'COAD': 'Colon adenocarcinoma (COAD)',
     'CNTL': 'Controls (CNTL)',
     'ESCA': 'Esophageal carcinoma (ESCA)',
     'FPPP': 'FFPE Pilot Phase II (FPPP)',
     'GBM': 'Glioblastoma multiforme (GBM)',
     'HNSC': 'Head and Neck squamous cell carcinoma (HNSC)',
     'KICH': 'Kidney Chromophobe (KICH)',
     'KIRC': 'Kidney renal clear cell carcinoma (KIRC)',
     'KIRP': 'Kidney renal papillary cell carcinoma (KIRP)',
     'LIHC': 'Liver hepatocellular carcinoma (LIHC)',
     'LUAD': 'Lung adenocarcinoma (LUAD)',
     'LUSC': 'Lung squamous cell carcinoma (LUSC)',
     'DLBC': 'Lymphoid Neoplasm Diffuse Large B-cell Lymphoma (DLBC)',
     'MESO': 'Mesothelioma (MESO)',
     'MISC': 'Miscellaneous (MISC)',
     'OV': 'Ovarian serous cystadenocarcinoma (OV)',
     'PAAD': 'Pancreatic adenocarcinoma (PAAD)',
     'PCPG': 'Pheochromocytoma and Paraganglioma (PCPG)',
     'PRAD': 'Prostate adenocarcinoma (PRAD)',
     'READ': 'Rectum adenocarcinoma (READ)',
     'SARC': 'Sarcoma (SARC)',
     'SKCM': 'Skin Cutaneous Melanoma (SKCM)',
     'STAD': 'Stomach adenocarcinoma (STAD)',
     'TGCT': 'Testicular Germ Cell Tumors (TGCT)',
     'THYM': 'Thymoma (THYM)',
     'THCA': 'Thyroid carcinoma (THCA)',
     'UCS': 'Uterine Carcinosarcoma (UCS)',
     'UCEC': 'Uterine Corpus Endometrial Carcinoma (UCEC)',
     'UVM': 'Uveal Melanoma (UVM)'}

```python
data["CancerType"].apply(tcga_study_abbr_dict.__getitem__).value_counts().plot(kind='bar')
```

```python
data['P.value'].plot(kind="hist", bins=100)
```

```python
{'all': data['P.value'].apply(1.00.__ge__).sum(),
'0.05': data['P.value'].apply(0.05.__ge__).sum(),
'0.01': data['P.value'].apply(0.01.__ge__).sum()
}
```

    {'all': 11373, '0.05': 6077, '0.01': 2893}

```python
data['SampleID'].value_counts().where(lambda x: x > 1.0).dropna()
```

    TCGA-A7-A13G-01B-04R-A22O-07    3.0
    TCGA-AC-A2QH-01B-04R-A22O-07    3.0
    TCGA-AC-A3OD-01B-06R-A22O-07    3.0
    TCGA-A7-A26I-01B-06R-A22O-07    3.0
    TCGA-A7-A26F-01B-04R-A22O-07    3.0
    TCGA-A7-A0DC-01B-04R-A22O-07    3.0
    TCGA-A7-A0DC-01A-11R-A00Z-07    3.0
    TCGA-AK-3425-01A-02R-1277-07    2.0
    TCGA-AC-A2QH-01A-11R-A18M-07    2.0
    TCGA-37-4132-01A-01R-1100-07    2.0
    TCGA-A6-5665-01B-03R-2302-07    2.0
    TCGA-AZ-4615-01A-01R-1410-07    2.0
    TCGA-FI-A2F8-01A-12R-A17B-07    2.0
    TCGA-37-4133-01A-01R-1100-07    2.0
    TCGA-HC-8261-01B-05R-2302-07    2.0
    TCGA-AZ-4614-01A-01R-1410-07    2.0
    TCGA-AK-3454-01A-02R-1277-07    2.0
    TCGA-AZ-4315-01A-01R-1410-07    2.0
    TCGA-A6-2684-01A-01R-1410-07    2.0
    TCGA-38-4625-01A-01R-1206-07    2.0
    TCGA-AK-3453-01A-02R-1277-07    2.0
    TCGA-CA-5256-01A-01R-1410-07    2.0
    TCGA-CM-4747-01A-01R-1410-07    2.0
    TCGA-A6-2685-01A-01R-1410-07    2.0
    TCGA-AW-A1PO-01A-12R-A157-07    2.0
    TCGA-HC-7740-01B-04R-2302-07    2.0
    TCGA-AK-3426-01A-02R-1325-07    2.0
    TCGA-AA-3502-01A-01R-1410-07    2.0
    TCGA-AA-A01Z-01A-11R-A083-07    2.0
    TCGA-AA-3492-01A-01R-1410-07    2.0
    TCGA-AA-3506-01A-01R-1410-07    2.0
    TCGA-A7-A13G-01A-11R-A13Q-07    2.0
    TCGA-AX-A1C7-01A-11R-A137-07    2.0
    TCGA-A6-2682-01A-01R-1410-07    2.0
    TCGA-AA-A01X-01A-21R-A083-07    2.0
    TCGA-A7-A0DC-11A-41R-A089-07    2.0
    TCGA-A7-A26F-01A-21R-A169-07    2.0
    TCGA-AZ-4313-01A-01R-1410-07    2.0
    TCGA-AC-A3OD-01A-11R-A21T-07    2.0
    TCGA-AZ-4684-01A-01R-1410-07    2.0
    TCGA-A6-2672-01B-03R-2302-07    2.0
    TCGA-A2-A0EM-01A-11R-A034-07    2.0
    TCGA-BH-A0B2-01A-11R-A10J-07    2.0
    TCGA-AA-3509-01A-01R-1410-07    2.0
    TCGA-HC-8258-01B-05R-2302-07    2.0
    TCGA-AA-A01P-01A-21R-A083-07    2.0
    TCGA-HC-8265-01B-04R-2302-07    2.0
    TCGA-AA-3495-01A-01R-1410-07    2.0
    TCGA-A7-A26I-01A-11R-A169-07    2.0
    TCGA-AC-A3QQ-01B-06R-A22O-07    2.0
    TCGA-CK-4951-01A-01R-1410-07    2.0
    TCGA-AC-A3QQ-01A-11R-A22K-07    2.0
    TCGA-AJ-A23O-01A-11R-A157-07    2.0
    TCGA-A6-5661-01B-05R-2302-07    2.0
    Name: SampleID, dtype: float64

```python
data[data['SampleID'] == 'TCGA-A7-A0DC-01B-04R-A22O-07']
```

      SampleID
      CancerType
      B.cells.naive
      B.cells.memory
      Plasma.cells
      T.cells.CD8
      T.cells.CD4.naive
      T.cells.CD4.memory.resting
      T.cells.CD4.memory.activated
      T.cells.follicular.helper
      ...
      Macrophages.M2
      Dendritic.cells.resting
      Dendritic.cells.activated
      Mast.cells.resting
      Mast.cells.activated
      Eosinophils
      Neutrophils
      P.value
      Correlation
      RMSE

      1762
      TCGA-A7-A0DC-01B-04R-A22O-07
      BRCA
      0.0
      0.095566
      0.045122
      0.018111
      0.130455
      0.172505
      0.0
      0.0
      ...
      0.115555
      0.0
      0.067555
      0.251592
      0.0
      0.000000
      0.0
      0.108
      0.099062
      1.040622

      1763
      TCGA-A7-A0DC-01B-04R-A22O-07
      BRCA
      0.0
      0.113939
      0.040357
      0.005743
      0.067493
      0.214965
      0.0
      0.0
      ...
      0.125661
      0.0
      0.058962
      0.278053
      0.0
      0.000000
      0.0
      0.138
      0.078372
      1.048953

      1764
      TCGA-A7-A0DC-01B-04R-A22O-07
      BRCA
      0.0
      0.133703
      0.046857
      0.018552
      0.031506
      0.225782
      0.0
      0.0
      ...
      0.104292
      0.0
      0.078465
      0.211148
      0.0
      0.001334
      0.0
      0.244
      0.045592
      1.056600

3 rows × 27 columns

## Leukocyte Fractions (LFs)

Leukocyte fractions from [The Immune Landscape of Cancer](https://www.sciencedirect.com/science/article/pii/S1074761318301213) downloaded from [NIH PanCanAtlas](https://gdc.cancer.gov/about-data/publications/panimmune).

```python
leuk = pd.read_csv("TCGA_all_leuk_estimate.masked.20170107.tsv", sep="\t", header=None)
leuk.columns = ['CancerType', 'SampleID', 'LF']
leuk
```

      CancerType
      SampleID
      LF

      0
      ACC
      TCGA-OR-A5J1-01A-11D-A29J-05
      0.046374

      1
      ACC
      TCGA-OR-A5J2-01A-11D-A29J-05
      0.057859

      2
      ACC
      TCGA-OR-A5J3-01A-11D-A29J-05
      0.048460

      3
      ACC
      TCGA-OR-A5J4-01A-11D-A29J-05
      0.043988

      4
      ACC
      TCGA-OR-A5J5-01A-11D-A29J-05
      0.016759

      ...
      ...
      ...
      ...

      10812
      TGCT
      TCGA-ZM-AA0D-01A-11D-A436-05
      0.578000

      10813
      TGCT
      TCGA-ZM-AA0E-01A-12D-A436-05
      0.512000

      10814
      TGCT
      TCGA-ZM-AA0F-01A-21D-A436-05
      0.625000

      10815
      TGCT
      TCGA-ZM-AA0H-01A-11D-A436-05
      0.431000

      10816
      TGCT
      TCGA-ZM-AA0N-01A-21D-A436-05
      0.390000

10817 rows × 3 columns

```python
display(leuk.LF.min())
display(leuk.LF.max())
```

    -9.29722165644888e-05

    0.9642001238693829

```python
leuk.LF[leuk.LF < 0] = 0
```

    <ipython-input-12-9cbbd3800d77>:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      leuk.LF[leuk.LF < 0] = 0

## Merge CIBERSORT and LF

We first investigate the number of samples available.
Remember that the format of TCGA barcode is:

| Label        | Project | TSS | Participant | Sample & Vial | Portion & Analyte | Plate | Center |
|--------------|---------|-----|-------------|---------------|-------------------|-------|--------|
| Subscription | 0       | 1   | 2           | 3             | 4                 | 5     | 6      |
| Example      | TCGA    | OR  | A5J1        | 01A           | 11D               | A29J  | 05     |

If we use Label-Project-TSS-Participant-Sample&Vial as the identifier, there are 10103 matched ones. It goes down to zero if we use up to Portion&Analyte. In fact, the CIBERSORT is based on RNA, while the LF is based on the DNA sample (by methylation analysis).

Finally, we use Label-Project-TSS-Participant-Sample&Vial-Portion, but exclude the Analyter, Plate, and Center as our identifier to match the samples.

Also note that there are "duplicates" in the original tables. For CIBERSORT, we keep the one with lowest p-value. For LF, we take the average.

It is worth noting that "match" here shall not be confused with "matched samples" in "paired test".

```python
id1 = set(data['SampleID'].apply(lambda x: '-'.join(x.split('-')[0:4])))
id2 = set(leuk['SampleID'].apply(lambda x: '-'.join(x.split('-')[0:4])))
{
    '1': len(id1),
    '2': len(id2),
    'intersection': len(id1.intersection(id2))
}
```

    {'1': 11273, '2': 10770, 'intersection': 10103}

```python
id1 = set(data['SampleID'].apply(lambda x: '-'.join(x.split('-')[0:5])))
id2 = set(leuk['SampleID'].apply(lambda x: '-'.join(x.split('-')[0:5])))
{
    '1': len(id1),
    '2': len(id2),
    'intersection': len(id1.intersection(id2))
}
```

    {'1': 11275, '2': 10782, 'intersection': 0}

```python
id1 = set(data['SampleID'].apply(lambda x: '-'.join(x.split('-')[0:4] + [x.split('-')[4][0:2]])))
id2 = set(leuk['SampleID'].apply(lambda x: '-'.join(x.split('-')[0:4] + [x.split('-')[4][0:2]])))
common_id = id1.intersection(id2)
{
    '1': len(id1),
    '2': len(id2),
    'intersection': len(common_id)
}
```

    {'1': 11275, '2': 10782, 'intersection': 10039}

```python
merged = pd.DataFrame()
for id in common_id:
    temp = data[data['SampleID'].apply(lambda x: id in x)]
    temp.loc[:, 'LF'] = leuk.loc[leuk.loc[:, 'SampleID'].apply(lambda x: id in x), 'LF'].mean()
    merged = merged.append(temp.loc[temp.loc[:, 'P.value'].idxmax()])
```

    /home/sliang3/miniconda3/envs/basic/lib/python3.8/site-packages/pandas/core/indexing.py:845: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      self.obj[key] = _infer_fill_value(value)
    /home/sliang3/miniconda3/envs/basic/lib/python3.8/site-packages/pandas/core/indexing.py:966: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      self.obj[item] = s

```python
merged = merged[data.columns.tolist() + ['LF']]
```

```python
merged
```

      SampleID
      CancerType
      B.cells.naive
      B.cells.memory
      Plasma.cells
      T.cells.CD8
      T.cells.CD4.naive
      T.cells.CD4.memory.resting
      T.cells.CD4.memory.activated
      T.cells.follicular.helper
      ...
      Dendritic.cells.resting
      Dendritic.cells.activated
      Mast.cells.resting
      Mast.cells.activated
      Eosinophils
      Neutrophils
      P.value
      Correlation
      RMSE
      LF

      6674
      TCGA-92-7341-01A-31R-2045-07
      LUSC
      0.147873
      0.000000
      0.087686
      0.033431
      0.0
      0.170879
      0.000000
      0.000000
      ...
      0.000000
      0.019767
      0.018538
      0.001042
      0.0
      0.004651
      0.006
      0.287252
      0.969659
      0.216818

      7939
      TCGA-CH-5754-01A-11R-1580-07
      PRAD
      0.065331
      0.000000
      0.097379
      0.145830
      0.0
      0.144164
      0.000000
      0.135648
      ...
      0.011441
      0.000000
      0.023868
      0.000000
      0.0
      0.000000
      0.014
      0.229421
      1.006918
      0.248506

      11182
      TCGA-BS-A0V7-01A-21R-A118-07
      UCEC
      0.000000
      0.129441
      0.112427
      0.166960
      0.0
      0.066348
      0.007972
      0.108577
      ...
      0.010174
      0.004734
      0.081979
      0.000000
      0.0
      0.000285
      0.004
      0.366437
      0.936339
      0.138962

      1419
      TCGA-A2-A0CQ-01A-21R-A034-07
      BRCA
      0.000000
      0.044607
      0.000000
      0.198327
      0.0
      0.041238
      0.000000
      0.018438
      ...
      0.006661
      0.000000
      0.059281
      0.000000
      0.0
      0.002902
      0.012
      0.221909
      1.001612
      0.120546

      6205
      TCGA-05-5420-01A-01R-1628-07
      LUAD
      0.031664
      0.000000
      0.013226
      0.087401
      0.0
      0.115560
      0.052406
      0.055071
      ...
      0.140423
      0.000000
      0.020737
      0.000000
      0.0
      0.014643
      0.216
      0.048965
      1.068250
      0.571093

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      3172
      TCGA-CV-5432-01A-02R-1686-07
      HNSC
      0.127065
      0.000000
      0.261089
      0.105192
      0.0
      0.023960
      0.000000
      0.042882
      ...
      0.057583
      0.000000
      0.039449
      0.000000
      0.0
      0.002197
      0.002
      0.437382
      0.903511
      0.179283

      2824
      TCGA-L5-A8NQ-01A-11R-A36D-31
      ESCA
      0.000000
      0.038928
      0.011305
      0.114216
      0.0
      0.036888
      0.000000
      0.100739
      ...
      0.000000
      0.000000
      0.053488
      0.000000
      0.0
      0.000000
      0.096
      0.101506
      1.059170
      0.392468

      9431
      TCGA-FP-8210-01A-11R-2343-13
      STAD
      0.187019
      0.099927
      0.000176
      0.146384
      0.0
      0.115863
      0.000000
      0.066405
      ...
      0.000000
      0.018263
      0.072636
      0.000000
      0.0
      0.000000
      0.002
      0.337942
      0.949999
      0.488038

      8279
      TCGA-KK-A8I8-01A-11R-A36G-07
      PRAD
      0.002727
      0.092736
      0.019580
      0.202959
      0.0
      0.177791
      0.000000
      0.036568
      ...
      0.014536
      0.000000
      0.064118
      0.000000
      0.0
      0.000000
      0.222
      0.065246
      1.063335
      0.066327

      1305
      TCGA-C8-A130-01A-31R-A115-07
      BRCA
      0.084378
      0.000000
      0.047390
      0.180525
      0.0
      0.119289
      0.000000
      0.061659
      ...
      0.002519
      0.000000
      0.092324
      0.000000
      0.0
      0.000000
      0.012
      0.211412
      0.997238
      0.143928

10039 rows × 28 columns

Multiply abundance of cell types with the leukocyte fraction.

```python
merged_adjusted = merged.copy()
for i in range(merged_adjusted.shape[0]):
    merged_adjusted.iloc[i, cell_types] *= merged.iloc[i, 27]
merged_adjusted
```

      SampleID
      CancerType
      B.cells.naive
      B.cells.memory
      Plasma.cells
      T.cells.CD8
      T.cells.CD4.naive
      T.cells.CD4.memory.resting
      T.cells.CD4.memory.activated
      T.cells.follicular.helper
      ...
      Dendritic.cells.resting
      Dendritic.cells.activated
      Mast.cells.resting
      Mast.cells.activated
      Eosinophils
      Neutrophils
      P.value
      Correlation
      RMSE
      LF

      6674
      TCGA-92-7341-01A-31R-2045-07
      LUSC
      0.032062
      0.000000
      0.019012
      0.007248
      0.0
      0.037050
      0.000000
      0.000000
      ...
      0.000000
      0.004286
      0.004019
      0.000226
      0.0
      0.001008
      0.006
      0.287252
      0.969659
      0.216818

      7939
      TCGA-CH-5754-01A-11R-1580-07
      PRAD
      0.016235
      0.000000
      0.024199
      0.036240
      0.0
      0.035825
      0.000000
      0.033709
      ...
      0.002843
      0.000000
      0.005931
      0.000000
      0.0
      0.000000
      0.014
      0.229421
      1.006918
      0.248506

      11182
      TCGA-BS-A0V7-01A-21R-A118-07
      UCEC
      0.000000
      0.017987
      0.015623
      0.023201
      0.0
      0.009220
      0.001108
      0.015088
      ...
      0.001414
      0.000658
      0.011392
      0.000000
      0.0
      0.000040
      0.004
      0.366437
      0.936339
      0.138962

      1419
      TCGA-A2-A0CQ-01A-21R-A034-07
      BRCA
      0.000000
      0.005377
      0.000000
      0.023907
      0.0
      0.004971
      0.000000
      0.002223
      ...
      0.000803
      0.000000
      0.007146
      0.000000
      0.0
      0.000350
      0.012
      0.221909
      1.001612
      0.120546

      6205
      TCGA-05-5420-01A-01R-1628-07
      LUAD
      0.018083
      0.000000
      0.007553
      0.049914
      0.0
      0.065995
      0.029929
      0.031451
      ...
      0.080195
      0.000000
      0.011843
      0.000000
      0.0
      0.008363
      0.216
      0.048965
      1.068250
      0.571093

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      3172
      TCGA-CV-5432-01A-02R-1686-07
      HNSC
      0.022781
      0.000000
      0.046809
      0.018859
      0.0
      0.004296
      0.000000
      0.007688
      ...
      0.010324
      0.000000
      0.007073
      0.000000
      0.0
      0.000394
      0.002
      0.437382
      0.903511
      0.179283

      2824
      TCGA-L5-A8NQ-01A-11R-A36D-31
      ESCA
      0.000000
      0.015278
      0.004437
      0.044826
      0.0
      0.014477
      0.000000
      0.039537
      ...
      0.000000
      0.000000
      0.020992
      0.000000
      0.0
      0.000000
      0.096
      0.101506
      1.059170
      0.392468

      9431
      TCGA-FP-8210-01A-11R-2343-13
      STAD
      0.091272
      0.048768
      0.000086
      0.071441
      0.0
      0.056546
      0.000000
      0.032408
      ...
      0.000000
      0.008913
      0.035449
      0.000000
      0.0
      0.000000
      0.002
      0.337942
      0.949999
      0.488038

      8279
      TCGA-KK-A8I8-01A-11R-A36G-07
      PRAD
      0.000181
      0.006151
      0.001299
      0.013462
      0.0
      0.011792
      0.000000
      0.002425
      ...
      0.000964
      0.000000
      0.004253
      0.000000
      0.0
      0.000000
      0.222
      0.065246
      1.063335
      0.066327

      1305
      TCGA-C8-A130-01A-31R-A115-07
      BRCA
      0.012144
      0.000000
      0.006821
      0.025982
      0.0
      0.017169
      0.000000
      0.008874
      ...
      0.000363
      0.000000
      0.013288
      0.000000
      0.0
      0.000000
      0.012
      0.211412
      0.997238
      0.143928

10039 rows × 28 columns

## Label samples
Label samples with its type and patient ID (i.e., Label-Project-TSS-Participant).

```python
tcga_sample_type_dict = {
    "01": "Primary Solid Tumor",
    "02": "Recurrent Solid Tumor",
    "03": "Primary Blood Derived Cancer - Peripheral Blood",
    "04": "Recurrent Blood Derived Cancer - Bone Marrow",
    "05": "Additional - New Primary",
    "06": "Metastatic",
    "07": "Additional Metastatic",
    "08": "Human Tumor Original Cells",
    "09": "Primary Blood Derived Cancer - Bone Marrow",
    "10": "Blood Derived Normal",
    "11": "Solid Tissue Normal",
    "12": "Buccal Cell Normal",
    "13": "EBV Immortalized Normal",
    "14": "Bone Marrow Normal",
    "15": "sample type 15",
    "16": "sample type 16",
    "20": "Control Analyte",
    "40": "Recurrent Blood Derived Cancer - Peripheral Blood",
    "50": "Cell Lines",
    "60": "Primary Xenograft Tissue",
    "61": "Cell Line Derived Xenograft Tissue",
    "99": "sample type 99"
}
```

```python
f = lambda x: tcga_sample_type_dict[x.split('-')[3][0:2]]
merged_adjusted['SampleType'] = merged_adjusted["SampleID"].apply(f)
```

```python
f = lambda x: '-'.join(x.split('-')[0:3])
merged_adjusted['PatientID'] = merged_adjusted["SampleID"].apply(f)
```

```python
merged_adjusted
```

      SampleID
      CancerType
      B.cells.naive
      B.cells.memory
      Plasma.cells
      T.cells.CD8
      T.cells.CD4.naive
      T.cells.CD4.memory.resting
      T.cells.CD4.memory.activated
      T.cells.follicular.helper
      ...
      Mast.cells.resting
      Mast.cells.activated
      Eosinophils
      Neutrophils
      P.value
      Correlation
      RMSE
      LF
      SampleType
      PatientID

      6674
      TCGA-92-7341-01A-31R-2045-07
      LUSC
      0.032062
      0.000000
      0.019012
      0.007248
      0.0
      0.037050
      0.000000
      0.000000
      ...
      0.004019
      0.000226
      0.0
      0.001008
      0.006
      0.287252
      0.969659
      0.216818
      Primary Solid Tumor
      TCGA-92-7341

      7939
      TCGA-CH-5754-01A-11R-1580-07
      PRAD
      0.016235
      0.000000
      0.024199
      0.036240
      0.0
      0.035825
      0.000000
      0.033709
      ...
      0.005931
      0.000000
      0.0
      0.000000
      0.014
      0.229421
      1.006918
      0.248506
      Primary Solid Tumor
      TCGA-CH-5754

      11182
      TCGA-BS-A0V7-01A-21R-A118-07
      UCEC
      0.000000
      0.017987
      0.015623
      0.023201
      0.0
      0.009220
      0.001108
      0.015088
      ...
      0.011392
      0.000000
      0.0
      0.000040
      0.004
      0.366437
      0.936339
      0.138962
      Primary Solid Tumor
      TCGA-BS-A0V7

      1419
      TCGA-A2-A0CQ-01A-21R-A034-07
      BRCA
      0.000000
      0.005377
      0.000000
      0.023907
      0.0
      0.004971
      0.000000
      0.002223
      ...
      0.007146
      0.000000
      0.0
      0.000350
      0.012
      0.221909
      1.001612
      0.120546
      Primary Solid Tumor
      TCGA-A2-A0CQ

      6205
      TCGA-05-5420-01A-01R-1628-07
      LUAD
      0.018083
      0.000000
      0.007553
      0.049914
      0.0
      0.065995
      0.029929
      0.031451
      ...
      0.011843
      0.000000
      0.0
      0.008363
      0.216
      0.048965
      1.068250
      0.571093
      Primary Solid Tumor
      TCGA-05-5420

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      3172
      TCGA-CV-5432-01A-02R-1686-07
      HNSC
      0.022781
      0.000000
      0.046809
      0.018859
      0.0
      0.004296
      0.000000
      0.007688
      ...
      0.007073
      0.000000
      0.0
      0.000394
      0.002
      0.437382
      0.903511
      0.179283
      Primary Solid Tumor
      TCGA-CV-5432

      2824
      TCGA-L5-A8NQ-01A-11R-A36D-31
      ESCA
      0.000000
      0.015278
      0.004437
      0.044826
      0.0
      0.014477
      0.000000
      0.039537
      ...
      0.020992
      0.000000
      0.0
      0.000000
      0.096
      0.101506
      1.059170
      0.392468
      Primary Solid Tumor
      TCGA-L5-A8NQ

      9431
      TCGA-FP-8210-01A-11R-2343-13
      STAD
      0.091272
      0.048768
      0.000086
      0.071441
      0.0
      0.056546
      0.000000
      0.032408
      ...
      0.035449
      0.000000
      0.0
      0.000000
      0.002
      0.337942
      0.949999
      0.488038
      Primary Solid Tumor
      TCGA-FP-8210

      8279
      TCGA-KK-A8I8-01A-11R-A36G-07
      PRAD
      0.000181
      0.006151
      0.001299
      0.013462
      0.0
      0.011792
      0.000000
      0.002425
      ...
      0.004253
      0.000000
      0.0
      0.000000
      0.222
      0.065246
      1.063335
      0.066327
      Primary Solid Tumor
      TCGA-KK-A8I8

      1305
      TCGA-C8-A130-01A-31R-A115-07
      BRCA
      0.012144
      0.000000
      0.006821
      0.025982
      0.0
      0.017169
      0.000000
      0.008874
      ...
      0.013288
      0.000000
      0.0
      0.000000
      0.012
      0.211412
      0.997238
      0.143928
      Primary Solid Tumor
      TCGA-C8-A130

10039 rows × 30 columns

# Basic statistics

```python
f = lambda x: tcga_sample_type_dict[x.split('-')[3][0:2]]
all_data = merged_adjusted.copy()
all_data["SampleType"] = all_data["SampleID"].apply(f)
all_data.groupby(["CancerType", "SampleType"]).SampleID.count().to_frame()
```

      SampleID

      CancerType
      SampleType

      ACC
      Primary Solid Tumor
      79

      BLCA
      Primary Solid Tumor
      411

      BRCA
      Metastatic
      7

      Primary Solid Tumor
      1084

      CESC
      Metastatic
      2

      Primary Solid Tumor
      304

      CHOL
      Primary Solid Tumor
      36

      COAD
      Metastatic
      1

      Primary Solid Tumor
      466

      Recurrent Solid Tumor
      1

      ESCA
      Metastatic
      1

      Primary Solid Tumor
      184

      GBM
      Primary Solid Tumor
      127

      Recurrent Solid Tumor
      13

      HNSC
      Metastatic
      2

      Primary Solid Tumor
      520

      KICH
      Primary Solid Tumor
      66

      KIRC
      Additional - New Primary
      1

      Primary Solid Tumor
      490

      KIRP
      Additional - New Primary
      1

      Primary Solid Tumor
      280

      LGG
      Primary Solid Tumor
      513

      Recurrent Solid Tumor
      18

      LIHC
      Primary Solid Tumor
      371

      Recurrent Solid Tumor
      3

      LUAD
      Primary Solid Tumor
      525

      Recurrent Solid Tumor
      2

      LUSC
      Primary Solid Tumor
      501

      MESO
      Primary Solid Tumor
      87

      OV
      Primary Solid Tumor
      419

      Recurrent Solid Tumor
      7

      PAAD
      Metastatic
      1

      Primary Solid Tumor
      178

      PCPG
      Additional - New Primary
      3

      Metastatic
      2

      Primary Solid Tumor
      179

      PRAD
      Metastatic
      1

      Primary Solid Tumor
      501

      READ
      Primary Solid Tumor
      164

      Recurrent Solid Tumor
      1

      SARC
      Metastatic
      1

      Primary Solid Tumor
      259

      Recurrent Solid Tumor
      3

      SKCM
      Additional Metastatic
      1

      Metastatic
      367

      Primary Solid Tumor
      103

      STAD
      Primary Solid Tumor
      416

      TGCT
      Additional - New Primary
      4

      Primary Solid Tumor
      133

      THCA
      Metastatic
      8

      Primary Solid Tumor
      505

      UCEC
      Primary Solid Tumor
      549

      Recurrent Solid Tumor
      1

      UCS
      Primary Solid Tumor
      57

      UVM
      Primary Solid Tumor
      80

```python
all_data.SampleType.unique().tolist()
```

    ['Primary Solid Tumor',
     'Metastatic',
     'Recurrent Solid Tumor',
     'Additional Metastatic',
     'Additional - New Primary']

```python
sample_size = pd.crosstab(all_data["CancerType"], all_data["SampleType"])
sample_size.T.style
```

        CancerType	        ACC	        BLCA	        BRCA	        CESC	        CHOL	        COAD	        ESCA	        GBM	        HNSC	        KICH	        KIRC	        KIRP	        LGG	        LIHC	        LUAD	        LUSC	        MESO	        OV	        PAAD	        PCPG	        PRAD	        READ	        SARC	        SKCM	        STAD	        TGCT	        THCA	        UCEC	        UCS	        UVM

        SampleType

                        Additional - New Primary
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        1
                        1
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        3
                        0
                        0
                        0
                        0
                        0
                        4
                        0
                        0
                        0
                        0

                        Additional Metastatic
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        1
                        0
                        0
                        0
                        0
                        0
                        0

                        Metastatic
                        0
                        0
                        7
                        2
                        0
                        1
                        1
                        0
                        2
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        0
                        1
                        2
                        1
                        0
                        1
                        367
                        0
                        0
                        8
                        0
                        0
                        0

                        Primary Solid Tumor
                        79
                        411
                        1084
                        304
                        36
                        466
                        184
                        127
                        520
                        66
                        490
                        280
                        513
                        371
                        525
                        501
                        87
                        419
                        178
                        179
                        501
                        164
                        259
                        103
                        416
                        133
                        505
                        549
                        57
                        80

                        Recurrent Solid Tumor
                        0
                        0
                        0
                        0
                        0
                        1
                        0
                        13
                        0
                        0
                        0
                        0
                        18
                        3
                        2
                        0
                        0
                        7
                        0
                        0
                        0
                        1
                        3
                        0
                        0
                        0
                        0
                        1
                        0
                        0

```python
all_data
```

      SampleID
      CancerType
      B.cells.naive
      B.cells.memory
      Plasma.cells
      T.cells.CD8
      T.cells.CD4.naive
      T.cells.CD4.memory.resting
      T.cells.CD4.memory.activated
      T.cells.follicular.helper
      ...
      Mast.cells.resting
      Mast.cells.activated
      Eosinophils
      Neutrophils
      P.value
      Correlation
      RMSE
      LF
      SampleType
      PatientID

      6674
      TCGA-92-7341-01A-31R-2045-07
      LUSC
      0.032062
      0.000000
      0.019012
      0.007248
      0.0
      0.037050
      0.000000
      0.000000
      ...
      0.004019
      0.000226
      0.0
      0.001008
      0.006
      0.287252
      0.969659
      0.216818
      Primary Solid Tumor
      TCGA-92-7341

      7939
      TCGA-CH-5754-01A-11R-1580-07
      PRAD
      0.016235
      0.000000
      0.024199
      0.036240
      0.0
      0.035825
      0.000000
      0.033709
      ...
      0.005931
      0.000000
      0.0
      0.000000
      0.014
      0.229421
      1.006918
      0.248506
      Primary Solid Tumor
      TCGA-CH-5754

      11182
      TCGA-BS-A0V7-01A-21R-A118-07
      UCEC
      0.000000
      0.017987
      0.015623
      0.023201
      0.0
      0.009220
      0.001108
      0.015088
      ...
      0.011392
      0.000000
      0.0
      0.000040
      0.004
      0.366437
      0.936339
      0.138962
      Primary Solid Tumor
      TCGA-BS-A0V7

      1419
      TCGA-A2-A0CQ-01A-21R-A034-07
      BRCA
      0.000000
      0.005377
      0.000000
      0.023907
      0.0
      0.004971
      0.000000
      0.002223
      ...
      0.007146
      0.000000
      0.0
      0.000350
      0.012
      0.221909
      1.001612
      0.120546
      Primary Solid Tumor
      TCGA-A2-A0CQ

      6205
      TCGA-05-5420-01A-01R-1628-07
      LUAD
      0.018083
      0.000000
      0.007553
      0.049914
      0.0
      0.065995
      0.029929
      0.031451
      ...
      0.011843
      0.000000
      0.0
      0.008363
      0.216
      0.048965
      1.068250
      0.571093
      Primary Solid Tumor
      TCGA-05-5420

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      3172
      TCGA-CV-5432-01A-02R-1686-07
      HNSC
      0.022781
      0.000000
      0.046809
      0.018859
      0.0
      0.004296
      0.000000
      0.007688
      ...
      0.007073
      0.000000
      0.0
      0.000394
      0.002
      0.437382
      0.903511
      0.179283
      Primary Solid Tumor
      TCGA-CV-5432

      2824
      TCGA-L5-A8NQ-01A-11R-A36D-31
      ESCA
      0.000000
      0.015278
      0.004437
      0.044826
      0.0
      0.014477
      0.000000
      0.039537
      ...
      0.020992
      0.000000
      0.0
      0.000000
      0.096
      0.101506
      1.059170
      0.392468
      Primary Solid Tumor
      TCGA-L5-A8NQ

      9431
      TCGA-FP-8210-01A-11R-2343-13
      STAD
      0.091272
      0.048768
      0.000086
      0.071441
      0.0
      0.056546
      0.000000
      0.032408
      ...
      0.035449
      0.000000
      0.0
      0.000000
      0.002
      0.337942
      0.949999
      0.488038
      Primary Solid Tumor
      TCGA-FP-8210

      8279
      TCGA-KK-A8I8-01A-11R-A36G-07
      PRAD
      0.000181
      0.006151
      0.001299
      0.013462
      0.0
      0.011792
      0.000000
      0.002425
      ...
      0.004253
      0.000000
      0.0
      0.000000
      0.222
      0.065246
      1.063335
      0.066327
      Primary Solid Tumor
      TCGA-KK-A8I8

      1305
      TCGA-C8-A130-01A-31R-A115-07
      BRCA
      0.012144
      0.000000
      0.006821
      0.025982
      0.0
      0.017169
      0.000000
      0.008874
      ...
      0.013288
      0.000000
      0.0
      0.000000
      0.012
      0.211412
      0.997238
      0.143928
      Primary Solid Tumor
      TCGA-C8-A130

10039 rows × 30 columns

```python
cell_types = ['B.cells.naive', 'B.cells.memory', 'Plasma.cells', 'T.cells.CD8',
       'T.cells.CD4.naive', 'T.cells.CD4.memory.resting',
       'T.cells.CD4.memory.activated', 'T.cells.follicular.helper',
       'T.cells.regulatory..Tregs.', 'T.cells.gamma.delta', 'NK.cells.resting',
       'NK.cells.activated', 'Monocytes', 'Macrophages.M0', 'Macrophages.M1',
       'Macrophages.M2', 'Dendritic.cells.resting',
       'Dendritic.cells.activated', 'Mast.cells.resting',
       'Mast.cells.activated', 'Eosinophils', 'Neutrophils']
```

```python
all_data['Leukocytes.all'] = all_data[cell_types].sum(1)

all_data['T.cells.all'] = all_data[['T.cells.CD8',
                                    'T.cells.CD4.naive',
                                    'T.cells.CD4.memory.resting',
                                    'T.cells.CD4.memory.activated',
                                    'T.cells.follicular.helper',
                                    'T.cells.regulatory..Tregs.',
                                    'T.cells.gamma.delta']].sum(1)

all_data['B.cells.all'] = all_data[['B.cells.naive', 'B.cells.memory']].sum(1)

all_data['Nk.cells.all'] = all_data[['NK.cells.resting', 'NK.cells.activated']].sum(1)

all_data['Macrophages.all'] = all_data[['Macrophages.M0', 'Macrophages.M1', 'Macrophages.M2']].sum(1)

all_data['Dendritic.cells.all'] = all_data[['Dendritic.cells.resting', 'Dendritic.cells.activated']].sum(1)

all_data['Mast.cells.all'] = all_data[['Mast.cells.resting', 'Mast.cells.activated']].sum(1)

augmented_cell_types = cell_types + ['T.cells.all', 'B.cells.all', 'Nk.cells.all', 'Macrophages.all',
                                     'Dendritic.cells.all', 'Mast.cells.all', 'Leukocytes.all']
```

## Mean and variance

Sample mean $\bar{x}$, confidence interval $(\bar{x}_L, \bar{x}_U)$.
Sample standard deviation $s$, confidence interval $(s_L, s_U)$.

\begin{align}
\bar{x} & = \frac{\sum{}x_i}{n} \\
s & = \sqrt{\frac{\sum{}(x_i - \bar{x})^2}{n}} \\
\bar{x}_L, \bar{x}_U & = \bar{x} \pm t_{1-\alpha/2, \nu}\frac{s}{\sqrt{n}} \\
s_L & = \sqrt{\frac{(n-1)s^2}{\chi^2_{1-\alpha/2, \nu}}} \\
s_U & = \sqrt{\frac{(n-1)s^2}{\chi^2_{\alpha/2, \nu}}} \\
\end{align}

In this context, degree of freedom $\nu = n - 1$.

```python
sample_size.columns
```

    Index(['Additional - New Primary', 'Additional Metastatic', 'Metastatic',
           'Primary Solid Tumor', 'Recurrent Solid Tumor'],
          dtype='object', name='SampleType')

```python
a = 0.05

num_data = all_data[augmented_cell_types]

sanitize = lambda x: 0.0 if x < 0 else 1.0 if x > 1 else x

for sample_type in sample_size.columns:
    sample_type_mean = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_mean_lower = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_mean_upper = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_sd = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_sd_lower = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_sd_upper = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)

    for j in sample_type_mean.columns:
        n = ((all_data.CancerType == j) & (all_data.SampleType == sample_type)).sum()

        # Mean
        sample_type_mean.loc[:, j] = num_data[(all_data.CancerType == j) & (all_data.SampleType == sample_type)].mean(axis=0)

        # Standard deviation
        sample_type_sd.loc[:, j] = all_data[(all_data.CancerType == j) & (all_data.SampleType == sample_type)].std(ddof=1, axis=0)

        # Mean CI
        err = scipy.stats.t.ppf(1 - a / 2, n - 1) * sample_type_sd.loc[:, j] / np.sqrt(n)
        sample_type_mean_lower.loc[:, j] = sample_type_mean.loc[:, j] - err
        sample_type_mean_upper.loc[:, j] = sample_type_mean.loc[:, j] + err

        # Standard deviation CI
        sample_type_sd_lower.loc[:, j] = np.sqrt((n - 1) * sample_type_sd.loc[:, j] ** 2 / scipy.stats.chi2.ppf(1 - a / 2, n - 1))
        sample_type_sd_upper.loc[:, j] = np.sqrt((n - 1) * sample_type_sd.loc[:, j] ** 2 / scipy.stats.chi2.ppf(a / 2, n - 1))

    index_str_len = sample_type_mean.index.to_series().apply(lambda x: len(x)).max()
    index_str_len

    with pd.ExcelWriter(sample_type + " all.xlsx", engine="xlsxwriter") as writer:
        sample_type_mean.to_excel(writer, sheet_name="mean")
        sample_type_mean_lower.applymap(sanitize).to_excel(writer, sheet_name="mean 95% CI lower")
        sample_type_mean_upper.applymap(sanitize).to_excel(writer, sheet_name="mean 95% CI upper")

        sample_type_sd.to_excel(writer, sheet_name="sd")
        sample_type_sd_lower.to_excel(writer, sheet_name="sd 95% CI lower")
        sample_type_sd_upper.to_excel(writer, sheet_name="sd 95% CI upper")
        for i in writer.sheets:
            writer.sheets[i].set_column('A:A', index_str_len)
```

```python
with pd.ExcelWriter("all.xlsx", engine="xlsxwriter") as writer:
    sample_size.T.to_excel(writer, sheet_name="summary")
```

```python
scipy.stats.t.ppf(1 - a / 2, n - 1) * sample_type_sd.loc[:, j] / n
```

```python
a = 0.05

loglog = lambda x: np.log(-np.log(x))
expexp = lambda x: np.exp(-np.exp(x))

num_data = all_data[augmented_cell_types]

loglog_num_data = loglog(num_data)

for sample_type in sample_size.columns:
    sample_type_mean = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_loglog_mean = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_loglog_sd = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_mean_lower = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_mean_upper = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_sd = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_sd_lower = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)
    sample_type_sd_upper = pd.DataFrame(index = augmented_cell_types, columns = sample_size.index)

    for j in sample_type_mean.columns:
        n = ((all_data.CancerType == j) & (all_data.SampleType == sample_type)).sum()

        # Mean
        sample_type_mean.loc[:, j] = num_data[(all_data.CancerType == j) & (all_data.SampleType == sample_type)].mean(axis=0)
        sample_type_loglog_mean.loc[:, j] = loglog_num_data[(all_data.CancerType == j) & (all_data.SampleType == sample_type)].mean(axis=0)
        sample_type_loglog_sd.loc[:, j] = loglog_num_data[(all_data.CancerType == j) & (all_data.SampleType == sample_type)].std(ddof=1, axis=0)

        # Mean CI
        err = scipy.stats.t.ppf(1 - a / 2, n - 1) * sample_type_loglog_sd.loc[:, j] / np.sqrt(n)
        sample_type_mean_lower.loc[:, j] = (sample_type_loglog_mean.loc[:, j] + err).apply(expexp)
        sample_type_mean_upper.loc[:, j] = (sample_type_loglog_mean.loc[:, j] - err).apply(expexp)

        # Standard deviation
        sample_type_sd.loc[:, j] = all_data[(all_data.CancerType == j) & (all_data.SampleType == sample_type)].std(ddof=1, axis=0)

        # Standard deviation CI
        sample_type_sd_lower.loc[:, j] = np.sqrt((n - 1) * sample_type_sd.loc[:, j] ** 2 / scipy.stats.chi2.ppf(1 - a / 2, n - 1))
        sample_type_sd_upper.loc[:, j] = np.sqrt((n - 1) * sample_type_sd.loc[:, j] ** 2 / scipy.stats.chi2.ppf(a / 2, n - 1))

    index_str_len = sample_type_mean.index.to_series().apply(lambda x: len(x)).max()
    index_str_len

    with pd.ExcelWriter(sample_type + ".xlsx", engine="xlsxwriter") as writer:
        sample_type_mean.to_excel(writer, sheet_name="mean")
        sample_type_mean_lower.to_excel(writer, sheet_name="mean 95% CI lower")
        sample_type_mean_upper.to_excel(writer, sheet_name="mean 95% CI upper")
        (sample_type_mean_upper - sample_type_mean_lower).to_excel(writer, sheet_name="mean 95% CI size")

        sample_type_sd.to_excel(writer, sheet_name="sd")
        sample_type_sd_lower.to_excel(writer, sheet_name="sd 95% CI lower")
        sample_type_sd_upper.to_excel(writer, sheet_name="sd 95% CI upper")
        (sample_type_sd_upper - sample_type_sd_lower).to_excel(writer, sheet_name="sd 95% CI size")
        for i in writer.sheets:
            writer.sheets[i].set_column('A:A', index_str_len)
```

```python
loglog_num_data[(all_data.CancerType == j) & (all_data.SampleType == sample_type)].mean(axis=0)
```

    B.cells.naive                        inf
    B.cells.memory                       inf
    Plasma.cells                         inf
    T.cells.CD8                          inf
    T.cells.CD4.naive                    inf
    T.cells.CD4.memory.resting           inf
    T.cells.CD4.memory.activated         inf
    T.cells.follicular.helper            inf
    T.cells.regulatory..Tregs.           inf
    T.cells.gamma.delta                  inf
    NK.cells.resting                     inf
    NK.cells.activated                   inf
    Monocytes                            inf
    Macrophages.M0                       inf
    Macrophages.M1                       inf
    Macrophages.M2                  1.497889
    Dendritic.cells.resting              inf
    Dendritic.cells.activated            inf
    Mast.cells.resting                   inf
    Mast.cells.activated                 inf
    Eosinophils                          inf
    Neutrophils                          inf
    T.cells.all                     1.576024
    B.cells.all                          inf
    Nk.cells.all                         inf
    Macrophages.all                 1.441598
    Dendritic.cells.all                  inf
    Mast.cells.all                       inf
    Leukocytes.all                  1.180608
    dtype: float64

```python
num_data.min()
```

    B.cells.naive                   0.0
    B.cells.memory                  0.0
    Plasma.cells                    0.0
    T.cells.CD8                     0.0
    T.cells.CD4.naive               0.0
    T.cells.CD4.memory.resting      0.0
    T.cells.CD4.memory.activated    0.0
    T.cells.follicular.helper       0.0
    T.cells.regulatory..Tregs.      0.0
    T.cells.gamma.delta             0.0
    NK.cells.resting                0.0
    NK.cells.activated              0.0
    Monocytes                       0.0
    Macrophages.M0                  0.0
    Macrophages.M1                  0.0
    Macrophages.M2                  0.0
    Dendritic.cells.resting         0.0
    Dendritic.cells.activated       0.0
    Mast.cells.resting              0.0
    Mast.cells.activated            0.0
    Eosinophils                     0.0
    Neutrophils                     0.0
    T.cells.all                     0.0
    B.cells.all                     0.0
    Nk.cells.all                    0.0
    Macrophages.all                 0.0
    Dendritic.cells.all             0.0
    Mast.cells.all                  0.0
    Leukocytes.all                  0.0
    dtype: float64


# **Modelagem**

## **Instalação e importação**

```python
%reset -f
```

```python
#Instalar e importar bibliotecas:

try:
    import CoolProp
except ImportError:
    !pip install CoolProp

import matplotlib.pyplot as plt
from matplotlib.ticker import AutoMinorLocator
import CoolProp.CoolProp as cp
from CoolProp.CoolProp import PropsSI as ps
from CoolProp.CoolProp import State as st
from tabulate import tabulate as tab
import numpy as np
from sympy import *
import sympy as sp
import pandas as pd
import ipywidgets as wd
from IPython.display import display

from IPython.core.display import HTML
HTML("""
<style>
.output_png {
    display: table-cell;
    text-align: center;
    vertical-align: middle;
}
</style>
""")
```

<style>
.output_png {
    display: table-cell;
    text-align: center;
    vertical-align: middle;
}
</style>

## **Escala de temperatura**

```python
# Selecionar escala de temperatura:

Temperatura = input('Temperatura em °C ou em K? (Digite c ou k): ')
```

    Temperatura em °C ou em K? (Digite c ou k): C

## **Declaração de variáveis**

```python
# Declarar estados:

n = 12

fld = "Water"
cp.set_reference_state(fld,'DEF')
if Temperatura == 'k':
    Tnn = "Temperatura [K]"
else:
    Tnn = "Temperatura [°C]"

mnn = "Vazão mássica [kg/s]"
hnn = "Entalpia [kJ/kg]"
vnn = "Volume específico [m³/kg]"
Pnn = "Pressão [kPa]"
snn = "Entropia [kJ/kg*K]"
Xnn = "Título"

estado = np.arange(1, n+1, 1)

T = np.linspace(0,0,n)
m = np.linspace(0,0,n)
h = np.linspace(0,0,n)
v = np.linspace(0,0,n)
P = np.linspace(0,0,n)
s = np.linspace(0,0,n)
X = np.linspace(0,0,n)
```

## **Fornalha**

```python
# Combustível:

P_CH4 = 0.8897
P_C2H6 = 0.0592
P_C3H8 = 0.0191
P_C4H10 = 0.0109
P_C02 = 0.0121
P_N2 = 0.0089
P_O2 = 0.0001
P_N2_Ar = 0.79
P_O2_Ar = 0.21

# Excesso de ar:

e = 1.5
```

```python
#Cálculos:

CH4 = np.array([1,4,0,0])
C2H6 = np.array([2,6,0,0])
C3H8 = np.array([3,8,0,0])
C4H10 = np.array([4,10,0,0])
CO2 = np.array([1,0,2,0])
N2 = np.array([0,0,0,2])
O2 = np.array([0,0,2,0])
H2O = np.array([0,2,1,0])

MM = np.array([12.011,1.008,15.999,14.007])
P_GN = [P_CH4,P_C2H6,P_C3H8,P_C4H10,P_C02,P_N2,P_O2]

M_CH4 = np.sum(CH4*MM)
M_C2H6 = np.sum(C2H6*MM)
M_C3H8 = np.sum(C3H8*MM)
M_C4H10 = np.sum(C4H10*MM)
M_CO2 = np.sum(CO2*MM)
M_N2 = np.sum(N2*MM)
M_O2 = np.sum(O2*MM)
M_H2O = np.sum(H2O*MM)
M_Air = M_N2*P_N2_Ar+M_O2*P_O2_Ar

M = [M_CH4,M_C2H6,M_C3H8,M_C4H10,M_CO2,M_N2,M_O2,M_N2,M_H2O]

F_M = []
for i in range(0,4):
    F_M.append(P_GN)

np.array(F_M)
quant_atomos_comb = np.array([CH4,C2H6,C3H8,C4H10,CO2,N2,O2])
m_comp_comb = np.sum(F_M*quant_atomos_comb.T,axis=1)
m_comp_comb[2] = m_comp_comb[2]*(-1)

P_GN.append(P_N2_Ar/P_O2_Ar) #acrecentar partes de O2
P_GN.append(P_O2_Ar/P_O2_Ar) ##acrecentar partes de N2

b = m_comp_comb[0]
c = m_comp_comb[1]/2
a = e*(m_comp_comb[2] + 2*b + c)/(2*P_GN[8])
d = (m_comp_comb[3] + 2*a*P_GN[7])/2
f = (2*e*(2*b+c)/(2*P_GN[8])-c-2*b)/2

BLNC = [1,1,1,1,1,1,1,a,a]

resposta = np.zeros((9,4))
quant_atomos_molecula = np.array([CH4,C2H6,C3H8,C4H10,CO2,N2,O2,N2,O2])

for i in range(0,9):
    for j in range(0,4):
        resposta[i,j] = quant_atomos_molecula[i,j] * MM[j] * P_GN[i] * BLNC[i]

fuel = resposta[0:7,0:4]
air = resposta[7:9,0:4]
soma = np.sum(resposta)
M_fuel = np.sum(fuel) #Massa total de combustível
soma_air = np.sum(air) #Massa total de ar
total_elem = np.sum(fuel, axis=0,keepdims=True) #C_tot, H_tot, O_tot, N_tot

F_tot = total_elem/M_fuel

PCS = 33900*F_tot[0,0] + 141800*(F_tot[0,1] - (F_tot[0,2]/8))

w = PCS/2440 - 9*F_tot[0,1]
F_w = w/M_fuel

PCI = PCS - 2440*(9*F_tot[0,1] - F_w)

ȦḞ = a*(1+79/21)
AF = ȦḞ*(M_Air/M_fuel)

print('A razão ar-combustível é de %.2f kg/kg\n' %AF)

elem = ['C', 'H', 'O', 'N']

for i in range(0,4):
    print(f'A razão {elem[i]}-combustível é de {round(F_tot[0,i],2)} kg/kg\n')

if e == 1:
    print('Mistura estequiométrica (e = 1)\n')
else:
    print(f'Mistura com {int((e-1)*100)}% de excesso de ar (e = {e})\n')

print('O PCI é de %.2f kJ/kg' %PCI)
```

    A razão ar-combustível é de 24.22 kg/kg

    A razão C-combustível é de 0.74 kg/kg

    A razão H-combustível é de 0.23 kg/kg

    A razão O-combustível é de 0.02 kg/kg

    A razão N-combustível é de 0.01 kg/kg

    Mistura com 50% de excesso de ar (e = 1.5)

    O PCI é de 54933.88 kJ/kg

## **Equação**

<p style="font-family: times; text-align: center; font-style:bold; font-size: 14.5pt">
    ({{P_GN[0]}})$CH_{4}$ + ({{P_GN[1]}})$C_{2}H_{6}$ + ({{P_GN[2]}})$C_{3}H_{8}$ + ({{P_GN[3]}})$C_{4}H_{10}$ + ({{P_GN[4]}})$CO_{2}$ + ({{P_GN[5]}})$N_{2}$ + ({{P_GN[6]}})$O_{2}$ + ({{round(a,4)}})$\cdot[$({{round(P_GN[7],4)}})$N_{2(Ar)}$ + ({{P_GN[8]}})$O_{2(Ar)}]$   $\longrightarrow$   ({{round(b,4)}})$CO_{2}$ + ({{round(c,4)}})$H_{2}O$ + ({{round(d,4)}})$N_{2}$ + ({{round(f,4)}})$O_{2}$
</p>

## **Estados**

```python
# Estado 1 - Entrada da turbina:

m[0] = 27.9 #kg/s
P[0] = 6495 #kPa                          ---> 8000kPa (Teste); 6495kPa (Original)
T[0] = 485 + 273.15 #K                    --->   600°C (Teste);   485°C (Original)
st1 = st(fld, {'P':P[0],'T':T[0]})
h[0] = st1.h         #entalpia
s[0] = st1.s         #entropia
X[0] = st1.Q
v[0] = 1/st1.rho
```

```python
# Estado 2 - Primeira extração da turbina:

P[1] = 900      #kPa
st_isen2 = st(fld,{'P':P[1],'S':s[0]})     #turbina/bomba isentrópica ideal
h_isen2 = st_isen2.h
η_turb = 0.85       #eficiência da turbina
h[1] = h[0] - (h[0] - h_isen2) * η_turb
st2 = st(fld,{'P':P[1],'H':h[1]})
s[1] = st2.s
T[1] = st2.T
X[1] = st2.Q
v[1] = 1/st2.rho
```

```python
# Estado 3 - Segunda extração da turbina:

P[2] = 250     #kPa
st_isen3 = st(fld,{'P':P[2],'S':s[1]})
h_isen3 = st_isen3.h
h[2] = h[1] - (h[1] - h_isen3) * η_turb
st3 = st(fld,{'P':P[2],'H':h[2]})
s[2] = st3.s
v[2] = 1/st3.rho
T[2] = st3.T
X[2] = st3.Q
```

```python
# Estado 4 - Saída final da turbina / entrada do condensador:

T[3] = 51 + 273.15 #K                     --->    30°C (Teste);    51°C (Original)
st4 = st(fld, {'T': T[3], 'Q': 1})
P[3] = st4.p
st_isen4 = st(fld,{'P':P[3],'S':s[2]})
h_isen4 = st_isen4.h
h[3] = h[2] - (h[2] - h_isen4) * η_turb
st4 = st(fld,{'P':P[3],'H':h[3]})
X[3] = st4.Q      #título
s[3] = st4.s
v[3] = 1/st4.rho
```

```python
# Estado 5 - Saída do condensador / entrada da bomba:

X[4] = 0
P[4] = P[3]
st5 = st(fld, {'P': P[4], 'Q': 0})
h[4] = st5.h
s[4] = st5.s
T[4] = st5.T
v[4] = 1/st5.rho
```

```python
# Estado 6 - Entrada do desaerador / saída da bomba:

P[5] = P[2]
st_isen6 = st(fld, {'P': P[5], 'S': s[4]})
h_isen6 = st_isen6.h
η_pump = 0.85       #eficiência das bombas
h[5] = (h_isen6 - h[4]) / η_pump + h[4]
st6 = st(fld, {'P':P[5], 'H': h[5]})
s[5] = st6.s
T[5] = st6.T
X[5] = st6.Q
v[5] = 1/st6.rho
```

```python
# Estado 7 - Saída do desaerador:

P[6] = P[2]
T[6] = 110 + 273.15
st7 = st(fld, {'P': P[6], 'T': T[6]})
h[6] = st7.h
s[6] = st7.s
X[6] = st7.Q
v[6] = 1/st7.rho
```

```python
# Estado 8 - Entrada da caldeira:

P[7] = P[0]
st_isen8 = st(fld, {'P': P[7], 'S': s[6]})
h_isen8 = st_isen8.h
h[7] = (h_isen8 - h[6]) / η_pump + h[6]
st8 = st(fld, {'P': P[7], 'H': h[7]})
T[7] = st8.T
s[7] = st8.s
X[7] = st8.Q
v[7] = 1/st8.rho
```

```python
# Estado 9 - Entrada da bomba:

st9 = st7
h[8] = st9.h
s[8] = st9.s
P[8] = st9.p
T[8] = st9.T
X[8] = st9.Q
v[8] = 1/st9.rho
```

```python
# Estado 10 - Saída da bomba / entrada do trocador de calor:

P[9] = P[1]
st_isen10 = st(fld, {'P': P[9], 'S': s[8]})
h_isen10 = st_isen10.h
h[9] = (h_isen10 - h[8]) / η_pump + h[8]
st10 = st(fld, {'P': P[9], 'H': h[9]})
s[9] = st10.s
T[9] = st10.T
X[9] = st10.Q
v[9] = 1/st10.rho
```

```python
# Estado 11 - Entrada do processo industrial vizinho:

X[10] = 1
P[10] = P[9]
st11 = st(fld, {'P': P[10], 'Q': X[10]})
h[10] = st11.h
s[10] = st11.s
T[10] = st11.T
v[10] = 1/st11.rho
```

```python
# Estado 12 - Saída do processo industrial vizinho:

X[11] = 0
P[11] = P[9]
st12 = st(fld, {'P':P[11], 'Q': X[11]})
h[11] = st12.h
s[11] = st12.s
T[11] = st12.T
v[11] = 1/st12.rho
```

## **Tabela - Estados**

```python
# Tabela 1:
headers=["Estado", Tnn, Pnn, Xnn, hnn, snn, vnn]

for i in range(0, n):
    if Temperatura == 'k':
        T[i] = round(float(T[i]),2)
    else:
        T[i] = round(float(T[i]-273.15),2)
        P[i] = round(float(P[i]),2)
        X[i] = round(float(X[i]),2)
        h[i] = round(float(h[i]),2)
        s[i] = round(float(s[i]),4)
        v[i] = round(float(v[i]),6)

X_arrumado = []

for i in range(0, n):
    if X[i] == -1:
        X_arrumado.append('-')
    else:
        X_arrumado.append(X[i])

df1 = pd.DataFrame()
tabela_estados = {Tnn: T, Pnn: P, Xnn: X_arrumado, hnn: h, snn: s, vnn: v}
df1 = pd.DataFrame(tabela_estados, index=["Estado 1", "Estado 2", "Estado 3", "Estado 4", "Estado 5", "Estado 6", "Estado 7", "Estado 8", "Estado 9", "Estado 10", "Estado 11", "Estado 12"])

#Voltar temperatura à condição inicial:

for i in range(0, n):
    if Temperatura == 'k':
        T[i] = round(float(T[i]),2)
    else:
        T[i] = round(float(T[i]+273.15),2)

st_antes = [st1, st2, st3, st4, st5, st6, st7, st8, st9, st10, st11, st12]
```

{{df1}}

## **Cálculos**

```python
# Cálculos:
m1 = m[0]
h1 = h[0]
h2 = h[1]
h3 = h[2]
h4 = h[3]
h5 = h[4]
h6 = h[5]
h7 = h[6]
h8 = h[7]
h9 = h[8]
h10 = h[9]
h11 = h[10]
h12 = h[11]

# Quantidade de iterações:
quant = 10

variacao = np.zeros(quant + 1, dtype = float)
resultados = np.zeros((n+1)*(quant + 1), dtype = float).reshape((n+1, quant + 1))

T_hout_cond = []
η_sys = []
W_liq =[]

STT_T = []
STT_P = []
STT_X = []
STT_h = []
STT_s = []
STT_v = []
Ti = np.linspace(0,0,n)
Xi = list(range(0,n))

# Vazões mássicas:
for i in range(quant + 1):
    processo = i/10

    m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12 = var('m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12')
    EqMassa = [m8-m1, m4-m5, m5-m6, m9-m10, m11-m12, m2+m10-m11, m2+m3+m4-m1, m12+m3+m6-m7, m2*h2+m10*h10-m11*h11, m12*h12+m3*h3+m6*h6-m7*h7, m12-6.94*processo]
    massa = linsolve(EqMassa, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12)
    (m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12) =  next(iter(massa))

    massa = np.array(list(massa))

    for j in range(quant + 1):
        massa[0,j] = round(float(massa[0,j]),2)

    variacao[i] = i*10
    resultados[0, i] = variacao[i]
    resultados[1, i] = m[0]
    resultados[2:(n+1), i] = massa

    # Saídas de energia:
    W_p1 = -m5*(h5 - h6) #(Perde energia "-")
    W_p2 = -m8*(h7 - h8) #(Perde energia "-")
    W_p3 = -m9*(h9 - h10) #(Perde energia "-")
    W_pump = W_p1 + W_p2 + W_p3 #(Somatório dos trabalhos da bombas)

    # Eficiência da caldeira:
    η_cald = 0.85
    #PCI = 10000 # kJ/kg

    # Encontra o trabalho gerado pela turbina e a vazão de combustível para a demanda:
    W_t = m1*h1 - m2*h2 - m3*h3 - m4*h4
    m_fuel = m1*(h1-h8)/(PCI*η_cald) #η_cald = m1*(h1-h8)/(m_fuel*PCI)

    # Entrada de energia:
    Q_in = m_fuel * PCI #(Ganha energia "+")

    η_sys.append(round((W_t - W_pump)/Q_in, 4))

    W_liq.append(round(η_sys[i]*Q_in,2))

    # Condensador:
    T_cout = 39.5
    T_cin = 29
    T_hin = T[3] - 273.15

    ϵ = (T_cout - T_cin)/(T_hin - T_cin)
    NUT = -ln(1 - ϵ)

    CT_in = st(fld, {'P': 101.325, 'T': T_cin + 273.15})
    CT_out = st(fld, {'P': 101.325, 'T': T_cout + 273.15})

    C_min = 4500/3600*(CT_in.rho + CT_out.rho)/2*(CT_in.cp + CT_out.cp)/2

    UA = NUT * C_min # Coeficiente global de tranferência de calor * Área

    # Sistema não-linear:
    ΔT_0, ΔT_L, T_hout, ΔT_ml, Q_out = var('ΔT_0, ΔT_L, T_hout, ΔT_ml, Q_out')
    EqCond = [ΔT_0-(T_hout-T_cin), ΔT_L-(T_hout-T_cout), ΔT_ml-(ΔT_0-ΔT_L)/ln(ΔT_0/ΔT_L), Q_out-UA*ΔT_ml, Q_out-m4*(h4-h5)]
    EqCond_solve = nonlinsolve(EqCond, ΔT_0, ΔT_L, T_hout, ΔT_ml, Q_out)
    (ΔT_0, ΔT_L, T_hout, ΔT_ml, Q_out) = next(iter(EqCond_solve))

    # Redefinindo estados:
    T[3] = T_hout + 273.15 #K
    #P[3] = psi('P','T',T[3],'Q', 1, fld)/1000  #divide por 1000 pra sair em kPa
    st4 = st(fld, {'T': T[3], 'Q': 1})
    P[3] = st4.p
    st_isen4 = st(fld,{'P':P[3],'S':s[2]})
    h_isen4 = st_isen4.h
    h[3] = h[2] - (h[2] - h_isen4) * η_turb
    st4 = st(fld,{'P':P[3],'H':h[3]})
    X[3] = st4.Q      #título
    s[3]=st4.s
    v[3] = 1/st4.rho

    X[4] = 0
    P[4] = P[3]
    st5 = st(fld, {'P': P[4], 'Q': X[4]})
    h[4] = st5.h
    s[4] = st5.s
    T[4] = st5.T
    v[4] = 1/st5.rho

    P[5] = P[2]
    st_isen6 = st(fld, {'P': P[5], 'S': s[4]})
    h_isen6 = st_isen6.h
    η_pump = 0.85       #eficiência das bombas
    h[5] = (h_isen6 - h[4]) / η_pump + h[4]
    st6 = st(fld, {'P':P[5], 'H': h[5]})
    s[5] = st6.s
    T[5] = st6.T
    X[5] = st6.Q
    v[5] = 1/st6.rho

    P[6] = P[2]
    T[6] = 110 + 273.15
    st7 = st(fld, {'P': P[6], 'T': T[6]})
    h[6] = st7.h
    s[6] = st7.s
    X[6] = st7.Q
    v[6] = 1/st7.rho

    P[7] = P[0]
    st_isen8 = st(fld, {'P': P[7], 'S': s[6]})
    h_isen8 = st_isen8.h
    h[7] = (h_isen8 - h[6]) / η_pump + h[6]
    st8 = st(fld, {'P': P[7], 'H': h[7]})
    T[7] = st8.T
    s[7] = st8.s
    X[7] = st8.Q
    v[7] = 1/st8.rho

    st9 = st7
    h[8] = st9.h
    s[8] = st9.s
    P[8] = st9.p
    T[8] = st9.T
    X[8] = st9.Q
    v[8] = 1/st9.rho

    P[9] = P[1]
    st_isen10 = st(fld, {'P': P[9], 'S': s[8]})
    h_isen10 = st_isen10.h
    h[9] = (h_isen10 - h[8]) / η_pump + h[8]
    st10 = st(fld, {'P': P[9], 'H': h[9]})
    s[9] = st10.s
    T[9] = st10.T
    X[9] = st10.Q
    v[9] = 1/st10.rho

    X[10] = 1
    P[10] = P[9]
    st11 = st(fld, {'P': P[10], 'Q': X[10]})
    h[10] = st11.h
    s[10] = st11.s
    T[10] = st11.T
    v[10] = 1/st11.rho

    X[11] = 0
    P[11] = P[9]
    st12 = st(fld, {'P':P[11], 'Q': X[11]})
    h[11] = st12.h
    s[11] = st12.s
    T[11] = st12.T
    v[11] = 1/st12.rho

    for i in range(0,n):
        if Temperatura == 'k':
            Ti[i] = round(T[i],2)
        else:
            Ti[i] = round(T[i]-273.15,2)
            P[i] = round(P[i],2)
            X[i] = round(X[i],2)
            h[i] = round(h[i],2)
            s[i] = round(s[i],4)
            v[i] = round(v[i],6)
        if X[i] == -1:
            Xi[i] = '-'
        else:
            Xi[i] = X[i]

    STT_T.append([Tnn,processo*100,Ti[0],Ti[1],Ti[2],Ti[3],Ti[4],Ti[5],Ti[6],Ti[7],Ti[8],Ti[9],Ti[10],Ti[11]])
    STT_P.append([Pnn,processo*100,P[0],P[1],P[2],P[3],P[4],P[5],P[6],P[7],P[8],P[9],P[10],P[11]])
    STT_X.append([Xnn,processo*100,Xi[0],Xi[1],Xi[2],Xi[3],Xi[4],Xi[5],Xi[6],Xi[7],Xi[8],Xi[9],Xi[10],Xi[11]])
    STT_h.append([hnn,processo*100,h[0],h[1],h[2],h[3],h[4],h[5],h[6],h[7],h[8],h[9],h[10],h[11]])
    STT_s.append([snn,processo*100,s[0],s[1],s[2],s[3],s[4],s[5],s[6],s[7],s[8],s[9],s[10],s[11]])
    STT_v.append([vnn,processo*100,v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7],v[8],v[9],v[10],v[11]])

    T_hout = round(T_hout, 2)
    T_hout_cond.append(T_hout)

'''# Adiciona linhas vazias entre as propriedades na tabela:
STT_T.append(["","","","","","","","","","","","","",""])
STT_P.append(["","","","","","","","","","","","","",""])
STT_X.append(["","","","","","","","","","","","","",""])
STT_h.append(["","","","","","","","","","","","","",""])
STT_s.append(["","","","","","","","","","","","","",""])
STT_v.append(["","","","","","","","","","","","","",""])
'''
# Incluir na tabela e gráfico:
np.reshape(variacao, (1,n-1))
np.reshape(η_sys, (1,n-1))
np.reshape(T_hout_cond, (1,n-1))
np.reshape(W_liq, (1,n-1))

for i in range(0, n):
    T[i] = round(T[i],2)
    P[i] = round(P[i],2)
    X[i] = round(X[i],2)
    h[i] = round(h[i],2)
    s[i] = round(s[i],2)
    v[i] = round(v[i],2)

x = []
y = []
z = []
w = []

for i in range(0, quant+1):
    x.append(variacao[i])
    y.append(T_hout_cond[i])
    z.append(η_sys[i])
    w.append(W_liq[i])

table2 = resultados.T

resultados2 = np.zeros((n+4)*(quant + 1), dtype = float).reshape((n+4, quant + 1))

for i in range(0, quant+1):
    variacao[i] = i*10
    resultados2[0, i] = resultados[0, i]
    resultados2[1, i] = resultados[1, i]
    resultados2[2:(n+1), i] = resultados[2:(n+1), i]
    resultados2[n+2,i] = round(float(w[i]),2)
    resultados2[n+3,i] = round(float(z[i]),4)
table2 = resultados2.T

st_depois = [st1, st2, st3, st4, st5, st6, st7, st8, st9, st10, st11, st12]
```

## **Temperaturas da fornalha**

```python
# Balanço de energia da caldeira:

m_air = AF*m_fuel
m_gas = m_air + m_fuel
m13 = m8
m14 = m_fuel
m15 = 0.1*m_air
m16 = 0.9*m_air
m17 = m16
m18 = m_gas
m19 = m18
m20 = m19

fld2 = 'Air'

T_amb = 20 + 273.15 #K
P_amb = 101.325 #kPa

T15 = T_amb
T16 = T_amb
T8 = T[7]
T20 = 200 + 273.15 #K
P20 = P_amb
P8 = P[7]

def myfunction(ΔT13, ΔT17):

    from CoolProp.CoolProp import State as st

    T20 = 200 + 273.15 #K

    #Economizador:
    st13 = st(fld, {'P': P8, 'Q': 0})

    T13 = st13.T - ΔT13   #chute

    st13_2 = st(fld, {'P': P8, 'T': T13})
    h13 = st13_2.h

    q13 = m13*(h13-h8)

    st20_1 = st('Water', {'P':P_amb, 'T':T20})
    st20_2 = st('CO2', {'P':P_amb, 'T':T20})
    st20_3 = st('Nitrogen', {'P':P_amb, 'T':T20})

    cp_20 = 0.0997*st20_1.cp+0.1771*st20_2.cp+0.7232*st20_3.cp

    #cachorrada
    cp_19 = cp_20
    cp_18 = cp_20

    T17 = T16 + 200 + ΔT17  #chute

    st16 = st(fld2, {'P':P_amb, 'T':T_amb})
    st17 = st(fld2, {'P':P_amb, 'T':T17})
    h16 = st16.h
    h17 = st17.h

    st17_1 = st('Water', {'P':P_amb, 'T':T17})
    st17_2 = st('CO2', {'P':P_amb, 'T':T17})
    st17_3 = st('Nitrogen', {'P':P_amb, 'T':T17})

    cp_17 = 0.0997*st20_1.cp+0.1771*st20_2.cp+0.7232*st20_3.cp

    T18, T19, T20  = var('T18, T19, T20')
    EqTemp = [m19*cp_19*(T19-T_amb) + m8*h8 - h13*m13 - m20*cp_20*(T20-T_amb), m18*cp_18*(T18-T_amb) - m17*cp_17*(T17-T_amb) - m19*cp_19*(T19-T_amb), m_fuel*PCI + m17*cp_17*(T17-T_amb)  + h13*m13 - m1*h1 - m18*cp_18*(T18-T_amb)]
    temp = linsolve(EqTemp, T18, T19, T20)
    (T18, T19, T20) =  next(iter(temp))

    Q_disp = m_fuel*PCI + m17*cp_17*(T17-T_amb)
    Q_util = Q_disp - (m_gas*cp_18*(T18-T_amb))
    η_forn = round(Q_util/Q_disp,4)

    df = pd.DataFrame()
    Valores = [T13,T17,T18,T19,T20,η_forn*100]
    for i in range(0,5):
        Valores[i] = round((Valores[i]-273.15),2)

    data = {'Valores':Valores}
    df = pd.DataFrame(data, index=['T13 [°C]','T17 [°C]','T18 [°C]','T19 [°C]','T20 [°C]','η fornalha [%]'])
    df.style.hide_index()
    return df

slider_ΔT13 = wd.IntSlider(min=30, max=100, step=10)
slider_ΔT17 = wd.IntSlider(min=-100, max=100, step=10)
wd.interact(myfunction, ΔT13=slider_ΔT13, ΔT17=slider_ΔT17)
```

    interactive(children=(IntSlider(value=30, description='ΔT13', min=30, step=10), IntSlider(value=0, description…

    <function __main__.myfunction(ΔT13, ΔT17)>

## **Tabela - Vazões mássicas**

```python
# Tabela 2:

percentagem = []

for i in range(0, quant+1):
    percentagem.append(str(int(resultados2[0,i])) + '%')
    if Temperatura == 'k':
        resultados2[n+1,i] = round(float(y[i]+273.15),2) #converter para K
        y[i] = resultados2[n+1,i]
    else:
        resultados2[n+1,i] = round(float(y[i]),2)

df2 = pd.DataFrame()
tabela_ṁ = {"ṁ1 [kg/s]": resultados2[1,:], "ṁ2 [kg/s]": resultados2[2,:], "ṁ3 [kg/s]": resultados2[3,:], "ṁ4 [kg/s]": resultados2[4,:], "ṁ5 [kg/s]": resultados2[5,:], "ṁ6 [kg/s]": resultados2[6,:], "ṁ7 [kg/s]": resultados2[7,:], "ṁ8 [kg/s]": resultados2[8,:], "ṁ9 [kg/s]": resultados2[9,:], "ṁ10 [kg/s]": resultados2[10,:], "ṁ11 [kg/s]": resultados2[11,:], "ṁ12 [kg/s]": resultados2[12,:], "T_hout [°C]": resultados2[13,:], "W_liq [kJ/s]": resultados2[14,:], "η_sistema": resultados2[15,:]}
df2 = pd.DataFrame(tabela_ṁ, index=percentagem)

print('A vazão mássica de combustível é de %.2f kg/s' %m_fuel)
print('A vazão mássica de ar é de %.2f kg/s' %m_air)
```

    A vazão mássica de combustível é de 1.74 kg/s
    A vazão mássica de ar é de 42.15 kg/s

{{df2}}

```python
# Gráficos:

escala1 = 0.975
plt.figure(dpi=100, figsize=[escala1*2*6.4, escala1*4.8])
fig1 = plt.figure(dpi=100, figsize=[escala1*2*6.4, escala1*4.8])
plt.subplot(1,2,1)
plt.xlabel('Processo [%]', fontsize=12)

if Temperatura == 'k':
    plt.ylabel('Temperatura de saída do condensador [K]', fontsize=12)
    plt.plot(x, y, color='r', marker = 'o', linestyle = 'solid')
else:
    plt.ylabel('Temperatura de saída do condensador [°C]', fontsize=12)
    plt.plot(x, y, color='r', marker = 'o', linestyle = 'solid')

plt.grid()
plt.subplot(1,2,2)
plt.xlabel('Processo [%]', fontsize=12)
plt.ylabel('$η_{sistema}$', fontsize=12)
plt.plot(x, z, color='b', marker = 'o', linestyle = 'solid')
plt.grid()
plt.show()
```

## **Gráfico completo Ciclo Rankine**

```python
# Gráfico do Ciclo Rankine:
escala2 = 1
plt.figure(dpi=150, figsize=[escala2*6.4, escala2*4.8])
fig2 = plt.figure(dpi=150, figsize=[escala2*6.4, escala2*4.8])
y1 = 273.15
y2 = 850
x1 = -0.5
x2 = 10
plt.ylim(y1,y2)
plt.xlim(x1,x2)
plt.title('Ciclo Rankine',fontsize=20)
plt.xlabel('Entropia, s[kJ/kg$\cdot$K]')
plt.ylabel('Temperatura, T[K]')

Tmin = ps('Tmin',fld )
Tcrit = ps('Tcrit',fld)
Pcrit = ps('Pcrit', fld)

Larg_isolines = .25
Larg_cycle = 1.5
Larg_scatters = 15

#Linhas de entalpia:
T = np.linspace(Tmin,Tcrit,1000)
Q = np.arange(0.1,1,0.1)
for i in Q:
    s = ps('S','T',T,'Q',i,'Water')/1000
    plt.plot(s,T,'#6ca8fa',alpha=0.7,lw=Larg_isolines)

#Domo:
T = np.linspace(Tmin,Tcrit,1000)
s = ps('S','T',T,'Q',0,fld)/1000
plt.plot(s,T,'black',lw=2*Larg_isolines)

T = np.linspace(Tcrit,Tmin,1000)
s = ps('S','T',T,'Q',1,fld)/1000
plt.plot(s,T,'black',lw=2*Larg_isolines)

#Linhas de pressão constante:
T = np.linspace(Tmin,1000,1000)
P = [st_antes[0].p,st_antes[1].p,st_antes[2].p,st_antes[3].p,st_depois[3].p] #P1, P2, P3, P4 e P4*
for i in P:
    s = ps('S','T',T,'P',i*1000,'Water')/1000
    plt.plot(s,T,'#cc00ff',alpha=.7,lw=Larg_isolines)

#Ciclo:
#(antes)
T = np.linspace(st_antes[7].T,st_antes[0].T,1000)
s = ps('S','P',st_antes[0].p * 1000,'T',T,'Water')/1000
plt.plot(s,T,'r',lw=Larg_cycle)

T = np.linspace(st_antes[9].T,st_antes[1].T,1000)
s = ps('S','T',T,'P',st_antes[9].p*1000,'Water')/1000
plt.plot(s,T,'r',lw=Larg_cycle)
plt.plot([st_antes[3].s,st_antes[4].s],[st_antes[3].T,st_antes[4].T],'r',lw=Larg_cycle)

plt.plot([st_antes[0].s,st_antes[1].s],[st_antes[0].T,st_antes[1].T],color='r',linestyle='dashed',lw=Larg_cycle)
plt.plot([st_antes[1].s,st_antes[2].s],[st_antes[1].T,st_antes[2].T],color='r',linestyle='dashed',lw=Larg_cycle)

s = np.linspace(st_antes[2].s,st_antes[6].s,1000)
T = ps('T','S',s*1000,'P',st_antes[2].p*1000,'Water')
plt.plot(s,T,'r',lw=Larg_cycle)

T = np.linspace(st_antes[5].T,st_antes[6].T,1000)
s = ps('S','P',st_antes[5].p*1000,'T',T,'Water')/1000
plt.plot(s,T,'r',lw=Larg_cycle)

s = np.linspace(st_antes[2].s,st_antes[3].s,1000)
plt.plot([st_antes[3].s,st_antes[2].s],[st_antes[3].T,st_antes[2].T],color='r',linestyle='dashed',lw=Larg_cycle)

#(depois)
T = np.linspace(st_depois[7].T,st_depois[0].T,1000)
s = ps('S','P',st_depois[0].p * 1000,'T',T,'Water')/1000
plt.plot(s,T,'lime',lw=Larg_cycle)

T = np.linspace(st_depois[9].T,st_depois[1].T,1000)
s = ps('S','T',T,'P',st_depois[9].p*1000,'Water')/1000
plt.plot(s,T,'lime',lw=Larg_cycle)
plt.plot([st_depois[3].s,st_depois[4].s],[st_depois[3].T,st_depois[4].T],'lime',lw=Larg_cycle)

plt.plot([st_depois[0].s,st_depois[1].s],[st_depois[0].T,st_depois[1].T],color='lime',linestyle='dashed',lw=Larg_cycle)
plt.plot([st_depois[1].s,st_depois[2].s],[st_depois[1].T,st_depois[2].T],color='lime',linestyle='dashed',lw=Larg_cycle)

s = np.linspace(st_depois[2].s,st_depois[6].s,1000)
T = ps('T','S',s*1000,'P',st_depois[2].p*1000,'Water')
plt.plot(s,T,'lime',lw=Larg_cycle)

T = np.linspace(st_depois[5].T,st_depois[6].T,1000)
s = ps('S','P',st_depois[5].p*1000,'T',T,'Water')/1000
plt.plot(s,T,'lime',lw=Larg_cycle)

s = np.linspace(st_depois[2].s,st_depois[3].s,1000)
plt.plot([st_depois[3].s,st_depois[2].s],[st_depois[3].T,st_depois[2].T],color='lime',linestyle='dashed',lw=Larg_cycle)

#Pontuar e nomear:
for i in range(0, n):
    plt.scatter(st_antes[i].s,st_antes[i].T,zorder=5,color='k',s=Larg_scatters)
    plt.scatter(st_depois[i].s,st_depois[i].T,zorder=5,color='k',s=Larg_scatters)

plt.text(st_antes[0].s+.1,st_antes[0].T,'1',ha='left')
plt.text(st_antes[1].s+.1,st_antes[1].T,'2',ha='left')
plt.text(st_antes[2].s+.1,st_antes[2].T,'3',ha='left')
plt.text(st_antes[3].s+.1,st_antes[3].T,'4',ha='left')
plt.text(st_antes[4].s-.1,st_antes[4].T,'5, 6',ha='right')
plt.text(st_antes[6].s-.1,st_antes[6].T,'7, 8, 9, 10',ha='right')
plt.text(st_antes[10].s-.1,st_antes[10].T,'11',va='bottom',ha='right')
plt.text(st_antes[11].s-.1,st_antes[11].T+.1,'12',ha='right')
plt.text(st_depois[3].s+.1,st_depois[3].T,'4*',va='top',ha='left')
plt.text(st_depois[4].s-.1,st_depois[4].T,'5*, 6*',va='top',ha='right')

#plt.grid(lw=Larg_isolines)
plt.twinx()
plt.ylim(y1-273.15,y2-273.15)
plt.ylabel('Temperatura, T[°C]')
#plt.grid(lw=Larg_isolines)
plt.show()

# Recobrar os estados anteriores ao gráfico:

del T
del P
del s

T = []
P = []
s = []

for i in range(0, n):
    T.append(round(st_depois[i].T,2))
    P.append(round(st_depois[i].p,2))
    s.append(round(st_depois[i].s,2))
```

## **Tabela relatório completo Ciclo Rankine**

```python
# Tabela 3:

table3 = np.array(STT_T + STT_P + STT_X + STT_h + STT_s + STT_v)

df3 = pd.DataFrame()
pd.set_option('display.max_rows', None)
tabela_geral = {"Propriedade": table3[:,0], "%": table3[:,1], "Estado 1": table3[:,2], "Estado 2": table3[:,3], "Estado 3": table3[:,4], "Estado 4": table3[:,5], "Estado 5": table3[:,6], "Estado 6": table3[:,7], "Estado 7": table3[:,8], "Estado 8": table3[:,9], "Estado 9": table3[:,10], "Estado 10": table3[:,11], "Estado 11": table3[:,12], "Estado 12": table3[:,13]}
df3 = pd.DataFrame(tabela_geral)
```

{{df3}}

## **Salvar dados e mídias**

```python
# Salvar tabelas em excel:

resp = 0
while resp != 's' or resp != 'S' or resp != 'n' or resp != 'N':
    resp = input('Salvar imagens dos gráficos? (S ou N) ')
    if resp == 's' or resp == 'S' or resp == 'n' or resp == 'N':
        break

if resp == 's' or resp == 'S':
    print('Aguarde...\n')
    fig1.savefig('Gráficos de temperatura e eficência do conddensador.png', dpi = 500)
    fig2.savefig('Diagrama T-s.png', dpi = 500)
del resp

resp = 0
while resp != 's' or resp != 'S' or resp != 'n' or resp != 'N':
    resp = input('Salvar tabelas em Excel? (S ou N) ')
    if resp == 's' or resp == 'S' or resp == 'n' or resp == 'N':
        break

if resp == 's' or resp == 'S':
    print('Aguarde...\n')
    with pd.ExcelWriter('Tabelas.xlsx') as writer:
        df1.to_excel(writer, sheet_name='Tabela 1')
        df2.to_excel(writer, sheet_name='Tabela 2')
        df3.to_excel(writer, sheet_name='Tabela 3', index=None)
del resp

class color:
    PURPLE = '\033[95m'
    CYAN = '\033[96m'
    DARKCYAN = '\033[36m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
    END = '\033[0m'

print(color.GREEN + color.BOLD + '\nPrograma executado com sucesso!')
```

    Salvar imagens dos gráficos? (S ou N) N
    Salvar tabelas em Excel? (S ou N) N

    Programa executado com sucesso!


# Aim of this notebook

* To construct the singular curve of universal type to finalize the solution of the optimal control problem

# Preamble

```python
from sympy import *
init_printing(use_latex='mathjax')

# Plotting
%matplotlib inline
## Make inline plots raster graphics
from IPython.display import set_matplotlib_formats
## Import modules for plotting and data analysis
import matplotlib.pyplot as plt
from matplotlib import gridspec,rc,colors
import matplotlib.ticker as plticker
# Parameters for seaborn plots
import seaborn as sns
clrs = sns.color_palette("Spectral", 6)
def set_plot_style(usetex=False):
    sns.set_style('white', {'axes.linewidth': 0.5})
    sns.set(style='white', font_scale=1.1,#context='paper',
            rc={'xtick.major.size': 6, 'ytick.major.size': 6, 'legend.fontsize': 14,
                'text.usetex': usetex, 'font.family': 'serif', 'font.serif': ['Verdana'],
                'text.latex.preamble': r"\usepackage{type1cm}"})
    plt.rcParams['xtick.major.size'] = 6
    plt.rcParams['xtick.major.width'] = 1
    plt.rcParams['ytick.major.size'] = 6
    plt.rcParams['ytick.major.width'] = 1
    plt.rcParams['xtick.bottom'] = True
    plt.rcParams['ytick.left'] = True

set_plot_style(True)

import pandas as pd
pd.set_option('mode.chained_assignment',None)

import numpy as np
from scipy.optimize import fsolve, root
from scipy.integrate import ode
backend = 'dopri5'
import warnings

# Timer
import time

from copy import deepcopy

from itertools import cycle
palette_size = 10;
clrs = sns.color_palette("Reds",palette_size)
iclrs = cycle(clrs) # iterated colors

clrs0 = sns.color_palette("Set1",palette_size)

# Suppress warnings
import warnings
warnings.filterwarnings("ignore")
```

# Parameter values

* Birth rate and const of downregulation are defined below in order to fit some experim. data

```python
d = .13 # death rate
α = .3 # low equilibrium point at expression of the main pathway (high equilibrium is at one)
θ = .45 # threshold value for the expression of the main pathway
κ = 40 # robustness parameter
```

* Symbolic variables - the list insludes μ & μbar, because they will be varied later

```python
σ, φ0, φ, x, μ, μbar = symbols('sigma, phi0, phi, x, mu, mubar')
```

* Main functions

```python
A = 1-σ*(1-θ)
Eminus = (α*A-θ)**2/2
ΔE = A*(1-α)*((1+α)*A/2-θ)
ΔEf = lambdify(σ,ΔE)
```

* Birth rate and cost of downregulation

```python
b = (0.1*(exp(κ*(ΔEf(1)))+1)-0.14*(exp(κ*ΔEf(0))+1))/(exp(κ*ΔEf(1))-exp(κ*ΔEf(0))) # birth rate
χ = 1-(0.14*(exp(κ*ΔEf(0))+1)-b*exp(κ*ΔEf(0)))/b
b, χ
```

$$\left ( 0.140168330860362, \quad 0.325961223954473\right )$$

```python
c_relative = 0.1
c = c_relative*(b-d)/b+(1-c_relative)*χ/(exp(κ*ΔEf(0))+1) # cost of resistance
c
```

$$0.00833519849448376$$

* Hamiltonian *H* and a part of it ρ that includes the control variable σ

```python
h = b*(χ/(exp(κ*ΔE)+1)*(1-x)+c*x)
H = -φ0 + φ*(b*(χ/(exp(κ*ΔE)+1)-c)*x*(1-x)+μ*(1-x)/(exp(κ*ΔE)+1)-μbar*exp(-κ*Eminus)*x) + h
ρ = (φ*(b*χ*x+μ)+b*χ)/(exp(κ*ΔE)+1)*(1-x)-φ*μbar*exp(-κ*Eminus)*x
ρ1 = (φ*(b*χ*x+μ)+b*χ)/(exp(κ*ΔE)+1)*(1-x)
ρ2 = φ*μbar*exp(-κ*Eminus)*x
n = b*(1-χ*(1-x)/(exp(κ*ΔE)+1)-c*x)-d
H, ρ, n
```

$$\left ( \phi \left(\frac{\mu \left(- x + 1\right)}{e^{40 \left(- 0.385 \sigma + 0.7\right) \left(- 0.3575 \sigma + 0.2\right)} + 1} - \bar{\mu} x e^{- 20 \left(- 0.165 \sigma - 0.15\right)^{2}} + x \left(-0.00116833086036159 + \frac{0.045689440686899}{e^{40 \left(- 0.385 \sigma + 0.7\right) \left(- 0.3575 \sigma + 0.2\right)} + 1}\right) \left(- x + 1\right)\right) - \phi_{0} + 0.00116833086036159 x + \frac{0.045689440686899 \left(- x + 1\right)}{e^{40 \left(- 0.385 \sigma + 0.7\right) \left(- 0.3575 \sigma + 0.2\right)} + 1}, \quad - \bar{\mu} \phi x e^{- 20 \left(- 0.165 \sigma - 0.15\right)^{2}} + \frac{\left(- x + 1\right) \left(\phi \left(\mu + 0.045689440686899 x\right) + 0.045689440686899\right)}{e^{40 \left(- 0.385 \sigma + 0.7\right) \left(- 0.3575 \sigma + 0.2\right)} + 1}, \quad - 0.00116833086036159 x - \frac{0.140168330860362 \left(- 0.325961223954473 x + 0.325961223954473\right)}{e^{40 \left(- 0.385 \sigma + 0.7\right) \left(- 0.3575 \sigma + 0.2\right)} + 1} + 0.0101683308603616\right )$$

* Same but for no treatment (σ = 0)

```python
h0 = h.subs(σ,0)
H0 = H.subs(σ,0)
ρ0 = ρ.subs(σ,0)
H0, ρ0
```

$$\left ( \phi \left(0.00368423989943599 \mu \left(- x + 1\right) - 0.637628151621773 \bar{\mu} x - 0.001 x \left(- x + 1\right)\right) - \phi_{0} + 0.001 x + 0.000168330860361587, \quad - 0.637628151621773 \bar{\mu} \phi x + 0.00368423989943599 \left(- x + 1\right) \left(\phi \left(\mu + 0.045689440686899 x\right) + 0.045689440686899\right)\right )$$

* Machinery: definition of the Poisson brackets

```python
PoissonBrackets = lambda H1, H2: diff(H1,x)*diff(H2,φ)-diff(H1,φ)*diff(H2,x)
```

* Necessary functions and defining the right hand side of dynamical equations

```python
ρf = lambdify((x,φ,σ,μ,μbar),ρ)
ρ1f = lambdify((x,φ,σ,μ,μbar),ρ1)
ρ2f = lambdify((x,φ,σ,μ,μbar),ρ2)
ρ0f = lambdify((x,φ,μ,μbar),ρ0)
dxdτ = lambdify((x,φ,σ,μ,μbar),-diff(H,φ))
dφdτ = lambdify((x,φ,σ,μ,μbar),diff(H,x))
# dndτ = lambdify((x,φ,σ,μ,μbar),-n)
dVdτ = lambdify((x,σ),h)
dρdσ = lambdify((σ,x,φ,μ,μbar),diff(ρ,σ))
dδρdτ = lambdify((x,φ,σ,μ,μbar),-PoissonBrackets(ρ0-ρ,H))
def ode_rhs(t,state,μ,μbar):
    x, φ, V, δρ = state
    σs = [0,1]
    if (dρdσ(1.,x,φ,μ,μbar)<0) and (dρdσ(θ,x,φ,μ,μbar)>0):
        σstar = fsolve(dρdσ,.8,args=(x,φ,μ,μbar,))[0]
    else:
        σstar = 1.;
    if ρf(x,φ,σstar,μ,μbar) < ρ0f(x,φ,μ,μbar):
        sgm = 0
    else:
        sgm = σstar
    return [dxdτ(x,φ,sgm,μ,μbar),dφdτ(x,φ,sgm,μ,μbar),dVdτ(x,sgm),dδρdτ(x,φ,σstar,μ,μbar)]
def σstarf(x,φ,μ,μbar):
    if (dρdσ(1.,x,φ,μ,μbar)<0) and (dρdσ(θ,x,φ,μ,μbar)>0):
        σstar = fsolve(dρdσ,.8,args=(x,φ,μ,μbar,))[0]
    else:
        σstar = 1.;
    if ρf(x,φ,σstar,μ,μbar) < ρ0f(x,φ,μ,μbar):
        sgm = 0
    else:
        sgm = σstar
    return sgm
```

```python
def get_primary_field(name, experiment,μ,μbar):
    solutions = {}
    solver = ode(ode_rhs).set_integrator(backend)
    τ0 = experiment['τ0']
    tms = np.linspace(τ0,experiment['T_end'],1e3+1)
    for x0 in experiment['x0']:
        δρ0 = ρ0.subs(x,x0).subs(φ,0)-ρ.subs(x,x0).subs(φ,0).subs(σ,1.)
        solver.set_initial_value([x0,0,0,δρ0],0.).set_f_params(μ,μbar)
        sol = []; k = 0;
        while (solver.t < experiment['T_end']) and (solver.y[0]<=1.) and (solver.y[0]>=0.):
            solver.integrate(tms[k])
            sol.append([solver.t]+list(solver.y))
            k += 1
        solutions[x0] = {'solution': sol}
    for x0, entry in solutions.items():
        entry['τ'] = [entry['solution'][j][0] for j in range(len(entry['solution']))]
        entry['x'] = [entry['solution'][j][1] for j in range(len(entry['solution']))]
        entry['φ'] = [entry['solution'][j][2] for j in range(len(entry['solution']))]
        entry['V'] = [entry['solution'][j][3] for j in range(len(entry['solution']))]
        entry['δρ'] = [entry['solution'][j][4] for j in range(len(entry['solution']))]
    return solutions
def get_δρ_value(tme,x0,μ,μbar):
    solver = ode(ode_rhs).set_integrator(backend)
    δρ0 = ρ0.subs(x,x0).subs(φ,0)-ρ.subs(x,x0).subs(φ,0).subs(σ,1.)
    solver.set_initial_value([x0,0,0,δρ0],0.).set_f_params(μ,μbar)
    while (solver.t < tme) and (solver.y[0]<=1.) and (solver.y[0]>=0.):
        solver.integrate(tme)
        sol = [solver.t]+list(solver.y)
    return solver.y[3]
def get_δρ_ending(params,μ,μbar):
    tme, x0 = params
    solver = ode(ode_rhs).set_integrator(backend)
    δρ0 = ρ0.subs(x,x0).subs(φ,0)-ρ.subs(x,x0).subs(φ,0).subs(σ,1.)
    solver.set_initial_value([x0,0,0,δρ0],0.).set_f_params(μ,μbar)
    δτ = 1.0e-8; tms = [tme,tme+δτ]
    _k = 0; sol = []
    while (_k<len(tms)):# and (solver.y[0]<=1.) and (solver.y[0]>=0.):
        solver.integrate(tms[_k])
        sol.append(solver.y)
        _k += 1
    #print(sol)
    return(sol[0][3],(sol[1][3]-sol[0][3])/δτ)
def get_state(tme,x0,μ,μbar):
    solver = ode(ode_rhs).set_integrator(backend)
    δρ0 = ρ0.subs(x,x0).subs(φ,0)-ρ.subs(x,x0).subs(φ,0).subs(σ,1.)
    solver.set_initial_value([x0,0,0,δρ0],0.).set_f_params(μ,μbar)
    δτ = 1.0e-8; tms = [tme,tme+δτ]
    _k = 0; sol = []
    while (solver.t < tms[-1]) and (solver.y[0]<=1.) and (solver.y[0]>=0.):
        solver.integrate(tms[_k])
        sol.append(solver.y)
        _k += 1
    return(list(sol[0])+[(sol[1][3]-sol[0][3])/δτ])
```

# Machinery for the universal line

* To find the universal singular curve we need to define two parameters

```python
γ0 = PoissonBrackets(PoissonBrackets(H,H0),H)
γ1 = PoissonBrackets(PoissonBrackets(H0,H),H0)
```

* The dynamics

```python
dxdτSingExpr = -(γ0*diff(H0,φ)+γ1*diff(H,φ))/(γ0+γ1)
dφdτSingExpr = (γ0*diff(H0,x)+γ1*diff(H,x))/(γ0+γ1)
dVdτSingExpr = (γ0*h0+γ1*h)/(γ0+γ1)
σSingExpr = γ1*σ/(γ0+γ1)
```

* Machinery for Python: lambdify the functions above

```python
dxdτSing = lambdify((x,φ,σ,μ,μbar),dxdτSingExpr)
dφdτSing = lambdify((x,φ,σ,μ,μbar),dφdτSingExpr)
dVdτSing = lambdify((x,φ,σ,μ,μbar),dVdτSingExpr)
σSing = lambdify((x,φ,σ,μ,μbar),σSingExpr)
```

```python
def ode_rhs_Sing(t,state,μ,μbar):
    x, φ, V = state
    if (dρdσ(1.,x,φ,μ,μbar)<0) and (dρdσ(θ,x,φ,μ,μbar)>0):
        σstar = fsolve(dρdσ,.8,args=(x,φ,μ,μbar,))[0]
    else:
        σstar = 1.;
    return [dxdτSing(x,φ,σstar,μ,μbar),dφdτSing(x,φ,σstar,μ,μbar),dVdτSing(x,φ,σstar,μ,μbar)]
def get_universal_curve(end_point,tmax,Nsteps,μ,μbar):
    tms =  np.linspace(end_point[0],tmax,Nsteps);
    solver = ode(ode_rhs_Sing).set_integrator(backend)
    solver.set_initial_value(end_point[1:4],end_point[0]).set_f_params(μ,μbar)
    _k = 0; sol = []
    while (solver.t < tms[-1]):
        solver.integrate(tms[_k])
        sol.append([solver.t]+list(solver.y))
        _k += 1
    return sol
def get_σ_universal(tme,end_point,μ,μbar):
    δτ = 1.0e-8; tms = [tme,tme+δτ]
    solver = ode(ode_rhs_Sing).set_integrator(backend)
    solver.set_initial_value(end_point[1:4],end_point[0]).set_f_params(μ,μbar)
    _k = 0; sol = []
    while (solver.t < tme+δτ):
        solver.integrate(tms[_k])
        sol.append([solver.t]+list(solver.y))
        _k += 1
    x, φ = sol[0][:2]
    sgm = fsolve(lambda σ: dxdτ(x,φ,σ,μ,μbar)-(sol[1][0]-sol[0][0])/δτ,θ/2)[0]
    return sgm
def get_state_universal(tme,end_point,μ,μbar):
    solver = ode(ode_rhs_Sing).set_integrator(backend)
    solver.set_initial_value(end_point[1:4],end_point[0]).set_f_params(μ,μbar)
    solver.integrate(tme)
    return [solver.t]+list(solver.y)
```

```python
def ode_rhs_with_σstar(t,state,μ,μbar):
    x, φ, V = state
    if (dρdσ(1.,x,φ,μ,μbar)<0) and (dρdσ(θ,x,φ,μ,μbar)>0):
        σ = fsolve(dρdσ,.8,args=(x,φ,μ,μbar,))[0]
    else:
        σ = 1.;
    return [dxdτ(x,φ,σ,μ,μbar),dφdτ(x,φ,σ,μ,μbar),dVdτ(x,σ)]
def ode_rhs_with_given_σ(t,state,σ,μ,μbar):
    x, φ, V = state
    return [dxdτ(x,φ,σ,μ,μbar),dφdτ(x,φ,σ,μ,μbar),dVdτ(x,σ)]
def get_trajectory_with_σstar(starting_point,tmax,Nsteps,μ,μbar):
    tms = np.linspace(starting_point[0],tmax,Nsteps)
    solver = ode(ode_rhs_with_σstar).set_integrator(backend)
    solver.set_initial_value(starting_point[1:],starting_point[0]).set_f_params(μ,μbar)
    sol = []; _k = 0;
    while solver.t < max(tms) and (solver.y[0]<=1.) and (solver.y[0]>=0.):
        solver.integrate(tms[_k])
        sol.append([solver.t]+list(solver.y))
        _k += 1
    return sol
def get_trajectory_with_given_σ(starting_point,tmax,Nsteps,σ,μ,μbar):
    tms = np.linspace(starting_point[0],tmax,100)
    solver = ode(ode_rhs_with_given_σ).set_integrator(backend)
    solver.set_initial_value(starting_point[1:],starting_point[0]).set_f_params(σ,μ,μbar)
    sol = []; _k = 0;
    while solver.t < max(tms) and (solver.y[0]<=1.) and (solver.y[0]>=0.):
        solver.integrate(tms[_k])
        sol.append([solver.t]+list(solver.y))
        _k += 1
    return sol
def get_state_with_σstar(tme,starting_point,μ,μbar):
    solver = ode(ode_rhs_with_σstar).set_integrator(backend)
    solver.set_initial_value(starting_point[1:4],starting_point[0]).set_f_params(μ,μbar)
    solver.integrate(tme)
    return [solver.t]+list(solver.y)
def get_finalizing_point_from_universal_curve(tme,tmx,end_point,μ,μbar):
    unv_point = get_state_universal(tme,end_point,μ,μbar)
    return get_state_with_σstar(tmx,unv_point,μ,μbar)[1]
```

# Field of optimal trajectories as the solution of the Bellman equation

* μ & μbar are varied by *T* and *T*bar ($\mu=1/T$ and $\bar\mu=1/\bar{T}$)

```python
tmx = 180.
end_switching_curve = {'t': 12., 'x': .7}
# for Τ, Τbar in zip([28]*5,[14,21,28,35,60]):
Τ = 10.5; Τbar = 14.0
μ = 1./Τ; μbar = 1./Τbar
print("Parameters: μ = %.5f, μbar = %.5f"%(μ,μbar))
end_switching_curve['t'], end_switching_curve['x'] = fsolve(get_δρ_ending,(end_switching_curve['t'],end_switching_curve['x']),args=(μ,μbar),xtol=1.0e-12)
end_point = [end_switching_curve['t']]+get_state(end_switching_curve['t'],end_switching_curve['x'],μ,μbar)
print("Ending point for the switching line: τ = %.1f days, x = %.1f%%" % (end_point[0], end_point[1]*100))
print("Checking the solution - should give zero values: ")
print(get_δρ_ending([end_switching_curve['t'],end_switching_curve['x']],μ,μbar))
print("* Constructing the primary field")
primary_field1 = []
experiments = {
    'sol1': { 'T_end': tmx, 'τ0': 0., 'x0': list(np.linspace(0,end_switching_curve['x']-(1e-3),7)) } }
for name, values in experiments.items():
    primary_field1.append(get_primary_field(name,values,μ,μbar))
primary_field2 = []
experiments = {
    'sol1': { 'T_end': tmx, 'τ0': 0., 'x0': list(np.linspace(end_switching_curve['x']+(3e-6),1.,7)) } }
for name, values in experiments.items():
    primary_field2.append(get_primary_field(name,values,μ,μbar))
print("* Constructing the switching curve")
switching_curve = []
x0s = np.linspace(end_switching_curve['x'],1,21); _y = end_switching_curve['t']

for x0 in x0s:
    tme = fsolve(get_δρ_value,_y,args=(x0,μ,μbar))[0]
    if (tme>0):
        switching_curve = switching_curve+[[tme,get_state(tme,x0,μ,μbar)[0]]]
        _y = tme
print("* Constructing the universal curve")
universal_curve = get_universal_curve(end_point,tmx,25,μ,μbar)
print("* Finding the last characteristic")
#time0 = time.time()
tuniv = fsolve(get_finalizing_point_from_universal_curve,tmx-40.,args=(tmx,end_point,μ,μbar,))[0]
#print("The proccess to find the last characteristic took %0.1f minutes" % ((time.time()-time0)/60.))
univ_point = get_state_universal(tuniv,end_point,μ,μbar)
print("The last point on the universal line:")
print(univ_point)
last_trajectory = get_trajectory_with_σstar(univ_point,tmx,50,μ,μbar)
print("Final state:")
final_state = get_state_with_σstar(tmx,univ_point,μ,μbar)
print(final_state)
```

    Parameters: μ = 0.09524, μbar = 0.07143
    Ending point for the switching line: τ = 11.1 days, x = 62.9%
    Checking the solution - should give zero values:
    (-1.6853786659789397e-09, -4.3053518384272386e-10)
    * Constructing the primary field
    * Constructing the switching curve
    * Constructing the universal curve
    * Finding the last characteristic
    The last point on the universal line:
    [168.21662930295125, 0.628557975772097, -0.2400762227850881, 1.3270349109165358]
    Final state:
    [180.0, -3.68066688238855e-13, -0.3878368709539695, 1.6191478873005891]

# Preparation for second figure

```python
for idx, tend in enumerate(np.r_[np.arange(30,240,30),np.arange(240,30*48,120)]):
    tuniv = fsolve(get_finalizing_point_from_universal_curve,tend-20.,args=(tend,end_point,μ,μbar,))[0]
    univ_point = get_state_universal(tuniv,end_point,μ,μbar)
    trajectory = get_trajectory_with_σstar(univ_point,tend,50,μ,μbar)
    plt.plot([x[0] for x in trajectory],[x[1] for x in trajectory],linewidth=1,color=clrs0[4])
    sol_ = [[tend,τ,σstarf(x,φ,μ,μbar),x,exp((b-d)*τ-V)] for τ,x,φ,V in trajectory]
    sol = sol+sol_ if idx else sol_
    universal_curve = get_universal_curve(end_point,univ_point[0],50,μ,μbar)
    plt.plot([x[0] for x in universal_curve],[x[1] for x in universal_curve],linewidth=3,color=clrs0[0],zorder=3)
    sol = [[tend,τ,get_σ_universal(τ,end_point,μ,μbar),x,exp((b-d)*τ-V)] for τ,x,φ,V in universal_curve] + sol
    trajectory = get_trajectory_with_σstar([0,end_switching_curve['x'],0,0],end_point[0],50,μ,μbar)
    sol = [[tend,τ,σstarf(x,φ,μ,μbar),x,exp((b-d)*τ-V)] for τ,x,φ,V in trajectory] + sol

pd.DataFrame(sol,columns=['T','time','sigma','resistance','fold_change']).\
    sort_values(['T','time']).to_csv('../figures/draft/Fig7X-trjs_optimal.csv',index=False,header=False)
```

```python

```

scripts/D-2. Field of optimal trajectories for different time horizons [Python].ipynb