{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "902dfc4a",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"import scanpy as sc\n",
"import anndata as ad\n",
"import sklearn as sk\n",
"\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.metrics import classification_report"
]
},
{
"cell_type": "markdown",
"id": "a5d9fb6e",
"metadata": {},
"source": [
"Load every available dataset."
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "d94b8e15",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 1min 59s, sys: 2min 25s, total: 4min 24s\n",
"Wall time: 4min 26s\n"
]
}
],
"source": [
"%%time\n",
"adatas = []\n",
"for i, row in pd.read_csv('personal.csv').iterrows():\n",
" try:\n",
" adata = sc.read(f'datasets/{row.Author}_{row.Year}.h5ad')\n",
" adata.obs['dataset'] = f'{row.Author}_{row.Year}'\n",
" adatas.append(adata)\n",
" except FileNotFoundError:\n",
" pass\n",
" \n",
"adata = ad.concat(adatas)"
]
},
{
"cell_type": "markdown",
"id": "2b65993f",
"metadata": {},
"source": [
"## Explore the data"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "ed4c66c8",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/icb/yuge.ji/miniconda3/envs/py37/lib/python3.7/site-packages/scanpy/_settings.py:447: DeprecationWarning: `set_matplotlib_formats` is deprecated since IPython 7.23, directly use `matplotlib_inline.backend_inline.set_matplotlib_formats()`\n",
" IPython.display.set_matplotlib_formats(*ipython_format)\n"
]
}
],
"source": [
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"sc.set_figure_params(dpi=100, frameon=False)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "c74e3c66",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" nth condition | \n",
" condition | \n",
" n_samples | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0 | \n",
" ELF1 | \n",
" 11676 | \n",
"
\n",
" \n",
" | 1 | \n",
" 1 | \n",
" control | \n",
" 8878 | \n",
"
\n",
" \n",
" | 2 | \n",
" 2 | \n",
" CREB1 | \n",
" 7587 | \n",
"
\n",
" \n",
" | 3 | \n",
" 3 | \n",
" ELK1 | \n",
" 7366 | \n",
"
\n",
" \n",
" | 4 | \n",
" 4 | \n",
" INTERGENIC393453 | \n",
" 7177 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"image/png": {
"height": 397,
"width": 863
},
"needs_background": "light"
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" nth condition | \n",
" condition | \n",
" n_samples | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0 | \n",
" | \n",
" 11835 | \n",
"
\n",
" \n",
" | 1 | \n",
" 1 | \n",
" KLF1 | \n",
" 1954 | \n",
"
\n",
" \n",
" | 2 | \n",
" 2 | \n",
" BAK1 | \n",
" 1451 | \n",
"
\n",
" \n",
" | 3 | \n",
" 3 | \n",
" CEBPE | \n",
" 1230 | \n",
"
\n",
" \n",
" | 4 | \n",
" 4 | \n",
" CEBPE,RUNX1T1 | \n",
" 1215 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"image/png": {
"height": 397,
"width": 863
},
"needs_background": "light"
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" nth condition | \n",
" condition | \n",
" n_samples | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0 | \n",
" JAK | \n",
" 1503 | \n",
"
\n",
" \n",
" | 1 | \n",
" 1 | \n",
" TIMP | \n",
" 1483 | \n",
"
\n",
" \n",
" | 2 | \n",
" 2 | \n",
" IFNGR | \n",
" 1427 | \n",
"
\n",
" \n",
" | 3 | \n",
" 3 | \n",
" SLC5A | \n",
" 1212 | \n",
"
\n",
" \n",
" | 4 | \n",
" 4 | \n",
" NPC | \n",
" 1151 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"image/png": {
"height": 397,
"width": 863
},
"needs_background": "light"
},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" nth condition | \n",
" condition | \n",
" n_samples | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0 | \n",
" Vehicle | \n",
" 12966 | \n",
"
\n",
" \n",
" | 1 | \n",
" 1 | \n",
" PD98059 | \n",
" 3754 | \n",
"
\n",
" \n",
" | 2 | \n",
" 2 | \n",
" Tranylcypromine | \n",
" 3735 | \n",
"
\n",
" \n",
" | 3 | \n",
" 3 | \n",
" WP1066 | \n",
" 3711 | \n",
"
\n",
" \n",
" | 4 | \n",
" 4 | \n",
" RG108 | \n",
" 3710 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"image/png": {
"height": 397,
"width": 863
},
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"for a in adatas:\n",
" df = pd.DataFrame(a.obs.perturbation_name.value_counts())\n",
" df = df.reset_index().reset_index()\n",
" df.columns = ['nth condition', 'condition', 'n_samples']\n",
" display(df.head(5))\n",
"\n",
" plt.figure(figsize=(10, 4))\n",
" sns.scatterplot(data=df, x='nth condition', y='n_samples', size=1, legend=None)\n",
" plt.yscale('log')\n",
" plt.title(a.obs.dataset.values[0])\n",
" plt.show()"
]
},
{
"cell_type": "markdown",
"id": "7f225cde",
"metadata": {},
"source": [
"Which perturbations are in these datasets?"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "cb5ae64a",
"metadata": {},
"outputs": [],
"source": [
"df = pd.crosstab(adata.obs.perturbation_name, adata.obs.dataset)\n",
"df['sum'] = df.sum(axis=1)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "71262c5e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"perturbation_name\n",
"(+)-JQ1 True\n",
"2-Methoxyestradiol True\n",
"A-366 True\n",
"ABT-737 True\n",
"AC480 True\n",
" ... \n",
"ZC3HAV1 True\n",
"ZM 447439 True\n",
"ZNF318 True\n",
"Zileuton True\n",
"control True\n",
"Length: 1364, dtype: bool"
]
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pd.crosstab(adata.obs.perturbation_name, adata.obs.dataset)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "bf7d558a",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | dataset | \n",
" Dixit_2016 | \n",
" Frangieh_2021 | \n",
" Norman_2019 | \n",
" Srivatsan_2019 | \n",
" sum | \n",
"
\n",
" \n",
" | perturbation_name | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | control | \n",
" 8878 | \n",
" 0 | \n",
" 11835 | \n",
" 0 | \n",
" 20713 | \n",
"
\n",
" \n",
" | DMSO | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 12966 | \n",
" 12966 | \n",
"
\n",
" \n",
" | ELF1 | \n",
" 11676 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 11676 | \n",
"
\n",
" \n",
" | CREB1 | \n",
" 7587 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 7587 | \n",
"
\n",
" \n",
" | ELK1 | \n",
" 7366 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 7366 | \n",
"
\n",
" \n",
" | INTERGENIC393453 | \n",
" 7177 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 7177 | \n",
"
\n",
" \n",
" | EGR1 | \n",
" 6181 | \n",
" 0 | \n",
" 321 | \n",
" 0 | \n",
" 6502 | \n",
"
\n",
" \n",
" | GABPA | \n",
" 5462 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 5462 | \n",
"
\n",
" \n",
" | NR2C2 | \n",
" 5030 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 5030 | \n",
"
\n",
" \n",
" | ETS1 | \n",
" 4815 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 4815 | \n",
"
\n",
" \n",
" | E2F4 | \n",
" 4516 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 4516 | \n",
"
\n",
" \n",
" | IRF1 | \n",
" 3837 | \n",
" 0 | \n",
" 427 | \n",
" 0 | \n",
" 4264 | \n",
"
\n",
" \n",
" | PD98059 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3754 | \n",
" 3754 | \n",
"
\n",
" \n",
" | Tranylcypromine | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3735 | \n",
" 3735 | \n",
"
\n",
" \n",
" | WP1066 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3711 | \n",
" 3711 | \n",
"
\n",
" \n",
" | RG108 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3710 | \n",
" 3710 | \n",
"
\n",
" \n",
" | Tubastatin A | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3706 | \n",
" 3706 | \n",
"
\n",
" \n",
" | Curcumin | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3685 | \n",
" 3685 | \n",
"
\n",
" \n",
" | GSK-LSD1 2 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3677 | \n",
" 3677 | \n",
"
\n",
" \n",
" | Busulfan | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3670 | \n",
" 3670 | \n",
"
\n",
" \n",
" | SRT2104 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3669 | \n",
" 3669 | \n",
"
\n",
" \n",
" | Baricitinib | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3664 | \n",
" 3664 | \n",
"
\n",
" \n",
" | Tacedinaline | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3652 | \n",
" 3652 | \n",
"
\n",
" \n",
" | Capecitabine | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3646 | \n",
" 3646 | \n",
"
\n",
" \n",
" | Tofacitinib | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3635 | \n",
" 3635 | \n",
"
\n",
" \n",
" | Tazemetostat | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3634 | \n",
" 3634 | \n",
"
\n",
" \n",
" | Entacapone | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3623 | \n",
" 3623 | \n",
"
\n",
" \n",
" | BRD4770 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3619 | \n",
" 3619 | \n",
"
\n",
" \n",
" | WHI-P154 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3614 | \n",
" 3614 | \n",
"
\n",
" \n",
" | Mesna | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3612 | \n",
" 3612 | \n",
"
\n",
" \n",
" | CEP-33779 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3598 | \n",
" 3598 | \n",
"
\n",
" \n",
" | Cerdulatinib | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3593 | \n",
" 3593 | \n",
"
\n",
" \n",
" | Anacardic Acid | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3593 | \n",
" 3593 | \n",
"
\n",
" \n",
" | GSK J1 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3592 | \n",
" 3592 | \n",
"
\n",
" \n",
" | Daphnetin | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3578 | \n",
" 3578 | \n",
"
\n",
" \n",
" | Altretamine | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3566 | \n",
" 3566 | \n",
"
\n",
" \n",
" | PJ34 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3561 | \n",
" 3561 | \n",
"
\n",
" \n",
" | Filgotinib | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3552 | \n",
" 3552 | \n",
"
\n",
" \n",
" | Triamcinolone Acetonide | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3546 | \n",
" 3546 | \n",
"
\n",
" \n",
" | UNC0631 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3543 | \n",
" 3543 | \n",
"
\n",
" \n",
" | Ofloxacin | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3543 | \n",
" 3543 | \n",
"
\n",
" \n",
" | Clevudine | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3534 | \n",
" 3534 | \n",
"
\n",
" \n",
" | Valproic acid sodium salt | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3529 | \n",
" 3529 | \n",
"
\n",
" \n",
" | Sirtinol | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3529 | \n",
" 3529 | \n",
"
\n",
" \n",
" | AICAR | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3527 | \n",
" 3527 | \n",
"
\n",
" \n",
" | AG-490 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3523 | \n",
" 3523 | \n",
"
\n",
" \n",
" | INTERGENIC216151 | \n",
" 3518 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3518 | \n",
"
\n",
" \n",
" | INTERGENIC1144056 | \n",
" 3518 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3518 | \n",
"
\n",
" \n",
" | NVP-BSK805 2 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3505 | \n",
" 3505 | \n",
"
\n",
" \n",
" | MK-0752 | \n",
" 0 | \n",
" 0 | \n",
" 0 | \n",
" 3495 | \n",
" 3495 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"image/png": {
"height": 1548,
"width": 444
},
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"sc.pl.dotplot(\n",
" freq[freq.obs.perturbation_name.isin(freq.obs.perturbation_name.values[:50])],\n",
" var_names=freq.var_names,\n",
" groupby='perturbation_name',\n",
" colorbar_title='n_samples'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "9340319e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 45,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"['a', 'b'].index('b')"
]
},
{
"cell_type": "markdown",
"id": "ad4a0817",
"metadata": {},
"source": [
"Which perturbations exist across multiple datasets?"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "d99119a5",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | dataset | \n",
" Dixit_2016 | \n",
" Frangieh_2021 | \n",
" Norman_2019 | \n",
" Srivatsan_2019 | \n",
"
\n",
" \n",
" | perturbation_name | \n",
" | \n",
" | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | EGR1 | \n",
" 6181 | \n",
" 0 | \n",
" 321 | \n",
" 0 | \n",
"
\n",
" \n",
" | IRF1 | \n",
" 3837 | \n",
" 0 | \n",
" 427 | \n",
" 0 | \n",
"
\n",
" \n",
" | NCL | \n",
" 0 | \n",
" 121 | \n",
" 237 | \n",
" 0 | \n",
"
\n",
" \n",
" | SET | \n",
" 0 | \n",
" 415 | \n",
" 985 | \n",
" 0 | \n",
"
\n",
" \n",
" | control | \n",
" 8878 | \n",
" 0 | \n",
" 11835 | \n",
" 0 | \n",
"
\n",
" \n",
"
\n",
"