{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# CyteOnto quick tutorial\n", "\n", "Compare author cell-type labels against one or more algorithm predictions using the Cell Ontology.\n", "\n", "This notebook uses the **primary** model pair (Nebius for descriptions and embeddings) with an optional **fallback** pair (Fireworks / OpenRouter) configured on `CyteOnto`.\n", "\n", "Set these in `.env` at the repo root (see `cyteonto/config.py` → `PROVIDER_API_KEY_ENV`):\n", "\n", "- `NEBIUS_API_KEY`\n", "- `FIREWORKS_API_KEY` (fallback LLM)\n", "- `OPENROUTER_API_KEY` (fallback embeddings)\n", "\n", "Optional: `NCBI_API_KEY` for PubMed tool use during description generation. Default is `False`." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import sys\n", "from pathlib import Path\n", "\n", "sys.path.insert(0, str(Path(\"..\").resolve()))\n", "\n", "from dotenv import load_dotenv\n", "\n", "load_dotenv(Path(\"..\") / \".env\")" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "import numpy as np\n", "import pandas as pd\n", "from pydantic_ai import Agent\n", "from pydantic_ai.models.openai import OpenAIChatModel\n", "from pydantic_ai.providers.openai import OpenAIProvider\n", "\n", "from cyteonto import CyteOnto, EmbdConfig, LlmConfig\n", "from cyteonto.config import Config\n", "from cyteonto.cyteonto import _hungarian_match_mean" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "cfg = Config()\n", "\n", "LLM_BASE_URLS = {\n", " \"nebius\": \"https://api.tokenfactory.nebius.com/v1/\",\n", " \"fireworks\": \"https://api.fireworks.ai/inference/v1\",\n", " \"openrouter\": \"https://openrouter.ai/api/v1/\",\n", " \"together\": \"https://api.together.xyz/v1/\",\n", "}\n", "\n", "\n", "def build_agent(provider: str, model: str) -> Agent:\n", " env_var = cfg.PROVIDER_API_KEY_ENV[provider]\n", " api_key = os.environ[env_var]\n", " return Agent(\n", " OpenAIChatModel(\n", " model,\n", " provider=OpenAIProvider(\n", " api_key=api_key,\n", " base_url=LLM_BASE_URLS[provider],\n", " ),\n", " )\n", " )\n", "\n", "\n", "primary_agent = build_agent(cfg.PRIMARY_LLM_PROVIDER, cfg.PRIMARY_LLM_MODEL)\n", "fallback_agent = build_agent(cfg.FALLBACK_LLM_PROVIDER, cfg.FALLBACK_LLM_MODEL)\n", "\n", "primary_embedding = EmbdConfig(\n", " provider=cfg.PRIMARY_EMBEDDING_PROVIDER,\n", " model=cfg.PRIMARY_EMBEDDING_MODEL,\n", " apiKey=os.environ[cfg.PROVIDER_API_KEY_ENV[cfg.PRIMARY_EMBEDDING_PROVIDER]],\n", " modelSettings={},\n", ")\n", "\n", "fallback_embedding = EmbdConfig(\n", " provider=cfg.FALLBACK_EMBEDDING_PROVIDER,\n", " model=cfg.FALLBACK_EMBEDDING_MODEL,\n", " apiKey=os.environ[cfg.PROVIDER_API_KEY_ENV[cfg.FALLBACK_EMBEDDING_PROVIDER]],\n", ")\n", "\n", "llm = LlmConfig(provider=cfg.PRIMARY_LLM_PROVIDER, model=cfg.PRIMARY_LLM_MODEL)\n", "fallback_llm = LlmConfig(\n", " provider=cfg.FALLBACK_LLM_PROVIDER, model=cfg.FALLBACK_LLM_MODEL\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Prepare ontology cache\n", "\n", "`from_config` ensures Cell Ontology descriptions and embeddings exist under `cyteonto/data/` for this model pair. The first run may take several minutes." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "cyto = await CyteOnto.from_config(\n", " agent=primary_agent,\n", " embedding=primary_embedding,\n", " llm=llm,\n", " fallback_agent=fallback_agent,\n", " fallback_embedding=fallback_embedding,\n", " fallback_llm=fallback_llm,\n", " use_pubmed_tool=False, # bool(os.getenv(\"NCBI_API_KEY\")),\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Run a comparison\n", "\n", "Each row is one `(algorithm, pair_index)` with ontology matches and a CyteScore similarity. Reuse `run_id` to benefit from on-disk caches." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "author_labels = [\n", " \"animal stem cell\",\n", " \"BFU-E\",\n", " \"neutrophilic granuloblast\",\n", "]\n", "\n", "algorithms = {\n", " \"method_a\": [\"stem cell\", \"blast forming unit erythroid\", \"spermatogonium\"],\n", " \"method_b\": [\"neuronal receptor cell\", \"stem cell\", \"ovum\"],\n", "}\n", "\n", "df = await cyto.compare(\n", " author_labels=author_labels,\n", " algorithms=algorithms,\n", " run_id=\"quick_tutorial_demo\",\n", " metric=\"cosine_kernel\",\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "id": "abe54d99", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
run_idalgorithmpair_indexauthor_labelalgorithm_labelauthor_ontology_idauthor_ontology_nameauthor_embedding_similarityalgorithm_ontology_idalgorithm_ontology_namealgorithm_embedding_similaritycytescore_similaritysimilarity_method
0quick_tutorial_demomethod_a0animal stem cellstem cellCL:0002248pluripotent stem cell0.8119CL:0000034stem cell0.82550.5450cytescore
1quick_tutorial_demomethod_a1BFU-Eblast forming unit erythroidCL:0000038erythroid progenitor cell0.8626CL:0000038erythroid progenitor cell0.82491.0000cytescore
2quick_tutorial_demomethod_a2neutrophilic granuloblastspermatogoniumCL:0000042neutrophilic myeloblast0.8360CL:0000020spermatogonium0.90610.1077cytescore
3quick_tutorial_demomethod_b0animal stem cellneuronal receptor cellCL:0002248pluripotent stem cell0.8119CL:0000006neuronal receptor cell0.86350.0671cytescore
4quick_tutorial_demomethod_b1BFU-Estem cellCL:0000038erythroid progenitor cell0.8626CL:0000034stem cell0.79250.1231cytescore
5quick_tutorial_demomethod_b2neutrophilic granuloblastovumCL:0000042neutrophilic myeloblast0.8360CL:0000025egg cell0.89050.0837cytescore
\n", "
" ], "text/plain": [ " run_id algorithm pair_index author_label \\\n", "0 quick_tutorial_demo method_a 0 animal stem cell \n", "1 quick_tutorial_demo method_a 1 BFU-E \n", "2 quick_tutorial_demo method_a 2 neutrophilic granuloblast \n", "3 quick_tutorial_demo method_b 0 animal stem cell \n", "4 quick_tutorial_demo method_b 1 BFU-E \n", "5 quick_tutorial_demo method_b 2 neutrophilic granuloblast \n", "\n", " algorithm_label author_ontology_id author_ontology_name \\\n", "0 stem cell CL:0002248 pluripotent stem cell \n", "1 blast forming unit erythroid CL:0000038 erythroid progenitor cell \n", "2 spermatogonium CL:0000042 neutrophilic myeloblast \n", "3 neuronal receptor cell CL:0002248 pluripotent stem cell \n", "4 stem cell CL:0000038 erythroid progenitor cell \n", "5 ovum CL:0000042 neutrophilic myeloblast \n", "\n", " author_embedding_similarity algorithm_ontology_id \\\n", "0 0.8119 CL:0000034 \n", "1 0.8626 CL:0000038 \n", "2 0.8360 CL:0000020 \n", "3 0.8119 CL:0000006 \n", "4 0.8626 CL:0000034 \n", "5 0.8360 CL:0000025 \n", "\n", " algorithm_ontology_name algorithm_embedding_similarity \\\n", "0 stem cell 0.8255 \n", "1 erythroid progenitor cell 0.8249 \n", "2 spermatogonium 0.9061 \n", "3 neuronal receptor cell 0.8635 \n", "4 stem cell 0.7925 \n", "5 egg cell 0.8905 \n", "\n", " cytescore_similarity similarity_method \n", "0 0.5450 cytescore \n", "1 1.0000 cytescore \n", "2 0.1077 cytescore \n", "3 0.0671 cytescore \n", "4 0.1231 cytescore \n", "5 0.0837 cytescore " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "run_id: quick_tutorial_demo\n", "model pair usage: {'llmTier': 'primary', 'embeddingTier': 'primary', 'llmProvider': 'nebius', 'llmModel': 'moonshotai/Kimi-K2.6', 'embeddingProvider': 'nebius', 'embeddingModel': 'Qwen/Qwen3-Embedding-8B'}\n", "LLM usage: {'agentName': 'CyteOnto', 'modelName': 'moonshotai/Kimi-K2.6', 'requests': 18, 'inputTokens': 8859, 'outputTokens': 1862, 'totalTokens': 10721, 'toolUsage': {'final_result': 19}}\n" ] } ], "source": [ "print(\"run_id:\", df[\"run_id\"].iloc[0])\n", "print(\"model pair usage:\", cyto.model_pair_usage.to_status_dict())\n", "print(\"LLM usage:\", cyto.usage.model_dump())" ] }, { "cell_type": "markdown", "id": "84651030", "metadata": {}, "source": [ "# Compound Label Scoring\n", "\n", "## Default Compound Comparison is Max-Score\n", "\n", "Mixture labels (doublets, mixed populations) are decomposed by the LLM into cell-type parts before scoring. Each input pair still produces **one row**.\n", "\n", "For compound pairs (`similarity_method == \"cytescore_compound\"`):\n" ] }, { "cell_type": "code", "execution_count": null, "id": "719ef730", "metadata": {}, "outputs": [], "source": [ "compound_scenarios = [\n", " (\"2x1_coverage\", \"AT2 cell-plasma cell doublet\", \"Plasma cell\"),\n", " (\"2x2_two_overlap\", \"AT2 cell-plasma cell doublet\", \"AT2 cell / Plasma cell\"),\n", " (\"2x2_one_overlap\", \"AT2 cell-plasma cell doublet\", \"AT2 cell / T cell\"),\n", " (\"2x2_zero_overlap\", \"AT2 cell-plasma cell doublet\", \"T cell / B cell\"),\n", " (\"3x2_one_overlap\", \"AT2 cell / plasma cell / NK cell\", \"AT2 cell / T cell\"),\n", " (\"simple_control\", \"T cell\", \"T cell\"),\n", "]\n", "\n", "compound_author_labels = [author for _, author, _ in compound_scenarios]\n", "compound_algorithm_labels = [algo for _, _, algo in compound_scenarios]\n", "\n", "compound_df_max = await cyto.compare(\n", " author_labels=compound_author_labels,\n", " algorithms={\"scenarios\": compound_algorithm_labels},\n", " run_id=\"quick_tutorial_compound\",\n", " metric=\"cosine_kernel\",\n", ")\n", "compound_df_max.insert(0, \"scenario\", [name for name, _, _ in compound_scenarios])" ] }, { "cell_type": "code", "execution_count": 8, "id": "95fbf3c0", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
scenariorun_idalgorithmpair_indexauthor_labelalgorithm_labelauthor_ontology_idauthor_ontology_nameauthor_embedding_similarityalgorithm_ontology_idalgorithm_ontology_namealgorithm_embedding_similaritycytescore_similaritysimilarity_method
02x1_coveragequick_tutorial_compoundscenarios0at2 cell-plasma cell doubletplasma cellCL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...0.9066;0.9281CL:0000786plasma cell0.91810.8975cytescore_compound
12x2_two_overlapquick_tutorial_compoundscenarios1at2 cell-plasma cell doubletat2 cell / plasma cellCL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...0.9066;0.9281CL:0002063;CL:0000786pulmonary alveolar type 2 cell;plasma cell0.8747;0.91811.0000cytescore_compound
22x2_one_overlapquick_tutorial_compoundscenarios2at2 cell-plasma cell doubletat2 cell / t cellCL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...0.9066;0.9281CL:0002063;CL:0000084pulmonary alveolar type 2 cell;T cell0.8747;0.88931.0000cytescore_compound
32x2_zero_overlapquick_tutorial_compoundscenarios3at2 cell-plasma cell doublett cell / b cellCL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...0.9066;0.9281CL:0000084;CL:0000945T cell;lymphocyte of B lineage0.8893;0.89530.4046cytescore_compound
43x2_one_overlapquick_tutorial_compoundscenarios4at2 cell / plasma cell / nk cellat2 cell / t cellCL:0002063;CL:0000946;CL:0000623pulmonary alveolar type 2 cell;antibody secret...0.9024;0.9281;0.8382CL:0002063;CL:0000084pulmonary alveolar type 2 cell;T cell0.8747;0.88931.0000cytescore_compound
5simple_controlquick_tutorial_compoundscenarios5t cellt cellCL:0000084T cell0.8688CL:0000084T cell0.88931.0000cytescore
\n", "
" ], "text/plain": [ " scenario run_id algorithm pair_index \\\n", "0 2x1_coverage quick_tutorial_compound scenarios 0 \n", "1 2x2_two_overlap quick_tutorial_compound scenarios 1 \n", "2 2x2_one_overlap quick_tutorial_compound scenarios 2 \n", "3 2x2_zero_overlap quick_tutorial_compound scenarios 3 \n", "4 3x2_one_overlap quick_tutorial_compound scenarios 4 \n", "5 simple_control quick_tutorial_compound scenarios 5 \n", "\n", " author_label algorithm_label \\\n", "0 at2 cell-plasma cell doublet plasma cell \n", "1 at2 cell-plasma cell doublet at2 cell / plasma cell \n", "2 at2 cell-plasma cell doublet at2 cell / t cell \n", "3 at2 cell-plasma cell doublet t cell / b cell \n", "4 at2 cell / plasma cell / nk cell at2 cell / t cell \n", "5 t cell t cell \n", "\n", " author_ontology_id \\\n", "0 CL:0002063;CL:0000946 \n", "1 CL:0002063;CL:0000946 \n", "2 CL:0002063;CL:0000946 \n", "3 CL:0002063;CL:0000946 \n", "4 CL:0002063;CL:0000946;CL:0000623 \n", "5 CL:0000084 \n", "\n", " author_ontology_name \\\n", "0 pulmonary alveolar type 2 cell;antibody secret... \n", "1 pulmonary alveolar type 2 cell;antibody secret... \n", "2 pulmonary alveolar type 2 cell;antibody secret... \n", "3 pulmonary alveolar type 2 cell;antibody secret... \n", "4 pulmonary alveolar type 2 cell;antibody secret... \n", "5 T cell \n", "\n", " author_embedding_similarity algorithm_ontology_id \\\n", "0 0.9066;0.9281 CL:0000786 \n", "1 0.9066;0.9281 CL:0002063;CL:0000786 \n", "2 0.9066;0.9281 CL:0002063;CL:0000084 \n", "3 0.9066;0.9281 CL:0000084;CL:0000945 \n", "4 0.9024;0.9281;0.8382 CL:0002063;CL:0000084 \n", "5 0.8688 CL:0000084 \n", "\n", " algorithm_ontology_name algorithm_embedding_similarity \\\n", "0 plasma cell 0.9181 \n", "1 pulmonary alveolar type 2 cell;plasma cell 0.8747;0.9181 \n", "2 pulmonary alveolar type 2 cell;T cell 0.8747;0.8893 \n", "3 T cell;lymphocyte of B lineage 0.8893;0.8953 \n", "4 pulmonary alveolar type 2 cell;T cell 0.8747;0.8893 \n", "5 T cell 0.8893 \n", "\n", " cytescore_similarity similarity_method \n", "0 0.8975 cytescore_compound \n", "1 1.0000 cytescore_compound \n", "2 1.0000 cytescore_compound \n", "3 0.4046 cytescore_compound \n", "4 1.0000 cytescore_compound \n", "5 1.0000 cytescore " ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "compound_df_max" ] }, { "cell_type": "markdown", "id": "d2a9d5a1", "metadata": {}, "source": [ "## Compound label scoring - Hungarian Matrix Scoring\n", "\n", "1. Build an **m×n** cytescore matrix `S` between author parts (rows) and algorithm parts (columns).\n", "2. Run **max-weight bipartite matching** (Hungarian algorithm): pick `k = min(m, n)` one-to-one pairs that maximize total score.\n", "3. **Match mean** = average score of the assigned pairs.\n", "4. If **m ≠ n**, multiply by **coverage** = `min(m, n) / max(m, n)`.\n", "\n", "Ontology IDs and names in the result list only the **matched** assignment pairs (semicolon-separated when `k > 1`).\n", "\n", "| Scenario | m×n | Illustrative score |\n", "|----------|-----|--------------------|\n", "| Same compound (two overlap) | 2×2 | ~1.0 |\n", "| One shared type | 2×2 | ~0.5 |\n", "| No shared types | 2×2 | ~0.07 |\n", "| Partial overlap, extra author type | 3×2 | ~0.35 |\n", "| Author doublet vs single type | 2×1 | best match × 0.5 |\n", "\n", "The next cell runs these matrices with `_hungarian_match_mean` using illustrative scores (no API calls). The section after that runs live comparisons for the same scenarios." ] }, { "cell_type": "markdown", "id": "9376d486", "metadata": {}, "source": [ "### Synthetic scoring examples (no API)\n", "\n", "Scores below are **illustrative** cytescores placed in `S` to show how the metric behaves. Live labels in the next section use the real pipeline (decompose → embed → match → Hungarian)." ] }, { "cell_type": "code", "execution_count": 8, "id": "b4c2dd21", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
scenariomncoverageassignmentsillustrative_cytescore
02x1_coverage210.5000[(1, 0)]0.4600
12x2_two_overlap221.0000[(0, 0), (1, 1)]0.9750
22x2_one_overlap221.0000[(0, 0), (1, 1)]0.5300
32x2_zero_overlap221.0000[(0, 0), (1, 1)]0.0650
43x2_one_overlap320.6667[(0, 0), (1, 1)]0.3533
\n", "
" ], "text/plain": [ " scenario m n coverage assignments illustrative_cytescore\n", "0 2x1_coverage 2 1 0.5000 [(1, 0)] 0.4600\n", "1 2x2_two_overlap 2 2 1.0000 [(0, 0), (1, 1)] 0.9750\n", "2 2x2_one_overlap 2 2 1.0000 [(0, 0), (1, 1)] 0.5300\n", "3 2x2_zero_overlap 2 2 1.0000 [(0, 0), (1, 1)] 0.0650\n", "4 3x2_one_overlap 3 2 0.6667 [(0, 0), (1, 1)] 0.3533" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def hungarian_demo(scenario: str, scores: list[list[float]]) -> dict:\n", " matrix = np.array(scores, dtype=np.float64)\n", " m, n = matrix.shape\n", " final, assignments = _hungarian_match_mean(matrix)\n", " return {\n", " \"scenario\": scenario,\n", " \"m\": m,\n", " \"n\": n,\n", " \"coverage\": min(m, n) / max(m, n) if max(m, n) else 0.0,\n", " \"assignments\": assignments,\n", " \"illustrative_cytescore\": round(final, 4),\n", " \"matrix\": matrix,\n", " }\n", "\n", "\n", "demo_scenarios = [\n", " (\"2x1_coverage\", [[0.05], [0.92]]),\n", " (\"2x2_two_overlap\", [[0.98, 0.06], [0.05, 0.97]]),\n", " (\"2x2_one_overlap\", [[0.98, 0.05], [0.06, 0.08]]),\n", " (\"2x2_zero_overlap\", [[0.06, 0.05], [0.08, 0.07]]),\n", " (\"3x2_one_overlap\", [[0.98, 0.05], [0.06, 0.08], [0.05, 0.07]]),\n", "]\n", "\n", "demos = [hungarian_demo(name, matrix) for name, matrix in demo_scenarios]\n", "\n", "demo_df = pd.DataFrame(\n", " [\n", " {\n", " \"scenario\": d[\"scenario\"],\n", " \"m\": d[\"m\"],\n", " \"n\": d[\"n\"],\n", " \"coverage\": round(d[\"coverage\"], 4),\n", " \"assignments\": d[\"assignments\"],\n", " \"illustrative_cytescore\": d[\"illustrative_cytescore\"],\n", " }\n", " for d in demos\n", " ]\n", ")\n", "demo_df" ] }, { "cell_type": "markdown", "id": "cd1fda09", "metadata": {}, "source": [ "### Live compound comparisons\n", "\n", "Each row below is a deliberate scenario. After `compare`, a `scenario` column is attached for readability." ] }, { "cell_type": "code", "execution_count": null, "id": "6273fd69", "metadata": {}, "outputs": [], "source": [ "compound_df = await cyto.compare(\n", " author_labels=compound_author_labels,\n", " algorithms={\"scenarios\": compound_algorithm_labels},\n", " run_id=\"quick_tutorial_compound\",\n", " metric=\"cosine_kernel\",\n", ")\n", "compound_df.insert(0, \"scenario\", [name for name, _, _ in compound_scenarios])" ] }, { "cell_type": "code", "execution_count": 10, "id": "d493be8d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
scenariorun_idalgorithmpair_indexauthor_labelalgorithm_labelauthor_ontology_idauthor_ontology_nameauthor_embedding_similarityalgorithm_ontology_idalgorithm_ontology_namealgorithm_embedding_similaritycytescore_similaritysimilarity_method
02x1_coveragequick_tutorial_compoundscenarios0AT2 cell-plasma cell doubletPlasma cellCL:0000946antibody secreting cell0.9124CL:0000946antibody secreting cell0.90230.5000cytescore_compound
12x2_two_overlapquick_tutorial_compoundscenarios1AT2 cell-plasma cell doubletAT2 cell / Plasma cellCL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...0.9124CL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...0.89791.0000cytescore_compound
22x2_one_overlapquick_tutorial_compoundscenarios2AT2 cell-plasma cell doubletAT2 cell / T cellCL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...0.9124CL:0002063;CL:0000084pulmonary alveolar type 2 cell;T cell0.86840.5984cytescore_compound
32x2_zero_overlapquick_tutorial_compoundscenarios3AT2 cell-plasma cell doubletT cell / B cellCL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...0.9124CL:0000084;CL:0000236T cell;B cell0.85760.3334cytescore_compound
43x2_one_overlapquick_tutorial_compoundscenarios4AT2 cell / plasma cell / NK cellAT2 cell / T cellCL:0002063;CL:0000623pulmonary alveolar type 2 cell;natural killer ...0.8855CL:0002063;CL:0000084pulmonary alveolar type 2 cell;T cell0.86840.4121cytescore_compound
5simple_controlquick_tutorial_compoundscenarios5T cellT cellCL:0000084T cell0.8753CL:0000084T cell0.84351.0000cytescore
\n", "
" ], "text/plain": [ " scenario run_id algorithm pair_index \\\n", "0 2x1_coverage quick_tutorial_compound scenarios 0 \n", "1 2x2_two_overlap quick_tutorial_compound scenarios 1 \n", "2 2x2_one_overlap quick_tutorial_compound scenarios 2 \n", "3 2x2_zero_overlap quick_tutorial_compound scenarios 3 \n", "4 3x2_one_overlap quick_tutorial_compound scenarios 4 \n", "5 simple_control quick_tutorial_compound scenarios 5 \n", "\n", " author_label algorithm_label \\\n", "0 AT2 cell-plasma cell doublet Plasma cell \n", "1 AT2 cell-plasma cell doublet AT2 cell / Plasma cell \n", "2 AT2 cell-plasma cell doublet AT2 cell / T cell \n", "3 AT2 cell-plasma cell doublet T cell / B cell \n", "4 AT2 cell / plasma cell / NK cell AT2 cell / T cell \n", "5 T cell T cell \n", "\n", " author_ontology_id author_ontology_name \\\n", "0 CL:0000946 antibody secreting cell \n", "1 CL:0002063;CL:0000946 pulmonary alveolar type 2 cell;antibody secret... \n", "2 CL:0002063;CL:0000946 pulmonary alveolar type 2 cell;antibody secret... \n", "3 CL:0002063;CL:0000946 pulmonary alveolar type 2 cell;antibody secret... \n", "4 CL:0002063;CL:0000623 pulmonary alveolar type 2 cell;natural killer ... \n", "5 CL:0000084 T cell \n", "\n", " author_embedding_similarity algorithm_ontology_id \\\n", "0 0.9124 CL:0000946 \n", "1 0.9124 CL:0002063;CL:0000946 \n", "2 0.9124 CL:0002063;CL:0000084 \n", "3 0.9124 CL:0000084;CL:0000236 \n", "4 0.8855 CL:0002063;CL:0000084 \n", "5 0.8753 CL:0000084 \n", "\n", " algorithm_ontology_name \\\n", "0 antibody secreting cell \n", "1 pulmonary alveolar type 2 cell;antibody secret... \n", "2 pulmonary alveolar type 2 cell;T cell \n", "3 T cell;B cell \n", "4 pulmonary alveolar type 2 cell;T cell \n", "5 T cell \n", "\n", " algorithm_embedding_similarity cytescore_similarity similarity_method \n", "0 0.9023 0.5000 cytescore_compound \n", "1 0.8979 1.0000 cytescore_compound \n", "2 0.8684 0.5984 cytescore_compound \n", "3 0.8576 0.3334 cytescore_compound \n", "4 0.8684 0.4121 cytescore_compound \n", "5 0.8435 1.0000 cytescore " ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "compound_df" ] }, { "cell_type": "code", "execution_count": 11, "id": "0288ec49", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
scenariomncoverageassignmentsillustrative_cytescoreauthor_labelalgorithm_labelauthor_ontology_namealgorithm_ontology_namelive_cytescoresimilarity_method
02x1_coverage2.01.00.5000[(1, 0)]0.4600AT2 cell-plasma cell doubletPlasma cellantibody secreting cellantibody secreting cell0.5000cytescore_compound
12x2_one_overlap2.02.01.0000[(0, 0), (1, 1)]0.5300AT2 cell-plasma cell doubletAT2 cell / T cellpulmonary alveolar type 2 cell;antibody secret...pulmonary alveolar type 2 cell;T cell0.5984cytescore_compound
22x2_two_overlap2.02.01.0000[(0, 0), (1, 1)]0.9750AT2 cell-plasma cell doubletAT2 cell / Plasma cellpulmonary alveolar type 2 cell;antibody secret...pulmonary alveolar type 2 cell;antibody secret...1.0000cytescore_compound
32x2_zero_overlap2.02.01.0000[(0, 0), (1, 1)]0.0650AT2 cell-plasma cell doubletT cell / B cellpulmonary alveolar type 2 cell;antibody secret...T cell;B cell0.3334cytescore_compound
43x2_one_overlap3.02.00.6667[(0, 0), (1, 1)]0.3533AT2 cell / plasma cell / NK cellAT2 cell / T cellpulmonary alveolar type 2 cell;natural killer ...pulmonary alveolar type 2 cell;T cell0.4121cytescore_compound
5simple_controlNaNNaNNaNNaNNaNT cellT cellT cellT cell1.0000cytescore
\n", "
" ], "text/plain": [ " scenario m n coverage assignments \\\n", "0 2x1_coverage 2.0 1.0 0.5000 [(1, 0)] \n", "1 2x2_one_overlap 2.0 2.0 1.0000 [(0, 0), (1, 1)] \n", "2 2x2_two_overlap 2.0 2.0 1.0000 [(0, 0), (1, 1)] \n", "3 2x2_zero_overlap 2.0 2.0 1.0000 [(0, 0), (1, 1)] \n", "4 3x2_one_overlap 3.0 2.0 0.6667 [(0, 0), (1, 1)] \n", "5 simple_control NaN NaN NaN NaN \n", "\n", " illustrative_cytescore author_label \\\n", "0 0.4600 AT2 cell-plasma cell doublet \n", "1 0.5300 AT2 cell-plasma cell doublet \n", "2 0.9750 AT2 cell-plasma cell doublet \n", "3 0.0650 AT2 cell-plasma cell doublet \n", "4 0.3533 AT2 cell / plasma cell / NK cell \n", "5 NaN T cell \n", "\n", " algorithm_label author_ontology_name \\\n", "0 Plasma cell antibody secreting cell \n", "1 AT2 cell / T cell pulmonary alveolar type 2 cell;antibody secret... \n", "2 AT2 cell / Plasma cell pulmonary alveolar type 2 cell;antibody secret... \n", "3 T cell / B cell pulmonary alveolar type 2 cell;antibody secret... \n", "4 AT2 cell / T cell pulmonary alveolar type 2 cell;natural killer ... \n", "5 T cell T cell \n", "\n", " algorithm_ontology_name live_cytescore \\\n", "0 antibody secreting cell 0.5000 \n", "1 pulmonary alveolar type 2 cell;T cell 0.5984 \n", "2 pulmonary alveolar type 2 cell;antibody secret... 1.0000 \n", "3 T cell;B cell 0.3334 \n", "4 pulmonary alveolar type 2 cell;T cell 0.4121 \n", "5 T cell 1.0000 \n", "\n", " similarity_method \n", "0 cytescore_compound \n", "1 cytescore_compound \n", "2 cytescore_compound \n", "3 cytescore_compound \n", "4 cytescore_compound \n", "5 cytescore " ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "summary_cols = [\n", " \"scenario\",\n", " \"author_label\",\n", " \"algorithm_label\",\n", " \"author_ontology_name\",\n", " \"algorithm_ontology_name\",\n", " \"cytescore_similarity\",\n", " \"similarity_method\",\n", "]\n", "\n", "live_summary = compound_df[summary_cols].rename(\n", " columns={\"cytescore_similarity\": \"live_cytescore\"}\n", ")\n", "\n", "pd.merge(demo_df, live_summary, on=\"scenario\", how=\"outer\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "f7d49d07", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
scenarioauthor_ontology_idalgorithm_ontology_idauthor_ontology_namealgorithm_ontology_namecytescore_similarity
02x1_coverageCL:0000946CL:0000946antibody secreting cellantibody secreting cell0.5000
12x2_two_overlapCL:0002063;CL:0000946CL:0002063;CL:0000946pulmonary alveolar type 2 cell;antibody secret...pulmonary alveolar type 2 cell;antibody secret...1.0000
22x2_one_overlapCL:0002063;CL:0000946CL:0002063;CL:0000084pulmonary alveolar type 2 cell;antibody secret...pulmonary alveolar type 2 cell;T cell0.5984
32x2_zero_overlapCL:0002063;CL:0000946CL:0000084;CL:0000236pulmonary alveolar type 2 cell;antibody secret...T cell;B cell0.3334
43x2_one_overlapCL:0002063;CL:0000623CL:0002063;CL:0000084pulmonary alveolar type 2 cell;natural killer ...pulmonary alveolar type 2 cell;T cell0.4121
\n", "
" ], "text/plain": [ " scenario author_ontology_id algorithm_ontology_id \\\n", "0 2x1_coverage CL:0000946 CL:0000946 \n", "1 2x2_two_overlap CL:0002063;CL:0000946 CL:0002063;CL:0000946 \n", "2 2x2_one_overlap CL:0002063;CL:0000946 CL:0002063;CL:0000084 \n", "3 2x2_zero_overlap CL:0002063;CL:0000946 CL:0000084;CL:0000236 \n", "4 3x2_one_overlap CL:0002063;CL:0000623 CL:0002063;CL:0000084 \n", "\n", " author_ontology_name \\\n", "0 antibody secreting cell \n", "1 pulmonary alveolar type 2 cell;antibody secret... \n", "2 pulmonary alveolar type 2 cell;antibody secret... \n", "3 pulmonary alveolar type 2 cell;antibody secret... \n", "4 pulmonary alveolar type 2 cell;natural killer ... \n", "\n", " algorithm_ontology_name cytescore_similarity \n", "0 antibody secreting cell 0.5000 \n", "1 pulmonary alveolar type 2 cell;antibody secret... 1.0000 \n", "2 pulmonary alveolar type 2 cell;T cell 0.5984 \n", "3 T cell;B cell 0.3334 \n", "4 pulmonary alveolar type 2 cell;T cell 0.4121 " ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "compound_df[compound_df[\"similarity_method\"] == \"cytescore_compound\"][\n", " [\n", " \"scenario\",\n", " \"author_ontology_id\",\n", " \"algorithm_ontology_id\",\n", " \"author_ontology_name\",\n", " \"algorithm_ontology_name\",\n", " \"cytescore_similarity\",\n", " ]\n", "]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Cache utilities\n", "\n", "Artifacts are stored under `cyteonto/data/user_files//` keyed by provider and model (schema v3)." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "cyto.cache_stats(run_id=\"quick_tutorial_demo\")" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "# cyto.clear_run(\"quick_tutorial_demo\") # uncomment to delete cached files for this run" ] } ], "metadata": { "kernelspec": { "display_name": ".venv (3.13.9)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.9" } }, "nbformat": 4, "nbformat_minor": 5 }