{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Allen dataset integration \n",
    "\n",
    "This notebook focuses demonstrate how to integrate Allen datasets into the Blue Brain Knowledge Graph\n",
    "\n",
    "The tasks to be demonstrated are the following:\n",
    "\n",
    "1. Configuration\n",
    "2. Retrieve human neuron morphologies from the Allen Cell Types Database\n",
    "3. Load the complete metadata of the neuron morphologies from Allen\n",
    "4. Load the transformation Mappings\n",
    "5. Map the neuron morphologies from Allen to the Neuroshapes Models\n",
    "6. Add the created entities from Allen to Nexus\n",
    "7. Retrieve the created entities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Session configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import getpass\n",
    "\n",
    "import allensdk\n",
    "\n",
    "from kgforge.core import KnowledgeGraphForge\n",
    "from kgforge.version import __version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check versions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Allensdk is 1.7.1 , and Nexus Fogre is 0.2.1.dev82+g0b66b09\n"
     ]
    }
   ],
   "source": [
    "print(\"Allensdk is\", allensdk.__version__, \", and Nexus Fogre is\", __version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please enter your BBP token:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# token = getpass.getpass()\n",
    "token = \"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI5T0R3Z1JSTFVsTTJHbFphVDZjVklnenJsb0lzUWJmbTBDck1icXNjNHQ4In0.eyJqdGkiOiI4NmJiNmUxNi04OWFhLTRjYjktODNhMC0zYjc2MDU0NDk5NGYiLCJleHAiOjE1OTExNjA4NTIsIm5iZiI6MCwiaWF0IjoxNTkxMTMyMDUyLCJpc3MiOiJodHRwczovL2JicGF1dGguZXBmbC5jaC9hdXRoL3JlYWxtcy9CQlAiLCJzdWIiOiJmOjBmZGFkZWY3LWIyYjktNDkyYi1hZjQ2LWM2NTQ5MmQ0NTljMjphZ2FyY2lhIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoibmV4dXMtd2ViIiwibm9uY2UiOiJjM2M5YmRhMTZkNTM0Zjk0ODgxZGRhYzZhZGQ2NjAzZCIsImF1dGhfdGltZSI6MTU5MTEzMjA1MSwic2Vzc2lvbl9zdGF0ZSI6ImIwM2FjYWM1LTIxOGItNDA5My05Y2ZmLTRlNTRhY2FhNTQxYSIsImFjciI6IjAiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cHM6Ly9kZXYubmV4dXMub2NwLmJicC5lcGZsLmNoIiwiaHR0cHM6Ly9iYnAuZXBmbC5jaCIsImh0dHA6Ly9kZXYubmV4dXMub2NwLmJicC5lcGZsLmNoIiwiaHR0cHM6Ly9zdGFnaW5nLm5leHVzLm9jcC5iYnAuZXBmbC5jaCIsImh0dHBzOi8vYmJwLW5leHVzLmVwZmwuY2giLCJodHRwczovL2JicHRlYW0uZXBmbC5jaCIsImh0dHA6Ly9zdGFnaW5nLm5leHVzLm9jcC5iYnAuZXBmbC5jaCJdLCJzY29wZSI6Im9wZW5pZCBwcm9maWxlIGdyb3VwcyBlbWFpbCIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJuYW1lIjoiQWxlamFuZHJhIEdhcmNpYSBSb2phcyBHYXJjaWEgUm9qYXMgTWFydGluZXoiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhZ2FyY2lhIiwiZ2l2ZW5fbmFtZSI6IkFsZWphbmRyYSBHYXJjaWEgUm9qYXMiLCJmYW1pbHlfbmFtZSI6IkdhcmNpYSBSb2phcyBNYXJ0aW5leiIsImVtYWlsIjoiYWxlamFuZHJhLmdhcmNpYXJvamFzQGVwZmwuY2gifQ.oB7SCkvflnTudSlDINGueJLZalRcMhTNPeevFpJRajCXXlRivH4JRENbVRuYZDO5__N3KjYigRkRIhrP-AXBxj8TLv8nFpyGP6G_1T3BRRqed9EjCxowoA13tEb7x40U1BNnoEPdGau2YiH3149MEALgKpmCLftiCd1ooEwpSzd-6NiVJuWbdlZWd12OJYg3D2oL62pt5n7tCgKUBCIphgG2Okc9StU1Wm-P6UsT--23q-0WXdxODMUrvrOiWi1d9V50LHQ3gzIOqHx4MruBA4NgvE_3QcIOuysWvhZVifErnjKmYAiLhcslkW8Ecilj7wkOu_ZBd64TnBTUdYKnkA\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Initialiting the forge may take a few seconds if the source is a directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:34:12.001710Z",
     "start_time": "2019-09-12T12:34:11.997386Z"
    }
   },
   "outputs": [],
   "source": [
    "forge = KnowledgeGraphForge(\"../../configurations/demo-forge-nexus-neuroshapes.yml\", token=token)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9W5M5Ck9Tq7q"
   },
   "source": [
    "## 2. Retrieve human neuron morphologies from the Allen Cell Types Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allensdk.core.cell_types_cache import CellTypesCache\n",
    "from allensdk.api.queries.cell_types_api import CellTypesApi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.A Downloaded files: Specify a directory where to download allen files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALLEN_DIR = \"allen_cell_types_database\"\n",
    "ctc = CellTypesCache(manifest_file=f\"{ALLEN_DIR}/manifest.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:34:12.102274Z",
     "start_time": "2019-09-12T12:34:12.043904Z"
    }
   },
   "outputs": [],
   "source": [
    "human_cells = ctc.get_cells(species=[CellTypesApi.HUMAN], require_reconstruction=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cells.json    manifest.json\n"
     ]
    }
   ],
   "source": [
    "!ls allen_cell_types_database"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.B Pick a subset of cells to integrate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "152"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(human_cells)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:34:12.108855Z",
     "start_time": "2019-09-12T12:34:12.104642Z"
    }
   },
   "outputs": [],
   "source": [
    "FROM = 8\n",
    "TO = 10\n",
    "human_cell_ids = [x[\"id\"] for x in human_cells][FROM:TO]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[527942865, 529807751]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "human_cell_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.C  Check the data has not integrated those already by trying to fetch them with the Forge\n",
    "\n",
    "For the picked cells, we check that they are not already integrated. If they are already integrated, get another couple of ids in human_cell_ids in the step 2.B.\n",
    "\n",
    "Note that `forge.format` method is used to create the identifier for the patchedcells to be attempted to be retreived from Nexus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://bbp.epfl.ch/neurosciencegraph/data/patchedcells/527942865\n",
      "<action> retrieve\n",
      "<error> RetrievalError: resource 'https://bbp.epfl.ch/neurosciencegraph/data/patchedcells/527942865' not found for schema 'https://bluebrain.github.io/nexus/schemas/unconstrained.json'\n",
      "\n",
      "https://bbp.epfl.ch/neurosciencegraph/data/patchedcells/529807751\n",
      "<action> retrieve\n",
      "<error> RetrievalError: resource 'https://bbp.epfl.ch/neurosciencegraph/data/patchedcells/529807751' not found for schema 'https://bluebrain.github.io/nexus/schemas/unconstrained.json'\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for id_ in human_cell_ids:\n",
    "    kg_id = forge.format(\"identifier\", \"patchedcells\", id_)\n",
    "    print(kg_id) \n",
    "    resource = forge.retrieve(kg_id)\n",
    "    if resource:\n",
    "        print(\"> already integrated\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:34:12.114296Z",
     "start_time": "2019-09-12T12:34:12.110781Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-06-02 23:31:21,701 allensdk.api.api.retrieve_file_over_http INFO     Downloading URL: http://api.brain-map.org/api/v2/well_known_file_download/601947673\n",
      "2020-06-02 23:31:23,344 allensdk.api.api.retrieve_file_over_http INFO     Downloading URL: http://api.brain-map.org/api/v2/well_known_file_download/667320244\n"
     ]
    }
   ],
   "source": [
    "human_cell_reconstructions = [ctc.get_reconstruction(x) for x in human_cell_ids]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Load the complete metadata of the neuron morphologies from Allen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:34:12.164676Z",
     "start_time": "2019-09-12T12:34:12.116514Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(f\"{ALLEN_DIR}/cells.json\") as f:\n",
    "    allen_cell_types_metadata = json.load(f)\n",
    "human_cell_metadata = [x for x in allen_cell_types_metadata if x[\"specimen__id\"] in human_cell_ids]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Have a look to a single record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cell_reporter_status': None,\n",
       " 'csl__normalized_depth': 0.694465802153644,\n",
       " 'csl__x': 67.0,\n",
       " 'csl__y': 256.0,\n",
       " 'csl__z': 110.0,\n",
       " 'donor__age': '24 yrs',\n",
       " 'donor__disease_state': 'epilepsy',\n",
       " 'donor__id': 527747035,\n",
       " 'donor__name': 'H16.06.008',\n",
       " 'donor__race': 'Hispanic',\n",
       " 'donor__sex': 'Female',\n",
       " 'donor__species': 'Homo Sapiens',\n",
       " 'donor__years_of_seizure_history': '7',\n",
       " 'ef__adaptation': 0.975858514620517,\n",
       " 'ef__avg_firing_rate': 2.71252644713286,\n",
       " 'ef__avg_isi': 368.66,\n",
       " 'ef__f_i_curve_slope': 0.0785947712418301,\n",
       " 'ef__fast_trough_v_long_square': -53.1875,\n",
       " 'ef__peak_t_ramp': 4.21818666666667,\n",
       " 'ef__ri': 136.718824505806,\n",
       " 'ef__tau': 26.8857603438525,\n",
       " 'ef__threshold_i_long_square': 70.0,\n",
       " 'ef__upstroke_downstroke_ratio_long_square': 3.90618458032267,\n",
       " 'ef__vrest': -67.7251434326172,\n",
       " 'ephys_inst_thresh_thumb_path': '/api/v2/well_known_file_download/529903821',\n",
       " 'ephys_thumb_path': '/api/v2/well_known_file_download/529903819',\n",
       " 'erwkf__id': 616980497,\n",
       " 'line_name': '',\n",
       " 'm__biophys': 1,\n",
       " 'm__biophys_all_active': 0,\n",
       " 'm__biophys_perisomatic': 1,\n",
       " 'm__glif': 0,\n",
       " 'morph_thumb_path': '/api/v2/well_known_file_download/618683378',\n",
       " 'nr__average_contraction': 0.845900877489379,\n",
       " 'nr__average_parent_daughter_ratio': 0.930432608365231,\n",
       " 'nr__max_euclidean_distance': 1042.45317093294,\n",
       " 'nr__number_bifurcations': 21,\n",
       " 'nr__number_stems': 6,\n",
       " 'nr__reconstruction_type': 'full',\n",
       " 'nrwkf__id': 601947673,\n",
       " 'si__height': 9632,\n",
       " 'si__path': '/external/humancelltypes/prod44/specimen_527942865/min_xy_527942865.aff',\n",
       " 'si__width': 14842,\n",
       " 'specimen__hemisphere': 'left',\n",
       " 'specimen__id': 527942865,\n",
       " 'specimen__name': 'H16.06.008.01.26.04',\n",
       " 'structure__acronym': 'MTG',\n",
       " 'structure__id': 12141,\n",
       " 'structure__layer': '5',\n",
       " 'structure__name': '\"middle temporal gyrus\"',\n",
       " 'structure_parent__acronym': 'MTG',\n",
       " 'structure_parent__id': 12141,\n",
       " 'tag__apical': 'truncated',\n",
       " 'tag__dendrite_type': 'spiny'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "human_cell_metadata[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Load the transformation Mappings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The forge a Dictionary Mapper that uses mapping files that provide the required transformation form a dictionary to another dictionary. The Dictionary Mappings are HJSON files containing the required transformations. This notebook has three mappings for: Subject, Patched Cell and Neuron Morphology."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 24\n",
      "-rw-r--r--  1 agarcia  INTRANET\\Domain Users  2059 Jun  2 23:23 NeuronMorphology.hjson\n",
      "-rw-r--r--  1 agarcia  INTRANET\\Domain Users  1043 Jun  2 23:23 PatchedCell.hjson\n",
      "-rw-r--r--  1 agarcia  INTRANET\\Domain Users   277 Jun  2 23:23 Subject.hjson\n"
     ]
    }
   ],
   "source": [
    "!ls -l ../../mappings/allen-database-mappings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR = \"../../mappings/allen-database-mappings\"\n",
    "subject_mapping_file = f\"{DIR}/Subject.hjson\"\n",
    "patched_cel_mapping_file = f\"{DIR}/PatchedCell.hjson\"\n",
    "neuronmorphology_mapping_file = f\"{DIR}/NeuronMorphology.hjson\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kgforge.specializations.mappings import DictionaryMapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "subject_mapping = DictionaryMapping.load(subject_mapping_file)\n",
    "patchedcell_mapping = DictionaryMapping.load(patched_cel_mapping_file)\n",
    "neuronmorphology_mapping = DictionaryMapping.load(neuronmorphology_mapping_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One of he mapping file context is shown next:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    id: forge.format(\"identifier\", \"subjects\", x.donor__id)\n",
      "    type: Subject\n",
      "    identifier: x.donor__id\n",
      "    name: x.donor__name\n",
      "    sex: forge.resolve(x.donor__sex, scope=\"terms\", target=\"sex\")\n",
      "    species: forge.resolve(x.donor__species, scope=\"terms\", target=\"species\")\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(subject_mapping)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inside mapping files, it is possible to use methos form the Forge such as:\n",
    "\n",
    "    - forge.format : used to format a string using a preconfigured string format (used previously in 2.C)\n",
    "    - forge.resolve: used to retrieve identifiers using a string that is part of the name of the desired resource \n",
    "    \n",
    "An example of the resolver is shown next:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kgforge.core.commons.strategies import ResolvingStrategy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check available resolvers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available scopes:\n",
      " -  agent :\n",
      "     - resolver:  AgentResolver\n",
      "         - targets:  agents\n",
      " -  ontology :\n",
      "     - resolver:  OntologyResolver\n",
      "         - targets:  terms\n",
      " -  terms :\n",
      "     - resolver:  DemoResolver\n",
      "         - targets:  species,sex,structure-layer\n"
     ]
    }
   ],
   "source": [
    "forge.resolvers()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Resolve the identifier for male in the terms scope and sex as target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    id: http://purl.obolibrary.org/obo/PATO_0000384\n",
      "    label: male\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(forge.resolve(\"male\", scope=\"terms\", target=\"sex\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Map the neuron morphologies from Allen to the Neuroshapes Models\n",
    "\n",
    "It is possible to provide a list of mappings to be applied to a single dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "mappings = [subject_mapping, patchedcell_mapping, neuronmorphology_mapping]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:34:12.223316Z",
     "start_time": "2019-09-12T12:34:12.209983Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "resources = forge.map(human_cell_metadata, mappings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the created resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(resources)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    id: https://bbp.epfl.ch/neurosciencegraph/data/neuronmorphologies/527942865\n",
      "    type: NeuronMorphology\n",
      "    apicalDendrite: truncated\n",
      "    brainLocation:\n",
      "    {\n",
      "        type: BrainLocation\n",
      "        brainRegion:\n",
      "        {\n",
      "            id: http://api.brain-map.org/api/v2/data/Structure/12141\n",
      "            label: MTG\n",
      "        }\n",
      "        coordinatesInBrainAtlas:\n",
      "        {\n",
      "            valueX: 67.0\n",
      "            valueY: 256.0\n",
      "            valueZ: 110.0\n",
      "        }\n",
      "        layer:\n",
      "        {\n",
      "            id: http://purl.obolibrary.org/obo/UBERON_0005394\n",
      "            label: layer 5\n",
      "        }\n",
      "    }\n",
      "    contribution:\n",
      "    {\n",
      "        type: Contribution\n",
      "        agent:\n",
      "        {\n",
      "            id: https://www.grid.ac/institutes/grid.417881.3\n",
      "            type: Organization\n",
      "        }\n",
      "    }\n",
      "    derivation:\n",
      "    [\n",
      "        {\n",
      "            type: Derivation\n",
      "            entity:\n",
      "            {\n",
      "                id: https://bbp.epfl.ch/neurosciencegraph/data/subjects/527747035\n",
      "                type: Subject\n",
      "            }\n",
      "        }\n",
      "        {\n",
      "            type: Derivation\n",
      "            entity:\n",
      "            {\n",
      "                id: https://bbp.epfl.ch/neurosciencegraph/data/patchedcells/527942865\n",
      "                type: PatchedCell\n",
      "            }\n",
      "        }\n",
      "    ]\n",
      "    distribution: LazyAction(operation=Store.upload, args=['./allen_cell_types_database/specimen_527942865/reconstruction.swc', 'application/swc'])\n",
      "    identifier: 527942865\n",
      "    name: H16.06.008.01.26.04\n",
      "    subject:\n",
      "    {\n",
      "        id: https://bbp.epfl.ch/neurosciencegraph/data/subjects/527747035\n",
      "        type: Subject\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(resources[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Register the created resources from Allen to Nexus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:34:12.248767Z",
     "start_time": "2019-09-12T12:34:12.243465Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<count> 6\n",
      "<action> _register_many\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(resources)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    context: https://bbp.neuroshapes.org\n",
      "    id: https://bbp.epfl.ch/neurosciencegraph/data/patchedcells/527942865\n",
      "    type: PatchedCell\n",
      "    brainLocation:\n",
      "    {\n",
      "        type: BrainLocation\n",
      "        brainRegion:\n",
      "        {\n",
      "            id: http://api.brain-map.org/api/v2/data/Structure/12141\n",
      "            label: MTG\n",
      "        }\n",
      "    }\n",
      "    contribution:\n",
      "    {\n",
      "        type: Contribution\n",
      "        agent:\n",
      "        {\n",
      "            id: https://www.grid.ac/institutes/grid.417881.3\n",
      "            type: Organization\n",
      "        }\n",
      "    }\n",
      "    derivation:\n",
      "    {\n",
      "        type: Derivation\n",
      "        entity:\n",
      "        {\n",
      "            id: https://bbp.epfl.ch/neurosciencegraph/data/subjects/527747035\n",
      "            type: Subject\n",
      "        }\n",
      "    }\n",
      "    identifier: 527942865\n",
      "    name: H16.06.008.01.26.04\n",
      "    subject:\n",
      "    {\n",
      "        id: https://bbp.epfl.ch/neurosciencegraph/data/subjects/527747035\n",
      "        type: Subject\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(resources[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Retrieve the created entities\n",
    "\n",
    "If you know exactly the ID you can just retreive as did in 2.C or you can use the `search()` method. To search for resource you can start by picking a Type in the available types."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To create a search based on the `PatchedCell` structure, use the paths() method which will load the structure of the givent type in a Python object and these fields can be accessed using auto-completition.\n",
    "Next, the `p` object will hold the properties of `PatchedCell` and can be used to create a search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:34:12.400246Z",
     "start_time": "2019-09-12T12:34:12.396228Z"
    }
   },
   "outputs": [],
   "source": [
    "p = forge.paths(\"PatchedCell\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = forge.search(p.type == \"PatchedCell\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "DISPLAY_LIMIT = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>type</th>\n",
       "      <th>brainLocation.type</th>\n",
       "      <th>brainLocation.brainRegion.id</th>\n",
       "      <th>brainLocation.brainRegion.label</th>\n",
       "      <th>contribution.type</th>\n",
       "      <th>contribution.agent.id</th>\n",
       "      <th>contribution.agent.type</th>\n",
       "      <th>derivation.type</th>\n",
       "      <th>derivation.entity.id</th>\n",
       "      <th>identifier</th>\n",
       "      <th>name</th>\n",
       "      <th>subject.id</th>\n",
       "      <th>subject.type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>MTG</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>596020931</td>\n",
       "      <td>H17.06.009.11.04.02</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>MTG</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>519832676</td>\n",
       "      <td>H16.03.001.01.09.01</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>AnG</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>569095789</td>\n",
       "      <td>H17.06.004.11.05.04</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>MTG</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>528706755</td>\n",
       "      <td>H16.06.009.01.01.15.01</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>MTG</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>616647103</td>\n",
       "      <td>H17.03.005.11.09.02</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>FroL</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>531520637</td>\n",
       "      <td>H16.06.007.01.05.03</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>MTG</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>542143598</td>\n",
       "      <td>H16.03.008.11.11.03</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>FroL</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>531520401</td>\n",
       "      <td>H16.06.007.01.07.02</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>MTG</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>527942865</td>\n",
       "      <td>H16.06.008.01.26.04</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/pat...</td>\n",
       "      <td>PatchedCell</td>\n",
       "      <td>BrainLocation</td>\n",
       "      <td>http://api.brain-map.org/api/v2/data/Structure...</td>\n",
       "      <td>MTG</td>\n",
       "      <td>Contribution</td>\n",
       "      <td>https://www.grid.ac/institutes/grid.417881.3</td>\n",
       "      <td>Organization</td>\n",
       "      <td>Derivation</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>529807751</td>\n",
       "      <td>H16.06.010.01.03.04.01</td>\n",
       "      <td>https://bbp.epfl.ch/neurosciencegraph/data/sub...</td>\n",
       "      <td>Subject</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  id         type  \\\n",
       "0  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "1  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "2  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "3  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "4  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "5  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "6  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "7  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "8  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "9  https://bbp.epfl.ch/neurosciencegraph/data/pat...  PatchedCell   \n",
       "\n",
       "  brainLocation.type                       brainLocation.brainRegion.id  \\\n",
       "0      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "1      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "2      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "3      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "4      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "5      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "6      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "7      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "8      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "9      BrainLocation  http://api.brain-map.org/api/v2/data/Structure...   \n",
       "\n",
       "  brainLocation.brainRegion.label contribution.type  \\\n",
       "0                             MTG      Contribution   \n",
       "1                             MTG      Contribution   \n",
       "2                             AnG      Contribution   \n",
       "3                             MTG      Contribution   \n",
       "4                             MTG      Contribution   \n",
       "5                            FroL      Contribution   \n",
       "6                             MTG      Contribution   \n",
       "7                            FroL      Contribution   \n",
       "8                             MTG      Contribution   \n",
       "9                             MTG      Contribution   \n",
       "\n",
       "                          contribution.agent.id contribution.agent.type  \\\n",
       "0  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "1  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "2  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "3  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "4  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "5  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "6  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "7  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "8  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "9  https://www.grid.ac/institutes/grid.417881.3            Organization   \n",
       "\n",
       "  derivation.type                               derivation.entity.id  \\\n",
       "0      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "1      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "2      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "3      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "4      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "5      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "6      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "7      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "8      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "9      Derivation  https://bbp.epfl.ch/neurosciencegraph/data/sub...   \n",
       "\n",
       "   identifier                    name  \\\n",
       "0   596020931     H17.06.009.11.04.02   \n",
       "1   519832676     H16.03.001.01.09.01   \n",
       "2   569095789     H17.06.004.11.05.04   \n",
       "3   528706755  H16.06.009.01.01.15.01   \n",
       "4   616647103     H17.03.005.11.09.02   \n",
       "5   531520637     H16.06.007.01.05.03   \n",
       "6   542143598     H16.03.008.11.11.03   \n",
       "7   531520401     H16.06.007.01.07.02   \n",
       "8   527942865     H16.06.008.01.26.04   \n",
       "9   529807751  H16.06.010.01.03.04.01   \n",
       "\n",
       "                                          subject.id subject.type  \n",
       "0  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "1  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "2  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "3  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "4  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "5  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "6  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "7  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "8  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  \n",
       "9  https://bbp.epfl.ch/neurosciencegraph/data/sub...      Subject  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forge.as_dataframe(results[:DISPLAY_LIMIT])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "! rm -R allen_cell_types_database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Step by step jupyter notebook for bringing data to Nexus v1.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "kgforge(v2)",
   "language": "python",
   "name": "kgforge"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}