{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-23T18:50:19.036357Z",
     "start_time": "2019-09-23T18:50:19.031896Z"
    }
   },
   "source": [
    "# Datasets\n",
    "\n",
    "A [Dataset](https://nexus-forge.readthedocs.io/en/latest/interaction.html#dataset) is a specialization of a [Resource](https://nexus-forge.readthedocs.io/en/latest/interaction.html#resource) that provides users with operations to handle files, record their provenance and describe them with metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-23T18:50:20.068658Z",
     "start_time": "2019-09-23T18:50:19.054054Z"
    }
   },
   "outputs": [],
   "source": [
    "from kgforge.core import KnowledgeGraphForge"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A configuration file is needed in order to create a KnowledgeGraphForge session. A configuration can be generated using the notebook [00-Initialization.ipynb](00%20-%20Initialization.ipynb).\n",
    "\n",
    "Note: DemoStore doesn't implement file operations yet. Use the BluBrainNexus store instead when creating a config file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "forge = KnowledgeGraphForge(\"../../configurations/forge.yml\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kgforge.core import Resource"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kgforge.specializations.resources import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creation with files added as parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "associations.tsv\n",
      "my_data.xwz\n",
      "my_data_derived.txt\n",
      "persons-with-id.csv\n",
      "persons.csv\n",
      "tfidfvectorizer_model_schemaorg_linking\n"
     ]
    }
   ],
   "source": [
    "! ls -p ../../data | egrep -v /$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "persons = Dataset(forge, name=\"Interesting Persons\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "persons.add_files(\"../../data/persons.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<action> _register_one\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/980a7cd9-36ef-4fc9-95b8-cbf622b49fd8',\n",
       " 'type': 'Dataset',\n",
       " 'hasPart': {'distribution': {'type': 'DataDownload',\n",
       "   'atLocation': {'type': 'Location',\n",
       "    'store': {'id': 'https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault',\n",
       "     'type': 'DiskStorage',\n",
       "     '_rev': 1}},\n",
       "   'contentSize': {'unitCode': 'bytes', 'value': 52},\n",
       "   'contentUrl': 'https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/2737f7f0-950a-471d-ae60-80b79c7451bd',\n",
       "   'digest': {'algorithm': 'SHA-256',\n",
       "    'value': '1dacd765946963fda4949753659089c5f532714b418d30788bedddfec47a389f'},\n",
       "   'encodingFormat': 'text/csv',\n",
       "   'name': 'persons.csv'}},\n",
       " 'name': 'Interesting Persons'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forge.as_json(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "associations = Dataset(forge, name=\"Associations data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "associations.add_files(\"../../data/associations.tsv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "associations.add_derivation(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<action> _register_one\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(associations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/80bd2bcb-b84f-4418-9d2e-42712a59fbfb',\n",
       " 'type': 'Dataset',\n",
       " 'derivation': {'type': 'Derivation',\n",
       "  'entity': {'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/980a7cd9-36ef-4fc9-95b8-cbf622b49fd8?rev=1',\n",
       "   'type': 'Dataset',\n",
       "   'name': 'Interesting Persons'}},\n",
       " 'hasPart': {'distribution': {'type': 'DataDownload',\n",
       "   'atLocation': {'type': 'Location',\n",
       "    'store': {'id': 'https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault',\n",
       "     'type': 'DiskStorage',\n",
       "     '_rev': 1}},\n",
       "   'contentSize': {'unitCode': 'bytes', 'value': 477},\n",
       "   'contentUrl': 'https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/d6b03d5b-4007-48d7-9432-5bf302c62999',\n",
       "   'digest': {'algorithm': 'SHA-256',\n",
       "    'value': '789aa07948683fe036ac29811814a826b703b562f7d168eb70dee1fabde26859'},\n",
       "   'encodingFormat': 'text/tab-separated-values',\n",
       "   'name': 'associations.tsv'}},\n",
       " 'name': 'Associations data'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forge.as_json(associations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# By default the files are downloaded in the current path (path=\".\"). The urls or the files to download can be collected from a different json path (by setting a value for \"follow\") and \n",
    "# the files downloaded to a different path (by setting a value for \"path\")\n",
    "# The argument overwrite: bool can be provided to decide whether to overwrite (True) existing files with the same name or\n",
    "# to create new ones (False) with their names suffixed with a timestamp.\n",
    "# A cross_bucket argument can be provided to download data from the configured bucket (cross_bucket=False - the default value) \n",
    "# or from a bucket different than the configured one (cross_bucket=True). The configured store should support crossing buckets for this to work.\n",
    "associations.download(source=\"parts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A specific path can be provided.\n",
    "associations.download(path=\"./downloaded/\", source=\"parts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A specific content type can be downloded.\n",
    "associations.download(path=\"./downloaded/\", source=\"parts\", content_type=\"text/tab-separated-values\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 8\n",
      "-rw-r--r--  1 mfsy  staff  477 Apr 12 17:13 associations.tsv\n"
     ]
    }
   ],
   "source": [
    "! ls -l ./downloaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! rm -R ./downloaded/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creation with files added as distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "persons = Dataset(forge, name=\"Interesting Persons\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "persons.add_distribution(\"../../data/associations.tsv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<action> _register_one\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/3579d0f7-dbf4-40be-90e5-cd641704dfb3',\n",
       " 'type': 'Dataset',\n",
       " 'distribution': {'type': 'DataDownload',\n",
       "  'atLocation': {'type': 'Location',\n",
       "   'store': {'id': 'https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault',\n",
       "    'type': 'DiskStorage',\n",
       "    '_rev': 1}},\n",
       "  'contentSize': {'unitCode': 'bytes', 'value': 477},\n",
       "  'contentUrl': 'https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/dbb6814c-ef9c-4320-b8a0-bf8190dd510a',\n",
       "  'digest': {'algorithm': 'SHA-256',\n",
       "   'value': '789aa07948683fe036ac29811814a826b703b562f7d168eb70dee1fabde26859'},\n",
       "  'encodingFormat': 'text/tab-separated-values',\n",
       "  'name': 'associations.tsv'},\n",
       " 'name': 'Interesting Persons'}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forge.as_json(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# When files are added as distributions, they can be directly downloaded without specifying which json path to use to collect the downlodable urls. In addition, content type and path arguments\n",
    "# can still be provided\n",
    "persons.download()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creation with resources added as parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "distribution_1 = forge.attach(\"../../data/associations.tsv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "distribution_2 = forge.attach(\"../../data/persons.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "jane = Resource(type=\"Person\", name=\"Jane Doe\", distribution=distribution_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "john = Resource(type=\"Person\", name=\"John Smith\", distribution=distribution_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "persons = [jane, john]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<count> 2\n",
      "<action> _register_many\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = Dataset(forge, name=\"Interesting people\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.add_parts(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<action> _register_one\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/46d34055-c662-4a7d-90f4-2c866f89cf57',\n",
       " 'type': 'Dataset',\n",
       " 'hasPart': [{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/0a2041a9-12f7-49aa-b302-48bb09450832?rev=1',\n",
       "   'type': 'Person',\n",
       "   'distribution': {'contentUrl': 'https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/901d4b2e-2b67-4504-aca7-3ab93966dbad'},\n",
       "   'name': 'Jane Doe'},\n",
       "  {'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/87652108-38ca-4fe2-8e41-9ba5ef22f32b?rev=1',\n",
       "   'type': 'Person',\n",
       "   'distribution': {'contentUrl': 'https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/86eb143e-89cb-462d-af0e-48afb7172f2d'},\n",
       "   'name': 'John Smith'}],\n",
       " 'name': 'Interesting people'}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forge.as_json(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.download(path=\"./downloaded/\", source=\"parts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 32\n",
      "-rw-r--r--  1 mfsy  staff  477 Apr 12 17:14 associations.tsv\n",
      "-rw-r--r--  1 mfsy  staff  477 Apr 12 17:14 associations.tsv.20220412171438\n",
      "-rw-r--r--  1 mfsy  staff   52 Apr 12 17:14 persons.csv\n",
      "-rw-r--r--  1 mfsy  staff   52 Apr 12 17:14 persons.csv.20220412171438\n"
     ]
    }
   ],
   "source": [
    "! ls -l ./downloaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! rm -R ./downloaded/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creation from resources converted as Dataset objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    id: https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/0a2041a9-12f7-49aa-b302-48bb09450832\n",
      "    type: Person\n",
      "    distribution:\n",
      "    {\n",
      "        type: DataDownload\n",
      "        atLocation:\n",
      "        {\n",
      "            type: Location\n",
      "            store:\n",
      "            {\n",
      "                id: https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault\n",
      "                type: DiskStorage\n",
      "                _rev: 1\n",
      "            }\n",
      "        }\n",
      "        contentSize:\n",
      "        {\n",
      "            unitCode: bytes\n",
      "            value: 477\n",
      "        }\n",
      "        contentUrl: https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/901d4b2e-2b67-4504-aca7-3ab93966dbad\n",
      "        digest:\n",
      "        {\n",
      "            algorithm: SHA-256\n",
      "            value: 789aa07948683fe036ac29811814a826b703b562f7d168eb70dee1fabde26859\n",
      "        }\n",
      "        encodingFormat: text/tab-separated-values\n",
      "        name: associations.tsv\n",
      "    }\n",
      "    name: Jane Doe\n",
      "}\n",
      "{\n",
      "    id: https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/87652108-38ca-4fe2-8e41-9ba5ef22f32b\n",
      "    type: Person\n",
      "    distribution:\n",
      "    {\n",
      "        type: DataDownload\n",
      "        atLocation:\n",
      "        {\n",
      "            type: Location\n",
      "            store:\n",
      "            {\n",
      "                id: https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault\n",
      "                type: DiskStorage\n",
      "                _rev: 1\n",
      "            }\n",
      "        }\n",
      "        contentSize:\n",
      "        {\n",
      "            unitCode: bytes\n",
      "            value: 52\n",
      "        }\n",
      "        contentUrl: https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/86eb143e-89cb-462d-af0e-48afb7172f2d\n",
      "        digest:\n",
      "        {\n",
      "            algorithm: SHA-256\n",
      "            value: 1dacd765946963fda4949753659089c5f532714b418d30788bedddfec47a389f\n",
      "        }\n",
      "        encodingFormat: text/csv\n",
      "        name: persons.csv\n",
      "    }\n",
      "    name: John Smith\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dataset = Dataset.from_resource(forge, [jane, john], store_metadata=True)\n",
    "print(*dataset, sep=\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creation from a dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See notebook `07 DataFrame IO.ipynb` for details on conversions of instances of Resource from a Pandas DataFrame."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### basics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe = pd.read_csv(\"../../data/persons.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Person</td>\n",
       "      <td>Marie Curie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Person</td>\n",
       "      <td>Albert Einstein</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     type             name\n",
       "0  Person      Marie Curie\n",
       "1  Person  Albert Einstein"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "persons = forge.from_dataframe(dataframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<count> 2\n",
      "<action> _register_many\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = Dataset(forge, name=\"Interesting people\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.add_parts(persons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<action> _register_one\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/5e1118bc-70b6-4b1d-b8ba-060a6f684230',\n",
       " 'type': 'Dataset',\n",
       " 'hasPart': [{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/58635f85-c6cc-4a7f-bf16-1558a5713080?rev=1',\n",
       "   'type': 'Person',\n",
       "   'name': 'Marie Curie'},\n",
       "  {'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/a9c2740b-2ef9-4557-841c-6ab425d35906?rev=1',\n",
       "   'type': 'Person',\n",
       "   'name': 'Albert Einstein'}],\n",
       " 'name': 'Interesting people'}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forge.as_json(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### advanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe = pd.read_csv(\"../../data/associations.tsv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>type</th>\n",
       "      <th>agent__type</th>\n",
       "      <th>agent__name</th>\n",
       "      <th>agent__gender__id</th>\n",
       "      <th>agent__gender__type</th>\n",
       "      <th>agent__gender__label</th>\n",
       "      <th>distribution</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(missing)</td>\n",
       "      <td>Curie Association</td>\n",
       "      <td>Association</td>\n",
       "      <td>Person</td>\n",
       "      <td>Marie Curie</td>\n",
       "      <td>http://purl.obolibrary.org/obo/PATO_0000383</td>\n",
       "      <td>LabeledOntologyEntity</td>\n",
       "      <td>female</td>\n",
       "      <td>../../data/scientists-database/marie_curie.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(missing)</td>\n",
       "      <td>Einstein Association</td>\n",
       "      <td>Association</td>\n",
       "      <td>Person</td>\n",
       "      <td>Albert Einstein</td>\n",
       "      <td>http://purl.obolibrary.org/obo/PATO_0000384</td>\n",
       "      <td>LabeledOntologyEntity</td>\n",
       "      <td>male</td>\n",
       "      <td>../../data/scientists-database/albert_einstein...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                  name         type agent__type      agent__name  \\\n",
       "0  (missing)     Curie Association  Association      Person      Marie Curie   \n",
       "1  (missing)  Einstein Association  Association      Person  Albert Einstein   \n",
       "\n",
       "                             agent__gender__id    agent__gender__type  \\\n",
       "0  http://purl.obolibrary.org/obo/PATO_0000383  LabeledOntologyEntity   \n",
       "1  http://purl.obolibrary.org/obo/PATO_0000384  LabeledOntologyEntity   \n",
       "\n",
       "  agent__gender__label                                       distribution  \n",
       "0               female     ../../data/scientists-database/marie_curie.txt  \n",
       "1                 male  ../../data/scientists-database/albert_einstein...  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe[\"distribution\"] = dataframe[\"distribution\"].map(lambda x: forge.attach(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "associations = forge.from_dataframe(dataframe, na=\"(missing)\", nesting=\"__\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    type: Association\n",
      "    agent:\n",
      "    {\n",
      "        type: Person\n",
      "        gender:\n",
      "        {\n",
      "            id: http://purl.obolibrary.org/obo/PATO_0000383\n",
      "            type: LabeledOntologyEntity\n",
      "            label: female\n",
      "        }\n",
      "        name: Marie Curie\n",
      "    }\n",
      "    distribution: LazyAction(operation=Store.upload, args=['../../data/scientists-database/marie_curie.txt', None])\n",
      "    name: Curie Association\n",
      "}\n",
      "{\n",
      "    type: Association\n",
      "    agent:\n",
      "    {\n",
      "        type: Person\n",
      "        gender:\n",
      "        {\n",
      "            id: http://purl.obolibrary.org/obo/PATO_0000384\n",
      "            type: LabeledOntologyEntity\n",
      "            label: male\n",
      "        }\n",
      "        name: Albert Einstein\n",
      "    }\n",
      "    distribution: LazyAction(operation=Store.upload, args=['../../data/scientists-database/albert_einstein.txt', None])\n",
      "    name: Einstein Association\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(*associations, sep=\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<count> 2\n",
      "<action> _register_many\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(associations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = Dataset(forge, name=\"Interesting associations\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.add_parts(associations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<action> _register_one\n",
      "<succeeded> True\n"
     ]
    }
   ],
   "source": [
    "forge.register(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/ae57773c-9e71-4ec0-85b9-6e5f52a04349',\n",
       " 'type': 'Dataset',\n",
       " 'hasPart': [{'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/8fc91342-42f7-4946-8a5d-a21be3448684?rev=1',\n",
       "   'type': 'Association',\n",
       "   'distribution': {'contentUrl': 'https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/5e4e7cb5-707a-4d4a-9682-36e5b993fe40'},\n",
       "   'name': 'Curie Association'},\n",
       "  {'id': 'https://bbp.epfl.ch/nexus/v1/resources/dke/kgforge/_/d78d842a-89f9-4273-bc70-9666c1f72781?rev=1',\n",
       "   'type': 'Association',\n",
       "   'distribution': {'contentUrl': 'https://bbp.epfl.ch/nexus/v1/files/dke/kgforge/1dc1c1a2-f13c-47d6-9533-92c97bf5a5d6'},\n",
       "   'name': 'Einstein Association'}],\n",
       " 'name': 'Interesting associations'}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forge.as_json(dataset)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 (nexusforgelatest)",
   "language": "python",
   "name": "nexusforgelatest"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}