{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pandas Profiling: NASA Meteorites example\n",
    "Source of data: https://data.nasa.gov/Space-Science/Meteorite-Landings/gh4g-9sfh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The autoreload instruction reloads modules automatically before code execution, which is helpful for the update below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make sure that we have the latest version of pandas-profiling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "!{sys.executable} -m pip install -U pandas-profiling[notebook]\n",
    "!jupyter nbextension enable --py widgetsnbextension"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You might want to restart the kernel now."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "\n",
    "import ydata_profiling\n",
    "from ydata_profiling.utils.cache import cache_file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and prepare example dataset\n",
    "We add some fake variables for illustrating pandas-profiling capabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = cache_file(\n",
    "    \"meteorites.csv\",\n",
    "    \"https://data.nasa.gov/api/views/gh4g-9sfh/rows.csv?accessType=DOWNLOAD\",\n",
    ")\n",
    "\n",
    "df = pd.read_csv(file_name)\n",
    "\n",
    "# Note: Pandas does not support dates before 1880, so we ignore these for this analysis\n",
    "df[\"year\"] = pd.to_datetime(df[\"year\"], errors=\"coerce\")\n",
    "\n",
    "# Example: Constant variable\n",
    "df[\"source\"] = \"NASA\"\n",
    "\n",
    "# Example: Boolean variable\n",
    "df[\"boolean\"] = np.random.choice([True, False], df.shape[0])\n",
    "\n",
    "# Example: Mixed with base types\n",
    "df[\"mixed\"] = np.random.choice([1, \"A\"], df.shape[0])\n",
    "\n",
    "# Example: Highly correlated variables\n",
    "df[\"reclat_city\"] = df[\"reclat\"] + np.random.normal(scale=5, size=(len(df)))\n",
    "\n",
    "# Example: Duplicate observations\n",
    "duplicates_to_add = pd.DataFrame(df.iloc[0:10])\n",
    "duplicates_to_add[\"name\"] = duplicates_to_add[\"name\"] + \" copy\"\n",
    "\n",
    "df = pd.concat([df, duplicates_to_add], ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inline report without saving object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = df.profile_report(\n",
    "    sort=None, html={\"style\": {\"full_width\": True}}, progress_bar=False\n",
    ")\n",
    "report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save report to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "profile_report = df.profile_report(html={\"style\": {\"full_width\": True}})\n",
    "profile_report.to_file(\"/tmp/example.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### More analysis (Unicode) and Print existing ProfileReport object inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "profile_report = df.profile_report(\n",
    "    explorative=True, html={\"style\": {\"full_width\": True}}\n",
    ")\n",
    "profile_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notebook Widgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "profile_report.to_widgets()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}