{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From Pandas to Polars: A Paradigm Shift in DataFrame Processing\n",
    "\n",
    "This notebook accompanies the blog post comparing Pandas and Polars. It contains all the code examples for you to run and experiment with."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concept 1: The Expression-Based Paradigm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "# Create identical DataFrames\n",
    "data = {'name': ['Alice', 'Bob', 'Charlie'], 'score': [85, 92, 78]}\n",
    "pd_df = pd.DataFrame(data)\n",
    "pl_df = pl.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pandas: Adding a new column - direct assignment\n",
    "pd_df['score_doubled'] = pd_df['score'] * 2\n",
    "pd_df['passed'] = pd_df['score'] >= 80\n",
    "print(\"Pandas Result:\")\n",
    "print(pd_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Polars: Adding new columns - expression-based approach\n",
    "pl_df = pl_df.with_columns(\n",
    "    (pl.col(\"score\") * 2).alias(\"score_doubled\"),\n",
    "    (pl.col(\"score\") >= 80).alias(\"passed\")\n",
    ")\n",
    "print(\"Polars Result:\")\n",
    "print(pl_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concept 2: The Four Essential Contexts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "data = {\n",
    "    'department': ['Sales', 'Sales', 'Engineering', 'Engineering'],\n",
    "    'employee': ['Alice', 'Bob', 'Charlie', 'Diana'],\n",
    "    'salary': [50000, 60000, 75000, 80000]\n",
    "}\n",
    "pd_df = pd.DataFrame(data)\n",
    "pl_df = pl.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pandas Operations\n",
    "# Select columns\n",
    "result_select = pd_df[['department', 'salary']]\n",
    "\n",
    "# Filter rows\n",
    "result_filter = pd_df[pd_df['salary'] > 55000]\n",
    "\n",
    "# Group and aggregate\n",
    "result_agg = pd_df.groupby('department')['salary'].agg(['mean', 'max']).reset_index()\n",
    "\n",
    "print(\"Pandas Aggregation:\")\n",
    "print(result_agg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Polars Operations\n",
    "# Select columns\n",
    "result_select = pl_df.select(\"department\", \"salary\")\n",
    "\n",
    "# Filter rows\n",
    "result_filter = pl_df.filter(pl.col(\"salary\") > 55000)\n",
    "\n",
    "# Group and aggregate\n",
    "result_agg = pl_df.group_by(\"department\").agg(\n",
    "    pl.col(\"salary\").mean().alias(\"salary_mean\"),\n",
    "    pl.col(\"salary\").max().alias(\"salary_max\")\n",
    ")\n",
    "\n",
    "print(\"Polars Aggregation:\")\n",
    "print(result_agg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concept 3: No More Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "data = {'id': [101, 102, 103], 'value': [10, 20, 30]}\n",
    "pd_df = pd.DataFrame(data)\n",
    "pl_df = pl.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pandas: Set index and access by index value\n",
    "pd_df_indexed = pd_df.set_index('id')\n",
    "result = pd_df_indexed.loc[102]  # Returns a Series\n",
    "print(\"Pandas loc result:\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Polars: Direct filtering - no index needed\n",
    "result = pl_df.filter(pl.col(\"id\") == 102)  # Returns a DataFrame\n",
    "print(\"Polars filter result:\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concept 4: Strict Data Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "import numpy as np\n",
    "\n",
    "# Pandas with NaN\n",
    "pd_df = pd.DataFrame({'values': [1, 2, np.nan, 4]})\n",
    "print(\"Pandas dtypes:\")\n",
    "print(pd_df.dtypes)  # float64 - silently converted!\n",
    "\n",
    "# Polars with null\n",
    "pl_df = pl.DataFrame({'values': [1, 2, None, 4]})\n",
    "print(\"\\nPolars schema:\")\n",
    "print(pl_df.schema)  # {'values': Int64} - stays integer!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concept 6: Conditional Logic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "import numpy as np\n",
    "\n",
    "data = {'score': [45, 65, 85, 92, 55]}\n",
    "pd_df = pd.DataFrame(data)\n",
    "pl_df = pl.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pandas: Using np.where\n",
    "pd_df['grade'] = np.where(pd_df['score'] >= 60, 'Pass', 'Fail')\n",
    "\n",
    "# Using nested np.where for multiple conditions\n",
    "pd_df['letter_grade'] = np.where(\n",
    "    pd_df['score'] >= 90, 'A',\n",
    "    np.where(pd_df['score'] >= 80, 'B',\n",
    "    np.where(pd_df['score'] >= 70, 'C',\n",
    "    np.where(pd_df['score'] >= 60, 'D', 'F'))))\n",
    "\n",
    "print(\"Pandas Result:\")\n",
    "print(pd_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Polars: Readable chained conditions\n",
    "pl_df = pl_df.with_columns(\n",
    "    pl.when(pl.col(\"score\") >= 60)\n",
    "      .then(pl.lit(\"Pass\"))\n",
    "      .otherwise(pl.lit(\"Fail\"))\n",
    "      .alias(\"grade\"),\n",
    "    \n",
    "    pl.when(pl.col(\"score\") >= 90).then(pl.lit(\"A\"))\n",
    "      .when(pl.col(\"score\") >= 80).then(pl.lit(\"B\"))\n",
    "      .when(pl.col(\"score\") >= 70).then(pl.lit(\"C\"))\n",
    "      .when(pl.col(\"score\") >= 60).then(pl.lit(\"D\"))\n",
    "      .otherwise(pl.lit(\"F\"))\n",
    "      .alias(\"letter_grade\")\n",
    ")\n",
    "\n",
    "print(\"Polars Result:\")\n",
    "print(pl_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concept 7: Window Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "data = {\n",
    "    'department': ['Sales', 'Sales', 'Engineering', 'Engineering', 'Sales'],\n",
    "    'employee': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'],\n",
    "    'salary': [50000, 60000, 75000, 80000, 55000]\n",
    "}\n",
    "pd_df = pd.DataFrame(data)\n",
    "pl_df = pl.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pandas: Add department average salary as a new column\n",
    "pd_df['dept_avg_salary'] = pd_df.groupby('department')['salary'].transform('mean')\n",
    "\n",
    "# Calculate deviation from department mean\n",
    "pd_df['salary_deviation'] = pd_df['salary'] - pd_df.groupby('department')['salary'].transform('mean')\n",
    "\n",
    "print(\"Pandas Result:\")\n",
    "print(pd_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Polars: Window functions with .over()\n",
    "pl_df = pl_df.with_columns(\n",
    "    pl.col(\"salary\").mean().over(\"department\").alias(\"dept_avg_salary\"),\n",
    "    (pl.col(\"salary\") - pl.col(\"salary\").mean().over(\"department\")).alias(\"salary_deviation\")\n",
    ")\n",
    "\n",
    "print(\"Polars Result:\")\n",
    "print(pl_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concept 8: Avoid apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "data = {'text': ['hello', 'world', 'polars'], 'value': [1, 2, 3]}\n",
    "pd_df = pd.DataFrame(data)\n",
    "pl_df = pl.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pandas: using apply\n",
    "pd_df['text_upper'] = pd_df['text'].apply(str.upper)\n",
    "pd_df['value_squared'] = pd_df['value'].apply(lambda x: x ** 2)\n",
    "\n",
    "print(\"Pandas Result:\")\n",
    "print(pd_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Polars: Preferred - Use native expression methods\n",
    "pl_df = pl_df.with_columns(\n",
    "    pl.col(\"text\").str.to_uppercase().alias(\"text_upper\"),\n",
    "    (pl.col(\"value\") ** 2).alias(\"value_squared\")\n",
    ")\n",
    "\n",
    "print(\"Polars Result:\")\n",
    "print(pl_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting Started"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "\n",
    "# Your first Polars DataFrame\n",
    "df = pl.DataFrame({\n",
    "    \"name\": [\"Alice\", \"Bob\", \"Charlie\"],\n",
    "    \"age\": [25, 30, 35],\n",
    "    \"city\": [\"NYC\", \"LA\", \"Chicago\"]\n",
    "})\n",
    "\n",
    "# Your first expression pipeline\n",
    "result = df.with_columns(\n",
    "    (pl.col(\"age\") + 5).alias(\"age_in_5_years\"),\n",
    "    pl.col(\"city\").str.to_uppercase().alias(\"city_upper\")\n",
    ").filter(\n",
    "    pl.col(\"age\") > 26\n",
    ")\n",
    "\n",
    "print(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}