{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "#### This notebook demonstrates the use of the learning fair representations algorithm for bias mitigation\n", "Learning fair representations [1] is a pre-processing technique that finds a latent representation which encodes the data well but obfuscates information about protected attributes. We will see how to use this algorithm for learning representations that encourage individual fairness and apply them on the Adult dataset.\n", "\n", "References:\n", "\n", "[1] R. Zemel, Y. Wu, K. Swersky, T. Pitassi, and C. Dwork, \"Learning Fair Representations.\" \n", "International Conference on Machine Learning, 2013." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "# Load all necessary packages\n", "import sys\n", "sys.path.append(\"../\")\n", "from aif360.datasets import BinaryLabelDataset\n", "from aif360.datasets import AdultDataset\n", "from aif360.metrics import BinaryLabelDatasetMetric\n", "from aif360.metrics import ClassificationMetric\n", "from aif360.metrics.utils import compute_boolean_conditioning_vector\n", "\n", "from aif360.algorithms.preprocessing.optim_preproc_helpers.data_preproc_functions import load_preproc_data_adult\n", "from aif360.algorithms.preprocessing.lfr import LFR\n", "\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.metrics import accuracy_score\n", "from sklearn.metrics import classification_report\n", "\n", "from IPython.display import Markdown, display\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "\n", "from common_utils import compute_metrics" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Load dataset and set options" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Get the dataset and split into train and test\n", "dataset_orig = load_preproc_data_adult()\n", "dataset_orig_train, dataset_orig_test = dataset_orig.split([0.7], shuffle=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Clean up training data" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "#### Training Dataset shape" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(34189, 18)\n" ] }, { "data": { "text/markdown": [ "#### Favorable and unfavorable labels" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "1.0 0.0\n" ] }, { "data": { "text/markdown": [ "#### Protected attribute names" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "['sex', 'race']\n" ] }, { "data": { "text/markdown": [ "#### Privileged and unprivileged protected attribute values" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[array([1.]), array([1.])] [array([0.]), array([0.])]\n" ] }, { "data": { "text/markdown": [ "#### Dataset feature names" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "['race', 'sex', 'Age (decade)=10', 'Age (decade)=20', 'Age (decade)=30', 'Age (decade)=40', 'Age (decade)=50', 'Age (decade)=60', 'Age (decade)=>=70', 'Education Years=6', 'Education Years=7', 'Education Years=8', 'Education Years=9', 'Education Years=10', 'Education Years=11', 'Education Years=12', 'Education Years=<6', 'Education Years=>12']\n" ] } ], "source": [ "# print out some labels, names, etc.\n", "display(Markdown(\"#### Training Dataset shape\"))\n", "print(dataset_orig_train.features.shape)\n", "display(Markdown(\"#### Favorable and unfavorable labels\"))\n", "print(dataset_orig_train.favorable_label, dataset_orig_train.unfavorable_label)\n", "display(Markdown(\"#### Protected attribute names\"))\n", "print(dataset_orig_train.protected_attribute_names)\n", "display(Markdown(\"#### Privileged and unprivileged protected attribute values\"))\n", "print(dataset_orig_train.privileged_protected_attributes, \n", " dataset_orig_train.unprivileged_protected_attributes)\n", "display(Markdown(\"#### Dataset feature names\"))\n", "print(dataset_orig_train.feature_names)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Metric for original training data" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "#### Original training dataset" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Difference in mean outcomes between unprivileged and privileged groups = -0.193139\n" ] }, { "data": { "text/markdown": [ "#### Original test dataset" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Difference in mean outcomes between unprivileged and privileged groups = -0.197697\n" ] } ], "source": [ "# Metric for the original dataset\n", "privileged_groups = [{'sex': 1.0}]\n", "unprivileged_groups = [{'sex': 0.0}]\n", "\n", "metric_orig_train = BinaryLabelDatasetMetric(dataset_orig_train, \n", " unprivileged_groups=unprivileged_groups,\n", " privileged_groups=privileged_groups)\n", "display(Markdown(\"#### Original training dataset\"))\n", "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % metric_orig_train.mean_difference())\n", "metric_orig_test = BinaryLabelDatasetMetric(dataset_orig_test, \n", " unprivileged_groups=unprivileged_groups,\n", " privileged_groups=privileged_groups)\n", "display(Markdown(\"#### Original test dataset\"))\n", "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % metric_orig_test.mean_difference())\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Train with and transform the original training data" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "scale_orig = StandardScaler()\n", "dataset_orig_train.features = scale_orig.fit_transform(dataset_orig_train.features)\n", "dataset_orig_test.features = scale_orig.transform(dataset_orig_test.features)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "step: 0, loss: 1.0939550595829053, L_x: 2.531834521858599, L_y: 0.8200826015334493, L_z: 0.010344502931797964\n", "step: 250, loss: 0.9162820270109503, L_x: 2.529109218043187, L_y: 0.6432961063010657, L_z: 0.010037499452782905\n", "step: 500, loss: 0.8207071510514392, L_x: 2.5204911168067197, L_y: 0.5500397646035967, L_z: 0.00930913738358528\n", "step: 750, loss: 0.8102771268166408, L_x: 2.511873834704061, L_y: 0.5427956868742799, L_z: 0.008147028235977415\n", "step: 1000, loss: 0.7996570283329768, L_x: 2.480828451323288, L_y: 0.5399446552800813, L_z: 0.00581476396028337\n", "step: 1250, loss: 0.7844631169970814, L_x: 2.4242508289183613, L_y: 0.5304307199052671, L_z: 0.005803657099989009\n", "step: 1500, loss: 0.7653305722023572, L_x: 2.3297047767431986, L_y: 0.5176248867874912, L_z: 0.007367603870273078\n", "step: 1750, loss: 0.7154304631442515, L_x: 2.085955877234543, L_y: 0.48081670080967953, L_z: 0.013009087305558827\n", "step: 2000, loss: 0.6906420918886886, L_x: 1.896344106091722, L_y: 0.4646651544564373, L_z: 0.018171263411539594\n", "step: 2250, loss: 0.6783680937630076, L_x: 1.7895665853948028, L_y: 0.4587714378849466, L_z: 0.020319998669290275\n", "step: 2500, loss: 0.6725576747654705, L_x: 1.742061633693402, L_y: 0.4577729094336143, L_z: 0.020289300981257967\n", "step: 2750, loss: 0.6694103860159343, L_x: 1.7548885984309939, L_y: 0.4545867175857845, L_z: 0.019667404293525217\n", "step: 3000, loss: 0.6658207636894926, L_x: 1.7515234617350093, L_y: 0.4539151313299769, L_z: 0.018376643093007367\n", "step: 3250, loss: 0.6481415219979564, L_x: 1.7252276686316934, L_y: 0.4491717858033674, L_z: 0.013223484665709846\n", "step: 3500, loss: 0.645366243737316, L_x: 1.7196207136719521, L_y: 0.4482843307446003, L_z: 0.012559920812760247\n", "step: 3750, loss: 0.6425278186287126, L_x: 1.7117758355776211, L_y: 0.4473063883366716, L_z: 0.012021923367139413\n", "step: 4000, loss: 0.6419409673076768, L_x: 1.7092609385556714, L_y: 0.44744616781598634, L_z: 0.011784352818061686\n", "step: 4250, loss: 0.6377801462539607, L_x: 1.6917081956472533, L_y: 0.4496335370425122, L_z: 0.009487894823361622\n" ] } ], "source": [ "# Input recontruction quality - Ax\n", "# Fairness constraint - Az\n", "# Output prediction error - Ay\n", "\n", "privileged_groups = [{'sex': 1}]\n", "unprivileged_groups = [{'sex': 0}]\n", " \n", "TR = LFR(unprivileged_groups=unprivileged_groups,\n", " privileged_groups=privileged_groups,\n", " k=10, Ax=0.1, Ay=1.0, Az=2.0,\n", " verbose=1\n", " )\n", "TR = TR.fit(dataset_orig_train, maxiter=5000, maxfun=5000)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Transform training data and align features\n", "dataset_transf_train = TR.transform(dataset_orig_train)\n", "dataset_transf_test = TR.transform(dataset_orig_test)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(classification_report(dataset_orig_test.labels, dataset_transf_test.labels))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metric_transf_train = BinaryLabelDatasetMetric(dataset_transf_train, \n", " unprivileged_groups=unprivileged_groups,\n", " privileged_groups=privileged_groups)\n", "display(Markdown(\"#### Transformed training dataset\"))\n", "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % metric_transf_train.mean_difference())\n", "metric_transf_test = BinaryLabelDatasetMetric(dataset_transf_test, \n", " unprivileged_groups=unprivileged_groups,\n", " privileged_groups=privileged_groups)\n", "display(Markdown(\"#### Transformed test dataset\"))\n", "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % metric_transf_test.mean_difference())\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from common_utils import compute_metrics\n", "\n", "display(Markdown(\"#### Predictions from transformed testing data\"))\n", "bal_acc_arr_transf = []\n", "disp_imp_arr_transf = []\n", "\n", "class_thresh_arr = np.linspace(0.01, 0.99, 100)\n", "\n", "dataset_transf_test_new = dataset_orig_test.copy(deepcopy=True)\n", "dataset_transf_test_new.scores = dataset_transf_test.scores\n", "\n", "\n", "for thresh in class_thresh_arr:\n", " \n", " fav_inds = dataset_transf_test_new.scores > thresh\n", " dataset_transf_test_new.labels[fav_inds] = 1.0\n", " dataset_transf_test_new.labels[~fav_inds] = 0.0\n", " \n", " metric_test_aft = compute_metrics(dataset_orig_test, dataset_transf_test_new, \n", " unprivileged_groups, privileged_groups,\n", " disp = False)\n", "\n", " bal_acc_arr_transf.append(metric_test_aft[\"Balanced accuracy\"])\n", " disp_imp_arr_transf.append(metric_test_aft[\"Disparate impact\"])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "fig, ax1 = plt.subplots(figsize=(10,7))\n", "ax1.plot(class_thresh_arr, bal_acc_arr_transf)\n", "ax1.set_xlabel('Classification Thresholds', fontsize=16, fontweight='bold')\n", "ax1.set_ylabel('Balanced Accuracy', color='b', fontsize=16, fontweight='bold')\n", "ax1.xaxis.set_tick_params(labelsize=14)\n", "ax1.yaxis.set_tick_params(labelsize=14)\n", "\n", "\n", "ax2 = ax1.twinx()\n", "ax2.plot(class_thresh_arr, np.abs(1.0-np.array(disp_imp_arr_transf)), color='r')\n", "ax2.set_ylabel('abs(1-disparate impact)', color='r', fontsize=16, fontweight='bold')\n", "ax2.yaxis.set_tick_params(labelsize=14)\n", "ax2.grid(True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "abs(1-disparate impact) must be small (close to 0) for classifier predictions to be fair." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "display(Markdown(\"#### Individual fairness metrics\"))\n", "print(\"Consistency of labels in transformed training dataset= %f\" %metric_transf_train.consistency())\n", "print(\"Consistency of labels in original training dataset= %f\" %metric_orig_train.consistency())\n", "print(\"Consistency of labels in transformed test dataset= %f\" %metric_transf_test.consistency())\n", "print(\"Consistency of labels in original test dataset= %f\" %metric_orig_test.consistency())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def check_algorithm_success():\n", " \"\"\"Transformed dataset consistency should be greater than original dataset.\"\"\"\n", " assert metric_transf_test.consistency() > metric_orig_test.consistency(), \"Transformed dataset consistency should be greater than original dataset.\"\n", "\n", "check_algorithm_success() " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.10" } }, "nbformat": 4, "nbformat_minor": 2 }