{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## collections_email" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "np.random.seed(24)\n", "n = 5000\n", "email = np.random.binomial(1, 0.5, n)\n", "\n", "credit_limit = np.random.gamma(6, 200, n)\n", "risk_score = np.random.beta(credit_limit, credit_limit.mean(), n)\n", "\n", "opened = np.random.normal(5 + 0.001*credit_limit - 4*risk_score, 2)\n", "opened = (opened > 4).astype(float) * email\n", "\n", "\n", "agreement = np.random.normal(30 +(-0.003*credit_limit - 10*risk_score), 7) * 2 * opened\n", "agreement = (agreement > 40).astype(float)\n", "\n", "payments = (np.random.normal(500 + 0.16*credit_limit - 40*risk_score + 11*agreement + email, 75).astype(int) // 10) * 10\n", "\n", "data = pd.DataFrame(dict(payments=payments,\n", " email=email,\n", " opened=opened,\n", " agreement=agreement,\n", " credit_limit=credit_limit,\n", " risk_score=risk_score))\n", "\n", "data.to_csv(\"collections_email.csv\", index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## hospital_treatment" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "np.random.seed(24)\n", "n = 80\n", "\n", "hospital = np.random.binomial(1, 0.5, n)\n", "\n", "treatment = np.where(hospital.astype(bool),\n", " np.random.binomial(1, 0.9, n),\n", " np.random.binomial(1, 0.1, n))\n", "\n", "severity = np.where(hospital.astype(bool), \n", " np.random.normal(20, 5, n),\n", " np.random.normal(10, 5, n))\n", "\n", "days = np.random.normal(15 + -5*treatment + 2*severity, 7).astype(int)\n", "\n", "hospital = pd.DataFrame(dict(hospital=hospital,\n", " treatment=treatment,\n", " severity=severity,\n", " days=days))\n", "\n", "hospital.to_csv(\"hospital_treatment.csv\", index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## app engagement push" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "np.random.seed(24)\n", "n = 10000\n", "\n", "push_assigned = np.random.binomial(1, 0.5, n)\n", "\n", "income = np.random.gamma(6, 200, n)\n", "\n", "push_delivered = np.random.normal(5 + 0.3+income, 500)\n", "push_delivered = ((push_delivered > 800) & (push_assigned == 1)).astype(int)\n", "\n", "in_app_purchase = (np.random.normal(100 + 20*push_delivered + 0.5*income, 75).astype(int) // 10)\n", "\n", "data = pd.DataFrame(dict(in_app_purchase=in_app_purchase,\n", " push_assigned=push_assigned,\n", " push_delivered=push_delivered))\n", "\n", "data.to_csv(\"app_engagement_push.csv\", index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Drug Impact" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "\n", "def make_confounded_data(N):\n", "\n", " def get_severity(df):\n", " return ((np.random.beta(1, 3, size=df.shape[0]) * (df[\"age\"] < 30)) +\n", " (np.random.beta(3, 1.5, size=df.shape[0]) * (df[\"age\"] >= 30)))\n", "\n", " def get_treatment(df):\n", " return ((.33 * df[\"sex\"] +\n", " 1.5 * df[\"severity\"] + df[\"severity\"] ** 2 +\n", " 0.15 * np.random.normal(size=df.shape[0])) > 1.5).astype(int)\n", "\n", " def get_recovery(df):\n", " return ((2 +\n", " 0.5 * df[\"sex\"] +\n", " 0.03 * df[\"age\"] + 0.03 * ((df[\"age\"] * 0.1) ** 2) +\n", " df[\"severity\"] + np.log(df[\"severity\"]) +\n", " df[\"sex\"] * df[\"severity\"] -\n", " df[\"medication\"]) * 10).astype(int)\n", "\n", " np.random.seed(1111)\n", " sexes = np.random.randint(0, 2, size=N)\n", " ages = np.random.gamma(8, scale=4, size=N)\n", " meds = np.random.beta(1, 1, size=N)\n", "\n", " # dados com designação aleatória\n", " df_rnd = pd.DataFrame(dict(sex=sexes, age=ages, medication=meds))\n", " df_rnd['severity'] = get_severity(df_rnd)\n", " df_rnd['recovery'] = get_recovery(df_rnd)\n", "\n", " features = ['sex', 'age', 'severity', 'medication', 'recovery']\n", " df_rnd = df_rnd[features] # to enforce column order\n", "\n", " # dados observacionais\n", " df_obs = df_rnd.copy()\n", " df_obs['medication'] = get_treatment(df_obs)\n", " df_obs['recovery'] = get_recovery(df_obs)\n", "\n", " # dados contrafactuais data\n", " df_ctf = df_obs.copy()\n", " df_ctf['medication'] = ((df_ctf['medication'] == 1) ^ 1).astype(float)\n", " df_ctf['recovery'] = get_recovery(df_ctf)\n", "\n", " return df_rnd, df_obs, df_ctf\n", "\n", "np.random.seed(1234)\n", "df_rnd, df_obs, df_ctf = make_confounded_data(20000)\n", "\n", "df_obs.to_csv(\"medicine_impact_recovery.csv\", index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Bilboard Mkt" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "np.random.seed(123)\n", "POAMay = np.random.gamma(7,10, 500) * np.random.binomial(1, .7, 500)\n", "POAJul = np.random.gamma(7,15, 800) * np.random.binomial(1, .8, 800)\n", "FLMay = np.random.gamma(10,20, 1300) * np.random.binomial(1, .85, 1300)\n", "FLJul = np.random.gamma(11,21, 2000) * np.random.binomial(1, .9, 2000)\n", "\n", "data = pd.concat([\n", " pd.DataFrame(dict(deposits = POAMay.astype(int), poa=1, jul=0)),\n", " pd.DataFrame(dict(deposits = POAJul.astype(int), poa=1, jul=1)),\n", " pd.DataFrame(dict(deposits = FLMay.astype(int), poa=0, jul=0)),\n", " pd.DataFrame(dict(deposits = FLJul.astype(int), poa=0, jul=1))\n", "])\n", "data.to_csv(\"billboard_impact.csv\", index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Customer Lifecicle" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.3721\n", "[1. 1. 1. 1. 1. 1. 1. 0.9999 0.9994 0.9984\n", " 0.9966 0.994 0.9886 0.9791 0.9663 0.944 0.9128 0.8726 0.8205 0.7603\n", " 0.6932 0.6138 0.5295 0.4424 0.3618 0.2919 0.2308 0.1769 0.1286 0.0942]\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "from matplotlib import pyplot as plt\n", "from toolz import merge\n", "from sklearn.preprocessing import LabelEncoder\n", "\n", "np.random.seed(12)\n", "\n", "n = 10000\n", "t = 30\n", "\n", "age = 18 + np.random.poisson(10, n)\n", "income = 500+np.random.exponential(2000, size=n).astype(int)\n", "region = np.random.choice(np.random.lognormal(4, size=50), size=n)\n", "\n", "freq = np.random.lognormal((1 + age/(18+10)).astype(int))\n", "churn = np.random.poisson((income-500)/2000 + 22, n)\n", "\n", "ones = np.ones((n, t))\n", "alive = (np.cumsum(ones, axis=1) <= churn.reshape(n, 1)).astype(int)\n", "buy = np.random.binomial(1, ((1/(freq+1)).reshape(n, 1) * ones))\n", "\n", "cacq = -1*abs(np.random.normal(region, 2, size=n).astype(int))\n", "transactions = np.random.lognormal(((income.mean() - 500) / 1000), size=(n, t)).astype(int) * buy * alive\n", "\n", "data = pd.DataFrame(merge({\"customer_id\": range(n), \"cacq\":cacq},\n", " {f\"day_{day}\": trans \n", " for day, trans in enumerate(transactions.T)}))\n", "\n", "encoced = {value:index for index, value in\n", " enumerate(np.random.permutation(np.unique(region)))}\n", "\n", "customer_features = pd.DataFrame(dict(customer_id=range(n), \n", " region=region,\n", " income=income,\n", " age=age)).replace({\"region\":encoced}).astype(int)\n", "\n", "print((data.drop(columns=[\"customer_id\"]).sum(axis=1) > 0).mean()) # proportion of profitable customers\n", "print((alive).mean(axis=0)) # alive customer per days\n", "\n", "data.to_csv(\"./causal-inference-for-the-brave-and-true/data/customer_transactions.csv\", index=False)\n", "customer_features.to_csv(\"./causal-inference-for-the-brave-and-true/data/customer_features.csv\", index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Prince and Sales" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "from matplotlib import pyplot as plt\n", "\n", "np.random.seed(5)\n", "\n", "def price_elast(price, temp, weekday, cost):\n", " return -4 + 0.2*price + 0.05*temp + 2*np.isin(weekday, [1,7]) + 0.3 * cost\n", "\n", "def sales(price, temp, weekday, cost):\n", " elast = -abs(price_elast(price, temp, weekday, cost))\n", " output = np.random.normal(200 + 20*np.isin(weekday, [1,7]) + 1.3 * temp +\n", " 5*elast * price, 5).astype(int)\n", " \n", " return output\n", "\n", "\n", "n_rnd = 5000\n", "\n", "temp = np.random.normal(24, 4, n_rnd).round(1)\n", "weekday = np.random.choice(list(range(1, 8)), n_rnd)\n", "cost = np.random.choice([0.3, 0.5, 1.0, 1.5], n_rnd)\n", "price_rnd = np.random.choice(list(range(3, 11)), n_rnd)\n", "\n", "price_df_rnd = pd.DataFrame(dict(temp=temp, weekday=weekday, cost=cost,\n", " price=price_rnd, sales=sales(price_rnd, temp, weekday, cost)))\n", "\n", "n = 10000\n", "temp = np.random.normal(24, 4, n).round(1)\n", "weekday = np.random.choice(list(range(1, 8)), n)\n", "cost = np.random.choice([0.3, 0.5, 1.0, 1.5], n)\n", "price = np.random.normal(5 + cost + np.isin(weekday, [1,7])).round(1)\n", "\n", "price_df = pd.DataFrame(dict(temp=temp, weekday=weekday, cost=cost,\n", " price=price, sales=sales(price, temp, weekday, cost)))\n", "\n", "price_df_rnd.to_csv(\"./causal-inference-for-the-brave-and-true/data/ice_cream_sales_rnd.csv\", index=False)\n", "price_df.to_csv(\"./causal-inference-for-the-brave-and-true/data/ice_cream_sales.csv\", index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Marketing Email" ] }, { "cell_type": "code", "execution_count": 216, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "from matplotlib import pyplot as plt\n", "from sklearn.preprocessing import MinMaxScaler\n", "\n", "scaler = MinMaxScaler((0, 1))\n", "\n", "np.random.seed(12321)\n", "\n", "n_rnd=5000\n", "\n", "age = 18 + np.random.normal(24, 4, n_rnd).round(1)\n", "income = 500 + np.random.gamma(1, age * 100, n_rnd).round(2)\n", "insurance = np.random.gamma(30/age, age*1000, n_rnd).round(2)\n", "invested = np.random.gamma(age/10, income/2, n_rnd).round(2)\n", "\n", "em1_ps = income.min()/(income + 10)\n", "em2_ps = invested/(invested.max())\n", "em3_ps = np.where(age > 40, scaler.fit_transform(-income.reshape(-1,1)).ravel(), 0)\n", "\n", "em1 = np.random.binomial(1, em1_ps)\n", "em2 = np.random.binomial(1, em2_ps)\n", "em3 = np.random.binomial(1, em3_ps)\n", "\n", "elast_em1 = scaler.fit_transform((-3*age + 0.005*invested).reshape(-1,1)).ravel()\n", "elast_em2 = scaler.fit_transform((age + income*0.005).reshape(-1,1)).ravel()\n", "elast_em3 = scaler.fit_transform((-insurance).reshape(-1,1)).ravel()\n", "\n", "buy = scaler.fit_transform((1 + 0.4*age - invested/10000).reshape(-1,1)).ravel()\n", "buy += elast_em1*em1 + elast_em2*em2 + elast_em3*em3\n", "buy = scaler.fit_transform(buy.reshape(-1,1)).ravel()\n", "buy = np.random.binomial(1, buy).round(2)\n", "\n", "df = pd.DataFrame(dict(age=age, income=income, insurance=insurance, invested=invested,\n", " em1_ps=em1_ps, em2_ps=em2_ps, em3_ps=em3_ps,\n", " em1=em1, em2=em2, em3=em3,\n", " converted=buy))\n", "\n", "df.to_csv(\"./causal-inference-for-the-brave-and-true/data/invest_email.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "from matplotlib import pyplot as plt\n", "from sklearn.preprocessing import MinMaxScaler\n", "\n", "scaler = MinMaxScaler((0.001, 0.999))\n", "\n", "np.random.seed(12321)\n", "\n", "n_rnd=15000\n", "\n", "age = 18 + np.random.normal(24, 4, n_rnd).round(1)\n", "income = 500 + np.random.gamma(1, age * 100, n_rnd).round(2)\n", "insurance = np.random.gamma(30/age, age*1000, n_rnd).round(2)\n", "invested = np.random.gamma(age/10, income/2, n_rnd).round(2)\n", "\n", "em1 = np.random.binomial(1, 0.5, n_rnd)\n", "em2 = np.random.binomial(1, 0.2, n_rnd)\n", "em3 = np.random.binomial(1, 0.9, n_rnd)\n", "\n", "elast_em1 = scaler.fit_transform((-3*age + 0.005*invested).reshape(-1,1)).ravel()\n", "elast_em2 = scaler.fit_transform((age + income*0.005).reshape(-1,1)).ravel()\n", "elast_em3 = scaler.fit_transform((-insurance).reshape(-1,1)).ravel()\n", "\n", "buy = (200*elast_em1*em1 + 100*elast_em2*em2 + 10*elast_em3*em3 \n", " + 1.5*age + 0.0005*invested - 0.0001*income)\n", "\n", "buy = scaler.fit_transform(buy.reshape(-1,1)).ravel()\n", "\n", "buy = np.random.binomial(1, buy)\n", "\n", "df = pd.DataFrame(dict(age=age, income=income, insurance=insurance, invested=invested,\n", " em1=em1, em2=em2, em3=em3,\n", " converted=buy))\n", "\n", "df.to_csv(\"./causal-inference-for-the-brave-and-true/data/invest_email_rnd.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "from matplotlib import pyplot as plt\n", "from sklearn.preprocessing import MinMaxScaler\n", "\n", "scaler = MinMaxScaler((0.001, 0.999))\n", "\n", "np.random.seed(12321)\n", "\n", "n_rnd=15000\n", "\n", "age = 18 + np.random.normal(24, 4, n_rnd).round(1)\n", "income = 500 + np.random.gamma(1, age * 100, n_rnd).round(2)\n", "insurance = np.random.gamma(30/age, age*1000, n_rnd).round(2)\n", "invested = np.random.gamma(age/10, income/2, n_rnd).round(2)\n", "\n", "em1_ps = income.min()/(income + 10)\n", "em2_ps = invested/(invested.max())\n", "em3_ps = np.where(age > 40, scaler.fit_transform(-income.reshape(-1,1)).ravel(), 0)\n", "\n", "\n", "em1 = np.random.binomial(1, em1_ps)\n", "em2 = np.random.binomial(1, em2_ps)\n", "em3 = np.random.binomial(1, em3_ps)\n", "\n", "elast_em1 = scaler.fit_transform((-3*age + 0.005*invested).reshape(-1,1)).ravel()\n", "elast_em2 = scaler.fit_transform((age + income*0.005).reshape(-1,1)).ravel()\n", "elast_em3 = scaler.fit_transform((-insurance).reshape(-1,1)).ravel()\n", "\n", "buy = (200*elast_em1*em1 + 100*elast_em2*em2 + 10*elast_em3*em3 \n", " + 1.5*age + 0.0005*invested - 0.0001*income)\n", "\n", "buy = scaler.fit_transform(buy.reshape(-1,1)).ravel()\n", "\n", "buy = np.random.binomial(1, buy)\n", "\n", "df = pd.DataFrame(dict(age=age, income=income, insurance=insurance, invested=invested,\n", " em1=em1, em2=em2, em3=em3,\n", " converted=buy))\n", "\n", "df.to_csv(\"./causal-inference-for-the-brave-and-true/data/invest_email_biased.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "causal-glory", "language": "python", "name": "causal-glory" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 2 }