{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZYchV5RWwdv5"
   },
   "source": [
    "# Install Comet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DJnmqphuY2eI"
   },
   "outputs": [],
   "source": [
    "%pip install -U \"comet_ml>=3.44.0\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "crOcPHobwhGL"
   },
   "source": [
    "# Import Comet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HNQRM0U3caiY"
   },
   "outputs": [],
   "source": [
    "import comet_ml\n",
    "\n",
    "comet_ml.login(project_name=\"keras-image-classification\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cgqwGSwtzVWD"
   },
   "source": [
    "# Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "e-5rRYaUw5AF"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3T8Qg7GYwwM2"
   },
   "source": [
    "# Create an Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "m-OWU2ynwp2u"
   },
   "outputs": [],
   "source": [
    "# create an experiment with your api key\n",
    "experiment = comet_ml.start()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TJuThf1TxP_G"
   },
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YFbozHiJxTax"
   },
   "outputs": [],
   "source": [
    "# the data, shuffled and split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "x_train = x_train.reshape(60000, 784)\n",
    "x_test = x_test.reshape(10000, 784)\n",
    "\n",
    "x_train = x_train.astype(\"float32\")\n",
    "x_test = x_test.astype(\"float32\")\n",
    "\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "print(x_train.shape[0], \"train samples\")\n",
    "print(x_test.shape[0], \"test samples\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dBDRqvzfxCHX"
   },
   "source": [
    "# Define Training Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xsZ8x-ntxCkV"
   },
   "outputs": [],
   "source": [
    "# these will all get logged\n",
    "params = {\n",
    "    \"batch_size\": 128,\n",
    "    \"epochs\": 2,\n",
    "    \"layer1_type\": \"Dense\",\n",
    "    \"layer1_num_nodes\": 64,\n",
    "    \"layer1_activation\": \"relu\",\n",
    "    \"optimizer\": \"adam\",\n",
    "}\n",
    "\n",
    "num_classes = 10\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "npaVMlZPxify"
   },
   "source": [
    "# Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "flH9VHctxjSB"
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(64, activation=\"relu\", input_shape=(784,)))\n",
    "model.add(Dense(num_classes, activation=\"softmax\"))\n",
    "\n",
    "# print model.summary() to preserve automatically in `Output` tab\n",
    "print(model.summary())\n",
    "\n",
    "model.compile(\n",
    "    loss=\"categorical_crossentropy\", optimizer=params[\"optimizer\"], metrics=[\"accuracy\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "U7U0MwO7yQ5g"
   },
   "source": [
    "# Log Experiment Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wtXRSEccyTEO"
   },
   "outputs": [],
   "source": [
    "experiment.log_parameters(params)\n",
    "experiment.log_dataset_hash(x_train)  # creates and logs a hash of your data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "c3jExnbxxoav"
   },
   "source": [
    "# Log Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Y8DNdezRxpng"
   },
   "outputs": [],
   "source": [
    "# will log metrics with the prefix 'train_'\n",
    "model.fit(\n",
    "    x_train,\n",
    "    y_train,\n",
    "    batch_size=params[\"batch_size\"],\n",
    "    epochs=params[\"epochs\"],\n",
    "    verbose=1,\n",
    "    validation_data=(x_test, y_test),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FSOLXEonyXoG"
   },
   "source": [
    "# Log Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "o5t36SlJ7evE"
   },
   "source": [
    "## Logging Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DwJRmFH8yYqI"
   },
   "outputs": [],
   "source": [
    "loss, accuracy = model.evaluate(x_test, y_test)\n",
    "metrics = {\"loss\": loss, \"accuracy\": accuracy}\n",
    "with experiment.test():\n",
    "    experiment.log_metrics(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L9q92OsE7g5X"
   },
   "source": [
    "## Logging the Confusion Matrix and Images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "K2KvVn3pF5m7"
   },
   "source": [
    "Comet gives you the option to log Images with the `experiment.log_image` method. We're going to use this method along with our Confusion Matrix so that we can log samples from our dataset and identify misclassified images in the UI.   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jWpEX1gz8gNX"
   },
   "outputs": [],
   "source": [
    "predictions = model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "h1hA_J0B7kNi"
   },
   "outputs": [],
   "source": [
    "# Logs the image corresponding to the model prediction\n",
    "def index_to_example(index):\n",
    "    image_array = x_test[index].reshape(28, 28)\n",
    "    image_name = \"confusion-matrix-%05d.png\" % index\n",
    "    results = experiment.log_image(image_array, name=image_name)\n",
    "    # Return sample, assetId (index is added automatically)\n",
    "    return {\"sample\": image_name, \"assetId\": results[\"imageId\"]}\n",
    "\n",
    "\n",
    "LABELS = [f\"class_{i}\" for i in range(10)]\n",
    "\n",
    "experiment.log_confusion_matrix(\n",
    "    y_test[:100],\n",
    "    predictions[:100],\n",
    "    labels=LABELS,\n",
    "    index_to_example_function=index_to_example,\n",
    "    title=\"Confusion Matrix: Evaluation\",\n",
    "    file_name=\"confusion-matrix-eval.json\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gGKVXqyDy3eA"
   },
   "source": [
    "# Ending an Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z_5zd8Pby2-b"
   },
   "outputs": [],
   "source": [
    "experiment.end()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}