{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "

Sky-Zoo

" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Getting the libs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!pip install opencv-python imutils" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Support Libs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from IPython.display import Image, display\n", "import os , random , json , requests\n", "import types\n", "import pandas as pd" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Libs for DeepLearning models\n", "ATTENTION - Do not use Keras from Tensorflow due incompatible with WML" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf\n", "import keras\n", "from keras.models import Sequential\n", "from keras.layers import Dense, Activation, Dropout, Flatten\n", "from keras.layers import Conv2D\n", "from keras.layers import MaxPooling2D\n", "from keras.layers import InputLayer\n", "from keras.preprocessing import image as Kimage\n", "from keras.preprocessing.image import img_to_array\n", "from sklearn.preprocessing import LabelBinarizer\n", "from sklearn.model_selection import train_test_split\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from imutils import paths" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Libs for getting the data from Object Storage and IBM Watson SDK" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from ibm_botocore.client import Config\n", "import ibm_boto3\n", "from watson_machine_learning_client import WatsonMachineLearningAPIClient" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Download do Dataset(imagens)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "## Area resevada para iserir as credenciais do dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "client = ibm_boto3.client(service_name='s3',\n", " ibm_api_key_id=credentials_1['IBM_API_KEY_ID'],\n", " ibm_auth_endpoint=credentials_1['IBM_AUTH_ENDPOINT'],\n", " config=Config(signature_version='oauth'),\n", " endpoint_url=credentials_1['ENDPOINT'])\n", "\n", "with open('dataset.zip', 'wb') as data:\n", " client.download_fileobj(credentials_1['BUCKET'], 'DATASET.zip', data)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Unzip the dataset and adequate the folders" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!rm -rf DATASET\n", "!unzip dataset.zip\n", "!mkdir DATASET\n", "!mv Elephant Giraffe Lion DATASET\n", "!ls" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Model definition" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "random.seed(1)\n", "model = Sequential()\n", "model.add(Conv2D(16, (3,3), input_shape=(96,96,3), padding='same', activation='relu'))\n", "model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n", "model.add(Conv2D(16, (3,3), padding='same', activation='relu'))\n", "model.add(Conv2D(32, (3,3), padding='same', activation='relu'))\n", "model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n", "model.add(Flatten())\n", "model.add(Dense(128, activation='relu'))\n", "model.add(Dense(3, activation='sigmoid'))\n", "\n", "model.compile(loss='binary_crossentropy',\n", " optimizer='adam',\n", " metrics=['accuracy'])\n", "\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Load das imagens, já com pre-processing, para utilizar no Train do modelo" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "imagePaths = sorted(list(paths.list_images('DATASET/')))\n", "\n", "random.seed(1)\n", "random.shuffle(imagePaths)\n", "\n", "data = []\n", "labels = []\n", "\n", "for imagePath in imagePaths:\n", " image = Kimage.load_img(imagePath,target_size=(96,96))\n", " image = img_to_array(image)\n", " data.append(image)\n", " label = imagePath.split(os.path.sep)[-2]\n", " labels.append(label)\n", " print(label)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Split do dataset em Train e Test" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "classes = ['Elephant', 'Giraffe', 'Lion']\n", "lb = LabelBinarizer()\n", "labels = lb.fit_transform(labels)\n", "data = np.array(data, dtype=\"float\") / 255.0\n", "\n", "(trainX, testX, trainY, testY) = train_test_split(data, labels, test_size=0.2, random_state=1)\n", "\n", "print(\"shape de X de treino :\",trainX.shape)\n", "print(\"shape de Y de treino :\",trainY.shape)\n", "print(\"shape de X de teste :\",testX.shape)\n", "print(\"shape de Y de teste :\",testY.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Treinamento do modelo - utilizando DataGenerator" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "aug = ImageDataGenerator(\n", " rotation_range=20,\n", " zoom_range=0.15,\n", " width_shift_range=0.2,\n", " height_shift_range=0.2,\n", " shear_range=0.15,\n", " horizontal_flip=True,\n", " fill_mode=\"nearest\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "random.seed(1)\n", "model.fit_generator(\n", " aug.flow(trainX, trainY, batch_size=16),\n", " validation_data=(testX, testY),\n", " steps_per_epoch=len(trainX) // 16,\n", " epochs=40)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Graphs for model valuation" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "H=model.history\n", "plt.rcParams['figure.figsize'] = 16, 8\n", "plt.figure()\n", "N = H.epoch[-1]+1\n", "plt.plot(np.arange(0, N), H.history[\"loss\"], label=\"train_loss\")\n", "plt.plot(np.arange(0, N), H.history[\"val_loss\"], label=\"val_loss\")\n", "plt.plot(np.arange(0, N), H.history[\"acc\"], label=\"train_acc\")\n", "plt.plot(np.arange(0, N), H.history[\"val_acc\"], label=\"val_acc\")\n", "plt.title(\"Training Loss and Accuracy\")\n", "plt.xlabel(\"Epoch #\")\n", "plt.ylabel(\"Loss/Accuracy\")\n", "plt.legend(loc=\"upper left\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Saving the models as model.h5" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.save(\"model_WSTUDIO.h5\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Upload the model to IBM Watson Machine Learning with Watson SDK\n", "The mdoel must be in .tar.gz format to upload to WML" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!tar -zcvf model_WSTUDIO.tar.gz model_WSTUDIO.h5" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Watson Machine Learning Credentials" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "wml_credentials = {\n", " \"Your WML\": \"Credentials\"\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Model Deployment" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "client = WatsonMachineLearningAPIClient( wml_credentials )\n", "\n", "sample_saved_model_filename = 'model_WSTUDIO.tar.gz'\n", "metadata = {\n", " client.repository.ModelMetaNames.NAME : 'TheZoo',\n", " client.repository.ModelMetaNames.FRAMEWORK_NAME : 'tensorflow',\n", " client.repository.ModelMetaNames.FRAMEWORK_VERSION : '1.13',\n", " client.repository.ModelMetaNames.RUNTIME_NAME : 'python',\n", " client.repository.ModelMetaNames.RUNTIME_VERSION : '3.6',\n", " client.repository.ModelMetaNames.FRAMEWORK_LIBRARIES : [{\"name\": \"keras\", \"version\": \"2.2.4\"}]\n", "}\n", "\n", "\n", "# Connection WML\n", "model_details = client.repository.store_model( sample_saved_model_filename, meta_props=metadata, training_data=None )\n", "\n", "# Deploy\n", "model_id = model_details[\"metadata\"][\"guid\"]\n", "model_deployment_details = client.deployments.create( artifact_uid=model_id, name=\"TheZoo\" )\n", "\n", "# Retrieve URL for API request\n", "model_endpoint_url = client.deployments.get_scoring_url( model_deployment_details )\n", "print(\"The URL of your API is: \",model_endpoint_url)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test API request" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Montagem da estrutura de JSON para chamada da API do WML\n", "ai_parms = { \"wml_credentials\" : wml_credentials, \"model_endpoint_url\" : model_endpoint_url }\n", "\n", "image = Kimage.load_img(\"./DATASET/IMAGEM VALIDA\")\n", "plt.imshow(image)\n", "image = image.resize(size=(96,96))\n", "image = img_to_array(image)\n", "image = np.array(image, dtype=\"float\") / 255.0\n", "image = np.expand_dims(image,axis=0)\n", "image = image.tolist()\n", "\n", "# Chamada da função SCORE no modelo (inference)\n", "model_payload = { \"values\" : image }\n", "model_result = client.deployments.score( ai_parms[\"model_endpoint_url\"], model_payload )\n", "print(model_result)\n", "\n", "print(\"\\nImage Classified as : \", classes[model_result['values'][0][1]])\n", "\n", "print(\"\\nConfident : \\n\\t\",\n", " classes[model_result['values'][0][1]],\" : %.2f\" %(model_result['values'][0][0][0]*100),\"%\\n\\t\",\n", " )" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 }