{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "N5gWCzvxL7Fo"
},
"source": [
"# EMOTION ANALYSIS"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "qecot2oyL7Fp"
},
"source": [
"
\n",
" \n",
" Run on Google Colab\n",
" | \n",
" \n",
" View source on GitHub\n",
" | \n",
" \n",
" Download notebook\n",
" | \n",
"
"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Xu7It58mL7Fq"
},
"source": [
"### Connect to EvaDB"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "c897YX__L7Fq",
"outputId": "f687c84c-5ca1-49b7-9343-b76a9dedcf30",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
" Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
" Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.9/108.9 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.6/137.6 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.4/139.4 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.7/7.7 MB\u001b[0m \u001b[31m21.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.6/17.6 MB\u001b[0m \u001b[31m62.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.6/13.6 MB\u001b[0m \u001b[31m71.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m631.1/631.1 kB\u001b[0m \u001b[31m42.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m76.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m58.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m70.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m79.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.8/71.8 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.7/98.7 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.0/302.0 kB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m58.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.2/3.2 MB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m90.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m24.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Building wheel for evadb (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"Downloading: \"http://ml.cs.tsinghua.edu.cn/~chenxi/pytorch-models/mnist-b07bb66b.pth\" to /root/.cache/torch/hub/checkpoints/mnist-b07bb66b.pth\n",
"100%|██████████| 1.03M/1.03M [00:01<00:00, 1.02MB/s]\n",
"Downloading: \"https://download.pytorch.org/models/fasterrcnn_resnet50_fpn_coco-258fb6c6.pth\" to /root/.cache/torch/hub/checkpoints/fasterrcnn_resnet50_fpn_coco-258fb6c6.pth\n",
"Downloading https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8m.pt to 'yolov8m.pt'...\n",
"100%|██████████| 49.7M/49.7M [00:00<00:00, 246MB/s]\n"
]
}
],
"source": [
"%pip install --quiet \"evadb[vision,notebook]\"\n",
"%pip install --quiet facenet_pytorch\n",
"import evadb\n",
"cursor = evadb.connect().cursor()\n",
"import warnings\n",
"warnings.filterwarnings(\"ignore\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "GFCfCX-mL7Fr"
},
"source": [
"### Download Video"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "Hi5IpPG4L7Fr",
"outputId": "049b9744-d551-4252-a2ac-c38485ab5377",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"--2023-10-11 05:24:07-- https://www.dropbox.com/s/gzfhwmib7u804zy/defhappy.mp4?raw=1\n",
"Resolving www.dropbox.com (www.dropbox.com)... 162.125.1.18, 2620:100:6016:18::a27d:112\n",
"Connecting to www.dropbox.com (www.dropbox.com)|162.125.1.18|:443... connected.\n",
"HTTP request sent, awaiting response... 302 Found\n",
"Location: /s/raw/gzfhwmib7u804zy/defhappy.mp4 [following]\n",
"--2023-10-11 05:24:07-- https://www.dropbox.com/s/raw/gzfhwmib7u804zy/defhappy.mp4\n",
"Reusing existing connection to www.dropbox.com:443.\n",
"HTTP request sent, awaiting response... 302 Found\n",
"Location: https://uc587966f383aa8250a80d1166f3.dl.dropboxusercontent.com/cd/0/inline/CFYdqNDefUE7BcgFctWTsqQutQc3aYOIH9rzKgzy-rrRYh46q_ZR8RJh1sE19Wt7mgc_ZZ6T_75C9xZ-JD4ON6kJ76L3sn1nblZ-laP74frLJjET21eOhVb3o-QkuOTPM8IitZ2CXP0xH7kcYjIDJVym/file# [following]\n",
"--2023-10-11 05:24:07-- https://uc587966f383aa8250a80d1166f3.dl.dropboxusercontent.com/cd/0/inline/CFYdqNDefUE7BcgFctWTsqQutQc3aYOIH9rzKgzy-rrRYh46q_ZR8RJh1sE19Wt7mgc_ZZ6T_75C9xZ-JD4ON6kJ76L3sn1nblZ-laP74frLJjET21eOhVb3o-QkuOTPM8IitZ2CXP0xH7kcYjIDJVym/file\n",
"Resolving uc587966f383aa8250a80d1166f3.dl.dropboxusercontent.com (uc587966f383aa8250a80d1166f3.dl.dropboxusercontent.com)... 162.125.1.15, 2620:100:6019:15::a27d:40f\n",
"Connecting to uc587966f383aa8250a80d1166f3.dl.dropboxusercontent.com (uc587966f383aa8250a80d1166f3.dl.dropboxusercontent.com)|162.125.1.15|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 2699034 (2.6M) [video/mp4]\n",
"Saving to: ‘defhappy.mp4’\n",
"\n",
"defhappy.mp4 100%[===================>] 2.57M --.-KB/s in 0.06s \n",
"\n",
"2023-10-11 05:24:08 (44.1 MB/s) - ‘defhappy.mp4’ saved [2699034/2699034]\n",
"\n",
"--2023-10-11 05:24:08-- https://raw.githubusercontent.com/georgia-tech-db/eva/master/evadb/functions/emotion_detector.py\n",
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 6085 (5.9K) [text/plain]\n",
"Saving to: ‘emotion_detector.py’\n",
"\n",
"emotion_detector.py 100%[===================>] 5.94K --.-KB/s in 0s \n",
"\n",
"2023-10-11 05:24:08 (39.1 MB/s) - ‘emotion_detector.py’ saved [6085/6085]\n",
"\n",
"--2023-10-11 05:24:08-- https://raw.githubusercontent.com/georgia-tech-db/eva/master/evadb/functions/face_detector.py\n",
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.111.133, ...\n",
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 2911 (2.8K) [text/plain]\n",
"Saving to: ‘face_detector.py’\n",
"\n",
"face_detector.py 100%[===================>] 2.84K --.-KB/s in 0s \n",
"\n",
"2023-10-11 05:24:08 (24.3 MB/s) - ‘face_detector.py’ saved [2911/2911]\n",
"\n"
]
}
],
"source": [
"# A video of a happy person\n",
"!wget -nc \"https://www.dropbox.com/s/gzfhwmib7u804zy/defhappy.mp4?raw=1\" -O defhappy.mp4\n",
"\n",
"# Adding Emotion detection\n",
"!wget -nc https://raw.githubusercontent.com/georgia-tech-db/eva/master/evadb/functions/emotion_detector.py\n",
"\n",
"# Adding Face Detector\n",
"!wget -nc https://raw.githubusercontent.com/georgia-tech-db/eva/master/evadb/functions/face_detector.py"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Qa7lIIkfL7Fr"
},
"source": [
"### Load video for analysis"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"id": "dL5b9QKuL7Fr",
"outputId": "9f0a2693-2a57-434e-aa1a-bc6dee7f5448",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 80
}
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" 0\n",
"0 Number of loaded VIDEO: 1"
],
"text/html": [
"\n",
" \n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" Number of loaded VIDEO: 1 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"
\n",
"
\n"
]
},
"metadata": {},
"execution_count": 3
}
],
"source": [
"cursor.query(\"DROP TABLE IF EXISTS HAPPY;\").df()\n",
"cursor.query(\"LOAD VIDEO 'defhappy.mp4' INTO HAPPY\").df()"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "MVsGXxEhL7Fr"
},
"source": [
"### Create a function for analyzing the frames"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"id": "azJnmlvtL7Fs",
"outputId": "4475ec7b-6fa5-45db-dc3c-b0b3df5880f6",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 98
}
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"100%|██████████| 76.5M/76.5M [00:01<00:00, 64.1MB/s]\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" 0\n",
"0 Function FaceDetector already exists, nothing ..."
],
"text/html": [
"\n",
" \n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" Function FaceDetector already exists, nothing ... | \n",
"
\n",
" \n",
"
\n",
"
\n",
"
\n",
"
\n"
]
},
"metadata": {},
"execution_count": 4
}
],
"source": [
"cursor.query(\"\"\"\n",
" CREATE FUNCTION IF NOT EXISTS EmotionDetector\n",
" INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))\n",
" OUTPUT (labels NDARRAY STR(ANYDIM), scores NDARRAY FLOAT32(ANYDIM))\n",
" TYPE Classification IMPL 'emotion_detector.py';\n",
"\"\"\").df()\n",
"\n",
"cursor.query(\"\"\"\n",
" CREATE FUNCTION IF NOT EXISTS FaceDetector\n",
" INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))\n",
" OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4),\n",
" scores NDARRAY FLOAT32(ANYDIM))\n",
" TYPE FaceDetection\n",
" IMPL 'face_detector.py';\n",
"\"\"\").df()"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7f8bLJV-L7Fs"
},
"source": [
"### Run the Face Detection UDF on video"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"id": "IyhT1CPbL7Fs",
"outputId": "f14f6c42-ed4b-46b1-a259-2edb49f7871f",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 362
}
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" id bboxes scores\n",
"0 0 [[502, 94, 762, 435], [238, 296, 325, 398]] [0.99990165, 0.79820216]\n",
"1 1 [[501, 96, 763, 435]] [0.999918]\n",
"2 2 [[504, 97, 766, 437]] [0.9999138]\n",
"3 3 [[498, 90, 776, 446]] [0.99996686]\n",
"4 4 [[496, 99, 767, 444]] [0.9999982]\n",
"5 5 [[499, 87, 777, 448], [236, 305, 324, 407]] [0.9999136, 0.83697325]\n",
"6 6 [[500, 89, 778, 449]] [0.9999131]\n",
"7 7 [[501, 89, 781, 452]] [0.9999124]\n",
"8 8 [[503, 90, 783, 450]] [0.99994683]\n",
"9 9 [[508, 87, 786, 447]] [0.999949]"
],
"text/html": [
"\n",
" \n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" id | \n",
" bboxes | \n",
" scores | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 0 | \n",
" [[502, 94, 762, 435], [238, 296, 325, 398]] | \n",
" [0.99990165, 0.79820216] | \n",
"
\n",
" \n",
" 1 | \n",
" 1 | \n",
" [[501, 96, 763, 435]] | \n",
" [0.999918] | \n",
"
\n",
" \n",
" 2 | \n",
" 2 | \n",
" [[504, 97, 766, 437]] | \n",
" [0.9999138] | \n",
"
\n",
" \n",
" 3 | \n",
" 3 | \n",
" [[498, 90, 776, 446]] | \n",
" [0.99996686] | \n",
"
\n",
" \n",
" 4 | \n",
" 4 | \n",
" [[496, 99, 767, 444]] | \n",
" [0.9999982] | \n",
"
\n",
" \n",
" 5 | \n",
" 5 | \n",
" [[499, 87, 777, 448], [236, 305, 324, 407]] | \n",
" [0.9999136, 0.83697325] | \n",
"
\n",
" \n",
" 6 | \n",
" 6 | \n",
" [[500, 89, 778, 449]] | \n",
" [0.9999131] | \n",
"
\n",
" \n",
" 7 | \n",
" 7 | \n",
" [[501, 89, 781, 452]] | \n",
" [0.9999124] | \n",
"
\n",
" \n",
" 8 | \n",
" 8 | \n",
" [[503, 90, 783, 450]] | \n",
" [0.99994683] | \n",
"
\n",
" \n",
" 9 | \n",
" 9 | \n",
" [[508, 87, 786, 447]] | \n",
" [0.999949] | \n",
"
\n",
" \n",
"
\n",
"
\n",
"
\n",
"
\n"
]
},
"metadata": {},
"execution_count": 5
}
],
"source": [
"cursor.query(\"SELECT id, FaceDetector(data) FROM HAPPY WHERE id < 10\").df()"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "QRX-TvuTL7Fs"
},
"source": [
"### Run the Emotion Detection UDF on the outputs of the Face Detection UDF"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"id": "t0E69JNIL7Fs",
"outputId": "5551bac6-9ac8-4e72-c78f-dd98582a04ba",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 613
}
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" id bbox labels scores\n",
"0 0 [502, 94, 762, 435] happy 0.999642\n",
"1 0 [238, 296, 325, 398] neutral 0.780949\n",
"2 1 [501, 96, 763, 435] happy 0.999644\n",
"3 2 [504, 97, 766, 437] happy 0.999668\n",
"4 3 [498, 90, 776, 446] happy 0.999654\n",
"5 4 [496, 99, 767, 444] happy 0.999649\n",
"6 5 [499, 87, 777, 448] happy 0.999710\n",
"7 5 [236, 305, 324, 407] neutral 0.760779\n",
"8 6 [500, 89, 778, 449] happy 0.999671\n",
"9 7 [501, 89, 781, 452] happy 0.999671\n",
"10 8 [503, 90, 783, 450] happy 0.999689\n",
"11 9 [508, 87, 786, 447] happy 0.999691\n",
"12 10 [505, 86, 788, 452] happy 0.999729\n",
"13 10 [235, 309, 322, 411] neutral 0.407872\n",
"14 11 [514, 85, 790, 454] happy 0.999745\n",
"15 12 [514, 86, 790, 454] happy 0.999729\n",
"16 13 [515, 87, 790, 454] happy 0.999718\n",
"17 14 [516, 86, 792, 455] happy 0.999739"
],
"text/html": [
"\n",
" \n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" id | \n",
" bbox | \n",
" labels | \n",
" scores | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 0 | \n",
" [502, 94, 762, 435] | \n",
" happy | \n",
" 0.999642 | \n",
"
\n",
" \n",
" 1 | \n",
" 0 | \n",
" [238, 296, 325, 398] | \n",
" neutral | \n",
" 0.780949 | \n",
"
\n",
" \n",
" 2 | \n",
" 1 | \n",
" [501, 96, 763, 435] | \n",
" happy | \n",
" 0.999644 | \n",
"
\n",
" \n",
" 3 | \n",
" 2 | \n",
" [504, 97, 766, 437] | \n",
" happy | \n",
" 0.999668 | \n",
"
\n",
" \n",
" 4 | \n",
" 3 | \n",
" [498, 90, 776, 446] | \n",
" happy | \n",
" 0.999654 | \n",
"
\n",
" \n",
" 5 | \n",
" 4 | \n",
" [496, 99, 767, 444] | \n",
" happy | \n",
" 0.999649 | \n",
"
\n",
" \n",
" 6 | \n",
" 5 | \n",
" [499, 87, 777, 448] | \n",
" happy | \n",
" 0.999710 | \n",
"
\n",
" \n",
" 7 | \n",
" 5 | \n",
" [236, 305, 324, 407] | \n",
" neutral | \n",
" 0.760779 | \n",
"
\n",
" \n",
" 8 | \n",
" 6 | \n",
" [500, 89, 778, 449] | \n",
" happy | \n",
" 0.999671 | \n",
"
\n",
" \n",
" 9 | \n",
" 7 | \n",
" [501, 89, 781, 452] | \n",
" happy | \n",
" 0.999671 | \n",
"
\n",
" \n",
" 10 | \n",
" 8 | \n",
" [503, 90, 783, 450] | \n",
" happy | \n",
" 0.999689 | \n",
"
\n",
" \n",
" 11 | \n",
" 9 | \n",
" [508, 87, 786, 447] | \n",
" happy | \n",
" 0.999691 | \n",
"
\n",
" \n",
" 12 | \n",
" 10 | \n",
" [505, 86, 788, 452] | \n",
" happy | \n",
" 0.999729 | \n",
"
\n",
" \n",
" 13 | \n",
" 10 | \n",
" [235, 309, 322, 411] | \n",
" neutral | \n",
" 0.407872 | \n",
"
\n",
" \n",
" 14 | \n",
" 11 | \n",
" [514, 85, 790, 454] | \n",
" happy | \n",
" 0.999745 | \n",
"
\n",
" \n",
" 15 | \n",
" 12 | \n",
" [514, 86, 790, 454] | \n",
" happy | \n",
" 0.999729 | \n",
"
\n",
" \n",
" 16 | \n",
" 13 | \n",
" [515, 87, 790, 454] | \n",
" happy | \n",
" 0.999718 | \n",
"
\n",
" \n",
" 17 | \n",
" 14 | \n",
" [516, 86, 792, 455] | \n",
" happy | \n",
" 0.999739 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"
\n",
"
\n"
]
},
"metadata": {},
"execution_count": 7
}
],
"source": [
"query = cursor.query(\"\"\"\n",
" SELECT id, bbox, EmotionDetector(Crop(data, bbox))\n",
" FROM HAPPY JOIN LATERAL UNNEST(FaceDetector(data)) AS Face(bbox, conf)\n",
" WHERE id < 15\n",
"\"\"\")\n",
"response = query.df()\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"id": "dm_QzeEEL7Ft"
},
"outputs": [],
"source": [
"import cv2\n",
"from pprint import pprint\n",
"from matplotlib import pyplot as plt\n",
"\n",
"def annotate_video(detections, input_video_path, output_video_path):\n",
" color1=(207, 248, 64)\n",
" color2=(255, 49, 49)\n",
" thickness=4\n",
"\n",
" vcap = cv2.VideoCapture(input_video_path)\n",
" width = int(vcap.get(3))\n",
" height = int(vcap.get(4))\n",
" fps = vcap.get(5)\n",
" fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') #codec\n",
" video=cv2.VideoWriter(output_video_path, fourcc, fps, (width,height))\n",
"\n",
" frame_id = 0\n",
" # Capture frame-by-frame\n",
" # ret = 1 if the video is captured; frame is the image\n",
" ret, frame = vcap.read()\n",
"\n",
" while ret:\n",
" df = detections\n",
" df = df[['bbox', 'labels', 'scores']][df.index == frame_id]\n",
" if df.size:\n",
"\n",
" x1, y1, x2, y2 = df['bbox'].values[0]\n",
" label = df['labels'].values[0]\n",
" score = df['scores'].values[0]\n",
" x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)\n",
" # object bbox\n",
" frame=cv2.rectangle(frame, (x1, y1), (x2, y2), color1, thickness)\n",
" # object label\n",
" cv2.putText(frame, label, (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, color1, thickness)\n",
" # object score\n",
" cv2.putText(frame, str(round(score, 5)), (x1+120, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, color1, thickness)\n",
" # frame label\n",
" cv2.putText(frame, 'Frame ID: ' + str(frame_id), (700, 500), cv2.FONT_HERSHEY_SIMPLEX, 1.2, color2, thickness)\n",
"\n",
" video.write(frame)\n",
" # Show every fifth frame\n",
" if frame_id % 5 == 0:\n",
" plt.imshow(frame)\n",
" plt.show()\n",
"\n",
" frame_id+=1\n",
" ret, frame = vcap.read()\n",
"\n",
" video.release()\n",
" vcap.release()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"id": "_ltdFviXL7Ft",
"outputId": "04e3aa01-b5be-44ed-91b4-95e518c33ae5",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"