{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Identification evaluation metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from pyannote.core import Annotation, Segment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHQAAACsCAYAAAAaLvvnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADPxJREFUeJzt3X+sJWdZB/DvQxeksSCWbQrZ1qxWsDaVrnYhCzSmbVCq\nJW2JxNCg6R8kQIKKiaJgYkpNiJqopX8YDQIWE7UQK6URYktKBUwsdBe2LQWrFEvqpnRFIdKIFdrH\nP86U3u6v3rN7t3Pfez+fZHNm5s7sPmf32fec8z0z71R3BwAAAIBxPG3uAgAAAABYjkAHAAAAYDAC\nHQAAAIDBCHQAAAAABiPQAQAAABiMQAcAAABgMAIdAAAAgMEIdAAAAAAGI9ABAAAAGIxABwAAAGAw\nQwY6VXVZVXVVnTl3LWupqh6pqr1VdUdVfbaqXjZ3TWulqp5XVddV1b1VtaeqPlpVL5y7LgAAABjR\nkIFOksuT/OP0uJF8q7t3dPc5Sd6e5HfnLmgtVFUl+VCSf+juM7r73Cye36nzVgYAAABjGi7QqaqT\nkpyX5PVJXjtzOcfTs5N8fe4i1sgFSb7d3X/62IbuvqO7PzVjTQAAADCsLXMXcBQuTfL33f0vVfWf\nVXVud++Zu6g1cmJV7U3yzCTPT3LhzPWslbOTbJR/IwAAAJjdMQU6+7ad3mtVSJJs23d/rWK3y5Nc\nMy1fN62veViw68qb1vS53XbVK1fz3L7V3TuSpKpemuQvqurs7l6zWi654eI1fV43XvaR1TwvAAAA\nYA0NdYZOVZ2cxVkrP1ZVneSEJF1Vb13L0GM96O5/qqqtSU5Jsn/ueo7R3UleM3cRAAAAsFHUSDlI\nVb0hybnd/cYV2z6R5Le7+5PzVbY2quqh7j5pWj4zi4mfT+3uR+at7NhMkyLfluS93f3uaduLknyf\neXQAAABgeaNNinx5FndLWun6bJy7XZ043bZ8b5IPJLli9DAnSaazp16d5BXTbcvvzuIOXl+dtzIA\nAAAY01Bn6AAAAAAw3hk6AAAAAJueQAcAAABgMAIdAAAAgMEIdAAAAAAGI9ABAAAAGMyWZXbeunVr\nb9++/TiVAgAAALD57Nmz52vdfcoyxywV6Gzfvj27d+9erioAAAAADquqvrLsMS65AgAAABiMQAcA\nAABgMAIdAAAAgMEIdAAAAAAGI9ABAAAAGIxABwAAAGAwAh0AAACAwQh0AAAAAAYj0AEAAAAYjEAH\nAAAAYDACHQAAAIDBCHQAAAAABiPQAQAAABiMQAcAAABgMAIdAAAAgMEIdAAAAAAGI9ABAAAAGIxA\nBwAAAGAwAh0AAACAwQh0AAAAAAYj0AEAAAAYjEAHAAAAYDACHQAAAIDBCHQAAAAABiPQAQAAABiM\nQAcAAABgMAIdAAAAgMEIdAAAAAAGI9ABAAAAGIxABwAAAGAwAh0AAACAwQh0AAAAAAYj0AEAAAAY\njEAHAAAAYDACHQAAAIDBCHQAAAAABiPQAQAAABiMQAcAAABgMAIdAAAAgMEIdAAAAAAGI9ABAAAA\nGIxABwAAAGAwAh0AAACAwQh0AAAAAAYj0AEAAAAYjEAHAAAAYDACHQAAAIDBLBXofPvOO49XHcAm\ntW/b6dm37fS5yzhudl15U3ZdedPcZcCm4//euDb668JTzd/n2rrkhotzyQ0Xz10GsMEc7bjiDB0A\nAACAwQh0AAAAAAYj0AEAAAAYzJZlD3ANLsDyzOUBsBzvOVnPzKMDrAfO0AEAAAAYzNJn6Gzbd//x\nqAPYpDbLN7C3XfXKuUuATcVZcePznnNtbJbX2afajZd9ZO4SgA3EXa4AAAAANgmBDgAAAMBgBDoA\nAAAAgxHoAAAAAAymunvVO+/cubN37959HMsBAAAA2Fyqak9371zmGGfoAAAAAAxGoAMAAAAwGIEO\nAAAAwGAEOgAAAACDEegAAAAADEagAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAMRqADAAAAMBiB\nDgAAAMBgBDoAAAAAgxHoAAAAAAxGoAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAADEagAwAAADAY\ngQ4AAADAYAQ6AAAAAIMR6AAAAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxGoAMAAAAw\nGIEOAAAAwGAEOgAAAACDEegAAAAADEagAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAMRqADAAAA\nMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxGoAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAADEagAwAA\nADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAg6nuXv3OVd9Mcs/x\nK4cNZGuSr81dBMPQL6yWXmEZ+oXV0issQ7+wWnqFZfxIdz9rmQO2LPkH3NPdO5c8hk2oqnbrFVZL\nv7BaeoVl6BdWS6+wDP3CaukVllFVu5c9xiVXAAAAAIMR6AAAAAAMZtlA593HpQo2Ir3CMvQLq6VX\nWIZ+YbX0CsvQL6yWXmEZS/fLUpMiAwAAADA/l1wBAAAADGZVgU5VXVRV91TVl6rqbce7KMZWVfdV\n1V1VtfdoZupmY6uq91XV/qr6/IptJ1fVx6rqX6fH75+zRtaHw/TKO6pq3zS+7K2qn52zRtaHqjq9\nqm6tqi9U1d1V9ZZpu7GFgxyhX4wvPEFVPbOqPlNVd0y9ctW0/Qer6tPTZ6MPVNUz5q6V+R2hX66t\nqn9bMbbsmLtW1oeqOqGqPldVfzetLz22PGmgU1UnJPnjJD+T5Kwkl1fVWcdaPBveBd29w236OIRr\nk1x0wLa3Jbmlu1+Q5JZpHa7Nwb2SJFdP48uO7v7oU1wT69N3kvxad5+VZFeSN0/vVYwtHMrh+iUx\nvvBEDye5sLvPSbIjyUVVtSvJ72fRKz+c5OtJXj9jjawfh+uXJHnrirFl73wlss68JckXV6wvPbas\n5gydlyT5Und/ubv/L8l1SS49imIB0t2fTPJfB2y+NMn7p+X3J7nsKS2KdekwvQIH6e4Huvuz0/I3\ns3hztC3GFg7hCP0CT9ALD02rT59+dZILk/zNtN3YQpIj9gscpKpOS3JxkvdM65WjGFtWE+hsS3L/\nivV/jxc9jqyT3FxVe6rqDXMXwxBO7e4HpuWvJjl1zmJY936pqu6cLslyCQ1PUFXbk/x4kk/H2MKT\nOKBfEuMLB5guidibZH+SjyW5N8k3uvs70y4+G/FdB/ZLdz82trxzGluurqrvmbFE1o93JfmNJI9O\n68/NUYwtJkXmeDivu38ii8v03lxVPzl3QYyjF7fe820Gh/MnSc7I4lTmB5L84bzlsJ5U1UlJrk/y\nq9393yt/ZmzhQIfoF+MLB+nuR7p7R5LTsrhy4cyZS2IdO7BfqursJG/Pom9enOTkJL85Y4msA1X1\nqiT7u3vPsf5eqwl09iU5fcX6adM2OKTu3jc97k/yoSxe/OBIHqyq5yfJ9Lh/5npYp7r7wenN0qNJ\n/izGFyZV9fQsPpz/ZXf/7bTZ2MIhHapfjC8cSXd/I8mtSV6a5DlVtWX6kc9GHGRFv1w0XebZ3f1w\nkj+PsYXk5Ukuqar7spjS5sIk1+QoxpbVBDq3J3nBNOPyM5K8NsmNR1M1G19VfW9VPeux5SQ/neTz\nRz4KcmOSK6blK5J8eMZaWMce+3A+eXWML+S7152/N8kXu/uPVvzI2MJBDtcvxhcOVFWnVNVzpuUT\nk/xUFnMu3ZrkNdNuxhaSHLZf/nnFFwuVxZwoxpZNrrvf3t2ndff2LPKVj3f363IUY0stzkB+kp0W\nt218V5ITkryvu995lLWzwVXVD2VxVk6SbEnyV/qFlarqr5Ocn2RrkgeTXJnkhiQfTPIDSb6S5Oe7\n22S4m9xheuX8LC6H6CT3JXnjijlS2KSq6rwkn0pyVx6/Fv23spgXxdjCExyhXy6P8YUVqupFWUxM\nekIWX4R/sLt/Z3q/e10Wl898LskvTGdfsIkdoV8+nuSUJJVkb5I3rZg8mU2uqs5P8uvd/aqjGVtW\nFegAAAAAsH6YFBkAAABgMAIdAAAAgMEIdAAAAAAGI9ABAAAAGIxABwAAAGAwW+YuAADgyVTVc5Pc\nMq0+L8kjSf5jWv+f7n7ZLIUBAMzEbcsBgKFU1TuSPNTdfzB3LQAAc3HJFQAwtKp6aHo8v6o+UVUf\nrqovV9XvVdXrquozVXVXVZ0x7XdKVV1fVbdPv14+7zMAAFieQAcA2EjOSfKmJD+a5BeTvLC7X5Lk\nPUl+edrnmiRXd/eLk/zc9DMAgKGYQwcA2Ehu7+4HkqSq7k1y87T9riQXTMuvSHJWVT12zLOr6qTu\nfugprRQA4BgIdACAjeThFcuPrlh/NI+/73lakl3d/b9PZWEAAGvJJVcAwGZzcx6//CpVtWPGWgAA\njopABwDYbH4lyc6qurOqvpDFnDsAAENx23IAAACAwThDBwAAAGAwAh0AAACAwQh0AAAAAAYj0AEA\nAAAYjEAHAAAAYDACHQAAAIDBCHQAAAAABiPQAQAAABjM/wNLTDmmMc0M8QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<pyannote.core.annotation.Annotation at 0x114c450f0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reference = Annotation()\n",
    "reference[Segment(0, 10)] = 'A'\n",
    "reference[Segment(12, 20)] = 'B'\n",
    "reference[Segment(24, 27)] = 'A'\n",
    "reference[Segment(30, 40)] = 'C'\n",
    "reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHQAAACsCAYAAAAaLvvnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADY5JREFUeJzt3W2MpWdZB/D/RQtCLIjQ5cVtzQqCtUFYZSHlJVoasdWS\ntkRiaMDwgaQlAYUEkZfEFEyIkigvH4wWedWChYi2jRCBAAFUCuzClvIiCliCm9oVhUijoLSXH85T\nOp196ZzZ2Xnmnvn9ks08zzPn7Fwn59p7z/nPfd+nujsAAAAAjOMecxcAAAAAwHIEOgAAAACDEegA\nAAAADEagAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAgxky\n0KmqS6qqq+qsuWvZSFV1W1UdrKobquozVfXEuWvaKFX1kKq6uqq+WlUHqup9VfXIues6USuesy9M\nz9uLq2rIf1cAAACMo7p77hqWVlXvSvJjST7c3VfMXc9Gqapbu/u06fj8JK/o7l+YuawTVlWV5B+S\nvL27/2S69pgk9+vuj89a3Ala9Zw9KMk7k/z9dupLAAAAtp7hZhJU1WlJnpzkuUmeOXM5J9P9knxr\n7iI2yFOS/N8dYU6SdPcNo4c5q3X34SSXJXnBFGIBAADASXHq3AWsw8VJ/ra7/6mq/qOqHtvdB+Yu\naoPcp6oOJrl3kocmOW/mejbKo5Jsl+fouLr7a1V1SpIHJbll7noAAADYnk4o0Dm0+8wNXa+1+9A3\n1jKr4dIkb5iOr57ONzwsOOeK92/oY7v+Veev5bH9T3fvTZKqekKSP6uqR/UGrou76JoLN/RxXXfJ\ne7fMTJQrL75qQx/b5dc+e8s8NgAAAFhpqBk6VfWALGat/ExVdZJTknRVvWQjQ4+toLs/UVWnJ9mV\n5PDc9ZygLyR5xtxFbIaqeliS2zL+cwYAAMAWNtSmyFV1WZLHdvflK659NMnvdPfH5qtsY6zaYPes\nJH+X5MHdfdu8lZ2YaT+Z65O8ubvfOF17dJIfGX0fnVXP2a4k70jyCZsiAwAAcDINNUMni+VVr1l1\n7T3T9eEDndy5h06SVJLnjB7mJEl3d1U9Pcnrq+qlSb6b5KYkL5q1sI1xx3N2zyTfT/LnSV47b0kA\nAABsd0PN0AEAAABgwI8tBwAAANjpBDoAAAAAgxHoAAAAAAxGoAMAAAAwGIEOAAAAwGCW+tjy008/\nvffs2XOSSgEAAADYeQ4cOPDN7t61zH2WCnT27NmT/fv3L1cVAAAAAMdUVV9f9j6WXAEAAAAMRqAD\nAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxGoAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAADEag\nAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxG\noAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAADEagAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAM\nRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxGoAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAA\nDEagAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAA\nAAxGoAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAADEagAwAAADAYgQ4AAADAYAQ6k0O7z8yh3WfO\nXQbAul158VW58uKrNu3nnXPF+3POFe/ftJ8Hy9ju/XnRNRfmomsunLsMANgUm/06d7Ot97EJdAAA\nAAAGI9ABAAAAGIxABwAAAGAwp85dwFZjHx1gWPt+L8n61+Aube+uJNnW+5TAVmcfHQB2ggtzaZJN\nfJ07CDN0AAAAAAZjhs4quw99Y+4SANZn+o3F5dc+e1N+3FunmTnXv+r8Tfl5sIydMnPsukveO3cJ\nAHDSXfnWzX2du9l8yhUAAADADiHQAQAAABiMQAcAAABgMAIdAAAAgMFUd6/5xvv27ev9+/efxHIA\nAAAAdpaqOtDd+5a5jxk6AAAAAIMR6AAAAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxG\noAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAADEagAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAM\nRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxGoAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAA\nDEagAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAAAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAA\nAAxGoAMAAAAwGIEOAAAAwGAEOgAAAACDEegAAAAADEagAwAAADAYgQ4AAADAYAQ6AAAAAIMR6AAA\nAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxGoAMAAAAwGIEOAAAAwGAEOgAAAACDEegA\nAAAADEagAwAAADAYgQ4AAADAYKq7137jqu8k+fLJK4dt5PQk35y7CIahX1grvcIy9AtrpVdYhn5h\nrfQKy/ip7r7vMnc4dckf8OXu3rfkfdiBqmq/XmGt9AtrpVdYhn5hrfQKy9AvrJVeYRlVtX/Z+1hy\nBQAAADAYgQ4AAADAYJYNdN54UqpgO9IrLEO/sFZ6hWXoF9ZKr7AM/cJa6RWWsXS/LLUpMgAAAADz\ns+QKAAAAYDBrCnSq6oKq+nJVfaWqXnayi2JsVXVTVd1YVQfXs1M321tVvaWqDlfV51dce0BVfbCq\n/nn6+qNz1sjWcIxeeWVVHZrGl4NV9Stz1sjWUFVnVtVHquqLVfWFqnrhdN3YwhGO0y/GF+6iqu5d\nVZ+qqhumXnnVdP0nquqT03ujd1XVveaulfkdp1/eVlX/smJs2Tt3rWwNVXVKVX22qv5mOl96bLnb\nQKeqTknyR0l+OcnZSS6tqrNPtHi2vad0914f08dRvC3JBauuvSzJh7r7EUk+NJ3D23JkryTJ66bx\nZW93v2+Ta2Jr+n6SF3f32UnOSfL86bWKsYWjOVa/JMYX7up7Sc7r7sck2Zvkgqo6J8lrsuiVn0zy\nrSTPnbFGto5j9UuSvGTF2HJwvhLZYl6Y5EsrzpceW9YyQ+fxSb7S3V/r7v9NcnWSi9dRLEC6+2NJ\n/nPV5YuTvH06fnuSSza1KLakY/QKHKG7b+7uz0zH38nixdHuGFs4iuP0C9xFL9w6nd5z+tNJzkvy\nl9N1YwtJjtsvcISqOiPJhUneNJ1X1jG2rCXQ2Z3kGyvO/zX+0+P4OskHqupAVV02dzEM4cHdffN0\n/G9JHjxnMWx5L6iqz01Lsiyh4S6qak+Sn03yyRhbuBur+iUxvrDKtCTiYJLDST6Y5KtJvt3d359u\n4r0RP7C6X7r7jrHl1dPY8rqq+qEZS2TreH2S305y+3T+wKxjbLEpMifDk7v757JYpvf8qvr5uQti\nHL346D2/zeBY/jjJw7OYynxzkj+ctxy2kqo6Lcl7kryou/9r5feMLax2lH4xvnCE7r6tu/cmOSOL\nlQtnzVwSW9jqfqmqRyV5eRZ987gkD0jy0hlLZAuoqqclOdzdB07071pLoHMoyZkrzs+YrsFRdfeh\n6evhJH+dxX9+cDy3VNVDk2T6enjmetiiuvuW6cXS7Un+NMYXJlV1zyzenL+ju/9qumxs4aiO1i/G\nF46nu7+d5CNJnpDk/lV16vQt7404wop+uWBa5tnd/b0kb42xheRJSS6qqpuy2NLmvCRvyDrGlrUE\nOp9O8ohpx+V7JXlmkuvWUzXbX1X9cFXd947jJL+U5PPHvxfkuiTPmY6fk+TaGWthC7vjzfnk6TG+\nkB+sO39zki9192tXfMvYwhGO1S/GF1arql1Vdf/p+D5JnprFnksfSfKM6WbGFpIcs1/+ccUvFiqL\nPVGMLTtcd7+8u8/o7j1Z5Csf7u5nZR1jSy1mIN/NjRYf2/j6JKckeUt3v3qdtbPNVdXDspiVkySn\nJnmnfmGlqvqLJOcmOT3JLUmuSHJNkncn+fEkX0/ya91tM9wd7hi9cm4WyyE6yU1JLl+xRwo7VFU9\nOcnHk9yYO9eivyKLfVGMLdzFcfrl0hhfWKGqHp3FxqSnZPGL8Hd39+9Or3evzmL5zGeTPHuafcEO\ndpx++XCSXUkqycEkz1uxeTI7XFWdm+S3uvtp6xlb1hToAAAAALB12BQZAAAAYDACHQAAAIDBCHQA\nAAAABiPQAQAAABiMQAcAAABgMKfOXQAAwN2pqgcm+dB0+pAktyX59+n8v7v7ibMUBgAwEx9bDgAM\npapemeTW7v6DuWsBAJiLJVcAwNCq6tbp67lV9dGquraqvlZVv19Vz6qqT1XVjVX18Ol2u6rqPVX1\n6enPk+Z9BAAAyxPoAADbyWOSPC/JTyf59SSP7O7HJ3lTkt+YbvOGJK/r7scl+dXpewAAQ7GHDgCw\nnXy6u29Okqr6apIPTNdvTPKU6fgXk5xdVXfc535VdVp337qplQIAnACBDgCwnXxvxfHtK85vz52v\ne+6R5Jzu/u5mFgYAsJEsuQIAdpoP5M7lV6mqvTPWAgCwLgIdAGCn+c0k+6rqc1X1xSz23AEAGIqP\nLQcAAAAYjBk6AAAAAIMR6AAAAAAMRqADAAAAMBiBDgAAAMBgBDoAAAAAgxHoAAAAAAxGoAMAAAAw\nGIEOAAAAwGD+H/JZbvqSS5q9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<pyannote.core.annotation.Annotation at 0x114c781d0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hypothesis = Annotation()\n",
    "hypothesis[Segment(2, 13)] = 'A'\n",
    "hypothesis[Segment(13, 14)] = 'D'\n",
    "hypothesis[Segment(14, 20)] = 'B'\n",
    "hypothesis[Segment(22, 38)] = 'C'\n",
    "hypothesis[Segment(38, 40)] = 'D'\n",
    "hypothesis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Identification error rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IER = 0.516\n"
     ]
    }
   ],
   "source": [
    "from pyannote.metrics.identification import IdentificationErrorRate\n",
    "identificationErrorRate = IdentificationErrorRate()\n",
    "print(\"IER = {0:.3f}\".format(identificationErrorRate(reference, hypothesis, uem=Segment(0, 40))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWIAAAD0CAYAAAChFCyQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFgJJREFUeJzt3X2wZVV95vHvQ9PQgi+o7UCHJoIJQwadCHiLtJpJEZWI\nHYtOVZgZmImCpdV5kUSnUjPBpAIZq6bGTE0l0WBC3UJGnBgk6WjsWK0EFItYDq0XbIEGiQ3JSLct\nTYPy4gvQ9z7546xDjjfn3LMvd52z94XnU7Wq9z7nx17rHvr87uq111pbtomIiPYc1nYDIiKe7ZKI\nIyJalkQcEdGyJOKIiJYlEUdEtCyJOCKiZUnEERETIOm/SNot6Q5J10haNyo2iTgiojJJxwO/AczY\nfgWwBjh/VHwScUTEZBwOPEfS4cBRwDdHBSYRR0RUZnsf8L+BbwD7gYdt/+2o+MOn1bCIiC57488e\n7Qcfmm8Ue8ttj+8GfjDw0qzt2f6JpBcCW4CTgO8Afynpl2z/2bDrJRFHRAAHH5pn53UbG8Wu3XDP\nD2zPLBHyBuAfbD8AIOnjwGuAJOKIiNHMvBdqXewbwCZJRwHfB14PzI0KTiKOiAAMLFBnN0rbOyVt\nA24FDgFfAWZHxa8oEUt6EXAtcCLwj8B/sP3tIXHzwO3l9Bu2z11JvRERtRnzpJuNETe6nn0ZcFmT\n2JXOmrgE+Kztk4HPlvNhvm/7tFKShCOikxZwo1LbShPxFuDqcnw18AsrvF5ERCsMzONGpbaVJuJj\nbe8vx98Cjh0Rt07SnKSbJSVZR0QntdUjHjtGLOkG4Lghb/3O4IltSxrVwpfa3ifpZcDnJN1u+54h\ndW0FtgIcfZRe9RM/fsTYH+DZ4O9vO6rtJnTGk8ce3XYTOmPt/d9tuwmd8SjfPmj7JSu5hoH5lh4d\nNzYR237DqPck3S9pg+39kjYAB0ZcY1/5815JnwdOB/5FIi4TomcBZl65zl+67oRGP8Qz3Rt/5LS2\nm9AZ3/rPr2m7CZ1x3B9+se0mdMYN3vb/V3oNY56cQG+3iZUOTWwHLizHFwKfXBwg6YWSjizH64HX\nAneusN6IiLoM8w1LbStNxO8Dzpb0dXorSd4HIGlG0pUl5t8Ac5K+CtwIvM92EnFEdEpvHnGzUtuK\n5hHbfpDeipHFr88B7yjHXwT+7UrqiYiYPDGPWqk5K+siIig94naGiJOIIyL60iOOiGiRgSfdzhbt\nScQREfRX1qVHHBHRGiPmW3poURJxRESx4PSIIyJak6GJiIjWifncrIuIaI+BJ1nTSt1JxBERgN1e\nj7idWiMiOmgBNSrjSDpF0q6B8oikd4+KT484IoL+zbo6fVPbdwOnAUhaA+wDPjEqPok4IgKY4M26\n1wP32B65Z3IScUQE/SXOE7lZdz5wzVIBScQRESx7Zd16SXMD57PlCUM/RNIRwLnAe5a6WBJxRESx\n0Hxo4qDtmQZxbwJutX3/UkFJxBER1L1ZN+ACxgxLQBJxRARQhiYq7jUh6WjgbOCXx8UmEUdEFAsV\ne8S2vwu8uElsEnFEBL2VdROaNTFWEnFEBGWMOJv+RES0KxvDR0S0yCgbw0dEtC094oiIFpllLeio\nqkqtks6RdLekPZIuGfL+kZKuLe/vlHRijXojImoxvVkTTUptK07EZYu3D9JbyncqcIGkUxeFvR34\ntu0fB/4Q+P2V1hsRUds8alRqq9EjPhPYY/te208AHwO2LIrZAlxdjrcBr5fUzqh4RMQQtljwYY1K\nbTWueDxw38D53vLa0Bjbh4CHGbLiRNJWSXOS5h54cL5C0yIimpv3YY1KbZ16VJLtWdsztmde8uJ2\nVrhExLOTqfeopOWqMWtiH3DCwPnG8tqwmL2SDgdeADxYoe6IiCqMeHKhnQ5gjR7xl4GTJZ1UNkE+\nH9i+KGY7cGE5Pg/4nG1XqDsiopp5DmtUaltxj9j2IUkXA9cBa4CrbO+W9F5gzvZ24EPA/5W0B3iI\nXrKOiOiMVb+yzvYOYMei1y4dOP4B8O9r1BURMSk1t8Fcjqysi4gAbKpuDL8cScQREcWqHpqIiFjt\n+kuc25BEHBFBf9OfdnrEnVrQERHRnrpLnCUdI2mbpK9JukvSq0fFpkccEVFUXjX3fuAzts8rayyO\nGhWYRBwRQd1ZE5JeAPwMcFHv2n4CeGJUfBJxRAS9m3WHmi9xXi9pbuB81vbswPlJwAPA/5H0SuAW\n4F22vzvsYknEERHFMoYmDtqeWeL9w4EzgF+3vVPS+4FLgN8dFpybdRER/POsiSalgb3AXts7y/k2\neol5qCTiiIii1qwJ298C7pN0Snnp9cCdo+IzNBERAdC8t9vUrwMfLTMm7gXeNiowiTgign/eGL7a\n9exdwFLjyE9JIo6IoJeIDy1k97WIiFZl05+IiBat+o3hIyKeCSbxYNAmkogjIgCcoYmIiFa1uQ1m\nEnFEBP29JjJrIiKiVU6POCKiXblZFxHRIudmXURE+9oamqgyMi3pHEl3S9oj6ZIh718k6QFJu0p5\nR416IyLqEfMLhzUqta24RyxpDfBB4Gx6e3B+WdJ224u3fLvW9sUrrS8iYhJW+1OczwT22L63PJfp\nY8CWCteNiJge98aJm5TaaiTi44H7Bs73ltcW+0VJt5XHS59Qod6IiKoWUKNS27Ru1v0NcI3txyX9\nMnA18LrFQZK2AlsB1q19PpvP/o9Tal63/cjN32y7Cd2x6Yttt6Az1rz8lPFBzxZ3rPwSZnXfrNsH\nDPZwN5bXnmL7QduPl9MrgVcNu5DtWdsztmeOWHNUhaZFRDTV7Hl1kxhHrpGIvwycLOmk8kiQ84Ht\ngwGSNgycngvcVaHeiIiqFhbUqNS24qEJ24ckXQxcB6wBrrK9W9J7gTnb24HfkHQucAh4CLhopfVG\nRNTUuxFXL8lK+kfgUWAeOGR75GOTqowR294B7Fj02qUDx+8B3lOjroiISZnAsMPP2j44Ligr6yIi\niklMTWuinT3fIiI6yFajAqyXNDdQtg67HPC3km4Z8f5T0iOOiKC3H/EyxogPLjXmW/y07X2S/hVw\nvaSv2b5pWGB6xBER8NSjkmpNX7O9r/x5APgEvVXIQyURR0T0uWEZQ9LRkp7XPwZ+jiWWnWRoIiKi\nqDh97VjgE5Kgl2f/3PZnRgUnEUdEFLVmTdi+F3hl0/gk4ogI2t1rIok4IgJ622BOYPlyE0nEERF9\nLS3oSCKOiABY3jziqpKIIyL60iOOiGhR5d3XliOJOCKiLz3iiIiWpUccEdGy9IgjIlpk0iOOiGhb\nWxvDJxFHRPQlEUdEtCxDExERLTJooZ2qk4gjIgBQesQREa3LGHFERMuSiCMiWtZSIq7y8FBJV0k6\nIGnow/HU8wFJeyTdJumMGvVGRFRj0IIalSYkrZH0FUmfGhdb6ynOHwbOWeL9NwEnl7IV+NNK9UZE\n1FPpKc7Fu4C7mgRWScS2bwIeWiJkC/AR99wMHCNpQ426IyK6RtJG4OeBK5vE1+oRj3M8cN/A+d7y\n2g+RtFXSnKS5J+a/N6WmRUT0yM0KsL6fq0rZuuhSfwT8N6DRzORO3ayzPQvMArzgORtaGjaPiGet\n5vOID9qeGfaGpDcDB2zfIumsJhebViLeB5wwcL6xvBYR0Q3LG/9dymuBcyVtBtYBz5f0Z7Z/adR/\nMK2hie3AW8vsiU3Aw7b3T6nuiIhGtNCsLMX2e2xvtH0icD7wuaWSMFTqEUu6BjiL3rjJXuAyYG1p\n1BXADmAzsAf4HvC2GvVGRFS1mhd02L5gzPsG3lmjroiIiamciG1/Hvj8uLhO3ayLiGjLwIyIqUsi\njojoy+5rEREtS484IqJd2Rg+IqJNGSOOiOiAJOKIiJYlEUdEtCtDExERbUsijohoUW7WRUR0QBJx\nRETLkogjItojMjQREdG+JOKIiBY5S5wjItqXHnFERLsyRhwR0bYk4oiIFtV7ijOS1gE3AUfSy7Pb\nbF82Kj6JOCKiqHiz7nHgdbYfk7QW+IKkT9u+eVhwEnFERFFrjLg8MPmxcrq2lJFXP6xOtRERzwBu\nWBqQtEbSLuAAcL3tnaNik4gjIqB5Eu4l4vWS5gbK1n9xOXve9mnARuBMSa8YVXWGJiIiKEucm4cf\ntD3TJND2dyTdCJwD3DEspkqPWNJVkg5IGlqJpLMkPSxpVymX1qg3IqKqSkMTkl4i6Zhy/BzgbOBr\no+Jr9Yg/DFwOfGSJmL+z/eZK9UVEVFdx1sQG4GpJa+h1eP/C9qdGBVdJxLZvknRijWtFRLSm3qyJ\n24DTm8ZP82bdqyV9VdKnJb18ivVGRIxXntDRpNQ2rZt1twIvLZObNwN/DZy8OKjcedwKsI6jmN99\n95Sa123f3NR2C7rjum/uarsJnfHv3vlTbTehO4benXoaWlriPJUese1HbD9WjncAayWtHxI3a3vG\n9sxajpxG0yIintJWj3gqiVjScZJUjs8s9T44jbojIhqruKBjOaoMTUi6BjiL3iTnvcBl9Jb0YfsK\n4DzgVyUdAr4PnF+WAEZEdMNq3xje9gVj3r+c3vS2iIjuyjaYERHtycNDIyK6IIk4IqJdaunWVRJx\nRASs/pt1ERHPCBmaiIhoV27WRUS0LYk4IqJFE1q+3EQScUREXxJxRER7BGgh09ciIlqVoYmIiDZN\naGe1Jqb5hI6IiE7TQrMy9jrSCZJulHSnpN2S3rVUfHrEERF99XrEh4DftH2rpOcBt0i63vadw4KT\niCMiilpjxLb3A/vL8aOS7gKOB5KIIyJG8mRmTZQn3J8O7BwVk0QcEdHXPA+vlzQ3cD5re3ZxkKTn\nAn8FvNv2I6MulkQcEcGyN4Y/aHtmyetJa+kl4Y/a/vhSsUnEEREAdq9UUB6W/CHgLtt/MC4+09ci\nIgq5WWngtcBbgNdJ2lXK5lHB6RFHRBS1Noa3/QV6ox2NJBFHREDvRl32moiIaFn2moiIaFc2/YmI\naFtLT3Fe8ayJJptbqOcDkvZIuk3SGSutNyKitoqzJpalRo+4yeYWbwJOLuWngD8tf0ZEdIImtMS5\niRX3iG3vt31rOX4U6G9uMWgL8BH33AwcI2nDSuuOiKhqoWGprOoY8RKbWxwP3Ddwvre8tn/Rf78V\n2AqwjqNqNi0iYiyt1jHivqabWyzF9qztGdszazmyVtMiIsbzMkplVXrEDTa32AecMHC+sbwWEdER\n9faaWK4asyaabG6xHXhrmT2xCXi4bJwcEdEZq3nWRH9zi9sl7Sqv/TbwowC2rwB2AJuBPcD3gLdV\nqDcioh6D5lfpEucmm1vYNvDOldYVETFRLQ1NZGVdRERfljhHRLSrrelrScQREX1JxBER7ZG9em/W\nRUQ8Y6zWecQREc8Y/QeIjitjSLpK0gFJdzSpNok4IgLKo5IalvE+DJzTtOoMTUREFLVmTdi+qWyC\n1kgScUREX2ZNRES0yIaFxpsNr5c0N3A+a3v26VadRBwR0dd80/eDtmdqVZtEHBFRrPqN4SMiVr16\n09euAf4fcIqkvZLevlR8esQREVCmr1WbNXHBcuKTiCMiAFjWzbqqkogjIvoyfS0iokUVhyaWK4k4\nIgLoPTw0QxMREe3K0ERERIsyNBER0QGZNRER0aZmizUmIYk4IgLK0ER6xBER7Vqte01IOkHSjZLu\nlLRb0ruGxJwl6WFJu0q5dKX1RkRUV2mvieWq0SM+BPym7VslPQ+4RdL1tu9cFPd3tt9cob6IiAnw\n6p01YXs/sL8cPyrpLuB4YHEijojoLoPn51upuuo2mOUZTacDO4e8/WpJX5X0aUkvr1lvREQVq3ho\nAgBJzwX+Cni37UcWvX0r8FLbj0naDPw1cPKQa2wFtpbTx2/wtkaPop6w9cDBtAHoQDvWbGi/DUUH\n2vFfO9AGoBOfBaes+ArLe1RSVVUSsaS19JLwR21/fPH7g4nZ9g5JfyJpve2Di+JmgdlyzbmajyJ5\nurrQji60oSvt6EIbutKOLrShK+1Y9Py4p2+1ziOWJOBDwF22/2BEzHHA/bYt6Ux6QyIPrrTuiIia\nvIp7xK8F3gLcLmlXee23gR8FsH0FcB7wq5IOAd8Hzrdb+tUTETGMDfOrNBHb/gKgMTGXA5cv89JP\n+9HUlXWhHV1oA3SjHV1oA3SjHV1oA3SjHXXa0NI2mErHNCICnn/Yi73p8Dc2ir3+yWtuGTcuLukc\n4P3AGuBK2+8bFZunOEdEQJmattCsjCFpDfBB4E3AqcAFkk4dFd+ZRCzpRZKul/T18ucLR8TNDyyV\n3l6x/nMk3S1pj6RLhrx/pKRry/s7y5zpqhq04SJJDwz8/O+YQBuuknRA0tCpg+r5QGnjbZLOaKEN\nU1ky33D5/kQ/j65sISBpnaQvlbUAuyX99yExE/2ONGzDir4jXnCj0sCZwB7b99p+AvgYsGV0xXYn\nCvC/gEvK8SXA74+Ie2wCda8B7gFeBhwBfBU4dVHMrwFXlOPzgWtbaMNFwOUT/v/wM8AZwB0j3t8M\nfJrefYFNwM4W2nAW8KlJfg6lng3AGeX4ecDfD/l/MtHPo2EbJv55lJ/vueV4Lb1FW5sWxUz6O9Kk\nDU/7OwJ8BphrWO5YdL510bXOozcc0T9/y1Lt6tLua1vo/YUCuBr4PPBbU6r7qd9eAJL6v70Gl2lv\nAX6vHG8DLpckl095Sm2YONs3jenJbAE+Un7umyUdI2mDe0vdp9WGqXCz5fsT/TwatmHiys/3WDld\nW8riv/sT/Y40bMNKrn9OrWstV2eGJoBjB/7yfgs4dkTcOklzkm6W9AuV6j4euG/gfG95bWiM7UPA\nw8CLK9XftA0Av1j+CbxN0gkV62+qaTsnbapL5jV6+f7UPo8l2gBT+DwkrSlTVA8A19se+VlM6DvS\npA3Q/ncEYB8wWPfG8tpQU03Ekm6QdMeQ8kNjJ+U336jfdC91727lfwL+SNKPTbrdHfI3wIm2fxK4\nnt6/HJ6N+kvmXwn8Mb0l8xOjpZfvT8WYNkzl87A9b/s0eknlTEmvmEQ9K2xDV74jXwZOlnSSpCPo\nDdWMvKc11URs+w22XzGkfBK4X9IGgPLngRHX2Ff+vJfe8MXpFZrW5LfXUzGSDgdeQN3VgWPbYPtB\n24+X0yuBV1Wsv6ll/aafBNuP2H6sHO8A1kpaP4m6NGb5PlP4PMa1YZqfR6njO8CNwOJ/yk/6OzK2\nDR35jvT/RXAxcB1wF/AXtnePiu/S0MR24MJyfCHwycUBkl4o6chyvJ7eqr4aY2VNfnsNtu884HMV\nx4cbtaH/i6o4l97/4GnbDry1zBbYBDxcc3y4CUnHSVI5ntiS+VLHksv3mfDn0aQN0/g8JL1E0jHl\n+DnA2cDXFoVN9DvSpA0d+Y4AvV+Ktv+17R+z/T/GBXei0BtL+izwdeAG4EXl9RnK3UfgNcDt9GYU\n3A68vWL9m+ndkb4H+J3y2nuBc8vxOuAvgT3Al4CXTeAzGNeG/wnsLj//jcBPTKAN19C7OfQkvfHO\ntwO/AvxKeV/05kfeU/4fzLTQhosHPoebgddM6O/kT9MbIrsN2FXK5ml+Hg3bMPHPA/hJ4CulHXcA\nl077O9KwDRP/jkyiZGVdRETLujQ0ERHxrJREHBHRsiTiiIiWJRFHRLQsiTgiomVJxBERLUsijoho\nWRJxRETL/gkIloarMJJ5WAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1151cc240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imshow(reference * hypothesis, interpolation='nearest'); colorbar();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Precision and coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision = 0.611\n"
     ]
    }
   ],
   "source": [
    "from pyannote.metrics.identification import IdentificationPrecision\n",
    "precision = IdentificationPrecision()\n",
    "print(\"Precision = {0:.3f}\".format(precision(reference, hypothesis, uem=Segment(0, 40))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall = 0.710\n"
     ]
    }
   ],
   "source": [
    "from pyannote.metrics.identification import IdentificationRecall\n",
    "recall = IdentificationRecall()\n",
    "print(\"Recall = {0:.3f}\".format(recall(reference, hypothesis, uem=Segment(0, 40))))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}