{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Example-5 (Sample weights)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Without weights" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from pycm import ConfusionMatrix\n", "y_test = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]\n", "y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "pycm.ConfusionMatrix(classes: [0, 1, 2])" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cm1=ConfusionMatrix(y_test, y_pred)\n", "cm1" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Predict 0 1 2 \n", "Actual\n", "0 3 0 0 \n", "\n", "1 0 1 2 \n", "\n", "2 2 1 3 \n", "\n", "\n", "\n", "\n", "\n", "Overall Statistics : \n", "\n", "95% CI (0.30439,0.86228)\n", "AUNP 0.66667\n", "AUNU 0.69444\n", "Bennett S 0.375\n", "CBA 0.47778\n", "Chi-Squared 6.6\n", "Chi-Squared DF 4\n", "Conditional Entropy 0.95915\n", "Cramer V 0.5244\n", "Cross Entropy 1.59352\n", "Gwet AC1 0.38931\n", "Hamming Loss 0.41667\n", "Joint Entropy 2.45915\n", "KL Divergence 0.09352\n", "Kappa 0.35484\n", "Kappa 95% CI (-0.07708,0.78675)\n", "Kappa No Prevalence 0.16667\n", "Kappa Standard Error 0.22036\n", "Kappa Unbiased 0.34426\n", "Lambda A 0.16667\n", "Lambda B 0.42857\n", "Mutual Information 0.52421\n", "NIR 0.5\n", "Overall ACC 0.58333\n", "Overall CEN 0.46381\n", "Overall J (1.225,0.40833)\n", "Overall MCC 0.36667\n", "Overall MCEN 0.51894\n", "Overall RACC 0.35417\n", "Overall RACCU 0.36458\n", "P-Value 0.38721\n", "PPV Macro 0.56667\n", "PPV Micro 0.58333\n", "Phi-Squared 0.55\n", "RCI 0.34947\n", "RR 4.0\n", "Reference Entropy 1.5\n", "Response Entropy 1.48336\n", "SOA1(Landis & Koch) Fair\n", "SOA2(Fleiss) Poor\n", "SOA3(Altman) Fair\n", "SOA4(Cicchetti) Poor\n", "Scott PI 0.34426\n", "Standard Error 0.14232\n", "TPR Macro 0.61111\n", "TPR Micro 0.58333\n", "Zero-one Loss 5\n", "\n", "Class Statistics :\n", "\n", "Classes 0 1 2 \n", "ACC(Accuracy) 0.83333 0.75 0.58333 \n", "AUC(Area under the roc curve) 0.88889 0.61111 0.58333 \n", "AUCI(Auc value interpretation) Very Good Fair Poor \n", "BM(Informedness or bookmaker informedness) 0.77778 0.22222 0.16667 \n", "CEN(Confusion entropy) 0.25 0.49658 0.60442 \n", "DOR(Diagnostic odds ratio) None 4.0 2.0 \n", "DP(Discriminant power) None 0.33193 0.16597 \n", "DPI(Discriminant power interpretation) None Poor Poor \n", "ERR(Error rate) 0.16667 0.25 0.41667 \n", "F0.5(F0.5 score) 0.65217 0.45455 0.57692 \n", "F1(F1 score - harmonic mean of precision and sensitivity) 0.75 0.4 0.54545 \n", "F2(F2 score) 0.88235 0.35714 0.51724 \n", "FDR(False discovery rate) 0.4 0.5 0.4 \n", "FN(False negative/miss/type 2 error) 0 2 3 \n", "FNR(Miss rate or false negative rate) 0.0 0.66667 0.5 \n", "FOR(False omission rate) 0.0 0.2 0.42857 \n", "FP(False positive/type 1 error/false alarm) 2 1 2 \n", "FPR(Fall-out or false positive rate) 0.22222 0.11111 0.33333 \n", "G(G-measure geometric mean of precision and sensitivity) 0.7746 0.40825 0.54772 \n", "GI(Gini index) 0.77778 0.22222 0.16667 \n", "IS(Information score) 1.26303 1.0 0.26303 \n", "J(Jaccard index) 0.6 0.25 0.375 \n", "MCC(Matthews correlation coefficient) 0.68313 0.2582 0.16903 \n", "MCEN(Modified confusion entropy) 0.26439 0.5 0.6875 \n", "MK(Markedness) 0.6 0.3 0.17143 \n", "N(Condition negative) 9 9 6 \n", "NLR(Negative likelihood ratio) 0.0 0.75 0.75 \n", "NPV(Negative predictive value) 1.0 0.8 0.57143 \n", "P(Condition positive or support) 3 3 6 \n", "PLR(Positive likelihood ratio) 4.5 3.0 1.5 \n", "PLRI(Positive likelihood ratio interpretation) Poor Poor Poor \n", "POP(Population) 12 12 12 \n", "PPV(Precision or positive predictive value) 0.6 0.5 0.6 \n", "PRE(Prevalence) 0.25 0.25 0.5 \n", "RACC(Random accuracy) 0.10417 0.04167 0.20833 \n", "RACCU(Random accuracy unbiased) 0.11111 0.0434 0.21007 \n", "TN(True negative/correct rejection) 7 8 4 \n", "TNR(Specificity or true negative rate) 0.77778 0.88889 0.66667 \n", "TON(Test outcome negative) 7 10 7 \n", "TOP(Test outcome positive) 5 2 5 \n", "TP(True positive/hit) 3 1 3 \n", "TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 0.33333 0.5 \n", "Y(Youden index) 0.77778 0.22222 0.16667 \n", "dInd(Distance index) 0.22222 0.67586 0.60093 \n", "sInd(Similarity index) 0.84287 0.52209 0.57508 \n", "\n" ] } ], "source": [ "print(cm1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## With random weights" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from random import randint\n", "weights = [randint(1,10) for i in range(len(y_test))]\n", "weights[2]*=9" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "pycm.ConfusionMatrix(classes: [0, 1, 2])" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cm2=ConfusionMatrix(y_test, y_pred, sample_weight = weights)\n", "cm2" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Predict 0 1 2 \n", "Actual\n", "0 13 0 0 \n", "\n", "1 0 4 11 \n", "\n", "2 11 6 74 \n", "\n", "\n", "\n", "\n", "\n", "Overall Statistics : \n", "\n", "95% CI (0.68849,0.84092)\n", "AUNP 0.72284\n", "AUNU 0.75426\n", "Bennett S 0.64706\n", "CBA 0.54051\n", "Chi-Squared 64.99179\n", "Chi-Squared DF 4\n", "Conditional Entropy 0.77055\n", "Cramer V 0.52257\n", "Cross Entropy 1.07391\n", "Gwet AC1 0.70255\n", "Hamming Loss 0.23529\n", "Joint Entropy 1.79211\n", "KL Divergence 0.05236\n", "Kappa 0.44131\n", "Kappa 95% CI (0.26035,0.62228)\n", "Kappa No Prevalence 0.52941\n", "Kappa Standard Error 0.09233\n", "Kappa Unbiased 0.43702\n", "Lambda A 0.07143\n", "Lambda B 0.38235\n", "Mutual Information 0.34228\n", "NIR 0.76471\n", "Overall ACC 0.76471\n", "Overall CEN 0.34008\n", "Overall J (1.45763,0.48588)\n", "Overall MCC 0.44913\n", "Overall MCEN 0.43775\n", "Overall RACC 0.57884\n", "Overall RACCU 0.58206\n", "P-Value 0.55048\n", "PPV Macro 0.60408\n", "PPV Micro 0.76471\n", "Phi-Squared 0.54615\n", "RCI 0.33505\n", "RR 39.66667\n", "Reference Entropy 1.02155\n", "Response Entropy 1.11283\n", "SOA1(Landis & Koch) Moderate\n", "SOA2(Fleiss) Intermediate to Good\n", "SOA3(Altman) Moderate\n", "SOA4(Cicchetti) Fair\n", "Scott PI 0.43702\n", "Standard Error 0.03888\n", "TPR Macro 0.69328\n", "TPR Micro 0.76471\n", "Zero-one Loss 28\n", "\n", "Class Statistics :\n", "\n", "Classes 0 1 2 \n", "ACC(Accuracy) 0.90756 0.85714 0.76471 \n", "AUC(Area under the roc curve) 0.94811 0.60449 0.71016 \n", "AUCI(Auc value interpretation) Excellent Fair Good \n", "BM(Informedness or bookmaker informedness) 0.89623 0.20897 0.42033 \n", "CEN(Confusion entropy) 0.26014 0.50764 0.33309 \n", "DOR(Diagnostic odds ratio) None 5.93939 6.72727 \n", "DP(Discriminant power) None 0.42659 0.45641 \n", "DPI(Discriminant power interpretation) None Poor Poor \n", "ERR(Error rate) 0.09244 0.14286 0.23529 \n", "F0.5(F0.5 score) 0.59633 0.36364 0.85847 \n", "F1(F1 score - harmonic mean of precision and sensitivity) 0.7027 0.32 0.84091 \n", "F2(F2 score) 0.85526 0.28571 0.82405 \n", "FDR(False discovery rate) 0.45833 0.6 0.12941 \n", "FN(False negative/miss/type 2 error) 0 11 17 \n", "FNR(Miss rate or false negative rate) 0.0 0.73333 0.18681 \n", "FOR(False omission rate) 0.0 0.10092 0.5 \n", "FP(False positive/type 1 error/false alarm) 11 6 11 \n", "FPR(Fall-out or false positive rate) 0.10377 0.05769 0.39286 \n", "G(G-measure geometric mean of precision and sensitivity) 0.73598 0.3266 0.8414 \n", "GI(Gini index) 0.89623 0.20897 0.42033 \n", "IS(Information score) 2.30986 1.666 0.18709 \n", "J(Jaccard index) 0.54167 0.19048 0.72549 \n", "MCC(Matthews correlation coefficient) 0.69675 0.25 0.39468 \n", "MCEN(Modified confusion entropy) 0.25793 0.50252 0.46672 \n", "MK(Markedness) 0.54167 0.29908 0.37059 \n", "N(Condition negative) 106 104 28 \n", "NLR(Negative likelihood ratio) 0.0 0.77823 0.30769 \n", "NPV(Negative predictive value) 1.0 0.89908 0.5 \n", "P(Condition positive or support) 13 15 91 \n", "PLR(Positive likelihood ratio) 9.63636 4.62222 2.06993 \n", "PLRI(Positive likelihood ratio interpretation) Fair Poor Poor \n", "POP(Population) 119 119 119 \n", "PPV(Precision or positive predictive value) 0.54167 0.4 0.87059 \n", "PRE(Prevalence) 0.10924 0.12605 0.76471 \n", "RACC(Random accuracy) 0.02203 0.01059 0.54622 \n", "RACCU(Random accuracy unbiased) 0.02417 0.01103 0.54685 \n", "TN(True negative/correct rejection) 95 98 17 \n", "TNR(Specificity or true negative rate) 0.89623 0.94231 0.60714 \n", "TON(Test outcome negative) 95 109 34 \n", "TOP(Test outcome positive) 24 10 85 \n", "TP(True positive/hit) 13 4 74 \n", "TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 0.26667 0.81319 \n", "Y(Youden index) 0.89623 0.20897 0.42033 \n", "dInd(Distance index) 0.10377 0.7356 0.43501 \n", "sInd(Similarity index) 0.92662 0.47985 0.6924 \n", "\n" ] } ], "source": [ "print(cm2)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 2 }