{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## 분석 목적 : Deep Learning 을 이용한 무인 자동차 핵심 기술별 논문 분류 분석\n",
    "\n",
    "## 본 Page는 R을 이용한 Deep Learning 연습용 페이지입니다.\n",
    "## 위 분석 중 일부의 분석을 가공하여 공개합니다.\n",
    "\n",
    "## 본 Page 설명\n",
    "## Data : 핵심 기술별 특허 정보 한국 DB from WIPS\n",
    "## 분석 : 특허 DB의 초록을 Deep Learning을 이용하여 텍스트 기반 특허 기술 분류 모형 학습\n",
    "\n",
    "## 실제 최종 분석에는 특허 DB 중 미국 DB를 사용하였습니다.\n",
    "## 한국 DATA는 기술 별 수가 적어, 결과 해석에 주의할 필요가 있습니다.\n",
    "\n",
    "## 배포 및 수정, 재배포를 삼가주세요.\n",
    "\n",
    "## 2015.04.28 by 김형준(Hyung-jun, Kim)\n",
    "## soeque1@gmail.com\n",
    "## http://soeque1.github.io/r_slide/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rm(list=ls())\n",
    "save_dir <- \"/Users/kimhyungjun/Dropbox/h2o/prac/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "options(repos='http://cran.nexr.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "install_lib <- function(x){\n",
    "  for( i in x ){\n",
    "    #  require returns TRUE invisibly if it was able to load package\n",
    "    if( ! require( i , character.only = TRUE ) ){\n",
    "      #  If package was not able to be loaded then re-install\n",
    "      install.packages( i , dependencies = TRUE )\n",
    "      #  Load package after installing\n",
    "            library( i , character.only = T)\n",
    "    } else {\n",
    "            library( i , character.only = T)\n",
    "    }\n",
    "  }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "suppressMessages(install_lib(c(\"readxl\", \n",
    "                               \"dplyr\", \"stringr\", \n",
    "                               \"tm\", \"lsa\",\n",
    "                               \"KoNLP\", \n",
    "                               \"h2o\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_dir <- \"/Users/kimhyungjun/Dropbox/h2o/prac/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_list <- list.files(paste(save_dir,\"data/wips\",sep=\"\"))\n",
    "data_list_kr <- data_list[grep(\"kr\",data_list)]\n",
    "\n",
    "data <- data.frame()\n",
    "for (i in 1:length(data_list_kr))\n",
    "{\n",
    "    data_temp <- read_excel(paste(save_dir,\"data/wips/\",data_list_kr[i],sep=\"\"))\n",
    "    data_temp <- cbind(rep(substr(data_list_kr[i],1,1),nrow(data_temp)),data_temp)\n",
    "    colnames(data_temp)[1] <- \"기술\"\n",
    "    data <- rbind(data,data_temp)\n",
    "    rm(data_temp)\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1] 126   6"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_kr <- data[,c(\"기술\", \"Original IPC Main\", \"출원일\",\"발명의 명칭\", \"요약\", \"국가코드\")]\n",
    "dim(dt_kr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dt_text <- dt_kr %>% dplyr::select(요약) %>% .[[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "DT <-  sapply(dt_text, extractNoun, USE.NAMES = F) %>%\n",
    "       sapply(function(x) paste(x, collapse = ' ')) %>%\n",
    "       as.data.frame"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "rm(dt_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              Docs\n",
      "Terms          1 2 3 4 5\n",
      "  가능         0 0 1 0 0\n",
      "  가능성       0 0 0 0 0\n",
      "  가능하다동일 0 0 0 0 0\n",
      "  가변         0 0 0 0 0\n",
      "  가변하는     0 0 0 0 0\n"
     ]
    }
   ],
   "source": [
    "tdm <- TermDocumentMatrix(Corpus(DataframeSource(DT)),\n",
    "                                control = list(\n",
    "                                    removeNumbers = TRUE,\n",
    "                                    wordLengths = c(2,Inf),\n",
    "                                    removePunctuation = TRUE,\n",
    "                                    weighting = function(x)\n",
    "                                        weightSMART(x, spec = \"nnn\")))\n",
    "tdm <- as.matrix(tdm)\n",
    "print(tdm[1:5,1:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tdm <- lw_logtf(tdm) * gw_entropy(tdm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              Docs\n",
      "Terms          1 2        3 4 5\n",
      "  가능         0 0 1.126579 0 0\n",
      "  가능성       0 0 0.000000 0 0\n",
      "  가능하다동일 0 0 0.000000 0 0\n",
      "  가변         0 0 0.000000 0 0\n",
      "  가변하는     0 0 0.000000 0 0\n"
     ]
    }
   ],
   "source": [
    "print(tdm[1:5,1:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rm(DT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "actual_y <- dt_kr%>%dplyr::select(기술)%>%.[[1]]\n",
    "save_data <- data.frame(cbind(t(tdm),actual_y))\n",
    "write.table(save_data, paste(save_dir,'data/patent_kr.csv', sep=\"\"),\n",
    "            row.names=F, col.names=F)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully connected to http://127.0.0.1:54321 \n",
      "\n",
      "R is connected to H2O cluster:\n",
      "    H2O cluster uptime:         1 hours 2 minutes \n",
      "    H2O cluster version:        2.8.4.4 \n",
      "    H2O cluster name:           H2O_started_from_R \n",
      "    H2O cluster total nodes:    1 \n",
      "    H2O cluster total memory:   5.33 GB \n",
      "    H2O cluster total cores:    4 \n",
      "    H2O cluster allowed cores:  4 \n",
      "    H2O cluster healthy:        TRUE \n",
      "\n"
     ]
    }
   ],
   "source": [
    "h2oServer <- h2o.init(nthreads=-1, max_mem_size = \"6g\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  |                                                                            \r",
      "  |                                                                      |   0%\r",
      "  |                                                                            \r",
      "  |======================================================================| 100%\n"
     ]
    }
   ],
   "source": [
    "data_hex <- h2o.importFile(h2oServer, path = paste(save_dir,\"data/patent_kr.csv\", sep=\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "random <- h2o.runif(data_hex, seed = 654321)\n",
    "train.hex <- h2o.assign(data_hex[random <= .8,], \"train.hex\")\n",
    "test.hex  <- h2o.assign(data_hex[random > .8,], \"test.hex\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "label_t <- test.hex %>% as.data.frame %>% select(ncol(test.hex)) %>% table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test.hex %>% as.data.frame %>% select(ncol(test.hex))\n",
       " A  B  C  E  H \n",
       " 4  2 23  2  1 "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "      C \n",
       "0.71875 "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_t[label_t == max(label_t)] / sum(label_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  |                                                                            \r",
      "  |                                                                      |   0%\r",
      "  |                                                                            \r",
      "  |==================                                                    |  26%\r",
      "  |                                                                            \r",
      "  |====================================                                  |  51%\r",
      "  |                                                                            \r",
      "  |======================================================                |  77%\r",
      "  |                                                                            \r",
      "  |======================================================================| 100%\n"
     ]
    }
   ],
   "source": [
    "my.dl <- h2o.deeplearning(x = 1:(ncol(train.hex)-1), y = ncol(train.hex), data=train.hex, validation=test.hex,\n",
    "                         variable_importances=T,\n",
    "                         activation = \"RectifierWithDropout\", \n",
    "                         input_dropout_ratio = 0.25, \n",
    "                         hidden_dropout_ratios = c(0.5,0.5,0.5), \n",
    "                         adaptive_rate = T,\n",
    "                         balance_classes = T, \n",
    "                         train_samples_per_iteration = 1500, \n",
    "                         hidden = c(250,250,250), \n",
    "                         epochs = 15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IP Address: 127.0.0.1 \n",
       "Port      : 54321 \n",
       "Parsed Data Key: train.hex \n",
       "\n",
       "Deep Learning Model Key: DeepLearning_b1d4668517cab735bd3788d3a204df6\n",
       "\n",
       "Training classification error: 0.01542416\n",
       "\n",
       "Validation classification error: 0.15625\n",
       "\n",
       "Confusion matrix:\n",
       "Reported on test.hex \n",
       "        Predicted\n",
       "Actual   A B  C D E G H   Error\n",
       "  A      2 0  1 0 1 0 0 0.50000\n",
       "  B      1 1  0 0 0 0 0 0.50000\n",
       "  C      0 0 23 0 0 0 0 0.00000\n",
       "  D      0 0  0 0 0 0 0     NaN\n",
       "  E      1 0  0 0 1 0 0 0.50000\n",
       "  G      0 0  0 0 0 0 0     NaN\n",
       "  H      1 0  0 0 0 0 0 1.00000\n",
       "  Totals 5 1 24 0 2 0 0 0.15625\n",
       "\n",
       "Hit Ratios for Multi-class Classification:\n",
       "  k hit_ratios\n",
       "1 1    0.84375\n",
       "2 2    0.84375\n",
       "3 3    0.87500\n",
       "4 4    0.96875\n",
       "5 5    1.00000\n",
       "6 6    1.00000\n",
       "7 7    1.00000\n",
       "\n",
       "Relative Variable Importance:\n",
       "  C966     C1129      C590     C326     C1002      C496     C686       C64\n",
       "1    1 0.8975929 0.8479354 0.846368 0.8334404 0.8257619 0.821659 0.8195269\n",
       "      C1307      C509      C618      C922      C388     C1032      C399\n",
       "1 0.8156956 0.8146315 0.8145919 0.8120914 0.8095592 0.8094196 0.8089094\n",
       "      C1130      C740     C1217     C1105     C1257      C467     C1025\n",
       "1 0.8085309 0.8078029 0.8077086 0.8062583 0.8059309 0.8057524 0.8045306\n",
       "       C271      C135      C111       C42      C947      C954      C199\n",
       "1 0.8035234 0.8022107 0.7999194 0.7996116 0.7982485 0.7972937 0.7971659\n",
       "       C319    C497     C517     C1194      C534      C905      C958     C1096\n",
       "1 0.7971356 0.79634 0.796011 0.7956779 0.7952835 0.7949308 0.7945616 0.7931651\n",
       "       C265     C1023     C1174       C33      C369      C884     C1276\n",
       "1 0.7927092 0.7920221 0.7913429 0.7905443 0.7903202 0.7902501 0.7901686\n",
       "       C588     C1005     C1326     C1065      C365      C463      C802\n",
       "1 0.7892069 0.7882137 0.7879453 0.7878744 0.7875574 0.7873529 0.7871828\n",
       "       C362      C376    C769     C1067      C546      C735      C622      C301\n",
       "1 0.7857954 0.7856193 0.78552 0.7852402 0.7847852 0.7847664 0.7835196 0.7833849\n",
       "       C829      C115     C1195      C744      C970     C1223     C1293\n",
       "1 0.7828824 0.7826813 0.7823161 0.7822815 0.7822725 0.7822048 0.7821126\n",
       "       C828      C430      C344       C37     C1154      C290      C140\n",
       "1 0.7816629 0.7814871 0.7814563 0.7814389 0.7809651 0.7809343 0.7798126\n",
       "       C702     C1116     C1298      C907      C426      C628      C725\n",
       "1 0.7796668 0.7795824 0.7792722 0.7791799 0.7791016 0.7789949 0.7789022\n",
       "      C1134      C383      C572      C996      C474     C1314      C471\n",
       "1 0.7784212 0.7784176 0.7783199 0.7782393 0.7781489 0.7780302 0.7779759\n",
       "       C291     C1224      C823     C1173      C929     C1331      C670\n",
       "1 0.7777364 0.7776245 0.7773763 0.7772556 0.7771392 0.7769874 0.7765389\n",
       "      C1074     C1121      C765       C84      C127      C594     C770\n",
       "1 0.7760179 0.7759684 0.7754998 0.7748432 0.7748062 0.7747998 0.774744\n",
       "       C323      C487     C1271      C334      C387      C293        C2\n",
       "1 0.7744636 0.7744355 0.7741257 0.7734792 0.7731457 0.7729978 0.7729579\n",
       "       C730     C1259     C1080      C345      C407       C87      C847\n",
       "1 0.7724191 0.7723939 0.7723885 0.7723081 0.7721801 0.7720025 0.7718574\n",
       "       C356     C1034      C894      C745      C143      C977      C840\n",
       "1 0.7713206 0.7711317 0.7711133 0.7703581 0.7702123 0.7701865 0.7698066\n",
       "      C1016      C794     C1296      C891      C835     C1141   C1047     C1272\n",
       "1 0.7697496 0.7697167 0.7694979 0.7694064 0.7690371 0.7690339 0.76896 0.7689471\n",
       "       C909      C116     C1160      C482      C714    C1013      C232\n",
       "1 0.7689156 0.7688924 0.7688897 0.7686064 0.7685998 0.767898 0.7675303\n",
       "       C734     C1055      C796      C134      C359     C1052     C1020\n",
       "1 0.7674907 0.7670408 0.7668033 0.7663577 0.7663179 0.7662599 0.7660069\n",
       "        C81      C825     C1309      C842      C309      C221      C928\n",
       "1 0.7659751 0.7659576 0.7658365 0.7657751 0.7657126 0.7656459 0.7655602\n",
       "      C1012       C24      C215     C133     C1069      C706     C935     C1198\n",
       "1 0.7654943 0.7653605 0.7653115 0.765112 0.7649566 0.7648462 0.764743 0.7647045\n",
       "        C55       C52     C1035    C1339      C375     C1006      C107\n",
       "1 0.7646913 0.7645694 0.7642381 0.764168 0.7641654 0.7638252 0.7637324\n",
       "        C67      C473      C410     C1327      C773    C984     C1205      C921\n",
       "1 0.7636613 0.7635288 0.7632498 0.7629954 0.7629081 0.76278 0.7627078 0.7626934\n",
       "       C855      C677      C522      C138      C630     C1254      C357\n",
       "1 0.7626634 0.7626478 0.7623096 0.7622596 0.7622589 0.7618416 0.7616351\n",
       "       C812     C1022      C421      C94     C1114      C480     C1297\n",
       "1 0.7615276 0.7614833 0.7611583 0.761064 0.7610591 0.7609112 0.7607001\n",
       "       C885      C687      C533     C1136      C151       C14     C1010\n",
       "1 0.7606875 0.7601503 0.7600196 0.7600086 0.7598212 0.7595975 0.7594127\n",
       "        C41     C1021     C1079      C614      C685      C395     C1182\n",
       "1 0.7594103 0.7594041 0.7593878 0.7592459 0.7592105 0.7592056 0.7591121\n",
       "      C1249     C1315      C715      C900    C1027     C1142     C1189\n",
       "1 0.7587667 0.7587557 0.7584937 0.7584094 0.758297 0.7582086 0.7580647\n",
       "       C914      C523      C122      C672      C461     C1329      C203\n",
       "1 0.7579346 0.7578264 0.7577965 0.7577749 0.7576365 0.7574875 0.7574754\n",
       "       C666     C1292      C844      C705      C181      C455     C913\n",
       "1 0.7573696 0.7572414 0.7571458 0.7569963 0.7568834 0.7568464 0.756492\n",
       "        C61      C165       C50      C694     C1294      C137      C780\n",
       "1 0.7563937 0.7562829 0.7562515 0.7559683 0.7557278 0.7557101 0.7556475\n",
       "      C1239      C435       C20     C1336     C1235      C485     C1113\n",
       "1 0.7556031 0.7554152 0.7553359 0.7552998 0.7552724 0.7551503 0.7551097\n",
       "       C650      C394      C279      C234     C521      C852      C554\n",
       "1 0.7550901 0.7549536 0.7549089 0.7548334 0.754686 0.7546334 0.7543668\n",
       "       C157      C749      C611     C1072      C417      C254      C128\n",
       "1 0.7543433 0.7541392 0.7540181 0.7539256 0.7539119 0.7539011 0.7538765\n",
       "       C305      C826     C1054     C1246      C576      C846      C633\n",
       "1 0.7538201 0.7537107 0.7537042 0.7536645 0.7536554 0.7535312 0.7533575\n",
       "      C1031      C736      C889      C610       C36      C989     C1001\n",
       "1 0.7532858 0.7532716 0.7532635 0.7531562 0.7527947 0.7527768 0.7526592\n",
       "      C1081       C29     C1221     C1150      C378      C296      C131\n",
       "1 0.7526432 0.7526221 0.7524357 0.7522174 0.7521967 0.7519562 0.7519069\n",
       "       C350      C727      C207     C1318     C916     C1026      C940\n",
       "1 0.7518244 0.7516847 0.7514795 0.7514294 0.751395 0.7513676 0.7512986\n",
       "       C807      C105      C488      C447     C1091     C1190      C915\n",
       "1 0.7512678 0.7511446 0.7509872 0.7509811 0.7508925 0.7507546 0.7506416\n",
       "       C321      C240     C1050       C90      C324      C903     C1218\n",
       "1 0.7505885 0.7505074 0.7504052 0.7503611 0.7502369 0.7501564 0.7499986\n",
       "      C1168       C43      C374     C848      C371      C578      C404\n",
       "1 0.7499676 0.7499409 0.7498293 0.749774 0.7496913 0.7495782 0.7495402\n",
       "      C1028      C713     C563     C1311      C830      C728      C269\n",
       "1 0.7495103 0.7494926 0.749487 0.7494801 0.7494037 0.7493975 0.7493664\n",
       "       C668     C446      C819      C144      C196      C104      C587\n",
       "1 0.7488936 0.748737 0.7484987 0.7484856 0.7484807 0.7484469 0.7483365\n",
       "       C729      C272      C208      C986     C1186      C760      C580\n",
       "1 0.7483054 0.7481934 0.7481154 0.7481051 0.7478497 0.7477873 0.7475753\n",
       "      C346      C662      C856      C717      C448      C136     C1222\n",
       "1 0.747523 0.7474669 0.7474087 0.7473926 0.7473556 0.7473185 0.7471813\n",
       "       C689     C1178      C742      C206     C883     C1125     C1062\n",
       "1 0.7471206 0.7469949 0.7469006 0.7467601 0.746563 0.7465131 0.7464874\n",
       "       C283     C1163     C1039      C567       C92     C1206      C544\n",
       "1 0.7463614 0.7463488 0.7462604 0.7462167 0.7462034 0.7460668 0.7458935\n",
       "      C1011     C1131      C758      C236      C616     C1058      C408\n",
       "1 0.7458351 0.7457771 0.7456776 0.7456732 0.7456213 0.7454973 0.7454961\n",
       "       C385     C1043      C179      C955      C813      C857      C145\n",
       "1 0.7454814 0.7453001 0.7451011 0.7450859 0.7449064 0.7448703 0.7448248\n",
       "       C753    C843      C739     C1036    C1341     C1090      C890      C259\n",
       "1 0.7447634 0.74475 0.7447041 0.7446395 0.744613 0.7445288 0.7444885 0.7444531\n",
       "      C1044      C464    C527      C267      C398      C329      C581      C861\n",
       "1 0.7443773 0.7442853 0.74422 0.7441939 0.7441821 0.7440794 0.7440608 0.7439879\n",
       "      C1045      C333      C505    C257      C386    C1145       C89      C316\n",
       "1 0.7439396 0.7438539 0.7436836 0.74361 0.7435558 0.743533 0.7435106 0.7433932\n",
       "       C601      C512     C542      C936      C458      C339      C816\n",
       "1 0.7433888 0.7433121 0.743299 0.7432885 0.7432319 0.7430946 0.7428714\n",
       "        C95      C893    C1323      C146     C681      C700     C1004      C639\n",
       "1 0.7428358 0.7428229 0.742514 0.7424703 0.742461 0.7424514 0.7424095 0.7423982\n",
       "       C997       C99      C552      C304      C432      C793      C332\n",
       "1 0.7423424 0.7423131 0.7421758 0.7421699 0.7420009 0.7419685 0.7419454\n",
       "       C420      C615      C172      C918     C320     C470       C63     C1082\n",
       "1 0.7418606 0.7418558 0.7415906 0.7415719 0.741483 0.741244 0.7412315 0.7412179\n",
       "      C1092    C1172     C1345     C1197      C462     C1165     C1322\n",
       "1 0.7410469 0.740999 0.7408158 0.7408131 0.7408046 0.7407908 0.7407196\n",
       "       C906      C575      C337     C555     C370      C800      C253      C476\n",
       "1 0.7407006 0.7406963 0.7404777 0.740415 0.740382 0.7403559 0.7403405 0.7400334\n",
       "        C54      C506      C845     C183      C431     C1071      C434\n",
       "1 0.7399247 0.7398568 0.7397155 0.739683 0.7394299 0.7393245 0.7392812\n",
       "        C25     C1000     C1184     C1304      C612      C406     C1015\n",
       "1 0.7392372 0.7390486 0.7388765 0.7388513 0.7388461 0.7387329 0.7386037\n",
       "      C1210     C1287       C35      C814      C295        C4     C1302\n",
       "1 0.7385069 0.7384486 0.7384242 0.7383907 0.7383103 0.7381767 0.7381145\n",
       "       C294      C427      C479     C1120      C310     C787       C27\n",
       "1 0.7379762 0.7379045 0.7377451 0.7377428 0.7376915 0.737655 0.7375934\n",
       "       C874      C680     C556      C106      C591      C557     C1179\n",
       "1 0.7374082 0.7373557 0.737317 0.7372713 0.7372595 0.7372367 0.7372044\n",
       "        C48      C440     C1237     C1305      C441      C632     C876\n",
       "1 0.7369888 0.7369506 0.7369368 0.7369273 0.7367464 0.7366913 0.736545\n",
       "       C472     C1338      C454     C1009      C879       C31      C520\n",
       "1 0.7364756 0.7364675 0.7364429 0.7364427 0.7363729 0.7363645 0.7363268\n",
       "      C1207      C962     C246      C676     C1075      C289      C963\n",
       "1 0.7363051 0.7362438 0.736175 0.7361508 0.7361286 0.7360806 0.7360426\n",
       "       C185     C1057      C465      C948     C1187      C311       C75\n",
       "1 0.7360024 0.7359349 0.7358131 0.7358015 0.7357221 0.7356907 0.7355947\n",
       "       C652      C312      C892      C260      C239     C709      C288\n",
       "1 0.7354209 0.7353572 0.7352065 0.7351296 0.7350991 0.735075 0.7350131\n",
       "       C466      C302     C949      C771      C274     C1146      C841\n",
       "1 0.7349718 0.7349485 0.734744 0.7345638 0.7344201 0.7343992 0.7343854\n",
       "      C1203      C721    C1029      C817    C1283      C862      C661      C911\n",
       "1 0.7343469 0.7342613 0.734135 0.7341335 0.734081 0.7340664 0.7340214 0.7340019\n",
       "      C1101     C561       C79     C1332     C1215     C451      C565     C1280\n",
       "1 0.7339368 0.733878 0.7338329 0.7336548 0.7336519 0.733615 0.7335549 0.7334427\n",
       "      C1244       C56      C193      C508      C860      C411     C1007\n",
       "1 0.7334313 0.7333252 0.7332497 0.7330612 0.7330523 0.7329103 0.7328154\n",
       "      C1196      C895      C397      C988      C737       C16      C297    C978\n",
       "1 0.7327754 0.7326934 0.7326428 0.7326143 0.7325304 0.7325301 0.7325003 0.73223\n",
       "       C985      C851      C530      C189      C62      C192     C638      C423\n",
       "1 0.7318763 0.7318415 0.7316546 0.7316399 0.731589 0.7315839 0.731581 0.7315204\n",
       "       C453      C202       C97      C859       C10      C888      C761\n",
       "1 0.7313812 0.7312518 0.7311867 0.7311819 0.7310823 0.7310578 0.7310466\n",
       "      C1328      C640     C1192     C1260     C1213      C822      C139\n",
       "1 0.7309218 0.7308531 0.7307048 0.7306598 0.7305744 0.7305577 0.7305316\n",
       "       C945     C1078     C946      C646     C1126      C262     C1041\n",
       "1 0.7305005 0.7304934 0.730439 0.7304055 0.7303486 0.7303273 0.7302767\n",
       "       C704     C1299     C1231     C1330      C166      C722      C930\n",
       "1 0.7302715 0.7302477 0.7302068 0.7301922 0.7300835 0.7299992 0.7299532\n",
       "       C194      C368      C102     C1306      C912      C317      C526\n",
       "1 0.7297704 0.7296972 0.7296708 0.7295907 0.7295895 0.7295792 0.7295696\n",
       "      C1188      C969     C1127    C1251    C1175     C1171      C537      C854\n",
       "1 0.7294126 0.7293707 0.7293646 0.729287 0.729272 0.7291312 0.7290774 0.7289547\n",
       "       C560     C1286     C110     C1076     C1112    C1132     C1137      C278\n",
       "1 0.7289056 0.7288549 0.728703 0.7286866 0.7286117 0.728604 0.7285699 0.7284995\n",
       "      C245      C201     C490      C690      C536     C1284     C1335      C460\n",
       "1 0.728353 0.7283059 0.728281 0.7282014 0.7281757 0.7281398 0.7280523 0.7280511\n",
       "       C821      C792      C636      C241      C114     C991     C1344\n",
       "1 0.7280304 0.7279805 0.7279483 0.7277631 0.7277523 0.727735 0.7277259\n",
       "      C1111      C983     C518     C1270      C968      C227      C524\n",
       "1 0.7276319 0.7274838 0.727406 0.7272971 0.7271523 0.7270928 0.7270865\n",
       "        C91     C1140     C1149     C1084      C188     C1083      C973\n",
       "1 0.7270833 0.7270638 0.7268338 0.7267823 0.7267478 0.7267203 0.7266377\n",
       "      C1099      C938      C276     C1204      C515     C168     C1085\n",
       "1 0.7265508 0.7264624 0.7263944 0.7263526 0.7262682 0.726178 0.7261704\n",
       "       C547      C777       C51      C352      C990      C452      C266\n",
       "1 0.7260259 0.7258962 0.7258841 0.7258764 0.7257587 0.7257494 0.7257239\n",
       "      C1110      C732       C77     C1153     C703      C498     C1048     C130\n",
       "1 0.7255935 0.7255872 0.7255705 0.7254947 0.725378 0.7252613 0.7252464 0.725246\n",
       "      C1320      C901      C209     C1264       C1       C45      C643\n",
       "1 0.7252022 0.7251865 0.7250204 0.7249654 0.724896 0.7248847 0.7248449\n",
       "        C49      C173      C824       C70      C149     C1252     C1266\n",
       "1 0.7248425 0.7247788 0.7247357 0.7247025 0.7246799 0.7246764 0.7246438\n",
       "       C235      C788      C865      C707     C898     C1030     C1324\n",
       "1 0.7246324 0.7246209 0.7246115 0.7245578 0.724535 0.7244304 0.7242343\n",
       "      C1042     C1159      C180     C1040     C693      C327     C1321\n",
       "1 0.7239729 0.7238959 0.7238867 0.7233977 0.723367 0.7233623 0.7233247\n",
       "       C281     C1093      C853      C784      C992      C868      C818\n",
       "1 0.7231582 0.7231197 0.7230434 0.7229472 0.7229041 0.7228208 0.7227647\n",
       "      C1295     C1046     C1301      C597      C229     C1214      C917\n",
       "1 0.7227563 0.7227126 0.7227007 0.7226906 0.7226747 0.7226316 0.7225092\n",
       "      C1242      C867     C1277      C342     C872    C1097      C964     C1063\n",
       "1 0.7224655 0.7224415 0.7222546 0.7221573 0.722146 0.722094 0.7219863 0.7219596\n",
       "       C934      C726      C325       C69    C1337      C927      C176\n",
       "1 0.7218661 0.7218481 0.7218267 0.7217745 0.721683 0.7216432 0.7216396\n",
       "       C535      C519      C212     C1258      C258      C401      C999\n",
       "1 0.7214982 0.7213431 0.7213139 0.7212271 0.7211434 0.7211424 0.7211385\n",
       "       C609     C1226      C161      C103      C264      C993     C1232\n",
       "1 0.7209745 0.7209169 0.7208147 0.7207948 0.7207845 0.7207643 0.7207557\n",
       "       C604       C47     C1247      C413      C804      C256      C228\n",
       "1 0.7207462 0.7207387 0.7207313 0.7206826 0.7204869 0.7204686 0.7204651\n",
       "       C340      C255      C529      C125      C965     C422      C887\n",
       "1 0.7204415 0.7202761 0.7202601 0.7202058 0.7201258 0.720008 0.7198415\n",
       "       C790      C216      C489     C1290       C88      C223       C11\n",
       "1 0.7198257 0.7198125 0.7197592 0.7197108 0.7197048 0.7195349 0.7194908\n",
       "      C1281      C459      C230      C389      C513      C762     C1157\n",
       "1 0.7194812 0.7194719 0.7194481 0.7194405 0.7194187 0.7193191 0.7192875\n",
       "       C658     C1180      C827      C108      C347     C1278      C213\n",
       "1 0.7188737 0.7187835 0.7187383 0.7187253 0.7186469 0.7185432 0.7185239\n",
       "        C66      C763     C1170      C994      C602      C226      C925\n",
       "1 0.7185237 0.7184799 0.7183663 0.7183149 0.7183145 0.7183065 0.7182727\n",
       "       C390      C647      C292     C1191      C328      C478      C785\n",
       "1 0.7182472 0.7182034 0.7181351 0.7181304 0.7181035 0.7180343 0.7179406\n",
       "      C1164      C871      C869      C723     C952     C1267      C156\n",
       "1 0.7178933 0.7178248 0.7177798 0.7177551 0.717751 0.7176986 0.7176813\n",
       "       C218     C1008      C910      C129       C83       C23      C445\n",
       "1 0.7175977 0.7175719 0.7174172 0.7171728 0.7171105 0.7170805 0.7170284\n",
       "       C486      C607      C834      C864      C897     C1056      C718\n",
       "1 0.7170102 0.7169808 0.7166972 0.7164092 0.7163653 0.7163365 0.7163192\n",
       "      C1139     C412     C1274      C920     C637      C225      C775      C808\n",
       "1 0.7161423 0.716037 0.7159036 0.7157712 0.715755 0.7152205 0.7152143 0.7150871\n",
       "      C562      C820      C338      C908      C998      C155      C810\n",
       "1 0.715073 0.7150667 0.7149462 0.7149453 0.7148497 0.7148433 0.7147877\n",
       "      C1185      C499     C664      C475      C237      C416      C381\n",
       "1 0.7146954 0.7145163 0.714487 0.7144051 0.7142384 0.7142098 0.7142019\n",
       "       C158      C944     C1268      C981      C273      C331      C653\n",
       "1 0.7141619 0.7141538 0.7140766 0.7140674 0.7139791 0.7137869 0.7137728\n",
       "      C1167      C280     C1166      C442      C396      C353     C1176\n",
       "1 0.7137065 0.7136951 0.7136383 0.7136252 0.7135556 0.7135518 0.7135348\n",
       "      C882     C1156      C300     C469     C1201      C571     C1343      C669\n",
       "1 0.713501 0.7134683 0.7133731 0.713302 0.7131832 0.7128011 0.7127619 0.7127088\n",
       "       C566      C655      C943      C214     C1162      C682     C1177\n",
       "1 0.7126492 0.7125909 0.7125428 0.7124574 0.7123262 0.7122599 0.7121975\n",
       "       C164     C516     C501     C1094     C1225      C211      C772      C873\n",
       "1 0.7121759 0.712017 0.711811 0.7116925 0.7116609 0.7116311 0.7115285 0.7111539\n",
       "         C3     C1250     C1333      C733      C961     C1123     C1233\n",
       "1 0.7111214 0.7109287 0.7106769 0.7106579 0.7106286 0.7106099 0.7105436\n",
       "      C1135       C44     C550      C341     C645      C141      C583      C159\n",
       "1 0.7105151 0.7104236 0.710413 0.7102728 0.710192 0.7101766 0.7100138 0.7099958\n",
       "      C1143      C960       C74      C778      C382    C1086      C167\n",
       "1 0.7099392 0.7098044 0.7097037 0.7096261 0.7096105 0.709577 0.7094892\n",
       "       C355     C1109      C979      C698     C880      C875      C449\n",
       "1 0.7094579 0.7093945 0.7092651 0.7092171 0.708971 0.7087178 0.7082565\n",
       "       C711      C789     C1325      C720     C1169      C148      C831\n",
       "1 0.7081269 0.7081051 0.7077674 0.7076669 0.7072759 0.7072732 0.7070149\n",
       "       C924     C275      C858      C330      C532      C343      C424\n",
       "1 0.7069317 0.706922 0.7069212 0.7068166 0.7067605 0.7067465 0.7067207\n",
       "       C716       C72     C1152     C1183      C153      C178     C1238\n",
       "1 0.7066548 0.7065266 0.7065162 0.7061791 0.7058851 0.7058771 0.7056066\n",
       "        C58      C372      C781      C392     C1241      C987      C549\n",
       "1 0.7055554 0.7054521 0.7054408 0.7052159 0.7050218 0.7049958 0.7049114\n",
       "       C608      C363     C1181      C678      C598     C774      C494\n",
       "1 0.7048945 0.7046925 0.7045758 0.7045578 0.7044086 0.704217 0.7039853\n",
       "       C361      C750      C953     C832    C1342     C1253      C673     C1104\n",
       "1 0.7039447 0.7038245 0.7037021 0.703631 0.703606 0.7035153 0.7034556 0.7033704\n",
       "      C1256      C623     C1227     C1103     C481      C200       C53\n",
       "1 0.7033493 0.7033337 0.7033017 0.7033001 0.703262 0.7030302 0.7029667\n",
       "       C939     C1151      C811    C1211      C799      C656      C649\n",
       "1 0.7029475 0.7028952 0.7028711 0.702709 0.7025641 0.7024982 0.7024234\n",
       "         C7     C1208      C282      C222      C170      C243      C150\n",
       "1 0.7023728 0.7022805 0.7022539 0.7021303 0.7021204 0.7019117 0.7018275\n",
       "       C806      C175      C162      C198     C160     C1124     C1269\n",
       "1 0.7016392 0.7016337 0.7014179 0.7013676 0.701191 0.7011765 0.7010739\n",
       "       C169      C249      C695     C1245     C1019      C982      C428\n",
       "1 0.7010584 0.7009816 0.7007717 0.7005556 0.7003686 0.7003679 0.7003365\n",
       "      C1340      C642        C9      C429      C191     C1024     C231\n",
       "1 0.7002858 0.7002758 0.7002059 0.7001151 0.7000586 0.6997445 0.699601\n",
       "       C629     C839      C746     C250     C1229       C86      C112      C456\n",
       "1 0.6995186 0.699373 0.6992595 0.699205 0.6991216 0.6991202 0.6990664 0.6990303\n",
       "       C30      C298      C373      C675     C805     C1313      C187      C731\n",
       "1 0.699007 0.6989816 0.6988905 0.6986963 0.698683 0.6986476 0.6985942 0.6984091\n",
       "       C699     C1220     C1308      C100      C195      C923       C59\n",
       "1 0.6982429 0.6981468 0.6979891 0.6978096 0.6977614 0.6976922 0.6976057\n",
       "       C641      C621      C224      C657      C815     C1202      C660\n",
       "1 0.6975431 0.6975341 0.6975082 0.6971343 0.6971101 0.6970732 0.6970658\n",
       "        C26     C1234      C468      C951      C665     C776      C118\n",
       "1 0.6966581 0.6965873 0.6965725 0.6964759 0.6960695 0.696057 0.6957356\n",
       "      C1312      C995      C625      C438      C950     C1003      C380\n",
       "1 0.6956233 0.6954372 0.6953285 0.6951095 0.6949871 0.6949301 0.6948329\n",
       "       C360      C152       C13      C942      C605      C620     C500\n",
       "1 0.6948285 0.6947499 0.6946539 0.6944026 0.6941826 0.6941012 0.693973\n",
       "      C1064     C1087     C1077      C177     C1100     C1334      C277\n",
       "1 0.6939357 0.6938794 0.6938523 0.6937278 0.6933864 0.6931501 0.6931373\n",
       "       C783     C1118     C1193      C551      C959     C303      C142\n",
       "1 0.6930872 0.6930514 0.6928251 0.6926186 0.6925067 0.692349 0.6923158\n",
       "       C976      C425      C558      C197     C1147        C5      C606\n",
       "1 0.6923003 0.6921511 0.6917399 0.6917113 0.6916144 0.6915811 0.6915093\n",
       "       C564      C541     C1230      C457     C691      C759      C896\n",
       "1 0.6914742 0.6909213 0.6907975 0.6906986 0.690504 0.6904957 0.6904862\n",
       "        C32     C577      C654      C593     C1199      C510      C748\n",
       "1 0.6903257 0.690174 0.6900639 0.6900535 0.6900114 0.6899878 0.6899688\n",
       "      C1346      C322       C15      C313     C252     C613      C634      C204\n",
       "1 0.6898377 0.6895236 0.6892427 0.6891936 0.689148 0.688661 0.6885933 0.6881501\n",
       "      C1017      C743      C418       C46      C335      C377      C543\n",
       "1 0.6881135 0.6880546 0.6879719 0.6879095 0.6879056 0.6876898 0.6875504\n",
       "       C957     C217      C393       C57      C450      C592    C1148      C174\n",
       "1 0.6872115 0.687151 0.6871322 0.6870043 0.6868423 0.6868161 0.686631 0.6865577\n",
       "       C336     C307      C484      C553    C1289      C314      C754        C6\n",
       "1 0.6862841 0.686172 0.6860717 0.6857949 0.685532 0.6853169 0.6852123 0.6851733\n",
       "       C803      C186     C1049     C1317      C741      C863      C491\n",
       "1 0.6848819 0.6848219 0.6839699 0.6835398 0.6828781 0.6826866 0.6824793\n",
       "      C1122      C507     C1059     C1240      C626     C1138      C747\n",
       "1 0.6823466 0.6822506 0.6815725 0.6815367 0.6813985 0.6803253 0.6799549\n",
       "        C21      C902     C1095      C931    C1088      C351      C559\n",
       "1 0.6797686 0.6789448 0.6789384 0.6787121 0.678239 0.6778748 0.6775958\n",
       "      C1066     C1288      C437      C409     C1236     C1255       C71\n",
       "1 0.6774592 0.6773075 0.6770167 0.6768748 0.6766678 0.6750192 0.6746039\n",
       "       C538      C569      C163      C248     C366     C932      C768     C1115\n",
       "1 0.6742165 0.6734675 0.6726171 0.6716926 0.671208 0.671065 0.6705245 0.6704519\n",
       "      C1089      C502      C124      C967    C1282      C710       C40\n",
       "1 0.6700866 0.6690288 0.6687425 0.6681562 0.667744 0.6676462 0.6672427\n",
       "       C833      C348      C242    C1038      C154      C132      C937\n",
       "1 0.6663449 0.6649933 0.6638626 0.662448 0.6615145 0.6553058 0.6417201"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my.dl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "        Predicted\n",
       "Actual   A B  C D E G H   Error\n",
       "  A      2 0  1 0 1 0 0 0.50000\n",
       "  B      1 1  0 0 0 0 0 0.50000\n",
       "  C      0 0 23 0 0 0 0 0.00000\n",
       "  D      0 0  0 0 0 0 0     NaN\n",
       "  E      1 0  0 0 1 0 0 0.50000\n",
       "  G      0 0  0 0 0 0 0     NaN\n",
       "  H      1 0  0 0 0 0 0 1.00000\n",
       "  Totals 5 1 24 0 2 0 0 0.15625"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[1] 0.84375"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my.dl@model[[5]]\n",
    "1 - my.dl@model[[7]]  ##  ACC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "   predict            A            B            C            D            E\n",
       "1        A 8.910353e-01 7.370919e-02 4.151384e-04 2.001426e-02 1.469966e-02\n",
       "2        C 6.575533e-02 4.006533e-03 6.922491e-01 1.994550e-03 2.357033e-01\n",
       "3        A 9.916058e-01 3.679702e-05 4.273191e-03 1.494910e-03 2.100528e-03\n",
       "4        E 8.663132e-04 2.727302e-04 1.563110e-02 2.535046e-05 9.809678e-01\n",
       "5        B 6.141900e-03 8.492556e-01 7.596204e-02 8.158414e-04 6.774489e-02\n",
       "6        A 4.749292e-01 3.193418e-02 3.677447e-01 6.163182e-02 5.019460e-02\n",
       "7        C 1.811038e-01 3.085864e-06 8.183260e-01 2.069992e-05 5.096302e-04\n",
       "8        C 5.756019e-03 2.522375e-03 9.888914e-01 1.087247e-03 1.734982e-03\n",
       "9        C 5.930470e-02 3.130129e-05 9.375899e-01 3.312651e-04 2.671929e-03\n",
       "10       C 1.825703e-03 4.023292e-06 9.747251e-01 1.472770e-05 2.341670e-02\n",
       "11       C 1.510467e-01 9.470678e-06 8.479494e-01 2.856549e-04 6.926252e-04\n",
       "12       C 8.396786e-04 9.822395e-07 9.987714e-01 2.709155e-06 3.833199e-04\n",
       "13       C 1.825993e-02 7.534517e-07 9.756461e-01 1.462772e-04 5.910906e-03\n",
       "14       C 1.339226e-02 4.830822e-05 9.745616e-01 5.765637e-05 1.158979e-02\n",
       "15       C 1.749968e-03 3.559534e-08 9.951811e-01 2.040259e-05 2.941091e-03\n",
       "16       C 4.439138e-03 8.681210e-05 9.923633e-01 5.276446e-04 2.578807e-03\n",
       "17       C 1.426087e-03 1.415208e-03 9.861260e-01 3.098939e-03 7.889327e-03\n",
       "18       C 2.582368e-02 6.033398e-06 9.713737e-01 7.237525e-05 2.713803e-03\n",
       "19       C 1.280009e-01 1.309878e-06 8.604778e-01 1.401316e-05 1.134650e-02\n",
       "20       C 4.581249e-04 3.253871e-06 9.955222e-01 1.269852e-06 4.000640e-03\n",
       "21       C 3.556316e-01 4.930092e-04 5.866958e-01 5.074530e-02 6.051867e-03\n",
       "22       C 4.196652e-03 2.281768e-03 9.871616e-01 4.027095e-04 5.698834e-03\n",
       "23       C 7.468453e-04 8.635247e-06 9.991198e-01 2.139740e-06 1.183868e-04\n",
       "24       C 1.859690e-03 1.281740e-04 9.570424e-01 4.032781e-02 4.311664e-04\n",
       "25       C 3.659647e-04 2.134902e-04 9.760911e-01 5.589099e-06 2.302772e-02\n",
       "26       C 8.512945e-04 4.381240e-06 9.776638e-01 1.415095e-05 2.142075e-02\n",
       "27       C 4.565042e-04 2.823675e-06 9.994981e-01 7.607645e-06 2.595043e-05\n",
       "28       C 3.962346e-01 4.569110e-06 5.495145e-01 1.112044e-03 4.062530e-02\n",
       "29       C 2.624084e-03 3.717237e-06 9.965943e-01 2.394627e-04 5.298649e-04\n",
       "30       E 2.945286e-06 1.138345e-07 3.548083e-06 4.594016e-08 9.999933e-01\n",
       "31       A 9.566181e-01 1.836613e-04 3.475539e-02 5.484897e-03 2.308514e-03\n",
       "32       A 9.819630e-01 3.784730e-08 1.782342e-02 1.398177e-04 9.965606e-06\n",
       "              G            H\n",
       "1  1.700104e-05 1.094636e-04\n",
       "2  1.429311e-04 1.482188e-04\n",
       "3  4.132097e-05 4.474512e-04\n",
       "4  2.197825e-03 3.890582e-05\n",
       "5  6.187477e-05 1.789151e-05\n",
       "6  7.559906e-03 6.005612e-03\n",
       "7  3.464324e-05 2.199473e-06\n",
       "8  5.861047e-06 2.077966e-06\n",
       "9  1.776990e-05 5.313415e-05\n",
       "10 6.773403e-06 6.934447e-06\n",
       "11 1.534041e-05 7.933132e-07\n",
       "12 1.854687e-06 3.544071e-08\n",
       "13 1.506946e-05 2.101103e-05\n",
       "14 3.349420e-04 1.544023e-05\n",
       "15 6.000408e-05 4.739714e-05\n",
       "16 3.315862e-06 9.583165e-07\n",
       "17 3.814938e-05 6.259146e-06\n",
       "18 1.024018e-05 1.188076e-07\n",
       "19 6.583014e-05 9.367479e-05\n",
       "20 1.441582e-05 8.449995e-08\n",
       "21 6.863641e-05 3.138040e-04\n",
       "22 2.417184e-04 1.673150e-05\n",
       "23 4.180110e-06 2.196789e-08\n",
       "24 4.772300e-06 2.060103e-04\n",
       "25 2.727209e-04 2.338081e-05\n",
       "26 3.735673e-05 8.244993e-06\n",
       "27 8.901210e-06 6.761185e-08\n",
       "28 1.999702e-04 1.230907e-02\n",
       "29 6.282980e-06 2.298442e-06\n",
       "30 1.389598e-09 3.232724e-08\n",
       "31 1.270388e-04 5.223532e-04\n",
       "32 5.418841e-06 5.831452e-05"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h2o.predict(my.dl, test.hex)%>%as.data.frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " [1] \"주차\"   \"통신\"   \"안전\"   \"목적지\" \"지도\"   \"센서\"   \"운전자\" \"경고\"  \n",
       " [9] \"lf\"     \"속도\"  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colnames(save_data)[str_replace_all(names(my.dl@model[[9]]), \"C\", \"\")%>%as.numeric%>%.[1:10]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  |                                                                            \r",
      "  |                                                                      |   0%\r",
      "  |                                                                            \r",
      "  |====                                                                  |   5%\r",
      "  |                                                                            \r",
      "  |======================================                                |  55%\r",
      "  |                                                                            \r",
      "  |======================================================================| 100%\n"
     ]
    }
   ],
   "source": [
    "my.rf <- h2o.randomForest(x = 1:(ncol(train.hex)-1), y = ncol(train.hex), data = train.hex, validation = test.hex,\n",
    "                         type=\"fast\", \n",
    "                         importance=TRUE, \n",
    "                         ntree=c(5), \n",
    "                         depth=c(5,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IP Address: 127.0.0.1 \n",
      "Port      : 54321 \n",
      "Parsed Data Key: train.hex \n",
      "\n",
      "Grid Search Model Key: GridSearch_854db4fb4e907d8477a4ddd9d52d429b \n",
      "\n",
      "Summary\n",
      "                                 model_key ntrees max_depth nbins\n",
      "1 SpeeDRF_8e1207d25a80c2ab78a9f42b87084dec      5         5  1024\n",
      "2 SpeeDRF_a5396d0c751d639fd2bf68a9c0224c25      5        10  1024\n",
      "  prediction_error run_time\n",
      "1             0.25     5893\n",
      "2          0.34375     6311\n"
     ]
    }
   ],
   "source": [
    "print(my.rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "        Predicted\n",
       "Actual   A B  C D E G H Error\n",
       "  A      1 0  2 0 1 0 0  0.75\n",
       "  B      0 0  2 0 0 0 0  1.00\n",
       "  C      0 0 23 0 0 0 0  0.00\n",
       "  D      0 0  0 0 0 0 0   NaN\n",
       "  E      0 0  2 0 0 0 0  1.00\n",
       "  G      0 0  0 0 0 0 0   NaN\n",
       "  H      0 0  1 0 0 0 0  1.00\n",
       "  Totals 1 0 30 0 1 0 0  0.25"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my.rf@model[[1]]@model$confusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 0.75\n",
      "[1] 0.65625\n"
     ]
    }
   ],
   "source": [
    "print(1 - my.rf@sumtable[[1]]$prediction_error)\n",
    "print(1 - my.rf@sumtable[[2]]$prediction_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Are you sure you want to shutdown the H2O instance running at http://127.0.0.1:54321 (Y/N)? \n"
     ]
    }
   ],
   "source": [
    "h2o.shutdown(h2oServer)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.1.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}