{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## CIFAR 10" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "%reload_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from fastai.conv_learner import *\n", "PATH = Path(\"data/cifar10/\")\n", "# PATH = Path(\"/home/ubuntu/data/cifar10/\")\n", "os.makedirs(PATH,exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from torchvision import transforms, datasets" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "torch.cuda.set_device(0)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n", "stats = (np.array([ 0.4914 , 0.48216, 0.44653]), np.array([ 0.24703, 0.24349, 0.26159]))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "bs=256" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "num_workers = num_cpus()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "num_workers = 16" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "traindir = str(PATH/'train')\n", "valdir = str(PATH/'test')\n", "tfms = [transforms.ToTensor(),\n", " transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]\n", "\n", "train_dataset = datasets.ImageFolder(\n", " traindir,\n", " transforms.Compose([\n", " transforms.RandomCrop(32, padding=4),\n", " transforms.RandomHorizontalFlip(),\n", " ] + tfms))\n", "\n", "train_loader = torch.utils.data.DataLoader(\n", " train_dataset, batch_size=bs, shuffle=True, num_workers=num_workers, pin_memory=True)\n", "\n", "val_dataset = datasets.ImageFolder(valdir, transforms.Compose(tfms))\n", "\n", "val_loader = torch.utils.data.DataLoader(\n", " val_dataset, batch_size=bs*2, shuffle=False, num_workers=num_workers, pin_memory=True)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "data = ModelData(PATH, train_loader, val_loader)\n", "data.sz=32" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "from models.wideresnet import WideResNet" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "m = WideResNet(3, 3, num_classes=10, k=6, drop_p=0.)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "m = nn.DataParallel(m, [0,1,2,3])" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "lr = 1.3" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "learn = ConvLearner.from_model_data(m, data)\n", "learn.crit = nn.CrossEntropyLoss()\n", "learn.metrics = [accuracy]\n", "wd=1e-4" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 22, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "64f26cb5419b404b9072e3520ba53cb0", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.33707 1.11173 0.594 \n", "\n" ] }, { "data": { "text/plain": [ "[array([1.11173]), 0.5940000004768372]" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# DP: m = WideResNet(depth=22, num_classes=10, widen_factor=6, dropRate=0.)\n", "learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "851a33350dfb4ab2b49594c3a19e0449", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.179957 1.432369 0.5427 \n", " 1 0.958488 1.338229 0.5676 \n", " 2 0.844668 0.902744 0.6965 \n", " 8%|▊ | 16/196 [00:02<00:22, 7.94it/s, loss=0.842]" ] } ], "source": [ "%time learn.fit(lr, 1, wds=wd, cycle_len=30, use_clr_beta=(20, 20, 0.95, 0.85))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "15d5973799a343df9534f932d048bdc4", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.294204 1.058497 0.6254 \n", "\n" ] }, { "data": { "text/plain": [ "[array([1.0585]), 0.6254]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# DP: m = WideResNet(depth=22, num_classes=10, widen_factor=6, dropRate=0.)\n", "learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b986659a675b4470a043605f06632f5d", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.057594 1.163891 0.6072 \n", " 1 0.791428 0.859953 0.7078 \n", " 2 0.642778 0.884299 0.7168 \n", " 3 0.586491 0.85431 0.7149 \n", " 4 0.530411 0.724244 0.7607 \n", " 5 0.492245 0.718871 0.764 \n", " 6 0.465657 0.546274 0.8112 \n", " 7 0.437823 0.536493 0.8182 \n", " 8 0.440909 0.69369 0.7729 \n", " 9 0.408925 1.115436 0.7126 \n", " 10 0.401172 0.902935 0.733 \n", " 11 0.397317 0.690258 0.7921 \n", " 12 0.376588 0.514558 0.8287 \n", " 13 0.366199 0.442919 0.8527 \n", " 14 0.345316 0.796473 0.7753 \n", " 15 0.333985 0.405802 0.8646 \n", " 16 0.306255 0.492593 0.8455 \n", " 17 0.307262 0.405131 0.8656 \n", " 18 0.285951 0.504936 0.8388 \n", " 19 0.262031 0.564698 0.8289 \n", " 20 0.255914 0.602085 0.8234 \n", " 21 0.20444 0.316775 0.9016 \n", " 22 0.161276 0.281515 0.9096 \n", " 23 0.110206 0.221838 0.929 \n", " 24 0.071019 0.203805 0.9368 \n", " 25 0.056447 0.220052 0.9346 \n", " 26 0.047433 0.206167 0.9388 \n", " 27 0.0333 0.212472 0.94 \n", " 28 0.025766 0.209915 0.9423 \n", " 29 0.021697 0.207422 0.9427 \n", "\n", "CPU times: user 12min 33s, sys: 6min 30s, total: 19min 4s\n", "Wall time: 15min 12s\n" ] }, { "data": { "text/plain": [ "[array([0.20742]), 0.9427]" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%time learn.fit(lr, 1, wds=wd, cycle_len=30, use_clr_beta=(20, 20, 0.95, 0.85))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b25c2101021d4818b3a66eca4e588da0", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.290646 1.08506 0.6043 \n", "\n" ] }, { "data": { "text/plain": [ "[array([1.08506]), 0.6043]" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "0c39738caa8a411ea2afbd4d82ca9e2b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.084963 1.085174 0.6082 \n", " 1 0.825962 1.001847 0.6643 \n", " 2 0.702493 0.930496 0.6788 \n", " 3 0.593327 0.76002 0.7503 \n", " 4 0.543732 0.654882 0.7788 \n", " 5 0.503807 0.954524 0.7058 \n", " 6 0.460451 0.520503 0.8216 \n", " 7 0.429511 0.73571 0.7748 \n", " 8 0.419969 0.555502 0.8179 \n", " 9 0.415771 0.593286 0.8055 \n", " 10 0.40077 0.642513 0.8029 \n", " 11 0.385437 0.965159 0.7204 \n", " 12 0.383747 0.691304 0.783 \n", " 13 0.35558 0.705928 0.774 \n", " 14 0.355323 0.631229 0.8075 \n", " 15 0.354718 0.796365 0.7543 \n", " 16 0.348135 0.771605 0.7833 \n", " 17 0.330313 0.57747 0.819 \n", " 18 0.319511 0.58501 0.8249 \n", " 19 0.305789 0.553965 0.8239 \n", " 20 0.309558 0.632576 0.794 \n", " 21 0.312331 0.490521 0.8399 \n", " 22 0.292812 0.740069 0.7778 \n", " 23 0.277894 0.671362 0.8029 \n", " 24 0.273641 0.489029 0.8439 \n", " 25 0.262769 0.492826 0.8458 \n", " 26 0.262545 0.417226 0.8648 \n", " 27 0.24185 0.522333 0.8441 \n", " 28 0.235303 0.573098 0.8327 \n", " 29 0.223754 0.468723 0.8581 \n", " 30 0.212939 0.421378 0.8665 \n", " 31 0.168992 0.31097 0.9021 \n", " 32 0.162031 0.339769 0.8921 \n", " 33 0.11894 0.280535 0.9163 \n", " 34 0.091013 0.250175 0.9251 \n", " 35 0.067888 0.2331 0.9318 \n", " 36 0.047799 0.226535 0.9368 \n", " 37 0.030331 0.216385 0.9405 \n", " 38 0.020049 0.204373 0.9429 \n", " 39 0.012407 0.20191 0.9448 \n", "\n", "CPU times: user 31min 18s, sys: 8min 10s, total: 39min 28s\n", "Wall time: 39min 28s\n" ] }, { "data": { "text/plain": [ "[array([0.20191]), 0.9448]" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%time learn.fit(lr, 1, wds=wd, cycle_len=40, use_clr_beta=(10, 15, 0.95, 0.85))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d124cf1084f8413c98109d51f2500695", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.264693 1.044528 0.6226 \n", "\n" ] }, { "data": { "text/plain": [ "[array([1.04453]), 0.6225999996185303]" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "scrolled": true }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "702050924bb5495eb84e724d350f9397", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.033732 1.263449 0.5967 \n", " 1 0.804623 1.398098 0.5774 \n", " 2 0.669485 0.68934 0.763 \n", " 6%|▌ | 11/196 [00:01<00:23, 7.82it/s, loss=0.658]" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Process Process-137:\n", "Process Process-129:\n", "Process Process-142:\n", "Process Process-139:\n", "Process Process-134:\n", "Process Process-141:\n", "Process Process-140:\n", "Process Process-143:\n", "Process Process-133:\n", "Process Process-138:\n", "Process Process-131:\n", "Process Process-135:\n", "Process Process-132:\n", "Process Process-144:\n", "Process Process-130:\n", "Process Process-136:\n", "Traceback (most recent call last):\n", "ERROR:root:Internal Python error in the inspect module.\n", "Below is the traceback from this internal error.\n", "\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", "Traceback (most recent call last):\n", "Traceback (most recent call last):\n", "Traceback (most recent call last):\n", "Traceback (most recent call last):\n", "Traceback (most recent call last):\n", "Traceback (most recent call last):\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", "Traceback (most recent call last):\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/IPython/core/magics/execution.py\", line 1230, in time\n", " out = eval(code, glob, local_ns)\n", " File \"\", line 1, in \n", " File \"/data1/jhoward/git/fastai/fastai/learner.py\", line 259, in fit\n", " return self.fit_gen(self.model, self.data, layer_opt, n_cycle, **kwargs)\n", " File \"/data1/jhoward/git/fastai/fastai/learner.py\", line 206, in fit_gen\n", " metrics=metrics, callbacks=callbacks, reg_fn=self.reg_fn, clip=self.clip, fp16=self.fp16, **kwargs)\n", " File \"/data1/jhoward/git/fastai/fastai/model.py\", line 120, in fit\n", " loss = stepper.step(V(x),V(y), epoch)\n", " File \"/data1/jhoward/git/fastai/fastai/model.py\", line 64, in step\n", " return torch_item(raw_loss.data)\n", " File \"/data1/jhoward/git/fastai/fastai/model.py\", line 28, in torch_item\n", " def torch_item(x): return x.item() if hasattr(x,'item') else x[0]\n", "KeyboardInterrupt\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 1828, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 1090, in get_records\n", " return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 311, in wrapped\n", " return f(*args, **kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 345, in _fixed_getinnerframes\n", " records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n", " File \"/home/jhoward/anaconda3/lib/python3.6/inspect.py\", line 1483, in getinnerframes\n", " frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/inspect.py\", line 1441, in getframeinfo\n", " filename = getsourcefile(frame) or getfile(frame)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/inspect.py\", line 696, in getsourcefile\n", " if getattr(getmodule(object, filename), '__loader__', None) is not None:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/inspect.py\", line 739, in getmodule\n", " f = getabsfile(module)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/inspect.py\", line 708, in getabsfile\n", " _filename = getsourcefile(object) or getfile(object)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/inspect.py\", line 693, in getsourcefile\n", " if os.path.exists(filename):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/genericpath.py\", line 19, in exists\n", " os.stat(path)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 175, in handler\n", " _error_if_any_worker_fails()\n", "RuntimeError: DataLoader worker (pid 5661) exited unexpectedly with exit code 1.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Traceback (most recent call last):\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", "Traceback (most recent call last):\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n", " r = index_queue.get()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", "KeyboardInterrupt\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in _worker_loop\n", " samples = collate_fn([dataset[i] for i in batch_indices])\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 335, in get\n", " res = self._reader.recv_bytes()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n", " with self._rlock:\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", "KeyboardInterrupt\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", "KeyboardInterrupt\n", "KeyboardInterrupt\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", "KeyboardInterrupt\n", "KeyboardInterrupt\n", "KeyboardInterrupt\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in \n", " samples = collate_fn([dataset[i] for i in batch_indices])\n", "KeyboardInterrupt\n", "KeyboardInterrupt\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n", " buf = self._recv_bytes(maxlength)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n", " return self._semlock.__enter__()\n", "KeyboardInterrupt\n", "KeyboardInterrupt\n", "KeyboardInterrupt\n", "KeyboardInterrupt\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torchvision/datasets/folder.py\", line 122, in __getitem__\n", " img = self.loader(path)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n", " buf = self._recv(4)\n", "KeyboardInterrupt\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torchvision/datasets/folder.py\", line 69, in default_loader\n", " return pil_loader(path)\n", " File \"/home/jhoward/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n", " chunk = read(handle, remaining)\n", "KeyboardInterrupt\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/torchvision/datasets/folder.py\", line 52, in pil_loader\n", " return img.convert('RGB')\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/PIL/Image.py\", line 879, in convert\n", " self.load()\n", " File \"/home/jhoward/anaconda3/lib/python3.6/site-packages/PIL/ImageFile.py\", line 242, in load\n", " self.load_end()\n", "KeyboardInterrupt\n" ] } ], "source": [ "%time learn.fit(1., 1, wds=wd, cycle_len=30, use_clr_beta=(10, 25, 0.95, 0.85))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5339f981678e4f309b2dfa7d4f3f17d8", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.272734 1.096014 0.6024 \n", "\n" ] }, { "data": { "text/plain": [ "[array([1.09601]), 0.6024]" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "48533ca4b34345ccb681c9d6ebc04fda", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.05031 1.058887 0.6364 \n", " 1 0.784812 1.010626 0.6695 \n", " 2 0.662347 0.962157 0.6972 \n", " 3 0.575188 0.63956 0.7755 \n", " 4 0.516394 0.830657 0.7348 \n", " 5 0.465992 0.802316 0.7552 \n", " 6 0.437692 0.666769 0.7819 \n", " 7 0.421664 0.699891 0.7839 \n", " 8 0.409448 0.716692 0.7687 \n", " 9 0.378664 0.57033 0.8045 \n", " 10 0.378573 0.713038 0.7787 \n", " 11 0.358377 0.843966 0.7419 \n", " 12 0.328598 0.72601 0.7865 \n", " 13 0.32008 0.702174 0.7723 \n", " 14 0.299527 0.478337 0.8429 \n", " 15 0.292695 0.692018 0.7925 \n", " 16 0.279109 0.502219 0.8366 \n", " 17 0.252726 0.3686 0.8795 \n", " 18 0.243707 0.375788 0.8743 \n", " 19 0.220037 0.447402 0.8516 \n", " 20 0.200196 0.408494 0.8703 \n", " 21 0.145898 0.295205 0.9067 \n", " 22 0.10881 0.255926 0.9216 \n", " 23 0.086912 0.239246 0.9252 \n", " 24 0.065587 0.299195 0.9173 \n", " 25 0.058427 0.239054 0.9322 \n", " 26 0.035484 0.233135 0.9357 \n", " 27 0.026134 0.229944 0.9382 \n", " 28 0.018713 0.225882 0.9424 \n", " 29 0.012363 0.228542 0.9416 \n", "\n", "CPU times: user 23min 22s, sys: 6min 6s, total: 29min 28s\n", "Wall time: 29min 27s\n" ] }, { "data": { "text/plain": [ "[array([0.22854]), 0.9416]" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%time learn.fit(1., 1, wds=wd, cycle_len=30, use_clr_beta=(10, 25, 0.95, 0.85))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 15, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b9082e96f04e44b082598f8c4bf3acfb", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.428835 1.412945 0.5194 \n", " 1 0.995178 2.106482 0.4173 \n", " 2 0.783325 1.311001 0.6189 \n", " 3 0.650752 0.948611 0.6966 \n", " 4 0.572025 0.611907 0.7906 \n", " 5 0.540067 0.853141 0.7339 \n", " 6 0.479793 0.552199 0.8167 \n", " 7 0.457727 0.951997 0.7204 \n", " 8 0.429621 0.825224 0.7421 \n", " 9 0.412101 1.17335 0.6826 \n", " 10 0.4009 0.808628 0.7315 \n", " 11 0.383221 0.60638 0.804 \n", " 12 0.36901 0.647075 0.7874 \n", " 13 0.368189 0.741168 0.7825 \n", " 14 0.366653 0.653589 0.7973 \n", " 15 0.355023 0.71174 0.7841 \n", " 16 0.354656 0.581478 0.8107 \n", " 17 0.335075 0.602485 0.8103 \n", " 18 0.336688 0.677685 0.7901 \n", " 19 0.318256 0.670932 0.7865 \n", " 20 0.30799 0.627659 0.8039 \n", " 21 0.298475 0.429212 0.8597 \n", " 22 0.297467 0.565836 0.8317 \n", " 23 0.273995 0.513813 0.8408 \n", " 24 0.270703 0.557105 0.822 \n", " 25 0.259869 0.370845 0.8781 \n", " 26 0.242922 0.517631 0.8333 \n", " 27 0.22433 0.424905 0.8628 \n", " 28 0.217046 0.460085 0.8528 \n", " 29 0.179096 0.329448 0.8916 \n", " 30 0.15454 0.34805 0.892 \n", " 31 0.125648 0.262553 0.919 \n", " 32 0.0801 0.244773 0.9244 \n", " 33 0.047182 0.206418 0.9396 \n", " 34 0.082104 0.277417 0.9182 \n", " 35 0.070456 0.292839 0.9197 \n", " 36 0.049607 0.245121 0.9325 \n", " 37 0.03186 0.245641 0.9357 \n", " 38 0.017482 0.21458 0.9405 \n", " 39 0.012877 0.207285 0.9427 \n", "\n", "CPU times: user 31min 8s, sys: 8min 12s, total: 39min 20s\n", "Wall time: 39min 22s\n" ] }, { "data": { "text/plain": [ "[array([0.20728]), 0.9427]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%time learn.fit(lr, 1, wds=wd, cycle_len=40, use_clr_beta=(100, 15, 0.95, 0.85))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 51, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4f4da4e01ebd4a999fbce8c10cd02d33", "version_major": 2, "version_minor": 0 }, "text/plain": [ "A Jupyter Widget" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "epoch trn_loss val_loss accuracy \n", " 0 1.533084 1.725181 0.4627 \n", " 1 1.224625 1.241789 0.5727 \n", " 2 0.995259 1.005162 0.6476 \n", " 3 0.865179 0.949657 0.6642 \n", " 4 0.756122 0.854677 0.6968 \n", " 5 0.692097 1.110497 0.6578 \n", " 6 0.635014 0.805524 0.7227 \n", " 7 0.588518 0.759759 0.7334 \n", " 8 0.567764 0.868924 0.7131 \n", " 9 0.547826 0.700656 0.7645 \n", " 10 0.524676 1.005113 0.6889 \n", " 11 0.50653 0.721323 0.7645 \n", " 12 0.493718 1.125408 0.6608 \n", " 13 0.479434 0.756994 0.7639 \n", " 14 0.475674 0.73913 0.7589 \n", " 15 0.464452 0.612312 0.7955 \n", " 16 0.453685 0.772014 0.757 \n", " 17 0.436029 0.60522 0.7943 \n", " 18 0.437321 0.555058 0.8158 \n", " 19 0.439846 0.819791 0.7449 \n", " 20 0.420495 0.994983 0.719 \n", " 21 0.416594 0.687188 0.7813 \n", " 22 0.413399 0.714974 0.7787 \n", " 23 0.421343 0.696471 0.7761 \n", " 24 0.41174 0.853185 0.7445 \n", " 25 0.411808 0.693145 0.7781 \n", " 26 0.412166 0.847656 0.7456 \n", " 27 0.402742 0.73174 0.772 \n", " 28 0.391636 0.685092 0.7868 \n", " 29 0.384671 0.635394 0.7931 \n", " 30 0.364357 0.856764 0.7271 \n", " 31 0.374435 0.490243 0.8325 \n", " 32 0.364152 0.685217 0.7872 \n", " 33 0.361441 0.724616 0.7843 \n", " 34 0.344948 0.541638 0.8189 \n", " 35 0.341661 0.604952 0.8152 \n", " 36 0.337969 0.571531 0.8172 \n", " 37 0.328699 0.55272 0.8177 \n", " 38 0.32664 0.429266 0.8554 \n", " 39 0.316233 0.424243 0.8555 \n", " 40 0.302454 0.455984 0.8502 \n", " 41 0.296169 0.61181 0.8123 \n", " 42 0.283048 0.572225 0.8267 \n", " 43 0.275228 0.453885 0.853 \n", " 44 0.273048 0.408815 0.863 \n", " 45 0.254404 0.397202 0.8715 \n", " 46 0.219166 0.403471 0.868 \n", " 47 0.215263 0.323341 0.8928 \n", " 48 0.192285 0.37336 0.8824 \n", " 49 0.163661 0.270863 0.9095 \n", " 50 0.118515 0.269602 0.9151 \n", " 51 0.089315 0.209591 0.9317 \n", " 52 0.058886 0.212586 0.9339 \n", " 53 0.05148 0.212392 0.9345 \n", " 54 0.046729 0.232031 0.9343 \n", " 55 0.038997 0.231949 0.9349 \n", " 56 0.035254 0.233632 0.9349 \n", " 57 0.03046 0.232361 0.937 \n", " 58 0.027203 0.22916 0.94 \n", " 59 0.020285 0.231641 0.9401 \n", " 60 0.017448 0.23432 0.9405 \n", " 61 0.016971 0.232452 0.9415 \n", " 62 0.011784 0.23313 0.9416 \n", " 63 0.011399 0.233199 0.9432 \n", " 64 0.009589 0.233732 0.9422 \n", "\n", "CPU times: user 1h 5min 54s, sys: 16min 4s, total: 1h 21min 59s\n", "Wall time: 57min 23s\n" ] }, { "data": { "text/plain": [ "[array([0.23373]), 0.9422]" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# darknet 2222 lr 1.3 65 cl\n", "%time learn.fit(lr, 1, wds=wd, cycle_len=65, use_clr_beta=(30, 20, 0.95, 0.85))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" }, "toc": { "colors": { "hover_highlight": "#DAA520", "navigate_num": "#000000", "navigate_text": "#333333", "running_highlight": "#FF0000", "selected_highlight": "#FFD700", "sidebar_border": "#EEEEEE", "wrapper_background": "#FFFFFF" }, "moveMenuLeft": true, "nav_menu": { "height": "266px", "width": "252px" }, "navigate_menu": true, "number_sections": true, "sideBar": true, "threshold": 4, "toc_cell": false, "toc_section_display": "block", "toc_window_display": false, "widenNotebook": false } }, "nbformat": 4, "nbformat_minor": 2 }