{ "cells": [ { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.010061, "end_time": "2020-02-18T22:36:11.168235", "exception": false, "start_time": "2020-02-18T22:36:11.158174", "status": "completed" }, "tags": [] }, "source": [ "# Job Time Statistics" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "papermill": { "duration": 6.352598, "end_time": "2020-02-18T22:36:17.529953", "exception": false, "start_time": "2020-02-18T22:36:11.177355", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "import pandas as pd\n", "import subprocess\n", "import sys\n", "import datetime as dt\n", "from io import StringIO" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "papermill": { "duration": 0.01505, "end_time": "2020-02-18T22:36:17.554344", "exception": false, "start_time": "2020-02-18T22:36:17.539294", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "job_name = 'umui-rerun'\n", "start_date = '2020-02-01'" ] }, { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.007975, "end_time": "2020-02-18T22:36:17.570127", "exception": false, "start_time": "2020-02-18T22:36:17.562152", "status": "completed" }, "tags": [] }, "source": [ "The `sacct` command will let us get info for job names:" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "papermill": { "duration": 0.0145, "end_time": "2020-02-18T22:36:17.592978", "exception": false, "start_time": "2020-02-18T22:36:17.578478", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "cmd = [\n", " 'sacct', '-P', '-o', 'JobID,Account,State,AllocCPUS,CpuTime,CpuTimeRAW,Elapsed,ElapsedRAW',\n", " '--name', job_name,\n", " '-S', start_date,\n", "]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "papermill": { "duration": 0.520463, "end_time": "2020-02-18T22:36:18.121754", "exception": false, "start_time": "2020-02-18T22:36:17.601291", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
JobIDAccountStateAllocCPUSCPUTimeCPUTimeRAWElapsedElapsedRaw
0210506mekstrandFAILED2800:03:1619600:00:077
1210506.batchmekstrandFAILED2800:03:1619600:00:077
2210507mekstrandFAILED2800:03:1619600:00:077
3210507.batchmekstrandFAILED2800:03:1619600:00:077
4210508mekstrandFAILED2811-20:35:48102454810:09:5136591
...........................
62210930mekstrandFAILED2800:07:0042000:00:1515
63210930.batchmekstrandFAILED2800:07:0042000:00:1515
64210931mekstrandFAILED2800:10:4464400:00:2323
65210975mekstrandCOMPLETED2825-06:19:28218276821:39:1677956
66210975.batchmekstrandCOMPLETED2825-06:19:28218276821:39:1677956
\n", "

67 rows × 8 columns

\n", "
" ], "text/plain": [ " JobID Account State AllocCPUS CPUTime CPUTimeRAW \\\n", "0 210506 mekstrand FAILED 28 00:03:16 196 \n", "1 210506.batch mekstrand FAILED 28 00:03:16 196 \n", "2 210507 mekstrand FAILED 28 00:03:16 196 \n", "3 210507.batch mekstrand FAILED 28 00:03:16 196 \n", "4 210508 mekstrand FAILED 28 11-20:35:48 1024548 \n", ".. ... ... ... ... ... ... \n", "62 210930 mekstrand FAILED 28 00:07:00 420 \n", "63 210930.batch mekstrand FAILED 28 00:07:00 420 \n", "64 210931 mekstrand FAILED 28 00:10:44 644 \n", "65 210975 mekstrand COMPLETED 28 25-06:19:28 2182768 \n", "66 210975.batch mekstrand COMPLETED 28 25-06:19:28 2182768 \n", "\n", " Elapsed ElapsedRaw \n", "0 00:00:07 7 \n", "1 00:00:07 7 \n", "2 00:00:07 7 \n", "3 00:00:07 7 \n", "4 10:09:51 36591 \n", ".. ... ... \n", "62 00:00:15 15 \n", "63 00:00:15 15 \n", "64 00:00:23 23 \n", "65 21:39:16 77956 \n", "66 21:39:16 77956 \n", "\n", "[67 rows x 8 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "child = subprocess.run(cmd, capture_output=True, encoding='utf8')\n", "if child.returncode:\n", " print(child.stderr, file=sys.stderr)\n", " raise ValueError('child failed with code ' + str(child.returncode))\n", "\n", "acct_text = child.stdout\n", "acct = pd.read_csv(StringIO(acct_text), sep='|')\n", "acct" ] }, { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.00922, "end_time": "2020-02-18T22:36:18.141069", "exception": false, "start_time": "2020-02-18T22:36:18.131849", "status": "completed" }, "tags": [] }, "source": [ "The jobs ending in `.batch` are duplicates - remove:" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "papermill": { "duration": 0.040492, "end_time": "2020-02-18T22:36:18.190441", "exception": false, "start_time": "2020-02-18T22:36:18.149949", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
JobIDAccountStateAllocCPUSCPUTimeCPUTimeRAWElapsedElapsedRaw
0210506mekstrandFAILED2800:03:1619600:00:077
2210507mekstrandFAILED2800:03:1619600:00:077
4210508mekstrandFAILED2811-20:35:48102454810:09:5136591
6210564mekstrandFAILED2804:03:361461600:08:42522
8210594mekstrandFAILED2800:04:4028000:00:1010
10210595mekstrandFAILED2811-16:12:36100875610:00:2736027
12210701mekstrandFAILED2800:04:4028000:00:1010
14210704mekstrandFAILED2800:09:4858800:00:2121
16210705mekstrandFAILED2800:04:1225200:00:099
18210712mekstrandCANCELLED by 10512806:39:002394000:14:15855
20210713mekstrandFAILED2800:02:2014000:00:055
22210714mekstrandFAILED2800:02:2014000:00:055
24210715mekstrandCANCELLED by 10512800:33:36201600:01:1272
26210716mekstrandFAILED2800:02:4816800:00:066
28210717mekstrandFAILED281-10:20:2012362001:13:354415
30210789mekstrandFAILED2800:57:24344400:02:03123
32210790mekstrandFAILED2800:03:4422400:00:088
33210791mekstrandFAILED2800:03:1619600:00:077
34210792mekstrandCANCELLED by 10512800:01:5211200:00:044
35210793mekstrandFAILED2811:15:444054400:24:081448
37210883mekstrandFAILED2800:22:52137200:00:4949
39210884mekstrandCANCELLED by 10512802:26:32879200:05:14314
41210886mekstrandFAILED2800:12:3675600:00:2727
43210888mekstrandFAILED2800:07:2844800:00:1616
45210891mekstrandFAILED2800:06:3239200:00:1414
47210892mekstrandFAILED2800:07:5647600:00:1717
49210893mekstrandFAILED2800:03:4422400:00:088
51210894mekstrandFAILED281-06:56:2411138401:06:183978
53210923mekstrandCANCELLED by 10512800:00:282800:00:011
54210924mekstrandCOMPLETED2800:18:40112000:00:4040
55210925mekstrandCOMPLETED2800:52:44316400:01:53113
56210926mekstrandFAILED2800:03:4422400:00:088
58210927mekstrandFAILED2800:04:1225200:00:099
60210928mekstrandFAILED283-22:59:2434196403:23:3312213
62210930mekstrandFAILED2800:07:0042000:00:1515
64210931mekstrandFAILED2800:10:4464400:00:2323
65210975mekstrandCOMPLETED2825-06:19:28218276821:39:1677956
\n", "
" ], "text/plain": [ " JobID Account State AllocCPUS CPUTime CPUTimeRAW \\\n", "0 210506 mekstrand FAILED 28 00:03:16 196 \n", "2 210507 mekstrand FAILED 28 00:03:16 196 \n", "4 210508 mekstrand FAILED 28 11-20:35:48 1024548 \n", "6 210564 mekstrand FAILED 28 04:03:36 14616 \n", "8 210594 mekstrand FAILED 28 00:04:40 280 \n", "10 210595 mekstrand FAILED 28 11-16:12:36 1008756 \n", "12 210701 mekstrand FAILED 28 00:04:40 280 \n", "14 210704 mekstrand FAILED 28 00:09:48 588 \n", "16 210705 mekstrand FAILED 28 00:04:12 252 \n", "18 210712 mekstrand CANCELLED by 1051 28 06:39:00 23940 \n", "20 210713 mekstrand FAILED 28 00:02:20 140 \n", "22 210714 mekstrand FAILED 28 00:02:20 140 \n", "24 210715 mekstrand CANCELLED by 1051 28 00:33:36 2016 \n", "26 210716 mekstrand FAILED 28 00:02:48 168 \n", "28 210717 mekstrand FAILED 28 1-10:20:20 123620 \n", "30 210789 mekstrand FAILED 28 00:57:24 3444 \n", "32 210790 mekstrand FAILED 28 00:03:44 224 \n", "33 210791 mekstrand FAILED 28 00:03:16 196 \n", "34 210792 mekstrand CANCELLED by 1051 28 00:01:52 112 \n", "35 210793 mekstrand FAILED 28 11:15:44 40544 \n", "37 210883 mekstrand FAILED 28 00:22:52 1372 \n", "39 210884 mekstrand CANCELLED by 1051 28 02:26:32 8792 \n", "41 210886 mekstrand FAILED 28 00:12:36 756 \n", "43 210888 mekstrand FAILED 28 00:07:28 448 \n", "45 210891 mekstrand FAILED 28 00:06:32 392 \n", "47 210892 mekstrand FAILED 28 00:07:56 476 \n", "49 210893 mekstrand FAILED 28 00:03:44 224 \n", "51 210894 mekstrand FAILED 28 1-06:56:24 111384 \n", "53 210923 mekstrand CANCELLED by 1051 28 00:00:28 28 \n", "54 210924 mekstrand COMPLETED 28 00:18:40 1120 \n", "55 210925 mekstrand COMPLETED 28 00:52:44 3164 \n", "56 210926 mekstrand FAILED 28 00:03:44 224 \n", "58 210927 mekstrand FAILED 28 00:04:12 252 \n", "60 210928 mekstrand FAILED 28 3-22:59:24 341964 \n", "62 210930 mekstrand FAILED 28 00:07:00 420 \n", "64 210931 mekstrand FAILED 28 00:10:44 644 \n", "65 210975 mekstrand COMPLETED 28 25-06:19:28 2182768 \n", "\n", " Elapsed ElapsedRaw \n", "0 00:00:07 7 \n", "2 00:00:07 7 \n", "4 10:09:51 36591 \n", "6 00:08:42 522 \n", "8 00:00:10 10 \n", "10 10:00:27 36027 \n", "12 00:00:10 10 \n", "14 00:00:21 21 \n", "16 00:00:09 9 \n", "18 00:14:15 855 \n", "20 00:00:05 5 \n", "22 00:00:05 5 \n", "24 00:01:12 72 \n", "26 00:00:06 6 \n", "28 01:13:35 4415 \n", "30 00:02:03 123 \n", "32 00:00:08 8 \n", "33 00:00:07 7 \n", "34 00:00:04 4 \n", "35 00:24:08 1448 \n", "37 00:00:49 49 \n", "39 00:05:14 314 \n", "41 00:00:27 27 \n", "43 00:00:16 16 \n", "45 00:00:14 14 \n", "47 00:00:17 17 \n", "49 00:00:08 8 \n", "51 01:06:18 3978 \n", "53 00:00:01 1 \n", "54 00:00:40 40 \n", "55 00:01:53 113 \n", "56 00:00:08 8 \n", "58 00:00:09 9 \n", "60 03:23:33 12213 \n", "62 00:00:15 15 \n", "64 00:00:23 23 \n", "65 21:39:16 77956 " ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "jobs = acct[~acct['JobID'].str.endswith('.batch')]\n", "jobs" ] }, { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.011089, "end_time": "2020-02-18T22:36:18.211635", "exception": false, "start_time": "2020-02-18T22:36:18.200546", "status": "completed" }, "tags": [] }, "source": [ "What is the CPU time?" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "papermill": { "duration": 0.024597, "end_time": "2020-02-18T22:36:18.246288", "exception": false, "start_time": "2020-02-18T22:36:18.221691", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "datetime.timedelta(days=56, seconds=60284)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cpu = dt.timedelta(seconds=int(jobs['CPUTimeRAW'].sum()))\n", "cpu" ] }, { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.01051, "end_time": "2020-02-18T22:36:18.267128", "exception": false, "start_time": "2020-02-18T22:36:18.256618", "status": "completed" }, "tags": [] }, "source": [ "How many hours?" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "papermill": { "duration": 0.059664, "end_time": "2020-02-18T22:36:18.337242", "exception": false, "start_time": "2020-02-18T22:36:18.277578", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "1360.7455555555555" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cpu.total_seconds() / 3600" ] }, { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.010886, "end_time": "2020-02-18T22:36:18.358926", "exception": false, "start_time": "2020-02-18T22:36:18.348040", "status": "completed" }, "tags": [] }, "source": [ "What is the wall clock time?" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "papermill": { "duration": 0.018364, "end_time": "2020-02-18T22:36:18.388169", "exception": false, "start_time": "2020-02-18T22:36:18.369805", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "datetime.timedelta(days=2, seconds=2153)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wall = dt.timedelta(seconds=int(jobs['ElapsedRaw'].sum()))\n", "wall" ] }, { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.011009, "end_time": "2020-02-18T22:36:18.410382", "exception": false, "start_time": "2020-02-18T22:36:18.399373", "status": "completed" }, "tags": [] }, "source": [ "How many hours?" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "papermill": { "duration": 0.017521, "end_time": "2020-02-18T22:36:18.439532", "exception": false, "start_time": "2020-02-18T22:36:18.422011", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "48.598055555555554" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wall.total_seconds() / 3600" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "papermill": { "duration": 0.011752, "end_time": "2020-02-18T22:36:18.462573", "exception": false, "start_time": "2020-02-18T22:36:18.450821", "status": "completed" }, "tags": [] }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.6" }, "papermill": { "duration": 9.2385, "end_time": "2020-02-18T22:36:19.081638", "environment_variables": {}, "exception": null, "input_path": "JobStats.ipynb", "output_path": "JobStats.temp.ipynb", "parameters": {}, "start_time": "2020-02-18T22:36:09.843138", "version": "1.2.1" } }, "nbformat": 4, "nbformat_minor": 4 }