{ "cells": [ { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.010061, "end_time": "2020-02-18T22:36:11.168235", "exception": false, "start_time": "2020-02-18T22:36:11.158174", "status": "completed" }, "tags": [] }, "source": [ "# Job Time Statistics" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "papermill": { "duration": 6.352598, "end_time": "2020-02-18T22:36:17.529953", "exception": false, "start_time": "2020-02-18T22:36:11.177355", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "import pandas as pd\n", "import subprocess\n", "import sys\n", "import datetime as dt\n", "from io import StringIO" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "papermill": { "duration": 0.01505, "end_time": "2020-02-18T22:36:17.554344", "exception": false, "start_time": "2020-02-18T22:36:17.539294", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "job_name = 'umui-rerun'\n", "start_date = '2020-02-01'" ] }, { "cell_type": "markdown", "metadata": { "papermill": { "duration": 0.007975, "end_time": "2020-02-18T22:36:17.570127", "exception": false, "start_time": "2020-02-18T22:36:17.562152", "status": "completed" }, "tags": [] }, "source": [ "The `sacct` command will let us get info for job names:" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "papermill": { "duration": 0.0145, "end_time": "2020-02-18T22:36:17.592978", "exception": false, "start_time": "2020-02-18T22:36:17.578478", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "cmd = [\n", " 'sacct', '-P', '-o', 'JobID,Account,State,AllocCPUS,CpuTime,CpuTimeRAW,Elapsed,ElapsedRAW',\n", " '--name', job_name,\n", " '-S', start_date,\n", "]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "papermill": { "duration": 0.520463, "end_time": "2020-02-18T22:36:18.121754", "exception": false, "start_time": "2020-02-18T22:36:17.601291", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", " | JobID | \n", "Account | \n", "State | \n", "AllocCPUS | \n", "CPUTime | \n", "CPUTimeRAW | \n", "Elapsed | \n", "ElapsedRaw | \n", "
---|---|---|---|---|---|---|---|---|
0 | \n", "210506 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:16 | \n", "196 | \n", "00:00:07 | \n", "7 | \n", "
1 | \n", "210506.batch | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:16 | \n", "196 | \n", "00:00:07 | \n", "7 | \n", "
2 | \n", "210507 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:16 | \n", "196 | \n", "00:00:07 | \n", "7 | \n", "
3 | \n", "210507.batch | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:16 | \n", "196 | \n", "00:00:07 | \n", "7 | \n", "
4 | \n", "210508 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "11-20:35:48 | \n", "1024548 | \n", "10:09:51 | \n", "36591 | \n", "
... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
62 | \n", "210930 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:07:00 | \n", "420 | \n", "00:00:15 | \n", "15 | \n", "
63 | \n", "210930.batch | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:07:00 | \n", "420 | \n", "00:00:15 | \n", "15 | \n", "
64 | \n", "210931 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:10:44 | \n", "644 | \n", "00:00:23 | \n", "23 | \n", "
65 | \n", "210975 | \n", "mekstrand | \n", "COMPLETED | \n", "28 | \n", "25-06:19:28 | \n", "2182768 | \n", "21:39:16 | \n", "77956 | \n", "
66 | \n", "210975.batch | \n", "mekstrand | \n", "COMPLETED | \n", "28 | \n", "25-06:19:28 | \n", "2182768 | \n", "21:39:16 | \n", "77956 | \n", "
67 rows × 8 columns
\n", "\n", " | JobID | \n", "Account | \n", "State | \n", "AllocCPUS | \n", "CPUTime | \n", "CPUTimeRAW | \n", "Elapsed | \n", "ElapsedRaw | \n", "
---|---|---|---|---|---|---|---|---|
0 | \n", "210506 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:16 | \n", "196 | \n", "00:00:07 | \n", "7 | \n", "
2 | \n", "210507 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:16 | \n", "196 | \n", "00:00:07 | \n", "7 | \n", "
4 | \n", "210508 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "11-20:35:48 | \n", "1024548 | \n", "10:09:51 | \n", "36591 | \n", "
6 | \n", "210564 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "04:03:36 | \n", "14616 | \n", "00:08:42 | \n", "522 | \n", "
8 | \n", "210594 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:04:40 | \n", "280 | \n", "00:00:10 | \n", "10 | \n", "
10 | \n", "210595 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "11-16:12:36 | \n", "1008756 | \n", "10:00:27 | \n", "36027 | \n", "
12 | \n", "210701 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:04:40 | \n", "280 | \n", "00:00:10 | \n", "10 | \n", "
14 | \n", "210704 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:09:48 | \n", "588 | \n", "00:00:21 | \n", "21 | \n", "
16 | \n", "210705 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:04:12 | \n", "252 | \n", "00:00:09 | \n", "9 | \n", "
18 | \n", "210712 | \n", "mekstrand | \n", "CANCELLED by 1051 | \n", "28 | \n", "06:39:00 | \n", "23940 | \n", "00:14:15 | \n", "855 | \n", "
20 | \n", "210713 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:02:20 | \n", "140 | \n", "00:00:05 | \n", "5 | \n", "
22 | \n", "210714 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:02:20 | \n", "140 | \n", "00:00:05 | \n", "5 | \n", "
24 | \n", "210715 | \n", "mekstrand | \n", "CANCELLED by 1051 | \n", "28 | \n", "00:33:36 | \n", "2016 | \n", "00:01:12 | \n", "72 | \n", "
26 | \n", "210716 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:02:48 | \n", "168 | \n", "00:00:06 | \n", "6 | \n", "
28 | \n", "210717 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "1-10:20:20 | \n", "123620 | \n", "01:13:35 | \n", "4415 | \n", "
30 | \n", "210789 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:57:24 | \n", "3444 | \n", "00:02:03 | \n", "123 | \n", "
32 | \n", "210790 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:44 | \n", "224 | \n", "00:00:08 | \n", "8 | \n", "
33 | \n", "210791 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:16 | \n", "196 | \n", "00:00:07 | \n", "7 | \n", "
34 | \n", "210792 | \n", "mekstrand | \n", "CANCELLED by 1051 | \n", "28 | \n", "00:01:52 | \n", "112 | \n", "00:00:04 | \n", "4 | \n", "
35 | \n", "210793 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "11:15:44 | \n", "40544 | \n", "00:24:08 | \n", "1448 | \n", "
37 | \n", "210883 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:22:52 | \n", "1372 | \n", "00:00:49 | \n", "49 | \n", "
39 | \n", "210884 | \n", "mekstrand | \n", "CANCELLED by 1051 | \n", "28 | \n", "02:26:32 | \n", "8792 | \n", "00:05:14 | \n", "314 | \n", "
41 | \n", "210886 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:12:36 | \n", "756 | \n", "00:00:27 | \n", "27 | \n", "
43 | \n", "210888 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:07:28 | \n", "448 | \n", "00:00:16 | \n", "16 | \n", "
45 | \n", "210891 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:06:32 | \n", "392 | \n", "00:00:14 | \n", "14 | \n", "
47 | \n", "210892 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:07:56 | \n", "476 | \n", "00:00:17 | \n", "17 | \n", "
49 | \n", "210893 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:44 | \n", "224 | \n", "00:00:08 | \n", "8 | \n", "
51 | \n", "210894 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "1-06:56:24 | \n", "111384 | \n", "01:06:18 | \n", "3978 | \n", "
53 | \n", "210923 | \n", "mekstrand | \n", "CANCELLED by 1051 | \n", "28 | \n", "00:00:28 | \n", "28 | \n", "00:00:01 | \n", "1 | \n", "
54 | \n", "210924 | \n", "mekstrand | \n", "COMPLETED | \n", "28 | \n", "00:18:40 | \n", "1120 | \n", "00:00:40 | \n", "40 | \n", "
55 | \n", "210925 | \n", "mekstrand | \n", "COMPLETED | \n", "28 | \n", "00:52:44 | \n", "3164 | \n", "00:01:53 | \n", "113 | \n", "
56 | \n", "210926 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:03:44 | \n", "224 | \n", "00:00:08 | \n", "8 | \n", "
58 | \n", "210927 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:04:12 | \n", "252 | \n", "00:00:09 | \n", "9 | \n", "
60 | \n", "210928 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "3-22:59:24 | \n", "341964 | \n", "03:23:33 | \n", "12213 | \n", "
62 | \n", "210930 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:07:00 | \n", "420 | \n", "00:00:15 | \n", "15 | \n", "
64 | \n", "210931 | \n", "mekstrand | \n", "FAILED | \n", "28 | \n", "00:10:44 | \n", "644 | \n", "00:00:23 | \n", "23 | \n", "
65 | \n", "210975 | \n", "mekstrand | \n", "COMPLETED | \n", "28 | \n", "25-06:19:28 | \n", "2182768 | \n", "21:39:16 | \n", "77956 | \n", "