#!/bin/bash # Initial version by Mats Rynge # Modified by Marco Mambelli - marco@hep.uchciago.edu ############################################################################# # # settings # DATA_DIR=/data/gratia/gratia-summary/data #DATA_DIR=/var/www/html/gratia-summary/data BASE_URL="http://gratiaweb.grid.iu.edu/gratia" VO_ALL="osg|OSG|OSG-Connect" VO="OSG-Connect" # Set to the hostname of the host running the probe (frontend or submit host) #SUBMIT_HOST_PROBE="login01.osgconnect.net" SUBMIT_HOST_PROBE=".*connect.*" # Set to the user used in the GWMS Frontend (from the frontend configuration) #GLIDEIN_USER="fechicago" GLIDEIN_USER="xsede-pilot" # For the query about the submit host you want to include all jobs, also the ones not # executed via the frontend GLIDEIN_USER2=".*" # Uncomment to include plots SHOST_PLOTS="yes" #VO_PLOTS="yes" # FORONTEND_PLOTS="yes" # FORONTEND_PLOTS2="yes" ERROR_PLOTS="yes" NEW_PLOTS="yes" #NEW_PLOTS2="yes" ####### Example configuration for a GWMS Frontend (uc3-vofe) # VO_ALL="UC3|uc3" # VO=$VO_ALL # SUBMIT_HOST_PROBE="uc3-vofe.uchicago.edu" # GLIDEIN_USER="feuc3" # GLIDEIN_USER2=$GLIDEIN_USER # # SHOST_PLOTS="yes" # ERROR_PLOTS="yes" # FORONTEND_PLOTS="yes" # #all other plots commented function help_msg { cat << EOF $0 [ options ] Edit $0 to set the default options (plots, data directory, submit host, glidein user) -a print all available plots -d DIR set an alternative data directory -h print this help message -s SH set the submit host probe to match -u GU set the glidein user to match -i use the OSG ITB Gratia server (http://gratiaweb-itb.grid.iu.edu/gratia) EOF } while getopts hiad:s:u: option do case "${option}" in "h") help_msg; exit 0;; "a") SHOST_PLOTS="yes" VO_PLOTS="yes" FORONTEND_PLOTS="yes" FORONTEND_PLOTS2="yes" ERROR_PLOTS="yes" NEW_PLOTS="yes" NEW_PLOTS2="yes" ;; "d") DATA_DIR=${OPTARG};; "s") SUBMIT_HOST_PROBE=${OPTARG};; "u") GLIDEIN_USER=${OPTARG};; "i") BASE_URL="http://gratiaweb-itb.grid.iu.edu/gratia";; esac done ############################################################################# START_DATE_YEAR=`date --utc -d "last year" +"%Y-%m-%d"` START_DATE_MONTH=`date --utc -d "last month" +"%Y-%m-%d"` START_DATE_WEEK=`date --utc -d "last week" +"%Y-%m-%d"` END_DATE=`date --utc +"%Y-%m-%d"` READABLE_DATE=`TZ=America/Chicago /bin/date` echo "Downloading data for $START_DATE_YEAR to $END_DATE" cd $DATA_DIR function include_graph() { description=$1 filename=$2 query_url=$3 graph_url=$4 wget -q -O $filename "$graph_url" if [ $? != 0 ]; then echo "FAILED: $graph_url" cp ../nodata_text.png $filename fi cat >>index.html.new < $description

Interactive Query EOF } cat >index.html.new < Gratia Summary Graphs

Last update: $READABLE_DATE

EOF if [ ! -z $SHOST_PLOTS ]; then include_graph \ "Hours By Project reported by Gratia" \ "daily_hours_by_project.png" \ "$BASE_URL/xml/project_hours_bar?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" \ "$BASE_URL/bar_graphs/project_hours_bar?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" include_graph \ "Hours By Users reported by Gratia" \ "daily_hours_by_user.png" \ "$BASE_URL/xml/glidein_dn_hours_bar?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" \ "$BASE_URL/glidein_bar_graphs/glidein_dn_hours_bar?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" fi if [ ! -z $VO_PLOTS ]; then include_graph \ "Hours By Project per VO $VO reported by Gratia" \ "daily_hours_by_project-2.png" \ "$BASE_URL/xml/project_hours_bar?facility=.*&probe=.*&resource-type=BatchPilot&vo=$VO&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" \ "$BASE_URL/bar_graphs/project_hours_bar?facility=.*&probe=.*&resource-type=BatchPilot&vo=$VO&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" include_graph \ "Hours By Users in VO $VO reported by Gratia" \ "daily_hours_by_user-2.png" \ "$BASE_URL/xml/glidein_dn_hours_bar?facility=.*&probe=.*&resource-type=BatchPilot&vo=$VO&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" \ "$BASE_URL/glidein_bar_graphs/glidein_dn_hours_bar?facility=.*&probe=.*&resource-type=BatchPilot&vo=$VO&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" fi if [ ! -z $ERROR_PLOTS ]; then include_graph \ "User exit codes as captured by Condor" \ "exit_codes.png" \ "$BASE_URL/xml/glidein_exitcode_bar?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" \ "$BASE_URL/glidein_bar_graphs/glidein_exitcode_bar?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&grid=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE&exclude-user=NONE&includeSuccess=true" fi if [ ! -z $FORONTEND_PLOTS ]; then include_graph \ "Daily Hours by Site, used by the GlideinWMS glideins, across sites" \ "daily_hours_by_site.png" \ "$BASE_URL/xml/dn_site_hours_bar?endtime=$END_DATE+23%3A59%3A59&span=86400&facility=.*&probe=.*&resource-type=%5EBatch%24&vo=$VO_ALL&role=.*&user=$GLIDEIN_USER&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE%7CGeneric%7CObsolete&exclude-user=NONE&includeSuccess=true&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE" \ "$BASE_URL/bar_graphs/dn_site_hours_bar?endtime=$END_DATE+23%3A59%3A59&span=86400&facility=.*&probe=.*&resource-type=%5EBatch%24&vo=$VO_ALL&role=.*&user=$GLIDEIN_USER&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE%7CGeneric%7CObsolete&exclude-user=NONE&includeSuccess=true&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE" include_graph \ "Cumulative Hours by Site, added up by site, over a month" \ "cumulative_hours_by_site.png" \ "$BASE_URL/xml/facility_success_cumulative_smry?span=86400&facility=.*&probe=.*&resource-type=%5EBatch%24&vo=.*&user=$GLIDEIN_USER&starttime=$START_DATE_MONTH+00%3A00%3A00&exclude-facility=NONE%7CGeneric%7CObsolete&exclude-user=NONE&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother" \ "$BASE_URL/cumulative_graphs/facility_success_cumulative_smry?span=86400&facility=.*&probe=.*&resource-type=^Batch$&vo=.*&user=$GLIDEIN_USER&starttime=$START_DATE_MONTH%2000:00:00&exclude-facility=NONE|Generic|Obsolete&exclude-user=NONE&endtime=$END_DATE%2023:59:59&exclude-vo=unknown|other&" fi if [ ! -z $FORONTEND_PLOTS2 ]; then include_graph \ "Daily Hours by Site, used by the GlideinWMS glideins ($VO), across sites" \ "daily_hours_by_site-2.png" \ "$BASE_URL/xml/dn_site_hours_bar?endtime=$END_DATE+23%3A59%3A59&span=86400&facility=.*&probe=.*&resource-type=%5EBatch%24&vo=$VO&role=.*&user=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE%7CGeneric%7CObsolete&exclude-user=NONE&includeSuccess=true&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE" \ "$BASE_URL/bar_graphs/dn_site_hours_bar?endtime=$END_DATE+23%3A59%3A59&span=86400&facility=.*&probe=.*&resource-type=%5EBatch%24&vo=$VO&role=.*&user=$GLIDEIN_USER2&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE%7CGeneric%7CObsolete&exclude-user=NONE&includeSuccess=true&exclude-vo=unknown%7Cother&includeFailed=true&exclude-role=NONE" include_graph \ "Cumulative Hours by Site, added up by site, over a month for VO $VO" \ "cumulative_hours_by_site-2.png" \ "$BASE_URL/xml/facility_success_cumulative_smry?span=86400&facility=.*&probe=.*&resource-type=%5EBatch%24&vo=$VO&user=.*&starttime=$START_DATE_MONTH+00%3A00%3A00&exclude-facility=NONE%7CGeneric%7CObsolete&exclude-user=NONE&endtime=$END_DATE+23%3A59%3A59&exclude-vo=unknown%7Cother" \ "$BASE_URL/cumulative_graphs/facility_success_cumulative_smry?span=86400&facility=.*&probe=.*&resource-type=^Batch$&vo=$VO&user=$GLIDEIN_USER2&starttime=$START_DATE_MONTH%2000:00:00&exclude-facility=NONE|Generic|Obsolete&exclude-user=NONE&endtime=$END_DATE%2023:59:59&exclude-vo=unknown|other&" fi if [ ! -z $NEW_PLOTS ]; then # http://gratiaweb-itb.grid.iu.edu/gratia/xml/project_site_hours_bar?facility=.*&probe=.*connect.*&resource-type=BatchPilot&vo=.*&user=.*&endtime=2013-08-21+23%3A59%3A59&exclude-vo=Unknown|unknown|other&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=2013-08-08+00%3A00%3A00&exclude-facility=NONE|Generic|Obsolete&exclude-user=NONE&includeSuccess=true&exclude-projectname=unknown|other # http://gratiaweb-itb.grid.iu.edu/gratia/bar_graphs/project_site_hours_bar?facility=.*&probe=.*connect.*&resource-type=BatchPilot&vo=.*&user=.*&endtime=2013-08-21%2023:59:59&exclude-vo=Unknown|unknown|other&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=2013-08-08%2000:00:00&exclude-facility=NONE|Generic|Obsolete&exclude-user=NONE&includeSuccess=true&exclude-projectname=unknown|other& include_graph \ "Hours by Project and Site" \ "hours_by_project_and_site.png" \ "$BASE_URL/xml/project_site_hours_bar?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=Unknown|unknown|other&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE|Generic|Obsolete&exclude-user=NONE&includeSuccess=true&exclude-projectname=unknown|other" \ "$BASE_URL/bar_graphs/project_site_hours_bar?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=Unknown|unknown|other&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_WEEK+00%3A00%3A00&exclude-facility=NONE|Generic|Obsolete&exclude-user=NONE&includeSuccess=true&exclude-projectname=unknown|other" fi if [ ! -z $NEW_PLOTS2 ]; then include_graph \ "Cumulative Hours by Site, submitted by OSG-Connect over a month" \ "cumulative_hours_by_site_sub.png" \ "$BASE_URL/xml/facility_success_cumulative_smry?span=86400&facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=Unknown|unknown|other&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_MONTH%2000:00:00&exclude-facility=NONE|Generic|Obsolete&exclude-user=NONE&includeSuccess=true&exclude-projectname=unknown|other" \ "$BASE_URL/cumulative_graphs/facility_success_cumulative_smry?facility=.*&probe=$SUBMIT_HOST_PROBE&resource-type=BatchPilot&vo=.*&user=.*&endtime=$END_DATE+23%3A59%3A59&exclude-vo=Unknown|unknown|other&includeFailed=true&exclude-role=NONE&span=86400&role=.*&starttime=$START_DATE_MONTH%2000:00:00&exclude-facility=NONE|Generic|Obsolete&exclude-user=NONE&includeSuccess=true&exclude-projectname=unknown|other" fi echo >>index.html.new < EOF mv index.html.new ../index.html