#!/usr/bin/env python3
"""psi2log - PSI metrics monitor and logger"""
from argparse import ArgumentParser
from ctypes import CDLL
from signal import SIGHUP, SIGINT, SIGQUIT, SIGTERM, signal
from sys import exit, stdout
from time import monotonic, sleep
def read_path(path):
"""
"""
try:
fd[path].seek(0)
except ValueError:
try:
fd[path] = open(path, 'rb', buffering=0)
except FileNotFoundError as e:
log(e)
return None
except KeyError:
try:
fd[path] = open(path, 'rb', buffering=0)
except FileNotFoundError as e:
log(e)
return None
try:
return fd[path].read(99999).decode()
except OSError as e:
log(e)
fd[path].close()
return None
def form1(num):
"""
"""
s = str(num).split('.')
return '{}.{:0<2}'.format(s[0], s[1])
def form2(num):
"""
"""
s = str(round(num, 1)).split('.')
return '{}.{:0<1}'.format(s[0], s[1])
def signal_handler(signum, frame):
"""
"""
def signal_handler_inner(signum, frame):
pass
for i in sig_list:
signal(i, signal_handler_inner)
if signum == SIGINT:
print('')
lpd = len(peaks_dict)
if lpd == 15:
log('=================================')
log('Peak values: avg10 avg60 avg300')
log('----------- ------ ------ ------')
log('some cpu {:>6} {:>6} {:>6}'.format(
form1(peaks_dict['c_some_avg10']),
form1(peaks_dict['c_some_avg60']),
form1(peaks_dict['c_some_avg300']),
))
log('----------- ------ ------ ------')
log('some io {:>6} {:>6} {:>6}'.format(
form1(peaks_dict['i_some_avg10']),
form1(peaks_dict['i_some_avg60']),
form1(peaks_dict['i_some_avg300']),
))
log('full io {:>6} {:>6} {:>6}'.format(
form1(peaks_dict['i_full_avg10']),
form1(peaks_dict['i_full_avg60']),
form1(peaks_dict['i_full_avg300']),
))
log('----------- ------ ------ ------')
log('some memory {:>6} {:>6} {:>6}'.format(
form1(peaks_dict['m_some_avg10']),
form1(peaks_dict['m_some_avg60']),
form1(peaks_dict['m_some_avg300']),
))
log('full memory {:>6} {:>6} {:>6}'.format(
form1(peaks_dict['m_full_avg10']),
form1(peaks_dict['m_full_avg60']),
form1(peaks_dict['m_full_avg300']),
))
if lpd == 5:
log('----- | ----- ----- | ----- ----- | --------')
log('{:>5} | {:>5} {:>5} | {:>5} {:>5} | peaks'.format(
form2(peaks_dict['avg_cs']),
form2(peaks_dict['avg_is']),
form2(peaks_dict['avg_if']),
form2(peaks_dict['avg_ms']),
form2(peaks_dict['avg_mf'])
))
if target == 'SYSTEM_WIDE':
log_stall_times()
if separate_log:
logging.info('')
exit()
def log_stall_times():
"""
"""
total_cs_1 = psi_file_cpu_to_total(cpu_file)
total_is_1, total_if_1 = psi_file_mem_to_total(io_file)
total_ms_1, total_mf_1 = psi_file_mem_to_total(memory_file)
t = monotonic() - t_0
M = 1000000
dcs = (total_cs_1 - total_cs_0) / M
dis = (total_is_1 - total_is_0) / M
dif = (total_if_1 - total_if_0) / M
dms = (total_ms_1 - total_ms_0) / M
dmf = (total_mf_1 - total_mf_0) / M
if mode == '0' or mode == '1':
log('=================================')
else:
log('--')
log('Stall times for the last {}s:'.format(round(t, 1)))
log('-----------')
log('some cpu {}s, avg {}%'.format(
round(dcs, 1),
round(dcs / t * 100, 1)
))
log('-----------')
log('some io {}s, avg {}%'.format(
round(dis, 1),
round(dis / t * 100, 1)
))
log('full io {}s, avg {}%'.format(
round(dif, 1),
round(dif / t * 100, 1)
))
log('-----------')
log('some memory {}s, avg {}%'.format(
round(dms, 1),
round(dms / t * 100, 1)
))
log('full memory {}s, avg {}%'.format(
round(dmf, 1),
round(dmf / t * 100, 1)
))
def cgroup2_root():
"""
"""
with open(mounts) as f:
for line in f:
if cgroup2_separator in line:
return line.partition(cgroup2_separator)[0].partition(' ')[2]
def mlockall():
"""
"""
MCL_CURRENT = 1
MCL_FUTURE = 2
MCL_ONFAULT = 4
libc = CDLL(None, use_errno=True)
result = libc.mlockall(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT)
if result != 0:
result = libc.mlockall(MCL_CURRENT | MCL_FUTURE)
if result != 0:
log('WARNING: cannot lock process memory: [Errno {}]'.format(
result))
else:
log('Prosess memory locked with MCL_CURRENT | MCL_FUTURE')
else:
log('Process memory locked with '
'MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT')
def psi_file_mem_to_metrics0(psi_path):
"""
"""
with open(psi_path) as f:
psi_list = f.readlines()
some_list, full_list = psi_list[0].split(' '), psi_list[1].split(' ')
some_avg10 = some_list[1].split('=')[1]
some_avg60 = some_list[2].split('=')[1]
some_avg300 = some_list[3].split('=')[1]
full_avg10 = full_list[1].split('=')[1]
full_avg60 = full_list[2].split('=')[1]
full_avg300 = full_list[3].split('=')[1]
return (some_avg10, some_avg60, some_avg300,
full_avg10, full_avg60, full_avg300)
def psi_file_mem_to_metrics(psi_path):
"""
"""
foo = read_path(psi_path)
if foo is None:
return None
try:
psi_list = foo.split('\n')
some_list, full_list = psi_list[0].split(' '), psi_list[1].split(' ')
some_avg10 = some_list[1].split('=')[1]
some_avg60 = some_list[2].split('=')[1]
some_avg300 = some_list[3].split('=')[1]
full_avg10 = full_list[1].split('=')[1]
full_avg60 = full_list[2].split('=')[1]
full_avg300 = full_list[3].split('=')[1]
return (some_avg10, some_avg60, some_avg300,
full_avg10, full_avg60, full_avg300)
except Exception as e:
log('{}'.format(e))
return None
def psi_file_cpu_to_metrics(psi_path):
"""
"""
foo = read_path(psi_path)
if foo is None:
return None
try:
psi_list = foo.split('\n')
some_list = psi_list[0].split(' ')
some_avg10 = some_list[1].split('=')[1]
some_avg60 = some_list[2].split('=')[1]
some_avg300 = some_list[3].split('=')[1]
return (some_avg10, some_avg60, some_avg300)
except Exception as e:
log('{}'.format(e))
return None
def psi_file_mem_to_total(psi_path):
"""
"""
foo = read_path(psi_path)
if foo is None:
return None
try:
psi_list = foo.split('\n')
some_list, full_list = psi_list[0].split(' '), psi_list[1].split(' ')
some_total = some_list[4].split('=')[1]
full_total = full_list[4].split('=')[1]
return int(some_total), int(full_total)
except Exception as e:
log('{}'.format(e))
return None
def psi_file_cpu_to_total(psi_path):
"""
"""
foo = read_path(psi_path)
if foo is None:
return None
try:
psi_list = foo.split('\n')
some_list = psi_list[0].split(' ')
some_total = some_list[4].split('=')[1]
return int(some_total)
except Exception as e:
log('{}'.format(e))
return None
def print_head_0():
"""
"""
log('==================================================================='
'============')
log(' cpu || io || memory')
log('============= || ============================= || ================='
'============')
log(' some || some | full || some | '
' full')
log('------------- || ------------- | ------------- || ------------- | -'
'------------')
log(' avg10 avg60 || avg10 avg60 | avg10 avg60 || avg10 avg60 | '
'avg10 avg60')
log('------ ------ || ------ ------ | ------ ------ || ------ ------ | -'
'----- ------')
def print_head_1():
"""
"""
log('===================================================================='
'==============================================')
log(' cpu || io '
'|| memory')
log('==================== || =========================================== '
'|| ===========================================')
log(' some || some | full '
'|| some | full')
log('-------------------- || -------------------- | -------------------- '
'|| -------------------- | --------------------')
log(' avg10 avg60 avg300 || avg10 avg60 avg300 | avg10 avg60 avg300 '
'|| avg10 avg60 avg300 | avg10 avg60 avg300')
log('------ ------ ------ || ------ ------ ------ | ------ ------ ------ '
'|| ------ ------ ------ | ------ ------ ------')
def print_head_2():
"""
"""
log('----- - ----------- - ----------- -')
log(' cpu | io | memory |')
log('----- | ----------- | ----------- |')
log(' some | some full | some full | interval')
log('----- | ----- ----- | ----- ----- | --------')
def log(*msg):
"""
"""
if not SUPPRESS_OUTPUT:
print(*msg)
if separate_log:
logging.info(*msg)
def log_head(*msg):
"""
"""
print(*msg)
if separate_log:
logging.info(*msg)
parser = ArgumentParser()
parser.add_argument(
'-t',
'--target',
help="""target (cgroup_v2 or SYSTEM_WIDE)""",
default='SYSTEM_WIDE',
type=str
)
parser.add_argument(
'-i',
'--interval',
help="""interval in sec""",
default=2,
type=float
)
parser.add_argument(
'-l',
'--log',
help="""path to log file""",
default=None,
type=str
)
parser.add_argument(
'-m',
'--mode',
help="""mode (0, 1 or 2)""",
default='0',
type=str
)
parser.add_argument(
'-s',
'--suppress-output',
help="""suppress output""",
default='False',
type=str
)
args = parser.parse_args()
target = args.target
mode = args.mode
interval = args.interval
log_file = args.log
suppress_output = args.suppress_output
if target != 'SYSTEM_WIDE':
target = '/' + target.strip('/')
if log_file is None:
separate_log = False
else:
separate_log = True
import logging
if separate_log:
try:
logging.basicConfig(
filename=log_file,
level=logging.INFO,
format="%(asctime)s: %(message)s")
except Exception as e:
print(e)
exit(1)
if suppress_output == 'False':
SUPPRESS_OUTPUT = False
elif suppress_output == 'True':
SUPPRESS_OUTPUT = True
else:
log_head('error: argument -s/--suppress-output: valid values are '
'False and True')
exit(1)
if log_file is not None:
logstring = 'log file: {}, '.format(log_file)
else:
logstring = 'log file is not set, '
if interval < 1:
log_head('error: argument -i/--interval: the value must be greater than or'
' equal to 1')
exit(1)
if not (mode == '0' or mode == '1' or mode == '2'):
log_head('ERROR: invalid mode. Valid values are 0, 1 and 2. Exit.')
exit(1)
try:
psi_file_mem_to_metrics0('/proc/pressure/memory')
except Exception as e:
log_head('ERROR: {}'.format(e))
log_head('PSI metrics are not provided by the kernel. Exit.')
exit(1)
log_head('Starting psi2log, target: {}, mode: {}, interval: {} sec, {}suppress'
' output: {}'.format(
target, mode, round(interval, 3), logstring, suppress_output))
fd = dict()
if target == 'SYSTEM_WIDE':
system_wide = True
source_dir = '/proc/pressure'
cpu_file = '/proc/pressure/cpu'
io_file = '/proc/pressure/io'
memory_file = '/proc/pressure/memory'
log_head('PSI source dir: /proc/pressure/, source files: cpu, io, memory')
else:
system_wide = False
mounts = '/proc/mounts'
cgroup2_separator = ' cgroup2 rw,'
cgroup2_mountpoint = cgroup2_root()
if cgroup2_mountpoint is None:
log('ERROR: unified cgroup hierarchy is not mounted, exit')
exit(1)
source_dir = cgroup2_mountpoint + target
cpu_file = source_dir + '/cpu.pressure'
io_file = source_dir + '/io.pressure'
memory_file = source_dir + '/memory.pressure'
log_head('PSI source dir: {}{}/, source files: cpu.pressure, io.pressure,'
' memory.pressure'.format(cgroup2_mountpoint, target))
abnormal_interval = 1.01 * interval
abnormal_inaccuracy = 0.05
if target == 'SYSTEM_WIDE':
total_cs_0 = psi_file_cpu_to_total(cpu_file)
total_is_0, total_if_0 = psi_file_mem_to_total(io_file)
total_ms_0, total_mf_0 = psi_file_mem_to_total(memory_file)
t_0 = monotonic()
peaks_dict = dict()
sig_list = [SIGTERM, SIGINT, SIGQUIT, SIGHUP]
for i in sig_list:
signal(i, signal_handler)
mlockall()
if mode == '0':
print_head_0()
while True:
try:
(c_some_avg10, c_some_avg60, c_some_avg300
) = psi_file_cpu_to_metrics(cpu_file)
(i_some_avg10, i_some_avg60, i_some_avg300,
i_full_avg10, i_full_avg60, i_full_avg300
) = psi_file_mem_to_metrics(io_file)
(m_some_avg10, m_some_avg60, m_some_avg300,
m_full_avg10, m_full_avg60, m_full_avg300
) = psi_file_mem_to_metrics(memory_file)
except TypeError:
stdout.flush()
sleep(interval)
continue
log('{:>6} {:>6} || {:>6} {:>6} | {:>6} {:>6} || {:>6} {:>6} | {:>6} '
'{:>6}'.format(
c_some_avg10, c_some_avg60,
i_some_avg10, i_some_avg60,
i_full_avg10, i_full_avg60,
m_some_avg10, m_some_avg60,
m_full_avg10, m_full_avg60
))
c_some_avg10 = float(c_some_avg10)
if ('c_some_avg10' not in peaks_dict or
peaks_dict['c_some_avg10'] < c_some_avg10):
peaks_dict['c_some_avg10'] = c_some_avg10
c_some_avg60 = float(c_some_avg60)
if ('c_some_avg60' not in peaks_dict or
peaks_dict['c_some_avg60'] < c_some_avg60):
peaks_dict['c_some_avg60'] = c_some_avg60
c_some_avg300 = float(c_some_avg300)
if ('c_some_avg300' not in peaks_dict or
peaks_dict['c_some_avg300'] < c_some_avg300):
peaks_dict['c_some_avg300'] = c_some_avg300
#######################################################################
i_some_avg10 = float(i_some_avg10)
if ('i_some_avg10' not in peaks_dict or
peaks_dict['i_some_avg10'] < i_some_avg10):
peaks_dict['i_some_avg10'] = i_some_avg10
i_some_avg60 = float(i_some_avg60)
if ('i_some_avg60' not in peaks_dict or
peaks_dict['i_some_avg60'] < i_some_avg60):
peaks_dict['i_some_avg60'] = i_some_avg60
i_some_avg300 = float(i_some_avg300)
if ('i_some_avg300' not in peaks_dict or
peaks_dict['i_some_avg300'] < i_some_avg300):
peaks_dict['i_some_avg300'] = i_some_avg300
i_full_avg10 = float(i_full_avg10)
if ('i_full_avg10' not in peaks_dict or
peaks_dict['i_full_avg10'] < i_full_avg10):
peaks_dict['i_full_avg10'] = i_full_avg10
i_full_avg60 = float(i_full_avg60)
if ('i_full_avg60' not in peaks_dict or
peaks_dict['i_full_avg60'] < i_full_avg60):
peaks_dict['i_full_avg60'] = i_full_avg60
i_full_avg300 = float(i_full_avg300)
if ('i_full_avg300' not in peaks_dict or
peaks_dict['i_full_avg300'] < i_full_avg300):
peaks_dict['i_full_avg300'] = i_full_avg300
#######################################################################
m_some_avg10 = float(m_some_avg10)
if ('m_some_avg10' not in peaks_dict or
peaks_dict['m_some_avg10'] < m_some_avg10):
peaks_dict['m_some_avg10'] = m_some_avg10
m_some_avg60 = float(m_some_avg60)
if ('m_some_avg60' not in peaks_dict or
peaks_dict['m_some_avg60'] < m_some_avg60):
peaks_dict['m_some_avg60'] = m_some_avg60
m_some_avg300 = float(m_some_avg300)
if ('m_some_avg300' not in peaks_dict or
peaks_dict['m_some_avg300'] < m_some_avg300):
peaks_dict['m_some_avg300'] = m_some_avg300
m_full_avg10 = float(m_full_avg10)
if ('m_full_avg10' not in peaks_dict or
peaks_dict['m_full_avg10'] < m_full_avg10):
peaks_dict['m_full_avg10'] = m_full_avg10
m_full_avg60 = float(m_full_avg60)
if ('m_full_avg60' not in peaks_dict or
peaks_dict['m_full_avg60'] < m_full_avg60):
peaks_dict['m_full_avg60'] = m_full_avg60
m_full_avg300 = float(m_full_avg300)
if ('m_full_avg300' not in peaks_dict or
peaks_dict['m_full_avg300'] < m_full_avg300):
peaks_dict['m_full_avg300'] = m_full_avg300
stdout.flush()
sleep(interval)
if mode == '1':
print_head_1()
while True:
try:
(c_some_avg10, c_some_avg60, c_some_avg300
) = psi_file_cpu_to_metrics(cpu_file)
(i_some_avg10, i_some_avg60, i_some_avg300,
i_full_avg10, i_full_avg60, i_full_avg300
) = psi_file_mem_to_metrics(io_file)
(m_some_avg10, m_some_avg60, m_some_avg300,
m_full_avg10, m_full_avg60, m_full_avg300
) = psi_file_mem_to_metrics(memory_file)
except TypeError:
stdout.flush()
sleep(interval)
continue
log('{:>6} {:>6} {:>6} || {:>6} {:>6} {:>6} | {:>6} {:>6} {:>6} || '
'{:>6} {:>6} {:>6} | {:>6} {:>6} {:>6}'.format(
c_some_avg10, c_some_avg60, c_some_avg300,
i_some_avg10, i_some_avg60, i_some_avg300,
i_full_avg10, i_full_avg60, i_full_avg300,
m_some_avg10, m_some_avg60, m_some_avg300,
m_full_avg10, m_full_avg60, m_full_avg300
))
c_some_avg10 = float(c_some_avg10)
if ('c_some_avg10' not in peaks_dict or
peaks_dict['c_some_avg10'] < c_some_avg10):
peaks_dict['c_some_avg10'] = c_some_avg10
c_some_avg60 = float(c_some_avg60)
if ('c_some_avg60' not in peaks_dict or
peaks_dict['c_some_avg60'] < c_some_avg60):
peaks_dict['c_some_avg60'] = c_some_avg60
c_some_avg300 = float(c_some_avg300)
if ('c_some_avg300' not in peaks_dict or
peaks_dict['c_some_avg300'] < c_some_avg300):
peaks_dict['c_some_avg300'] = c_some_avg300
#######################################################################
i_some_avg10 = float(i_some_avg10)
if ('i_some_avg10' not in peaks_dict or
peaks_dict['i_some_avg10'] < i_some_avg10):
peaks_dict['i_some_avg10'] = i_some_avg10
i_some_avg60 = float(i_some_avg60)
if ('i_some_avg60' not in peaks_dict or
peaks_dict['i_some_avg60'] < i_some_avg60):
peaks_dict['i_some_avg60'] = i_some_avg60
i_some_avg300 = float(i_some_avg300)
if ('i_some_avg300' not in peaks_dict or
peaks_dict['i_some_avg300'] < i_some_avg300):
peaks_dict['i_some_avg300'] = i_some_avg300
i_full_avg10 = float(i_full_avg10)
if ('i_full_avg10' not in peaks_dict or
peaks_dict['i_full_avg10'] < i_full_avg10):
peaks_dict['i_full_avg10'] = i_full_avg10
i_full_avg60 = float(i_full_avg60)
if ('i_full_avg60' not in peaks_dict or
peaks_dict['i_full_avg60'] < i_full_avg60):
peaks_dict['i_full_avg60'] = i_full_avg60
i_full_avg300 = float(i_full_avg300)
if ('i_full_avg300' not in peaks_dict or
peaks_dict['i_full_avg300'] < i_full_avg300):
peaks_dict['i_full_avg300'] = i_full_avg300
#######################################################################
m_some_avg10 = float(m_some_avg10)
if ('m_some_avg10' not in peaks_dict or
peaks_dict['m_some_avg10'] < m_some_avg10):
peaks_dict['m_some_avg10'] = m_some_avg10
m_some_avg60 = float(m_some_avg60)
if ('m_some_avg60' not in peaks_dict or
peaks_dict['m_some_avg60'] < m_some_avg60):
peaks_dict['m_some_avg60'] = m_some_avg60
m_some_avg300 = float(m_some_avg300)
if ('m_some_avg300' not in peaks_dict or
peaks_dict['m_some_avg300'] < m_some_avg300):
peaks_dict['m_some_avg300'] = m_some_avg300
m_full_avg10 = float(m_full_avg10)
if ('m_full_avg10' not in peaks_dict or
peaks_dict['m_full_avg10'] < m_full_avg10):
peaks_dict['m_full_avg10'] = m_full_avg10
m_full_avg60 = float(m_full_avg60)
if ('m_full_avg60' not in peaks_dict or
peaks_dict['m_full_avg60'] < m_full_avg60):
peaks_dict['m_full_avg60'] = m_full_avg60
m_full_avg300 = float(m_full_avg300)
if ('m_full_avg300' not in peaks_dict or
peaks_dict['m_full_avg300'] < m_full_avg300):
peaks_dict['m_full_avg300'] = m_full_avg300
stdout.flush()
sleep(interval)
print_head_2()
try:
total_cs0 = psi_file_cpu_to_total(cpu_file)
total_is0, total_if0 = psi_file_mem_to_total(io_file)
total_ms0, total_mf0 = psi_file_mem_to_total(memory_file)
monotonic0 = monotonic()
stdout.flush()
sleep(interval)
except TypeError:
stdout.flush()
sleep(interval)
TT = 10000
while True:
try:
total_cs1 = psi_file_cpu_to_total(cpu_file)
total_is1, total_if1 = psi_file_mem_to_total(io_file)
total_ms1, total_mf1 = psi_file_mem_to_total(memory_file)
monotonic1 = monotonic()
dm = monotonic1 - monotonic0
if dm > abnormal_interval and dm - interval > abnormal_inaccuracy:
log('WARNING: abnormal interval ({} sec), metrics may be prov'
'ided incorrect'.format(round(dm, 3)))
monotonic0 = monotonic1
except TypeError:
stdout.flush()
sleep(interval)
continue
dtotal_cs = total_cs1 - total_cs0
avg_cs = dtotal_cs / dm / TT
if 'avg_cs' not in peaks_dict or peaks_dict['avg_cs'] < avg_cs:
peaks_dict['avg_cs'] = avg_cs
total_cs0 = total_cs1
dtotal_is = total_is1 - total_is0
avg_is = dtotal_is / dm / TT
if 'avg_is' not in peaks_dict or peaks_dict['avg_is'] < avg_is:
peaks_dict['avg_is'] = avg_is
total_is0 = total_is1
dtotal_if = total_if1 - total_if0
avg_if = dtotal_if / dm / TT
if 'avg_if' not in peaks_dict or peaks_dict['avg_if'] < avg_if:
peaks_dict['avg_if'] = avg_if
total_if0 = total_if1
dtotal_ms = total_ms1 - total_ms0
avg_ms = dtotal_ms / dm / TT
if 'avg_ms' not in peaks_dict or peaks_dict['avg_ms'] < avg_ms:
peaks_dict['avg_ms'] = avg_ms
total_ms0 = total_ms1
dtotal_mf = total_mf1 - total_mf0
avg_mf = dtotal_mf / dm / TT
if 'avg_mf' not in peaks_dict or peaks_dict['avg_mf'] < avg_mf:
peaks_dict['avg_mf'] = avg_mf
total_mf0 = total_mf1
log('{:>5} | {:>5} {:>5} | {:>5} {:>5} | {}'.format(
round(avg_cs, 1),
round(avg_is, 1),
round(avg_if, 1),
round(avg_ms, 1),
round(avg_mf, 1),
round(dm, 2)
))
stdout.flush()
sleep(interval)