#!/usr/bin/env python3 """psi2log - PSI metrics monitor and logger""" from argparse import ArgumentParser from ctypes import CDLL from signal import SIGHUP, SIGINT, SIGQUIT, SIGTERM, signal from sys import exit, stdout from time import monotonic, sleep def read_path(path): """ """ try: fd[path].seek(0) except ValueError: try: fd[path] = open(path, 'rb', buffering=0) except FileNotFoundError as e: log(e) return None except KeyError: try: fd[path] = open(path, 'rb', buffering=0) except FileNotFoundError as e: log(e) return None try: return fd[path].read(99999).decode() except OSError as e: log(e) fd[path].close() return None def form1(num): """ """ s = str(num).split('.') return '{}.{:0<2}'.format(s[0], s[1]) def form2(num): """ """ s = str(round(num, 1)).split('.') return '{}.{:0<1}'.format(s[0], s[1]) def signal_handler(signum, frame): """ """ def signal_handler_inner(signum, frame): pass for i in sig_list: signal(i, signal_handler_inner) if signum == SIGINT: print('') lpd = len(peaks_dict) if lpd == 15: log('=================================') log('Peak values: avg10 avg60 avg300') log('----------- ------ ------ ------') log('some cpu {:>6} {:>6} {:>6}'.format( form1(peaks_dict['c_some_avg10']), form1(peaks_dict['c_some_avg60']), form1(peaks_dict['c_some_avg300']), )) log('----------- ------ ------ ------') log('some io {:>6} {:>6} {:>6}'.format( form1(peaks_dict['i_some_avg10']), form1(peaks_dict['i_some_avg60']), form1(peaks_dict['i_some_avg300']), )) log('full io {:>6} {:>6} {:>6}'.format( form1(peaks_dict['i_full_avg10']), form1(peaks_dict['i_full_avg60']), form1(peaks_dict['i_full_avg300']), )) log('----------- ------ ------ ------') log('some memory {:>6} {:>6} {:>6}'.format( form1(peaks_dict['m_some_avg10']), form1(peaks_dict['m_some_avg60']), form1(peaks_dict['m_some_avg300']), )) log('full memory {:>6} {:>6} {:>6}'.format( form1(peaks_dict['m_full_avg10']), form1(peaks_dict['m_full_avg60']), form1(peaks_dict['m_full_avg300']), )) if lpd == 5: log('----- | ----- ----- | ----- ----- | --------') log('{:>5} | {:>5} {:>5} | {:>5} {:>5} | peaks'.format( form2(peaks_dict['avg_cs']), form2(peaks_dict['avg_is']), form2(peaks_dict['avg_if']), form2(peaks_dict['avg_ms']), form2(peaks_dict['avg_mf']) )) if target == 'SYSTEM_WIDE': log_stall_times() if separate_log: logging.info('') exit() def log_stall_times(): """ """ total_cs_1 = psi_file_cpu_to_total(cpu_file) total_is_1, total_if_1 = psi_file_mem_to_total(io_file) total_ms_1, total_mf_1 = psi_file_mem_to_total(memory_file) t = monotonic() - t_0 M = 1000000 dcs = (total_cs_1 - total_cs_0) / M dis = (total_is_1 - total_is_0) / M dif = (total_if_1 - total_if_0) / M dms = (total_ms_1 - total_ms_0) / M dmf = (total_mf_1 - total_mf_0) / M if mode == '0' or mode == '1': log('=================================') else: log('--') log('Stall times for the last {}s:'.format(round(t, 1))) log('-----------') log('some cpu {}s, avg {}%'.format( round(dcs, 1), round(dcs / t * 100, 1) )) log('-----------') log('some io {}s, avg {}%'.format( round(dis, 1), round(dis / t * 100, 1) )) log('full io {}s, avg {}%'.format( round(dif, 1), round(dif / t * 100, 1) )) log('-----------') log('some memory {}s, avg {}%'.format( round(dms, 1), round(dms / t * 100, 1) )) log('full memory {}s, avg {}%'.format( round(dmf, 1), round(dmf / t * 100, 1) )) def cgroup2_root(): """ """ with open(mounts) as f: for line in f: if cgroup2_separator in line: return line.partition(cgroup2_separator)[0].partition(' ')[2] def mlockall(): """ """ MCL_CURRENT = 1 MCL_FUTURE = 2 MCL_ONFAULT = 4 libc = CDLL(None, use_errno=True) result = libc.mlockall(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT) if result != 0: result = libc.mlockall(MCL_CURRENT | MCL_FUTURE) if result != 0: log('WARNING: cannot lock process memory: [Errno {}]'.format( result)) else: log('Prosess memory locked with MCL_CURRENT | MCL_FUTURE') else: log('Process memory locked with ' 'MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT') def psi_file_mem_to_metrics0(psi_path): """ """ with open(psi_path) as f: psi_list = f.readlines() some_list, full_list = psi_list[0].split(' '), psi_list[1].split(' ') some_avg10 = some_list[1].split('=')[1] some_avg60 = some_list[2].split('=')[1] some_avg300 = some_list[3].split('=')[1] full_avg10 = full_list[1].split('=')[1] full_avg60 = full_list[2].split('=')[1] full_avg300 = full_list[3].split('=')[1] return (some_avg10, some_avg60, some_avg300, full_avg10, full_avg60, full_avg300) def psi_file_mem_to_metrics(psi_path): """ """ foo = read_path(psi_path) if foo is None: return None try: psi_list = foo.split('\n') some_list, full_list = psi_list[0].split(' '), psi_list[1].split(' ') some_avg10 = some_list[1].split('=')[1] some_avg60 = some_list[2].split('=')[1] some_avg300 = some_list[3].split('=')[1] full_avg10 = full_list[1].split('=')[1] full_avg60 = full_list[2].split('=')[1] full_avg300 = full_list[3].split('=')[1] return (some_avg10, some_avg60, some_avg300, full_avg10, full_avg60, full_avg300) except Exception as e: log('{}'.format(e)) return None def psi_file_cpu_to_metrics(psi_path): """ """ foo = read_path(psi_path) if foo is None: return None try: psi_list = foo.split('\n') some_list = psi_list[0].split(' ') some_avg10 = some_list[1].split('=')[1] some_avg60 = some_list[2].split('=')[1] some_avg300 = some_list[3].split('=')[1] return (some_avg10, some_avg60, some_avg300) except Exception as e: log('{}'.format(e)) return None def psi_file_mem_to_total(psi_path): """ """ foo = read_path(psi_path) if foo is None: return None try: psi_list = foo.split('\n') some_list, full_list = psi_list[0].split(' '), psi_list[1].split(' ') some_total = some_list[4].split('=')[1] full_total = full_list[4].split('=')[1] return int(some_total), int(full_total) except Exception as e: log('{}'.format(e)) return None def psi_file_cpu_to_total(psi_path): """ """ foo = read_path(psi_path) if foo is None: return None try: psi_list = foo.split('\n') some_list = psi_list[0].split(' ') some_total = some_list[4].split('=')[1] return int(some_total) except Exception as e: log('{}'.format(e)) return None def print_head_0(): """ """ log('===================================================================' '============') log(' cpu || io || memory') log('============= || ============================= || =================' '============') log(' some || some | full || some | ' ' full') log('------------- || ------------- | ------------- || ------------- | -' '------------') log(' avg10 avg60 || avg10 avg60 | avg10 avg60 || avg10 avg60 | ' 'avg10 avg60') log('------ ------ || ------ ------ | ------ ------ || ------ ------ | -' '----- ------') def print_head_1(): """ """ log('====================================================================' '==============================================') log(' cpu || io ' '|| memory') log('==================== || =========================================== ' '|| ===========================================') log(' some || some | full ' '|| some | full') log('-------------------- || -------------------- | -------------------- ' '|| -------------------- | --------------------') log(' avg10 avg60 avg300 || avg10 avg60 avg300 | avg10 avg60 avg300 ' '|| avg10 avg60 avg300 | avg10 avg60 avg300') log('------ ------ ------ || ------ ------ ------ | ------ ------ ------ ' '|| ------ ------ ------ | ------ ------ ------') def print_head_2(): """ """ log('----- - ----------- - ----------- -') log(' cpu | io | memory |') log('----- | ----------- | ----------- |') log(' some | some full | some full | interval') log('----- | ----- ----- | ----- ----- | --------') def log(*msg): """ """ if not SUPPRESS_OUTPUT: print(*msg) if separate_log: logging.info(*msg) def log_head(*msg): """ """ print(*msg) if separate_log: logging.info(*msg) parser = ArgumentParser() parser.add_argument( '-t', '--target', help="""target (cgroup_v2 or SYSTEM_WIDE)""", default='SYSTEM_WIDE', type=str ) parser.add_argument( '-i', '--interval', help="""interval in sec""", default=2, type=float ) parser.add_argument( '-l', '--log', help="""path to log file""", default=None, type=str ) parser.add_argument( '-m', '--mode', help="""mode (0, 1 or 2)""", default='0', type=str ) parser.add_argument( '-s', '--suppress-output', help="""suppress output""", default='False', type=str ) args = parser.parse_args() target = args.target mode = args.mode interval = args.interval log_file = args.log suppress_output = args.suppress_output if target != 'SYSTEM_WIDE': target = '/' + target.strip('/') if log_file is None: separate_log = False else: separate_log = True import logging if separate_log: try: logging.basicConfig( filename=log_file, level=logging.INFO, format="%(asctime)s: %(message)s") except Exception as e: print(e) exit(1) if suppress_output == 'False': SUPPRESS_OUTPUT = False elif suppress_output == 'True': SUPPRESS_OUTPUT = True else: log_head('error: argument -s/--suppress-output: valid values are ' 'False and True') exit(1) if log_file is not None: logstring = 'log file: {}, '.format(log_file) else: logstring = 'log file is not set, ' if interval < 1: log_head('error: argument -i/--interval: the value must be greater than or' ' equal to 1') exit(1) if not (mode == '0' or mode == '1' or mode == '2'): log_head('ERROR: invalid mode. Valid values are 0, 1 and 2. Exit.') exit(1) try: psi_file_mem_to_metrics0('/proc/pressure/memory') except Exception as e: log_head('ERROR: {}'.format(e)) log_head('PSI metrics are not provided by the kernel. Exit.') exit(1) log_head('Starting psi2log, target: {}, mode: {}, interval: {} sec, {}suppress' ' output: {}'.format( target, mode, round(interval, 3), logstring, suppress_output)) fd = dict() if target == 'SYSTEM_WIDE': system_wide = True source_dir = '/proc/pressure' cpu_file = '/proc/pressure/cpu' io_file = '/proc/pressure/io' memory_file = '/proc/pressure/memory' log_head('PSI source dir: /proc/pressure/, source files: cpu, io, memory') else: system_wide = False mounts = '/proc/mounts' cgroup2_separator = ' cgroup2 rw,' cgroup2_mountpoint = cgroup2_root() if cgroup2_mountpoint is None: log('ERROR: unified cgroup hierarchy is not mounted, exit') exit(1) source_dir = cgroup2_mountpoint + target cpu_file = source_dir + '/cpu.pressure' io_file = source_dir + '/io.pressure' memory_file = source_dir + '/memory.pressure' log_head('PSI source dir: {}{}/, source files: cpu.pressure, io.pressure,' ' memory.pressure'.format(cgroup2_mountpoint, target)) abnormal_interval = 1.01 * interval abnormal_inaccuracy = 0.05 if target == 'SYSTEM_WIDE': total_cs_0 = psi_file_cpu_to_total(cpu_file) total_is_0, total_if_0 = psi_file_mem_to_total(io_file) total_ms_0, total_mf_0 = psi_file_mem_to_total(memory_file) t_0 = monotonic() peaks_dict = dict() sig_list = [SIGTERM, SIGINT, SIGQUIT, SIGHUP] for i in sig_list: signal(i, signal_handler) mlockall() if mode == '0': print_head_0() while True: try: (c_some_avg10, c_some_avg60, c_some_avg300 ) = psi_file_cpu_to_metrics(cpu_file) (i_some_avg10, i_some_avg60, i_some_avg300, i_full_avg10, i_full_avg60, i_full_avg300 ) = psi_file_mem_to_metrics(io_file) (m_some_avg10, m_some_avg60, m_some_avg300, m_full_avg10, m_full_avg60, m_full_avg300 ) = psi_file_mem_to_metrics(memory_file) except TypeError: stdout.flush() sleep(interval) continue log('{:>6} {:>6} || {:>6} {:>6} | {:>6} {:>6} || {:>6} {:>6} | {:>6} ' '{:>6}'.format( c_some_avg10, c_some_avg60, i_some_avg10, i_some_avg60, i_full_avg10, i_full_avg60, m_some_avg10, m_some_avg60, m_full_avg10, m_full_avg60 )) c_some_avg10 = float(c_some_avg10) if ('c_some_avg10' not in peaks_dict or peaks_dict['c_some_avg10'] < c_some_avg10): peaks_dict['c_some_avg10'] = c_some_avg10 c_some_avg60 = float(c_some_avg60) if ('c_some_avg60' not in peaks_dict or peaks_dict['c_some_avg60'] < c_some_avg60): peaks_dict['c_some_avg60'] = c_some_avg60 c_some_avg300 = float(c_some_avg300) if ('c_some_avg300' not in peaks_dict or peaks_dict['c_some_avg300'] < c_some_avg300): peaks_dict['c_some_avg300'] = c_some_avg300 ####################################################################### i_some_avg10 = float(i_some_avg10) if ('i_some_avg10' not in peaks_dict or peaks_dict['i_some_avg10'] < i_some_avg10): peaks_dict['i_some_avg10'] = i_some_avg10 i_some_avg60 = float(i_some_avg60) if ('i_some_avg60' not in peaks_dict or peaks_dict['i_some_avg60'] < i_some_avg60): peaks_dict['i_some_avg60'] = i_some_avg60 i_some_avg300 = float(i_some_avg300) if ('i_some_avg300' not in peaks_dict or peaks_dict['i_some_avg300'] < i_some_avg300): peaks_dict['i_some_avg300'] = i_some_avg300 i_full_avg10 = float(i_full_avg10) if ('i_full_avg10' not in peaks_dict or peaks_dict['i_full_avg10'] < i_full_avg10): peaks_dict['i_full_avg10'] = i_full_avg10 i_full_avg60 = float(i_full_avg60) if ('i_full_avg60' not in peaks_dict or peaks_dict['i_full_avg60'] < i_full_avg60): peaks_dict['i_full_avg60'] = i_full_avg60 i_full_avg300 = float(i_full_avg300) if ('i_full_avg300' not in peaks_dict or peaks_dict['i_full_avg300'] < i_full_avg300): peaks_dict['i_full_avg300'] = i_full_avg300 ####################################################################### m_some_avg10 = float(m_some_avg10) if ('m_some_avg10' not in peaks_dict or peaks_dict['m_some_avg10'] < m_some_avg10): peaks_dict['m_some_avg10'] = m_some_avg10 m_some_avg60 = float(m_some_avg60) if ('m_some_avg60' not in peaks_dict or peaks_dict['m_some_avg60'] < m_some_avg60): peaks_dict['m_some_avg60'] = m_some_avg60 m_some_avg300 = float(m_some_avg300) if ('m_some_avg300' not in peaks_dict or peaks_dict['m_some_avg300'] < m_some_avg300): peaks_dict['m_some_avg300'] = m_some_avg300 m_full_avg10 = float(m_full_avg10) if ('m_full_avg10' not in peaks_dict or peaks_dict['m_full_avg10'] < m_full_avg10): peaks_dict['m_full_avg10'] = m_full_avg10 m_full_avg60 = float(m_full_avg60) if ('m_full_avg60' not in peaks_dict or peaks_dict['m_full_avg60'] < m_full_avg60): peaks_dict['m_full_avg60'] = m_full_avg60 m_full_avg300 = float(m_full_avg300) if ('m_full_avg300' not in peaks_dict or peaks_dict['m_full_avg300'] < m_full_avg300): peaks_dict['m_full_avg300'] = m_full_avg300 stdout.flush() sleep(interval) if mode == '1': print_head_1() while True: try: (c_some_avg10, c_some_avg60, c_some_avg300 ) = psi_file_cpu_to_metrics(cpu_file) (i_some_avg10, i_some_avg60, i_some_avg300, i_full_avg10, i_full_avg60, i_full_avg300 ) = psi_file_mem_to_metrics(io_file) (m_some_avg10, m_some_avg60, m_some_avg300, m_full_avg10, m_full_avg60, m_full_avg300 ) = psi_file_mem_to_metrics(memory_file) except TypeError: stdout.flush() sleep(interval) continue log('{:>6} {:>6} {:>6} || {:>6} {:>6} {:>6} | {:>6} {:>6} {:>6} || ' '{:>6} {:>6} {:>6} | {:>6} {:>6} {:>6}'.format( c_some_avg10, c_some_avg60, c_some_avg300, i_some_avg10, i_some_avg60, i_some_avg300, i_full_avg10, i_full_avg60, i_full_avg300, m_some_avg10, m_some_avg60, m_some_avg300, m_full_avg10, m_full_avg60, m_full_avg300 )) c_some_avg10 = float(c_some_avg10) if ('c_some_avg10' not in peaks_dict or peaks_dict['c_some_avg10'] < c_some_avg10): peaks_dict['c_some_avg10'] = c_some_avg10 c_some_avg60 = float(c_some_avg60) if ('c_some_avg60' not in peaks_dict or peaks_dict['c_some_avg60'] < c_some_avg60): peaks_dict['c_some_avg60'] = c_some_avg60 c_some_avg300 = float(c_some_avg300) if ('c_some_avg300' not in peaks_dict or peaks_dict['c_some_avg300'] < c_some_avg300): peaks_dict['c_some_avg300'] = c_some_avg300 ####################################################################### i_some_avg10 = float(i_some_avg10) if ('i_some_avg10' not in peaks_dict or peaks_dict['i_some_avg10'] < i_some_avg10): peaks_dict['i_some_avg10'] = i_some_avg10 i_some_avg60 = float(i_some_avg60) if ('i_some_avg60' not in peaks_dict or peaks_dict['i_some_avg60'] < i_some_avg60): peaks_dict['i_some_avg60'] = i_some_avg60 i_some_avg300 = float(i_some_avg300) if ('i_some_avg300' not in peaks_dict or peaks_dict['i_some_avg300'] < i_some_avg300): peaks_dict['i_some_avg300'] = i_some_avg300 i_full_avg10 = float(i_full_avg10) if ('i_full_avg10' not in peaks_dict or peaks_dict['i_full_avg10'] < i_full_avg10): peaks_dict['i_full_avg10'] = i_full_avg10 i_full_avg60 = float(i_full_avg60) if ('i_full_avg60' not in peaks_dict or peaks_dict['i_full_avg60'] < i_full_avg60): peaks_dict['i_full_avg60'] = i_full_avg60 i_full_avg300 = float(i_full_avg300) if ('i_full_avg300' not in peaks_dict or peaks_dict['i_full_avg300'] < i_full_avg300): peaks_dict['i_full_avg300'] = i_full_avg300 ####################################################################### m_some_avg10 = float(m_some_avg10) if ('m_some_avg10' not in peaks_dict or peaks_dict['m_some_avg10'] < m_some_avg10): peaks_dict['m_some_avg10'] = m_some_avg10 m_some_avg60 = float(m_some_avg60) if ('m_some_avg60' not in peaks_dict or peaks_dict['m_some_avg60'] < m_some_avg60): peaks_dict['m_some_avg60'] = m_some_avg60 m_some_avg300 = float(m_some_avg300) if ('m_some_avg300' not in peaks_dict or peaks_dict['m_some_avg300'] < m_some_avg300): peaks_dict['m_some_avg300'] = m_some_avg300 m_full_avg10 = float(m_full_avg10) if ('m_full_avg10' not in peaks_dict or peaks_dict['m_full_avg10'] < m_full_avg10): peaks_dict['m_full_avg10'] = m_full_avg10 m_full_avg60 = float(m_full_avg60) if ('m_full_avg60' not in peaks_dict or peaks_dict['m_full_avg60'] < m_full_avg60): peaks_dict['m_full_avg60'] = m_full_avg60 m_full_avg300 = float(m_full_avg300) if ('m_full_avg300' not in peaks_dict or peaks_dict['m_full_avg300'] < m_full_avg300): peaks_dict['m_full_avg300'] = m_full_avg300 stdout.flush() sleep(interval) print_head_2() try: total_cs0 = psi_file_cpu_to_total(cpu_file) total_is0, total_if0 = psi_file_mem_to_total(io_file) total_ms0, total_mf0 = psi_file_mem_to_total(memory_file) monotonic0 = monotonic() stdout.flush() sleep(interval) except TypeError: stdout.flush() sleep(interval) TT = 10000 while True: try: total_cs1 = psi_file_cpu_to_total(cpu_file) total_is1, total_if1 = psi_file_mem_to_total(io_file) total_ms1, total_mf1 = psi_file_mem_to_total(memory_file) monotonic1 = monotonic() dm = monotonic1 - monotonic0 if dm > abnormal_interval and dm - interval > abnormal_inaccuracy: log('WARNING: abnormal interval ({} sec), metrics may be prov' 'ided incorrect'.format(round(dm, 3))) monotonic0 = monotonic1 except TypeError: stdout.flush() sleep(interval) continue dtotal_cs = total_cs1 - total_cs0 avg_cs = dtotal_cs / dm / TT if 'avg_cs' not in peaks_dict or peaks_dict['avg_cs'] < avg_cs: peaks_dict['avg_cs'] = avg_cs total_cs0 = total_cs1 dtotal_is = total_is1 - total_is0 avg_is = dtotal_is / dm / TT if 'avg_is' not in peaks_dict or peaks_dict['avg_is'] < avg_is: peaks_dict['avg_is'] = avg_is total_is0 = total_is1 dtotal_if = total_if1 - total_if0 avg_if = dtotal_if / dm / TT if 'avg_if' not in peaks_dict or peaks_dict['avg_if'] < avg_if: peaks_dict['avg_if'] = avg_if total_if0 = total_if1 dtotal_ms = total_ms1 - total_ms0 avg_ms = dtotal_ms / dm / TT if 'avg_ms' not in peaks_dict or peaks_dict['avg_ms'] < avg_ms: peaks_dict['avg_ms'] = avg_ms total_ms0 = total_ms1 dtotal_mf = total_mf1 - total_mf0 avg_mf = dtotal_mf / dm / TT if 'avg_mf' not in peaks_dict or peaks_dict['avg_mf'] < avg_mf: peaks_dict['avg_mf'] = avg_mf total_mf0 = total_mf1 log('{:>5} | {:>5} {:>5} | {:>5} {:>5} | {}'.format( round(avg_cs, 1), round(avg_is, 1), round(avg_if, 1), round(avg_ms, 1), round(avg_mf, 1), round(dm, 2) )) stdout.flush() sleep(interval)