/* * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "DcgmHealthWatch.h" #include "DcgmImexManager.h" #include "DcgmMutex.h" #include "dcgm_fields.h" #include #include #include #include #include #include #include #include #include const char *EntityToString(dcgm_field_entity_group_t entityGroupId) { switch (entityGroupId) { case DCGM_FE_GPU: return "GPU"; case DCGM_FE_VGPU: return "VGPU"; case DCGM_FE_SWITCH: return "NvSwitch"; case DCGM_FE_GPU_I: return "GPU Instance"; case DCGM_FE_GPU_CI: return "Compute Instance"; case DCGM_FE_LINK: return "Link"; default: return "Unknown"; } return ""; } // Adds a watch for the specified field that will poll every 10 seconds for the last hour's events #define ADD_WATCH(fieldId) \ do \ { \ bool updateOnFirstWatch = false; /* All of the callers of this call UpdateFields() right after */ \ bool wereFirstWatcher = false; \ ret = mpCoreProxy.AddFieldWatch(entityGroupId, \ entityId, \ fieldId, \ updateInterval, \ maxKeepAge, \ 0, \ watcher, \ false, \ updateOnFirstWatch, \ wereFirstWatcher); \ if (DCGM_ST_OK != ret) \ { \ log_error("Failed to set watch for field {} on {} {} group {}", \ fieldId, \ EntityToString(entityGroupId), \ entityId, \ entityGroupId); \ return ret; \ } \ } while (0) #define ADD_GLOBAL_WATCH(fieldId) \ do \ { \ bool updateOnFirstWatch = false; /* callers call UpdateFields() right after */ \ bool wereFirstWatcher = false; \ ret = mpCoreProxy.AddFieldWatch(DCGM_FE_NONE, \ 0, \ fieldId, \ updateInterval, \ maxKeepAge, \ 0, \ watcher, \ false, \ updateOnFirstWatch, \ wereFirstWatcher); \ if (DCGM_ST_OK != ret) \ { \ log_error("Failed to set global watch for field {}", fieldId); \ return ret; \ } \ } while (0) /*****************************************************************************/ DcgmHealthWatch::DcgmHealthWatch(dcgmCoreCallbacks_t &dcc) : mpCoreProxy(dcc) { m_mutex = new DcgmMutex(0); mGroupWatchState.clear(); BuildFieldLists(); BuildXidMappings(); } /*****************************************************************************/ DcgmHealthWatch::~DcgmHealthWatch() { if (m_mutex) { delete (m_mutex); m_mutex = 0; } } /*****************************************************************************/ void DcgmHealthWatch::BuildFieldLists(void) { // all the non-fatal error field ids. m_nvSwitchNonFatalFieldIds.push_back(DCGM_FI_DEV_NVSWITCH_NON_FATAL_ERRORS); // all the fatal error field ids. m_nvSwitchFatalFieldIds.push_back(DCGM_FI_DEV_NVSWITCH_FATAL_ERRORS); } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetNvSwitchWatches(std::vector &groupSwitchIds, dcgmHealthSystems_t systems, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { dcgmReturn_t dcgmReturn; std::vector::iterator switchIter; unsigned int i; bool updateOnFirstWatch = false; /* All of the callers of this call UpdateFields() right after */ bool wereFirstWatcher = false; for (switchIter = groupSwitchIds.begin(); switchIter != groupSwitchIds.end(); ++switchIter) { if (systems & DCGM_HEALTH_WATCH_NVSWITCH_NONFATAL) { for (i = 0; i < m_nvSwitchNonFatalFieldIds.size(); i++) { dcgmReturn = mpCoreProxy.AddFieldWatch(DCGM_FE_SWITCH, *switchIter, m_nvSwitchNonFatalFieldIds[i], updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (dcgmReturn != DCGM_ST_OK) { log_error("Error {} from AddEntityFieldWatch() for NvSwitch fields", (int)dcgmReturn); return dcgmReturn; } } } if (systems & DCGM_HEALTH_WATCH_NVSWITCH_FATAL) { for (i = 0; i < m_nvSwitchFatalFieldIds.size(); i++) { dcgmReturn = mpCoreProxy.AddFieldWatch(DCGM_FE_SWITCH, *switchIter, m_nvSwitchFatalFieldIds[i], updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (dcgmReturn != DCGM_ST_OK) { log_error("Error {} from AddEntityFieldWatch() for NvSwitch fields", (int)dcgmReturn); return dcgmReturn; } } } } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::AttachGpus() { unsigned int allGpuGroupId = DCGM_GROUP_ALL_GPUS; if (auto ret = mpCoreProxy.VerifyAndUpdateGroupId(&allGpuGroupId); ret != DCGM_ST_OK) { return ret; } dcgm_mutex_lock(m_mutex); if (!mGroupWatchState.contains(allGpuGroupId)) { dcgm_mutex_unlock(m_mutex); return DCGM_ST_OK; } auto healthWatchState = mGroupWatchState[allGpuGroupId]; dcgm_mutex_unlock(m_mutex); return SetWatches(allGpuGroupId, healthWatchState.systems, healthWatchState.connectionId, healthWatchState.updateInterval, healthWatchState.maxKeepAge); } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetWatches(unsigned int groupId, dcgmHealthSystems_t systems, dcgm_connection_id_t connectionId, long long updateInterval, double maxKeepAge) { dcgmReturn_t ret = DCGM_ST_OK; dcgmReturn_t tmpRet = DCGM_ST_OK; std::vector groupGpuIds; std::vector groupSwitchIds; DcgmWatcher watcher(DcgmWatcherTypeHealthWatch, connectionId); std::vector entities; ret = mpCoreProxy.GetGroupEntities(groupId, EntityListOption::ActiveOnly, entities); if (ret != DCGM_ST_OK) { log_error("Got st {} from GetGroupEntities()", ret); return ret; } dcgm_mutex_lock(m_mutex); mGroupWatchState[groupId] = HealthWatchState { systems, connectionId, updateInterval, maxKeepAge }; dcgm_mutex_unlock(m_mutex); /* Capture the entities that are GPUs as a separate list */ for (size_t i = 0; i < entities.size(); i++) { switch (entities[i].entityGroupId) { // Handle GPUs, GPU instances, and compute instances the same for setting watches case DCGM_FE_GPU: case DCGM_FE_GPU_I: case DCGM_FE_GPU_CI: for (unsigned int bitIndex = 0; bitIndex < DCGM_HEALTH_WATCH_COUNT_V2; bitIndex++) { unsigned int bit = 1 << bitIndex; switch (bit) { case DCGM_HEALTH_WATCH_PCIE: tmpRet = SetPcie(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; case DCGM_HEALTH_WATCH_MEM: tmpRet = SetMem(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; case DCGM_HEALTH_WATCH_INFOROM: tmpRet = SetInforom(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; case DCGM_HEALTH_WATCH_THERMAL: tmpRet = SetThermal(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; case DCGM_HEALTH_WATCH_POWER: tmpRet = SetPower(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; case DCGM_HEALTH_WATCH_NVLINK: tmpRet = SetNVLink(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; case DCGM_HEALTH_WATCH_DRIVER: tmpRet = SetDriver(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; default: // ignore everything else for now break; } if (DCGM_ST_OK != tmpRet) { log_error("Error {} from bit {}, entity group {} entityId {}", tmpRet, bit, entities[i].entityGroupId, entities[i].entityId); break; // exit on error } } break; case DCGM_FE_SWITCH: groupSwitchIds.push_back(entities[i].entityId); break; case DCGM_FE_LINK: /** * DCGM-2836. Examine what we should do in this case. Watch * for the health status of the associated switch or GPU, * perhaps? */ break; case DCGM_FE_CPU: for (unsigned int bitIndex = 0; bitIndex < DCGM_HEALTH_WATCH_COUNT_V2; bitIndex++) { unsigned int bit = 1 << bitIndex; switch (bit) { case DCGM_HEALTH_WATCH_THERMAL: tmpRet = SetCpuThermal(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; case DCGM_HEALTH_WATCH_POWER: tmpRet = SetCpuPower(entities[i].entityGroupId, entities[i].entityId, (systems & bit) ? true : false, watcher, updateInterval, maxKeepAge); break; default: // ignore everything else for now break; } if (DCGM_ST_OK != tmpRet) { log_error("Error {} from bit {}, entity group {} entityId {}", tmpRet, bit, entities[i].entityGroupId, entities[i].entityId); break; // exit on error } } break; default: // NO-OP break; } } // Set global (system-wide) health checks once per group, not per entity tmpRet = SetGlobalChecks(systems, watcher, updateInterval, maxKeepAge); if (tmpRet != DCGM_ST_OK) { log_error("Error {} from SetGlobalChecks", tmpRet); ret = tmpRet; } if (groupSwitchIds.size() > 0) { ret = SetNvSwitchWatches(groupSwitchIds, systems, watcher, updateInterval, maxKeepAge); } /* Make sure every field has updated */ tmpRet = mpCoreProxy.UpdateAllFields(1); if (tmpRet != DCGM_ST_OK) { DCGM_LOG_ERROR << "UpdateAllFields() returned " << tmpRet; ret = tmpRet; } return ret; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::GetWatches(unsigned int groupId, dcgmHealthSystems_t *systems) { dcgmReturn_t ret = DCGM_ST_OK; groupWatchTable_t::iterator groupWatchIter; std::vector entities; ret = mpCoreProxy.GetGroupEntities(groupId, EntityListOption::ActiveOnly, entities); if (ret != DCGM_ST_OK) { log_error("Got st {} from GetGroupEntities()", ret); return ret; } dcgm_mutex_lock(m_mutex); groupWatchIter = mGroupWatchState.find(groupId); if (groupWatchIter == mGroupWatchState.end()) { *systems = (dcgmHealthSystems_enum)0; } else { *systems = groupWatchIter->second.systems; } dcgm_mutex_unlock(m_mutex); return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorWatchesForGpu(unsigned int gpuId, long long startTime, long long endTime, dcgmHealthSystems_t healthSystemsMask, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; if (gpuId >= DCGM_MAX_NUM_DEVICES) { DCGM_LOG_ERROR << "Bad gpuId: " << gpuId; return DCGM_ST_BADPARAM; } auto const gpuStatus = mpCoreProxy.GetGpuStatus(gpuId); if (gpuStatus == DcgmEntityStatusDetached) { return DCGM_ST_GPU_IS_LOST; } MonitorDevastatingXids(DCGM_FE_GPU, gpuId, response); for (unsigned int index = 0; index < DCGM_HEALTH_WATCH_COUNT_V1; index++) { unsigned int bit = 1 << index; dcgmReturn_t tmpRet = DCGM_ST_OK; switch (bit) { case DCGM_HEALTH_WATCH_PCIE: if (bit & healthSystemsMask) tmpRet = MonitorPcie(DCGM_FE_GPU, gpuId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_MEM: if (bit & healthSystemsMask) tmpRet = MonitorMem(DCGM_FE_GPU, gpuId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_INFOROM: if (bit & healthSystemsMask) tmpRet = MonitorInforom(DCGM_FE_GPU, gpuId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_THERMAL: if (bit & healthSystemsMask) tmpRet = MonitorThermal(DCGM_FE_GPU, gpuId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_POWER: if (bit & healthSystemsMask) tmpRet = MonitorPower(DCGM_FE_GPU, gpuId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_NVLINK: if (bit & healthSystemsMask) tmpRet = MonitorNVLink(DCGM_FE_GPU, gpuId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_DRIVER: if (bit & healthSystemsMask) tmpRet = MonitorGpuRecoveryAction(DCGM_FE_GPU, gpuId, response); break; default: // ignore everything else for now, other bugs break; } if ((ret == DCGM_ST_OK) && (tmpRet != DCGM_ST_OK)) ret = tmpRet; } return ret; } bool DcgmHealthWatch::FitsGpuHardwareCheck(dcgm_field_entity_group_t entityGroupId) { return (entityGroupId == DCGM_FE_GPU || entityGroupId == DCGM_FE_GPU_I || entityGroupId == DCGM_FE_GPU_CI); } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorWatches(unsigned int groupId, long long startTime, long long endTime, DcgmHealthResponse &response) { unsigned int index; dcgmReturn_t ret = DCGM_ST_OK; std::vector entities; dcgmHealthSystems_t healthSystemsMask = (dcgmHealthSystems_t)0; /* Cached version of this group's watch mask */ std::vector gpuIds; /* Handle BLANK start-time and end-time */ if (DCGM_INT64_IS_BLANK(startTime)) startTime = 0; if (DCGM_INT64_IS_BLANK(endTime)) endTime = 0; ret = mpCoreProxy.GetGroupEntities(groupId, EntityListOption::ActiveOnly, entities); if (ret != DCGM_ST_OK) { log_error("Got st {} from GetGroupEntities()", ret); return ret; } dcgm_mutex_lock(m_mutex); groupWatchTable_t::iterator groupWatchIter = mGroupWatchState.find(groupId); if (groupWatchIter != mGroupWatchState.end()) { healthSystemsMask = groupWatchIter->second.systems; log_debug("Found health systems mask {:X} for groupId {}", (unsigned int)healthSystemsMask, groupId); } else { log_debug("Found NO health systems mask for groupId {}", groupId); } dcgm_mutex_unlock(m_mutex); if (healthSystemsMask == 0) return DCGM_ST_OK; /* This is the same as walking over the loops below and doing nothing */ // Perform global (system-wide) health checks before per-entity checks dcgmReturn_t globalRet = MonitorGlobalHealthChecks(healthSystemsMask, response); if (globalRet != DCGM_ST_OK) { log_debug("Global health checks returned error {}", (int)globalRet); ret = globalRet; } for (size_t entityIndex = 0; entityIndex < entities.size(); entityIndex++) { dcgm_field_entity_group_t entityGroupId = entities[entityIndex].entityGroupId; dcgm_field_eid_t entityId = entities[entityIndex].entityId; MonitorDevastatingXids(entityGroupId, entityId, response); for (index = 0; index < DCGM_HEALTH_WATCH_COUNT_V2; index++) { unsigned int bit = 1 << index; switch (bit) { case DCGM_HEALTH_WATCH_PCIE: if (bit & healthSystemsMask && FitsGpuHardwareCheck(entityGroupId)) ret = MonitorPcie(entityGroupId, entityId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_MEM: if (bit & healthSystemsMask && FitsGpuHardwareCheck(entityGroupId)) ret = MonitorMem(entityGroupId, entityId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_INFOROM: if (bit & healthSystemsMask && FitsGpuHardwareCheck(entityGroupId)) ret = MonitorInforom(entityGroupId, entityId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_THERMAL: if (bit & healthSystemsMask && FitsGpuHardwareCheck(entityGroupId)) { ret = MonitorThermal(entityGroupId, entityId, startTime, endTime, response); if (ret != DCGM_ST_OK) { break; } } if (bit & healthSystemsMask && entityGroupId == DCGM_FE_CPU) { ret = MonitorCpuThermal(entityGroupId, entityId, startTime, endTime, response); } break; case DCGM_HEALTH_WATCH_POWER: if (bit & healthSystemsMask && FitsGpuHardwareCheck(entityGroupId)) { ret = MonitorPower(entityGroupId, entityId, startTime, endTime, response); if (ret != DCGM_ST_OK) { break; } } if (bit & healthSystemsMask && entityGroupId == DCGM_FE_CPU) { ret = MonitorCpuPower(entityGroupId, entityId, startTime, endTime, response); } break; case DCGM_HEALTH_WATCH_NVLINK: if (bit & healthSystemsMask && FitsGpuHardwareCheck(entityGroupId)) ret = MonitorNVLink(entityGroupId, entityId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_DRIVER: if (bit & healthSystemsMask && FitsGpuHardwareCheck(entityGroupId)) ret = MonitorGpuRecoveryAction(entityGroupId, entityId, response); break; case DCGM_HEALTH_WATCH_NVSWITCH_NONFATAL: if (bit & healthSystemsMask && entityGroupId == DCGM_FE_SWITCH) ret = MonitorNvSwitchErrorCounts(false, entityGroupId, entityId, startTime, endTime, response); break; case DCGM_HEALTH_WATCH_NVSWITCH_FATAL: if (bit & healthSystemsMask && entityGroupId == DCGM_FE_SWITCH) ret = MonitorNvSwitchErrorCounts(true, entityGroupId, entityId, startTime, endTime, response); break; default: // reduce the logging level as this may pollute the log file if unsupported fields are watched // continuously. log_debug("Unhandled health bit {}", bit); break; } } } return ret; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorGlobalHealthChecks(dcgmHealthSystems_t healthSystemsMask, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; if (healthSystemsMask & DCGM_HEALTH_WATCH_NVLINK) { dcgmReturn_t tmpRet = MonitorImex(DCGM_FE_NONE, 0, response); if (tmpRet != DCGM_ST_OK) { log_error("Got error {} from MonitorImex (global check)", (int)tmpRet); ret = tmpRet; } } return ret; } std::string DcgmHealthWatch::GetHealthSystemAsString(dcgmHealthSystems_t system) { switch (system) { case DCGM_HEALTH_WATCH_PCIE: return "PCIe"; break; // NOT REACHED case DCGM_HEALTH_WATCH_NVLINK: return "NVLink"; break; // NOT REACHED case DCGM_HEALTH_WATCH_PMU: return "PMU"; break; // NOT REACHED case DCGM_HEALTH_WATCH_MCU: return "MCU"; break; // NOT REACHED case DCGM_HEALTH_WATCH_MEM: return "Memory"; break; // NOT REACHED case DCGM_HEALTH_WATCH_SM: return "SM"; break; // NOT REACHED case DCGM_HEALTH_WATCH_INFOROM: return "Inforom"; break; // NOT REACHED case DCGM_HEALTH_WATCH_THERMAL: return "Thermal"; break; // NOT REACHED case DCGM_HEALTH_WATCH_POWER: return "Power"; break; // NOT REACHED case DCGM_HEALTH_WATCH_DRIVER: return "Driver"; break; // NOT REACHED case DCGM_HEALTH_WATCH_NVSWITCH_NONFATAL: return "NVSwitch non-fatal errors"; break; // NOT REACHED case DCGM_HEALTH_WATCH_NVSWITCH_FATAL: return "NVSwitch fatal errors"; break; // NOT REACHED default: return "Unknown"; break; // NOT REACHED } } std::string DcgmHealthWatch::GetHealthResultAsString(dcgmHealthWatchResults_t result) { switch (result) { case DCGM_HEALTH_RESULT_PASS: return "PASS"; break; // NOT REACHED case DCGM_HEALTH_RESULT_WARN: return "WARNING"; break; // NOT REACHED case DCGM_HEALTH_RESULT_FAIL: return "FAILURE"; break; // NOT REACHED default: return "UNKNOWN"; break; // NOT REACHED } } /*****************************************************************************/ void DcgmHealthWatch::SetResponse(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, dcgmHealthWatchResults_t status, dcgmHealthSystems_t system, DcgmError &d, DcgmHealthResponse &response) { dcgmDiagErrorDetail_t error; snprintf(error.msg, sizeof(error.msg), "%s", d.GetMessage().c_str()); error.code = d.GetCode(); response.AddIncident(system, status, error, entityGroupId, entityId); log_error("Detected a {} in health system {}: '{}'", GetHealthResultAsString(status), GetHealthSystemAsString(system), d.GetMessage()); } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetPcie(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, bool enable, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { // currently if a watch is removed it removes for the entire system (i.e. no reference counter) // thus ignore the "enable" flag for now dcgmReturn_t ret = DCGM_ST_OK; if (!enable) // ignore return ret; ADD_WATCH(DCGM_FI_DEV_PCIE_REPLAY_COUNTER); ADD_WATCH(DCGM_FI_DEV_PCIE_LINK_GEN); ADD_WATCH(DCGM_FI_DEV_PCIE_LINK_WIDTH); return ret; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetMem(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, bool enable, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { // currently if a watch is removed it removes for the entire system (i.e. no reference counter) // thus ignore the "enable" flag for now dcgmReturn_t ret = DCGM_ST_OK; if (!enable) // ignore return ret; ADD_WATCH(DCGM_FI_DEV_ECC_DBE_VOL_TOTAL); // the sampling of 1 second is fine for the above, these however should have a longer sampling rate updateInterval = std::max(30000000ll, updateInterval); bool updateOnFirstWatch = false; /* The caller calls UpdateFields() after this */ bool wereFirstWatcher = false; // single and double bit retired pages ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_RETIRED_SBE, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_RETIRED_SBE, EntityToString(entityGroupId), entityId); return ret; } ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_RETIRED_DBE, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_RETIRED_DBE, EntityToString(entityGroupId), entityId); return ret; } ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_RETIRED_PENDING, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_RETIRED_PENDING, EntityToString(entityGroupId), entityId); return ret; } /* Note that we're subscribing for XID updates so that OnFieldValuesUpdate and eventually ProcessXidFv get called */ ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_XID_ERRORS, updateInterval, maxKeepAge, 0, watcher, true, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_XID_ERRORS, EntityToString(entityGroupId), entityId); return ret; } ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_ROW_REMAP_FAILURE, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_ROW_REMAP_FAILURE, EntityToString(entityGroupId), entityId); return ret; } ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG, EntityToString(entityGroupId), entityId); return ret; } return ret; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetDriver(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, bool enableWatch, DcgmWatcher const &watcher, long long updateInterval, double maxKeepAge) { bool updateOnFirstWatch = true; bool wereFirstWatcher = false; dcgmReturn_t ret; if (!enableWatch) { mpCoreProxy.RemoveFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION, 0, watcher); return DCGM_ST_OK; } // Watch GPU recovery action field ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (ret != DCGM_ST_OK) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION, EntityToString(entityGroupId), entityId); return ret; } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetInforom(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, bool enable, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { // currently if a watch is removed it removes for the entire system (i.e. no reference counter) // thus ignore the "enable" flag for now dcgmReturn_t ret = DCGM_ST_OK; if (!enable) // ignore return ret; updateInterval = std::max(3600000000ll, updateInterval); maxKeepAge = std::max(7200.0, maxKeepAge); /* Keep at least 2 hours of data so we can get a sample */ bool updateOnFirstWatch = false; /* The caller calls UpdateFields() after this */ bool wereFirstWatcher = false; ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_INFOROM_CONFIG_VALID, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_INFOROM_CONFIG_VALID, EntityToString(entityGroupId), entityId); return ret; } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetThermal(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, bool enable, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { // currently if a watch is removed it removes for the entire system (i.e. no reference counter) // thus ignore the "enable" flag for now dcgmReturn_t ret = DCGM_ST_OK; if (!enable) // ignore return ret; /* Enforce a minimum sample rate of every 30 seconds */ updateInterval = std::max(30000000ll, updateInterval); bool updateOnFirstWatch = false; /* The caller calls UpdateFields() after this */ bool wereFirstWatcher = false; ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_THERMAL_VIOLATION, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_THERMAL_VIOLATION, EntityToString(entityGroupId), entityId); return ret; } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetPower(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, bool enable, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { // currently if a watch is removed it removes for the entire system (i.e. no reference counter) // thus ignore the "enable" flag for now dcgmReturn_t ret = DCGM_ST_OK; if (!enable) // ignore return ret; /* Enforce a minimum sample rate of every 30 seconds */ updateInterval = std::max(30000000ll, updateInterval); bool updateOnFirstWatch = false; /* The caller calls UpdateFields() after this */ bool wereFirstWatcher = false; ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_POWER_VIOLATION, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_POWER_VIOLATION, EntityToString(entityGroupId), entityId); return ret; } ret = mpCoreProxy.AddFieldWatch(entityGroupId, entityId, DCGM_FI_DEV_POWER_USAGE, updateInterval, maxKeepAge, 0, watcher, false, updateOnFirstWatch, wereFirstWatcher); if (DCGM_ST_OK != ret) { log_error("Failed to set watch for field {} on {} {}", DCGM_FI_DEV_POWER_USAGE, EntityToString(entityGroupId), entityId); return ret; } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetCpuThermal(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, bool enable, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { // currently if a watch is removed it removes for the entire system (i.e. no reference counter) // thus ignore the "enable" flag for now dcgmReturn_t ret = DCGM_ST_OK; if (!enable) // ignore return ret; /* Enforce a minimum sample rate of every 30 seconds */ updateInterval = std::max(30000000ll, updateInterval); ADD_WATCH(DCGM_FI_DEV_CPU_TEMP_CURRENT); ADD_WATCH(DCGM_FI_DEV_CPU_TEMP_WARNING); ADD_WATCH(DCGM_FI_DEV_CPU_TEMP_CRITICAL); return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetCpuPower(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, bool enable, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { // currently if a watch is removed it removes for the entire system (i.e. no reference counter) // thus ignore the "enable" flag for now dcgmReturn_t ret = DCGM_ST_OK; if (!enable) // ignore return ret; /* Enforce a minimum sample rate of every 30 seconds */ updateInterval = std::max(30000000ll, updateInterval); ADD_WATCH(DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT); ADD_WATCH(DCGM_FI_DEV_CPU_POWER_LIMIT); return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetNVLink(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, bool enable, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { // currently if a watch is removed it removes for the entire system (i.e. no reference counter) // thus ignore the "enable" flag for now dcgmReturn_t ret = DCGM_ST_OK; if (!enable) // ignore return ret; ADD_WATCH(DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL); ADD_WATCH(DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL); ADD_WATCH(DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL); ADD_WATCH(DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_RX_SYMBOL_ERRORS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_RX_MALFORMED_PACKET_ERRORS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_RX_BUFFER_OVERRUN_ERRORS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_RX_REMOTE_ERRORS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_RX_GENERAL_ERRORS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS); ADD_WATCH(DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER); ADD_WATCH(DCGM_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL); ADD_WATCH(DCGM_FI_DEV_FABRIC_HEALTH_MASK); ADD_WATCH(DCGM_FI_DEV_FABRIC_MANAGER_STATUS); return ret; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::SetGlobalChecks(dcgmHealthSystems_t systems, DcgmWatcher watcher, long long updateInterval, double maxKeepAge) { dcgmReturn_t ret = DCGM_ST_OK; // Add global watches for IMEX status fields (part of NVLink health system) if (systems & DCGM_HEALTH_WATCH_NVLINK) { ADD_GLOBAL_WATCH(DCGM_FI_IMEX_DOMAIN_STATUS); ADD_GLOBAL_WATCH(DCGM_FI_IMEX_DAEMON_STATUS); } // Future global checks can be added here return ret; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::GetExpectedPcieReplayRate(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, int &expectedPcieReplayRate) { std::vector fieldIds { DCGM_FI_DEV_PCIE_LINK_GEN, DCGM_FI_DEV_PCIE_LINK_WIDTH }; std::unordered_map samples {}; dcgmReturn_t ret = DCGM_ST_OK; /* Get PCIe generation and lanes/width. */ for (auto fieldId : fieldIds) { ret = mpCoreProxy.GetLatestSample(entityGroupId, entityId, fieldId, &samples[fieldId], 0); if (DCGM_ST_NO_DATA == ret) { log_debug("No PCIe data for field {} of gpuId {}.", fieldId, entityId); return DCGM_ST_NO_DATA; } else if (DCGM_ST_NOT_WATCHED == ret) { log_warning("PCIe field {} not watched for gpuId {}", fieldId, entityId); return DCGM_ST_NOT_WATCHED; } else if (DCGM_ST_OK != ret) { log_error("Unable to retrieve PCIe field {} from cache for gpuId {}, returned {}", fieldId, entityId, ret); return ret; } if (DCGM_INT64_IS_BLANK(samples[fieldId].val.i64)) { log_debug("Blank PCIe data for field {} of gpuId {}.", fieldId, entityId); return DCGM_ST_NO_DATA; } } unsigned long long pcieGen = samples[DCGM_FI_DEV_PCIE_LINK_GEN].val.i64; unsigned long long pcieLanes = samples[DCGM_FI_DEV_PCIE_LINK_WIDTH].val.i64; /* * PCIe replay rate thresholds for each generation per lane. * Note that ((10^-12) * 10^9) is simpliefied as 10^-3 i.e. ( divide by 1000) in following table. */ static double pcieGenReplayRatePerLane[] = { 2.5 / 1000 * 60, /* Gen1 speed = 2.5 Gbps, (1x10^-12) * (2.5x10^9) * 60 = 0.15 errors/min per lane. */ 5.0 / 1000 * 60, /* Gen2 speed = 5 Gbps, (1x10^-12) * (5x10^9) * 60 = 0.3 errors/min per lane. */ 8.0 / 1000 * 60, /* Gen3 speed = 8 Gbps, (1x10^-12) * (8x10^9) * 60 = 0.48 errors/min per lane. */ 16.0 / 1000 * 60, /* Gen4 speed = 16 Gbps, (1x10^-12) * (16x10^9) * 60 = 0.96 errors/min per lane. */ 32.0 / 1000 * 60, /* Gen5 speed = 32 Gbps, (1x10^-12) * (32x10^9) * 60 = 1.92 errors/min per lane. */ 64.0 / 1000 * 60 /* Gen6 speed = 64 Gbps, (1x10^-12) * (64x10^9) * 60 = 3.84 errors/min per lane. */ }; unsigned int pcieGenMax = (sizeof(pcieGenReplayRatePerLane) / sizeof(pcieGenReplayRatePerLane[0])); if (pcieGen > pcieGenMax) { /* Cap the version at the max we know about. */ log_warning("PCIe generation {} is more than max we know {}, capping it to max.", pcieGen, pcieGenMax); pcieGen = pcieGenMax; } /* Expected replay rate = Replay rate per lane * Num of pcie Lanes. Note that value is ceiled. */ expectedPcieReplayRate = std::ceil(pcieGenReplayRatePerLane[pcieGen - 1] * pcieLanes); log_debug("expected PCIe replay rate is {} for gen {}, replay rate per lane {} and lanes {}", expectedPcieReplayRate, pcieGen, pcieGenReplayRatePerLane[pcieGen - 1], pcieLanes); return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorPcie(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; unsigned short fieldId = DCGM_FI_DEV_PCIE_REPLAY_COUNTER; dcgmcm_sample_t startValue = {}; dcgmcm_sample_t endValue = {}; int count = 0; unsigned int oneMinuteInUsec = 60000000; timelib64_t now = timelib_usecSince1970(); // Add XID monitoring for PCIe subsystem MonitorSubsystemXids(entityGroupId, entityId, DCGM_HEALTH_WATCH_PCIE, response); /* Update the start time if it is blank. Allow endTime to be in the future */ if (!startTime) { startTime = now - oneMinuteInUsec; } /* Get the value of the field at the StartTime*/ count = 1; ret = mpCoreProxy.GetSamples( entityGroupId, entityId, fieldId, &startValue, &count, startTime, endTime, DCGM_ORDER_ASCENDING); if (DCGM_ST_NO_DATA == ret) { log_debug("No data for PCIe for gpuId {}", entityId); return DCGM_ST_OK; } else if (DCGM_ST_NOT_WATCHED == ret) { log_warning("PCIe not watched for gpuId {}", entityId); return DCGM_ST_OK; } else if (DCGM_ST_OK != ret) { log_error("mpCoreProxy.GetSamples returned {} for gpuId {}", (int)ret, entityId); return ret; } if (DCGM_INT64_IS_BLANK(startValue.val.i64)) return DCGM_ST_OK; /* Get the value of the field at the endTime*/ count = 1; ret = mpCoreProxy.GetSamples( entityGroupId, entityId, fieldId, &endValue, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (DCGM_ST_NO_DATA == ret) { log_debug("No data for PCIe for gpuId {}", entityId); return DCGM_ST_OK; } else if (DCGM_ST_NOT_WATCHED == ret) { log_warning("PCIe not watched for gpuId {}", entityId); return DCGM_ST_OK; } else if (DCGM_ST_OK != ret) { log_error("mpCoreProxy.GetSamples returned {} for gpuId {}", (int)ret, entityId); return ret; } if (DCGM_INT64_IS_BLANK(endValue.val.i64)) return DCGM_ST_OK; // NO DATA is handled automatically so here we can assume we have the values from the last minute // both values have been checked for BLANK values so can be used here int pciReplayRate = (startValue.val.i64 >= endValue.val.i64) ? (startValue.val.i64 - endValue.val.i64) : (endValue.val.i64 - startValue.val.i64); /* Get expected replay rate which depends on the PCIe generation and number of lanes. */ int expectedPcieReplayRate = 0; ret = GetExpectedPcieReplayRate(entityGroupId, entityId, expectedPcieReplayRate); if (DCGM_ST_OK != ret) { log_debug("GetExpectedPcieReplayRate returned, ret {}.", ret); if (DCGM_ST_NO_DATA == ret || DCGM_ST_NOT_WATCHED == ret) return DCGM_ST_OK; return ret; } if (pciReplayRate > expectedPcieReplayRate) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_PCI_REPLAY_RATE, d, expectedPcieReplayRate, entityId, pciReplayRate); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_PCIE, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ std::string DcgmHealthWatch::MemFieldToString(unsigned short fieldId) { switch (fieldId) { case DCGM_FI_DEV_ECC_SBE_VOL_TOTAL: return "Volatile SBEs"; case DCGM_FI_DEV_ECC_DBE_VOL_TOTAL: return "Volatile DBEs"; default: return "Error"; } } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorMemVolatileDbes(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { // first handle the actual error counts // if our stored value is greater than the returned value then someone likely // reset the volatile counter. Just reset ours dcgmReturn_t ret; int count = 1; dcgmcm_sample_t sample = {}; ret = mpCoreProxy.GetSamples(entityGroupId, entityId, DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, &sample, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA && ret != DCGM_ST_NOT_WATCHED) { DCGM_LOG_ERROR << "GetSamples got ret " << errorString(ret); return ret; } if (DCGM_INT64_IS_BLANK(sample.val.i64)) { DCGM_LOG_DEBUG << "DCGM_FI_DEV_ECC_DBE_VOL_TOTAL was blank for eg " << entityGroupId << ", eid " << entityId; return DCGM_ST_OK; } // Fail for any volatile DBEs if (sample.val.i64 > 0) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE( DCGM_FR_VOLATILE_DBE_DETECTED, d, static_cast(sample.val.i64), entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_MEM, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorMemRetiredPending(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmcm_sample_t retiredPending = {}; int count = 1; dcgmReturn_t ret = mpCoreProxy.GetSamples(entityGroupId, entityId, DCGM_FI_DEV_RETIRED_PENDING, &retiredPending, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA && ret != DCGM_ST_NOT_WATCHED) { DCGM_LOG_ERROR << "GetSamples got ret " << errorString(ret); return ret; } if (!DCGM_INT64_IS_BLANK(retiredPending.val.i64) && retiredPending.val.i64 != 0) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_PENDING_PAGE_RETIREMENTS, d, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_MEM, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorMemSbeDbeRetiredPages(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmcm_sample_t sbeRetiredPage = {}; dcgmcm_sample_t dbeRetiredPage = {}; int count = 1; dcgmReturn_t ret = mpCoreProxy.GetSamples(entityGroupId, entityId, DCGM_FI_DEV_RETIRED_DBE, &dbeRetiredPage, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA && ret != DCGM_ST_NOT_WATCHED) { DCGM_LOG_ERROR << "GetSamples got ret " << errorString(ret); return ret; } count = 1; ret = mpCoreProxy.GetSamples(entityGroupId, entityId, DCGM_FI_DEV_RETIRED_SBE, &sbeRetiredPage, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA && ret != DCGM_ST_NOT_WATCHED) { DCGM_LOG_ERROR << "GetSamples got ret " << errorString(ret); return ret; } long long totalRetiredPages = 0; if (!DCGM_INT64_IS_BLANK(sbeRetiredPage.val.i64)) { totalRetiredPages += sbeRetiredPage.val.i64; } if (!DCGM_INT64_IS_BLANK(dbeRetiredPage.val.i64)) { totalRetiredPages += dbeRetiredPage.val.i64; } // the combined total of retired pages should not be more than or equal to DCGM_LIMIT_MAX_RETIRED_PAGES // which is set via bug 1665722 if (totalRetiredPages >= DCGM_LIMIT_MAX_RETIRED_PAGES) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_RETIRED_PAGES_LIMIT, d, DCGM_LIMIT_MAX_RETIRED_PAGES, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_MEM, d, response); return ret; } // The dbe retired pages should not be more than DCGM_LIMIT_MAX_RETIRED_PAGES_SOFT_LIMIT // *AND* be accumulating more than 1 per week after the limit has been met // JIRA DCGM-458 if (!DCGM_INT64_IS_BLANK(dbeRetiredPage.val.i64) && dbeRetiredPage.val.i64 > DCGM_LIMIT_MAX_RETIRED_PAGES_SOFT_LIMIT) { // Check whether the rate of continuing page retirments (after the SOFT_LIMIT) meets the failure condition. dcgmReturn_t localReturn = DCGM_ST_OK; dcgmcm_sample_t oneWeekAgoDbeRetiredPages = {}; timelib64_t oneWeekInUsec = 604800000000; timelib64_t now = timelib_usecSince1970(); count = 1; // Get the number of dbe retired pages before current week localReturn = mpCoreProxy.GetSamples(entityGroupId, entityId, DCGM_FI_DEV_RETIRED_DBE, &oneWeekAgoDbeRetiredPages, &count, 0, now - oneWeekInUsec, DCGM_ORDER_DESCENDING); if (localReturn != DCGM_ST_OK && localReturn != DCGM_ST_NO_DATA) { DCGM_LOG_ERROR << "GetSamples got ret " << errorString(ret); return ret; } if (DCGM_INT64_IS_BLANK(oneWeekAgoDbeRetiredPages.val.i64)) { DCGM_LOG_DEBUG << "oneWeekAgoDbeRetiredPages was blank"; return DCGM_ST_OK; } int64_t dbePagesRetiredThisWeek = dbeRetiredPage.val.i64 - oneWeekAgoDbeRetiredPages.val.i64; if (dbePagesRetiredThisWeek > 1) { // More than one page retired due to DBE in the past week, failure condition met. DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE( DCGM_FR_RETIRED_PAGES_DBE_LIMIT, d, DCGM_LIMIT_MAX_RETIRED_PAGES_SOFT_LIMIT, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_MEM, d, response); } } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorMemRowRemapFailures(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { // first handle the actual error counts // if our stored value is greater than the returned value then someone likely // reset the volatile counter. Just reset ours dcgmReturn_t ret; int count = 1; dcgmcm_sample_t sample = {}; ret = mpCoreProxy.GetSamples(entityGroupId, entityId, DCGM_FI_DEV_ROW_REMAP_FAILURE, &sample, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA && ret != DCGM_ST_NOT_WATCHED) { DCGM_LOG_ERROR << "GetSamples got ret " << errorString(ret); return ret; } if (DCGM_INT64_IS_BLANK(sample.val.i64)) { DCGM_LOG_DEBUG << "DCGM_FI_DEV_ROW_REMAP_FAILURE was blank for eg " << entityGroupId << ", eid " << entityId; return DCGM_ST_OK; } // Fail for any volatile DBEs if (sample.val.i64 > 0) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_ROW_REMAP_FAILURE, d); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_MEM, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorMemUnrepairableFlag(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, DcgmHealthResponse &response) { dcgmReturn_t ret; dcgmcm_sample_t sample = {}; ret = mpCoreProxy.GetLatestSample(entityGroupId, entityId, DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG, &sample, nullptr); if (ret == DCGM_ST_NO_DATA) { log_debug("No data for unrepairable memory flag for gpuId {}", entityId); return DCGM_ST_OK; } else if (ret == DCGM_ST_NOT_WATCHED) { log_warning("Not watched for unrepairable memory flag for gpuId {}", entityId); return DCGM_ST_OK; } else if (ret != DCGM_ST_OK) { log_error("Unable to retrieve field DCGM_FI_DEV_MEMORY_UNREPAIRABLE_FLAG from cache. gpuId {}", entityId); return ret; } if (DCGM_INT64_IS_BLANK(sample.val.i64)) { return DCGM_ST_OK; } // Fail if unrepairable memory flag is set (non-zero) if (sample.val.i64 != 0) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_FAULTY_MEMORY, d, sample.val.i64, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_MEM, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorImex(dcgm_field_entity_group_t /* entityGroupId */, dcgm_field_eid_t /* entityId */, DcgmHealthResponse &response) { dcgmReturn_t ret; dcgmcm_sample_t sample = {}; dcgm_field_eid_t constexpr normalizedEntityId = 0; dcgm_field_entity_group_t constexpr normalizedEntityGroupId = DCGM_FE_NONE; // Check IMEX domain status (string field) ret = mpCoreProxy.GetLatestSample( normalizedEntityGroupId, normalizedEntityId, DCGM_FI_IMEX_DOMAIN_STATUS, &sample, nullptr); if (ret == DCGM_ST_NO_DATA || ret == DCGM_ST_NOT_WATCHED) { log_debug("No data or not watched for IMEX domain status (global scope)"); // Continue to check daemon status } else if (ret != DCGM_ST_OK) { log_warning("Unable to retrieve DCGM_FI_IMEX_DOMAIN_STATUS from cache (global scope)"); return ret; } else if (!DCGM_STR_IS_BLANK(sample.val.str)) { // Check if domain status is healthy // PASS: UP, NOT_INSTALLED, NOT_CONFIGURED, UNAVAILABLE // FAIL: DOWN, DEGRADED auto const domainStatus = std::string_view(sample.val.str); bool isDomainHealthy = (domainStatus == "UP") || (domainStatus == "NOT_INSTALLED") || (domainStatus == "NOT_CONFIGURED") || (domainStatus == "UNAVAILABLE"); if (!isDomainHealthy) { DcgmError d { normalizedEntityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_IMEX_UNHEALTHY, d, "domain", sample.val.str, "unhealthy"); SetResponse(normalizedEntityGroupId, normalizedEntityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); } } // Check IMEX daemon status (int64 field) sample = {}; ret = mpCoreProxy.GetLatestSample( normalizedEntityGroupId, normalizedEntityId, DCGM_FI_IMEX_DAEMON_STATUS, &sample, nullptr); if (ret == DCGM_ST_NO_DATA || ret == DCGM_ST_NOT_WATCHED) { log_debug("No data or not watched for IMEX daemon status (global scope)"); return DCGM_ST_OK; } else if (ret != DCGM_ST_OK) { log_warning("Unable to retrieve DCGM_FI_IMEX_DAEMON_STATUS from cache (global scope)"); return ret; } if (DCGM_INT64_IS_BLANK(sample.val.i64)) { return DCGM_ST_OK; } // Check if daemon status is healthy // PASS: UNAVAILABLE (7), READY (5), NOT_INSTALLED (-1), NOT_CONFIGURED (-2) // FAIL: All other states (INITIALIZING, STARTING_AUTH_SERVER, etc.) DcgmImexDaemonStatus const daemonStatus = static_cast(sample.val.i64); bool const isDaemonHealthy = (daemonStatus == DcgmImexDaemonStatus::UNAVAILABLE) || (daemonStatus == DcgmImexDaemonStatus::READY) || (daemonStatus == DcgmImexDaemonStatus::NOT_INSTALLED) || (daemonStatus == DcgmImexDaemonStatus::NOT_CONFIGURED); if (!isDaemonHealthy) { DcgmError d { normalizedEntityId }; auto const statusStr = std::to_string(sample.val.i64); DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_IMEX_UNHEALTHY, d, "daemon", statusStr.c_str(), "not READY"); SetResponse(normalizedEntityGroupId, normalizedEntityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorGpuRecoveryAction(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, DcgmHealthResponse &response) { dcgmReturn_t ret; dcgmcm_sample_t sample = {}; ret = mpCoreProxy.GetLatestSample(entityGroupId, entityId, DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION, &sample, nullptr); if (ret == DCGM_ST_NO_DATA) { log_verbose("No data for GPU recovery action for eg {} eid {}", entityGroupId, entityId); return DCGM_ST_OK; } else if (ret == DCGM_ST_NOT_WATCHED) { log_debug("Not watched for GPU recovery action for eg {} eid {}", entityGroupId, entityId); return DCGM_ST_OK; } else if (ret != DCGM_ST_OK) { log_warning( "Unable to retrieve DCGM_FI_DEV_GET_GPU_RECOVERY_ACTION from cache. eg {} eid {}", entityGroupId, entityId); return ret; } if (DCGM_INT64_IS_BLANK(sample.val.i64)) { return DCGM_ST_OK; } if (sample.val.i64 < NVML_GPU_RECOVERY_ACTION_NONE || sample.val.i64 > NVML_GPU_RECOVERY_ACTION_DRAIN_AND_RESET) { log_warning("Invalid GPU recovery action value {} for eg {} eid {}", sample.val.i64, entityGroupId, entityId); return DCGM_ST_OK; } auto recoveryAction = static_cast(sample.val.i64); switch (recoveryAction) { case NVML_GPU_RECOVERY_ACTION_NONE: // Healthy - no action needed break; case NVML_GPU_RECOVERY_ACTION_GPU_RESET: { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_GPU_RECOVERY_RESET, d, entityId, sample.val.i64); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_DRIVER, d, response); break; } case NVML_GPU_RECOVERY_ACTION_NODE_REBOOT: { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_GPU_RECOVERY_REBOOT, d, entityId, sample.val.i64); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_DRIVER, d, response); break; } case NVML_GPU_RECOVERY_ACTION_DRAIN_P2P: { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_GPU_RECOVERY_DRAIN_P2P, d, entityId, sample.val.i64); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_DRIVER, d, response); break; } case NVML_GPU_RECOVERY_ACTION_DRAIN_AND_RESET: { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_GPU_RECOVERY_DRAIN_RESET, d, entityId, sample.val.i64); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_DRIVER, d, response); break; } } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorMem(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; dcgmReturn_t localReturn = DCGM_ST_OK; unsigned int oneMinuteInUsec = 60000000; timelib64_t now = timelib_usecSince1970(); // Add XID monitoring for Memory subsystem MonitorSubsystemXids(entityGroupId, entityId, DCGM_HEALTH_WATCH_MEM, response); /* Update the start time if it is blank */ if (!startTime) { startTime = now - oneMinuteInUsec; } /* Note: Allow endTime to be in the future. 0 = blank = most recent record in time series */ localReturn = MonitorMemVolatileDbes(entityGroupId, entityId, startTime, endTime, response); if (localReturn != DCGM_ST_OK) { ret = localReturn; } localReturn = MonitorMemRetiredPending(entityGroupId, entityId, startTime, endTime, response); if (localReturn != DCGM_ST_OK) { ret = localReturn; } localReturn = MonitorMemSbeDbeRetiredPages(entityGroupId, entityId, startTime, endTime, response); if (localReturn != DCGM_ST_OK) { ret = localReturn; } localReturn = MonitorMemRowRemapFailures(entityGroupId, entityId, startTime, endTime, response); if (localReturn != DCGM_ST_OK) { ret = localReturn; } localReturn = MonitorMemUnrepairableFlag(entityGroupId, entityId, response); if (localReturn != DCGM_ST_OK) { ret = localReturn; } return ret; } namespace { /** * Collect latest sample for a given field. * * @param coreProxy[in] : The cache manager proxy * @param entityGroupId[in] : The entity group ID * @param entityId[in] : The entity ID * @param fieldId[in] : The field ID * @param sample[out] : The sample * * @return expected, error> where: * - Success with valid data: expected contains optional with sample * - Success with no data, not watched: expected contains empty optional * - Success with blank value, not supported: expected contains empty optional * - Error: expected contains error code */ std::expected, dcgmReturn_t> CollectLatestInt64Sample( DcgmCoreProxy &coreProxy, dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, unsigned short fieldId) { dcgmcm_sample_t sample {}; dcgmReturn_t ret = coreProxy.GetLatestSample(entityGroupId, entityId, fieldId, &sample, 0); if (ret == DCGM_ST_NO_DATA || ret == DCGM_ST_NOT_WATCHED) { log_debug("{} for fieldId {} for eg {} eid {}", ret == DCGM_ST_NO_DATA ? "No data" : "Not watched", fieldId, entityGroupId, entityId); return std::nullopt; // Success, but no data available } if (ret != DCGM_ST_OK) { log_warning( "Got error {} from GetLatestSample for fieldId {} for eg {} eid {}", ret, fieldId, entityGroupId, entityId); return std::unexpected(ret); // Actual error } if (DCGM_INT64_IS_BLANK(sample.val.i64) || sample.val.i64 == DCGM_INT64_NOT_SUPPORTED) { log_debug("Got {} for fieldId {} for eg {} eid {}", DCGM_INT64_IS_BLANK(sample.val.i64) ? "Blank" : "Not Supported", fieldId, entityGroupId, entityId); return std::nullopt; // Success, but no data available } return sample; // Success with valid data } } // namespace dcgmReturn_t DcgmHealthWatch::MonitorInforom(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long /* startTime */, long long /* endTime */, DcgmHealthResponse &response) { unsigned short fieldId = DCGM_FI_DEV_INFOROM_CONFIG_VALID; // Add XID monitoring for InfoROM subsystem MonitorSubsystemXids(entityGroupId, entityId, DCGM_HEALTH_WATCH_INFOROM, response); auto const result = CollectLatestInt64Sample(mpCoreProxy, entityGroupId, entityId, fieldId); if (!result.has_value()) { return result.error(); } if (!result.value().has_value()) { return DCGM_ST_OK; } auto const &sample = result.value().value(); if (!(sample.val.i64)) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_CORRUPT_INFOROM, d, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_INFOROM, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorThermal(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; unsigned short fieldId = DCGM_FI_DEV_THERMAL_VIOLATION; dcgmcm_sample_t startValue = {}; dcgmcm_sample_t endValue = {}; int count = 1; long long int violationTime = 0; timelib64_t now = timelib_usecSince1970(); unsigned int oneMinuteInUsec = 60000000; // Add XID monitoring for Thermal subsystem MonitorSubsystemXids(entityGroupId, entityId, DCGM_HEALTH_WATCH_THERMAL, response); /* Update the start and the end time if they are blank */ if (!startTime) { startTime = now - oneMinuteInUsec; } /* Note: Allow endTime to be in the future. 0 = blank = most recent record in time series */ /* Get the value at the startTime */ ret = mpCoreProxy.GetSamples( entityGroupId, entityId, fieldId, &startValue, &count, startTime, endTime, DCGM_ORDER_ASCENDING); if (DCGM_ST_NO_DATA == ret) return DCGM_ST_OK; if (DCGM_ST_NOT_WATCHED == ret) { return DCGM_ST_OK; } if (DCGM_ST_OK != ret) return ret; if (DCGM_INT64_IS_BLANK(startValue.val.i64)) return DCGM_ST_OK; /* Get the value at the endTime*/ ret = mpCoreProxy.GetLatestSample(entityGroupId, entityId, fieldId, &endValue, 0); if (DCGM_ST_NO_DATA == ret) return DCGM_ST_OK; if (DCGM_ST_NOT_WATCHED == ret) { return DCGM_ST_OK; } if (DCGM_ST_OK != ret) return ret; if (DCGM_INT64_IS_BLANK(endValue.val.i64)) return DCGM_ST_OK; // NO DATA is handled automatically so here we can assume we have the values from the last minute // both values have been checked for BLANK values so can be used here violationTime = startValue.val.i64 >= endValue.val.i64 ? (startValue.val.i64 - endValue.val.i64) : (endValue.val.i64 - startValue.val.i64); if (violationTime) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_CLOCKS_EVENT_THERMAL, d, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_THERMAL, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorPower(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; unsigned short fieldId = DCGM_FI_DEV_POWER_VIOLATION; dcgmcm_sample_t startValue = {}; dcgmcm_sample_t endValue = {}; unsigned int oneMinuteInUsec = 60000000; int count = 0; long long int violationTime = 0; dcgmcm_sample_t sample = {}; timelib64_t now = timelib_usecSince1970(); // Add XID monitoring for Power subsystem MonitorSubsystemXids(entityGroupId, entityId, DCGM_HEALTH_WATCH_POWER, response); // Warn if we cannot read the power on this entity if (entityGroupId == DCGM_FE_GPU) { ret = mpCoreProxy.GetLatestSample(entityGroupId, entityId, DCGM_FI_DEV_POWER_USAGE, &sample, 0); if (ret == DCGM_ST_OK && DCGM_FP64_IS_BLANK(sample.val.d) && sample.val.d != DCGM_FP64_NOT_SUPPORTED) { // We aren't successfully reading the power for this GPU, add a warning DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_POWER_UNREADABLE, d, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_POWER, d, response); } } /* Update the start and the end time if they are blank */ if (!startTime) { startTime = now - oneMinuteInUsec; } /* Note: Allow endTime to be in the future. 0 = blank = most recent record in time series */ /* Update the value at the start time*/ count = 1; ret = mpCoreProxy.GetSamples( entityGroupId, entityId, fieldId, &startValue, &count, startTime, endTime, DCGM_ORDER_ASCENDING); if (DCGM_ST_NO_DATA == ret) return DCGM_ST_OK; if (DCGM_ST_NOT_WATCHED == ret) { return DCGM_ST_OK; } if (DCGM_ST_OK != ret) return ret; if (DCGM_INT64_IS_BLANK(startValue.val.i64)) return DCGM_ST_OK; /* Update the value at the end time */ count = 1; ret = mpCoreProxy.GetSamples( entityGroupId, entityId, fieldId, &endValue, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (DCGM_ST_NO_DATA == ret) return DCGM_ST_OK; if (DCGM_ST_NOT_WATCHED == ret) { return DCGM_ST_OK; } if (DCGM_ST_OK != ret) return ret; if (DCGM_INT64_IS_BLANK(endValue.val.i64)) return DCGM_ST_OK; // NO DATA is handled automatically so here we can assume we have the values from the last minute // both values have been checked for BLANK values so can be used here violationTime = startValue.val.i64 >= endValue.val.i64 ? (startValue.val.i64 - endValue.val.i64) : (endValue.val.i64 - startValue.val.i64); if (violationTime) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_CLOCKS_EVENT_POWER, d, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_POWER, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorCpuThermal(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; std::vector fieldId { DCGM_FI_DEV_CPU_TEMP_CURRENT, DCGM_FI_DEV_CPU_TEMP_WARNING, DCGM_FI_DEV_CPU_TEMP_CRITICAL }; std::unordered_map startValue {}; std::unordered_map endValue {}; int count = 1; timelib64_t now = timelib_usecSince1970(); unsigned int oneMinuteInUsec = 60000000; /* Update the start and the end time if they are blank */ if (!startTime) { startTime = now - oneMinuteInUsec; } /* Note: Allow endTime to be in the future. 0 = blank = most recent record in time series */ /* Get the value at the startTime */ for (auto field : fieldId) { ret = mpCoreProxy.GetSamples( entityGroupId, entityId, field, &startValue[field], &count, startTime, endTime, DCGM_ORDER_ASCENDING); if (DCGM_ST_NO_DATA == ret) return DCGM_ST_OK; if (DCGM_ST_OK != ret) return ret; if (DCGM_INT64_IS_BLANK(startValue[field].val.i64)) return DCGM_ST_OK; } /* Get the value at the endTime*/ for (auto field : fieldId) { ret = mpCoreProxy.GetLatestSample(entityGroupId, entityId, field, &endValue[field], 0); if (DCGM_ST_NO_DATA == ret) return DCGM_ST_OK; if (DCGM_ST_OK != ret) return ret; if (DCGM_INT64_IS_BLANK(endValue[field].val.i64)) return DCGM_ST_OK; } // First check: start and end samples are over the warning threshold (WARN) if (((startValue[DCGM_FI_DEV_CPU_TEMP_CURRENT].val.d + endValue[DCGM_FI_DEV_CPU_TEMP_CURRENT].val.d) / 2) >= endValue[DCGM_FI_DEV_CPU_TEMP_WARNING].val.d) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_FIELD_THRESHOLD_DBL, d, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_THERMAL, d, response); } // If the latest sample is over the critical threshold (FAIL) if (endValue[DCGM_FI_DEV_CPU_TEMP_CURRENT].val.d >= endValue[DCGM_FI_DEV_CPU_TEMP_CRITICAL].val.d) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_FIELD_THRESHOLD_DBL, d, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_THERMAL, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorCpuPower(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long /* startTime */, long long /* endTime */, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; std::vector fieldId { DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT, DCGM_FI_DEV_CPU_POWER_LIMIT }; std::unordered_map currValue {}; /* Get the value at the endTime*/ for (auto field : fieldId) { ret = mpCoreProxy.GetLatestSample(entityGroupId, entityId, field, &currValue[field], 0); if (DCGM_ST_NO_DATA == ret) return DCGM_ST_OK; if (DCGM_ST_OK != ret) return ret; if (DCGM_INT64_IS_BLANK(currValue[field].val.i64)) return DCGM_ST_OK; } // If the sample is over the power limit (FAIL) if (currValue[DCGM_FI_DEV_CPU_POWER_UTIL_CURRENT].val.d >= currValue[DCGM_FI_DEV_CPU_POWER_LIMIT].val.d) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_FIELD_THRESHOLD_DBL, d, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_POWER, d, response); } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorNVLink(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; auto tmpRet = MonitorNVLink4Fields(entityGroupId, entityId, startTime, endTime, response); if (tmpRet != DCGM_ST_OK) { log_error("Got error {} from MonitorNVLink4Fields gpuId {}", (int)tmpRet, entityId); ret = tmpRet; } tmpRet = MonitorNVLink5Fields(entityGroupId, entityId, startTime, endTime, response); if (tmpRet != DCGM_ST_OK) { log_error("Got error {} from MonitorNVLink5Fields gpuId {}", (int)tmpRet, entityId); ret = tmpRet; } tmpRet = MonitorNVLinkStatus(entityGroupId, entityId, response); if (tmpRet != DCGM_ST_OK) { log_error("Got error {} from MonitorNVLinkStatus gpuId {}", (int)tmpRet, entityId); ret = tmpRet; } tmpRet = MonitorFabricFields(entityGroupId, entityId, startTime, endTime, response); if (tmpRet != DCGM_ST_OK) { log_error("Got error {} from MonitorFabricHealth gpuId {}", (int)tmpRet, entityId); ret = tmpRet; } return ret; } dcgmReturn_t DcgmHealthWatch::MonitorNVLink4Fields(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; /* Various NVLink error counters to be monitored */ std::array constexpr fieldIds = { DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL }; dcgmcm_sample_t startValue = {}; dcgmcm_sample_t endValue = {}; int count = 0; unsigned int oneMinuteInUsec = 60000000; timelib64_t now = timelib_usecSince1970(); /* Update the start and the end time if they are blank */ if (!startTime) { startTime = now - oneMinuteInUsec; } /* Note: Allow endTime to be in the future. 0 = blank = most recent record in time series */ for (auto const field : fieldIds) { count = 1; ret = mpCoreProxy.GetSamples( entityGroupId, entityId, field, &startValue, &count, startTime, endTime, DCGM_ORDER_ASCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA && ret != DCGM_ST_NOT_WATCHED) return ret; /* If the field is not supported, continue with others */ if (ret == DCGM_ST_NO_DATA || startValue.val.i64 == DCGM_INT64_NOT_SUPPORTED || DCGM_INT64_IS_BLANK(startValue.val.i64)) continue; count = 1; ret = mpCoreProxy.GetSamples( entityGroupId, entityId, field, &endValue, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA) return ret; /* Continue with other fields if this value is BLANK or has no data */ if (ret == DCGM_ST_NO_DATA || DCGM_INT64_IS_BLANK(endValue.val.i64)) continue; // NO DATA is handled automatically so here we can assume we have the values from the last minute // both values have been checked for BLANK values so can be used here int64_t nvLinkError = (startValue.val.i64 >= endValue.val.i64) ? (startValue.val.i64 - endValue.val.i64) : (endValue.val.i64 - startValue.val.i64); if (nvLinkError >= DCGM_LIMIT_MAX_NVLINK_ERROR) { dcgm_field_meta_p fm = DcgmFieldGetById(field); char fieldTag[128]; dcgmHealthWatchResults_t res = DCGM_HEALTH_RESULT_WARN; DcgmError d { entityId }; if (fm != NULL) { snprintf(fieldTag, sizeof(fieldTag), "%s", fm->tag); } else { snprintf(fieldTag, sizeof(fieldTag), "Unknown field %hu", field); } if ((field == DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL) || (field == DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL)) { // Replay and recovery errors are failures, not warnings. res = DCGM_HEALTH_RESULT_FAIL; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_NVLINK_ERROR_CRITICAL, d, nvLinkError, fieldTag, entityId); } else { // CRC errors are only an error if more than 100 are happening per second double timeDiffInSec; if (endTime == 0) { // Use now as the end time timeDiffInSec = (now - startTime) / 1000000.0; } else { timeDiffInSec = (endTime - startTime) / 1000000.0; } double perSec = static_cast(nvLinkError) / timeDiffInSec; if (perSec >= DCGM_LIMIT_MAX_NVLINK_CRC_ERROR) { res = DCGM_HEALTH_RESULT_FAIL; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_NVLINK_CRC_ERROR_THRESHOLD, d, perSec, fieldTag, entityId); } else { DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_NVLINK_ERROR_THRESHOLD, d, nvLinkError, fieldTag, entityId, DCGM_LIMIT_MAX_NVLINK_ERROR); } } SetResponse(entityGroupId, entityId, res, DCGM_HEALTH_WATCH_NVLINK, d, response); } } return DCGM_ST_OK; } dcgmReturn_t DcgmHealthWatch::MonitorNVLink5Fields(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { dcgmReturn_t ret = DCGM_ST_OK; std::array constexpr fieldIds = { DCGM_FI_DEV_NVLINK_COUNT_RX_SYMBOL_ERRORS, DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER, DCGM_FI_DEV_NVLINK_COUNT_RX_MALFORMED_PACKET_ERRORS, DCGM_FI_DEV_NVLINK_COUNT_RX_BUFFER_OVERRUN_ERRORS, DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS, DCGM_FI_DEV_NVLINK_COUNT_RX_REMOTE_ERRORS, DCGM_FI_DEV_NVLINK_COUNT_RX_GENERAL_ERRORS, DCGM_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS, DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS, DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS, // Shouldn't see Symbol at all. Symbol means FEC didn't succeed to fix the error. DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER, DCGM_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL }; dcgmcm_sample_t startValue = {}; dcgmcm_sample_t endValue = {}; int count = 0; auto constexpr oneMinuteInUsec = std::chrono::duration_cast(std::chrono::minutes(1)); timelib64_t now = timelib_usecSince1970(); /* Update the start and the end time if they are blank */ if (!startTime) { startTime = now - oneMinuteInUsec.count(); } for (auto const field : fieldIds) { dcgm_field_meta_p fm = DcgmFieldGetById(field); std::string fieldTag; DcgmError d { entityId }; if (fm != nullptr) { fieldTag = fm->tag; } else { fieldTag = fmt::format("Unknown field {}", field); } count = 1; ret = mpCoreProxy.GetSamples( entityGroupId, entityId, field, &startValue, &count, startTime, endTime, DCGM_ORDER_ASCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA && ret != DCGM_ST_NOT_WATCHED) { return ret; } /* If the field is not supported, continue with others */ if (ret == DCGM_ST_NO_DATA || startValue.val.i64 == DCGM_INT64_NOT_SUPPORTED || DCGM_INT64_IS_BLANK(startValue.val.i64)) { log_debug("MonitorNVLink5Fields field {} start value is not supported, ret: {}, value: {}", field, ret, startValue.val.i64); continue; } count = 1; ret = mpCoreProxy.GetSamples( entityGroupId, entityId, field, &endValue, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (ret != DCGM_ST_OK && ret != DCGM_ST_NO_DATA) { return ret; } /* Continue with other fields if this value is BLANK or has no data */ if (ret == DCGM_ST_NO_DATA || DCGM_INT64_IS_BLANK(endValue.val.i64)) { log_debug("MonitorNVLink5Fields field {} end value is not supported", field); continue; } if (field == DCGM_FI_DEV_NVLINK_COUNT_SYMBOL_BER || field == DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER) { auto const berValue = endValue.val.i64; // BER is the bit error rate itself, and a rate of 0 indicates a healthy system. // However, in NVML the smallest value is 1.5e-254, to be able to compare and avoid precision issues // with the decoded value, we decode it here and compare the mantissa and exponent to 15 and 255 // directly, rather than using the decoded field (i.e., DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER_FLOAT). auto const [mantissa, exponent, ber] = DcgmNs::Utils::NvmlBerParser(berValue); if (mantissa != 15 && exponent != 255) { DcgmError d { entityId }; auto const errorCode = (field == DCGM_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER) ? DCGM_FR_NVLINK_EFFECTIVE_BER_THRESHOLD : DCGM_FR_NVLINK_SYMBOL_BER_THRESHOLD; DCGM_ERROR_FORMAT_MESSAGE(errorCode, d, ber, entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); } } else { // Handle all other error counters int64_t const nvLinkError = (startValue.val.i64 >= endValue.val.i64) ? (startValue.val.i64 - endValue.val.i64) : (endValue.val.i64 - startValue.val.i64); if (nvLinkError >= DCGM_LIMIT_MAX_NVLINK_ERROR) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_NVLINK_ERROR_CRITICAL, d, nvLinkError, fieldTag.c_str(), entityId); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); } } } return DCGM_ST_OK; } /*****************************************************************************/ bool DcgmHealthWatch::IsGpuMigEnabled(dcgm_field_eid_t gpuId) { dcgmcm_sample_t sample = {}; dcgmReturn_t ret = mpCoreProxy.GetLatestSample(DCGM_FE_GPU, gpuId, DCGM_FI_DEV_MIG_MODE, &sample, nullptr); if (ret == DCGM_ST_OK) { if (DCGM_INT64_IS_BLANK(sample.val.i64)) { log_debug("MIG mode for GPU {} is blank, treating as disabled", gpuId); return false; } return sample.val.i64 != 0; } log_debug("Could not get MIG status for GPU {}", gpuId); return false; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorNVLinkStatus(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, DcgmHealthResponse &response) { // Add XID monitoring for NVLink subsystem MonitorSubsystemXids(entityGroupId, entityId, DCGM_HEALTH_WATCH_NVLINK, response); if (entityGroupId != DCGM_FE_GPU) { log_debug("MonitorNVLinkStatus called with entityGroupId {}, expected {}", entityGroupId, DCGM_FE_GPU); return DCGM_ST_OK; } dcgmNvLinkLinkState_t linkStates[DCGM_NVLINK_MAX_LINKS_PER_GPU]; dcgmReturn_t ret = mpCoreProxy.GetEntityNvLinkLinkStatus(DCGM_FE_GPU, entityId, linkStates); if (ret != DCGM_ST_OK) { log_error("Got error {} from GetEntityNvLinkLinkStatus gpuId {}", (int)ret, entityId); return ret; } bool const migIsEnabled = IsGpuMigEnabled(entityId); for (int i = 0; i < DCGM_NVLINK_MAX_LINKS_PER_GPU; i++) { dcgmNvLinkLinkState_t linkState = linkStates[i]; if (linkState == DcgmNvLinkLinkStateNotSupported || linkState == DcgmNvLinkLinkStateDisabled) { continue; } if (linkState == DcgmNvLinkLinkStateDown) { // Bug 200682374 - NVML reports links to MIG enabled GPUs as down. This causes // health checks to fail. As a WaR, we'll treat any Down links as Disabled if // this GPU is in MIG mode if (migIsEnabled) { log_debug("GPU {} NvLink {} was reported down in MIG mode, treating as disabled", entityId, i); continue; } else { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_NVLINK_DOWN, d, entityId, i); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); } } // link state is up } return DCGM_ST_OK; } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorNvSwitchErrorCounts(bool fatal, dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long startTime, long long endTime, DcgmHealthResponse &response) { std::vector::iterator fieldIdIter; dcgmReturn_t dcgmReturn; std::vector *fieldIds; dcgmcm_sample_t sample = {}; dcgmHealthWatchResults_t healthWatchResult; dcgmHealthSystems_t healthWatchSystems; std::string errorTypeString; unsigned int oneMinuteInUsec = 60000000; timelib64_t now = timelib_usecSince1970(); /* Update the start and the end time if they are blank */ if (!startTime) { startTime = now - oneMinuteInUsec; } /* Note: Allow endTime to be in the future. 0 = blank = most recent record in time series */ if (fatal) { fieldIds = &m_nvSwitchFatalFieldIds; healthWatchResult = DCGM_HEALTH_RESULT_FAIL; healthWatchSystems = DCGM_HEALTH_WATCH_NVSWITCH_FATAL; errorTypeString = "fatal"; } else /* Non-fatal */ { fieldIds = &m_nvSwitchNonFatalFieldIds; healthWatchResult = DCGM_HEALTH_RESULT_WARN; healthWatchSystems = DCGM_HEALTH_WATCH_NVSWITCH_NONFATAL; errorTypeString = "nonfatal"; } memset(&sample, 0, sizeof(sample)); for (fieldIdIter = fieldIds->begin(); fieldIdIter != fieldIds->end(); ++fieldIdIter) { int count = 1; dcgmReturn = mpCoreProxy.GetSamples( entityGroupId, entityId, *fieldIdIter, &sample, &count, startTime, endTime, DCGM_ORDER_DESCENDING); if (dcgmReturn != DCGM_ST_OK) { log_debug("return {} for GetSamples eg {}, eid {}, fieldId {}, start {}, end {}", dcgmReturn, entityGroupId, entityId, *fieldIdIter, startTime, endTime); continue; } if (!DCGM_INT64_IS_BLANK(sample.val.i64) && sample.val.i64 > 0) { unsigned int linkId = (*fieldIdIter) - fieldIds->at(0); DcgmError d { entityId }; if (fatal) { DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_NVSWITCH_FATAL_ERROR, d, entityId, linkId); } else { DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_NVSWITCH_NON_FATAL_ERROR, d, entityId, linkId); } SetResponse(entityGroupId, entityId, healthWatchResult, healthWatchSystems, d, response); } } /* See if any links are down. Only do this for the fatal case so we don't get duplicate errors for both fatal * and non-fatal */ if (fatal) { dcgmNvLinkLinkState_t linkStates[DCGM_NVLINK_MAX_LINKS_PER_NVSWITCH]; dcgmReturn = mpCoreProxy.GetEntityNvLinkLinkStatus(DCGM_FE_SWITCH, entityId, linkStates); if (dcgmReturn != DCGM_ST_OK) { log_error("Got error {} from GetEntityNvLinkLinkStatus eid {}", (int)dcgmReturn, entityId); return dcgmReturn; } for (int i = 0; i < DCGM_NVLINK_MAX_LINKS_PER_NVSWITCH; i++) { if (linkStates[i] == DcgmNvLinkLinkStateDown) { DcgmError d { entityId }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_NVLINK_DOWN, d, entityId, i); SetResponse(entityGroupId, entityId, healthWatchResult, healthWatchSystems, d, response); } } } return DCGM_ST_OK; } /*****************************************************************************/ void DcgmHealthWatch::OnGroupRemove(unsigned int groupId) { groupWatchTable_t::iterator groupWatchIter; dcgm_mutex_lock(m_mutex); groupWatchIter = mGroupWatchState.find(groupId); if (groupWatchIter == mGroupWatchState.end()) { log_debug("OnGroupRemove didn't find groupId {}", groupId); } else { mGroupWatchState.erase(groupWatchIter); log_debug("OnGroupRemove found and removed groupId {}", groupId); } dcgm_mutex_unlock(m_mutex); } /*****************************************************************************/ void DcgmHealthWatch::ProcessXidFv(dcgmBufferedFv_t *fv) { uint64_t xid = static_cast(fv->value.i64); bool isHandled = false; log_debug("Processing XID {} for gpuId {}", xid, fv->entityId); DcgmLockGuard dlg(m_mutex); // Check if devastating XID is hit auto devastatingXid = m_devastatingXids.find(xid); if (devastatingXid != m_devastatingXids.end()) { // Record it for this gpu m_gpuXidHistory[xid].insert(fv->entityId); log_error("Devastating XID {} detected on gpuId {}", xid, fv->entityId); isHandled = true; } // Check if subsystem level XID is hit for (auto const &[subsystem, xidMap] : m_subsystemXids) { bool subsystemEnabled = false; for (const auto &[groupId, systems] : mGroupWatchState) { if (systems.systems & subsystem) { subsystemEnabled = true; break; } } if (subsystemEnabled) { auto xidIt = xidMap.find(xid); if (xidIt != xidMap.end()) { // Record it for this gpu m_gpuXidHistory[xid].insert(fv->entityId); log_warning( "gpuId {} hit XID {} for subsystem {}", fv->entityId, xid, GetHealthSystemAsString(subsystem)); isHandled = true; } } } if (!isHandled) { log_debug("Ignored XID {} for gpuId {}", xid, fv->entityId); } } /*****************************************************************************/ void DcgmHealthWatch::OnFieldValuesUpdate(DcgmFvBuffer *fvBuffer) { dcgmBufferedFv_t *fv; dcgmBufferedFvCursor_t cursor = 0; /* This is a bit coarse-grained for now, but it's clean */ dcgmMutexReturn_t mutexSt = dcgm_mutex_lock(m_mutex); for (fv = fvBuffer->GetNextFv(&cursor); fv; fv = fvBuffer->GetNextFv(&cursor)) { /* Policy only pertains to GPUs for now */ if (fv->entityGroupId != DCGM_FE_GPU) { log_debug("Ignored non-GPU eg {}", fv->entityGroupId); continue; } switch (fv->fieldId) { case DCGM_FI_DEV_XID_ERRORS: ProcessXidFv(fv); break; default: /* This is partially expected since the cache manager will broadcast any FVs that updated during the same loop as FVs we care about */ DCGM_LOG_DEBUG << "Ignoring unhandled field " << fv->fieldId; break; } } if (mutexSt != DCGM_MUTEX_ST_LOCKEDBYME) dcgm_mutex_unlock(m_mutex); } /*****************************************************************************/ void DcgmHealthWatch::BuildXidMappings() { // Define always-monitored "devastating" XIDs - these are critical hardware errors that should // always be monitored regardless of which health subsystems are enabled m_devastatingXids = { { 48, { DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_ALL, "Double Bit ECC Error", DCGM_FR_XID_ERROR } }, { 74, { DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_ALL, "NVLink Critical Error", DCGM_FR_XID_ERROR } }, { 79, { DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_ALL, DCGM_FR_FALLEN_OFF_BUS_MSG, DCGM_FR_FALLEN_OFF_BUS } }, { 94, { DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_ALL, "Contained Error", DCGM_FR_XID_ERROR } }, { 95, { DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_ALL, DCGM_FR_UNCONTAINED_ERROR_MSG, DCGM_FR_UNCONTAINED_ERROR } }, { 119, { DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_ALL, "GSP RPC Timeout", DCGM_FR_XID_ERROR } }, { 120, { DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_ALL, "GSP Error", DCGM_FR_XID_ERROR } }, { 140, { DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_ALL, "ECC unrecovered error", DCGM_FR_XID_ERROR } } }; // Define subsystem-specific XIDs // Memory subsystem m_subsystemXids[DCGM_HEALTH_WATCH_MEM] = { // ECC errors { 31, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_MEM, "MMU Error", DCGM_FR_XID_ERROR } }, { 32, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_MEM, "PBDMA Error", DCGM_FR_XID_ERROR } }, { 43, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_MEM, "Reset Channel Verif Error", DCGM_FR_XID_ERROR } }, { 63, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_MEM, DCGM_FR_PENDING_PAGE_RETIREMENTS_MSG, DCGM_FR_PENDING_PAGE_RETIREMENTS } }, { 64, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_MEM, DCGM_FR_ROW_REMAP_FAILURE_MSG, DCGM_FR_ROW_REMAP_FAILURE } } }; // PCIe subsystem m_subsystemXids[DCGM_HEALTH_WATCH_PCIE] = { { 38, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_PCIE, "PCIe Bus Error", DCGM_FR_XID_ERROR } }, { 39, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_PCIE, "PCIe Fabric Error", DCGM_FR_XID_ERROR } }, { 42, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_PCIE, DCGM_FR_PCI_REPLAY_RATE_MSG, DCGM_FR_PCI_REPLAY_RATE } }, }; // Thermal subsystem m_subsystemXids[DCGM_HEALTH_WATCH_THERMAL] = { { 60, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_THERMAL, DCGM_FR_CLOCKS_EVENT_THERMAL_MSG, DCGM_FR_CLOCKS_EVENT_THERMAL } }, { 61, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_THERMAL, "EDPP Power Brake Thermal limit", DCGM_FR_XID_ERROR } }, { 62, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_THERMAL, DCGM_FR_THERMAL_VIOLATIONS_MSG, DCGM_FR_THERMAL_VIOLATIONS } }, }; // Power subsystem m_subsystemXids[DCGM_HEALTH_WATCH_POWER] = { { 54, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_POWER, "Power state change", DCGM_FR_XID_ERROR } }, { 56, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_POWER, "Clock change", DCGM_FR_XID_ERROR } }, { 57, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_POWER, "Clock change due to power", DCGM_FR_CLOCKS_EVENT_POWER } }, { 58, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_POWER, "Clock change due to thermal", DCGM_FR_XID_ERROR } }, { 78, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_POWER, "Power state forced change", DCGM_FR_XID_ERROR } } }; // NVLink subsystem m_subsystemXids[DCGM_HEALTH_WATCH_NVLINK] = { { 67, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_NVLINK, DCGM_FR_NVLINK_ERROR_THRESHOLD_MSG, DCGM_FR_NVLINK_ERROR_THRESHOLD } }, { 73, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_NVLINK, "NVLink Flow Control Error", DCGM_FR_XID_ERROR } }, { 121, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_NVLINK, "C2C Link corrected error", DCGM_FR_XID_ERROR } }, { 137, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_NVLINK, "NVLink FLA privilege error", DCGM_FR_XID_ERROR } } }; // InfoROM subsystem m_subsystemXids[DCGM_HEALTH_WATCH_INFOROM] = { { 93, { DCGM_HEALTH_RESULT_WARN, DCGM_HEALTH_WATCH_INFOROM, DCGM_FR_CORRUPT_INFOROM_MSG, DCGM_FR_CORRUPT_INFOROM } } }; } /*****************************************************************************/ void DcgmHealthWatch::MonitorSubsystemXids(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, dcgmHealthSystems_t system, DcgmHealthResponse &response) { log_debug("Monitoring XIDs for {} {}, system {}", EntityToString(entityGroupId), entityId, GetHealthSystemAsString(system)); DcgmLockGuard dlg(m_mutex); // Check subsystem-specific XIDs auto subsystemIt = m_subsystemXids.find(system); if (subsystemIt != m_subsystemXids.end()) { log_debug( "Found {} XIDs to check for subsystem {}", subsystemIt->second.size(), GetHealthSystemAsString(system)); for (const auto &[xid, xidInfo] : subsystemIt->second) { if (m_gpuXidHistory.count(xid) && m_gpuXidHistory[xid].count(entityId)) { log_error("Found subsystem XID {} ({}) for {} {} in system {}", xid, xidInfo.errorMessage, EntityToString(entityGroupId), entityId, GetHealthSystemAsString(system)); DcgmError d { entityId }; if (xidInfo.errorCode == DCGM_FR_XID_ERROR) { DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_XID_ERROR, d, xid, entityId); } else { DCGM_ERROR_FORMAT_MESSAGE(xidInfo.errorCode, d, entityId); } SetResponse(entityGroupId, entityId, xidInfo.severity, system, d, response); } } } else { log_debug("No XIDs found for subsystem {}", GetHealthSystemAsString(system)); } } /*****************************************************************************/ void DcgmHealthWatch::MonitorDevastatingXids(dcgm_field_entity_group_t entityGroupId, unsigned int entityId, DcgmHealthResponse &response) { log_debug("Monitoring Devastating XIDs for {} {}, all systems {}", EntityToString(entityGroupId), entityId, GetHealthSystemAsString(DCGM_HEALTH_WATCH_ALL)); DcgmLockGuard dlg(m_mutex); // Check devastating XIDs always for (const auto &[xid, xidInfo] : m_devastatingXids) { if (m_gpuXidHistory.count(xid) && m_gpuXidHistory[xid].count(entityId)) { log_error("Found devastating XID {} ({}) for {} {}", xid, xidInfo.errorMessage, EntityToString(entityGroupId), entityId); DcgmError d { entityId }; if (xidInfo.errorCode == DCGM_FR_XID_ERROR) { DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_XID_ERROR, d, xid, entityId); } else { DCGM_ERROR_FORMAT_MESSAGE(xidInfo.errorCode, d, entityId); } SetResponse(entityGroupId, entityId, xidInfo.severity, DCGM_HEALTH_WATCH_ALL, d, response); } } } /*****************************************************************************/ dcgmReturn_t DcgmHealthWatch::MonitorFabricFields(dcgm_field_entity_group_t entityGroupId, dcgm_field_eid_t entityId, long long /* startTime */, long long /* endTime */, DcgmHealthResponse &response) { log_debug("Monitoring Fabric Fields for {} {}, all systems {}", EntityToString(entityGroupId), entityId, GetHealthSystemAsString(DCGM_HEALTH_WATCH_ALL)); dcgmGroupEntityPair_t const entityPair = { entityGroupId, entityId }; unsigned short const fieldId = DCGM_FI_DEV_FABRIC_HEALTH_MASK; auto const result = CollectLatestInt64Sample(mpCoreProxy, entityGroupId, entityId, fieldId); if (result.has_value() && result.value().has_value()) { auto const &sample = result.value().value(); if (DCGM_GPU_FABRIC_HEALTH_TEST(sample.val.i64, _ROUTE_UNHEALTHY, _TRUE)) { DcgmError d { entityPair }; DCGM_ERROR_FORMAT_MESSAGE( DCGM_FR_FIELD_VIOLATION, d, sample.val.i64, "Fabric Health: Route Unhealthy", entityId); d.AddDetail("Route Unhealthy"); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); log_verbose("Found Route Unhealthy for gpuId {}, health_mask: 0x{:x}", entityId, sample.val.i64); } if (DCGM_GPU_FABRIC_HEALTH_TEST(sample.val.i64, _DEGRADED_BW, _TRUE)) { DcgmError d { entityPair }; DCGM_ERROR_FORMAT_MESSAGE( DCGM_FR_FIELD_VIOLATION, d, sample.val.i64, "Fabric Health: Bandwidth Degraded", entityId); d.AddDetail("Bandwidth Degraded"); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); log_verbose("Found Bandwidth Degraded for gpuId {}, health_mask: 0x{:x}", entityId, sample.val.i64); } if (DCGM_GPU_FABRIC_HEALTH_TEST(sample.val.i64, _ROUTE_RECOVERY, _TRUE)) { DcgmError d { entityPair }; DCGM_ERROR_FORMAT_MESSAGE( DCGM_FR_FIELD_VIOLATION, d, sample.val.i64, "Fabric Health: Route Recovery", entityId); d.AddDetail("Route Recovery"); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); log_verbose("Found Route Recovery for gpuId {}, health_mask: 0x{:x}", entityId, sample.val.i64); } if (DCGM_GPU_FABRIC_HEALTH_TEST(sample.val.i64, _ACCESS_TIMEOUT_RECOVERY, _TRUE)) { DcgmError d { entityPair }; DCGM_ERROR_FORMAT_MESSAGE( DCGM_FR_FIELD_VIOLATION, d, sample.val.i64, "Fabric Health: Access Timeout Recovery", entityId); d.AddDetail("Access Timeout Recovery"); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); log_verbose("Found Access Timeout Recovery for gpuId {}, health_mask: 0x{:x}", entityId, sample.val.i64); } if (!DCGM_GPU_FABRIC_HEALTH_TEST(sample.val.i64, _INCORRECT_CONFIGURATION, _NOT_SUPPORTED) && !DCGM_GPU_FABRIC_HEALTH_TEST(sample.val.i64, _INCORRECT_CONFIGURATION, _NONE)) { DcgmError d { entityPair }; DCGM_ERROR_FORMAT_MESSAGE( DCGM_FR_FIELD_VIOLATION, d, sample.val.i64, "Fabric Health: Incorrect Configuration", entityId); d.AddDetail("Incorrect Configuration"); SetResponse(entityGroupId, entityId, DCGM_HEALTH_RESULT_FAIL, DCGM_HEALTH_WATCH_NVLINK, d, response); log_verbose("Found Incorrect Configuration for gpuId {}, health_mask: 0x{:x}", entityId, sample.val.i64); } } else if (!result.has_value()) { /* Errors logged by CollectLatestInt64Sample() */ return result.error(); } unsigned short const fmStatusFieldId = DCGM_FI_DEV_FABRIC_MANAGER_STATUS; auto const fmResult = CollectLatestInt64Sample(mpCoreProxy, entityGroupId, entityId, fmStatusFieldId); if (fmResult.has_value() && fmResult.value().has_value()) { auto const &fmSample = fmResult.value().value(); auto const status = static_cast(fmSample.val.i64); auto handleError = [&](std::string_view statusStr, std::string detail, dcgmHealthWatchResults_t health) { DcgmError d { entityPair }; DCGM_ERROR_FORMAT_MESSAGE(DCGM_FR_FABRIC_PROBE_STATE, d, entityId, statusStr.data(), fmSample.val.i64); d.AddDetail(detail); SetResponse(entityGroupId, entityId, health, DCGM_HEALTH_WATCH_NVLINK, d, response); log_verbose("Fabric State {} for {} {}, status: {}", statusStr, EntityToString(entityGroupId), entityId, fmSample.val.i64); }; switch (status) { case DcgmFMStatusSuccess: case DcgmFMStatusNotSupported: break; case DcgmFMStatusInProgress: handleError("In Progress", "Fabric Manager training in progress.", DCGM_HEALTH_RESULT_WARN); break; case DcgmFMStatusNotStarted: handleError("Not Started", "Fabric Manager training has not started.", DCGM_HEALTH_RESULT_FAIL); break; case DcgmFMStatusFailure: handleError("Failed", "Fabric Manager training failed.", DCGM_HEALTH_RESULT_FAIL); break; case DcgmFMStatusUnrecognized: handleError("Unrecognized", "Fabric Manager status is unrecognized.", DCGM_HEALTH_RESULT_FAIL); break; case DcgmFMStatusNvmlTooOld: handleError( "NvmlTooOld", "NVML version is too old to query Fabric Manager status.", DCGM_HEALTH_RESULT_FAIL); break; default: handleError("Unknown", fmt::format("Unknown Fabric Manager status code: {}", fmSample.val.i64), DCGM_HEALTH_RESULT_FAIL); break; } } else if (!fmResult.has_value()) { /* Errors logged by CollectLatestInt64Sample() */ return fmResult.error(); } return DCGM_ST_OK; }