#!/bin/sh # SPDX-License-Identifier: MIT OR Apache-2.0 # fstart AMD RAS test: RAM only, no block drivers, no disk mounts. The steps # come from the kernel command line, `fstart.ras=step,step,...` (the image's # --linux-bootargs), in order: # # info the RAS tables, BERT's records, the eSPI decodes and KCS ports # smm every CPU armed in SMM (amd-ras smm) # probe one benign SEL event through SMM # kcs:PORT:SPACING one IPMI Get Device ID from the OS on a KCS channel # sel:PORT:SPACING the SMM probe's SEL record from the OS on a KCS channel # (Add SEL Entry); event:PORT:SPACING the same as a Platform # Event Message; sel-info:PORT:SPACING Get SEL Info # ce:ADDR:N[:BANK] N correctable memory injections at ADDR (EDAC must count # each), then BANK's thresholding counters # sp3-einj:N[:BANK] N of SP3's correctable injections (Rome's EINJ, on the # firmware's own targets), then BANK's thresholding counters # pcie:TYPE:SEG:BUS:DEV.FN one PCIe injection (0x40 correctable) at a # function: the injection must succeed and AER report it # from that function afterwards # aer:SEG:BUS:DEV.FN a function's AER status # ue:ADDR one uncorrectable memory injection, consumed: the machine # goes down. The boot after it (a mark in the RTC's CMOS says # which) checks the BERT instead; one injection per mark. # plant:CPU:BANK:STATUS[:ADDR] a synthetic bank error, a mark of it in the # RTC's CMOS, then at once a warm reset (CF9h). The boot after # it checks the BERT for that bank and status instead of # planting again; one plant per mark. # smn:ADDR one SMN register (read only) # storm SMM's storm state (interrupts or polling, the periodic SMI) # burst:ADDR:N N correctable injections at ADDR back to back: SMM must poll # through the storm and interrupt again after it # flood:ADDR:N one correctable injection at ADDR, then N uncached reads of # it (each a corrected error while the error stays in DRAM) # ipmi-load:S[:MS] the OS's IPMI driver busy for S seconds in the background # (Get Device ID through /dev/ipmi0 back to back, or MS apart); ipmi-wait # waits for it: every request must have completed # ipmi-sel Get SEL Info through the OS's driver # sleep:S wait S seconds # off power off (the default end); `hold` sleeps instead # # Every line it prints starts with FSTART-RAS, results with FSTART_RAS_. PATH=/bin export PATH mount -t proc proc /proc mount -t sysfs sysfs /sys mount -t devtmpfs devtmpfs /dev mount -t tmpfs tmpfs /tmp say() { echo "<6>FSTART-RAS $*" > /dev/kmsg; } run() { "$@" > /tmp/out 2>&1; s=$?; while read -r l; do say "$l"; done < /tmp/out; return $s; } say 'RAM-only init: no disk drivers, no disk mounts' say "CPUs online=$(cat /sys/devices/system/cpu/online) cmdline=$(cat /proc/cmdline)" for b in /sys/block/*; do [ -e "$b" ] && say "UNEXPECTED block device $b"; done echo 1 > /sys/devices/system/machinecheck/machinecheck0/check_interval steps= for a in $(cat /proc/cmdline); do case $a in fstart.ras=*) steps=${a#*=};; esac; done dmesg | grep -E 'BERT|HEST|GHES|EINJ|APEI|\[Hardware Error\]|EDAC MC[0-9]|mce:' | while read -r l; do say "dmesg: $l"; done end=off for step in $(echo "$steps" | tr ',' ' '); do set -- $(echo "$step" | tr ':' ' ') case $1 in info) run amd-ras info; run amd-ras espi ;; smm) run amd-ras smm && say FSTART_RAS_SMM_OK ;; probe) run amd-ras probe && say FSTART_RAS_PROBE_OK ;; kcs) run amd-ras kcs "$2" "$3" && say FSTART_RAS_KCS_OK ;; sel) run amd-ras sel "$2" "$3" && say FSTART_RAS_SEL_OK ;; event) run amd-ras event "$2" "$3" && say FSTART_RAS_EVENT_OK ;; sel-info) run amd-ras sel-info "$2" "$3" ;; ce) run amd-ras einj 0x8 "$2" "${3:-1}" && say FSTART_RAS_CE_OK [ -n "$4" ] && run amd-ras misc "$4" dmesg | grep -E '\[Hardware Error\]|EDAC' | tail -6 | while read -r l; do say "dmesg: $l"; done ;; sp3-einj) run amd-ras sp3-einj "${2:-1}" && say FSTART_RAS_CE_OK [ -n "$3" ] && run amd-ras misc "$3" dmesg | grep -E '\[Hardware Error\]|EDAC' | tail -6 | while read -r l; do say "dmesg: $l"; done ;; pcie) run amd-ras einj-pcie "$2" "$3:$4:$5" && say FSTART_RAS_PCIE_OK ;; ue) run amd-ras ue-check; case $? in 0) say "FSTART_RAS_BERT_OK: BERT holds the previous boot's errors; no new injection" ;; 2) say 'FSTART_RAS_UE: injecting an uncorrectable error; the machine goes down' run amd-ras mark-ue && run amd-ras einj 0x10 "$2" 1 sleep 10 say 'FSTART_RAS_UE_SURVIVED' ;; *) say 'FSTART_RAS_BERT_MISSING: no BERT record after the injection; not injecting again' ;; esac ;; plant) run amd-ras plant-check; case $? in 0) say "FSTART_RAS_BERT_OK: BERT holds the planted bank and status" ;; 2) say 'FSTART_RAS_PLANT: a synthetic error, then a warm reset' run amd-ras plant-reset "$2" "$3" "$4" ${5:+"$5"} ;; *) say 'FSTART_RAS_BERT_MISSING: the planted error is not in the BERT; not planting again' ;; esac ;; aer) run amd-ras aer "$2:$3:$4" ;; storm) run amd-ras storm ;; smn) run amd-ras smn "$2" ;; burst) run amd-ras ce-burst "$2" "$3" && say FSTART_RAS_STORM_OK dmesg | grep -E '\[Hardware Error\]|EDAC|mce:' | tail -6 | while read -r l; do say "dmesg: $l"; done ;; flood) run amd-ras ce-flood "$2" "$3" && say FSTART_RAS_STORM_OK dmesg | grep -E '\[Hardware Error\]|EDAC|mce:' | tail -6 | while read -r l; do say "dmesg: $l"; done ;; ipmi-load) amd-ras ipmi-load "$2" ${3:+"$3"} > /tmp/ipmi-load 2>&1 & load=$! ;; ipmi-wait) wait "$load"; s=$? while read -r l; do say "$l"; done < /tmp/ipmi-load [ "$s" = 0 ] && say FSTART_RAS_IPMI_OK ;; ipmi-sel) run amd-ras ipmi-sel ;; sleep) sleep "$2" ;; hold) end=hold ;; off) end=off ;; *) say "unknown step $step" ;; esac done say 'FSTART_RAS_DONE' sleep 2 [ "$end" = off ] && poweroff -f while :; do sleep 3600; done