// SPDX-License-Identifier: MIT OR Apache-2.0 //! The SMM machine-check handler of the SoCs with an SMI trigger for their //! MCA (SP3: [`crate::sp3::smm`], RS: [`crate::rs::smm`]), per the board's //! [`Policy`] (`Board::RAS`). //! //! Every CPU that enters SMM runs [`enter`]. On its first SMI (the //! installer's self-SMI, after SMRAM is locked) it arms its SMI trigger //! I/O cycle to the FCH's SMI command port and the thresholding counters //! of its banks (`ccx::ras::arm_thresholds`: [`Policy::threshold`] //! corrected errors each, then an SMI; the blocks are locked for the OS, //! which keeps the banks), or with [`Policy::observe_mce`] PFEH as well //! (`ccx::ras::arm`). On every SMI it reads its local SMI status: a //! counter overflow or a deferred error (SmiSrcMca) or a machine check //! (MceRedirSts) is recorded in a journal in SMRAM, the counter re-armed, //! and with PFEH the event percolated back to the OS on RSM. An OS write //! to a bank status PFEH intercepts is replayed if its saved registers //! still name a clear; otherwise the CPU leaves PFEH (the vendors' //! AmdPlatformRasRsSmm "MCE to Software SMI error handler" takes the same //! sources but cloaks the banks and reports to the OS through GHES). //! //! The journal goes to the board's BMC ([`deliver`]): an [`Event`] at a //! time through the board's hook, which owns the transport (a KCS //! channel the OS does not use, or the OS's when it is idle) and may use //! [`sel`] for the record. Delivery is bounded: [`SEL_PER_BOOT`] events //! per boot, the first failure stops it, a counter is re-armed at most //! [`REARMS_PER_CPU`] times per CPU (an error storm then stays with the //! OS, whose banks still log it), and nothing waits for another CPU. //! //! No heap, flash writes, OS buffers, GHES, address-to-DIMM guesses, bank //! clearing or all-CPU barrier. The first records survive in locked SMRAM //! until reset; this is deliberately not a persistent error store. /// A board's RAS policy ([`crate::board::Board::RAS`]). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Policy { /// Corrected errors a bank counts before its thresholding interrupt /// (the vendors' "MCA error thresh count" default is 10); 0 counts /// none. Nonzero: SMM counts them, reports each overflow and leaves /// the errors in the banks for the OS; zero: the OS counts, with its /// own limit. pub threshold: u16, /// Machine checks and deferred errors observed in SMM too (PFEH), /// then percolated to the OS. Experimental: PFEH turns every OS write /// to a bank status into an SMI that arrives after the write, whose /// registers Linux 7.3 has already cleared (`ccx::ras`). pub observe_mce: bool, } impl Policy { /// OS-first: no SMM counting, observation or SEL. pub const OS_FIRST: Self = Self { threshold: 0, observe_mce: false, }; /// SMM counts corrected errors with the vendors' threshold of 10. pub const THRESHOLDS: Self = Self { threshold: 10, observe_mce: false, }; /// SMM handles machine-check events at all. pub const fn smm(&self) -> bool { self.threshold != 0 || self.observe_mce } } use fstart_driver_amd::ccx::ras::{Kind, Record}; use fstart_driver_amd::ccx::tsc::Tsc; use fstart_driver_ipmi::SelRecord; use zerocopy::little_endian::{U16, U32}; /// What the handler, and the boot for the last boot's errors, report to /// the board's BMC (`Board::BMC_EVENT`). #[derive(Debug, Clone, Copy)] pub enum Event<'a> { /// A machine check, a deferred error or a thresholding overflow, or /// an error of the last boot. Mca(&'a Record), /// The one benign transport check the diagnostic ABI requests. Probe, } /// A board without a BMC to report to. pub fn no_bmc(_: &Event<'_>, _: Tsc) -> bool { false } /// The KCS poll budget of one event (`Kcs::with_poll_budget`): 25000 /// readiness checks 10 us apart, about 250 ms. The S8040's BMC takes /// about 25 ms to answer an event. pub const SEL_POLL_BUDGET: u32 = 25_000; /// The generator of the records: system software ID 10h, the first of /// the SMI handler's (IPMI 2.0 ยง5.5, table 5-4). const GENERATOR: u16 = 0x10 << 1 | 1; // Standard system events work on the bench BMCs; OEM E0 entries are // silently acknowledged without storage. FF names no SDR sensor. fn event(sensor_type: u8, event_data: [u8; 3]) -> SelRecord { SelRecord { record_id: U16::ZERO, record_type: 2, timestamp: U32::ZERO, generator_id: U16::new(GENERATOR), event_message_revision: 4, sensor_type, sensor_number: 0xff, event_dir_type: 0x6f, event_data, } } /// A standard SEL record of `event` (IPMI 2.0 table 42-3). A DRAM /// error is a memory event (sensor type 0Ch): correctable ECC logging /// limit reached (05h) for a counter overflow, uncorrectable ECC (01h) /// for an uncorrected one, correctable ECC (00h) otherwise; any other /// bank's is a processor event (07h): uncorrectable (0Bh) or /// correctable (0Ch) machine check. Event data 2 and 3 are OEM data: /// the logical CPU that read the bank and the bank, not a guessed DIMM. pub fn sel(event: &Event<'_>) -> SelRecord { let record = match event { Event::Mca(record) => record, // Entry added to auxiliary log, OEM data 'FS': a transport // check, never a manufactured machine-check record. Event::Probe => return self::event(0x12, [0xa3, b'F', b'S']), }; let (sensor, offset) = match (record.memory(), record.uncorrected()) { (true, true) => (0x0c, 0x01), (true, false) if record.kind == Kind::Threshold => (0x0c, 0x05), (true, false) => (0x0c, 0x00), (false, true) => (0x07, 0x0b), (false, false) => (0x07, 0x0c), }; self::event(sensor, [0xa0 | offset, record.cpu as u8, record.bank as u8]) } #[cfg(feature = "smm")] pub use handler::{ DIAG_COMMAND, DIAG_REQUEST, MCE_SMI_COMMAND, REARMS_PER_CPU, SEL_PER_BOOT, deliver, enter, }; #[cfg(feature = "smm")] mod handler { use core::cell::UnsafeCell; use core::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, Ordering}; use fstart_arch::x86::cpu::amd::msr::SMM_BASE; use fstart_arch::x86::cpu::msr::{MsrAccess, NativeMsr}; use fstart_driver_amd::ccx::ras::{self, Kind, LocalSmiStatus, Record}; use fstart_driver_amd::ccx::tsc::Tsc; use fstart_driver_amd::cpu::Silicon; use fstart_driver_amd::window::{Mmio, Window}; use tock_registers::interfaces::Readable; use super::{Event, Policy}; /// Logical CPUs the handler tracks: two Genoa sockets of 96 cores and /// two threads fit. Indices beyond stay OS-first, never armed. const MAX_CPUS: usize = 512; const RECORDS: usize = 64; /// Events sent to the BMC per boot. pub const SEL_PER_BOOT: u32 = 16; /// Events sent per delivery, which runs in one SMI. const SEL_PER_SMI: u32 = 2; /// Thresholding re-arms per CPU per boot. pub const REARMS_PER_CPU: u8 = 32; /// The I/O cycle data of the local SMI trigger; never sufficient to /// establish an MCA cause. pub const MCE_SMI_COMMAND: u8 = 0x80; /// Diagnostic I/O ABI: OUT AL,DX with RAX=DIAG_REQUEST, RDX=B2, RCX selector. /// 0 reads PFEH/cloak/mask into RAX/RCX/RDX. 1 requests one benign SEL probe; /// 2 reads its result (0 idle, 1 queued, 4 running, 2 success, 3 failure). /// 3 checks saved-SMBASE offsets (RAX=FF00 match, RCX=FE00 match, RDX=revision). /// 4 reads RAX=records journaled, RCX=dropped, RDX=refused CPUs. /// No pointers, error injection or arbitrary MSRs are accepted. pub const DIAG_COMMAND: u8 = 0xd1; pub const DIAG_REQUEST: u64 = 0x4653_5241_5300_00d1; /// Per CPU: 0 not tried, [`TRIGGER`] armed, with [`PFEH`] too, or /// [`REFUSED`]. static CPU_STATE: [AtomicU8; MAX_CPUS] = [const { AtomicU8::new(0) }; MAX_CPUS]; const TRIGGER: u8 = 1; const PFEH: u8 = 2; const REFUSED: u8 = 4; static REARMS: [AtomicU8; MAX_CPUS] = [const { AtomicU8::new(0) }; MAX_CPUS]; static REFUSED_CPUS: AtomicU32 = AtomicU32::new(0); static PROBE: AtomicU8 = AtomicU8::new(0); static DELIVERING: AtomicBool = AtomicBool::new(false); static SEL_FAILED: AtomicBool = AtomicBool::new(false); static JOURNAL: Journal = Journal::new(); struct Journal { locked: AtomicBool, dropped: AtomicU32, state: UnsafeCell, } struct State { records: [Record; RECORDS], count: usize, pending: usize, sent: u32, } // SAFETY: every access to state is behind `with`'s acquire/release lock; // producers never spin and no reference to the contents escapes its closure. unsafe impl Sync for Journal {} impl Journal { const fn new() -> Self { Self { locked: AtomicBool::new(false), dropped: AtomicU32::new(0), state: UnsafeCell::new(State { records: [Record::EMPTY; RECORDS], count: 0, pending: 0, sent: 0, }), } } fn with(&self, f: impl FnOnce(&mut State) -> T) -> Option { if self .locked .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) .is_err() { return None; } struct Unlock<'a>(&'a AtomicBool); impl Drop for Unlock<'_> { fn drop(&mut self) { self.0.store(false, Ordering::Release); } } let _unlock = Unlock(&self.locked); // SAFETY: exclusive access until Unlock drops; `f` cannot leak the borrow. Some(f(unsafe { &mut *self.state.get() })) } /// Keep `record`, unless it is a deferred error already kept (it /// stays in its bank until the OS clears it, and another SMI of /// the bank's thread reads it again). #[inline(always)] fn push(&self, record: Record) { let stored = self .with(|state| { let kept = state.records.get(..state.count).unwrap_or_default(); if record.kind == Kind::Deferred && kept.iter().any(|r| r.same(&record)) { return true; } let Some(slot) = state.records.get_mut(state.count) else { return false; }; *slot = record; state.count += 1; true }) .unwrap_or(false); if !stored { self.dropped.fetch_add(1, Ordering::Relaxed); } } fn next(&self) -> Option { self.with(|state| { if state.pending >= state.count || state.sent >= SEL_PER_BOOT { return None; } let record = *state.records.get(state.pending)?; state.pending += 1; state.sent += 1; Some(record) }) .flatten() } fn waiting(&self) -> bool { self.with(|state| state.pending < state.count && state.sent < SEL_PER_BOOT) .unwrap_or(true) } } /// Every entrant runs this, including CPUs that do not own the FCH /// dispatch: the local MCA work of `policy`, then delivery of what it /// recorded through `bmc` unless another CPU delivers. Arming happens /// on the installer's per-CPU self-SMI, after SMRAM is locked. No CPU /// waits for a sibling and a journal collision never delays native #MC. #[inline(always)] pub fn enter( policy: &Policy, logical_cpu: u32, silicon: Silicon, bmc: impl FnMut(&Event<'_>, Tsc) -> bool, ) { if policy.smm() && enter_enabled(logical_cpu, silicon, policy.threshold, policy.observe_mce) { deliver(policy, bmc); } } // Keep SMRAM static references in their defining crate. Cross-crate // inlining into the board entry can materialize a GOT access, forbidden // in PIC SMM. True if a record was journaled. #[inline(never)] fn enter_enabled( logical_cpu: u32, silicon: Silicon, threshold: u16, observe_mce: bool, ) -> bool { let index = logical_cpu as usize; let (Some(state), Some(rearms)) = (CPU_STATE.get(index), REARMS.get(index)) else { return false; }; // SAFETY: SMM, ring 0. let cpu = unsafe { NativeMsr::new() }; let smbase = cpu.reg(SMM_BASE).get() as usize; // SAFETY: this thread's locked SMBASE window, entirely identity-mapped. let saved = unsafe { fstart_smm::X86SaveState::from_top( (smbase + 0x10000) as *mut u8, fstart_smm::X86SaveStateFormat::Amd64, ) }; if state.load(Ordering::Relaxed) == 0 { let stepping = core::arch::x86_64::__cpuid(1).eax as u8 & 0xf; let (port, command) = (fstart_driver_amd::fch::smi::APM_CNT, MCE_SMI_COMMAND); let armed = saved.revision().is_ok() && if observe_mce { ras::arm(&cpu, silicon, stepping, port, command) } else { ras::arm_trigger(&cpu, port, command) } .is_ok(); if armed && threshold != 0 { ras::arm_thresholds(&cpu, threshold); } state.store( match (armed, observe_mce) { (false, _) => REFUSED, (true, false) => TRIGGER, (true, true) => TRIGGER | PFEH, }, Ordering::Relaxed, ); if !armed { REFUSED_CPUS.fetch_add(1, Ordering::Relaxed); } } let armed = state.load(Ordering::Relaxed); if armed & TRIGGER == 0 { return false; } // SAFETY: this thread's installed AMD64 SMBASE; local status at +FEC4 // is in its locked save-state window, mapped by the SMM identity tables. let save = unsafe { Mmio::new(smbase) }; let status = save .reg::(ras::LOCAL_SMI_STATUS) .get(); diagnostics(&cpu, saved); let mut journaled = false; let mut sink = |record| { JOURNAL.push(record); journaled = true; }; if armed & PFEH != 0 { ras::observe(&cpu, logical_cpu, status, &mut sink); } let rearm = (threshold != 0 && rearms.load(Ordering::Relaxed) < REARMS_PER_CPU) .then_some(threshold); if let Some(events) = ras::observe_mca(&cpu, logical_cpu, status, rearm, &mut sink) && events.thresholds != 0 && rearm.is_some() { rearms.fetch_add(1, Ordering::Relaxed); } if armed & PFEH != 0 && LocalSmiStatus::WRMSR.is_set(status) { use fstart_smm::save_state::Amd64Register::{Rax, Rcx, Rdx}; // The trigger's SMI arrives after the write: the registers // are the ones at the SMI, which may no longer name it. let replayed = match ( saved.read_amd64(Rcx), saved.read_amd64(Rax), saved.read_amd64(Rdx), ) { (Ok(index), Ok(low), Ok(high)) => { let value = (high as u32 as u64) << 32 | low as u32 as u64; ras::replay_clear(&cpu, index as u32, value) } _ => false, }; if !replayed { // The write is lost (PFEH turned it into this SMI); the // OS's next poll logs the bank again and clears it, now // natively: this CPU leaves PFEH for good. ras::disarm(&cpu); state.store(TRIGGER, Ordering::Relaxed); REFUSED_CPUS.fetch_add(1, Ordering::Relaxed); } } journaled } fn diagnostics(cpu: &M, saved: fstart_smm::X86SaveState) { use fstart_smm::save_state::Amd64Register::{Rax, Rcx, Rdx}; if saved.read_amd64(Rax) != Ok(DIAG_REQUEST) || saved.read_amd64(Rdx) != Ok(0xb2) { return; } match saved.read_amd64(Rcx) { Ok(0) => { let _ = saved.write_amd64(Rax, cpu.reg(ras::PFEH_CONFIG).get()); let _ = saved.write_amd64(Rcx, cpu.reg(ras::PFEH_CLOAK).get()); let _ = saved.write_amd64(Rdx, cpu.reg(ras::PFEH_DEFERRED_MASK).get()); } Ok(3) => { let base = cpu.reg(SMM_BASE).get(); // SAFETY: this CPU's locked, mapped save-state window. let memory = unsafe { Mmio::new(base as usize) }; let _ = saved.write_amd64(Rax, u64::from(memory.read32(0xff00) == base as u32)); let _ = saved.write_amd64(Rcx, u64::from(memory.read32(0xfe00) == base as u32)); let _ = saved.write_amd64(Rdx, u64::from(saved.revision().unwrap_or(0))); } Ok(4) => { let count = JOURNAL.with(|state| state.count).unwrap_or(usize::MAX); let _ = saved.write_amd64(Rax, count as u64); let _ = saved.write_amd64(Rcx, JOURNAL.dropped.load(Ordering::Relaxed).into()); let _ = saved.write_amd64(Rdx, REFUSED_CPUS.load(Ordering::Relaxed).into()); } Ok(selector @ (1 | 2)) => { if selector == 1 { let _ = PROBE.compare_exchange(0, 1, Ordering::Release, Ordering::Relaxed); } let _ = saved.write_amd64(Rax, u64::from(PROBE.load(Ordering::Acquire))); } _ => {} } } /// Send what the journal holds to the board's BMC through `bmc`, at /// most [`SEL_PER_SMI`] events now; on the shared owner every SMI, /// and on a CPU right after it journaled. One CPU delivers at a time: /// the others leave their records to it. The first failed event stops /// delivery until reset (no retry, no wait for the BMC); OS handoff /// never depends on it. #[inline(always)] pub fn deliver(policy: &Policy, mut bmc: impl FnMut(&Event<'_>, Tsc) -> bool) { if !policy.smm() { return; } // SAFETY: SMM, ring 0; the P-state MSRs. let timer = Tsc::native(&unsafe { NativeMsr::new() }); while begin() { let mut sent = 0; while sent < SEL_PER_SMI { let Some(event) = next() else { break }; let ok = match event { Some(record) => bmc(&Event::Mca(&record), timer), None => bmc(&Event::Probe, timer), }; done(event.is_none(), ok); sent += 1; if !ok { break; } } if !end() { break; } } } /// Take delivery, unless another CPU has it or the BMC failed. #[inline(never)] fn begin() -> bool { if SEL_FAILED.load(Ordering::Relaxed) { if PROBE.load(Ordering::Acquire) == 1 { PROBE.store(3, Ordering::Release); } return false; } DELIVERING .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) .is_ok() } /// The next event: a record (`Some`), or the requested probe (`None`). #[inline(never)] fn next() -> Option> { if let Some(record) = JOURNAL.next() { return Some(Some(record)); } PROBE .compare_exchange(1, 4, Ordering::AcqRel, Ordering::Relaxed) .is_ok() .then_some(None) } #[inline(never)] fn done(probe: bool, ok: bool) { if !ok { SEL_FAILED.store(true, Ordering::Relaxed); } if probe { PROBE.store(if ok { 2 } else { 3 }, Ordering::Release); } } /// Give delivery up; true if a record came in meanwhile, whose CPU /// left it to this one, and delivery should go on. #[inline(never)] fn end() -> bool { DELIVERING.store(false, Ordering::Release); !SEL_FAILED.load(Ordering::Relaxed) && JOURNAL.waiting() } #[cfg(test)] mod tests { use super::super::sel; use super::*; use zerocopy::IntoBytes; #[test] fn journal_preserves_first_evidence_and_reports_overflow() { let journal = Journal::new(); for bank in 0..RECORDS as u32 + 3 { journal.push(Record { bank, status: 1 << 63, ..Record::EMPTY }); } assert_eq!(journal.dropped.load(Ordering::Relaxed), 3); journal .with(|s| { assert_eq!(s.count, RECORDS); assert_eq!(s.records[0].bank, 0); assert_eq!(s.records[RECORDS - 1].bank, RECORDS as u32 - 1); }) .unwrap(); for bank in 0..SEL_PER_BOOT { assert_eq!(journal.next().unwrap().bank, bank); } assert!(journal.next().is_none()); assert!(!journal.waiting(), "the per-boot budget is spent"); assert_eq!( journal.with(|s| s.count), Some(RECORDS), "delivery does not erase evidence" ); } #[test] fn a_deferred_error_read_again_is_kept_once() { let journal = Journal::new(); let deferred = Record { kind: Kind::Deferred, bank: 3, status: 1 << 63 | 1 << 44, ..Record::EMPTY }; journal.push(deferred); journal.push(Record { cpu: 9, ..deferred }); journal.push(Record { kind: Kind::Threshold, ..deferred }); journal.push(Record { kind: Kind::Threshold, ..deferred }); assert_eq!(journal.with(|s| s.count), Some(3)); } #[test] fn contended_producer_drops_instead_of_waiting() { let journal = Journal::new(); journal.with(|_| journal.push(Record::EMPTY)).unwrap(); assert_eq!(journal.dropped.load(Ordering::Relaxed), 1); assert_eq!(journal.with(|s| s.count), Some(0)); } #[test] fn diagnostics_only_return_registers_to_the_requesting_context() { use fstart_smm::save_state::Amd64Register::{Rax, Rcx, Rdx}; use fstart_smm::{X86SaveState, X86SaveStateFormat}; struct Cpu; impl MsrAccess for Cpu { fn read(&self, index: u32) -> u64 { match index { 0xc0010120 => 1, 0xc0010121 | 0xc0010122 => 0, _ => panic!("unexpected MSR"), } } fn write(&self, _: u32, _: u64) { panic!("diagnostics must not write MSRs"); } } let mut memory = [0u8; 0x400]; memory[0x2fc..0x300].copy_from_slice(&0x30064u32.to_le_bytes()); let saved = unsafe { X86SaveState::from_top(memory.as_mut_ptr().add(0x400), X86SaveStateFormat::Amd64) }; saved.write_amd64(Rax, DIAG_REQUEST).unwrap(); saved.write_amd64(Rcx, 0).unwrap(); saved.write_amd64(Rdx, 0xb3).unwrap(); diagnostics(&Cpu, saved); assert_eq!(saved.read_amd64(Rax), Ok(DIAG_REQUEST)); saved.write_amd64(Rdx, 0xb2).unwrap(); diagnostics(&Cpu, saved); assert_eq!(saved.read_amd64(Rax), Ok(1)); assert_eq!(saved.read_amd64(Rcx), Ok(0)); assert_eq!(saved.read_amd64(Rdx), Ok(0)); } #[test] fn standard_sel_has_no_guessed_dimm_or_sensor() { let record = Record { cpu: 255, bank: 63, status: 0xdead_beef_1234_5678, ..Record::EMPTY }; assert_eq!( sel(&Event::Mca(&record)).as_bytes(), &[0, 0, 2, 0, 0, 0, 0, 0x21, 0, 4, 7, 255, 0x6f, 0xab, 255, 63] ); } #[test] fn dram_errors_are_memory_events() { let umc = Record { cpu: 3, bank: 21, status: 1 << 63, ipid: 0x96 << 32, ..Record::EMPTY }; let threshold = Record { kind: Kind::Threshold, ..umc }; let uncorrected = Record { status: 1 << 63 | 1 << 61, ..umc }; assert_eq!( sel(&Event::Mca(&threshold)).as_bytes()[10..], [0xc, 0xff, 0x6f, 0xa5, 3, 21] ); assert_eq!( sel(&Event::Mca(&uncorrected)).as_bytes()[10..], [0xc, 0xff, 0x6f, 0xa1, 3, 21] ); assert_eq!( sel(&Event::Probe).as_bytes()[10..], [0x12, 0xff, 0x6f, 0xa3, b'F', b'S'] ); } } }