From b0e70da834594d6f497b9c6faead60a507478f00 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:32 +0000 Subject: [PATCH 01/31] x86/MCE/AMD, EDAC/amd64: Move address translation to AMD64 EDAC The address translation code used for current AMD systems is non-architectural. So move it to EDAC. Cc: Signed-off-by: Yazen Ghannam --- arch/x86/include/asm/mce.h | 3 - arch/x86/kernel/cpu/mce/amd.c | 200 ---------------------------------- drivers/edac/amd64_edac.c | 199 +++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 203 deletions(-) diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index da9321548..6bcc432b7 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -357,7 +357,6 @@ extern int mce_threshold_create_device(unsigned int cpu); extern int mce_threshold_remove_device(unsigned int cpu); void mce_amd_feature_init(struct cpuinfo_x86 *c); -int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr); #else @@ -365,8 +364,6 @@ static inline int mce_threshold_create_device(unsigned int cpu) { return 0; }; static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; }; static inline bool amd_mce_is_memory_error(struct mce *m) { return false; }; static inline void mce_amd_feature_init(struct cpuinfo_x86 *c) { } -static inline int -umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { return -EINVAL; }; #endif static inline void mce_hygon_feature_init(struct cpuinfo_x86 *c) { return mce_amd_feature_init(c); } diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 08831acc1..4e5ba898a 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -688,206 +688,6 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c) deferred_error_interrupt_enable(c); } -int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) -{ - u64 dram_base_addr, dram_limit_addr, dram_hole_base; - /* We start from the normalized address */ - u64 ret_addr = norm_addr; - - u32 tmp; - - u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; - u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; - u8 intlv_addr_sel, intlv_addr_bit; - u8 num_intlv_bits, hashed_bit; - u8 lgcy_mmio_hole_en, base = 0; - u8 cs_mask, cs_id = 0; - bool hash_enabled = false; - - /* Read D18F0x1B4 (DramOffset), check if base 1 is used. */ - if (amd_df_indirect_read(nid, 0, 0x1B4, umc, &tmp)) - goto out_err; - - /* Remove HiAddrOffset from normalized address, if enabled: */ - if (tmp & BIT(0)) { - u64 hi_addr_offset = (tmp & GENMASK_ULL(31, 20)) << 8; - - if (norm_addr >= hi_addr_offset) { - ret_addr -= hi_addr_offset; - base = 1; - } - } - - /* Read D18F0x110 (DramBaseAddress). */ - if (amd_df_indirect_read(nid, 0, 0x110 + (8 * base), umc, &tmp)) - goto out_err; - - /* Check if address range is valid. */ - if (!(tmp & BIT(0))) { - pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n", - __func__, tmp); - goto out_err; - } - - lgcy_mmio_hole_en = tmp & BIT(1); - intlv_num_chan = (tmp >> 4) & 0xF; - intlv_addr_sel = (tmp >> 8) & 0x7; - dram_base_addr = (tmp & GENMASK_ULL(31, 12)) << 16; - - /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ - if (intlv_addr_sel > 3) { - pr_err("%s: Invalid interleave address select %d.\n", - __func__, intlv_addr_sel); - goto out_err; - } - - /* Read D18F0x114 (DramLimitAddress). */ - if (amd_df_indirect_read(nid, 0, 0x114 + (8 * base), umc, &tmp)) - goto out_err; - - intlv_num_sockets = (tmp >> 8) & 0x1; - intlv_num_dies = (tmp >> 10) & 0x3; - dram_limit_addr = ((tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); - - intlv_addr_bit = intlv_addr_sel + 8; - - /* Re-use intlv_num_chan by setting it equal to log2(#channels) */ - switch (intlv_num_chan) { - case 0: intlv_num_chan = 0; break; - case 1: intlv_num_chan = 1; break; - case 3: intlv_num_chan = 2; break; - case 5: intlv_num_chan = 3; break; - case 7: intlv_num_chan = 4; break; - - case 8: intlv_num_chan = 1; - hash_enabled = true; - break; - default: - pr_err("%s: Invalid number of interleaved channels %d.\n", - __func__, intlv_num_chan); - goto out_err; - } - - num_intlv_bits = intlv_num_chan; - - if (intlv_num_dies > 2) { - pr_err("%s: Invalid number of interleaved nodes/dies %d.\n", - __func__, intlv_num_dies); - goto out_err; - } - - num_intlv_bits += intlv_num_dies; - - /* Add a bit if sockets are interleaved. */ - num_intlv_bits += intlv_num_sockets; - - /* Assert num_intlv_bits <= 4 */ - if (num_intlv_bits > 4) { - pr_err("%s: Invalid interleave bits %d.\n", - __func__, num_intlv_bits); - goto out_err; - } - - if (num_intlv_bits > 0) { - u64 temp_addr_x, temp_addr_i, temp_addr_y; - u8 die_id_bit, sock_id_bit, cs_fabric_id; - - /* - * Read FabricBlockInstanceInformation3_CS[BlockFabricID]. - * This is the fabric id for this coherent slave. Use - * umc/channel# as instance id of the coherent slave - * for FICAA. - */ - if (amd_df_indirect_read(nid, 0, 0x50, umc, &tmp)) - goto out_err; - - cs_fabric_id = (tmp >> 8) & 0xFF; - die_id_bit = 0; - - /* If interleaved over more than 1 channel: */ - if (intlv_num_chan) { - die_id_bit = intlv_num_chan; - cs_mask = (1 << die_id_bit) - 1; - cs_id = cs_fabric_id & cs_mask; - } - - sock_id_bit = die_id_bit; - - /* Read D18F1x208 (SystemFabricIdMask). */ - if (intlv_num_dies || intlv_num_sockets) - if (amd_df_indirect_read(nid, 1, 0x208, umc, &tmp)) - goto out_err; - - /* If interleaved over more than 1 die. */ - if (intlv_num_dies) { - sock_id_bit = die_id_bit + intlv_num_dies; - die_id_shift = (tmp >> 24) & 0xF; - die_id_mask = (tmp >> 8) & 0xFF; - - cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; - } - - /* If interleaved over more than 1 socket. */ - if (intlv_num_sockets) { - socket_id_shift = (tmp >> 28) & 0xF; - socket_id_mask = (tmp >> 16) & 0xFF; - - cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit; - } - - /* - * The pre-interleaved address consists of XXXXXXIIIYYYYY - * where III is the ID for this CS, and XXXXXXYYYYY are the - * address bits from the post-interleaved address. - * "num_intlv_bits" has been calculated to tell us how many "I" - * bits there are. "intlv_addr_bit" tells us how many "Y" bits - * there are (where "I" starts). - */ - temp_addr_y = ret_addr & GENMASK_ULL(intlv_addr_bit-1, 0); - temp_addr_i = (cs_id << intlv_addr_bit); - temp_addr_x = (ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits; - ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; - } - - /* Add dram base address */ - ret_addr += dram_base_addr; - - /* If legacy MMIO hole enabled */ - if (lgcy_mmio_hole_en) { - if (amd_df_indirect_read(nid, 0, 0x104, umc, &tmp)) - goto out_err; - - dram_hole_base = tmp & GENMASK(31, 24); - if (ret_addr >= dram_hole_base) - ret_addr += (BIT_ULL(32) - dram_hole_base); - } - - if (hash_enabled) { - /* Save some parentheses and grab ls-bit at the end. */ - hashed_bit = (ret_addr >> 12) ^ - (ret_addr >> 18) ^ - (ret_addr >> 21) ^ - (ret_addr >> 30) ^ - cs_id; - - hashed_bit &= BIT(0); - - if (hashed_bit != ((ret_addr >> intlv_addr_bit) & BIT(0))) - ret_addr ^= BIT(intlv_addr_bit); - } - - /* Is calculated system address is above DRAM limit address? */ - if (ret_addr > dram_limit_addr) - goto out_err; - - *sys_addr = ret_addr; - return 0; - -out_err: - return -EINVAL; -} -EXPORT_SYMBOL_GPL(umc_normaddr_to_sysaddr); - bool amd_mce_is_memory_error(struct mce *m) { /* ErrCodeExt[20:16] */ diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 99b06a3e8..fba375253 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -988,6 +988,205 @@ static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) return csrow; } +static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) +{ + u64 dram_base_addr, dram_limit_addr, dram_hole_base; + /* We start from the normalized address */ + u64 ret_addr = norm_addr; + + u32 tmp; + + u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; + u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; + u8 intlv_addr_sel, intlv_addr_bit; + u8 num_intlv_bits, hashed_bit; + u8 lgcy_mmio_hole_en, base = 0; + u8 cs_mask, cs_id = 0; + bool hash_enabled = false; + + /* Read D18F0x1B4 (DramOffset), check if base 1 is used. */ + if (amd_df_indirect_read(nid, 0, 0x1B4, umc, &tmp)) + goto out_err; + + /* Remove HiAddrOffset from normalized address, if enabled: */ + if (tmp & BIT(0)) { + u64 hi_addr_offset = (tmp & GENMASK_ULL(31, 20)) << 8; + + if (norm_addr >= hi_addr_offset) { + ret_addr -= hi_addr_offset; + base = 1; + } + } + + /* Read D18F0x110 (DramBaseAddress). */ + if (amd_df_indirect_read(nid, 0, 0x110 + (8 * base), umc, &tmp)) + goto out_err; + + /* Check if address range is valid. */ + if (!(tmp & BIT(0))) { + pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n", + __func__, tmp); + goto out_err; + } + + lgcy_mmio_hole_en = tmp & BIT(1); + intlv_num_chan = (tmp >> 4) & 0xF; + intlv_addr_sel = (tmp >> 8) & 0x7; + dram_base_addr = (tmp & GENMASK_ULL(31, 12)) << 16; + + /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ + if (intlv_addr_sel > 3) { + pr_err("%s: Invalid interleave address select %d.\n", + __func__, intlv_addr_sel); + goto out_err; + } + + /* Read D18F0x114 (DramLimitAddress). */ + if (amd_df_indirect_read(nid, 0, 0x114 + (8 * base), umc, &tmp)) + goto out_err; + + intlv_num_sockets = (tmp >> 8) & 0x1; + intlv_num_dies = (tmp >> 10) & 0x3; + dram_limit_addr = ((tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); + + intlv_addr_bit = intlv_addr_sel + 8; + + /* Re-use intlv_num_chan by setting it equal to log2(#channels) */ + switch (intlv_num_chan) { + case 0: intlv_num_chan = 0; break; + case 1: intlv_num_chan = 1; break; + case 3: intlv_num_chan = 2; break; + case 5: intlv_num_chan = 3; break; + case 7: intlv_num_chan = 4; break; + + case 8: intlv_num_chan = 1; + hash_enabled = true; + break; + default: + pr_err("%s: Invalid number of interleaved channels %d.\n", + __func__, intlv_num_chan); + goto out_err; + } + + num_intlv_bits = intlv_num_chan; + + if (intlv_num_dies > 2) { + pr_err("%s: Invalid number of interleaved nodes/dies %d.\n", + __func__, intlv_num_dies); + goto out_err; + } + + num_intlv_bits += intlv_num_dies; + + /* Add a bit if sockets are interleaved. */ + num_intlv_bits += intlv_num_sockets; + + /* Assert num_intlv_bits <= 4 */ + if (num_intlv_bits > 4) { + pr_err("%s: Invalid interleave bits %d.\n", + __func__, num_intlv_bits); + goto out_err; + } + + if (num_intlv_bits > 0) { + u64 temp_addr_x, temp_addr_i, temp_addr_y; + u8 die_id_bit, sock_id_bit, cs_fabric_id; + + /* + * Read FabricBlockInstanceInformation3_CS[BlockFabricID]. + * This is the fabric id for this coherent slave. Use + * umc/channel# as instance id of the coherent slave + * for FICAA. + */ + if (amd_df_indirect_read(nid, 0, 0x50, umc, &tmp)) + goto out_err; + + cs_fabric_id = (tmp >> 8) & 0xFF; + die_id_bit = 0; + + /* If interleaved over more than 1 channel: */ + if (intlv_num_chan) { + die_id_bit = intlv_num_chan; + cs_mask = (1 << die_id_bit) - 1; + cs_id = cs_fabric_id & cs_mask; + } + + sock_id_bit = die_id_bit; + + /* Read D18F1x208 (SystemFabricIdMask). */ + if (intlv_num_dies || intlv_num_sockets) + if (amd_df_indirect_read(nid, 1, 0x208, umc, &tmp)) + goto out_err; + + /* If interleaved over more than 1 die. */ + if (intlv_num_dies) { + sock_id_bit = die_id_bit + intlv_num_dies; + die_id_shift = (tmp >> 24) & 0xF; + die_id_mask = (tmp >> 8) & 0xFF; + + cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; + } + + /* If interleaved over more than 1 socket. */ + if (intlv_num_sockets) { + socket_id_shift = (tmp >> 28) & 0xF; + socket_id_mask = (tmp >> 16) & 0xFF; + + cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit; + } + + /* + * The pre-interleaved address consists of XXXXXXIIIYYYYY + * where III is the ID for this CS, and XXXXXXYYYYY are the + * address bits from the post-interleaved address. + * "num_intlv_bits" has been calculated to tell us how many "I" + * bits there are. "intlv_addr_bit" tells us how many "Y" bits + * there are (where "I" starts). + */ + temp_addr_y = ret_addr & GENMASK_ULL(intlv_addr_bit-1, 0); + temp_addr_i = (cs_id << intlv_addr_bit); + temp_addr_x = (ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits; + ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; + } + + /* Add dram base address */ + ret_addr += dram_base_addr; + + /* If legacy MMIO hole enabled */ + if (lgcy_mmio_hole_en) { + if (amd_df_indirect_read(nid, 0, 0x104, umc, &tmp)) + goto out_err; + + dram_hole_base = tmp & GENMASK(31, 24); + if (ret_addr >= dram_hole_base) + ret_addr += (BIT_ULL(32) - dram_hole_base); + } + + if (hash_enabled) { + /* Save some parentheses and grab ls-bit at the end. */ + hashed_bit = (ret_addr >> 12) ^ + (ret_addr >> 18) ^ + (ret_addr >> 21) ^ + (ret_addr >> 30) ^ + cs_id; + + hashed_bit &= BIT(0); + + if (hashed_bit != ((ret_addr >> intlv_addr_bit) & BIT(0))) + ret_addr ^= BIT(intlv_addr_bit); + } + + /* Is calculated system address is above DRAM limit address? */ + if (ret_addr > dram_limit_addr) + goto out_err; + + *sys_addr = ret_addr; + return 0; + +out_err: + return -EINVAL; +} + static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); /* -- 2.33.1.711.g9d530dc002 From cfdba55f332e860ba63936df41bd8b57f44c8e51 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:33 +0000 Subject: [PATCH 02/31] x86/amd_nb, EDAC/amd64: Move DF Indirect Read to AMD64 EDAC The df_indirect_read() function is only used for address translation. Move this to EDAC along with the translation code. Cc: Signed-off-by: Yazen Ghannam --- arch/x86/include/asm/amd_nb.h | 1 - arch/x86/kernel/amd_nb.c | 50 +---------------------------------- drivers/edac/amd64_edac.c | 50 +++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h index 455066a06..00d1a400b 100644 --- a/arch/x86/include/asm/amd_nb.h +++ b/arch/x86/include/asm/amd_nb.h @@ -24,7 +24,6 @@ extern int amd_set_subcaches(int, unsigned long); extern int amd_smn_read(u16 node, u32 address, u32 *value); extern int amd_smn_write(u16 node, u32 address, u32 value); -extern int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo); struct amd_l3_cache { unsigned indices; diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index c92c9c774..a43d1ef74 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -29,7 +29,7 @@ #define PCI_DEVICE_ID_AMD_19H_M40H_DF_F4 0x167d #define PCI_DEVICE_ID_AMD_19H_M50H_DF_F4 0x166e -/* Protect the PCI config register pairs used for SMN and DF indirect access. */ +/* Protect the PCI config register pairs used for SMN. */ static DEFINE_MUTEX(smn_mutex); static u32 *flush_words; @@ -182,54 +182,6 @@ int amd_smn_write(u16 node, u32 address, u32 value) } EXPORT_SYMBOL_GPL(amd_smn_write); -/* - * Data Fabric Indirect Access uses FICAA/FICAD. - * - * Fabric Indirect Configuration Access Address (FICAA): Constructed based - * on the device's Instance Id and the PCI function and register offset of - * the desired register. - * - * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO - * and FICAD HI registers but so far we only need the LO register. - */ -int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo) -{ - struct pci_dev *F4; - u32 ficaa; - int err = -ENODEV; - - if (node >= amd_northbridges.num) - goto out; - - F4 = node_to_amd_nb(node)->link; - if (!F4) - goto out; - - ficaa = 1; - ficaa |= reg & 0x3FC; - ficaa |= (func & 0x7) << 11; - ficaa |= instance_id << 16; - - mutex_lock(&smn_mutex); - - err = pci_write_config_dword(F4, 0x5C, ficaa); - if (err) { - pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa); - goto out_unlock; - } - - err = pci_read_config_dword(F4, 0x98, lo); - if (err) - pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa); - -out_unlock: - mutex_unlock(&smn_mutex); - -out: - return err; -} -EXPORT_SYMBOL_GPL(amd_df_indirect_read); - int amd_cache_northbridges(void) { const struct pci_device_id *misc_ids = amd_nb_misc_ids; diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index fba375253..409916c0e 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -988,6 +988,56 @@ static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) return csrow; } +/* Protect the PCI config register pairs used for DF indirect access. */ +static DEFINE_MUTEX(df_indirect_mutex); + +/* + * Data Fabric Indirect Access uses FICAA/FICAD. + * + * Fabric Indirect Configuration Access Address (FICAA): Constructed based + * on the device's Instance Id and the PCI function and register offset of + * the desired register. + * + * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO + * and FICAD HI registers but so far we only need the LO register. + */ +static int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo) +{ + struct pci_dev *F4; + u32 ficaa; + int err = -ENODEV; + + if (node >= amd_nb_num()) + goto out; + + F4 = node_to_amd_nb(node)->link; + if (!F4) + goto out; + + ficaa = 1; + ficaa |= reg & 0x3FC; + ficaa |= (func & 0x7) << 11; + ficaa |= instance_id << 16; + + mutex_lock(&df_indirect_mutex); + + err = pci_write_config_dword(F4, 0x5C, ficaa); + if (err) { + pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa); + goto out_unlock; + } + + err = pci_read_config_dword(F4, 0x98, lo); + if (err) + pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa); + +out_unlock: + mutex_unlock(&df_indirect_mutex); + +out: + return err; +} + static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { u64 dram_base_addr, dram_limit_addr, dram_hole_base; -- 2.33.1.711.g9d530dc002 From 55cf1a8cc6dcd67cbf7f4d1ca9127da0cabd6936 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:34 +0000 Subject: [PATCH 03/31] EDAC/amd64: Don't use naked values for DF registers AMD Data Fabric registers are defined using a combination of PCI function number and offset. Define a struct to hold these values, and update the DF Indirect Access function to accept a struct of this type. Update the address translation code to include a list of the needed DF registers using this new format. Define an enumeration to give the registers more human-readable names. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 60 ++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 409916c0e..6985fe9bb 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -991,6 +991,11 @@ static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) /* Protect the PCI config register pairs used for DF indirect access. */ static DEFINE_MUTEX(df_indirect_mutex); +struct df_reg { + u8 func; + u16 offset; +}; + /* * Data Fabric Indirect Access uses FICAA/FICAD. * @@ -1001,7 +1006,7 @@ static DEFINE_MUTEX(df_indirect_mutex); * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO * and FICAD HI registers but so far we only need the LO register. */ -static int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo) +static int amd_df_indirect_read(u16 node, struct df_reg reg, u8 instance_id, u32 *lo) { struct pci_dev *F4; u32 ficaa; @@ -1015,8 +1020,8 @@ static int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 goto out; ficaa = 1; - ficaa |= reg & 0x3FC; - ficaa |= (func & 0x7) << 11; + ficaa |= reg.offset & 0x3FC; + ficaa |= (reg.func & 0x7) << 11; ficaa |= instance_id << 16; mutex_lock(&df_indirect_mutex); @@ -1038,6 +1043,33 @@ static int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 return err; } +enum df_reg_names { + /* Function 0 */ + FAB_BLK_INST_INFO_3, + DRAM_HOLE_CTL, + DRAM_BASE_ADDR, + DRAM_LIMIT_ADDR, + DRAM_OFFSET, + + /* Function 1 */ + SYS_FAB_ID_MASK, +}; + +static struct df_reg df_regs[] = { + /* D18F0x50 (FabricBlockInstanceInformation3_CS) */ + [FAB_BLK_INST_INFO_3] = {0, 0x50}, + /* D18F0x104 (DramHoleControl) */ + [DRAM_HOLE_CTL] = {0, 0x104}, + /* D18F0x110 (DramBaseAddress) */ + [DRAM_BASE_ADDR] = {0, 0x110}, + /* D18F0x114 (DramLimitAddress) */ + [DRAM_LIMIT_ADDR] = {0, 0x114}, + /* D18F0x1B4 (DramOffset) */ + [DRAM_OFFSET] = {0, 0x1B4}, + /* D18F1x208 (SystemFabricIdMask) */ + [SYS_FAB_ID_MASK] = {1, 0x208}, +}; + static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { u64 dram_base_addr, dram_limit_addr, dram_hole_base; @@ -1054,8 +1086,9 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr u8 cs_mask, cs_id = 0; bool hash_enabled = false; - /* Read D18F0x1B4 (DramOffset), check if base 1 is used. */ - if (amd_df_indirect_read(nid, 0, 0x1B4, umc, &tmp)) + struct df_reg reg; + + if (amd_df_indirect_read(nid, df_regs[DRAM_OFFSET], umc, &tmp)) goto out_err; /* Remove HiAddrOffset from normalized address, if enabled: */ @@ -1068,8 +1101,9 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr } } - /* Read D18F0x110 (DramBaseAddress). */ - if (amd_df_indirect_read(nid, 0, 0x110 + (8 * base), umc, &tmp)) + reg = df_regs[DRAM_BASE_ADDR]; + reg.offset += base * 8; + if (amd_df_indirect_read(nid, reg, umc, &tmp)) goto out_err; /* Check if address range is valid. */ @@ -1091,8 +1125,9 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr goto out_err; } - /* Read D18F0x114 (DramLimitAddress). */ - if (amd_df_indirect_read(nid, 0, 0x114 + (8 * base), umc, &tmp)) + reg = df_regs[DRAM_LIMIT_ADDR]; + reg.offset += base * 8; + if (amd_df_indirect_read(nid, reg, umc, &tmp)) goto out_err; intlv_num_sockets = (tmp >> 8) & 0x1; @@ -1148,7 +1183,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr * umc/channel# as instance id of the coherent slave * for FICAA. */ - if (amd_df_indirect_read(nid, 0, 0x50, umc, &tmp)) + if (amd_df_indirect_read(nid, df_regs[FAB_BLK_INST_INFO_3], umc, &tmp)) goto out_err; cs_fabric_id = (tmp >> 8) & 0xFF; @@ -1163,9 +1198,8 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr sock_id_bit = die_id_bit; - /* Read D18F1x208 (SystemFabricIdMask). */ if (intlv_num_dies || intlv_num_sockets) - if (amd_df_indirect_read(nid, 1, 0x208, umc, &tmp)) + if (amd_df_indirect_read(nid, df_regs[SYS_FAB_ID_MASK], umc, &tmp)) goto out_err; /* If interleaved over more than 1 die. */ @@ -1204,7 +1238,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr /* If legacy MMIO hole enabled */ if (lgcy_mmio_hole_en) { - if (amd_df_indirect_read(nid, 0, 0x104, umc, &tmp)) + if (amd_df_indirect_read(nid, df_regs[DRAM_HOLE_CTL], umc, &tmp)) goto out_err; dram_hole_base = tmp & GENMASK(31, 24); -- 2.33.1.711.g9d530dc002 From 3a303fbb7059f44dcb8801a9f5cf8441fda4a200 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:35 +0000 Subject: [PATCH 04/31] EDAC/amd64: Allow for DF Indirect Broadcast reads The DF Indirect Access method allows for "Broadcast" accesses in which case no specific instance is targeted. Add support using a reserved instance ID of 0xFF to indicate a broadcast access. Set the FICAA register appropriately. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 6985fe9bb..e2abb4ac8 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1005,7 +1005,11 @@ struct df_reg { * * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO * and FICAD HI registers but so far we only need the LO register. + * + * Use Instance Id 0xFF to indicate a broadcast read. */ + +#define DF_BROADCAST 0xFF static int amd_df_indirect_read(u16 node, struct df_reg reg, u8 instance_id, u32 *lo) { struct pci_dev *F4; @@ -1019,7 +1023,7 @@ static int amd_df_indirect_read(u16 node, struct df_reg reg, u8 instance_id, u32 if (!F4) goto out; - ficaa = 1; + ficaa = (instance_id == DF_BROADCAST) ? 0 : 1; ficaa |= reg.offset & 0x3FC; ficaa |= (reg.func & 0x7) << 11; ficaa |= instance_id << 16; -- 2.33.1.711.g9d530dc002 From 7591e8aac56679adccf4fb33677a5f91a0e4756f Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:36 +0000 Subject: [PATCH 05/31] EDAC/amd64: Add context struct Define an address translation context struct. This will hold values that will be passed between multiple functions. Save return address, Node ID, and the Instance ID number to start. Currently, we use the UMC number as the Instance ID, but future DF versions may use another value. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 49 +++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index e2abb4ac8..98bea2dbd 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1074,12 +1074,15 @@ static struct df_reg df_regs[] = { [SYS_FAB_ID_MASK] = {1, 0x208}, }; +struct addr_ctx { + u64 ret_addr; + u16 nid; + u8 inst_id; +}; + static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { u64 dram_base_addr, dram_limit_addr, dram_hole_base; - /* We start from the normalized address */ - u64 ret_addr = norm_addr; - u32 tmp; u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; @@ -1092,6 +1095,16 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr struct df_reg reg; + struct addr_ctx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + /* We start from the normalized address */ + ctx.ret_addr = norm_addr; + + ctx.nid = nid; + ctx.inst_id = umc; + if (amd_df_indirect_read(nid, df_regs[DRAM_OFFSET], umc, &tmp)) goto out_err; @@ -1100,7 +1113,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr u64 hi_addr_offset = (tmp & GENMASK_ULL(31, 20)) << 8; if (norm_addr >= hi_addr_offset) { - ret_addr -= hi_addr_offset; + ctx.ret_addr -= hi_addr_offset; base = 1; } } @@ -1231,14 +1244,14 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr * bits there are. "intlv_addr_bit" tells us how many "Y" bits * there are (where "I" starts). */ - temp_addr_y = ret_addr & GENMASK_ULL(intlv_addr_bit-1, 0); + temp_addr_y = ctx.ret_addr & GENMASK_ULL(intlv_addr_bit - 1, 0); temp_addr_i = (cs_id << intlv_addr_bit); - temp_addr_x = (ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits; - ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; + temp_addr_x = (ctx.ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits; + ctx.ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; } /* Add dram base address */ - ret_addr += dram_base_addr; + ctx.ret_addr += dram_base_addr; /* If legacy MMIO hole enabled */ if (lgcy_mmio_hole_en) { @@ -1246,29 +1259,29 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr goto out_err; dram_hole_base = tmp & GENMASK(31, 24); - if (ret_addr >= dram_hole_base) - ret_addr += (BIT_ULL(32) - dram_hole_base); + if (ctx.ret_addr >= dram_hole_base) + ctx.ret_addr += (BIT_ULL(32) - dram_hole_base); } if (hash_enabled) { /* Save some parentheses and grab ls-bit at the end. */ - hashed_bit = (ret_addr >> 12) ^ - (ret_addr >> 18) ^ - (ret_addr >> 21) ^ - (ret_addr >> 30) ^ + hashed_bit = (ctx.ret_addr >> 12) ^ + (ctx.ret_addr >> 18) ^ + (ctx.ret_addr >> 21) ^ + (ctx.ret_addr >> 30) ^ cs_id; hashed_bit &= BIT(0); - if (hashed_bit != ((ret_addr >> intlv_addr_bit) & BIT(0))) - ret_addr ^= BIT(intlv_addr_bit); + if (hashed_bit != ((ctx.ret_addr >> intlv_addr_bit) & BIT(0))) + ctx.ret_addr ^= BIT(intlv_addr_bit); } /* Is calculated system address is above DRAM limit address? */ - if (ret_addr > dram_limit_addr) + if (ctx.ret_addr > dram_limit_addr) goto out_err; - *sys_addr = ret_addr; + *sys_addr = ctx.ret_addr; return 0; out_err: -- 2.33.1.711.g9d530dc002 From 7e8a8faa31558babaac4c3c68b51df7a4cf2c805 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:37 +0000 Subject: [PATCH 06/31] EDAC/amd64: Define Data Fabric operations Define a stub to hold operations for different Data Fabric versions. This will be filled in following patches. Define a function to set the appropriate operations. Use a return code because future updates have checks that may fail. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 98bea2dbd..92e7b51f8 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1080,6 +1080,21 @@ struct addr_ctx { u8 inst_id; }; +struct data_fabric_ops { +}; + +struct data_fabric_ops df2_ops = { +}; + +struct data_fabric_ops *df_ops; + +static int set_df_ops(struct addr_ctx *ctx) +{ + df_ops = &df2_ops; + + return 0; +} + static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { u64 dram_base_addr, dram_limit_addr, dram_hole_base; @@ -1105,6 +1120,9 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr ctx.nid = nid; ctx.inst_id = umc; + if (set_df_ops(&ctx)) + return -EINVAL; + if (amd_df_indirect_read(nid, df_regs[DRAM_OFFSET], umc, &tmp)) goto out_err; -- 2.33.1.711.g9d530dc002 From 89d7047f23fb1b4d72f67769d2cf138afec1c249 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:38 +0000 Subject: [PATCH 07/31] EDAC/amd64: Define functions for DramOffset Add helper functions to read the DramOffset register and to remove the offset from the calculated address. The helper functions will be expanded in future DF versions. Rename the "base" variable to "map_num" to indicate that this is the address map number. An address map is defined with a base and limit value. The map_num variable is used to select the proper base and limit registers to use for the address translation. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 57 +++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 92e7b51f8..1a15c2f4c 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1074,16 +1074,26 @@ static struct df_reg df_regs[] = { [SYS_FAB_ID_MASK] = {1, 0x208}, }; +/* Use "reg_" prefix for raw register values. */ struct addr_ctx { u64 ret_addr; + u32 reg_dram_offset; u16 nid; u8 inst_id; + u8 map_num; }; struct data_fabric_ops { + u64 (*get_hi_addr_offset)(struct addr_ctx *ctx); }; +static u64 get_hi_addr_offset_df2(struct addr_ctx *ctx) +{ + return (ctx->reg_dram_offset & GENMASK_ULL(31, 20)) << 8; +} + struct data_fabric_ops df2_ops = { + .get_hi_addr_offset = &get_hi_addr_offset_df2, }; struct data_fabric_ops *df_ops; @@ -1095,6 +1105,35 @@ static int set_df_ops(struct addr_ctx *ctx) return 0; } +static int get_dram_offset_reg(struct addr_ctx *ctx) +{ + if (amd_df_indirect_read(ctx->nid, df_regs[DRAM_OFFSET], + ctx->inst_id, &ctx->reg_dram_offset)) + return -EINVAL; + + return 0; +} + +static int remove_dram_offset(struct addr_ctx *ctx) +{ + if (get_dram_offset_reg(ctx)) + return -EINVAL; + + ctx->map_num = 0; + + /* Remove HiAddrOffset from normalized address, if enabled: */ + if (ctx->reg_dram_offset & BIT(0)) { + u64 hi_addr_offset = df_ops->get_hi_addr_offset(ctx); + + if (ctx->ret_addr >= hi_addr_offset) { + ctx->ret_addr -= hi_addr_offset; + ctx->map_num = 1; + } + } + + return 0; +} + static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { u64 dram_base_addr, dram_limit_addr, dram_hole_base; @@ -1104,7 +1143,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; u8 intlv_addr_sel, intlv_addr_bit; u8 num_intlv_bits, hashed_bit; - u8 lgcy_mmio_hole_en, base = 0; + u8 lgcy_mmio_hole_en; u8 cs_mask, cs_id = 0; bool hash_enabled = false; @@ -1123,21 +1162,11 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (set_df_ops(&ctx)) return -EINVAL; - if (amd_df_indirect_read(nid, df_regs[DRAM_OFFSET], umc, &tmp)) + if (remove_dram_offset(&ctx)) goto out_err; - /* Remove HiAddrOffset from normalized address, if enabled: */ - if (tmp & BIT(0)) { - u64 hi_addr_offset = (tmp & GENMASK_ULL(31, 20)) << 8; - - if (norm_addr >= hi_addr_offset) { - ctx.ret_addr -= hi_addr_offset; - base = 1; - } - } - reg = df_regs[DRAM_BASE_ADDR]; - reg.offset += base * 8; + reg.offset += ctx.map_num * 8; if (amd_df_indirect_read(nid, reg, umc, &tmp)) goto out_err; @@ -1161,7 +1190,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr } reg = df_regs[DRAM_LIMIT_ADDR]; - reg.offset += base * 8; + reg.offset += ctx.map_num * 8; if (amd_df_indirect_read(nid, reg, umc, &tmp)) goto out_err; -- 2.33.1.711.g9d530dc002 From 7584ffe89612a9115c5982577fa3e9920e9ebe39 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:39 +0000 Subject: [PATCH 08/31] EDAC/amd64: Define function to read DRAM address map registers Move the reading of the base and limit registers into a helper function. Save the raw values in the context struct as they will be parsed later. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 58 +++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 1a15c2f4c..fc6581a22 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1078,6 +1078,8 @@ static struct df_reg df_regs[] = { struct addr_ctx { u64 ret_addr; u32 reg_dram_offset; + u32 reg_base_addr; + u32 reg_limit_addr; u16 nid; u8 inst_id; u8 map_num; @@ -1134,6 +1136,30 @@ static int remove_dram_offset(struct addr_ctx *ctx) return 0; } +static int get_dram_addr_map(struct addr_ctx *ctx) +{ + struct df_reg reg = df_regs[DRAM_BASE_ADDR]; + + reg.offset += ctx->map_num * 8; + + if (amd_df_indirect_read(ctx->nid, reg, ctx->inst_id, &ctx->reg_base_addr)) + return -EINVAL; + + /* Check if address range is valid. */ + if (!(ctx->reg_base_addr & BIT(0))) { + pr_debug("Invalid DramBaseAddress range: 0x%x.\n", ctx->reg_base_addr); + return -EINVAL; + } + + reg = df_regs[DRAM_LIMIT_ADDR]; + reg.offset += ctx->map_num * 8; + + if (amd_df_indirect_read(ctx->nid, reg, ctx->inst_id, &ctx->reg_limit_addr)) + return -EINVAL; + + return 0; +} + static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { u64 dram_base_addr, dram_limit_addr, dram_hole_base; @@ -1147,8 +1173,6 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr u8 cs_mask, cs_id = 0; bool hash_enabled = false; - struct df_reg reg; - struct addr_ctx ctx; memset(&ctx, 0, sizeof(ctx)); @@ -1165,22 +1189,13 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (remove_dram_offset(&ctx)) goto out_err; - reg = df_regs[DRAM_BASE_ADDR]; - reg.offset += ctx.map_num * 8; - if (amd_df_indirect_read(nid, reg, umc, &tmp)) - goto out_err; - - /* Check if address range is valid. */ - if (!(tmp & BIT(0))) { - pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n", - __func__, tmp); + if (get_dram_addr_map(&ctx)) goto out_err; - } - lgcy_mmio_hole_en = tmp & BIT(1); - intlv_num_chan = (tmp >> 4) & 0xF; - intlv_addr_sel = (tmp >> 8) & 0x7; - dram_base_addr = (tmp & GENMASK_ULL(31, 12)) << 16; + lgcy_mmio_hole_en = ctx.reg_base_addr & BIT(1); + intlv_num_chan = (ctx.reg_base_addr >> 4) & 0xF; + intlv_addr_sel = (ctx.reg_base_addr >> 8) & 0x7; + dram_base_addr = (ctx.reg_base_addr & GENMASK_ULL(31, 12)) << 16; /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ if (intlv_addr_sel > 3) { @@ -1189,14 +1204,9 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr goto out_err; } - reg = df_regs[DRAM_LIMIT_ADDR]; - reg.offset += ctx.map_num * 8; - if (amd_df_indirect_read(nid, reg, umc, &tmp)) - goto out_err; - - intlv_num_sockets = (tmp >> 8) & 0x1; - intlv_num_dies = (tmp >> 10) & 0x3; - dram_limit_addr = ((tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); + intlv_num_sockets = (ctx.reg_limit_addr >> 8) & 0x1; + intlv_num_dies = (ctx.reg_limit_addr >> 10) & 0x3; + dram_limit_addr = ((ctx.reg_limit_addr & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); intlv_addr_bit = intlv_addr_sel + 8; -- 2.33.1.711.g9d530dc002 From f8b8a4a2ce4899de1bb9d5aeb0d1e5d2f27d731b Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:40 +0000 Subject: [PATCH 09/31] EDAC/amd64: Define function to find interleaving mode Define a helper function to find the interleaving mode. Define a DF2-specific function now. Future DF versions will have their own functions. Use an enumeration for the interleaving modes to give a human-readable value. Save the interleaving mode in the context struct, since this will be used in multiple functions. Multiple interleaving modes support hashing, so save a boolean in the context struct to check if hashing is enabled. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index fc6581a22..d4f39e7f2 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1074,8 +1074,16 @@ static struct df_reg df_regs[] = { [SYS_FAB_ID_MASK] = {1, 0x208}, }; +/* These are mapped 1:1 to the hardware values. Special cases are set at > 0x20. */ +enum intlv_modes { + NONE = 0x00, + NOHASH_2CH = 0x01, + DF2_HASH_2CH = 0x21, +}; + /* Use "reg_" prefix for raw register values. */ struct addr_ctx { + enum intlv_modes intlv_mode; u64 ret_addr; u32 reg_dram_offset; u32 reg_base_addr; @@ -1083,10 +1091,12 @@ struct addr_ctx { u16 nid; u8 inst_id; u8 map_num; + bool hash_enabled; }; struct data_fabric_ops { u64 (*get_hi_addr_offset)(struct addr_ctx *ctx); + int (*get_intlv_mode)(struct addr_ctx *ctx); }; static u64 get_hi_addr_offset_df2(struct addr_ctx *ctx) @@ -1094,8 +1104,26 @@ static u64 get_hi_addr_offset_df2(struct addr_ctx *ctx) return (ctx->reg_dram_offset & GENMASK_ULL(31, 20)) << 8; } +static int get_intlv_mode_df2(struct addr_ctx *ctx) +{ + ctx->intlv_mode = (ctx->reg_base_addr >> 4) & 0xF; + + if (ctx->intlv_mode == 8) { + ctx->intlv_mode = DF2_HASH_2CH; + ctx->hash_enabled = true; + } + + if (ctx->intlv_mode != NONE && + ctx->intlv_mode != NOHASH_2CH && + ctx->intlv_mode != DF2_HASH_2CH) + return -EINVAL; + + return 0; +} + struct data_fabric_ops df2_ops = { .get_hi_addr_offset = &get_hi_addr_offset_df2, + .get_intlv_mode = &get_intlv_mode_df2, }; struct data_fabric_ops *df_ops; @@ -1171,7 +1199,6 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr u8 num_intlv_bits, hashed_bit; u8 lgcy_mmio_hole_en; u8 cs_mask, cs_id = 0; - bool hash_enabled = false; struct addr_ctx ctx; @@ -1192,6 +1219,9 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (get_dram_addr_map(&ctx)) goto out_err; + if (df_ops->get_intlv_mode(&ctx)) + goto out_err; + lgcy_mmio_hole_en = ctx.reg_base_addr & BIT(1); intlv_num_chan = (ctx.reg_base_addr >> 4) & 0xF; intlv_addr_sel = (ctx.reg_base_addr >> 8) & 0x7; @@ -1219,7 +1249,6 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr case 7: intlv_num_chan = 4; break; case 8: intlv_num_chan = 1; - hash_enabled = true; break; default: pr_err("%s: Invalid number of interleaved channels %d.\n", @@ -1320,7 +1349,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr ctx.ret_addr += (BIT_ULL(32) - dram_hole_base); } - if (hash_enabled) { + if (ctx.hash_enabled) { /* Save some parentheses and grab ls-bit at the end. */ hashed_bit = (ctx.ret_addr >> 12) ^ (ctx.ret_addr >> 18) ^ -- 2.33.1.711.g9d530dc002 From 2a9ccf59c9de2d9b7c51bb609d8d9f3f06f9c5c9 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:41 +0000 Subject: [PATCH 10/31] EDAC/amd64: Define function to denormalize address Move the address denormalization into a separate helper function. This will be further refactored in later patches. Add the interleave address bit and the CS ID to the context struct. These values will be used by multiple functions. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 128 ++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 55 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index d4f39e7f2..0920b97f8 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1091,6 +1091,8 @@ struct addr_ctx { u16 nid; u8 inst_id; u8 map_num; + u8 intlv_addr_bit; + u8 cs_id; bool hash_enabled; }; @@ -1188,57 +1190,26 @@ static int get_dram_addr_map(struct addr_ctx *ctx) return 0; } -static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) +static int denormalize_addr(struct addr_ctx *ctx) { - u64 dram_base_addr, dram_limit_addr, dram_hole_base; u32 tmp; u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; - u8 intlv_addr_sel, intlv_addr_bit; - u8 num_intlv_bits, hashed_bit; - u8 lgcy_mmio_hole_en; - u8 cs_mask, cs_id = 0; - - struct addr_ctx ctx; - - memset(&ctx, 0, sizeof(ctx)); - - /* We start from the normalized address */ - ctx.ret_addr = norm_addr; - - ctx.nid = nid; - ctx.inst_id = umc; - - if (set_df_ops(&ctx)) - return -EINVAL; - - if (remove_dram_offset(&ctx)) - goto out_err; - - if (get_dram_addr_map(&ctx)) - goto out_err; - - if (df_ops->get_intlv_mode(&ctx)) - goto out_err; - - lgcy_mmio_hole_en = ctx.reg_base_addr & BIT(1); - intlv_num_chan = (ctx.reg_base_addr >> 4) & 0xF; - intlv_addr_sel = (ctx.reg_base_addr >> 8) & 0x7; - dram_base_addr = (ctx.reg_base_addr & GENMASK_ULL(31, 12)) << 16; + u8 intlv_addr_sel = (ctx->reg_base_addr >> 8) & 0x7; + u8 num_intlv_bits, cs_mask = 0; /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ if (intlv_addr_sel > 3) { pr_err("%s: Invalid interleave address select %d.\n", __func__, intlv_addr_sel); - goto out_err; + return -EINVAL; } - intlv_num_sockets = (ctx.reg_limit_addr >> 8) & 0x1; - intlv_num_dies = (ctx.reg_limit_addr >> 10) & 0x3; - dram_limit_addr = ((ctx.reg_limit_addr & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); + intlv_num_sockets = (ctx->reg_limit_addr >> 8) & 0x1; + intlv_num_dies = (ctx->reg_limit_addr >> 10) & 0x3; - intlv_addr_bit = intlv_addr_sel + 8; + ctx->intlv_addr_bit = intlv_addr_sel + 8; /* Re-use intlv_num_chan by setting it equal to log2(#channels) */ switch (intlv_num_chan) { @@ -1253,7 +1224,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr default: pr_err("%s: Invalid number of interleaved channels %d.\n", __func__, intlv_num_chan); - goto out_err; + return -EINVAL; } num_intlv_bits = intlv_num_chan; @@ -1261,7 +1232,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (intlv_num_dies > 2) { pr_err("%s: Invalid number of interleaved nodes/dies %d.\n", __func__, intlv_num_dies); - goto out_err; + return -EINVAL; } num_intlv_bits += intlv_num_dies; @@ -1273,7 +1244,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (num_intlv_bits > 4) { pr_err("%s: Invalid interleave bits %d.\n", __func__, num_intlv_bits); - goto out_err; + return -EINVAL; } if (num_intlv_bits > 0) { @@ -1286,8 +1257,9 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr * umc/channel# as instance id of the coherent slave * for FICAA. */ - if (amd_df_indirect_read(nid, df_regs[FAB_BLK_INST_INFO_3], umc, &tmp)) - goto out_err; + if (amd_df_indirect_read(ctx->nid, df_regs[FAB_BLK_INST_INFO_3], + ctx->inst_id, &tmp)) + return -EINVAL; cs_fabric_id = (tmp >> 8) & 0xFF; die_id_bit = 0; @@ -1296,14 +1268,15 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (intlv_num_chan) { die_id_bit = intlv_num_chan; cs_mask = (1 << die_id_bit) - 1; - cs_id = cs_fabric_id & cs_mask; + ctx->cs_id = cs_fabric_id & cs_mask; } sock_id_bit = die_id_bit; if (intlv_num_dies || intlv_num_sockets) - if (amd_df_indirect_read(nid, df_regs[SYS_FAB_ID_MASK], umc, &tmp)) - goto out_err; + if (amd_df_indirect_read(ctx->nid, df_regs[SYS_FAB_ID_MASK], + ctx->inst_id, &tmp)) + return -EINVAL; /* If interleaved over more than 1 die. */ if (intlv_num_dies) { @@ -1311,7 +1284,8 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr die_id_shift = (tmp >> 24) & 0xF; die_id_mask = (tmp >> 8) & 0xFF; - cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; + ctx->cs_id |= ((cs_fabric_id & die_id_mask) + >> die_id_shift) << die_id_bit; } /* If interleaved over more than 1 socket. */ @@ -1319,7 +1293,8 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr socket_id_shift = (tmp >> 28) & 0xF; socket_id_mask = (tmp >> 16) & 0xFF; - cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit; + ctx->cs_id |= ((cs_fabric_id & socket_id_mask) + >> socket_id_shift) << sock_id_bit; } /* @@ -1330,12 +1305,55 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr * bits there are. "intlv_addr_bit" tells us how many "Y" bits * there are (where "I" starts). */ - temp_addr_y = ctx.ret_addr & GENMASK_ULL(intlv_addr_bit - 1, 0); - temp_addr_i = (cs_id << intlv_addr_bit); - temp_addr_x = (ctx.ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits; - ctx.ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; + temp_addr_y = ctx->ret_addr & GENMASK_ULL(ctx->intlv_addr_bit - 1, 0); + temp_addr_i = (ctx->cs_id << ctx->intlv_addr_bit); + temp_addr_x = (ctx->ret_addr & GENMASK_ULL(63, ctx->intlv_addr_bit)) + << num_intlv_bits; + ctx->ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; } + return 0; +} + +static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) +{ + u64 dram_base_addr, dram_limit_addr, dram_hole_base; + + u32 tmp; + + u8 hashed_bit; + u8 lgcy_mmio_hole_en; + + struct addr_ctx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + /* We start from the normalized address */ + ctx.ret_addr = norm_addr; + + ctx.nid = nid; + ctx.inst_id = umc; + + if (set_df_ops(&ctx)) + return -EINVAL; + + if (remove_dram_offset(&ctx)) + return -EINVAL; + + if (get_dram_addr_map(&ctx)) + goto out_err; + + if (df_ops->get_intlv_mode(&ctx)) + goto out_err; + + if (denormalize_addr(&ctx)) + goto out_err; + + lgcy_mmio_hole_en = ctx.reg_base_addr & BIT(1); + dram_base_addr = (ctx.reg_base_addr & GENMASK_ULL(31, 12)) << 16; + + dram_limit_addr = ((ctx.reg_limit_addr & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); + /* Add dram base address */ ctx.ret_addr += dram_base_addr; @@ -1355,12 +1373,12 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr (ctx.ret_addr >> 18) ^ (ctx.ret_addr >> 21) ^ (ctx.ret_addr >> 30) ^ - cs_id; + ctx.cs_id; hashed_bit &= BIT(0); - if (hashed_bit != ((ctx.ret_addr >> intlv_addr_bit) & BIT(0))) - ctx.ret_addr ^= BIT(intlv_addr_bit); + if (hashed_bit != ((ctx.ret_addr >> ctx.intlv_addr_bit) & BIT(0))) + ctx.ret_addr ^= BIT(ctx.intlv_addr_bit); } /* Is calculated system address is above DRAM limit address? */ -- 2.33.1.711.g9d530dc002 From 6e00b7035bc5d0e41a0b1aaa38fce1dba6e4d22a Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:42 +0000 Subject: [PATCH 11/31] EDAC/amd64: Define function to add DRAM base and hole Move adding of DRAM base and hole into a separate helper function. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 45 ++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 0920b97f8..57e50d390 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1315,14 +1315,34 @@ static int denormalize_addr(struct addr_ctx *ctx) return 0; } -static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) +static int add_base_and_hole(struct addr_ctx *ctx) { - u64 dram_base_addr, dram_limit_addr, dram_hole_base; + u64 dram_base_addr = (ctx->reg_base_addr & GENMASK_ULL(31, 12)) << 16; - u32 tmp; + /* Add dram base address */ + ctx->ret_addr += dram_base_addr; + + /* If legacy MMIO hole enabled */ + if (ctx->reg_base_addr & BIT(1)) { + u32 dram_hole_base; + + if (amd_df_indirect_read(0, df_regs[DRAM_HOLE_CTL], + DF_BROADCAST, &dram_hole_base)) + return -EINVAL; + + dram_hole_base &= GENMASK(31, 24); + if (ctx->ret_addr >= dram_hole_base) + ctx->ret_addr += (BIT_ULL(32) - dram_hole_base); + } + + return 0; +} + +static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) +{ + u64 dram_limit_addr; u8 hashed_bit; - u8 lgcy_mmio_hole_en; struct addr_ctx ctx; @@ -1349,23 +1369,10 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (denormalize_addr(&ctx)) goto out_err; - lgcy_mmio_hole_en = ctx.reg_base_addr & BIT(1); - dram_base_addr = (ctx.reg_base_addr & GENMASK_ULL(31, 12)) << 16; - dram_limit_addr = ((ctx.reg_limit_addr & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); - /* Add dram base address */ - ctx.ret_addr += dram_base_addr; - - /* If legacy MMIO hole enabled */ - if (lgcy_mmio_hole_en) { - if (amd_df_indirect_read(nid, df_regs[DRAM_HOLE_CTL], umc, &tmp)) - goto out_err; - - dram_hole_base = tmp & GENMASK(31, 24); - if (ctx.ret_addr >= dram_hole_base) - ctx.ret_addr += (BIT_ULL(32) - dram_hole_base); - } + if (add_base_and_hole(&ctx)) + goto out_err; if (ctx.hash_enabled) { /* Save some parentheses and grab ls-bit at the end. */ -- 2.33.1.711.g9d530dc002 From 5d847119c41edef293d43b1d11de682889d35e50 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:43 +0000 Subject: [PATCH 12/31] EDAC/amd64: Define function to dehash address Move the dehashing code into a separate helper function. Define a DF2-specific function for the current code. Specific helper functions will be added for future DF versions. The dehashing code is tied to interleaving mode rather than Data Fabric version, so save the function pointer in the ctx struct. The name include "df2" because this mode only exists on DF2. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 57e50d390..6629137c2 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1093,7 +1093,7 @@ struct addr_ctx { u8 map_num; u8 intlv_addr_bit; u8 cs_id; - bool hash_enabled; + int (*dehash_addr)(struct addr_ctx *ctx); }; struct data_fabric_ops { @@ -1106,13 +1106,29 @@ static u64 get_hi_addr_offset_df2(struct addr_ctx *ctx) return (ctx->reg_dram_offset & GENMASK_ULL(31, 20)) << 8; } +static int dehash_addr_df2(struct addr_ctx *ctx) +{ + u8 hashed_bit = (ctx->ret_addr >> 12) ^ + (ctx->ret_addr >> 18) ^ + (ctx->ret_addr >> 21) ^ + (ctx->ret_addr >> 30) ^ + ctx->cs_id; + + hashed_bit &= BIT(0); + + if (hashed_bit != ((ctx->ret_addr >> ctx->intlv_addr_bit) & BIT(0))) + ctx->ret_addr ^= BIT(ctx->intlv_addr_bit); + + return 0; +} + static int get_intlv_mode_df2(struct addr_ctx *ctx) { ctx->intlv_mode = (ctx->reg_base_addr >> 4) & 0xF; if (ctx->intlv_mode == 8) { ctx->intlv_mode = DF2_HASH_2CH; - ctx->hash_enabled = true; + ctx->dehash_addr = &dehash_addr_df2; } if (ctx->intlv_mode != NONE && @@ -1342,8 +1358,6 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr { u64 dram_limit_addr; - u8 hashed_bit; - struct addr_ctx ctx; memset(&ctx, 0, sizeof(ctx)); @@ -1374,19 +1388,8 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (add_base_and_hole(&ctx)) goto out_err; - if (ctx.hash_enabled) { - /* Save some parentheses and grab ls-bit at the end. */ - hashed_bit = (ctx.ret_addr >> 12) ^ - (ctx.ret_addr >> 18) ^ - (ctx.ret_addr >> 21) ^ - (ctx.ret_addr >> 30) ^ - ctx.cs_id; - - hashed_bit &= BIT(0); - - if (hashed_bit != ((ctx.ret_addr >> ctx.intlv_addr_bit) & BIT(0))) - ctx.ret_addr ^= BIT(ctx.intlv_addr_bit); - } + if (ctx.dehash_addr && ctx.dehash_addr(&ctx)) + goto out_err; /* Is calculated system address is above DRAM limit address? */ if (ctx.ret_addr > dram_limit_addr) -- 2.33.1.711.g9d530dc002 From 0d92dd306cf2a7b9958bf496b0abc8a6a53daf73 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:44 +0000 Subject: [PATCH 13/31] EDAC/amd64: Define function to check DRAM limit address Move the DRAM limit check into a separate helper function. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 6629137c2..b41894523 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1354,10 +1354,20 @@ static int add_base_and_hole(struct addr_ctx *ctx) return 0; } -static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) +static int addr_over_limit(struct addr_ctx *ctx) { - u64 dram_limit_addr; + u64 dram_limit_addr = ((ctx->reg_limit_addr & GENMASK_ULL(31, 12)) << 16) + | GENMASK_ULL(27, 0); + + /* Is calculated system address above DRAM limit address? */ + if (ctx->ret_addr > dram_limit_addr) + return -EINVAL; + return 0; +} + +static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) +{ struct addr_ctx ctx; memset(&ctx, 0, sizeof(ctx)); @@ -1383,16 +1393,13 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (denormalize_addr(&ctx)) goto out_err; - dram_limit_addr = ((ctx.reg_limit_addr & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); - if (add_base_and_hole(&ctx)) goto out_err; if (ctx.dehash_addr && ctx.dehash_addr(&ctx)) goto out_err; - /* Is calculated system address is above DRAM limit address? */ - if (ctx.ret_addr > dram_limit_addr) + if (addr_over_limit(&ctx)) goto out_err; *sys_addr = ctx.ret_addr; -- 2.33.1.711.g9d530dc002 From af3ceaa2b28622a92457ca18024fad65b0b7e5ca Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:45 +0000 Subject: [PATCH 14/31] EDAC/amd64: Remove goto statements ...and just return error codes directly. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index b41894523..23333f199 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1385,28 +1385,25 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr return -EINVAL; if (get_dram_addr_map(&ctx)) - goto out_err; + return -EINVAL; if (df_ops->get_intlv_mode(&ctx)) - goto out_err; + return -EINVAL; if (denormalize_addr(&ctx)) - goto out_err; + return -EINVAL; if (add_base_and_hole(&ctx)) - goto out_err; + return -EINVAL; if (ctx.dehash_addr && ctx.dehash_addr(&ctx)) - goto out_err; + return -EINVAL; if (addr_over_limit(&ctx)) - goto out_err; + return -EINVAL; *sys_addr = ctx.ret_addr; return 0; - -out_err: - return -EINVAL; } static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); -- 2.33.1.711.g9d530dc002 From 58a4c50393bfe3d73b8d5039ca43d31ee3b4bafc Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:46 +0000 Subject: [PATCH 15/31] EDAC/amd64: Simplify function parameters Use a single address parameter for input and result to reduce the number of parameters. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 23333f199..1e5348ace 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1366,14 +1366,14 @@ static int addr_over_limit(struct addr_ctx *ctx) return 0; } -static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) +static int umc_normaddr_to_sysaddr(u64 *addr, u16 nid, u8 umc) { struct addr_ctx ctx; memset(&ctx, 0, sizeof(ctx)); /* We start from the normalized address */ - ctx.ret_addr = norm_addr; + ctx.ret_addr = *addr; ctx.nid = nid; ctx.inst_id = umc; @@ -1402,7 +1402,7 @@ static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr if (addr_over_limit(&ctx)) return -EINVAL; - *sys_addr = ctx.ret_addr; + *addr = ctx.ret_addr; return 0; } @@ -3309,7 +3309,7 @@ static void decode_umc_error(int node_id, struct mce *m) struct mem_ctl_info *mci; struct amd64_pvt *pvt; struct err_info err; - u64 sys_addr; + u64 sys_addr = m->addr; mci = edac_mc_find(node_id); if (!mci) @@ -3340,7 +3340,7 @@ static void decode_umc_error(int node_id, struct mce *m) err.csrow = m->synd & 0x7; - if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) { + if (umc_normaddr_to_sysaddr(&sys_addr, pvt->mc_node_id, err.channel)) { err.err_code = ERR_NORM_ADDR; goto log_error; } -- 2.33.1.711.g9d530dc002 From 8b38acbc71bba944446d55d76d43edfca763fc01 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:47 +0000 Subject: [PATCH 16/31] EDAC/amd64: Define function to get Interleave Address Bit Move code to find the interleave address bit into a separate helper function. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 1e5348ace..329ae0ebb 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1206,27 +1206,35 @@ static int get_dram_addr_map(struct addr_ctx *ctx) return 0; } +static int get_intlv_addr_bit(struct addr_ctx *ctx) +{ + u8 intlv_addr_sel = (ctx->reg_base_addr >> 8) & 0x7; + + /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ + if (intlv_addr_sel > 3) { + pr_debug("Invalid interleave address select %d.\n", intlv_addr_sel); + return -EINVAL; + } + + ctx->intlv_addr_bit = intlv_addr_sel + 8; + + return 0; +} + static int denormalize_addr(struct addr_ctx *ctx) { u32 tmp; u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; - u8 intlv_addr_sel = (ctx->reg_base_addr >> 8) & 0x7; u8 num_intlv_bits, cs_mask = 0; - /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ - if (intlv_addr_sel > 3) { - pr_err("%s: Invalid interleave address select %d.\n", - __func__, intlv_addr_sel); + if (get_intlv_addr_bit(ctx)) return -EINVAL; - } intlv_num_sockets = (ctx->reg_limit_addr >> 8) & 0x1; intlv_num_dies = (ctx->reg_limit_addr >> 10) & 0x3; - ctx->intlv_addr_bit = intlv_addr_sel + 8; - /* Re-use intlv_num_chan by setting it equal to log2(#channels) */ switch (intlv_num_chan) { case 0: intlv_num_chan = 0; break; -- 2.33.1.711.g9d530dc002 From 3b5a0b7e7c63d1539dd7d23187b40327cc4a21b8 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:48 +0000 Subject: [PATCH 17/31] EDAC/amd64: Skip denormalization if no interleaving Denormalization doesn't apply to the "no interleaving" mode, so return early without error in this case. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 329ae0ebb..1fba766df 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1229,6 +1229,10 @@ static int denormalize_addr(struct addr_ctx *ctx) u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; u8 num_intlv_bits, cs_mask = 0; + /* Return early if no interleaving. */ + if (ctx->intlv_mode == NONE) + return 0; + if (get_intlv_addr_bit(ctx)) return -EINVAL; -- 2.33.1.711.g9d530dc002 From e28b073ca687ddd40efb33cb200fafd32d54d079 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:49 +0000 Subject: [PATCH 18/31] EDAC/amd64: Define function to get number of interleaved channels Move number of interleaved channel calculation to a separate helper function. Drop unused cases. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 42 +++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 1fba766df..9027538a4 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1092,6 +1092,7 @@ struct addr_ctx { u8 inst_id; u8 map_num; u8 intlv_addr_bit; + u8 intlv_num_chan; u8 cs_id; int (*dehash_addr)(struct addr_ctx *ctx); }; @@ -1221,12 +1222,29 @@ static int get_intlv_addr_bit(struct addr_ctx *ctx) return 0; } +static void get_intlv_num_chan(struct addr_ctx *ctx) +{ + /* Save the log2(# of channels). */ + switch (ctx->intlv_mode) { + case NONE: + ctx->intlv_num_chan = 0; + break; + case NOHASH_2CH: + case DF2_HASH_2CH: + ctx->intlv_num_chan = 1; + break; + default: + /* Valid interleaving modes where checked earlier. */ + break; + } +} + static int denormalize_addr(struct addr_ctx *ctx) { u32 tmp; u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; - u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; + u8 intlv_num_dies, intlv_num_sockets; u8 num_intlv_bits, cs_mask = 0; /* Return early if no interleaving. */ @@ -1239,23 +1257,9 @@ static int denormalize_addr(struct addr_ctx *ctx) intlv_num_sockets = (ctx->reg_limit_addr >> 8) & 0x1; intlv_num_dies = (ctx->reg_limit_addr >> 10) & 0x3; - /* Re-use intlv_num_chan by setting it equal to log2(#channels) */ - switch (intlv_num_chan) { - case 0: intlv_num_chan = 0; break; - case 1: intlv_num_chan = 1; break; - case 3: intlv_num_chan = 2; break; - case 5: intlv_num_chan = 3; break; - case 7: intlv_num_chan = 4; break; - - case 8: intlv_num_chan = 1; - break; - default: - pr_err("%s: Invalid number of interleaved channels %d.\n", - __func__, intlv_num_chan); - return -EINVAL; - } + get_intlv_num_chan(ctx); - num_intlv_bits = intlv_num_chan; + num_intlv_bits = ctx->intlv_num_chan; if (intlv_num_dies > 2) { pr_err("%s: Invalid number of interleaved nodes/dies %d.\n", @@ -1293,8 +1297,8 @@ static int denormalize_addr(struct addr_ctx *ctx) die_id_bit = 0; /* If interleaved over more than 1 channel: */ - if (intlv_num_chan) { - die_id_bit = intlv_num_chan; + if (ctx->intlv_num_chan) { + die_id_bit = ctx->intlv_num_chan; cs_mask = (1 << die_id_bit) - 1; ctx->cs_id = cs_fabric_id & cs_mask; } -- 2.33.1.711.g9d530dc002 From fe4d7382a91c3c2a3a7881ccc9e7bf85c5b1b268 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:50 +0000 Subject: [PATCH 19/31] EDAC/amd64: Define function to get number of interleaved dies Move parsing of the number of interleaved dies to a separate helper function. This will be expanded for future DF versions. Also, drop an unneeded assert to match the reference code. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 9027538a4..ce54da03e 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1093,6 +1093,7 @@ struct addr_ctx { u8 map_num; u8 intlv_addr_bit; u8 intlv_num_chan; + u8 intlv_num_dies; u8 cs_id; int (*dehash_addr)(struct addr_ctx *ctx); }; @@ -1100,6 +1101,7 @@ struct addr_ctx { struct data_fabric_ops { u64 (*get_hi_addr_offset)(struct addr_ctx *ctx); int (*get_intlv_mode)(struct addr_ctx *ctx); + void (*get_intlv_num_dies)(struct addr_ctx *ctx); }; static u64 get_hi_addr_offset_df2(struct addr_ctx *ctx) @@ -1140,9 +1142,15 @@ static int get_intlv_mode_df2(struct addr_ctx *ctx) return 0; } +static void get_intlv_num_dies_df2(struct addr_ctx *ctx) +{ + ctx->intlv_num_dies = (ctx->reg_limit_addr >> 10) & 0x3; +} + struct data_fabric_ops df2_ops = { .get_hi_addr_offset = &get_hi_addr_offset_df2, .get_intlv_mode = &get_intlv_mode_df2, + .get_intlv_num_dies = &get_intlv_num_dies_df2, }; struct data_fabric_ops *df_ops; @@ -1244,7 +1252,7 @@ static int denormalize_addr(struct addr_ctx *ctx) u32 tmp; u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; - u8 intlv_num_dies, intlv_num_sockets; + u8 intlv_num_sockets; u8 num_intlv_bits, cs_mask = 0; /* Return early if no interleaving. */ @@ -1255,19 +1263,12 @@ static int denormalize_addr(struct addr_ctx *ctx) return -EINVAL; intlv_num_sockets = (ctx->reg_limit_addr >> 8) & 0x1; - intlv_num_dies = (ctx->reg_limit_addr >> 10) & 0x3; get_intlv_num_chan(ctx); + df_ops->get_intlv_num_dies(ctx); num_intlv_bits = ctx->intlv_num_chan; - - if (intlv_num_dies > 2) { - pr_err("%s: Invalid number of interleaved nodes/dies %d.\n", - __func__, intlv_num_dies); - return -EINVAL; - } - - num_intlv_bits += intlv_num_dies; + num_intlv_bits += ctx->intlv_num_dies; /* Add a bit if sockets are interleaved. */ num_intlv_bits += intlv_num_sockets; @@ -1305,14 +1306,14 @@ static int denormalize_addr(struct addr_ctx *ctx) sock_id_bit = die_id_bit; - if (intlv_num_dies || intlv_num_sockets) + if (ctx->intlv_num_dies || intlv_num_sockets) if (amd_df_indirect_read(ctx->nid, df_regs[SYS_FAB_ID_MASK], ctx->inst_id, &tmp)) return -EINVAL; /* If interleaved over more than 1 die. */ - if (intlv_num_dies) { - sock_id_bit = die_id_bit + intlv_num_dies; + if (ctx->intlv_num_dies) { + sock_id_bit = die_id_bit + ctx->intlv_num_dies; die_id_shift = (tmp >> 24) & 0xF; die_id_mask = (tmp >> 8) & 0xFF; -- 2.33.1.711.g9d530dc002 From b83e427395f01a32d885925912c95e98d1d8051f Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:51 +0000 Subject: [PATCH 20/31] EDAC/amd64: Define function to get number of interleaved sockets Move parsing of the number of interleaved sockets to a separate helper function. This will be expanded for future DF versions. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index ce54da03e..2f73ba0d8 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1094,6 +1094,7 @@ struct addr_ctx { u8 intlv_addr_bit; u8 intlv_num_chan; u8 intlv_num_dies; + u8 intlv_num_sockets; u8 cs_id; int (*dehash_addr)(struct addr_ctx *ctx); }; @@ -1102,6 +1103,7 @@ struct data_fabric_ops { u64 (*get_hi_addr_offset)(struct addr_ctx *ctx); int (*get_intlv_mode)(struct addr_ctx *ctx); void (*get_intlv_num_dies)(struct addr_ctx *ctx); + void (*get_intlv_num_sockets)(struct addr_ctx *ctx); }; static u64 get_hi_addr_offset_df2(struct addr_ctx *ctx) @@ -1147,10 +1149,16 @@ static void get_intlv_num_dies_df2(struct addr_ctx *ctx) ctx->intlv_num_dies = (ctx->reg_limit_addr >> 10) & 0x3; } +static void get_intlv_num_sockets_df2(struct addr_ctx *ctx) +{ + ctx->intlv_num_sockets = (ctx->reg_limit_addr >> 8) & 0x1; +} + struct data_fabric_ops df2_ops = { .get_hi_addr_offset = &get_hi_addr_offset_df2, .get_intlv_mode = &get_intlv_mode_df2, .get_intlv_num_dies = &get_intlv_num_dies_df2, + .get_intlv_num_sockets = &get_intlv_num_sockets_df2, }; struct data_fabric_ops *df_ops; @@ -1252,7 +1260,6 @@ static int denormalize_addr(struct addr_ctx *ctx) u32 tmp; u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; - u8 intlv_num_sockets; u8 num_intlv_bits, cs_mask = 0; /* Return early if no interleaving. */ @@ -1262,16 +1269,13 @@ static int denormalize_addr(struct addr_ctx *ctx) if (get_intlv_addr_bit(ctx)) return -EINVAL; - intlv_num_sockets = (ctx->reg_limit_addr >> 8) & 0x1; - get_intlv_num_chan(ctx); df_ops->get_intlv_num_dies(ctx); + df_ops->get_intlv_num_sockets(ctx); num_intlv_bits = ctx->intlv_num_chan; num_intlv_bits += ctx->intlv_num_dies; - - /* Add a bit if sockets are interleaved. */ - num_intlv_bits += intlv_num_sockets; + num_intlv_bits += ctx->intlv_num_sockets; /* Assert num_intlv_bits <= 4 */ if (num_intlv_bits > 4) { @@ -1306,7 +1310,7 @@ static int denormalize_addr(struct addr_ctx *ctx) sock_id_bit = die_id_bit; - if (ctx->intlv_num_dies || intlv_num_sockets) + if (ctx->intlv_num_dies || ctx->intlv_num_sockets) if (amd_df_indirect_read(ctx->nid, df_regs[SYS_FAB_ID_MASK], ctx->inst_id, &tmp)) return -EINVAL; @@ -1322,7 +1326,7 @@ static int denormalize_addr(struct addr_ctx *ctx) } /* If interleaved over more than 1 socket. */ - if (intlv_num_sockets) { + if (ctx->intlv_num_sockets) { socket_id_shift = (tmp >> 28) & 0xF; socket_id_mask = (tmp >> 16) & 0xFF; -- 2.33.1.711.g9d530dc002 From d26239223d365436fbf001f9f880a74e16236ea7 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:52 +0000 Subject: [PATCH 21/31] EDAC/amd64: Remove unnecessary assert It was removed in the reference code, so remove it here. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 2f73ba0d8..6d94843d7 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1277,13 +1277,6 @@ static int denormalize_addr(struct addr_ctx *ctx) num_intlv_bits += ctx->intlv_num_dies; num_intlv_bits += ctx->intlv_num_sockets; - /* Assert num_intlv_bits <= 4 */ - if (num_intlv_bits > 4) { - pr_err("%s: Invalid interleave bits %d.\n", - __func__, num_intlv_bits); - return -EINVAL; - } - if (num_intlv_bits > 0) { u64 temp_addr_x, temp_addr_i, temp_addr_y; u8 die_id_bit, sock_id_bit, cs_fabric_id; -- 2.33.1.711.g9d530dc002 From 4f7103cfa0d0d57bc1c75257620ddd5c73655d5b Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:53 +0000 Subject: [PATCH 22/31] EDAC/amd64: Define function to make space for CS ID Move code that makes a gap for the CS ID into a separate helper function. The exact bits to use vary based on interleaving mode. New interleaving modes in future DF versions will be added as new cases. Also, introduce a helper function that does the bit manipulation to make the gap. The current version of this function is "simple", and future interleaving modes may reuse this or use a more advanced function. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 6d94843d7..39b6a6de7 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1097,6 +1097,7 @@ struct addr_ctx { u8 intlv_num_sockets; u8 cs_id; int (*dehash_addr)(struct addr_ctx *ctx); + void (*make_space_for_cs_id)(struct addr_ctx *ctx); }; struct data_fabric_ops { @@ -1106,6 +1107,29 @@ struct data_fabric_ops { void (*get_intlv_num_sockets)(struct addr_ctx *ctx); }; +static void expand_bits(u8 start_bit, u8 num_bits, u64 *value) +{ + u64 temp1, temp2; + + if (start_bit == 0) { + *value <<= num_bits; + return; + } + + temp1 = *value & GENMASK_ULL(start_bit - 1, 0); + temp2 = (*value & GENMASK_ULL(63, start_bit)) << num_bits; + *value = temp1 | temp2; +} + +static void make_space_for_cs_id_simple(struct addr_ctx *ctx) +{ + u8 num_intlv_bits = ctx->intlv_num_chan; + + num_intlv_bits += ctx->intlv_num_dies; + num_intlv_bits += ctx->intlv_num_sockets; + expand_bits(ctx->intlv_addr_bit, num_intlv_bits, &ctx->ret_addr); +} + static u64 get_hi_addr_offset_df2(struct addr_ctx *ctx) { return (ctx->reg_dram_offset & GENMASK_ULL(31, 20)) << 8; @@ -1136,6 +1160,8 @@ static int get_intlv_mode_df2(struct addr_ctx *ctx) ctx->dehash_addr = &dehash_addr_df2; } + ctx->make_space_for_cs_id = &make_space_for_cs_id_simple; + if (ctx->intlv_mode != NONE && ctx->intlv_mode != NOHASH_2CH && ctx->intlv_mode != DF2_HASH_2CH) @@ -1273,13 +1299,11 @@ static int denormalize_addr(struct addr_ctx *ctx) df_ops->get_intlv_num_dies(ctx); df_ops->get_intlv_num_sockets(ctx); - num_intlv_bits = ctx->intlv_num_chan; - num_intlv_bits += ctx->intlv_num_dies; - num_intlv_bits += ctx->intlv_num_sockets; + ctx->make_space_for_cs_id(ctx); if (num_intlv_bits > 0) { - u64 temp_addr_x, temp_addr_i, temp_addr_y; u8 die_id_bit, sock_id_bit, cs_fabric_id; + u64 temp_addr_i; /* * Read FabricBlockInstanceInformation3_CS[BlockFabricID]. @@ -1335,11 +1359,8 @@ static int denormalize_addr(struct addr_ctx *ctx) * bits there are. "intlv_addr_bit" tells us how many "Y" bits * there are (where "I" starts). */ - temp_addr_y = ctx->ret_addr & GENMASK_ULL(ctx->intlv_addr_bit - 1, 0); temp_addr_i = (ctx->cs_id << ctx->intlv_addr_bit); - temp_addr_x = (ctx->ret_addr & GENMASK_ULL(63, ctx->intlv_addr_bit)) - << num_intlv_bits; - ctx->ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; + ctx->ret_addr |= temp_addr_i; } return 0; -- 2.33.1.711.g9d530dc002 From 1ea6036ae064927eb56eff902593c638a1ff3fa3 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:54 +0000 Subject: [PATCH 23/31] EDAC/amd64: Define function to calculate CS ID Move code used to calculate the CS ID into a separate helper function. Drop redundant code comment about reading DF register. The "num_intlv_bits" variable is left uninitialized as it will be removed in a later patch. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 98 +++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 39b6a6de7..6056d4499 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1281,12 +1281,54 @@ static void get_intlv_num_chan(struct addr_ctx *ctx) } } -static int denormalize_addr(struct addr_ctx *ctx) +static int calculate_cs_id(struct addr_ctx *ctx) { + u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; + u8 die_id_bit, sock_id_bit, cs_fabric_id, cs_mask = 0; u32 tmp; - u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; - u8 num_intlv_bits, cs_mask = 0; + if (amd_df_indirect_read(ctx->nid, df_regs[FAB_BLK_INST_INFO_3], ctx->inst_id, &tmp)) + return -EINVAL; + + cs_fabric_id = (tmp >> 8) & 0xFF; + die_id_bit = 0; + + /* If interleaved over more than 1 channel: */ + if (ctx->intlv_num_chan) { + die_id_bit = ctx->intlv_num_chan; + cs_mask = (1 << die_id_bit) - 1; + ctx->cs_id = cs_fabric_id & cs_mask; + } + + sock_id_bit = die_id_bit; + + if (ctx->intlv_num_dies || ctx->intlv_num_sockets) + if (amd_df_indirect_read(ctx->nid, df_regs[SYS_FAB_ID_MASK], ctx->inst_id, &tmp)) + return -EINVAL; + + /* If interleaved over more than 1 die: */ + if (ctx->intlv_num_dies) { + sock_id_bit = die_id_bit + ctx->intlv_num_dies; + die_id_shift = (tmp >> 24) & 0xF; + die_id_mask = (tmp >> 8) & 0xFF; + + ctx->cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; + } + + /* If interleaved over more than 1 socket: */ + if (ctx->intlv_num_sockets) { + socket_id_shift = (tmp >> 28) & 0xF; + socket_id_mask = (tmp >> 16) & 0xFF; + + ctx->cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit; + } + + return 0; +} + +static int denormalize_addr(struct addr_ctx *ctx) +{ + u8 num_intlv_bits; /* Return early if no interleaving. */ if (ctx->intlv_mode == NONE) @@ -1301,56 +1343,12 @@ static int denormalize_addr(struct addr_ctx *ctx) ctx->make_space_for_cs_id(ctx); + if (calculate_cs_id(ctx)) + return -EINVAL; + if (num_intlv_bits > 0) { - u8 die_id_bit, sock_id_bit, cs_fabric_id; u64 temp_addr_i; - /* - * Read FabricBlockInstanceInformation3_CS[BlockFabricID]. - * This is the fabric id for this coherent slave. Use - * umc/channel# as instance id of the coherent slave - * for FICAA. - */ - if (amd_df_indirect_read(ctx->nid, df_regs[FAB_BLK_INST_INFO_3], - ctx->inst_id, &tmp)) - return -EINVAL; - - cs_fabric_id = (tmp >> 8) & 0xFF; - die_id_bit = 0; - - /* If interleaved over more than 1 channel: */ - if (ctx->intlv_num_chan) { - die_id_bit = ctx->intlv_num_chan; - cs_mask = (1 << die_id_bit) - 1; - ctx->cs_id = cs_fabric_id & cs_mask; - } - - sock_id_bit = die_id_bit; - - if (ctx->intlv_num_dies || ctx->intlv_num_sockets) - if (amd_df_indirect_read(ctx->nid, df_regs[SYS_FAB_ID_MASK], - ctx->inst_id, &tmp)) - return -EINVAL; - - /* If interleaved over more than 1 die. */ - if (ctx->intlv_num_dies) { - sock_id_bit = die_id_bit + ctx->intlv_num_dies; - die_id_shift = (tmp >> 24) & 0xF; - die_id_mask = (tmp >> 8) & 0xFF; - - ctx->cs_id |= ((cs_fabric_id & die_id_mask) - >> die_id_shift) << die_id_bit; - } - - /* If interleaved over more than 1 socket. */ - if (ctx->intlv_num_sockets) { - socket_id_shift = (tmp >> 28) & 0xF; - socket_id_mask = (tmp >> 16) & 0xFF; - - ctx->cs_id |= ((cs_fabric_id & socket_id_mask) - >> socket_id_shift) << sock_id_bit; - } - /* * The pre-interleaved address consists of XXXXXXIIIYYYYY * where III is the ID for this CS, and XXXXXXYYYYY are the -- 2.33.1.711.g9d530dc002 From a7d12ef2a31cc6f70442a3041b72bb809607207c Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:55 +0000 Subject: [PATCH 24/31] EDAC/amd64: Define function to insert CS ID into address Move the code that inserts the CS ID into the address into a separate helper function. This will be expanded for future DF versions. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 6056d4499..cf8d663ca 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1098,6 +1098,7 @@ struct addr_ctx { u8 cs_id; int (*dehash_addr)(struct addr_ctx *ctx); void (*make_space_for_cs_id)(struct addr_ctx *ctx); + void (*insert_cs_id)(struct addr_ctx *ctx); }; struct data_fabric_ops { @@ -1130,6 +1131,11 @@ static void make_space_for_cs_id_simple(struct addr_ctx *ctx) expand_bits(ctx->intlv_addr_bit, num_intlv_bits, &ctx->ret_addr); } +static void insert_cs_id_simple(struct addr_ctx *ctx) +{ + ctx->ret_addr |= (ctx->cs_id << ctx->intlv_addr_bit); +} + static u64 get_hi_addr_offset_df2(struct addr_ctx *ctx) { return (ctx->reg_dram_offset & GENMASK_ULL(31, 20)) << 8; @@ -1161,6 +1167,7 @@ static int get_intlv_mode_df2(struct addr_ctx *ctx) } ctx->make_space_for_cs_id = &make_space_for_cs_id_simple; + ctx->insert_cs_id = &insert_cs_id_simple; if (ctx->intlv_mode != NONE && ctx->intlv_mode != NOHASH_2CH && @@ -1328,8 +1335,6 @@ static int calculate_cs_id(struct addr_ctx *ctx) static int denormalize_addr(struct addr_ctx *ctx) { - u8 num_intlv_bits; - /* Return early if no interleaving. */ if (ctx->intlv_mode == NONE) return 0; @@ -1346,20 +1351,7 @@ static int denormalize_addr(struct addr_ctx *ctx) if (calculate_cs_id(ctx)) return -EINVAL; - if (num_intlv_bits > 0) { - u64 temp_addr_i; - - /* - * The pre-interleaved address consists of XXXXXXIIIYYYYY - * where III is the ID for this CS, and XXXXXXYYYYY are the - * address bits from the post-interleaved address. - * "num_intlv_bits" has been calculated to tell us how many "I" - * bits there are. "intlv_addr_bit" tells us how many "Y" bits - * there are (where "I" starts). - */ - temp_addr_i = (ctx->cs_id << ctx->intlv_addr_bit); - ctx->ret_addr |= temp_addr_i; - } + ctx->insert_cs_id(ctx); return 0; } -- 2.33.1.711.g9d530dc002 From d8ba05a5145b390dd6d7df406816747e6defa56c Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:56 +0000 Subject: [PATCH 25/31] EDAC/amd64: Define function to get CS Fabric ID Move code that gets the CS Fabric ID into a separate helper function. This will be expanded for future DF versions. The bitfield used for this value may be larger than the 8 bits currently used. So make it a u16 type which is large enough to hold all known sizes of this bitfield across DF versions. Also, call this function early as future DF versions may need the value early. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index cf8d663ca..8bffcb920 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1088,6 +1088,7 @@ struct addr_ctx { u32 reg_dram_offset; u32 reg_base_addr; u32 reg_limit_addr; + u16 cs_fabric_id; u16 nid; u8 inst_id; u8 map_num; @@ -1104,6 +1105,7 @@ struct addr_ctx { struct data_fabric_ops { u64 (*get_hi_addr_offset)(struct addr_ctx *ctx); int (*get_intlv_mode)(struct addr_ctx *ctx); + int (*get_cs_fabric_id)(struct addr_ctx *ctx); void (*get_intlv_num_dies)(struct addr_ctx *ctx); void (*get_intlv_num_sockets)(struct addr_ctx *ctx); }; @@ -1187,11 +1189,24 @@ static void get_intlv_num_sockets_df2(struct addr_ctx *ctx) ctx->intlv_num_sockets = (ctx->reg_limit_addr >> 8) & 0x1; } +static int get_cs_fabric_id_df2(struct addr_ctx *ctx) +{ + u32 tmp; + + if (amd_df_indirect_read(ctx->nid, df_regs[FAB_BLK_INST_INFO_3], ctx->inst_id, &tmp)) + return -EINVAL; + + ctx->cs_fabric_id = (tmp >> 8) & 0xFF; + + return 0; +} + struct data_fabric_ops df2_ops = { .get_hi_addr_offset = &get_hi_addr_offset_df2, .get_intlv_mode = &get_intlv_mode_df2, .get_intlv_num_dies = &get_intlv_num_dies_df2, .get_intlv_num_sockets = &get_intlv_num_sockets_df2, + .get_cs_fabric_id = &get_cs_fabric_id_df2, }; struct data_fabric_ops *df_ops; @@ -1291,20 +1306,16 @@ static void get_intlv_num_chan(struct addr_ctx *ctx) static int calculate_cs_id(struct addr_ctx *ctx) { u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; - u8 die_id_bit, sock_id_bit, cs_fabric_id, cs_mask = 0; + u8 die_id_bit, sock_id_bit, cs_mask = 0; u32 tmp; - if (amd_df_indirect_read(ctx->nid, df_regs[FAB_BLK_INST_INFO_3], ctx->inst_id, &tmp)) - return -EINVAL; - - cs_fabric_id = (tmp >> 8) & 0xFF; die_id_bit = 0; /* If interleaved over more than 1 channel: */ if (ctx->intlv_num_chan) { die_id_bit = ctx->intlv_num_chan; cs_mask = (1 << die_id_bit) - 1; - ctx->cs_id = cs_fabric_id & cs_mask; + ctx->cs_id = ctx->cs_fabric_id & cs_mask; } sock_id_bit = die_id_bit; @@ -1319,7 +1330,7 @@ static int calculate_cs_id(struct addr_ctx *ctx) die_id_shift = (tmp >> 24) & 0xF; die_id_mask = (tmp >> 8) & 0xFF; - ctx->cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; + ctx->cs_id |= ((ctx->cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; } /* If interleaved over more than 1 socket: */ @@ -1327,7 +1338,8 @@ static int calculate_cs_id(struct addr_ctx *ctx) socket_id_shift = (tmp >> 28) & 0xF; socket_id_mask = (tmp >> 16) & 0xFF; - ctx->cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit; + ctx->cs_id |= ((ctx->cs_fabric_id & socket_id_mask) + >> socket_id_shift) << sock_id_bit; } return 0; @@ -1406,6 +1418,9 @@ static int umc_normaddr_to_sysaddr(u64 *addr, u16 nid, u8 umc) if (set_df_ops(&ctx)) return -EINVAL; + if (df_ops->get_cs_fabric_id(&ctx)) + return -EINVAL; + if (remove_dram_offset(&ctx)) return -EINVAL; -- 2.33.1.711.g9d530dc002 From 2231bc4d6cb6e018656c0fdec64bf9f03acb100e Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:57 +0000 Subject: [PATCH 26/31] EDAC/amd64: Define function to find shift and mask values Move code to find the shift and mask values used in die and socket interleaving into separate helper functions. These will be expanded for future DF versions. Make the die_id_mask and socket_id_mask values u16 type to accommodate larger bitfields in future DF versions. Also, move reading of the System Fabric ID Mask register into set_df_ops(). This will be expanded for future DF versions and will be used to determine DF version levels. And call this function early since future DF versions may need these values early. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 61 +++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 8bffcb920..9198d7870 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1088,7 +1088,10 @@ struct addr_ctx { u32 reg_dram_offset; u32 reg_base_addr; u32 reg_limit_addr; + u32 reg_fab_id_mask0; u16 cs_fabric_id; + u16 die_id_mask; + u16 socket_id_mask; u16 nid; u8 inst_id; u8 map_num; @@ -1104,8 +1107,11 @@ struct addr_ctx { struct data_fabric_ops { u64 (*get_hi_addr_offset)(struct addr_ctx *ctx); + u8 (*get_die_id_shift)(struct addr_ctx *ctx); + u8 (*get_socket_id_shift)(struct addr_ctx *ctx); int (*get_intlv_mode)(struct addr_ctx *ctx); int (*get_cs_fabric_id)(struct addr_ctx *ctx); + int (*get_masks)(struct addr_ctx *ctx); void (*get_intlv_num_dies)(struct addr_ctx *ctx); void (*get_intlv_num_sockets)(struct addr_ctx *ctx); }; @@ -1201,18 +1207,43 @@ static int get_cs_fabric_id_df2(struct addr_ctx *ctx) return 0; } +static int get_masks_df2(struct addr_ctx *ctx) +{ + ctx->die_id_mask = (ctx->reg_fab_id_mask0 >> 8) & 0xFF; + ctx->socket_id_mask = (ctx->reg_fab_id_mask0 >> 16) & 0xFF; + + return 0; +} + +static u8 get_die_id_shift_df2(struct addr_ctx *ctx) +{ + return (ctx->reg_fab_id_mask0 >> 24) & 0xF; +} + +static u8 get_socket_id_shift_df2(struct addr_ctx *ctx) +{ + return (ctx->reg_fab_id_mask0 >> 28) & 0xF; +} + struct data_fabric_ops df2_ops = { .get_hi_addr_offset = &get_hi_addr_offset_df2, .get_intlv_mode = &get_intlv_mode_df2, .get_intlv_num_dies = &get_intlv_num_dies_df2, .get_intlv_num_sockets = &get_intlv_num_sockets_df2, .get_cs_fabric_id = &get_cs_fabric_id_df2, + .get_masks = &get_masks_df2, + .get_die_id_shift = &get_die_id_shift_df2, + .get_socket_id_shift = &get_socket_id_shift_df2, }; struct data_fabric_ops *df_ops; static int set_df_ops(struct addr_ctx *ctx) { + if (amd_df_indirect_read(0, df_regs[SYS_FAB_ID_MASK], + DF_BROADCAST, &ctx->reg_fab_id_mask0)) + return -EINVAL; + df_ops = &df2_ops; return 0; @@ -1305,11 +1336,7 @@ static void get_intlv_num_chan(struct addr_ctx *ctx) static int calculate_cs_id(struct addr_ctx *ctx) { - u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; - u8 die_id_bit, sock_id_bit, cs_mask = 0; - u32 tmp; - - die_id_bit = 0; + u8 die_id_bit = 0, sock_id_bit, cs_mask = 0; /* If interleaved over more than 1 channel: */ if (ctx->intlv_num_chan) { @@ -1318,27 +1345,26 @@ static int calculate_cs_id(struct addr_ctx *ctx) ctx->cs_id = ctx->cs_fabric_id & cs_mask; } - sock_id_bit = die_id_bit; + /* Return early if no die interleaving and no socket interleaving. */ + if (!(ctx->intlv_num_dies || ctx->intlv_num_sockets)) + return 0; - if (ctx->intlv_num_dies || ctx->intlv_num_sockets) - if (amd_df_indirect_read(ctx->nid, df_regs[SYS_FAB_ID_MASK], ctx->inst_id, &tmp)) - return -EINVAL; + sock_id_bit = die_id_bit; /* If interleaved over more than 1 die: */ if (ctx->intlv_num_dies) { - sock_id_bit = die_id_bit + ctx->intlv_num_dies; - die_id_shift = (tmp >> 24) & 0xF; - die_id_mask = (tmp >> 8) & 0xFF; + u8 die_id_shift = df_ops->get_die_id_shift(ctx); - ctx->cs_id |= ((ctx->cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; + sock_id_bit = die_id_bit + ctx->intlv_num_dies; + ctx->cs_id |= ((ctx->cs_fabric_id & ctx->die_id_mask) + >> die_id_shift) << die_id_bit; } /* If interleaved over more than 1 socket: */ if (ctx->intlv_num_sockets) { - socket_id_shift = (tmp >> 28) & 0xF; - socket_id_mask = (tmp >> 16) & 0xFF; + u8 socket_id_shift = df_ops->get_socket_id_shift(ctx); - ctx->cs_id |= ((ctx->cs_fabric_id & socket_id_mask) + ctx->cs_id |= ((ctx->cs_fabric_id & ctx->socket_id_mask) >> socket_id_shift) << sock_id_bit; } @@ -1418,6 +1444,9 @@ static int umc_normaddr_to_sysaddr(u64 *addr, u16 nid, u8 umc) if (set_df_ops(&ctx)) return -EINVAL; + if (df_ops->get_masks(&ctx)) + return -EINVAL; + if (df_ops->get_cs_fabric_id(&ctx)) return -EINVAL; -- 2.33.1.711.g9d530dc002 From 2b3f2f60c2832408c7eb158dcfc66ba5e3cb319d Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:58 +0000 Subject: [PATCH 27/31] EDAC/amd64: Update CS ID calculation to match reference code Redo the current CS ID calculations to match the reference code. Helper functions are introduced that will be expanded for future DF versions. Use u16 type for dst_fabric_id and component_id_mask values to accommodate larger bitfields in future DF versions. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 52 ++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 9198d7870..b8e1eb956 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1107,6 +1107,8 @@ struct addr_ctx { struct data_fabric_ops { u64 (*get_hi_addr_offset)(struct addr_ctx *ctx); + u16 (*get_dst_fabric_id)(struct addr_ctx *ctx); + u16 (*get_component_id_mask)(struct addr_ctx *ctx); u8 (*get_die_id_shift)(struct addr_ctx *ctx); u8 (*get_socket_id_shift)(struct addr_ctx *ctx); int (*get_intlv_mode)(struct addr_ctx *ctx); @@ -1225,6 +1227,16 @@ static u8 get_socket_id_shift_df2(struct addr_ctx *ctx) return (ctx->reg_fab_id_mask0 >> 28) & 0xF; } +static u16 get_dst_fabric_id_df2(struct addr_ctx *ctx) +{ + return ctx->reg_limit_addr & 0xFF; +} + +static u16 get_component_id_mask_df2(struct addr_ctx *ctx) +{ + return (~(ctx->socket_id_mask | ctx->die_id_mask)) & 0xFF; +} + struct data_fabric_ops df2_ops = { .get_hi_addr_offset = &get_hi_addr_offset_df2, .get_intlv_mode = &get_intlv_mode_df2, @@ -1234,6 +1246,8 @@ struct data_fabric_ops df2_ops = { .get_masks = &get_masks_df2, .get_die_id_shift = &get_die_id_shift_df2, .get_socket_id_shift = &get_socket_id_shift_df2, + .get_dst_fabric_id = &get_dst_fabric_id_df2, + .get_component_id_mask = &get_component_id_mask_df2, }; struct data_fabric_ops *df_ops; @@ -1334,38 +1348,42 @@ static void get_intlv_num_chan(struct addr_ctx *ctx) } } -static int calculate_cs_id(struct addr_ctx *ctx) +static u8 calc_level_bits(u8 id, u8 level_mask, u8 shift, u8 mask, u8 num_bits) { - u8 die_id_bit = 0, sock_id_bit, cs_mask = 0; + return (((id & level_mask) >> shift) & mask) << num_bits; +} - /* If interleaved over more than 1 channel: */ - if (ctx->intlv_num_chan) { - die_id_bit = ctx->intlv_num_chan; - cs_mask = (1 << die_id_bit) - 1; - ctx->cs_id = ctx->cs_fabric_id & cs_mask; - } +static int calculate_cs_id(struct addr_ctx *ctx) +{ + u16 dst_fabric_id = df_ops->get_dst_fabric_id(ctx); + u16 mask, num_intlv_bits = ctx->intlv_num_chan; - /* Return early if no die interleaving and no socket interleaving. */ - if (!(ctx->intlv_num_dies || ctx->intlv_num_sockets)) - return 0; + mask = df_ops->get_component_id_mask(ctx); + ctx->cs_id = (ctx->cs_fabric_id & mask) - (dst_fabric_id & mask); - sock_id_bit = die_id_bit; + mask = (1 << num_intlv_bits) - 1; + ctx->cs_id &= mask; /* If interleaved over more than 1 die: */ if (ctx->intlv_num_dies) { u8 die_id_shift = df_ops->get_die_id_shift(ctx); - sock_id_bit = die_id_bit + ctx->intlv_num_dies; - ctx->cs_id |= ((ctx->cs_fabric_id & ctx->die_id_mask) - >> die_id_shift) << die_id_bit; + mask = (1 << ctx->intlv_num_dies) - 1; + + ctx->cs_id |= calc_level_bits(ctx->cs_fabric_id, ctx->die_id_mask, + die_id_shift, mask, num_intlv_bits); + + num_intlv_bits += ctx->intlv_num_dies; } /* If interleaved over more than 1 socket: */ if (ctx->intlv_num_sockets) { u8 socket_id_shift = df_ops->get_socket_id_shift(ctx); - ctx->cs_id |= ((ctx->cs_fabric_id & ctx->socket_id_mask) - >> socket_id_shift) << sock_id_bit; + mask = (1 << ctx->intlv_num_sockets) - 1; + + ctx->cs_id |= calc_level_bits(ctx->cs_fabric_id, ctx->socket_id_mask, + socket_id_shift, mask, num_intlv_bits); } return 0; -- 2.33.1.711.g9d530dc002 From 61ca52ac68afa9a4e6ce807d3ee07c2ff9900ce7 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:19:59 +0000 Subject: [PATCH 28/31] EDAC/amd64: Match hash function to reference code The reference code for DF2 hashing was changed to XOR the interleave address bit rather than the CS ID. Match that here. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index b8e1eb956..054c3698b 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1157,7 +1157,7 @@ static int dehash_addr_df2(struct addr_ctx *ctx) (ctx->ret_addr >> 18) ^ (ctx->ret_addr >> 21) ^ (ctx->ret_addr >> 30) ^ - ctx->cs_id; + (ctx->ret_addr >> ctx->intlv_addr_bit); hashed_bit &= BIT(0); -- 2.33.1.711.g9d530dc002 From d3b746444d59d36dc6a7554d81ce1698c1eebaa9 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:20:00 +0000 Subject: [PATCH 29/31] EDAC/amd64: Define helper function to get interleave address select bit ...this will be expanded for future Data Fabric versions. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 054c3698b..4f9c77344 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1111,6 +1111,7 @@ struct data_fabric_ops { u16 (*get_component_id_mask)(struct addr_ctx *ctx); u8 (*get_die_id_shift)(struct addr_ctx *ctx); u8 (*get_socket_id_shift)(struct addr_ctx *ctx); + u8 (*get_intlv_addr_sel)(struct addr_ctx *ctx); int (*get_intlv_mode)(struct addr_ctx *ctx); int (*get_cs_fabric_id)(struct addr_ctx *ctx); int (*get_masks)(struct addr_ctx *ctx); @@ -1187,6 +1188,11 @@ static int get_intlv_mode_df2(struct addr_ctx *ctx) return 0; } +static u8 get_intlv_addr_sel_df2(struct addr_ctx *ctx) +{ + return (ctx->reg_base_addr >> 8) & 0x7; +} + static void get_intlv_num_dies_df2(struct addr_ctx *ctx) { ctx->intlv_num_dies = (ctx->reg_limit_addr >> 10) & 0x3; @@ -1240,6 +1246,7 @@ static u16 get_component_id_mask_df2(struct addr_ctx *ctx) struct data_fabric_ops df2_ops = { .get_hi_addr_offset = &get_hi_addr_offset_df2, .get_intlv_mode = &get_intlv_mode_df2, + .get_intlv_addr_sel = &get_intlv_addr_sel_df2, .get_intlv_num_dies = &get_intlv_num_dies_df2, .get_intlv_num_sockets = &get_intlv_num_sockets_df2, .get_cs_fabric_id = &get_cs_fabric_id_df2, @@ -1318,7 +1325,7 @@ static int get_dram_addr_map(struct addr_ctx *ctx) static int get_intlv_addr_bit(struct addr_ctx *ctx) { - u8 intlv_addr_sel = (ctx->reg_base_addr >> 8) & 0x7; + u8 intlv_addr_sel = df_ops->get_intlv_addr_sel(ctx); /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ if (intlv_addr_sel > 3) { -- 2.33.1.711.g9d530dc002 From f8ecf9744b7cf117e17afbc928409fcf4ff7a010 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:20:01 +0000 Subject: [PATCH 30/31] EDAC/amd64: Add support for address translation on DF3 systems DF3-based systems (Rome and later) support new interleaving modes and a number of bit fields have changed or moved entirely. Add support for these new modes and fields. Refactoring should be minimal due to earlier changes, and most updates will be additions. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 189 +++++++++++++++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 2 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 4f9c77344..80635e07d 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1054,9 +1054,11 @@ enum df_reg_names { DRAM_BASE_ADDR, DRAM_LIMIT_ADDR, DRAM_OFFSET, + DF_GLOBAL_CTL, /* Function 1 */ SYS_FAB_ID_MASK, + SYS_FAB_ID_MASK_1, }; static struct df_reg df_regs[] = { @@ -1070,14 +1072,23 @@ static struct df_reg df_regs[] = { [DRAM_LIMIT_ADDR] = {0, 0x114}, /* D18F0x1B4 (DramOffset) */ [DRAM_OFFSET] = {0, 0x1B4}, + /* D18F0x3F8 (DfGlobalCtrl) */ + [DF_GLOBAL_CTL] = {0, 0x3F8}, /* D18F1x208 (SystemFabricIdMask) */ [SYS_FAB_ID_MASK] = {1, 0x208}, + /* D18F1x20C (SystemFabricIdMask1) */ + [SYS_FAB_ID_MASK_1] = {1, 0x20C}, }; /* These are mapped 1:1 to the hardware values. Special cases are set at > 0x20. */ enum intlv_modes { NONE = 0x00, NOHASH_2CH = 0x01, + NOHASH_4CH = 0x03, + NOHASH_8CH = 0x05, + HASH_COD4_2CH = 0x0C, + HASH_COD2_4CH = 0x0D, + HASH_COD1_8CH = 0x0E, DF2_HASH_2CH = 0x21, }; @@ -1089,6 +1100,7 @@ struct addr_ctx { u32 reg_base_addr; u32 reg_limit_addr; u32 reg_fab_id_mask0; + u32 reg_fab_id_mask1; u16 cs_fabric_id; u16 die_id_mask; u16 socket_id_mask; @@ -1100,6 +1112,7 @@ struct addr_ctx { u8 intlv_num_dies; u8 intlv_num_sockets; u8 cs_id; + u8 node_id_shift; int (*dehash_addr)(struct addr_ctx *ctx); void (*make_space_for_cs_id)(struct addr_ctx *ctx); void (*insert_cs_id)(struct addr_ctx *ctx); @@ -1257,6 +1270,164 @@ struct data_fabric_ops df2_ops = { .get_component_id_mask = &get_component_id_mask_df2, }; +static u64 get_hi_addr_offset_df3(struct addr_ctx *ctx) +{ + return (ctx->reg_dram_offset & GENMASK_ULL(31, 12)) << 16; +} + +static void make_space_for_cs_id_cod_hash(struct addr_ctx *ctx) +{ + u8 num_intlv_bits = ctx->intlv_num_chan; + + num_intlv_bits += ctx->intlv_num_sockets; + expand_bits(ctx->intlv_addr_bit, 1, &ctx->ret_addr); + if (num_intlv_bits > 1) + expand_bits(12, num_intlv_bits - 1, &ctx->ret_addr); +} + +static void insert_cs_id_cod_hash(struct addr_ctx *ctx) +{ + ctx->ret_addr |= ((ctx->cs_id & 0x1) << ctx->intlv_addr_bit); + ctx->ret_addr |= ((ctx->cs_id & 0xE) << 11); +} + +static int dehash_addr_df3(struct addr_ctx *ctx) +{ + u8 hashed_bit, intlv_ctl_64k, intlv_ctl_2M, intlv_ctl_1G; + u32 tmp; + + if (amd_df_indirect_read(0, df_regs[DF_GLOBAL_CTL], DF_BROADCAST, &tmp)) + return -EINVAL; + + intlv_ctl_64k = !!((tmp >> 20) & 0x1); + intlv_ctl_2M = !!((tmp >> 21) & 0x1); + intlv_ctl_1G = !!((tmp >> 22) & 0x1); + + hashed_bit = (ctx->ret_addr >> 14) ^ + ((ctx->ret_addr >> 18) & intlv_ctl_64k) ^ + ((ctx->ret_addr >> 23) & intlv_ctl_2M) ^ + ((ctx->ret_addr >> 32) & intlv_ctl_1G) ^ + (ctx->ret_addr >> ctx->intlv_addr_bit); + + hashed_bit &= BIT(0); + + if (hashed_bit != ((ctx->ret_addr >> ctx->intlv_addr_bit) & BIT(0))) + ctx->ret_addr ^= BIT(ctx->intlv_addr_bit); + + if (ctx->intlv_mode != HASH_COD2_4CH && + ctx->intlv_mode != HASH_COD1_8CH) + return 0; + + hashed_bit = (ctx->ret_addr >> 12) ^ + ((ctx->ret_addr >> 16) & intlv_ctl_64k) ^ + ((ctx->ret_addr >> 21) & intlv_ctl_2M) ^ + ((ctx->ret_addr >> 30) & intlv_ctl_1G); + + hashed_bit &= BIT(0); + + if (hashed_bit != ((ctx->ret_addr >> 12) & BIT(0))) + ctx->ret_addr ^= BIT(12); + + if (ctx->intlv_mode != HASH_COD1_8CH) + return 0; + + hashed_bit = (ctx->ret_addr >> 13) ^ + ((ctx->ret_addr >> 17) & intlv_ctl_64k) ^ + ((ctx->ret_addr >> 22) & intlv_ctl_2M) ^ + ((ctx->ret_addr >> 31) & intlv_ctl_1G); + + hashed_bit &= BIT(0); + + if (hashed_bit != ((ctx->ret_addr >> 13) & BIT(0))) + ctx->ret_addr ^= BIT(13); + + return 0; +} + +static int get_intlv_mode_df3(struct addr_ctx *ctx) +{ + ctx->intlv_mode = (ctx->reg_base_addr >> 2) & 0xF; + + if (ctx->intlv_mode == HASH_COD4_2CH || + ctx->intlv_mode == HASH_COD2_4CH || + ctx->intlv_mode == HASH_COD1_8CH) { + ctx->make_space_for_cs_id = &make_space_for_cs_id_cod_hash; + ctx->insert_cs_id = &insert_cs_id_cod_hash; + ctx->dehash_addr = &dehash_addr_df3; + } else { + ctx->make_space_for_cs_id = &make_space_for_cs_id_simple; + ctx->insert_cs_id = &insert_cs_id_simple; + } + + return 0; +} + +static u8 get_intlv_addr_sel_df3(struct addr_ctx *ctx) +{ + return (ctx->reg_base_addr >> 9) & 0x7; +} + +static void get_intlv_num_dies_df3(struct addr_ctx *ctx) +{ + ctx->intlv_num_dies = (ctx->reg_base_addr >> 6) & 0x3; +} + +static void get_intlv_num_sockets_df3(struct addr_ctx *ctx) +{ + ctx->intlv_num_sockets = (ctx->reg_base_addr >> 8) & 0x1; +} + +static u8 get_die_id_shift_df3(struct addr_ctx *ctx) +{ + return ctx->node_id_shift; +} + +static u8 get_socket_id_shift_df3(struct addr_ctx *ctx) +{ + return ((ctx->reg_fab_id_mask1 >> 8) & 0x3) + ctx->node_id_shift; +} + +static int get_masks_df3(struct addr_ctx *ctx) +{ + if (amd_df_indirect_read(0, df_regs[SYS_FAB_ID_MASK_1], + DF_BROADCAST, &ctx->reg_fab_id_mask1)) + return -EINVAL; + + ctx->node_id_shift = ctx->reg_fab_id_mask1 & 0xF; + + ctx->die_id_mask = (ctx->reg_fab_id_mask1 >> 16) & 0x7; + ctx->die_id_mask <<= ctx->node_id_shift; + + ctx->socket_id_mask = (ctx->reg_fab_id_mask1 >> 24) & 0x7; + ctx->socket_id_mask <<= ctx->node_id_shift; + + return 0; +} + +static u16 get_dst_fabric_id_df3(struct addr_ctx *ctx) +{ + return ctx->reg_limit_addr & 0x3FF; +} + +static u16 get_component_id_mask_df3(struct addr_ctx *ctx) +{ + return ctx->reg_fab_id_mask0 & 0x3FF; +} + +struct data_fabric_ops df3_ops = { + .get_hi_addr_offset = &get_hi_addr_offset_df3, + .get_intlv_mode = &get_intlv_mode_df3, + .get_intlv_addr_sel = &get_intlv_addr_sel_df3, + .get_intlv_num_dies = &get_intlv_num_dies_df3, + .get_intlv_num_sockets = &get_intlv_num_sockets_df3, + .get_cs_fabric_id = &get_cs_fabric_id_df2, + .get_masks = &get_masks_df3, + .get_die_id_shift = &get_die_id_shift_df3, + .get_socket_id_shift = &get_socket_id_shift_df3, + .get_dst_fabric_id = &get_dst_fabric_id_df3, + .get_component_id_mask = &get_component_id_mask_df3, +}; + struct data_fabric_ops *df_ops; static int set_df_ops(struct addr_ctx *ctx) @@ -1265,6 +1436,11 @@ static int set_df_ops(struct addr_ctx *ctx) DF_BROADCAST, &ctx->reg_fab_id_mask0)) return -EINVAL; + if ((ctx->reg_fab_id_mask0 & 0xFF) != 0) { + df_ops = &df3_ops; + return 0; + } + df_ops = &df2_ops; return 0; @@ -1327,8 +1503,8 @@ static int get_intlv_addr_bit(struct addr_ctx *ctx) { u8 intlv_addr_sel = df_ops->get_intlv_addr_sel(ctx); - /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ - if (intlv_addr_sel > 3) { + /* {0, 1, 2, 3, 4} map to address bits {8, 9, 10, 11, 12} respectively */ + if (intlv_addr_sel > 4) { pr_debug("Invalid interleave address select %d.\n", intlv_addr_sel); return -EINVAL; } @@ -1346,9 +1522,18 @@ static void get_intlv_num_chan(struct addr_ctx *ctx) ctx->intlv_num_chan = 0; break; case NOHASH_2CH: + case HASH_COD4_2CH: case DF2_HASH_2CH: ctx->intlv_num_chan = 1; break; + case NOHASH_4CH: + case HASH_COD2_4CH: + ctx->intlv_num_chan = 2; + break; + case NOHASH_8CH: + case HASH_COD1_8CH: + ctx->intlv_num_chan = 3; + break; default: /* Valid interleaving modes where checked earlier. */ break; -- 2.33.1.711.g9d530dc002 From 5b66c25dfe8779638cc982bf21408ad334626cc7 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 23 Jun 2021 19:20:02 +0000 Subject: [PATCH 31/31] EDAC/amd64: Add glossary of acronyms for address translation Add a short glossary of the acronyms used for address translation. Some terms may be seen with other AMD code, and some are only used within the translation code. Signed-off-by: Yazen Ghannam --- drivers/edac/amd64_edac.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 80635e07d..4783dbef1 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -988,6 +988,15 @@ static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) return csrow; } +/* + * Glossary of acronyms used in address translation for Zen-based systems + * + * COD = Cluster-on-Die + * CS = Coherent Slave + * DF = Data Fabric + * UMC = Unified Memory Controller + */ + /* Protect the PCI config register pairs used for DF indirect access. */ static DEFINE_MUTEX(df_indirect_mutex); -- 2.33.1.711.g9d530dc002