[Skiboot] [PATCH V3 06/15] pau: enabling opencapi
Christophe Lombard
clombard at linux.vnet.ibm.com
Fri Oct 15 02:56:55 AEDT 2021
Enable OpenCAPI mode for each brick which are connected to be used in
this mode. This is be done through 7 steps as described in the
P10 OCAPI 5.0 Processing Unit Workbook document, section:
17.1.3.1 Enabling OpenCAPI.
The following sequences must be performed:
1. Set Transport MUX controls to select OpenCAPI
2. Enable Clocks in XSL
3. Enable Clocks in MISC
4. Set NPCQ configuration
5. Enable XSL-XTS Interfaces
6. Enable State-machine allocation
Enabling the NTL/GENID BARS allows to access to the MMIO registers.
Signed-off-by: Christophe Lombard <clombard at linux.vnet.ibm.com>
Reviewed-by: Frederic Barrat <fbarrat at linux.ibm.com>
---
hw/pau.c | 222 +++++++++++++++++++++++++++++++++++++++++++++
include/pau-regs.h | 43 +++++++++
include/pau.h | 1 +
3 files changed, 266 insertions(+)
diff --git a/hw/pau.c b/hw/pau.c
index 1a370a97..5f181011 100644
--- a/hw/pau.c
+++ b/hw/pau.c
@@ -199,6 +199,42 @@ static void pau_device_detect_fixup(struct pau_dev *dev)
dt_add_property_strings(dn, "ibm,pau-link-type", "unknown");
}
+#define CQ_CTL_STATUS_TIMEOUT 10 /* milliseconds */
+
+static int pau_opencapi_set_fence_control(struct pau_dev *dev,
+ uint8_t state_requested)
+{
+ uint64_t timeout = mftb() + msecs_to_tb(CQ_CTL_STATUS_TIMEOUT);
+ uint8_t status;
+ struct pau *pau = dev->pau;
+ uint64_t reg, val;
+
+ reg = PAU_CTL_MISC_FENCE_CTRL(dev->index);
+ val = pau_read(pau, reg);
+ val = SETFIELD(PAU_CTL_MISC_FENCE_REQUEST, val, state_requested);
+ pau_write(pau, reg, val);
+
+ /* Wait for fence status to update */
+ do {
+ reg = PAU_CTL_MISC_STATUS(dev->index);
+ val = pau_read(pau, reg);
+ status = GETFIELD(PAU_CTL_MISC_STATUS_AM_FENCED(dev->index), val);
+ if (status == state_requested)
+ return OPAL_SUCCESS;
+ time_wait_ms(1);
+ } while (tb_compare(mftb(), timeout) == TB_ABEFOREB);
+
+ /*
+ * @fwts-label OCAPIFenceStatusTimeout
+ * @fwts-advice The PAU fence status did not update as expected. This
+ * could be the result of a firmware or hardware bug. OpenCAPI
+ * functionality could be broken.
+ */
+ PAUDEVERR(dev, "Bad fence status: expected 0x%x, got 0x%x\n",
+ state_requested, status);
+ return OPAL_HARDWARE;
+}
+
static void pau_opencapi_assign_bars(struct pau *pau)
{
struct pau_dev *dev;
@@ -256,6 +292,37 @@ static void pau_opencapi_assign_bars(struct pau *pau)
}
}
+static void pau_opencapi_enable_bars(struct pau_dev *dev, bool enable)
+{
+ struct pau *pau = dev->pau;
+ uint64_t reg, val;
+
+ if (dev->ntl_bar.enable == enable) /* No state change */
+ return;
+
+ dev->ntl_bar.enable = enable;
+ dev->genid_bar.enable = enable;
+
+ reg = PAU_NTL_BAR(dev->index);
+ val = pau_read(pau, reg);
+ val = SETFIELD(PAU_NTL_BAR_ENABLE, val, enable);
+ pau_write(pau, reg, val);
+
+ /*
+ * Generation IDs are a single space in the hardware but we split them
+ * per device. Only disable in hardware if every device has disabled.
+ */
+ if (!enable)
+ pau_for_each_dev(dev, pau)
+ if (dev->genid_bar.enable)
+ return;
+
+ reg = PAU_GENID_BAR;
+ val = pau_read(pau, reg);
+ val = SETFIELD(PAU_GENID_BAR_ENABLE, val, enable);
+ pau_write(pau, reg, val);
+}
+
static void pau_opencapi_create_phb_slot(struct pau_dev *dev)
{
struct pci_slot *slot;
@@ -474,6 +541,135 @@ static void pau_opencapi_dt_add_props(struct pau_dev *dev)
pau_opencapi_dt_add_mmio_window(dev);
}
+static void pau_opencapi_set_transport_mux_controls(struct pau_dev *dev)
+{
+ struct pau *pau = dev->pau;
+ uint32_t typemap = 0;
+ uint64_t reg, val = 0;
+
+ PAUDEVDBG(dev, "Setting transport mux controls\n");
+ typemap = 0x2 >> dev->index;
+
+ reg = PAU_MISC_OPTICAL_IO_CONFIG;
+ val = pau_read(pau, reg);
+ typemap |= GETFIELD(PAU_MISC_OPTICAL_IO_CONFIG_OTL, val);
+ val = SETFIELD(PAU_MISC_OPTICAL_IO_CONFIG_OTL, val, typemap);
+ pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_xsl_clocks(struct pau *pau)
+{
+ uint64_t reg, val;
+
+ PAUDBG(pau, "Enable clocks in XSL\n");
+
+ reg = PAU_XSL_WRAP_CFG;
+ val = pau_read(pau, reg);
+ val |= PAU_XSL_WRAP_CFG_CLOCK_ENABLE;
+ pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_misc_clocks(struct pau *pau)
+{
+ uint64_t reg, val;
+
+ PAUDBG(pau, "Enable clocks in MISC\n");
+
+ /* clear any spurious NDL stall or no_stall_c_err_rpts */
+ reg = PAU_MISC_HOLD;
+ val = pau_read(pau, reg);
+ val = SETFIELD(PAU_MISC_HOLD_NDL_STALL, val, 0b0000);
+ pau_write(pau, reg, val);
+
+ reg = PAU_MISC_CONFIG;
+ val = pau_read(pau, reg);
+ val |= PAU_MISC_CONFIG_OC_MODE;
+ pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_set_npcq_config(struct pau *pau)
+{
+ struct pau_dev *dev;
+ uint8_t oc_typemap = 0;
+ uint64_t reg, val;
+
+ /* MCP_MISC_CFG0
+ * SNP_MISC_CFG0 done in pau_opencapi_enable_pb
+ */
+ pau_for_each_opencapi_dev(dev, pau)
+ oc_typemap |= 0x10 >> dev->index;
+
+ PAUDBG(pau, "Set NPCQ Config\n");
+ reg = PAU_CTL_MISC_CFG2;
+ val = pau_read(pau, reg);
+ val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_MODE, val, oc_typemap);
+ val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_4, val, oc_typemap);
+ val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_C2, val, oc_typemap);
+ val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_AMO, val, oc_typemap);
+ val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_MEM_OS_BIT, val, oc_typemap);
+ pau_write(pau, reg, val);
+
+ reg = PAU_DAT_MISC_CFG1;
+ val = pau_read(pau, reg);
+ val = SETFIELD(PAU_DAT_MISC_CFG1_OCAPI_MODE, val, oc_typemap);
+ pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_xsl_xts_interfaces(struct pau *pau)
+{
+ uint64_t reg, val;
+
+ PAUDBG(pau, "Enable XSL-XTS Interfaces\n");
+ reg = PAU_XTS_CFG;
+ val = pau_read(pau, reg);
+ val |= PAU_XTS_CFG_OPENCAPI;
+ pau_write(pau, reg, val);
+
+ reg = PAU_XTS_CFG2;
+ val = pau_read(pau, reg);
+ val |= PAU_XTS_CFG2_XSL2_ENA;
+ pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_sm_allocation(struct pau *pau)
+{
+ uint64_t reg, val;
+
+ PAUDBG(pau, "Enable State Machine Allocation\n");
+
+ reg = PAU_MISC_MACHINE_ALLOC;
+ val = pau_read(pau, reg);
+ val |= PAU_MISC_MACHINE_ALLOC_ENABLE;
+ pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_powerbus(struct pau *pau)
+{
+ struct pau_dev *dev;
+ uint8_t oc_typemap = 0;
+ uint64_t reg, val;
+
+ PAUDBG(pau, "Enable PowerBus\n");
+
+ pau_for_each_opencapi_dev(dev, pau)
+ oc_typemap |= 0x10 >> dev->index;
+
+ /* PowerBus interfaces must be enabled prior to MMIO */
+ reg = PAU_MCP_MISC_CFG0;
+ val = pau_read(pau, reg);
+ val |= PAU_MCP_MISC_CFG0_ENABLE_PBUS;
+ val |= PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP;
+ val = SETFIELD(PAU_MCP_MISC_CFG0_OCAPI_MODE, val, oc_typemap);
+ pau_write(pau, reg, val);
+
+ reg = PAU_SNP_MISC_CFG0;
+ val = pau_read(pau, reg);
+ val |= PAU_SNP_MISC_CFG0_ENABLE_PBUS;
+ val = SETFIELD(PAU_SNP_MISC_CFG0_OCAPI_MODE, val, oc_typemap);
+ val = SETFIELD(PAU_SNP_MISC_CFG0_OCAPI_C2, val, oc_typemap);
+ pau_write(pau, reg, val);
+}
+
static void pau_opencapi_init_hw(struct pau *pau)
{
struct pau_dev *dev = NULL;
@@ -482,9 +678,35 @@ static void pau_opencapi_init_hw(struct pau *pau)
/* Create phb */
pau_for_each_opencapi_dev(dev, pau) {
+ PAUDEVINF(dev, "Create phb\n");
pau_opencapi_create_phb(dev);
+ pau_opencapi_enable_bars(dev, true);
pau_opencapi_dt_add_props(dev);
}
+
+ /* Procedure 17.1.3.1 - Enabling OpenCAPI */
+ pau_for_each_opencapi_dev(dev, pau) {
+ PAUDEVINF(dev, "Configuring link ...\n");
+ pau_opencapi_set_transport_mux_controls(dev); /* step 1 */
+ }
+ pau_opencapi_enable_xsl_clocks(pau); /* step 2 */
+ pau_opencapi_enable_misc_clocks(pau); /* step 3 */
+
+ /* OTL disabled */
+ pau_for_each_opencapi_dev(dev, pau)
+ pau_opencapi_set_fence_control(dev, 0b01);
+
+ pau_opencapi_set_npcq_config(pau); /* step 4 */
+ pau_opencapi_enable_xsl_xts_interfaces(pau); /* step 5 */
+ pau_opencapi_enable_sm_allocation(pau); /* step 6 */
+ pau_opencapi_enable_powerbus(pau); /* step 7 */
+
+ /*
+ * access to the PAU registers through mmio requires setting
+ * up the PAU mmio BAR (in pau_opencapi_assign_bars() above)
+ * and machine state allocation
+ */
+ pau->mmio_access = true;
}
static void pau_opencapi_init(struct pau *pau)
diff --git a/include/pau-regs.h b/include/pau-regs.h
index 57796920..6aeb7589 100644
--- a/include/pau-regs.h
+++ b/include/pau-regs.h
@@ -27,6 +27,10 @@
#define PAU_BLOCK_CQ_SM(n) PAU_BLOCK(4, (n))
#define PAU_BLOCK_CQ_CTL PAU_BLOCK(4, 4)
+#define PAU_BLOCK_CQ_DAT PAU_BLOCK(4, 5)
+#define PAU_BLOCK_XSL PAU_BLOCK(4, 0xE)
+#define PAU_BLOCK_PAU_XTS PAU_BLOCK(7, 1)
+#define PAU_BLOCK_PAU_MISC PAU_BLOCK(7, 2)
/*
* CQ_SM block registers
@@ -37,21 +41,38 @@
#define PAU_MCP_MISC_CFG0 (PAU_BLOCK_CQ_SM(0) + 0x000)
#define PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP PPC_BIT(9)
#define PAU_MCP_MISC_CFG0_ENABLE_PBUS PPC_BIT(26)
+#define PAU_MCP_MISC_CFG0_OCAPI_MODE PPC_BITMASK(44, 48)
#define PAU_SNP_MISC_CFG0 (PAU_BLOCK_CQ_SM(0) + 0x180)
#define PAU_SNP_MISC_CFG0_ENABLE_PBUS PPC_BIT(2)
+#define PAU_SNP_MISC_CFG0_OCAPI_MODE PPC_BITMASK(32, 36)
+#define PAU_SNP_MISC_CFG0_OCAPI_C2 PPC_BITMASK(45, 49)
#define PAU_NTL_BAR(brk) (PAU_BLOCK_CQ_SM(0) + 0x1b8 + (brk) * 8)
+#define PAU_NTL_BAR_ENABLE PPC_BIT(0)
#define PAU_NTL_BAR_ADDR PPC_BITMASK(3, 35)
#define PAU_NTL_BAR_SIZE PPC_BITMASK(39, 43)
#define PAU_MMIO_BAR (PAU_BLOCK_CQ_SM(0) + 0x1e0)
#define PAU_MMIO_BAR_ENABLE PPC_BIT(0)
#define PAU_MMIO_BAR_ADDR PPC_BITMASK(3, 27)
#define PAU_GENID_BAR (PAU_BLOCK_CQ_SM(0) + 0x1e8)
+#define PAU_GENID_BAR_ENABLE PPC_BIT(0)
#define PAU_GENID_BAR_ADDR PPC_BITMASK(3, 32)
+#define PAU_MISC_MACHINE_ALLOC (PAU_BLOCK_CQ_SM(0) + 0x268)
+#define PAU_MISC_MACHINE_ALLOC_ENABLE PPC_BIT(0)
/* CQ_CTL block registers */
+#define PAU_CTL_MISC_CFG2 (PAU_BLOCK_CQ_CTL + 0x010)
+#define PAU_CTL_MISC_CFG2_OCAPI_MODE PPC_BITMASK(0, 4)
+#define PAU_CTL_MISC_CFG2_OCAPI_4 PPC_BITMASK(10, 14)
+#define PAU_CTL_MISC_CFG2_OCAPI_C2 PPC_BITMASK(15, 19)
+#define PAU_CTL_MISC_CFG2_OCAPI_AMO PPC_BITMASK(20, 24)
+#define PAU_CTL_MISC_CFG2_OCAPI_MEM_OS_BIT PPC_BITMASK(25, 29)
+#define PAU_CTL_MISC_STATUS(brk) (PAU_BLOCK_CQ_CTL + 0x060 + (brk) * 8)
+#define PAU_CTL_MISC_STATUS_AM_FENCED(brk) (PPC_BITMASK(41, 42) << ((brk)*32))
#define PAU_CTL_MISC_MMIOPA_CONFIG(brk) (PAU_BLOCK_CQ_CTL + 0x098 + (brk) * 8)
#define PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR PPC_BITMASK(1, 35)
#define PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE PPC_BITMASK(39, 43)
+#define PAU_CTL_MISC_FENCE_CTRL(brk) (PAU_BLOCK_CQ_CTL + 0x108 + (brk) * 8)
+#define PAU_CTL_MISC_FENCE_REQUEST PPC_BITMASK(0, 1)
#define PAU_CTL_MISC_CFG_ADDR(brk) (PAU_BLOCK_CQ_CTL + 0x250 + (brk) * 8)
#define PAU_CTL_MISC_CFG_ADDR_ENABLE PPC_BIT(0)
#define PAU_CTL_MISC_CFG_ADDR_STATUS PPC_BITMASK(1, 3)
@@ -61,4 +82,26 @@
#define PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR PPC_BITMASK(20, 31)
#define PAU_CTL_MISC_CFG_ADDR_TYPE PPC_BIT(32)
+/* CQ_DAT block registers */
+#define PAU_DAT_MISC_CFG1 (PAU_BLOCK_CQ_DAT + 0x008)
+#define PAU_DAT_MISC_CFG1_OCAPI_MODE PPC_BITMASK(40, 44)
+
+/* XSL block registers */
+#define PAU_XSL_WRAP_CFG (PAU_BLOCK_XSL + 0x100)
+#define PAU_XSL_WRAP_CFG_CLOCK_ENABLE PPC_BIT(0)
+
+/* XTS block registers */
+#define PAU_XTS_CFG (PAU_BLOCK_PAU_XTS + 0x020)
+#define PAU_XTS_CFG_OPENCAPI PPC_BIT(15)
+#define PAU_XTS_CFG2 (PAU_BLOCK_PAU_XTS + 0x028)
+#define PAU_XTS_CFG2_XSL2_ENA PPC_BIT(55)
+
+/* MISC block registers */
+#define PAU_MISC_OPTICAL_IO_CONFIG (PAU_BLOCK_PAU_MISC + 0x018)
+#define PAU_MISC_OPTICAL_IO_CONFIG_OTL PPC_BITMASK(2, 3)
+#define PAU_MISC_HOLD (PAU_BLOCK_PAU_MISC + 0x020)
+#define PAU_MISC_HOLD_NDL_STALL PPC_BITMASK(0, 3)
+#define PAU_MISC_CONFIG (PAU_BLOCK_PAU_MISC + 0x030)
+#define PAU_MISC_CONFIG_OC_MODE PPC_BIT(16)
+
#endif /* __PAU_REGS_H */
diff --git a/include/pau.h b/include/pau.h
index fdf85f85..0246a63d 100644
--- a/include/pau.h
+++ b/include/pau.h
@@ -22,6 +22,7 @@ enum pau_dev_type {
/* Used to expose a hardware BAR (or logical slice of it) outside skiboot */
struct pau_bar {
+ bool enable;
uint64_t addr;
uint64_t size;
uint64_t cfg;
--
2.31.1
More information about the Skiboot
mailing list