[Skiboot] [PATCH 07/16] [PATCH 07/16] opencapi5: enabling opencapi
Frederic Barrat
fbarrat at linux.ibm.com
Wed Sep 8 22:55:24 AEST 2021
On 20/08/2021 11:45, Christophe Lombard wrote:
> Enable OpenCAPI mode for each brick which are connected to be used in
> this mode. This is be done through 7 steps as described in the
> P10 OCAPI 5.0 Processing Unit Workbook document, section:
> 17.1.3.1 Enabling OpenCAPI.
> The following sequences must be performed:
> 1. Set Transport MUX controls to select OpenCAPI
> 2. Enable Clocks in XSL
> 3. Enable Clocks in MISC
> 4. Set NPCQ configuration
> 5. Enable XSL-XTS Interfaces
> 6. Enable State-machine allocation
>
> Enabling the NTL/GENID BARS allows to access to the MMIO registers.
>
> Signed-off-by: Christophe Lombard <clombard at linux.vnet.ibm.com>
> ---
I must admit I haven't (re-)checked all the bits and registers for all
those inits in this patch (and the next 2), that's what bringup is for
:) And most of those date from Swift... So:
Reviewed-by: Frederic Barrat <fbarrat at linux.ibm.com>
> hw/pau.c | 222 +++++++++++++++++++++++++++++++++++++++++++++
> include/pau-regs.h | 43 +++++++++
> include/pau.h | 1 +
> 3 files changed, 266 insertions(+)
>
> diff --git a/hw/pau.c b/hw/pau.c
> index 5caafe6b..d7b51ee5 100644
> --- a/hw/pau.c
> +++ b/hw/pau.c
> @@ -200,6 +200,42 @@ static void pau_device_detect_fixup(struct pau_dev *dev)
> dt_add_property_strings(dn, "ibm,pau-link-type", "unknown");
> }
>
> +#define CQ_CTL_STATUS_TIMEOUT 10 /* milliseconds */
> +
> +static int pau_opencapi_set_fence_control(struct pau_dev *dev,
> + uint8_t state_requested)
> +{
> + uint64_t timeout = mftb() + msecs_to_tb(CQ_CTL_STATUS_TIMEOUT);
> + uint8_t status;
> + struct pau *pau = dev->pau;
> + uint64_t reg, val;
> +
> + reg = PAU_CTL_MISC_FENCE_CTRL(dev->index);
> + val = pau_read(pau, reg);
> + val = SETFIELD(PAU_CTL_MISC_FENCE_REQUEST, val, state_requested);
> + pau_write(pau, reg, val);
> +
> + /* Wait for fence status to update */
> + do {
> + reg = PAU_CTL_MISC_STATUS(dev->index);
> + val = pau_read(pau, reg);
> + status = GETFIELD(PAU_CTL_MISC_STATUS_AM_FENCED(dev->index), val);
> + if (status == state_requested)
> + return OPAL_SUCCESS;
> + time_wait_ms(1);
> + } while (tb_compare(mftb(), timeout) == TB_ABEFOREB);
> +
> + /*
> + * @fwts-label OCAPIFenceStatusTimeout
> + * @fwts-advice The PAU fence status did not update as expected. This
> + * could be the result of a firmware or hardware bug. OpenCAPI
> + * functionality could be broken.
> + */
> + PAUDEVERR(dev, "Bad fence status: expected 0x%x, got 0x%x\n",
> + state_requested, status);
> + return OPAL_HARDWARE;
> +}
> +
> static void pau_opencapi_assign_bars(struct pau *pau)
> {
> struct pau_dev *dev;
> @@ -257,6 +293,37 @@ static void pau_opencapi_assign_bars(struct pau *pau)
> }
> }
>
> +static void pau_opencapi_enable_bars(struct pau_dev *dev, bool enable)
> +{
> + struct pau *pau = dev->pau;
> + uint64_t reg, val;
> +
> + if (dev->ntl_bar.enable == enable) /* No state change */
> + return;
> +
> + dev->ntl_bar.enable = enable;
> + dev->genid_bar.enable = enable;
> +
> + reg = PAU_NTL_BAR(dev->index);
> + val = pau_read(pau, reg);
> + val = SETFIELD(PAU_NTL_BAR_ENABLE, val, enable);
> + pau_write(pau, reg, val);
> +
> + /*
> + * Generation IDs are a single space in the hardware but we split them
> + * per device. Only disable in hardware if every device has disabled.
> + */
> + if (!enable)
> + pau_for_each_dev(dev, pau)
> + if (dev->genid_bar.enable)
> + return;
> +
> + reg = PAU_GENID_BAR;
> + val = pau_read(pau, reg);
> + val = SETFIELD(PAU_GENID_BAR_ENABLE, val, enable);
> + pau_write(pau, reg, val);
> +}
> +
> static void pau_opencapi_create_phb_slot(struct pau_dev *dev)
> {
> struct pci_slot *slot;
> @@ -475,6 +542,135 @@ static void pau_opencapi_dt_add_props(struct pau_dev *dev)
> pau_opencapi_dt_add_mmio_window(dev);
> }
>
> +static void pau_opencapi_set_transport_mux_controls(struct pau_dev *dev)
> +{
> + struct pau *pau = dev->pau;
> + uint32_t typemap = 0;
> + uint64_t reg, val = 0;
> +
> + PAUDEVDBG(dev, "Setting transport mux controls\n");
> + typemap = 0x2 >> dev->index;
> +
> + reg = PAU_MISC_OPTICAL_IO_CONFIG;
> + val = pau_read(pau, reg);
> + typemap |= GETFIELD(PAU_MISC_OPTICAL_IO_CONFIG_OTL, val);
> + val = SETFIELD(PAU_MISC_OPTICAL_IO_CONFIG_OTL, val, typemap);
> + pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_xsl_clocks(struct pau *pau)
> +{
> + uint64_t reg, val;
> +
> + PAUDBG(pau, "Enable clocks in XSL\n");
> +
> + reg = PAU_XSL_WRAP_CFG;
> + val = pau_read(pau, reg);
> + val |= PAU_XSL_WRAP_CFG_CLOCK_ENABLE;
> + pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_misc_clocks(struct pau *pau)
> +{
> + uint64_t reg, val;
> +
> + PAUDBG(pau, "Enable clocks in MISC\n");
> +
> + /* clear any spurious NDL stall or no_stall_c_err_rpts */
> + reg = PAU_MISC_HOLD;
> + val = pau_read(pau, reg);
> + val = SETFIELD(PAU_MISC_HOLD_NDL_STALL, val, 0b0000);
> + pau_write(pau, reg, val);
> +
> + reg = PAU_MISC_CONFIG;
> + val = pau_read(pau, reg);
> + val |= PAU_MISC_CONFIG_OC_MODE;
> + pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_set_npcq_config(struct pau *pau)
> +{
> + struct pau_dev *dev;
> + uint8_t oc_typemap = 0;
> + uint64_t reg, val;
> +
> + /* MCP_MISC_CFG0
> + * SNP_MISC_CFG0 done in pau_opencapi_enable_pb
> + */
> + pau_for_each_opencapi_dev(dev, pau)
> + oc_typemap |= 0x10 >> dev->index;
> +
> + PAUDBG(pau, "Set NPCQ Config\n");
> + reg = PAU_CTL_MISC_CFG2;
> + val = pau_read(pau, reg);
> + val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_MODE, val, oc_typemap);
> + val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_4, val, oc_typemap);
> + val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_C2, val, oc_typemap);
> + val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_AMO, val, oc_typemap);
> + val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_MEM_OS_BIT, val, oc_typemap);
> + pau_write(pau, reg, val);
> +
> + reg = PAU_DAT_MISC_CFG1;
> + val = pau_read(pau, reg);
> + val = SETFIELD(PAU_DAT_MISC_CFG1_OCAPI_MODE, val, oc_typemap);
> + pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_xsl_xts_interfaces(struct pau *pau)
> +{
> + uint64_t reg, val;
> +
> + PAUDBG(pau, "Enable XSL-XTS Interfaces\n");
> + reg = PAU_XTS_CFG;
> + val = pau_read(pau, reg);
> + val |= PAU_XTS_CFG_OPENCAPI;
> + pau_write(pau, reg, val);
> +
> + reg = PAU_XTS_CFG2;
> + val = pau_read(pau, reg);
> + val |= PAU_XTS_CFG2_XSL2_ENA;
> + pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_sm_allocation(struct pau *pau)
> +{
> + uint64_t reg, val;
> +
> + PAUDBG(pau, "Enable State Machine Allocation\n");
> +
> + reg = PAU_MISC_MACHINE_ALLOC;
> + val = pau_read(pau, reg);
> + val |= PAU_MISC_MACHINE_ALLOC_ENABLE;
> + pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_powerbus(struct pau *pau)
> +{
> + struct pau_dev *dev;
> + uint8_t oc_typemap = 0;
> + uint64_t reg, val;
> +
> + PAUDBG(pau, "Enable PowerBus\n");
> +
> + pau_for_each_opencapi_dev(dev, pau)
> + oc_typemap |= 0x10 >> dev->index;
> +
> + /* PowerBus interfaces must be enabled prior to MMIO */
> + reg = PAU_MCP_MISC_CFG0;
> + val = pau_read(pau, reg);
> + val |= PAU_MCP_MISC_CFG0_ENABLE_PBUS;
> + val |= PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP;
> + val = SETFIELD(PAU_MCP_MISC_CFG0_OCAPI_MODE, val, oc_typemap);
> + pau_write(pau, reg, val);
> +
> + reg = PAU_SNP_MISC_CFG0;
> + val = pau_read(pau, reg);
> + val |= PAU_SNP_MISC_CFG0_ENABLE_PBUS;
> + val = SETFIELD(PAU_SNP_MISC_CFG0_OCAPI_MODE, val, oc_typemap);
> + val = SETFIELD(PAU_SNP_MISC_CFG0_OCAPI_C2, val, oc_typemap);
> + pau_write(pau, reg, val);
> +}
> +
> static void pau_opencapi_init_hw(struct pau *pau)
> {
> struct pau_dev *dev = NULL;
> @@ -483,9 +679,35 @@ static void pau_opencapi_init_hw(struct pau *pau)
>
> /* Create phb */
> pau_for_each_opencapi_dev(dev, pau) {
> + PAUDEVINF(dev, "Create phb\n");
> pau_opencapi_create_phb(dev);
> + pau_opencapi_enable_bars(dev, true);
> pau_opencapi_dt_add_props(dev);
> }
> +
> + /* Procedure 17.1.3.1 - Enabling OpenCAPI */
> + pau_for_each_opencapi_dev(dev, pau) {
> + PAUDEVINF(dev, "Configuring link ...\n");
> + pau_opencapi_set_transport_mux_controls(dev); /* step 1 */
> + }
> + pau_opencapi_enable_xsl_clocks(pau); /* step 2 */
> + pau_opencapi_enable_misc_clocks(pau); /* step 3 */
> +
> + /* OTL disabled */
> + pau_for_each_opencapi_dev(dev, pau)
> + pau_opencapi_set_fence_control(dev, 0b01);
> +
> + pau_opencapi_set_npcq_config(pau); /* step 4 */
> + pau_opencapi_enable_xsl_xts_interfaces(pau); /* step 5 */
> + pau_opencapi_enable_sm_allocation(pau); /* step 6 */
> + pau_opencapi_enable_powerbus(pau); /* step 7 */
> +
> + /*
> + * access to the PAU registers through mmio requires setting
> + * up the PAU mmio BAR (in pau_opencapi_assign_bars() above)
> + * and machine state allocation
> + */
> + pau->mmio_access = true;
> }
>
> static void pau_opencapi_init(struct pau *pau)
> diff --git a/include/pau-regs.h b/include/pau-regs.h
> index 57796920..6aeb7589 100644
> --- a/include/pau-regs.h
> +++ b/include/pau-regs.h
> @@ -27,6 +27,10 @@
>
> #define PAU_BLOCK_CQ_SM(n) PAU_BLOCK(4, (n))
> #define PAU_BLOCK_CQ_CTL PAU_BLOCK(4, 4)
> +#define PAU_BLOCK_CQ_DAT PAU_BLOCK(4, 5)
> +#define PAU_BLOCK_XSL PAU_BLOCK(4, 0xE)
> +#define PAU_BLOCK_PAU_XTS PAU_BLOCK(7, 1)
> +#define PAU_BLOCK_PAU_MISC PAU_BLOCK(7, 2)
>
> /*
> * CQ_SM block registers
> @@ -37,21 +41,38 @@
> #define PAU_MCP_MISC_CFG0 (PAU_BLOCK_CQ_SM(0) + 0x000)
> #define PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP PPC_BIT(9)
> #define PAU_MCP_MISC_CFG0_ENABLE_PBUS PPC_BIT(26)
> +#define PAU_MCP_MISC_CFG0_OCAPI_MODE PPC_BITMASK(44, 48)
> #define PAU_SNP_MISC_CFG0 (PAU_BLOCK_CQ_SM(0) + 0x180)
> #define PAU_SNP_MISC_CFG0_ENABLE_PBUS PPC_BIT(2)
> +#define PAU_SNP_MISC_CFG0_OCAPI_MODE PPC_BITMASK(32, 36)
> +#define PAU_SNP_MISC_CFG0_OCAPI_C2 PPC_BITMASK(45, 49)
> #define PAU_NTL_BAR(brk) (PAU_BLOCK_CQ_SM(0) + 0x1b8 + (brk) * 8)
> +#define PAU_NTL_BAR_ENABLE PPC_BIT(0)
> #define PAU_NTL_BAR_ADDR PPC_BITMASK(3, 35)
> #define PAU_NTL_BAR_SIZE PPC_BITMASK(39, 43)
> #define PAU_MMIO_BAR (PAU_BLOCK_CQ_SM(0) + 0x1e0)
> #define PAU_MMIO_BAR_ENABLE PPC_BIT(0)
> #define PAU_MMIO_BAR_ADDR PPC_BITMASK(3, 27)
> #define PAU_GENID_BAR (PAU_BLOCK_CQ_SM(0) + 0x1e8)
> +#define PAU_GENID_BAR_ENABLE PPC_BIT(0)
> #define PAU_GENID_BAR_ADDR PPC_BITMASK(3, 32)
> +#define PAU_MISC_MACHINE_ALLOC (PAU_BLOCK_CQ_SM(0) + 0x268)
> +#define PAU_MISC_MACHINE_ALLOC_ENABLE PPC_BIT(0)
>
> /* CQ_CTL block registers */
> +#define PAU_CTL_MISC_CFG2 (PAU_BLOCK_CQ_CTL + 0x010)
> +#define PAU_CTL_MISC_CFG2_OCAPI_MODE PPC_BITMASK(0, 4)
> +#define PAU_CTL_MISC_CFG2_OCAPI_4 PPC_BITMASK(10, 14)
> +#define PAU_CTL_MISC_CFG2_OCAPI_C2 PPC_BITMASK(15, 19)
> +#define PAU_CTL_MISC_CFG2_OCAPI_AMO PPC_BITMASK(20, 24)
> +#define PAU_CTL_MISC_CFG2_OCAPI_MEM_OS_BIT PPC_BITMASK(25, 29)
> +#define PAU_CTL_MISC_STATUS(brk) (PAU_BLOCK_CQ_CTL + 0x060 + (brk) * 8)
> +#define PAU_CTL_MISC_STATUS_AM_FENCED(brk) (PPC_BITMASK(41, 42) << ((brk)*32))
> #define PAU_CTL_MISC_MMIOPA_CONFIG(brk) (PAU_BLOCK_CQ_CTL + 0x098 + (brk) * 8)
> #define PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR PPC_BITMASK(1, 35)
> #define PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE PPC_BITMASK(39, 43)
> +#define PAU_CTL_MISC_FENCE_CTRL(brk) (PAU_BLOCK_CQ_CTL + 0x108 + (brk) * 8)
> +#define PAU_CTL_MISC_FENCE_REQUEST PPC_BITMASK(0, 1)
> #define PAU_CTL_MISC_CFG_ADDR(brk) (PAU_BLOCK_CQ_CTL + 0x250 + (brk) * 8)
> #define PAU_CTL_MISC_CFG_ADDR_ENABLE PPC_BIT(0)
> #define PAU_CTL_MISC_CFG_ADDR_STATUS PPC_BITMASK(1, 3)
> @@ -61,4 +82,26 @@
> #define PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR PPC_BITMASK(20, 31)
> #define PAU_CTL_MISC_CFG_ADDR_TYPE PPC_BIT(32)
>
> +/* CQ_DAT block registers */
> +#define PAU_DAT_MISC_CFG1 (PAU_BLOCK_CQ_DAT + 0x008)
> +#define PAU_DAT_MISC_CFG1_OCAPI_MODE PPC_BITMASK(40, 44)
> +
> +/* XSL block registers */
> +#define PAU_XSL_WRAP_CFG (PAU_BLOCK_XSL + 0x100)
> +#define PAU_XSL_WRAP_CFG_CLOCK_ENABLE PPC_BIT(0)
> +
> +/* XTS block registers */
> +#define PAU_XTS_CFG (PAU_BLOCK_PAU_XTS + 0x020)
> +#define PAU_XTS_CFG_OPENCAPI PPC_BIT(15)
> +#define PAU_XTS_CFG2 (PAU_BLOCK_PAU_XTS + 0x028)
> +#define PAU_XTS_CFG2_XSL2_ENA PPC_BIT(55)
> +
> +/* MISC block registers */
> +#define PAU_MISC_OPTICAL_IO_CONFIG (PAU_BLOCK_PAU_MISC + 0x018)
> +#define PAU_MISC_OPTICAL_IO_CONFIG_OTL PPC_BITMASK(2, 3)
> +#define PAU_MISC_HOLD (PAU_BLOCK_PAU_MISC + 0x020)
> +#define PAU_MISC_HOLD_NDL_STALL PPC_BITMASK(0, 3)
> +#define PAU_MISC_CONFIG (PAU_BLOCK_PAU_MISC + 0x030)
> +#define PAU_MISC_CONFIG_OC_MODE PPC_BIT(16)
> +
> #endif /* __PAU_REGS_H */
> diff --git a/include/pau.h b/include/pau.h
> index 6f2eef6e..4d78cbb6 100644
> --- a/include/pau.h
> +++ b/include/pau.h
> @@ -22,6 +22,7 @@ enum pau_dev_type {
>
> /* Used to expose a hardware BAR (or logical slice of it) outside skiboot */
> struct pau_bar {
> + bool enable;
> uint64_t addr;
> uint64_t size;
> uint64_t cfg;
>
More information about the Skiboot
mailing list