[Skiboot] [PATCH 07/16] [PATCH 07/16] opencapi5: enabling opencapi

Frederic Barrat fbarrat at linux.ibm.com
Wed Sep 8 22:55:24 AEST 2021



On 20/08/2021 11:45, Christophe Lombard wrote:
> Enable OpenCAPI mode for each brick which are connected to be used in
> this mode. This is be done through 7 steps as described in the
> P10 OCAPI 5.0 Processing Unit Workbook document, section:
> 17.1.3.1 Enabling OpenCAPI.
> The following sequences must be performed:
> 1. Set Transport MUX controls to select OpenCAPI
> 2. Enable Clocks in XSL
> 3. Enable Clocks in MISC
> 4. Set NPCQ configuration
> 5. Enable XSL-XTS Interfaces
> 6. Enable State-machine allocation
> 
> Enabling the NTL/GENID BARS allows to access to the MMIO registers.
> 
> Signed-off-by: Christophe Lombard <clombard at linux.vnet.ibm.com>
> ---


I must admit I haven't (re-)checked all the bits and registers for all 
those inits in this patch (and the next 2), that's what bringup is for 
:) And most of those date from Swift... So:
Reviewed-by: Frederic Barrat <fbarrat at linux.ibm.com>




>   hw/pau.c           | 222 +++++++++++++++++++++++++++++++++++++++++++++
>   include/pau-regs.h |  43 +++++++++
>   include/pau.h      |   1 +
>   3 files changed, 266 insertions(+)
> 
> diff --git a/hw/pau.c b/hw/pau.c
> index 5caafe6b..d7b51ee5 100644
> --- a/hw/pau.c
> +++ b/hw/pau.c
> @@ -200,6 +200,42 @@ static void pau_device_detect_fixup(struct pau_dev *dev)
>   	dt_add_property_strings(dn, "ibm,pau-link-type", "unknown");
>   }
> 
> +#define CQ_CTL_STATUS_TIMEOUT  10 /* milliseconds */
> +
> +static int pau_opencapi_set_fence_control(struct pau_dev *dev,
> +					  uint8_t state_requested)
> +{
> +	uint64_t timeout = mftb() + msecs_to_tb(CQ_CTL_STATUS_TIMEOUT);
> +	uint8_t status;
> +	struct pau *pau = dev->pau;
> +	uint64_t reg, val;
> +
> +	reg = PAU_CTL_MISC_FENCE_CTRL(dev->index);
> +	val = pau_read(pau, reg);
> +	val = SETFIELD(PAU_CTL_MISC_FENCE_REQUEST, val, state_requested);
> +	pau_write(pau, reg, val);
> +
> +	/* Wait for fence status to update */
> +	do {
> +		reg = PAU_CTL_MISC_STATUS(dev->index);
> +		val = pau_read(pau, reg);
> +		status = GETFIELD(PAU_CTL_MISC_STATUS_AM_FENCED(dev->index), val);
> +		if (status == state_requested)
> +			return OPAL_SUCCESS;
> +		time_wait_ms(1);
> +	} while (tb_compare(mftb(), timeout) == TB_ABEFOREB);
> +
> +	/*
> +	 * @fwts-label OCAPIFenceStatusTimeout
> +	 * @fwts-advice The PAU fence status did not update as expected. This
> +	 * could be the result of a firmware or hardware bug. OpenCAPI
> +	 * functionality could be broken.
> +	 */
> +	PAUDEVERR(dev, "Bad fence status: expected 0x%x, got 0x%x\n",
> +		       state_requested, status);
> +	return OPAL_HARDWARE;
> +}
> +
>   static void pau_opencapi_assign_bars(struct pau *pau)
>   {
>   	struct pau_dev *dev;
> @@ -257,6 +293,37 @@ static void pau_opencapi_assign_bars(struct pau *pau)
>   	}
>   }
> 
> +static void pau_opencapi_enable_bars(struct pau_dev *dev, bool enable)
> +{
> +	struct pau *pau = dev->pau;
> +	uint64_t reg, val;
> +
> +	if (dev->ntl_bar.enable == enable) /* No state change */
> +		return;
> +
> +	dev->ntl_bar.enable = enable;
> +	dev->genid_bar.enable = enable;
> +
> +	reg = PAU_NTL_BAR(dev->index);
> +	val = pau_read(pau, reg);
> +	val = SETFIELD(PAU_NTL_BAR_ENABLE, val, enable);
> +	pau_write(pau, reg, val);
> +
> +	/*
> +	 * Generation IDs are a single space in the hardware but we split them
> +	 * per device. Only disable in hardware if every device has disabled.
> +	 */
> +	if (!enable)
> +		pau_for_each_dev(dev, pau)
> +			if (dev->genid_bar.enable)
> +				return;
> +
> +	reg = PAU_GENID_BAR;
> +	val = pau_read(pau, reg);
> +	val = SETFIELD(PAU_GENID_BAR_ENABLE, val, enable);
> +	pau_write(pau, reg, val);
> +}
> +
>   static void pau_opencapi_create_phb_slot(struct pau_dev *dev)
>   {
>   	struct pci_slot *slot;
> @@ -475,6 +542,135 @@ static void pau_opencapi_dt_add_props(struct pau_dev *dev)
>   	pau_opencapi_dt_add_mmio_window(dev);
>   }
> 
> +static void pau_opencapi_set_transport_mux_controls(struct pau_dev *dev)
> +{
> +	struct pau *pau = dev->pau;
> +	uint32_t typemap = 0;
> +	uint64_t reg, val = 0;
> +
> +	PAUDEVDBG(dev, "Setting transport mux controls\n");
> +	typemap = 0x2 >> dev->index;
> +
> +	reg = PAU_MISC_OPTICAL_IO_CONFIG;
> +	val = pau_read(pau, reg);
> +	typemap |= GETFIELD(PAU_MISC_OPTICAL_IO_CONFIG_OTL, val);
> +	val = SETFIELD(PAU_MISC_OPTICAL_IO_CONFIG_OTL, val, typemap);
> +	pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_xsl_clocks(struct pau *pau)
> +{
> +	uint64_t reg, val;
> +
> +	PAUDBG(pau, "Enable clocks in XSL\n");
> +
> +	reg = PAU_XSL_WRAP_CFG;
> +	val = pau_read(pau, reg);
> +	val |= PAU_XSL_WRAP_CFG_CLOCK_ENABLE;
> +	pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_misc_clocks(struct pau *pau)
> +{
> +	uint64_t reg, val;
> +
> +	PAUDBG(pau, "Enable clocks in MISC\n");
> +
> +	/* clear any spurious NDL stall or no_stall_c_err_rpts */
> +	reg = PAU_MISC_HOLD;
> +	val = pau_read(pau, reg);
> +	val = SETFIELD(PAU_MISC_HOLD_NDL_STALL, val, 0b0000);
> +	pau_write(pau, reg, val);
> +
> +	reg = PAU_MISC_CONFIG;
> +	val = pau_read(pau, reg);
> +	val |= PAU_MISC_CONFIG_OC_MODE;
> +	pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_set_npcq_config(struct pau *pau)
> +{
> +	struct pau_dev *dev;
> +	uint8_t oc_typemap = 0;
> +	uint64_t reg, val;
> +
> +	/* MCP_MISC_CFG0
> +	 * SNP_MISC_CFG0 done in pau_opencapi_enable_pb
> +	 */
> +	pau_for_each_opencapi_dev(dev, pau)
> +		oc_typemap |= 0x10 >> dev->index;
> +
> +	PAUDBG(pau, "Set NPCQ Config\n");
> +	reg = PAU_CTL_MISC_CFG2;
> +	val = pau_read(pau, reg);
> +	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_MODE, val, oc_typemap);
> +	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_4, val, oc_typemap);
> +	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_C2, val, oc_typemap);
> +	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_AMO, val, oc_typemap);
> +	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_MEM_OS_BIT, val, oc_typemap);
> +	pau_write(pau, reg, val);
> +
> +	reg = PAU_DAT_MISC_CFG1;
> +	val = pau_read(pau, reg);
> +	val = SETFIELD(PAU_DAT_MISC_CFG1_OCAPI_MODE, val, oc_typemap);
> +	pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_xsl_xts_interfaces(struct pau *pau)
> +{
> +	uint64_t reg, val;
> +
> +	PAUDBG(pau, "Enable XSL-XTS Interfaces\n");
> +	reg = PAU_XTS_CFG;
> +	val = pau_read(pau, reg);
> +	val |= PAU_XTS_CFG_OPENCAPI;
> +	pau_write(pau, reg, val);
> +
> +	reg = PAU_XTS_CFG2;
> +	val = pau_read(pau, reg);
> +	val |= PAU_XTS_CFG2_XSL2_ENA;
> +	pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_sm_allocation(struct pau *pau)
> +{
> +	uint64_t reg, val;
> +
> +	PAUDBG(pau, "Enable State Machine Allocation\n");
> +
> +	reg = PAU_MISC_MACHINE_ALLOC;
> +	val = pau_read(pau, reg);
> +	val |= PAU_MISC_MACHINE_ALLOC_ENABLE;
> +	pau_write(pau, reg, val);
> +}
> +
> +static void pau_opencapi_enable_powerbus(struct pau *pau)
> +{
> +	struct pau_dev *dev;
> +	uint8_t oc_typemap = 0;
> +	uint64_t reg, val;
> +
> +	PAUDBG(pau, "Enable PowerBus\n");
> +
> +	pau_for_each_opencapi_dev(dev, pau)
> +		oc_typemap |= 0x10 >> dev->index;
> +
> +	/* PowerBus interfaces must be enabled prior to MMIO */
> +	reg = PAU_MCP_MISC_CFG0;
> +	val = pau_read(pau, reg);
> +	val |= PAU_MCP_MISC_CFG0_ENABLE_PBUS;
> +	val |= PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP;
> +	val = SETFIELD(PAU_MCP_MISC_CFG0_OCAPI_MODE, val, oc_typemap);
> +	pau_write(pau, reg, val);
> +
> +	reg = PAU_SNP_MISC_CFG0;
> +	val = pau_read(pau, reg);
> +	val |= PAU_SNP_MISC_CFG0_ENABLE_PBUS;
> +	val = SETFIELD(PAU_SNP_MISC_CFG0_OCAPI_MODE, val, oc_typemap);
> +	val = SETFIELD(PAU_SNP_MISC_CFG0_OCAPI_C2, val, oc_typemap);
> +	pau_write(pau, reg, val);
> +}
> +
>   static void pau_opencapi_init_hw(struct pau *pau)
>   {
>   	struct pau_dev *dev = NULL;
> @@ -483,9 +679,35 @@ static void pau_opencapi_init_hw(struct pau *pau)
> 
>   	/* Create phb */
>   	pau_for_each_opencapi_dev(dev, pau) {
> +		PAUDEVINF(dev, "Create phb\n");
>   		pau_opencapi_create_phb(dev);
> +		pau_opencapi_enable_bars(dev, true);
>   		pau_opencapi_dt_add_props(dev);
>   	}
> +
> +	/* Procedure 17.1.3.1 - Enabling OpenCAPI */
> +	pau_for_each_opencapi_dev(dev, pau) {
> +		PAUDEVINF(dev, "Configuring link ...\n");
> +		pau_opencapi_set_transport_mux_controls(dev);	/* step 1 */
> +	}
> +	pau_opencapi_enable_xsl_clocks(pau);		/* step 2 */
> +	pau_opencapi_enable_misc_clocks(pau);		/* step 3 */
> +
> +	/* OTL disabled */
> +	pau_for_each_opencapi_dev(dev, pau)
> +		pau_opencapi_set_fence_control(dev, 0b01);
> +
> +	pau_opencapi_set_npcq_config(pau);		/* step 4 */
> +	pau_opencapi_enable_xsl_xts_interfaces(pau);	/* step 5 */
> +	pau_opencapi_enable_sm_allocation(pau);		/* step 6 */
> +	pau_opencapi_enable_powerbus(pau);		/* step 7 */
> +
> +	/*
> +	 * access to the PAU registers through mmio requires setting
> +	 * up the PAU mmio BAR (in pau_opencapi_assign_bars() above)
> +	 * and machine state allocation
> +	 */
> +	pau->mmio_access = true;
>   }
> 
>   static void pau_opencapi_init(struct pau *pau)
> diff --git a/include/pau-regs.h b/include/pau-regs.h
> index 57796920..6aeb7589 100644
> --- a/include/pau-regs.h
> +++ b/include/pau-regs.h
> @@ -27,6 +27,10 @@
> 
>   #define PAU_BLOCK_CQ_SM(n)			PAU_BLOCK(4, (n))
>   #define PAU_BLOCK_CQ_CTL			PAU_BLOCK(4, 4)
> +#define PAU_BLOCK_CQ_DAT			PAU_BLOCK(4, 5)
> +#define PAU_BLOCK_XSL				PAU_BLOCK(4, 0xE)
> +#define PAU_BLOCK_PAU_XTS			PAU_BLOCK(7, 1)
> +#define PAU_BLOCK_PAU_MISC			PAU_BLOCK(7, 2)
> 
>   /*
>    * CQ_SM block registers
> @@ -37,21 +41,38 @@
>   #define PAU_MCP_MISC_CFG0			(PAU_BLOCK_CQ_SM(0) + 0x000)
>   #define   PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP	PPC_BIT(9)
>   #define   PAU_MCP_MISC_CFG0_ENABLE_PBUS		PPC_BIT(26)
> +#define   PAU_MCP_MISC_CFG0_OCAPI_MODE		PPC_BITMASK(44, 48)
>   #define PAU_SNP_MISC_CFG0			(PAU_BLOCK_CQ_SM(0) + 0x180)
>   #define   PAU_SNP_MISC_CFG0_ENABLE_PBUS		PPC_BIT(2)
> +#define   PAU_SNP_MISC_CFG0_OCAPI_MODE		PPC_BITMASK(32, 36)
> +#define   PAU_SNP_MISC_CFG0_OCAPI_C2		PPC_BITMASK(45, 49)
>   #define PAU_NTL_BAR(brk)			(PAU_BLOCK_CQ_SM(0) + 0x1b8 + (brk) * 8)
> +#define   PAU_NTL_BAR_ENABLE			PPC_BIT(0)
>   #define   PAU_NTL_BAR_ADDR			PPC_BITMASK(3, 35)
>   #define   PAU_NTL_BAR_SIZE			PPC_BITMASK(39, 43)
>   #define PAU_MMIO_BAR				(PAU_BLOCK_CQ_SM(0) + 0x1e0)
>   #define   PAU_MMIO_BAR_ENABLE			PPC_BIT(0)
>   #define   PAU_MMIO_BAR_ADDR			PPC_BITMASK(3, 27)
>   #define PAU_GENID_BAR				(PAU_BLOCK_CQ_SM(0) + 0x1e8)
> +#define   PAU_GENID_BAR_ENABLE			PPC_BIT(0)
>   #define   PAU_GENID_BAR_ADDR			PPC_BITMASK(3, 32)
> +#define PAU_MISC_MACHINE_ALLOC			(PAU_BLOCK_CQ_SM(0) + 0x268)
> +#define   PAU_MISC_MACHINE_ALLOC_ENABLE		PPC_BIT(0)
> 
>   /* CQ_CTL block registers */
> +#define PAU_CTL_MISC_CFG2			(PAU_BLOCK_CQ_CTL + 0x010)
> +#define   PAU_CTL_MISC_CFG2_OCAPI_MODE		PPC_BITMASK(0, 4)
> +#define   PAU_CTL_MISC_CFG2_OCAPI_4		PPC_BITMASK(10, 14)
> +#define   PAU_CTL_MISC_CFG2_OCAPI_C2		PPC_BITMASK(15, 19)
> +#define   PAU_CTL_MISC_CFG2_OCAPI_AMO		PPC_BITMASK(20, 24)
> +#define   PAU_CTL_MISC_CFG2_OCAPI_MEM_OS_BIT	PPC_BITMASK(25, 29)
> +#define PAU_CTL_MISC_STATUS(brk)		(PAU_BLOCK_CQ_CTL + 0x060 + (brk) * 8)
> +#define   PAU_CTL_MISC_STATUS_AM_FENCED(brk)	(PPC_BITMASK(41, 42) << ((brk)*32))
>   #define PAU_CTL_MISC_MMIOPA_CONFIG(brk)		(PAU_BLOCK_CQ_CTL + 0x098 + (brk) * 8)
>   #define   PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR	PPC_BITMASK(1, 35)
>   #define   PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE	PPC_BITMASK(39, 43)
> +#define PAU_CTL_MISC_FENCE_CTRL(brk)		(PAU_BLOCK_CQ_CTL + 0x108 + (brk) * 8)
> +#define   PAU_CTL_MISC_FENCE_REQUEST		PPC_BITMASK(0, 1)
>   #define PAU_CTL_MISC_CFG_ADDR(brk)		(PAU_BLOCK_CQ_CTL + 0x250 + (brk) * 8)
>   #define   PAU_CTL_MISC_CFG_ADDR_ENABLE		PPC_BIT(0)
>   #define   PAU_CTL_MISC_CFG_ADDR_STATUS		PPC_BITMASK(1, 3)
> @@ -61,4 +82,26 @@
>   #define   PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR	PPC_BITMASK(20, 31)
>   #define   PAU_CTL_MISC_CFG_ADDR_TYPE		PPC_BIT(32)
> 
> +/* CQ_DAT block registers */
> +#define PAU_DAT_MISC_CFG1			(PAU_BLOCK_CQ_DAT + 0x008)
> +#define   PAU_DAT_MISC_CFG1_OCAPI_MODE		PPC_BITMASK(40, 44)
> +
> +/* XSL block registers */
> +#define PAU_XSL_WRAP_CFG			(PAU_BLOCK_XSL + 0x100)
> +#define   PAU_XSL_WRAP_CFG_CLOCK_ENABLE		PPC_BIT(0)
> +
> +/* XTS block registers */
> +#define PAU_XTS_CFG				(PAU_BLOCK_PAU_XTS + 0x020)
> +#define   PAU_XTS_CFG_OPENCAPI			PPC_BIT(15)
> +#define PAU_XTS_CFG2				(PAU_BLOCK_PAU_XTS + 0x028)
> +#define   PAU_XTS_CFG2_XSL2_ENA			PPC_BIT(55)
> +
> +/* MISC block registers */
> +#define PAU_MISC_OPTICAL_IO_CONFIG		(PAU_BLOCK_PAU_MISC + 0x018)
> +#define   PAU_MISC_OPTICAL_IO_CONFIG_OTL	PPC_BITMASK(2, 3)
> +#define PAU_MISC_HOLD				(PAU_BLOCK_PAU_MISC + 0x020)
> +#define   PAU_MISC_HOLD_NDL_STALL		PPC_BITMASK(0, 3)
> +#define PAU_MISC_CONFIG				(PAU_BLOCK_PAU_MISC + 0x030)
> +#define   PAU_MISC_CONFIG_OC_MODE		PPC_BIT(16)
> +
>   #endif /* __PAU_REGS_H */
> diff --git a/include/pau.h b/include/pau.h
> index 6f2eef6e..4d78cbb6 100644
> --- a/include/pau.h
> +++ b/include/pau.h
> @@ -22,6 +22,7 @@ enum pau_dev_type {
> 
>   /* Used to expose a hardware BAR (or logical slice of it) outside skiboot */
>   struct pau_bar {
> +	bool			enable;
>   	uint64_t		addr;
>   	uint64_t		size;
>   	uint64_t		cfg;
> 


More information about the Skiboot mailing list