[Skiboot] [PATCH 05/16] [PATCH 05/16] opencapi5: assing bars

Frederic Barrat fbarrat at linux.ibm.com
Wed Sep 8 22:40:33 AEST 2021



On 20/08/2021 11:45, Christophe Lombard wrote:
> Configure early PAU Global MMIO BAR registers to allow PAU MMIO
> register accesses. This is done for each PAU. Enable the Powerbus
> interface is mandatory for MMIO accesses.
> For each OpenCAPI device, configure the bar registers to access to
> the AFU MMIO and to the AFU Config Addr/Data registers.
> 
> AFU Config/Data registers = GENID_ADDR (from phy_map file) + 320K
> (= 0x50000)
> 
> Signed-off-by: Christophe Lombard <clombard at linux.vnet.ibm.com>
> ---

hmmm, that patch title :-)
Other than that:
Reviewed-by: Frederic Barrat <fbarrat at linux.ibm.com>



>   hw/pau.c           | 75 +++++++++++++++++++++++++++++++++++++++++++
>   hw/phys-map.c      | 46 ++++++++++++++++++++++++---
>   include/pau-regs.h | 26 +++++++++++++++
>   include/pau.h      | 79 ++++++++++++++++++++++++++++++++++++++++++++++
>   include/phys-map.h |  4 +++
>   5 files changed, 225 insertions(+), 5 deletions(-)
> 
> diff --git a/hw/pau.c b/hw/pau.c
> index b02b0034..fb6a175e 100644
> --- a/hw/pau.c
> +++ b/hw/pau.c
> @@ -194,6 +194,80 @@ static void pau_device_detect_fixup(struct pau_dev *dev)
>   	dt_add_property_strings(dn, "ibm,pau-link-type", "unknown");
>   }
> 
> +static void pau_opencapi_assign_bars(struct pau *pau)
> +{
> +	struct pau_dev *dev;
> +	uint64_t addr, size, val;
> +
> +	/* Global MMIO bar (per pau)
> +	 * 16M aligned address -> 0x1000000 (bit 24)
> +	 */
> +	phys_map_get(pau->chip_id, PAU_REGS, pau->index, &addr, &size);
> +	val = SETFIELD(PAU_MMIO_BAR_ADDR, 0ull, addr >> 24);
> +	val |= PAU_MMIO_BAR_ENABLE;
> +	pau_write(pau, PAU_MMIO_BAR, val);
> +
> +	PAUINF(pau, "MMIO base: 0x%016llx (%lldMB)\n", addr, size >> 20);
> +	pau->regs[0] = addr;
> +	pau->regs[1] = size;
> +
> +	/* NTL bar (per device)
> +	 * 64K aligned address -> 0x10000 (bit 16)
> +	 */
> +	pau_for_each_dev(dev, pau) {
> +		if (dev->type == PAU_DEV_TYPE_UNKNOWN)
> +			continue;
> +
> +		phys_map_get(pau->chip_id, PAU_OCAPI_MMIO,
> +			     pau_dev_index(dev, PAU_LINKS_OPENCAPI_PER_PAU),
> +			     &addr, &size);
> +
> +		val = SETFIELD(PAU_NTL_BAR_ADDR, 0ull, addr >> 16);
> +		val = SETFIELD(PAU_NTL_BAR_SIZE, val, ilog2(size >> 16));
> +		pau_write(pau, PAU_NTL_BAR(dev->index), val);
> +
> +		val = SETFIELD(PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR, 0ull, addr >> 16);
> +		val = SETFIELD(PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE, val, ilog2(size >> 16));
> +		pau_write(pau, PAU_CTL_MISC_MMIOPA_CONFIG(dev->index), val);
> +
> +		dev->ntl_bar.addr = addr;
> +		dev->ntl_bar.size = size;
> +	}
> +
> +	/* GENID bar (logically divided per device)
> +	 * 512K aligned address -> 0x80000 (bit 19)
> +	 */
> +	phys_map_get(pau->chip_id, PAU_GENID, pau->index, &addr, &size);
> +	val = SETFIELD(PAU_GENID_BAR_ADDR, 0ull, addr >> 19);
> +	pau_write(pau, PAU_GENID_BAR, val);
> +
> +	pau_for_each_dev(dev, pau) {
> +		if (dev->type == PAU_DEV_TYPE_UNKNOWN)
> +			continue;
> +
> +		dev->genid_bar.size = size;
> +		/* +320K = Bricks 0-4 Config Addr/Data registers */
> +		dev->genid_bar.cfg = addr + 0x50000;
> +	}
> +}
> +
> +static void pau_opencapi_init_hw(struct pau *pau)
> +{
> +	pau_opencapi_assign_bars(pau);
> +}
> +
> +static void pau_opencapi_init(struct pau *pau)
> +{
> +	if (!pau_next_dev(pau, NULL, PAU_DEV_TYPE_OPENCAPI))
> +		return;
> +
> +	assert(platform.ocapi);
> +
> +	pau_opencapi_init_hw(pau);
> +
> +	disable_fast_reboot("OpenCAPI device enabled");
> +}
> +
>   static void pau_init(struct pau *pau)
>   {
>   	struct pau_dev *dev;
> @@ -202,6 +276,7 @@ static void pau_init(struct pau *pau)
>   	pau_for_each_dev(dev, pau)
>   		pau_device_detect_fixup(dev);
> 
> +	pau_opencapi_init(pau);
>   }
> 
>   void probe_pau(void)
> diff --git a/hw/phys-map.c b/hw/phys-map.c
> index d6ff99fd..7b44fc61 100644
> --- a/hw/phys-map.c
> +++ b/hw/phys-map.c
> @@ -82,8 +82,44 @@ static const struct phys_map_entry phys_map_table_p10[] = {
>   	{ VAS_HYP_WIN    , 0, 0x00060302fe000000ull, 0x0000000002000000ull },
>   	{ VAS_USER_WIN   , 0, 0x0006030300000000ull, 0x0000000100000000ull },
> 
> -	/* TODO: MC, OCMB, PAU */
> +	/* TODO: MC, OCMB */
>   	{ RESV		 , 8, 0x0006030400000000ull, 0x000000f800000000ull },
> +	{ PAU_OCAPI_MMIO,  0, 0x0006038800000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  1, 0x0006039000000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  2, 0x0006039800000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  3, 0x000603a000000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  4, 0x000603a800000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  5, 0x000603b000000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  6, 0x000603b800000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  7, 0x000603c000000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  8, 0x000603c800000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO,  9, 0x000603d000000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO, 10, 0x000603d800000000ull, 0x0000000800000000ull },
> +	{ PAU_OCAPI_MMIO, 11, 0x000603e000000000ull, 0x0000000800000000ull },
> +	{ PAU_REGS,        0, 0x000603e800000000ull, 0x0000000001000000ull },
> +	{ PAU_REGS,        1, 0x000603e801000000ull, 0x0000000001000000ull },
> +	{ PAU_REGS,        2, 0x000603e802000000ull, 0x0000000001000000ull },
> +	{ PAU_REGS,        3, 0x000603e803000000ull, 0x0000000001000000ull },
> +	{ PAU_REGS,        4, 0x000603e804000000ull, 0x0000000001000000ull },
> +	{ PAU_REGS,        5, 0x000603e805000000ull, 0x0000000001000000ull },
> +	{ PAU_GENID,       0, 0x000603e806080000ull, 0x0000000000080000ull },
> +	{ PAU_GENID,       1, 0x000603e806180000ull, 0x0000000000080000ull },
> +	{ PAU_GENID,       2, 0x000603e806280000ull, 0x0000000000080000ull },
> +	{ PAU_GENID,       3, 0x000603e806380000ull, 0x0000000000080000ull },
> +	{ PAU_GENID,       4, 0x000603e806480000ull, 0x0000000000080000ull },
> +	{ PAU_GENID,       5, 0x000603e806580000ull, 0x0000000000080000ull },
> +	{ PAU_NTL,         0, 0x000603e806040000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         1, 0x000603e806060000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         2, 0x000603e806140000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         3, 0x000603e806160000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         4, 0x000603e806240000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         5, 0x000603e806260000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         6, 0x000603e806340000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         7, 0x000603e806360000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         8, 0x000603e806440000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,         9, 0x000603e806460000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,        10, 0x000603e806540000ull, 0x0000000000020000ull },
> +	{ PAU_NTL,        11, 0x000603e806560000ull, 0x0000000000020000ull },
>   	{ XSCOM          , 0, 0x000603fc00000000ull, 0x0000000400000000ull },
> 
>   	/* 4 TB offset */
> @@ -130,10 +166,10 @@ static const struct phys_map_entry phys_map_table_nimbus[] = {
>   	 *
>   	 * We don't currently support >4TB ranges.
>   	 */
> -	{ OCAPI_MEM,	   0, 0x0002000000000000ull, 0x0000040000000000ull },
> -	{ OCAPI_MEM,	   1, 0x0002800000000000ull, 0x0000040000000000ull },
> -	{ OCAPI_MEM,	   2, 0x0003000000000000ull, 0x0000040000000000ull },
> -	{ OCAPI_MEM,	   3, 0x0003800000000000ull, 0x0000040000000000ull },
> +	{ OCAPI_MEM,       0, 0x0002000000000000ull, 0x0000040000000000ull },
> +	{ OCAPI_MEM,       1, 0x0002800000000000ull, 0x0000040000000000ull },
> +	{ OCAPI_MEM,       2, 0x0003000000000000ull, 0x0000040000000000ull },
> +	{ OCAPI_MEM,       3, 0x0003800000000000ull, 0x0000040000000000ull },
> 
>   	/* 0 TB offset @ MMIO 0x0006000000000000ull */
>   	{ PHB4_64BIT_MMIO, 0, 0x0006000000000000ull, 0x0000004000000000ull },
> diff --git a/include/pau-regs.h b/include/pau-regs.h
> index a35668f1..afe6f958 100644
> --- a/include/pau-regs.h
> +++ b/include/pau-regs.h
> @@ -26,5 +26,31 @@
>   #define PAU_REG_OFFSET(reg)			((reg) & 0xffff)
> 
>   #define PAU_BLOCK_CQ_SM(n)			PAU_BLOCK(4, (n))
> +#define PAU_BLOCK_CQ_CTL			PAU_BLOCK(4, 4)
> +
> +/*
> + * CQ_SM block registers
> + *
> + * Definitions here use PAU_BLOCK_CQ_SM(0), but when pau_write() is given
> + * one of these, it will do corresponding writes to every CQ_SM block.
> + */
> +#define PAU_MCP_MISC_CFG0			(PAU_BLOCK_CQ_SM(0) + 0x000)
> +#define   PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP	PPC_BIT(9)
> +#define   PAU_MCP_MISC_CFG0_ENABLE_PBUS		PPC_BIT(26)
> +#define PAU_SNP_MISC_CFG0			(PAU_BLOCK_CQ_SM(0) + 0x180)
> +#define   PAU_SNP_MISC_CFG0_ENABLE_PBUS		PPC_BIT(2)
> +#define PAU_NTL_BAR(brk)			(PAU_BLOCK_CQ_SM(0) + 0x1b8 + (brk) * 8)
> +#define   PAU_NTL_BAR_ADDR			PPC_BITMASK(3, 35)
> +#define   PAU_NTL_BAR_SIZE			PPC_BITMASK(39, 43)
> +#define PAU_MMIO_BAR				(PAU_BLOCK_CQ_SM(0) + 0x1e0)
> +#define   PAU_MMIO_BAR_ENABLE			PPC_BIT(0)
> +#define   PAU_MMIO_BAR_ADDR			PPC_BITMASK(3, 27)
> +#define PAU_GENID_BAR				(PAU_BLOCK_CQ_SM(0) + 0x1e8)
> +#define   PAU_GENID_BAR_ADDR			PPC_BITMASK(3, 32)
> +
> +/* CQ_CTL block registers */
> +#define PAU_CTL_MISC_MMIOPA_CONFIG(brk)		(PAU_BLOCK_CQ_CTL + 0x098 + (brk) * 8)
> +#define   PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR	PPC_BITMASK(1, 35)
> +#define   PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE	PPC_BITMASK(39, 43)
> 
>   #endif /* __PAU_REGS_H */
> diff --git a/include/pau.h b/include/pau.h
> index d91ffa6d..05b27196 100644
> --- a/include/pau.h
> +++ b/include/pau.h
> @@ -19,11 +19,21 @@ enum pau_dev_type {
>   	PAU_DEV_TYPE_ANY = INT_MAX
>   };
> 
> +/* Used to expose a hardware BAR (or logical slice of it) outside skiboot */
> +struct pau_bar {
> +	uint64_t		addr;
> +	uint64_t		size;
> +	uint64_t		cfg;
> +};
> +
>   struct pau_dev {
>   	enum pau_dev_type	type;
>   	uint32_t		index;
>   	struct dt_node		*dn;
> 
> +	struct pau_bar		ntl_bar;
> +	struct pau_bar		genid_bar;
> +
>   	/* Associated I2C information */
>   	uint8_t			i2c_bus_id;
> 
> @@ -44,6 +54,7 @@ struct pau {
> 
>   	/* Global MMIO window (all PAU regs) */
>   	uint64_t		regs[2];
> +	bool			mmio_access;
> 
>   	struct lock		lock;
> 
> @@ -87,4 +98,72 @@ struct pau_dev *pau_next_dev(struct pau *pau, struct pau_dev *dev,
>   #define pau_for_each_dev(dev, pau) \
>   	pau_for_each_dev_type(dev, pau, PAU_DEV_TYPE_ANY)
> 
> +/*
> + * We use the indirect method because it uses the same addresses as
> + * the MMIO offsets (PAU RING)
> + */
> +static inline void pau_scom_sel(struct pau *pau, uint64_t reg,
> +				uint64_t size)
> +{
> +	uint64_t val;
> +
> +	val = SETFIELD(PAU_MISC_DA_ADDR, 0ull, reg);
> +	val = SETFIELD(PAU_MISC_DA_LEN, val, size);
> +	xscom_write(pau->chip_id,
> +		    pau->xscom_base + PAU_MISC_SCOM_IND_SCOM_ADDR,
> +		    val);
> +}
> +
> +static inline void pau_scom_write(struct pau *pau, uint64_t reg,
> +				  uint64_t size,
> +				  uint64_t val)
> +{
> +	pau_scom_sel(pau, reg, size);
> +	xscom_write(pau->chip_id,
> +		    pau->xscom_base + PAU_MISC_SCOM_IND_SCOM_DATA,
> +		    val);
> +}
> +
> +static inline uint64_t pau_scom_read(struct pau *pau, uint64_t reg,
> +				     uint64_t size)
> +{
> +	uint64_t val;
> +
> +	pau_scom_sel(pau, reg, size);
> +	xscom_read(pau->chip_id,
> +		   pau->xscom_base + PAU_MISC_SCOM_IND_SCOM_DATA,
> +		   &val);
> +
> +	return val;
> +}
> +
> +static inline void pau_write(struct pau *pau, uint64_t reg,
> +			     uint64_t val)
> +{
> +	void *mmio = (void *)pau->regs[0];
> +
> +	if (pau->mmio_access)
> +		out_be64(mmio + reg, val);
> +	else
> +		pau_scom_write(pau, reg, PAU_MISC_DA_LEN_8B, val);
> +
> +	/* CQ_SM writes should be mirrored in all four blocks */
> +	if (PAU_REG_BLOCK(reg) != PAU_BLOCK_CQ_SM(0))
> +		return;
> +
> +	for (uint32_t i = 1; i < 4; i++)
> +		pau_write(pau, PAU_BLOCK_CQ_SM(i) + PAU_REG_OFFSET(reg),
> +			   val);
> +}
> +
> +static inline uint64_t pau_read(struct pau *pau, uint64_t reg)
> +{
> +	void *mmio = (void *)pau->regs[0];
> +
> +	if (pau->mmio_access)
> +		return in_be64(mmio + reg);
> +
> +	return pau_scom_read(pau, reg, PAU_MISC_DA_LEN_8B);
> +}
> +
>   #endif /* __PAU_H */
> diff --git a/include/phys-map.h b/include/phys-map.h
> index 1dd337a5..a53bcd04 100644
> --- a/include/phys-map.h
> +++ b/include/phys-map.h
> @@ -51,6 +51,10 @@ enum phys_map_type {
>   	XIVE_NVPG,
>   	XIVE_ESB,
>   	XIVE_END,
> +	PAU_OCAPI_MMIO,
> +	PAU_REGS,
> +	PAU_GENID,
> +	PAU_NTL,
>   };
> 
>   extern void phys_map_get(uint64_t gcid, enum phys_map_type type,
> 


More information about the Skiboot mailing list