[Skiboot] [PATCH v3 05/10] npu2-opencapi: Configure NPU for OpenCAPI

Alistair Popple alistair at popple.id.au
Tue Feb 6 15:37:09 AEDT 2018


Actually I think I get it. You're trying to figure out if this NPU should be in
nvlink or OpenCAPI mode based on what type of link nodes have been created
underneath the NPU root node.

Given we explicitly don't support a NPU with both OCAPI and NVLink active why
not just create the root npu@ node with a different compatible property? That
would make the initialisation flow much clearer as at present it looks like you
are probing the npu@ nodes twice and there is really nothing stopping both the
NVLink and OCAPI NPU inits running on the same system which is explicitly not
supported and could lead to all sorts of "fun".

- Alistair

On Tuesday, 6 February 2018 3:07:03 PM AEDT Alistair Popple wrote:
> > +	dt_for_each_compatible(dn, np, "ibm,npu-link") {
> > +		links++;
> 
> Is there any reason can't you just read the existing ibm,npu-links property
> here?
> 
> Also why does the existing code need to be reordered here?
> 
> Thanks.
> 
> - Alistair
> 
> > +	}
> > +
> > +	index = dt_prop_get_u32(dn, "ibm,npu-index");
> > +	phb_index = dt_prop_get_u32(dn, "ibm,phb-index");
> > +
> > +	prlog(PR_INFO, "NPU2: Chip %d Found NPU2#%d (%d links) at %s\n",
> > +	      gcid, index, links, path);
> > +	free(path);
> > +
> > +	/* Retrieve scom base address */
> > +	scom = dt_get_address(dn, 0, NULL);
> > +	prlog(PR_INFO, "   SCOM Base:  %08x\n", scom);
> > +
> > +	if (!links) {
> > +		prlog(PR_INFO, "   No NVLink links found\n");
> > +		return;
> > +	}
> > +
> >  	if (!is_p9dd1()) {
> >  		/* TODO: Clean this up with register names, etc. when we get
> >  		 * time. This just turns NVLink mode on in each brick and should
> > @@ -1353,17 +1373,6 @@ static void npu2_probe_phb(struct dt_node *dn)
> >  		xscom_write_mask(gcid, 0x5011530, PPC_BIT(0), PPC_BIT(0));
> >  	}
> >  
> > -	index = dt_prop_get_u32(dn, "ibm,npu-index");
> > -	phb_index = dt_prop_get_u32(dn, "ibm,phb-index");
> > -	links = dt_prop_get_u32(dn, "ibm,npu-links");
> > -	prlog(PR_INFO, "NPU2: Chip %d Found NPU2#%d (%d links) at %s\n",
> > -	      gcid, index, links, path);
> > -	free(path);
> > -
> > -	/* Retrieve scom base address */
> > -	scom = dt_get_address(dn, 0, NULL);
> > -	prlog(PR_INFO, "   SCOM Base:  %08x\n", scom);
> > -
> >  	/* Reassign the BARs */
> >  	assign_mmio_bars(gcid, scom, reg, mm_win);
> >  
> > diff --git a/include/npu2-regs.h b/include/npu2-regs.h
> > index b6dae7b..9732ec9 100644
> > --- a/include/npu2-regs.h
> > +++ b/include/npu2-regs.h
> > @@ -90,6 +90,11 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_BLOCK_DAT	0x5
> >  #define NPU2_BLOCK_NTL0	0x9
> >  #define NPU2_BLOCK_NTL1	0xb
> > +#define NPU2_BLOCK_OTL0 0xc
> > +#define NPU2_BLOCK_OTL1 0xd
> > +
> > +/* OpenCAPI blocks */
> > +#define NPU2_BLOCK_XSL 0xe
> >  
> >  /* MISC stack blocks */
> >  #define NPU2_BLOCK_ATS	0
> > @@ -108,6 +113,8 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  
> >  /* SM block registers */
> >  #define NPU2_CQ_SM_MISC_CFG0			0x000
> > +#define   NPU2_CQ_SM_MISC_CFG0_CONFIG_ENABLE_PBUS PPC_BIT(38)
> > +#define   NPU2_CQ_SM_MISC_CFG0_CONFIG_OCAPI_MODE PPC_BIT(57)
> >  #define NPU2_CQ_SM_MISC_CFG1			0x008
> >  #define NPU2_PB_EPSILON				0x010
> >  #define NPU2_TIMER_CFG				0x018
> > @@ -138,6 +145,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define   NPU2_GENID_BAR_ADDR			PPC_BITMASK(3,35)
> >  #define   NPU2_GENID_BAR_POISON			PPC_BIT(39)
> >  #define NPU2_LOW_WATER_MARKS			0x040
> > +#define   NPU2_LOW_WATER_MARKS_ENABLE_MACHINE_ALLOC PPC_BIT(51)
> >  #define NPU2_HIGH_WATER_MARKS			0x048
> >  #define NPU2_RELAXED_ORDERING_CFG0		0x050
> >  #define NPU2_RELAXED_ORDERING_CFG1		0x058
> > @@ -176,6 +184,9 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  
> >  /* CTL block registers */
> >  #define NPU2_CQ_CTL_MISC_CFG			0x000
> > +#define   NPU2_CQ_CTL_MISC_CFG_CONFIG_OCAPI_MODE PPC_BIT(52)
> > +#define   NPU2_CQ_CTL_MISC_CFG_CONFIG_OTL0_ENABLE PPC_BIT(55)
> > +#define   NPU2_CQ_CTL_MISC_CFG_CONFIG_OTL1_ENABLE PPC_BIT(56)
> >  #define NPU2_CQ_FUTURE_CFG1			0x008
> >  #define NPU2_CQ_FUTURE_CFG2			0x010
> >  #define NPU2_CQ_FUTURE_CFG3			0x018
> > @@ -198,6 +209,8 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_CQ_LPC_THRESHOLD_CFG		0x080
> >  #define NPU2_CQ_INHIBIT_CFG			0x088
> >  #define NPU2_CQ_CTL_STATUS			0x090
> > +#define   NPU2_CQ_CTL_STATUS_BRK0_AM_FENCED	PPC_BITMASK(48, 49)
> > +#define   NPU2_CQ_CTL_STATUS_BRK1_AM_FENCED	PPC_BITMASK(50, 51)
> >  #define NPU2_CQ_C_ERR_RPT_MSG0			0x0C0
> >  #define NPU2_CQ_C_ERR_RPT_MSG1			0x0C8
> >  #define NPU2_CQ_C_ERR_RPT_FIRST0		0x0D0
> > @@ -206,9 +219,13 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_CQ_C_ERR_RPT_MASK1			0x0E8
> >  #define NPU2_CQ_C_ERR_RPT_HOLD0			0x0F0
> >  #define NPU2_CQ_C_ERR_RPT_HOLD1			0x0F8
> > +#define NPU2_CQ_CTL_FENCE_CONTROL_0		0x140
> > +#define NPU2_CQ_CTL_FENCE_CONTROL_1		0x148
> > +#define   NPU2_CQ_CTL_FENCE_CONTROL_REQUEST_FENCE PPC_BITMASK(0, 1)
> >  
> >  /* DAT block registers */
> >  #define NPU2_CQ_DAT_MISC_CFG			0x008
> > +#define	  NPU2_CQ_DAT_MISC_CFG_CONFIG_OCAPI_MODE PPC_BIT(40)
> >  #define NPU2_CQ_DAT_ECC_CFG			0x010
> >  #define NPU2_CQ_DAT_SCRATCH0			0x018
> >  #define NPU2_CQ_DAT_ECC_STATUS			0x020
> > @@ -289,6 +306,43 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_NTL_DL_CONFIG(ndev)		NPU2_DL_REG_OFFSET(ndev, 0xFFF8)
> >  #define NPU2_NTL_DL_CLK_CTRL(ndev)		NPU2_DL_REG_OFFSET(ndev, 0x001C)
> >  
> > +/* OpenCAPI - XSL registers */
> > +#define NPU2_XSL_PSL_SCNTL_A0			0x010
> > +#define   NPU2_XSL_PSL_SCNTL_A0_MULTI_AFU_DIAL	PPC_BIT(0)
> > +#define NPU2_XSL_DEF				0x040
> > +#define NPU2_XSL_GP				0x058
> > +#define   NPU2_XSL_GP_BLOOM_FILTER_ENABLE	PPC_BIT(16)
> > +#define NPU2_XSL_WRAP_CFG			0x0C0
> > +#define   NPU2_XSL_WRAP_CFG_XSLO_CLOCK_ENABLE	PPC_BIT(0)
> > +
> > +/* OpenCAPI - OTL registers */
> > +#define NPU2_OTL_CONFIG0(stack, block)		NPU2_REG_OFFSET(stack, block, 0x000)
> > +#define   NPU2_OTL_CONFIG0_EN			PPC_BIT(0)
> > +#define   NPU2_OTL_CONFIG0_BLOCK_PE_HANDLE	PPC_BIT(1)
> > +#define   NPU2_OTL_CONFIG0_BRICKID		PPC_BITMASK(2, 3)
> > +#define   NPU2_OTL_CONFIG0_PE_MASK		PPC_BITMASK(4, 7)
> > +#define   NPU2_OTL_CONFIG0_ERAT_HASH_0		PPC_BITMASK(8, 13)
> > +#define   NPU2_OTL_CONFIG0_ERAT_HASH_1		PPC_BITMASK(14, 19)
> > +#define   NPU2_OTL_CONFIG0_ERAT_HASH_2		PPC_BITMASK(20, 25)
> > +#define   NPU2_OTL_CONFIG0_ERAT_HASH_3		PPC_BITMASK(26, 31)
> > +#define   NPU2_OTL_CONFIG0_BLOCK_TID_OVERRIDE	PPC_BIT(50)
> > +#define NPU2_OTL_TLX_CREDITS(stack, block)	NPU2_REG_OFFSET(stack, block, 0x050)
> > +#define   NPU2_OTL_TLX_CREDITS_VC0_CREDITS	PPC_BITMASK(0, 7)
> > +#define   NPU2_OTL_TLX_CREDITS_VC3_CREDITS	PPC_BITMASK(24, 31)
> > +#define   NPU2_OTL_TLX_CREDITS_DCP0_CREDITS	PPC_BITMASK(32, 39)
> > +#define   NPU2_OTL_TLX_CREDITS_DCP1_CREDITS	PPC_BITMASK(56, 63)
> > +#define NPU2_OTL_CONFIG1(stack, block)		NPU2_REG_OFFSET(stack, block, 0x058)
> > +#define   NPU2_OTL_CONFIG1_TX_DRDY_WAIT		PPC_BITMASK(5, 7)
> > +#define   NPU2_OTL_CONFIG1_TX_TEMP0_RATE	PPC_BITMASK(8, 11)
> > +#define   NPU2_OTL_CONFIG1_TX_CRET_FREQ 	PPC_BITMASK(32, 34)
> > +#define   NPU2_OTL_CONFIG1_TX_AGE_FREQ		PPC_BITMASK(35, 39)
> > +#define   NPU2_OTL_CONFIG1_TX_RS2_HPWAIT	PPC_BITMASK(40, 45)
> > +#define   NPU2_OTL_CONFIG1_TX_RQ4_HPWAIT	PPC_BITMASK(46, 51)
> > +#define   NPU2_OTL_CONFIG1_TX_RQ6_HPWAIT	PPC_BITMASK(52, 57)
> > +#define   NPU2_OTL_CONFIG1_TX_CBUF_ECC_DIS	PPC_BIT(58)
> > +#define   NPU2_OTL_CONFIG1_TX_STOP_LINK		PPC_BIT(59)
> > +#define   NPU2_OTL_CONFIG1_TX_STOP_ON_UE	PPC_BIT(60)
> > +
> >  /* Misc block registers. Unlike the SM/CTL/DAT/NTL registers above
> >   * there is only a single instance of each of these in the NPU so we
> >   * define them as absolute offsets. */
> > @@ -296,6 +350,9 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_MISC_RELAXTED_CFG			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x008)
> >  #define NPU2_MISC_LCO_CFG			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x010)
> >  #define NPU2_MISC_OPTICAL_IO_CFG0		NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x018)
> > +#define   NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2 PPC_BITMASK(0, 2)
> > +#define   NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1 PPC_BITMASK(3, 4)
> > +#define   NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5 PPC_BITMASK(5, 6)
> >  #define NPU2_MISC_ERR_RPT_HOLD			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x020)
> >  #define NPU2_MISC_ERR_RPT_MASK			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x028)
> >  #define NPU2_MISC_CFG				NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x030)
> > @@ -320,6 +377,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_MISC_ERR_CTL_CFG			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x088)
> >  #define NPU2_MISC_RING_ACK_ERR			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x090)
> >  #define NPU2_MISC_IRQ_BASE			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x098)
> > +#define   NPU2_MISC_IRQ_BASE_MASK		PPC_BITMASK(13, 51)
> >  #define NPU2_MISC_ERR_BRICK_GROUP		NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0A0)
> >  #define NPU2_MISC_FREEZE_STATE			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0A8)
> >  #define NPU2_MISC_FENCE_STATE			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0B0)
> > @@ -362,6 +420,10 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_MISC_IRQ_LOG13			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x368)
> >  #define NPU2_MISC_IRQ_LOG14			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x370)
> >  #define NPU2_MISC_IRQ_LOG15			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x378)
> > +#define NPU2_MISC_IRQ_ENABLE2			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x408)
> > +
> > +/* Misc register, direct access only */
> > +#define NPU2_MISC_FIR_MASK1		0x2C43
> >  
> >  /* ATS block registers */
> >  #define NPU2_ATS_PMU_CTL			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_ATS, 0x000)
> > @@ -406,8 +468,11 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_XTS_CFG				NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x020)
> >  #define   NPU2_XTS_CFG_MMIOSD			PPC_BIT(1)
> >  #define   NPU2_XTS_CFG_TRY_ATR_RO		PPC_BIT(6)
> > +#define   NPU2_XTS_CFG_OPENCAPI			PPC_BIT(15)
> >  #define NPU2_XTS_CFG2				NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x028)
> >  #define   NPU2_XTS_CFG2_NO_FLUSH_ENA		PPC_BIT(49)
> > +#define   NPU2_XTS_CFG2_XSL1_ENA		PPC_BIT(54)
> > +#define   NPU2_XTS_CFG2_XSL2_ENA		PPC_BIT(55)
> >  #define NPU2_XTS_DBG_CFG0			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x030)
> >  #define NPU2_XTS_DBG_CFG1			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x038)
> >  #define NPU2_XTS_PMU_CNT			NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x040)
> > @@ -463,4 +528,29 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >  #define NPU2_DD1_MISC_SCOM_IND_SCOM_DATA	0x38f
> >  #define NPU2_MISC_SCOM_IND_SCOM_DATA		0x68f
> >  
> > +
> > +/* OpenCAPI - PowerBus and OBus configuration SCOM addresses */
> > +#define PU_IOE_PB_MISC_CFG			0x5013823
> > +#define   PU_IOE_PB_MISC_CFG_SEL_03_NPU_NOT_PB	PPC_BIT(13)
> > +#define   PU_IOE_PB_MISC_CFG_SEL_04_NPU_NOT_PB	PPC_BIT(14)
> > +#define   PU_IOE_PB_MISC_CFG_SEL_05_NPU_NOT_PB	PPC_BIT(15)
> > +#define OBUS_LL0_IOOL_PHY_CONFIG		0x901080C
> > +#define OBUS_LL3_IOOL_PHY_CONFIG		0xC01080C
> > +#define   OBUS_IOOL_PHY_CONFIG_ODL0_ENABLED	PPC_BIT(52)
> > +#define   OBUS_IOOL_PHY_CONFIG_ODL1_ENABLED	PPC_BIT(53)
> > +#define   OBUS_IOOL_PHY_CONFIG_ODL_PHY_SWAP	PPC_BIT(54)
> > +#define   OBUS_IOOL_PHY_CONFIG_LINK0_OLL_ENABLED PPC_BIT(58)
> > +#define   OBUS_IOOL_PHY_CONFIG_LINK1_OLL_ENABLED PPC_BIT(59)
> > +#define   OBUS_IOOL_PHY_CONFIG_NV0_NPU_ENABLED	PPC_BIT(61)
> > +#define   OBUS_IOOL_PHY_CONFIG_NV1_NPU_ENABLED	PPC_BIT(62)
> > +#define   OBUS_IOOL_PHY_CONFIG_NV2_NPU_ENABLED	PPC_BIT(63)
> > +#define PU_IOE_PB_FP01_CFG			0x501380A
> > +#define PU_IOE_PB_FP23_CFG			0x501380B
> > +#define PU_IOE_PB_FP45_CFG			0x501380C
> > +#define PU_IOE_PB_FP67_CFG			0x501380D
> > +#define    PU_IOE_PB_FP_CFG_FP0_FMR_DISABLE	PPC_BIT(20)
> > +#define    PU_IOE_PB_FP_CFG_FP0_PRS_DISABLE	PPC_BIT(25)
> > +#define    PU_IOE_PB_FP_CFG_FP1_FMR_DISABLE	PPC_BIT(52)
> > +#define    PU_IOE_PB_FP_CFG_FP1_PRS_DISABLE	PPC_BIT(57)
> > +
> >  #endif /* __NPU2_REGS_H */
> > diff --git a/include/npu2.h b/include/npu2.h
> > index 94e537b..7b2b509 100644
> > --- a/include/npu2.h
> > +++ b/include/npu2.h
> > @@ -138,6 +138,7 @@ struct npu2_dev {
> >  
> >  struct npu2 {
> >  	uint32_t	index;
> > +	struct dt_node	*dt_node;
> >  	uint32_t	flags;
> >  	uint32_t	chip_id;
> >  	uint64_t	xscom_base;
> > @@ -146,6 +147,7 @@ struct npu2 {
> >  	uint64_t	mm_base;
> >  	uint64_t	mm_size;
> >  	uint32_t	base_lsi;
> > +	uint32_t	irq_base;
> >  	uint32_t	total_devices;
> >  	struct npu2_dev	*devices;
> >  	enum phys_map_type gpu_map_type;
> > diff --git a/include/skiboot.h b/include/skiboot.h
> > index e94f212..1218367 100644
> > --- a/include/skiboot.h
> > +++ b/include/skiboot.h
> > @@ -232,6 +232,7 @@ extern int preload_capp_ucode(void);
> >  extern void preload_io_vpd(void);
> >  extern void probe_npu(void);
> >  extern void probe_npu2(void);
> > +extern void probe_npu2_opencapi(void);
> >  extern void uart_init(void);
> >  extern void mbox_init(void);
> >  extern void early_uart_init(void);
> > 
> 
> 




More information about the Skiboot mailing list