[Skiboot] [PATCH v3 05/10] npu2-opencapi: Configure NPU for OpenCAPI
Alistair Popple
alistair at popple.id.au
Tue Feb 6 15:37:09 AEDT 2018
Actually I think I get it. You're trying to figure out if this NPU should be in
nvlink or OpenCAPI mode based on what type of link nodes have been created
underneath the NPU root node.
Given we explicitly don't support a NPU with both OCAPI and NVLink active why
not just create the root npu@ node with a different compatible property? That
would make the initialisation flow much clearer as at present it looks like you
are probing the npu@ nodes twice and there is really nothing stopping both the
NVLink and OCAPI NPU inits running on the same system which is explicitly not
supported and could lead to all sorts of "fun".
- Alistair
On Tuesday, 6 February 2018 3:07:03 PM AEDT Alistair Popple wrote:
> > + dt_for_each_compatible(dn, np, "ibm,npu-link") {
> > + links++;
>
> Is there any reason can't you just read the existing ibm,npu-links property
> here?
>
> Also why does the existing code need to be reordered here?
>
> Thanks.
>
> - Alistair
>
> > + }
> > +
> > + index = dt_prop_get_u32(dn, "ibm,npu-index");
> > + phb_index = dt_prop_get_u32(dn, "ibm,phb-index");
> > +
> > + prlog(PR_INFO, "NPU2: Chip %d Found NPU2#%d (%d links) at %s\n",
> > + gcid, index, links, path);
> > + free(path);
> > +
> > + /* Retrieve scom base address */
> > + scom = dt_get_address(dn, 0, NULL);
> > + prlog(PR_INFO, " SCOM Base: %08x\n", scom);
> > +
> > + if (!links) {
> > + prlog(PR_INFO, " No NVLink links found\n");
> > + return;
> > + }
> > +
> > if (!is_p9dd1()) {
> > /* TODO: Clean this up with register names, etc. when we get
> > * time. This just turns NVLink mode on in each brick and should
> > @@ -1353,17 +1373,6 @@ static void npu2_probe_phb(struct dt_node *dn)
> > xscom_write_mask(gcid, 0x5011530, PPC_BIT(0), PPC_BIT(0));
> > }
> >
> > - index = dt_prop_get_u32(dn, "ibm,npu-index");
> > - phb_index = dt_prop_get_u32(dn, "ibm,phb-index");
> > - links = dt_prop_get_u32(dn, "ibm,npu-links");
> > - prlog(PR_INFO, "NPU2: Chip %d Found NPU2#%d (%d links) at %s\n",
> > - gcid, index, links, path);
> > - free(path);
> > -
> > - /* Retrieve scom base address */
> > - scom = dt_get_address(dn, 0, NULL);
> > - prlog(PR_INFO, " SCOM Base: %08x\n", scom);
> > -
> > /* Reassign the BARs */
> > assign_mmio_bars(gcid, scom, reg, mm_win);
> >
> > diff --git a/include/npu2-regs.h b/include/npu2-regs.h
> > index b6dae7b..9732ec9 100644
> > --- a/include/npu2-regs.h
> > +++ b/include/npu2-regs.h
> > @@ -90,6 +90,11 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_BLOCK_DAT 0x5
> > #define NPU2_BLOCK_NTL0 0x9
> > #define NPU2_BLOCK_NTL1 0xb
> > +#define NPU2_BLOCK_OTL0 0xc
> > +#define NPU2_BLOCK_OTL1 0xd
> > +
> > +/* OpenCAPI blocks */
> > +#define NPU2_BLOCK_XSL 0xe
> >
> > /* MISC stack blocks */
> > #define NPU2_BLOCK_ATS 0
> > @@ -108,6 +113,8 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >
> > /* SM block registers */
> > #define NPU2_CQ_SM_MISC_CFG0 0x000
> > +#define NPU2_CQ_SM_MISC_CFG0_CONFIG_ENABLE_PBUS PPC_BIT(38)
> > +#define NPU2_CQ_SM_MISC_CFG0_CONFIG_OCAPI_MODE PPC_BIT(57)
> > #define NPU2_CQ_SM_MISC_CFG1 0x008
> > #define NPU2_PB_EPSILON 0x010
> > #define NPU2_TIMER_CFG 0x018
> > @@ -138,6 +145,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_GENID_BAR_ADDR PPC_BITMASK(3,35)
> > #define NPU2_GENID_BAR_POISON PPC_BIT(39)
> > #define NPU2_LOW_WATER_MARKS 0x040
> > +#define NPU2_LOW_WATER_MARKS_ENABLE_MACHINE_ALLOC PPC_BIT(51)
> > #define NPU2_HIGH_WATER_MARKS 0x048
> > #define NPU2_RELAXED_ORDERING_CFG0 0x050
> > #define NPU2_RELAXED_ORDERING_CFG1 0x058
> > @@ -176,6 +184,9 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >
> > /* CTL block registers */
> > #define NPU2_CQ_CTL_MISC_CFG 0x000
> > +#define NPU2_CQ_CTL_MISC_CFG_CONFIG_OCAPI_MODE PPC_BIT(52)
> > +#define NPU2_CQ_CTL_MISC_CFG_CONFIG_OTL0_ENABLE PPC_BIT(55)
> > +#define NPU2_CQ_CTL_MISC_CFG_CONFIG_OTL1_ENABLE PPC_BIT(56)
> > #define NPU2_CQ_FUTURE_CFG1 0x008
> > #define NPU2_CQ_FUTURE_CFG2 0x010
> > #define NPU2_CQ_FUTURE_CFG3 0x018
> > @@ -198,6 +209,8 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_CQ_LPC_THRESHOLD_CFG 0x080
> > #define NPU2_CQ_INHIBIT_CFG 0x088
> > #define NPU2_CQ_CTL_STATUS 0x090
> > +#define NPU2_CQ_CTL_STATUS_BRK0_AM_FENCED PPC_BITMASK(48, 49)
> > +#define NPU2_CQ_CTL_STATUS_BRK1_AM_FENCED PPC_BITMASK(50, 51)
> > #define NPU2_CQ_C_ERR_RPT_MSG0 0x0C0
> > #define NPU2_CQ_C_ERR_RPT_MSG1 0x0C8
> > #define NPU2_CQ_C_ERR_RPT_FIRST0 0x0D0
> > @@ -206,9 +219,13 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_CQ_C_ERR_RPT_MASK1 0x0E8
> > #define NPU2_CQ_C_ERR_RPT_HOLD0 0x0F0
> > #define NPU2_CQ_C_ERR_RPT_HOLD1 0x0F8
> > +#define NPU2_CQ_CTL_FENCE_CONTROL_0 0x140
> > +#define NPU2_CQ_CTL_FENCE_CONTROL_1 0x148
> > +#define NPU2_CQ_CTL_FENCE_CONTROL_REQUEST_FENCE PPC_BITMASK(0, 1)
> >
> > /* DAT block registers */
> > #define NPU2_CQ_DAT_MISC_CFG 0x008
> > +#define NPU2_CQ_DAT_MISC_CFG_CONFIG_OCAPI_MODE PPC_BIT(40)
> > #define NPU2_CQ_DAT_ECC_CFG 0x010
> > #define NPU2_CQ_DAT_SCRATCH0 0x018
> > #define NPU2_CQ_DAT_ECC_STATUS 0x020
> > @@ -289,6 +306,43 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_NTL_DL_CONFIG(ndev) NPU2_DL_REG_OFFSET(ndev, 0xFFF8)
> > #define NPU2_NTL_DL_CLK_CTRL(ndev) NPU2_DL_REG_OFFSET(ndev, 0x001C)
> >
> > +/* OpenCAPI - XSL registers */
> > +#define NPU2_XSL_PSL_SCNTL_A0 0x010
> > +#define NPU2_XSL_PSL_SCNTL_A0_MULTI_AFU_DIAL PPC_BIT(0)
> > +#define NPU2_XSL_DEF 0x040
> > +#define NPU2_XSL_GP 0x058
> > +#define NPU2_XSL_GP_BLOOM_FILTER_ENABLE PPC_BIT(16)
> > +#define NPU2_XSL_WRAP_CFG 0x0C0
> > +#define NPU2_XSL_WRAP_CFG_XSLO_CLOCK_ENABLE PPC_BIT(0)
> > +
> > +/* OpenCAPI - OTL registers */
> > +#define NPU2_OTL_CONFIG0(stack, block) NPU2_REG_OFFSET(stack, block, 0x000)
> > +#define NPU2_OTL_CONFIG0_EN PPC_BIT(0)
> > +#define NPU2_OTL_CONFIG0_BLOCK_PE_HANDLE PPC_BIT(1)
> > +#define NPU2_OTL_CONFIG0_BRICKID PPC_BITMASK(2, 3)
> > +#define NPU2_OTL_CONFIG0_PE_MASK PPC_BITMASK(4, 7)
> > +#define NPU2_OTL_CONFIG0_ERAT_HASH_0 PPC_BITMASK(8, 13)
> > +#define NPU2_OTL_CONFIG0_ERAT_HASH_1 PPC_BITMASK(14, 19)
> > +#define NPU2_OTL_CONFIG0_ERAT_HASH_2 PPC_BITMASK(20, 25)
> > +#define NPU2_OTL_CONFIG0_ERAT_HASH_3 PPC_BITMASK(26, 31)
> > +#define NPU2_OTL_CONFIG0_BLOCK_TID_OVERRIDE PPC_BIT(50)
> > +#define NPU2_OTL_TLX_CREDITS(stack, block) NPU2_REG_OFFSET(stack, block, 0x050)
> > +#define NPU2_OTL_TLX_CREDITS_VC0_CREDITS PPC_BITMASK(0, 7)
> > +#define NPU2_OTL_TLX_CREDITS_VC3_CREDITS PPC_BITMASK(24, 31)
> > +#define NPU2_OTL_TLX_CREDITS_DCP0_CREDITS PPC_BITMASK(32, 39)
> > +#define NPU2_OTL_TLX_CREDITS_DCP1_CREDITS PPC_BITMASK(56, 63)
> > +#define NPU2_OTL_CONFIG1(stack, block) NPU2_REG_OFFSET(stack, block, 0x058)
> > +#define NPU2_OTL_CONFIG1_TX_DRDY_WAIT PPC_BITMASK(5, 7)
> > +#define NPU2_OTL_CONFIG1_TX_TEMP0_RATE PPC_BITMASK(8, 11)
> > +#define NPU2_OTL_CONFIG1_TX_CRET_FREQ PPC_BITMASK(32, 34)
> > +#define NPU2_OTL_CONFIG1_TX_AGE_FREQ PPC_BITMASK(35, 39)
> > +#define NPU2_OTL_CONFIG1_TX_RS2_HPWAIT PPC_BITMASK(40, 45)
> > +#define NPU2_OTL_CONFIG1_TX_RQ4_HPWAIT PPC_BITMASK(46, 51)
> > +#define NPU2_OTL_CONFIG1_TX_RQ6_HPWAIT PPC_BITMASK(52, 57)
> > +#define NPU2_OTL_CONFIG1_TX_CBUF_ECC_DIS PPC_BIT(58)
> > +#define NPU2_OTL_CONFIG1_TX_STOP_LINK PPC_BIT(59)
> > +#define NPU2_OTL_CONFIG1_TX_STOP_ON_UE PPC_BIT(60)
> > +
> > /* Misc block registers. Unlike the SM/CTL/DAT/NTL registers above
> > * there is only a single instance of each of these in the NPU so we
> > * define them as absolute offsets. */
> > @@ -296,6 +350,9 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_MISC_RELAXTED_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x008)
> > #define NPU2_MISC_LCO_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x010)
> > #define NPU2_MISC_OPTICAL_IO_CFG0 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x018)
> > +#define NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2 PPC_BITMASK(0, 2)
> > +#define NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1 PPC_BITMASK(3, 4)
> > +#define NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5 PPC_BITMASK(5, 6)
> > #define NPU2_MISC_ERR_RPT_HOLD NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x020)
> > #define NPU2_MISC_ERR_RPT_MASK NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x028)
> > #define NPU2_MISC_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x030)
> > @@ -320,6 +377,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_MISC_ERR_CTL_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x088)
> > #define NPU2_MISC_RING_ACK_ERR NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x090)
> > #define NPU2_MISC_IRQ_BASE NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x098)
> > +#define NPU2_MISC_IRQ_BASE_MASK PPC_BITMASK(13, 51)
> > #define NPU2_MISC_ERR_BRICK_GROUP NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0A0)
> > #define NPU2_MISC_FREEZE_STATE NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0A8)
> > #define NPU2_MISC_FENCE_STATE NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0B0)
> > @@ -362,6 +420,10 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_MISC_IRQ_LOG13 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x368)
> > #define NPU2_MISC_IRQ_LOG14 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x370)
> > #define NPU2_MISC_IRQ_LOG15 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x378)
> > +#define NPU2_MISC_IRQ_ENABLE2 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x408)
> > +
> > +/* Misc register, direct access only */
> > +#define NPU2_MISC_FIR_MASK1 0x2C43
> >
> > /* ATS block registers */
> > #define NPU2_ATS_PMU_CTL NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_ATS, 0x000)
> > @@ -406,8 +468,11 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_XTS_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x020)
> > #define NPU2_XTS_CFG_MMIOSD PPC_BIT(1)
> > #define NPU2_XTS_CFG_TRY_ATR_RO PPC_BIT(6)
> > +#define NPU2_XTS_CFG_OPENCAPI PPC_BIT(15)
> > #define NPU2_XTS_CFG2 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x028)
> > #define NPU2_XTS_CFG2_NO_FLUSH_ENA PPC_BIT(49)
> > +#define NPU2_XTS_CFG2_XSL1_ENA PPC_BIT(54)
> > +#define NPU2_XTS_CFG2_XSL2_ENA PPC_BIT(55)
> > #define NPU2_XTS_DBG_CFG0 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x030)
> > #define NPU2_XTS_DBG_CFG1 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x038)
> > #define NPU2_XTS_PMU_CNT NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x040)
> > @@ -463,4 +528,29 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> > #define NPU2_DD1_MISC_SCOM_IND_SCOM_DATA 0x38f
> > #define NPU2_MISC_SCOM_IND_SCOM_DATA 0x68f
> >
> > +
> > +/* OpenCAPI - PowerBus and OBus configuration SCOM addresses */
> > +#define PU_IOE_PB_MISC_CFG 0x5013823
> > +#define PU_IOE_PB_MISC_CFG_SEL_03_NPU_NOT_PB PPC_BIT(13)
> > +#define PU_IOE_PB_MISC_CFG_SEL_04_NPU_NOT_PB PPC_BIT(14)
> > +#define PU_IOE_PB_MISC_CFG_SEL_05_NPU_NOT_PB PPC_BIT(15)
> > +#define OBUS_LL0_IOOL_PHY_CONFIG 0x901080C
> > +#define OBUS_LL3_IOOL_PHY_CONFIG 0xC01080C
> > +#define OBUS_IOOL_PHY_CONFIG_ODL0_ENABLED PPC_BIT(52)
> > +#define OBUS_IOOL_PHY_CONFIG_ODL1_ENABLED PPC_BIT(53)
> > +#define OBUS_IOOL_PHY_CONFIG_ODL_PHY_SWAP PPC_BIT(54)
> > +#define OBUS_IOOL_PHY_CONFIG_LINK0_OLL_ENABLED PPC_BIT(58)
> > +#define OBUS_IOOL_PHY_CONFIG_LINK1_OLL_ENABLED PPC_BIT(59)
> > +#define OBUS_IOOL_PHY_CONFIG_NV0_NPU_ENABLED PPC_BIT(61)
> > +#define OBUS_IOOL_PHY_CONFIG_NV1_NPU_ENABLED PPC_BIT(62)
> > +#define OBUS_IOOL_PHY_CONFIG_NV2_NPU_ENABLED PPC_BIT(63)
> > +#define PU_IOE_PB_FP01_CFG 0x501380A
> > +#define PU_IOE_PB_FP23_CFG 0x501380B
> > +#define PU_IOE_PB_FP45_CFG 0x501380C
> > +#define PU_IOE_PB_FP67_CFG 0x501380D
> > +#define PU_IOE_PB_FP_CFG_FP0_FMR_DISABLE PPC_BIT(20)
> > +#define PU_IOE_PB_FP_CFG_FP0_PRS_DISABLE PPC_BIT(25)
> > +#define PU_IOE_PB_FP_CFG_FP1_FMR_DISABLE PPC_BIT(52)
> > +#define PU_IOE_PB_FP_CFG_FP1_PRS_DISABLE PPC_BIT(57)
> > +
> > #endif /* __NPU2_REGS_H */
> > diff --git a/include/npu2.h b/include/npu2.h
> > index 94e537b..7b2b509 100644
> > --- a/include/npu2.h
> > +++ b/include/npu2.h
> > @@ -138,6 +138,7 @@ struct npu2_dev {
> >
> > struct npu2 {
> > uint32_t index;
> > + struct dt_node *dt_node;
> > uint32_t flags;
> > uint32_t chip_id;
> > uint64_t xscom_base;
> > @@ -146,6 +147,7 @@ struct npu2 {
> > uint64_t mm_base;
> > uint64_t mm_size;
> > uint32_t base_lsi;
> > + uint32_t irq_base;
> > uint32_t total_devices;
> > struct npu2_dev *devices;
> > enum phys_map_type gpu_map_type;
> > diff --git a/include/skiboot.h b/include/skiboot.h
> > index e94f212..1218367 100644
> > --- a/include/skiboot.h
> > +++ b/include/skiboot.h
> > @@ -232,6 +232,7 @@ extern int preload_capp_ucode(void);
> > extern void preload_io_vpd(void);
> > extern void probe_npu(void);
> > extern void probe_npu2(void);
> > +extern void probe_npu2_opencapi(void);
> > extern void uart_init(void);
> > extern void mbox_init(void);
> > extern void early_uart_init(void);
> >
>
>
More information about the Skiboot
mailing list