[Skiboot] [PATCH v3 05/10] npu2-opencapi: Configure NPU for OpenCAPI
Alistair Popple
alistair at popple.id.au
Tue Feb 6 16:57:32 AEDT 2018
On Tuesday, 6 February 2018 4:08:31 PM AEDT Andrew Donnellan wrote:
> On 06/02/18 15:07, Alistair Popple wrote:
> >> + dt_for_each_compatible(dn, np, "ibm,npu-link") {
> >> + links++;
> >
> > Is there any reason can't you just read the existing ibm,npu-links property
> > here?
>
> There was definitely a super good reason why I did this...
>
> Ah, yes. It's because I'm matching NVLink links via the compatible
> string "ibm,npu-link" and OpenCAPI links as "ibm,npu-link-opencapi".
>
> > Also why does the existing code need to be reordered here?
>
> So we can abort if there are no NVLink links.
>
> This will all change for mixed setups...
>
> Perhaps the more sensible way to do what I'm doing here would be to
> explicitly scan for any nodes matching "ibm,npu-link-opencapi" and back
> out, rather than checking to see if there are 0 "ibm,npu-link"s there...
Yep, I think that would be preferable. It would at least avoid a situation where
we try to init the NPU twice if someone accidently tries mixed mode.
- Alistair
> >
> > Thanks.
> >
> > - Alistair
> >
> >> + }
> >> +
> >> + index = dt_prop_get_u32(dn, "ibm,npu-index");
> >> + phb_index = dt_prop_get_u32(dn, "ibm,phb-index");
> >> +
> >> + prlog(PR_INFO, "NPU2: Chip %d Found NPU2#%d (%d links) at %s\n",
> >> + gcid, index, links, path);
> >> + free(path);
> >> +
> >> + /* Retrieve scom base address */
> >> + scom = dt_get_address(dn, 0, NULL);
> >> + prlog(PR_INFO, " SCOM Base: %08x\n", scom);
> >> +
> >> + if (!links) {
> >> + prlog(PR_INFO, " No NVLink links found\n");
> >> + return;
> >> + }
> >> +
> >> if (!is_p9dd1()) {
> >> /* TODO: Clean this up with register names, etc. when we get
> >> * time. This just turns NVLink mode on in each brick and should
> >> @@ -1353,17 +1373,6 @@ static void npu2_probe_phb(struct dt_node *dn)
> >> xscom_write_mask(gcid, 0x5011530, PPC_BIT(0), PPC_BIT(0));
> >> }
> >>
> >> - index = dt_prop_get_u32(dn, "ibm,npu-index");
> >> - phb_index = dt_prop_get_u32(dn, "ibm,phb-index");
> >> - links = dt_prop_get_u32(dn, "ibm,npu-links");
> >> - prlog(PR_INFO, "NPU2: Chip %d Found NPU2#%d (%d links) at %s\n",
> >> - gcid, index, links, path);
> >> - free(path);
> >> -
> >> - /* Retrieve scom base address */
> >> - scom = dt_get_address(dn, 0, NULL);
> >> - prlog(PR_INFO, " SCOM Base: %08x\n", scom);
> >> -
> >> /* Reassign the BARs */
> >> assign_mmio_bars(gcid, scom, reg, mm_win);
> >>
> >> diff --git a/include/npu2-regs.h b/include/npu2-regs.h
> >> index b6dae7b..9732ec9 100644
> >> --- a/include/npu2-regs.h
> >> +++ b/include/npu2-regs.h
> >> @@ -90,6 +90,11 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_BLOCK_DAT 0x5
> >> #define NPU2_BLOCK_NTL0 0x9
> >> #define NPU2_BLOCK_NTL1 0xb
> >> +#define NPU2_BLOCK_OTL0 0xc
> >> +#define NPU2_BLOCK_OTL1 0xd
> >> +
> >> +/* OpenCAPI blocks */
> >> +#define NPU2_BLOCK_XSL 0xe
> >>
> >> /* MISC stack blocks */
> >> #define NPU2_BLOCK_ATS 0
> >> @@ -108,6 +113,8 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >>
> >> /* SM block registers */
> >> #define NPU2_CQ_SM_MISC_CFG0 0x000
> >> +#define NPU2_CQ_SM_MISC_CFG0_CONFIG_ENABLE_PBUS PPC_BIT(38)
> >> +#define NPU2_CQ_SM_MISC_CFG0_CONFIG_OCAPI_MODE PPC_BIT(57)
> >> #define NPU2_CQ_SM_MISC_CFG1 0x008
> >> #define NPU2_PB_EPSILON 0x010
> >> #define NPU2_TIMER_CFG 0x018
> >> @@ -138,6 +145,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_GENID_BAR_ADDR PPC_BITMASK(3,35)
> >> #define NPU2_GENID_BAR_POISON PPC_BIT(39)
> >> #define NPU2_LOW_WATER_MARKS 0x040
> >> +#define NPU2_LOW_WATER_MARKS_ENABLE_MACHINE_ALLOC PPC_BIT(51)
> >> #define NPU2_HIGH_WATER_MARKS 0x048
> >> #define NPU2_RELAXED_ORDERING_CFG0 0x050
> >> #define NPU2_RELAXED_ORDERING_CFG1 0x058
> >> @@ -176,6 +184,9 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >>
> >> /* CTL block registers */
> >> #define NPU2_CQ_CTL_MISC_CFG 0x000
> >> +#define NPU2_CQ_CTL_MISC_CFG_CONFIG_OCAPI_MODE PPC_BIT(52)
> >> +#define NPU2_CQ_CTL_MISC_CFG_CONFIG_OTL0_ENABLE PPC_BIT(55)
> >> +#define NPU2_CQ_CTL_MISC_CFG_CONFIG_OTL1_ENABLE PPC_BIT(56)
> >> #define NPU2_CQ_FUTURE_CFG1 0x008
> >> #define NPU2_CQ_FUTURE_CFG2 0x010
> >> #define NPU2_CQ_FUTURE_CFG3 0x018
> >> @@ -198,6 +209,8 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_CQ_LPC_THRESHOLD_CFG 0x080
> >> #define NPU2_CQ_INHIBIT_CFG 0x088
> >> #define NPU2_CQ_CTL_STATUS 0x090
> >> +#define NPU2_CQ_CTL_STATUS_BRK0_AM_FENCED PPC_BITMASK(48, 49)
> >> +#define NPU2_CQ_CTL_STATUS_BRK1_AM_FENCED PPC_BITMASK(50, 51)
> >> #define NPU2_CQ_C_ERR_RPT_MSG0 0x0C0
> >> #define NPU2_CQ_C_ERR_RPT_MSG1 0x0C8
> >> #define NPU2_CQ_C_ERR_RPT_FIRST0 0x0D0
> >> @@ -206,9 +219,13 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_CQ_C_ERR_RPT_MASK1 0x0E8
> >> #define NPU2_CQ_C_ERR_RPT_HOLD0 0x0F0
> >> #define NPU2_CQ_C_ERR_RPT_HOLD1 0x0F8
> >> +#define NPU2_CQ_CTL_FENCE_CONTROL_0 0x140
> >> +#define NPU2_CQ_CTL_FENCE_CONTROL_1 0x148
> >> +#define NPU2_CQ_CTL_FENCE_CONTROL_REQUEST_FENCE PPC_BITMASK(0, 1)
> >>
> >> /* DAT block registers */
> >> #define NPU2_CQ_DAT_MISC_CFG 0x008
> >> +#define NPU2_CQ_DAT_MISC_CFG_CONFIG_OCAPI_MODE PPC_BIT(40)
> >> #define NPU2_CQ_DAT_ECC_CFG 0x010
> >> #define NPU2_CQ_DAT_SCRATCH0 0x018
> >> #define NPU2_CQ_DAT_ECC_STATUS 0x020
> >> @@ -289,6 +306,43 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_NTL_DL_CONFIG(ndev) NPU2_DL_REG_OFFSET(ndev, 0xFFF8)
> >> #define NPU2_NTL_DL_CLK_CTRL(ndev) NPU2_DL_REG_OFFSET(ndev, 0x001C)
> >>
> >> +/* OpenCAPI - XSL registers */
> >> +#define NPU2_XSL_PSL_SCNTL_A0 0x010
> >> +#define NPU2_XSL_PSL_SCNTL_A0_MULTI_AFU_DIAL PPC_BIT(0)
> >> +#define NPU2_XSL_DEF 0x040
> >> +#define NPU2_XSL_GP 0x058
> >> +#define NPU2_XSL_GP_BLOOM_FILTER_ENABLE PPC_BIT(16)
> >> +#define NPU2_XSL_WRAP_CFG 0x0C0
> >> +#define NPU2_XSL_WRAP_CFG_XSLO_CLOCK_ENABLE PPC_BIT(0)
> >> +
> >> +/* OpenCAPI - OTL registers */
> >> +#define NPU2_OTL_CONFIG0(stack, block) NPU2_REG_OFFSET(stack, block, 0x000)
> >> +#define NPU2_OTL_CONFIG0_EN PPC_BIT(0)
> >> +#define NPU2_OTL_CONFIG0_BLOCK_PE_HANDLE PPC_BIT(1)
> >> +#define NPU2_OTL_CONFIG0_BRICKID PPC_BITMASK(2, 3)
> >> +#define NPU2_OTL_CONFIG0_PE_MASK PPC_BITMASK(4, 7)
> >> +#define NPU2_OTL_CONFIG0_ERAT_HASH_0 PPC_BITMASK(8, 13)
> >> +#define NPU2_OTL_CONFIG0_ERAT_HASH_1 PPC_BITMASK(14, 19)
> >> +#define NPU2_OTL_CONFIG0_ERAT_HASH_2 PPC_BITMASK(20, 25)
> >> +#define NPU2_OTL_CONFIG0_ERAT_HASH_3 PPC_BITMASK(26, 31)
> >> +#define NPU2_OTL_CONFIG0_BLOCK_TID_OVERRIDE PPC_BIT(50)
> >> +#define NPU2_OTL_TLX_CREDITS(stack, block) NPU2_REG_OFFSET(stack, block, 0x050)
> >> +#define NPU2_OTL_TLX_CREDITS_VC0_CREDITS PPC_BITMASK(0, 7)
> >> +#define NPU2_OTL_TLX_CREDITS_VC3_CREDITS PPC_BITMASK(24, 31)
> >> +#define NPU2_OTL_TLX_CREDITS_DCP0_CREDITS PPC_BITMASK(32, 39)
> >> +#define NPU2_OTL_TLX_CREDITS_DCP1_CREDITS PPC_BITMASK(56, 63)
> >> +#define NPU2_OTL_CONFIG1(stack, block) NPU2_REG_OFFSET(stack, block, 0x058)
> >> +#define NPU2_OTL_CONFIG1_TX_DRDY_WAIT PPC_BITMASK(5, 7)
> >> +#define NPU2_OTL_CONFIG1_TX_TEMP0_RATE PPC_BITMASK(8, 11)
> >> +#define NPU2_OTL_CONFIG1_TX_CRET_FREQ PPC_BITMASK(32, 34)
> >> +#define NPU2_OTL_CONFIG1_TX_AGE_FREQ PPC_BITMASK(35, 39)
> >> +#define NPU2_OTL_CONFIG1_TX_RS2_HPWAIT PPC_BITMASK(40, 45)
> >> +#define NPU2_OTL_CONFIG1_TX_RQ4_HPWAIT PPC_BITMASK(46, 51)
> >> +#define NPU2_OTL_CONFIG1_TX_RQ6_HPWAIT PPC_BITMASK(52, 57)
> >> +#define NPU2_OTL_CONFIG1_TX_CBUF_ECC_DIS PPC_BIT(58)
> >> +#define NPU2_OTL_CONFIG1_TX_STOP_LINK PPC_BIT(59)
> >> +#define NPU2_OTL_CONFIG1_TX_STOP_ON_UE PPC_BIT(60)
> >> +
> >> /* Misc block registers. Unlike the SM/CTL/DAT/NTL registers above
> >> * there is only a single instance of each of these in the NPU so we
> >> * define them as absolute offsets. */
> >> @@ -296,6 +350,9 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_MISC_RELAXTED_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x008)
> >> #define NPU2_MISC_LCO_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x010)
> >> #define NPU2_MISC_OPTICAL_IO_CFG0 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x018)
> >> +#define NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2 PPC_BITMASK(0, 2)
> >> +#define NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1 PPC_BITMASK(3, 4)
> >> +#define NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5 PPC_BITMASK(5, 6)
> >> #define NPU2_MISC_ERR_RPT_HOLD NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x020)
> >> #define NPU2_MISC_ERR_RPT_MASK NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x028)
> >> #define NPU2_MISC_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x030)
> >> @@ -320,6 +377,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_MISC_ERR_CTL_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x088)
> >> #define NPU2_MISC_RING_ACK_ERR NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x090)
> >> #define NPU2_MISC_IRQ_BASE NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x098)
> >> +#define NPU2_MISC_IRQ_BASE_MASK PPC_BITMASK(13, 51)
> >> #define NPU2_MISC_ERR_BRICK_GROUP NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0A0)
> >> #define NPU2_MISC_FREEZE_STATE NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0A8)
> >> #define NPU2_MISC_FENCE_STATE NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x0B0)
> >> @@ -362,6 +420,10 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_MISC_IRQ_LOG13 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x368)
> >> #define NPU2_MISC_IRQ_LOG14 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x370)
> >> #define NPU2_MISC_IRQ_LOG15 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x378)
> >> +#define NPU2_MISC_IRQ_ENABLE2 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC, 0x408)
> >> +
> >> +/* Misc register, direct access only */
> >> +#define NPU2_MISC_FIR_MASK1 0x2C43
> >>
> >> /* ATS block registers */
> >> #define NPU2_ATS_PMU_CTL NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_ATS, 0x000)
> >> @@ -406,8 +468,11 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_XTS_CFG NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x020)
> >> #define NPU2_XTS_CFG_MMIOSD PPC_BIT(1)
> >> #define NPU2_XTS_CFG_TRY_ATR_RO PPC_BIT(6)
> >> +#define NPU2_XTS_CFG_OPENCAPI PPC_BIT(15)
> >> #define NPU2_XTS_CFG2 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x028)
> >> #define NPU2_XTS_CFG2_NO_FLUSH_ENA PPC_BIT(49)
> >> +#define NPU2_XTS_CFG2_XSL1_ENA PPC_BIT(54)
> >> +#define NPU2_XTS_CFG2_XSL2_ENA PPC_BIT(55)
> >> #define NPU2_XTS_DBG_CFG0 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x030)
> >> #define NPU2_XTS_DBG_CFG1 NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x038)
> >> #define NPU2_XTS_PMU_CNT NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_XTS, 0x040)
> >> @@ -463,4 +528,29 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
> >> #define NPU2_DD1_MISC_SCOM_IND_SCOM_DATA 0x38f
> >> #define NPU2_MISC_SCOM_IND_SCOM_DATA 0x68f
> >>
> >> +
> >> +/* OpenCAPI - PowerBus and OBus configuration SCOM addresses */
> >> +#define PU_IOE_PB_MISC_CFG 0x5013823
> >> +#define PU_IOE_PB_MISC_CFG_SEL_03_NPU_NOT_PB PPC_BIT(13)
> >> +#define PU_IOE_PB_MISC_CFG_SEL_04_NPU_NOT_PB PPC_BIT(14)
> >> +#define PU_IOE_PB_MISC_CFG_SEL_05_NPU_NOT_PB PPC_BIT(15)
> >> +#define OBUS_LL0_IOOL_PHY_CONFIG 0x901080C
> >> +#define OBUS_LL3_IOOL_PHY_CONFIG 0xC01080C
> >> +#define OBUS_IOOL_PHY_CONFIG_ODL0_ENABLED PPC_BIT(52)
> >> +#define OBUS_IOOL_PHY_CONFIG_ODL1_ENABLED PPC_BIT(53)
> >> +#define OBUS_IOOL_PHY_CONFIG_ODL_PHY_SWAP PPC_BIT(54)
> >> +#define OBUS_IOOL_PHY_CONFIG_LINK0_OLL_ENABLED PPC_BIT(58)
> >> +#define OBUS_IOOL_PHY_CONFIG_LINK1_OLL_ENABLED PPC_BIT(59)
> >> +#define OBUS_IOOL_PHY_CONFIG_NV0_NPU_ENABLED PPC_BIT(61)
> >> +#define OBUS_IOOL_PHY_CONFIG_NV1_NPU_ENABLED PPC_BIT(62)
> >> +#define OBUS_IOOL_PHY_CONFIG_NV2_NPU_ENABLED PPC_BIT(63)
> >> +#define PU_IOE_PB_FP01_CFG 0x501380A
> >> +#define PU_IOE_PB_FP23_CFG 0x501380B
> >> +#define PU_IOE_PB_FP45_CFG 0x501380C
> >> +#define PU_IOE_PB_FP67_CFG 0x501380D
> >> +#define PU_IOE_PB_FP_CFG_FP0_FMR_DISABLE PPC_BIT(20)
> >> +#define PU_IOE_PB_FP_CFG_FP0_PRS_DISABLE PPC_BIT(25)
> >> +#define PU_IOE_PB_FP_CFG_FP1_FMR_DISABLE PPC_BIT(52)
> >> +#define PU_IOE_PB_FP_CFG_FP1_PRS_DISABLE PPC_BIT(57)
> >> +
> >> #endif /* __NPU2_REGS_H */
> >> diff --git a/include/npu2.h b/include/npu2.h
> >> index 94e537b..7b2b509 100644
> >> --- a/include/npu2.h
> >> +++ b/include/npu2.h
> >> @@ -138,6 +138,7 @@ struct npu2_dev {
> >>
> >> struct npu2 {
> >> uint32_t index;
> >> + struct dt_node *dt_node;
> >> uint32_t flags;
> >> uint32_t chip_id;
> >> uint64_t xscom_base;
> >> @@ -146,6 +147,7 @@ struct npu2 {
> >> uint64_t mm_base;
> >> uint64_t mm_size;
> >> uint32_t base_lsi;
> >> + uint32_t irq_base;
> >> uint32_t total_devices;
> >> struct npu2_dev *devices;
> >> enum phys_map_type gpu_map_type;
> >> diff --git a/include/skiboot.h b/include/skiboot.h
> >> index e94f212..1218367 100644
> >> --- a/include/skiboot.h
> >> +++ b/include/skiboot.h
> >> @@ -232,6 +232,7 @@ extern int preload_capp_ucode(void);
> >> extern void preload_io_vpd(void);
> >> extern void probe_npu(void);
> >> extern void probe_npu2(void);
> >> +extern void probe_npu2_opencapi(void);
> >> extern void uart_init(void);
> >> extern void mbox_init(void);
> >> extern void early_uart_init(void);
> >>
> >
> >
>
>
More information about the Skiboot
mailing list