[Skiboot] [PATCH v2 10/15] hw/npu2: Combined NPU brick configuration procedure

Andrew Donnellan andrew.donnellan at au1.ibm.com
Thu Jan 31 15:39:09 AEDT 2019


On 21/1/19 5:18 pm, Alexey Kardashevskiy wrote:
> 
> 
> On 11/01/2019 12:09, Andrew Donnellan wrote:
>> To support the case where we have both OpenCAPI and NVLink devices
>> connected to the same NPU, we need to be able to configure each NPU
>> stack/brick separately depending on the device type it is connected to.
>>
>> Refactor the existing code so that we can set the transport muxes and other
>> relevant NPU registers on a per-brick/stack basis. Replace the existing
>> hard-coded register values for NVLink setup with code to do it properly
>> only on the devices that need it.
> 
> This "replace" is in fact obfuscation of the actual change :( Frederic
> must be a superman to have this reviewed :)

"Replace" is a fairly accurate description IMHO. :)

> 
> 
>>
>> Signed-off-by: Andrew Donnellan <andrew.donnellan at au1.ibm.com>
>> Reviewed-by: Frederic Barrat <fbarrat at linux.ibm.com>
>>
>> ---
>> v1->v2:
>> - commit message (Fred)
>> ---
>>   hw/npu2-common.c    | 126 +++++++++++++++++++++++++++++++++++++++++++++-
>>   hw/npu2-opencapi.c  | 109 +---------------------------------------
>>   hw/npu2.c           | 121 +++++++++++++++++++++++--------------------
>>   include/npu2-regs.h |   4 +-
>>   include/npu2.h      |   1 +-
>>   5 files changed, 196 insertions(+), 165 deletions(-)
>>
>> diff --git a/hw/npu2-common.c b/hw/npu2-common.c
>> index 282e4873125a..e323e71cb8d5 100644
>> --- a/hw/npu2-common.c
>> +++ b/hw/npu2-common.c
>> @@ -411,6 +411,130 @@ void npu2_i2c_presence_detect(struct npu2 *npu)
>>   	}
>>   }
>>   
>> +/* Procedure 13.1.3.1, OpenCAPI NPU Workbook - Select OCAPI vs NVLink */
>> +static void set_brick_config(struct npu2 *npu)
>> +{
>> +	/* Step 1 - Set Transport MUX controls to select correct OTL or NTL */
>> +	uint64_t reg;
>> +	uint64_t ndlmux_brk0to2, ocmux_brk0to1, ocmux_brk4to5;
>> +	uint64_t phy_config_scom;
>> +	struct npu2_dev *dev;
>> +
>> +	prlog(PR_DEBUG, "NPU: %s: Setting transport mux controls\n", __func__);
>> +
>> +	/*
>> +	 * Disable NVLink link layers on PHYs being used for OpenCAPI.
>> +	 * Experience suggests that this needs to be done early
>> +	 * (before setting the transport muxes) otherwise we get link
>> +	 * internal errors.
>> +	 */
>> +	for (int i = 0; i < npu->total_devices; i++) {
>> +		dev = &npu->devices[i];
>> +		if (dev->type != NPU2_DEV_TYPE_OPENCAPI)
>> +			continue;
>> +		switch (dev->brick_index) {
>> +		case 2:
>> +		case 3:
>> +			phy_config_scom = OBUS_LL0_IOOL_PHY_CONFIG;
>> +			break;
>> +		case 4:
>> +		case 5:
>> +			phy_config_scom = OBUS_LL3_IOOL_PHY_CONFIG;
>> +			break;
>> +		default:
>> +			assert(false);
>> +		}
>> +		/* Disable NV-Link link layers */
>> +		xscom_read(npu->chip_id, phy_config_scom, &reg);
>> +		reg &= ~OBUS_IOOL_PHY_CONFIG_NV0_NPU_ENABLED;
>> +		reg &= ~OBUS_IOOL_PHY_CONFIG_NV1_NPU_ENABLED;
>> +		reg &= ~OBUS_IOOL_PHY_CONFIG_NV2_NPU_ENABLED;
>> +		xscom_write(npu->chip_id, phy_config_scom, reg);
>> +	}
>> +
>> +	/* Optical IO Transport Mux Config for Bricks 0-2 and 4-5 */
>> +	reg = npu2_scom_read(npu->chip_id, npu->xscom_base, NPU2_MISC_OPTICAL_IO_CFG0,
>> +			     NPU2_MISC_DA_LEN_8B);
>> +	ndlmux_brk0to2 = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2, reg);
>> +	ocmux_brk0to1 = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1, reg);
>> +	ocmux_brk4to5 = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5, reg);
>> +	for (int i = 0; i < npu->total_devices; i++) {
>> +		dev = &npu->devices[i];
>> +		if (dev->type == NPU2_DEV_TYPE_UNKNOWN)
>> +			continue;
>> +
>> +		switch (dev->brick_index) {
>> +		case 0:  /* NTL0.0 */
>> +			assert(dev->type == NPU2_DEV_TYPE_NVLINK);
>> +			ndlmux_brk0to2 |= 0b100;
>> +			break;
>> +		case 1:  /* NTL0.1 */
>> +			assert(dev->type == NPU2_DEV_TYPE_NVLINK);
>> +			ndlmux_brk0to2 |= 0b010;
>> +			break;
>> +		case 2:	 /* NTL1.0 / OTL1.0 */
>> +			if (dev->type == NPU2_DEV_TYPE_OPENCAPI) {
>> +				ndlmux_brk0to2 &= ~0b100;
>> +				ocmux_brk0to1 |= 0b10;
>> +			} else {
>> +				ndlmux_brk0to2 |= 0b001;
>> +			}
>> +			break;
>> +		case 3:	 /* NTL1.1 / OTL1.1 */
>> +			if (dev->type == NPU2_DEV_TYPE_OPENCAPI) {
>> +				ndlmux_brk0to2 &= ~0b010;
>> +				ocmux_brk0to1 |= 0b01;
>> +			}
>> +			break;
>> +		case 4:	 /* NTL2.0 / OTL2.0 */
>> +			if (dev->type == NPU2_DEV_TYPE_OPENCAPI) {
>> +				ocmux_brk4to5 |= 0b10;
>> +			} else {
>> +				ocmux_brk4to5 &= ~0b10;
>> +			}
>> +			break;
>> +		case 5:	 /* NTL2.1 / OTL2.1 */
>> +			if (dev->type == NPU2_DEV_TYPE_OPENCAPI) {
>> +				ocmux_brk4to5 |= 0b01;
>> +			} else {
>> +				ocmux_brk4to5 &= ~0b01;
>> +			}
>> +			break;
>> +		default:
>> +			assert(false);
>> +		}
>> +	}
>> +
>> +	reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2, reg, ndlmux_brk0to2);
>> +	reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1, reg, ocmux_brk0to1);
>> +	reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5, reg, ocmux_brk4to5);
>> +	npu2_scom_write(npu->chip_id, npu->xscom_base, NPU2_MISC_OPTICAL_IO_CFG0,
>> +			NPU2_MISC_DA_LEN_8B, reg);
>> +
>> +	/*
>> +	 * PowerBus Optical Miscellaneous Config Register
>> +	 */
>> +	xscom_read(npu->chip_id, PU_IOE_PB_MISC_CFG, &reg);
>> +	for (int i = 0; i < npu->total_devices; i++) {
>> +		dev = &npu->devices[i];
>> +		switch (dev->brick_index) {
>> +		case 3:
>> +			if (dev->type == NPU2_DEV_TYPE_NVLINK)
>> +				reg = SETFIELD(PU_IOE_PB_MISC_CFG_SEL_03_NPU_NOT_PB, reg, 1);
>> +			break;
>> +		case 4:
>> +			reg = SETFIELD(PU_IOE_PB_MISC_CFG_SEL_04_NPU_NOT_PB, reg, 1);
>> +			break;
>> +		case 5:
>> +			reg = SETFIELD(PU_IOE_PB_MISC_CFG_SEL_05_NPU_NOT_PB, reg, 1);
>> +			break;
>> +		default:
>> +			break;
>> +		}
>> +	}
>> +	xscom_write(npu->chip_id, PU_IOE_PB_MISC_CFG, reg);
>> +}
>> +
>>   static struct npu2 *setup_npu(struct dt_node *dn)
>>   {
>>   	struct npu2 *npu;
>> @@ -471,6 +595,7 @@ static struct npu2 *setup_npu(struct dt_node *dn)
>>   		dev->link_index = dt_prop_get_u32(np, "ibm,npu-link-index");
>>   		/* May be overridden by platform presence detection */
>>   		dev->brick_index = dev->link_index;
>> +		dev->group_id = dt_prop_get_u32(np, "ibm,npu-group-id");
>>   		/* Will be overridden by presence detection */
>>   		dev->type = NPU2_DEV_TYPE_UNKNOWN;
>>   		dev->npu = npu;
>> @@ -575,6 +700,7 @@ void probe_npu2(void)
>>   		if (!npu)
>>   			continue;
>>   		platform.npu2_device_detect(npu);
>> +		set_brick_config(npu);
>>   		setup_devices(npu);
>>   	}
>>   }
>> diff --git a/hw/npu2-opencapi.c b/hw/npu2-opencapi.c
>> index 1b1a06ff81ad..d3e95a45e9a4 100644
>> --- a/hw/npu2-opencapi.c
>> +++ b/hw/npu2-opencapi.c
>> @@ -196,112 +196,6 @@ static uint64_t get_odl_endpoint_info(uint32_t gcid, uint64_t index)
>>   	return reg;
>>   }
>>   
>> -static void disable_nvlink(uint32_t gcid, int index)
>> -{
>> -	uint64_t phy_config_scom, reg;
>> -
>> -	switch (index) {
>> -	case 2:
>> -	case 3:
>> -		phy_config_scom = OBUS_LL0_IOOL_PHY_CONFIG;
>> -		break;
>> -	case 4:
>> -	case 5:
>> -		phy_config_scom = OBUS_LL3_IOOL_PHY_CONFIG;
>> -		break;
>> -	default:
>> -		assert(false);
>> -	}
>> -	/* Disable NV-Link link layers */
>> -	xscom_read(gcid, phy_config_scom, &reg);
>> -	reg &= ~OBUS_IOOL_PHY_CONFIG_NV0_NPU_ENABLED;
>> -	reg &= ~OBUS_IOOL_PHY_CONFIG_NV1_NPU_ENABLED;
>> -	reg &= ~OBUS_IOOL_PHY_CONFIG_NV2_NPU_ENABLED;
>> -	xscom_write(gcid, phy_config_scom, reg);
>> -}
>> -
>> -/* Procedure 13.1.3.1 - select OCAPI vs NVLink for bricks 2-3/4-5 */
>> -
>> -static void set_transport_mux_controls(uint32_t gcid, uint32_t scom_base,
>> -				       int index, enum npu2_dev_type type)
>> -{
>> -	/* Step 1 - Set Transport MUX controls to select correct OTL or NTL */
>> -	uint64_t reg;
>> -	uint64_t field;
>> -
>> -	/* TODO: Rework this to select for NVLink too */
>> -	assert(type == NPU2_DEV_TYPE_OPENCAPI);
>> -
>> -	prlog(PR_DEBUG, "OCAPI: %s: Setting transport mux controls\n", __func__);
>> -
>> -	/* Optical IO Transport Mux Config for Bricks 0-2 and 4-5 */
>> -	reg = npu2_scom_read(gcid, scom_base, NPU2_MISC_OPTICAL_IO_CFG0,
>> -			     NPU2_MISC_DA_LEN_8B);
>> -	switch (index) {
>> -	case 0:
>> -	case 1:
>> -		/* not valid for OpenCAPI */
>> -		assert(false);
>> -		break;
>> -	case 2:	 /* OTL1.0 */
>> -		field = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2, reg);
>> -		field &= ~0b100;
>> -		reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2, reg,
>> -			       field);
>> -		field = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1, reg);
>> -		field |= 0b10;
>> -		reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1, reg,
>> -			       field);
>> -		break;
>> -	case 3:	 /* OTL1.1 */
>> -		field = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2, reg);
>> -		field &= ~0b010;
>> -		reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_NDLMUX_BRK0TO2, reg,
>> -			       field);
>> -		field = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1, reg);
>> -		field |= 0b01;
>> -		reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK0TO1, reg,
>> -			       field);
>> -		break;
>> -	case 4:	 /* OTL2.0 */
>> -		field = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5, reg);
>> -		field |= 0b10;
>> -		reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5, reg,
>> -			       field);
>> -		break;
>> -	case 5:	 /* OTL2.1 */
>> -		field = GETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5, reg);
>> -		field |= 0b01;
>> -		reg = SETFIELD(NPU2_MISC_OPTICAL_IO_CFG0_OCMUX_BRK4TO5, reg,
>> -			       field);
>> -		break;
>> -	default:
>> -		assert(false);
>> -	}
>> -	npu2_scom_write(gcid, scom_base, NPU2_MISC_OPTICAL_IO_CFG0,
>> -			NPU2_MISC_DA_LEN_8B, reg);
>> -
>> -	/*
>> -	 * PowerBus Optical Miscellaneous Config Register - select
>> -	 * OpenCAPI for b4/5 and A-Link for b3
>> -	 */
>> -	xscom_read(gcid, PU_IOE_PB_MISC_CFG, &reg);
>> -	switch (index) {
>> -	case 0:
>> -	case 1:
>> -	case 2:
>> -	case 3:
>> -		break;
>> -	case 4:
>> -		reg = SETFIELD(PU_IOE_PB_MISC_CFG_SEL_04_NPU_NOT_PB, reg, 1);
>> -		break;
>> -	case 5:
>> -		reg = SETFIELD(PU_IOE_PB_MISC_CFG_SEL_05_NPU_NOT_PB, reg, 1);
>> -		break;
>> -	}
>> -	xscom_write(gcid, PU_IOE_PB_MISC_CFG, reg);
>> -}
>> -
>>   static void enable_odl_phy_mux(uint32_t gcid, int index)
>>   {
>>   	uint64_t reg;
>> @@ -579,9 +473,6 @@ static void brick_config(uint32_t gcid, uint32_t scom_base, int index)
>>   	 * We assume at this point that the PowerBus Hotplug Mode Control
>>   	 * register is correctly set by Hostboot
>>   	 */
>> -	disable_nvlink(gcid, index);
>> -	set_transport_mux_controls(gcid, scom_base, index,
>> -				   NPU2_DEV_TYPE_OPENCAPI);
>>   	enable_odl_phy_mux(gcid, index);
>>   	disable_alink_fp(gcid);
>>   	enable_xsl_clocks(gcid, scom_base, index);
>> diff --git a/hw/npu2.c b/hw/npu2.c
>> index 6c720764e614..a8744f4af7f7 100644
>> --- a/hw/npu2.c
>> +++ b/hw/npu2.c
>> @@ -1308,6 +1308,62 @@ static const struct phb_ops npu_ops = {
>>   	.tce_kill		= npu2_tce_kill,
>>   };
>>   
>> +static void enable_nvlink(struct npu2_dev *dev)
>> +{
>> +	struct npu2 *npu = dev->npu;
>> +	int stack = NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev);
>> +	int block;
>> +	uint64_t addr, val;
>> +
>> +	/* CQ_SM Misc Config #0 - enable NVLink mode */
>> +	for (block = NPU2_BLOCK_SM_0; block <= NPU2_BLOCK_SM_3; block++) {
>> +		addr = NPU2_REG_OFFSET(stack, block, NPU2_CQ_SM_MISC_CFG0);
>> +		val = npu2_scom_read(npu->chip_id, npu->xscom_base, addr,
>> +				     NPU2_MISC_DA_LEN_8B);
>> +		val |= NPU2_CQ_SM_MISC_CFG0_CONFIG_NVLINK_MODE;
>> +		npu2_scom_write(npu->chip_id, npu->xscom_base, addr,
>> +				NPU2_MISC_DA_LEN_8B, val);
>> +	}
>> +
>> +	/* CQ_CTL Misc Config #0 - enable NVLink mode */
>> +	addr = NPU2_REG_OFFSET(stack, NPU2_BLOCK_CTL, NPU2_CQ_CTL_MISC_CFG);
>> +	val = npu2_scom_read(npu->chip_id, npu->xscom_base, addr,
>> +			     NPU2_MISC_DA_LEN_8B);
>> +	val |= NPU2_CQ_CTL_MISC_CFG_CONFIG_NVLINK_MODE;
>> +	npu2_scom_write(npu->chip_id, npu->xscom_base, addr,
>> +			NPU2_MISC_DA_LEN_8B, val);
>> +
>> +	/* CQ_DAT Misc Config #1 - enable NVLink mode */
>> +	addr = NPU2_REG_OFFSET(stack, NPU2_BLOCK_DAT, NPU2_CQ_DAT_MISC_CFG);
>> +	val = npu2_scom_read(npu->chip_id, npu->xscom_base, addr,
>> +			     NPU2_MISC_DA_LEN_8B);
>> +	val |= NPU2_CQ_DAT_MISC_CFG_CONFIG_NVLINK_MODE;
>> +	npu2_scom_write(npu->chip_id, npu->xscom_base, addr,
>> +			NPU2_MISC_DA_LEN_8B, val);
>> +
>> +	/* NTL Misc Config 2 - enable NTL brick and checks */
>> +	addr = NPU2_NTL_MISC_CFG2(dev);
>> +	val = npu2_scom_read(npu->chip_id, npu->xscom_base, addr,
>> +			     NPU2_MISC_DA_LEN_8B);
>> +	val |= NPU2_NTL_MISC_CFG2_BRICK_ENABLE;
>> +	val |= NPU2_NTL_MISC_CFG2_NDL_TX_PARITY_ENA;
>> +	val |= NPU2_NTL_MISC_CFG2_NDL_PRI_PARITY_ENA;
>> +	val |= NPU2_NTL_MISC_CFG2_RCV_CREDIT_OVERFLOW_ENA;
>> +	npu2_scom_write(npu->chip_id, npu->xscom_base, addr,
>> +			NPU2_MISC_DA_LEN_8B, val);
>> +
>> +	/* High Water Marks */
>> +	for (block = NPU2_BLOCK_SM_0; block <= NPU2_BLOCK_SM_3; block++) {
>> +		addr = NPU2_REG_OFFSET(stack, block, NPU2_HIGH_WATER_MARKS);
>> +		val = npu2_scom_read(npu->chip_id, npu->xscom_base, addr,
>> +				     NPU2_MISC_DA_LEN_8B);
>> +		val &= ~NPU2_HIGH_WATER_MARKS_PWR0;
>> +		val |= PPC_BIT(6) | PPC_BIT(7) | PPC_BIT(11);
>> +		npu2_scom_write(npu->chip_id, npu->xscom_base, addr,
>> +				NPU2_MISC_DA_LEN_8B, val);
>> +	}
>> +}
>> +
>>   /*
>>    * Set up NPU for NVLink and create PCI root device node
>>    * accordingly.
>> @@ -1315,59 +1371,12 @@ static const struct phb_ops npu_ops = {
>>   int npu2_nvlink_init_npu(struct npu2 *npu)
>>   {
>>   	struct dt_node *np;
>> -	uint64_t reg[2], mm_win[2], val;
>> +	uint64_t reg[2], mm_win[2];
>>   
>> -	/* TODO: Clean this up with register names, etc. when we get
>> -	 * time. This just turns NVLink mode on in each brick and should
>> -	 * get replaced with a patch from ajd once we've worked out how
>> -	 * things are going to work there.
>> -	 *
>> -	 * Obviously if the year is now 2020 that didn't happen and you
>> -	 * should fix this :-) */
>> -	xscom_write_mask(npu->chip_id, 0x5011000, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011030, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011060, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011090, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011200, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011230, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011260, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011290, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011400, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011430, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011460, PPC_BIT(58), PPC_BIT(58));
>> -	xscom_write_mask(npu->chip_id, 0x5011490, PPC_BIT(58), PPC_BIT(58));
>> -
>> -	xscom_write_mask(npu->chip_id, 0x50110c0, PPC_BIT(53), PPC_BIT(53));
>> -	xscom_write_mask(npu->chip_id, 0x50112c0, PPC_BIT(53), PPC_BIT(53));
>> -	xscom_write_mask(npu->chip_id, 0x50114c0, PPC_BIT(53), PPC_BIT(53));
>> -	xscom_write_mask(npu->chip_id, 0x50110f1, PPC_BIT(41), PPC_BIT(41));
>> -	xscom_write_mask(npu->chip_id, 0x50112f1, PPC_BIT(41), PPC_BIT(41));
>> -	xscom_write_mask(npu->chip_id, 0x50114f1, PPC_BIT(41), PPC_BIT(41));
>> -
>> -	val = NPU2_NTL_MISC_CFG2_BRICK_ENABLE |
>> -	      NPU2_NTL_MISC_CFG2_NDL_TX_PARITY_ENA |
>> -	      NPU2_NTL_MISC_CFG2_NDL_PRI_PARITY_ENA |
>> -	      NPU2_NTL_MISC_CFG2_RCV_CREDIT_OVERFLOW_ENA;
>> -	xscom_write_mask(npu->chip_id, 0x5011110, val, val);
>> -	xscom_write_mask(npu->chip_id, 0x5011130, val, val);
>> -	xscom_write_mask(npu->chip_id, 0x5011310, val, val);
>> -	xscom_write_mask(npu->chip_id, 0x5011330, val, val);
>> -	xscom_write_mask(npu->chip_id, 0x5011510, val, val);
>> -	xscom_write_mask(npu->chip_id, 0x5011530, val, val);
>> -
>> -	val = PPC_BIT(6) | PPC_BIT(7) | PPC_BIT(11);
>> -	xscom_write_mask(npu->chip_id, 0x5011009, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011039, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011069, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011099, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011209, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011239, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011269, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011299, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011409, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011439, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011469, val, PPC_BITMASK(6,11));
>> -	xscom_write_mask(npu->chip_id, 0x5011499, val, PPC_BITMASK(6,11));
>> +	for (int i = 0; i < npu->total_devices; i++) {
>> +		if (npu->devices[i].type == NPU2_DEV_TYPE_NVLINK)
>> +			enable_nvlink(&npu->devices[i]);
>> +	}
>>   
>>   	/* Populate PCI root device node */
>>   	reg[0] = (uint64_t)npu->regs;
>> @@ -1599,6 +1608,9 @@ static uint32_t npu_allocate_bdfn(struct npu2 *p, uint32_t group, int size)
>>   	int bdfn = (group << 3);
>>   
>>   	for (i = 0; i < size; i++) {
>> +		if (p->devices[i].type != NPU2_DEV_TYPE_NVLINK)
>> +			continue;
>> +
>>   		if ((p->devices[i].bdfn & 0xf8) == (bdfn & 0xf8))
>>   			bdfn++;
>>   	}
>> @@ -1612,14 +1624,11 @@ static void npu2_configure_devices(struct npu2 *p)
>>   	uint32_t index = 0;
>>   
>>   	for (index = 0; index < p->total_devices; index++) {
>> -		uint32_t group_id;
>> -
>>   		dev = &p->devices[index];
>>   		if (dev->type != NPU2_DEV_TYPE_NVLINK)
>>   			continue;
>>   
>> -		group_id = dt_prop_get_u32(dev->dt_node, "ibm,npu-group-id");
>> -		dev->bdfn = npu_allocate_bdfn(p, group_id, index);
>> +		dev->bdfn = npu_allocate_bdfn(p, dev->group_id, index);
>>   
>>   		/* This must be done after calling
>>   		 * npu_allocate_bdfn() */
>> diff --git a/include/npu2-regs.h b/include/npu2-regs.h
>> index 8273b2be26a2..a8f59dcdbe2f 100644
>> --- a/include/npu2-regs.h
>> +++ b/include/npu2-regs.h
>> @@ -119,6 +119,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
>>   #define NPU2_CQ_SM_MISC_CFG0			0x000
>>   #define   NPU2_CQ_SM_MISC_CFG0_CONFIG_ENABLE_PBUS PPC_BIT(38)
>>   #define   NPU2_CQ_SM_MISC_CFG0_CONFIG_OCAPI_MODE PPC_BIT(57)
>> +#define   NPU2_CQ_SM_MISC_CFG0_CONFIG_NVLINK_MODE PPC_BIT(58)
>>   #define NPU2_CQ_SM_MISC_CFG1			0x008
>>   #define NPU2_CQ_SM_MISC_CFG2			0x148
>>   #define NPU2_PB_EPSILON				0x010
>> @@ -152,6 +153,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
>>   #define NPU2_LOW_WATER_MARKS			0x040
>>   #define   NPU2_LOW_WATER_MARKS_ENABLE_MACHINE_ALLOC PPC_BIT(51)
>>   #define NPU2_HIGH_WATER_MARKS			0x048
>> +#define   NPU2_HIGH_WATER_MARKS_PWR0		PPC_BITMASK(6, 11);
>>   #define NPU2_RELAXED_ORDERING_CFG(n)		(0x050 + (n)*8)
>>   #define   NPU2_RELAXED_ORDERING_SOURCE(n)	(PPC_BITMASK(0,31) >> ((n)*32))
>>   #define     NPU2_RELAXED_ORDERING_SOURCE_ENA	PPC_BITMASK32(0,3)
>> @@ -207,6 +209,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
>>   /* CTL block registers */
>>   #define NPU2_CQ_CTL_MISC_CFG			0x000
>>   #define   NPU2_CQ_CTL_MISC_CFG_CONFIG_OCAPI_MODE PPC_BIT(52)
>> +#define   NPU2_CQ_CTL_MISC_CFG_CONFIG_NVLINK_MODE PPC_BIT(53)
>>   #define   NPU2_CQ_CTL_MISC_CFG_CONFIG_OTL0_ENABLE PPC_BIT(55)
>>   #define   NPU2_CQ_CTL_MISC_CFG_CONFIG_OTL1_ENABLE PPC_BIT(56)
>>   #define NPU2_CQ_CTL_MISC_MMIOPA0_CONFIG		0x0B0
>> @@ -261,6 +264,7 @@ void npu2_scom_write(uint64_t gcid, uint64_t scom_base,
>>   /* DAT block registers */
>>   #define NPU2_CQ_DAT_MISC_CFG			0x008
>>   #define	  NPU2_CQ_DAT_MISC_CFG_CONFIG_OCAPI_MODE PPC_BIT(40)
>> +#define	  NPU2_CQ_DAT_MISC_CFG_CONFIG_NVLINK_MODE PPC_BIT(41)
>>   #define NPU2_CQ_DAT_ECC_CFG			0x010
>>   #define NPU2_CQ_DAT_SCRATCH0			0x018
>>   #define NPU2_CQ_DAT_ECC_STATUS			0x020
>> diff --git a/include/npu2.h b/include/npu2.h
>> index 75c11ff3c26f..347a69b30d7d 100644
>> --- a/include/npu2.h
>> +++ b/include/npu2.h
>> @@ -114,6 +114,7 @@ struct npu2_dev {
>>   	enum npu2_dev_type	type;
>>   	uint32_t		link_index;
>>   	uint32_t		brick_index;
>> +	uint32_t		group_id;
> 
> Why this change? You do not seem to do dt_prop_get_u32(dev->dt_node,
> "ibm,npu-group-id") often anyway.

Because I want to keep all our device tree accessing together, as much 
as possible, and have everything else avoid touching the device tree.

> 
> 
>>   	uint64_t		pl_xscom_base;
>>   	struct dt_node		*dt_node;
>>   	struct npu2_bar		ntl_bar;
>>
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan at au1.ibm.com  IBM Australia Limited



More information about the Skiboot mailing list