[Skiboot] [PATCH v2 04/15] hw/npu2: Move PHY/NTL/GENID BAR assignment to common code

Andrew Donnellan andrew.donnellan at au1.ibm.com
Fri Jan 11 12:09:24 AEDT 2019


Assignment of PHY/NTL/GENID BARs is currently duplicated between NVLink
and OpenCAPI. This is going to cause us particular issues later on when we
implement support for mixed-mode setups with NVLink and OpenCAPI on the
same NPU.

Centralise the assignment of PHY/NTL/GENID BARs in common code.

Signed-off-by: Andrew Donnellan <andrew.donnellan at au1.ibm.com>
---
v1->v2:
- keep config bar enablement in separate function (Fred)
- fix GENID BAR assignment for OpenCAPI OTL1
- split patch (Alexey)
---
 hw/npu2-common.c   | 78 +++++++++++++++++++++++++++++++++++++-
 hw/npu2-opencapi.c | 61 +++--------------------------
 hw/npu2.c          | 98 ++---------------------------------------------
 include/npu2.h     |  1 +-
 4 files changed, 91 insertions(+), 147 deletions(-)

diff --git a/hw/npu2-common.c b/hw/npu2-common.c
index 3446acb45bea..944719d2489a 100644
--- a/hw/npu2-common.c
+++ b/hw/npu2-common.c
@@ -17,6 +17,7 @@
 #include <skiboot.h>
 #include <xscom.h>
 #include <pci.h>
+#include <pci-cfg.h>
 #include <npu2.h>
 #include <npu2-regs.h>
 #include <bitutils.h>
@@ -174,6 +175,81 @@ void npu2_write_bar(struct npu2 *p, struct npu2_bar *bar, uint32_t gcid,
 	}
 }
 
+static void assign_bars(struct npu2 *npu)
+{
+	uint32_t i;
+	struct npu2_bar *bar;
+	struct npu2_dev *dev;
+	struct npu2_bar phy_bars[] = {
+		/* NPU_REGS must be first in this list */
+		{ .type = NPU_REGS, .index = 0,
+		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR),
+		  .enabled = true },
+		{ .type = NPU_PHY, .index = 0,
+		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR),
+		  .enabled = true },
+		{ .type = NPU_PHY, .index = 1,
+		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR),
+		  .enabled = true },
+	};
+	struct npu2_bar last_bar =
+		{ .type = NPU_GENID, .index = 2,
+		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_GENID_BAR) };
+
+	/* Set common PHY BARs */
+	for (i = 0; i < ARRAY_SIZE(phy_bars); i++) {
+		bar = &phy_bars[i];
+		npu2_get_bar(npu->chip_id, bar);
+		npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base);
+	}
+
+	/* Device BARs */
+	for (i = 0; i < npu->total_devices; i++) {
+		dev = &npu->devices[i];
+		if (dev->type == NPU2_DEV_TYPE_UNKNOWN)
+			continue;
+
+		/* NTL / OCAPI_MMIO BAR */
+		bar = &dev->ntl_bar;
+		if (dev->type == NPU2_DEV_TYPE_NVLINK)
+			bar->type = NPU_NTL;
+		else if (dev->type == NPU2_DEV_TYPE_OPENCAPI)
+			bar->type = NPU_OCAPI_MMIO;
+		bar->index = dev->brick_index;
+		bar->reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev),
+					   0, NPU2DEV_BRICK(dev) == 0 ?
+					   NPU2_NTL0_BAR : NPU2_NTL1_BAR);
+		bar->flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
+		npu2_get_bar(npu->chip_id, bar);
+		npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base);
+
+		/* GENID BAR */
+		bar = &dev->genid_bar;
+		bar->type = NPU_GENID;
+		bar->index = NPU2DEV_STACK(dev);
+		bar->reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev),
+					   0, NPU2_GENID_BAR);
+		bar->flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
+		npu2_get_bar(npu->chip_id, bar);
+		if (dev->type == NPU2_DEV_TYPE_NVLINK) {
+			bar->size = 0x10000;
+			if (NPU2DEV_BRICK(dev))
+				bar->base += 0x10000;
+		}
+		npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base);
+	};
+
+	/* Global MMIO BAR */
+	npu->regs = (uint64_t *)phy_bars[0].base;
+	npu->regs_size = phy_bars[0].size;
+
+	/* NTL and GENID BARs are exposed to kernel via the mm
+	 * window */
+	npu2_get_bar(npu->chip_id, &last_bar);
+	npu->mm_base = phy_bars[2].base + phy_bars[2].size;
+	npu->mm_size = last_bar.base + last_bar.size - npu->mm_base;
+}
+
 static bool _i2c_presence_detect(struct npu2_dev *dev)
 {
 	uint8_t state, data;
@@ -350,6 +426,8 @@ static void setup_devices(struct npu2 *npu)
 		return;
 	}
 
+	assign_bars(npu);
+
 	if (nvlink_detected)
 		npu2_nvlink_init_npu(npu);
 	else if (ocapi_detected)
diff --git a/hw/npu2-opencapi.c b/hw/npu2-opencapi.c
index 8dac552adea8..17a16be00eba 100644
--- a/hw/npu2-opencapi.c
+++ b/hw/npu2-opencapi.c
@@ -730,53 +730,19 @@ static void address_translation_config(uint32_t gcid, uint32_t scom_base,
 	}
 }
 
-static void setup_global_mmio_bar(uint32_t gcid, uint32_t scom_base,
-				  uint64_t reg[])
-{
-	struct npu2_bar *bar;
-	struct npu2_bar phy_bars[] = {
-		{ .type = NPU_PHY, .index = 0,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR),
-		  .enabled = true },
-		{ .type = NPU_PHY, .index = 1,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR),
-		  .enabled = true },
-		{ .type = NPU_REGS, .index = 0,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR),
-		  .enabled = true },
-	};
-
-	for (int i = 0; i < ARRAY_SIZE(phy_bars); i++) {
-		bar = &phy_bars[i];
-		npu2_get_bar(gcid, bar);
-		npu2_write_bar(NULL, bar, gcid, scom_base);
-	}
-	reg[0] = phy_bars[2].base;
-	reg[1] = phy_bars[2].size;
-}
-
 /* Procedure 13.1.3.8 - AFU MMIO Range BARs */
 static void setup_afu_mmio_bars(uint32_t gcid, uint32_t scom_base,
 				struct npu2_dev *dev)
 {
 	uint64_t stack = index_to_stack(dev->brick_index);
-	uint64_t offset = index_to_block(dev->brick_index) == NPU2_BLOCK_OTL0 ?
-		NPU2_NTL0_BAR : NPU2_NTL1_BAR;
 	uint64_t pa_offset = index_to_block(dev->brick_index) == NPU2_BLOCK_OTL0 ?
 		NPU2_CQ_CTL_MISC_MMIOPA0_CONFIG :
 		NPU2_CQ_CTL_MISC_MMIOPA1_CONFIG;
 	uint64_t reg;
 
 	prlog(PR_DEBUG, "OCAPI: %s: Setup AFU MMIO BARs\n", __func__);
-	dev->ntl_bar.type = NPU_OCAPI_MMIO;
-	dev->ntl_bar.index = dev->brick_index;
-	dev->ntl_bar.reg = NPU2_REG_OFFSET(stack, 0, offset);
 	dev->ntl_bar.enabled = true;
-	npu2_get_bar(gcid, &dev->ntl_bar);
-
-	prlog(PR_DEBUG, "OCAPI: AFU MMIO set to %llx, size %llx\n",
-	      dev->ntl_bar.base, dev->ntl_bar.size);
-	npu2_write_bar(NULL, &dev->ntl_bar, gcid, scom_base);
+	npu2_write_bar(dev->npu, &dev->ntl_bar, gcid, scom_base);
 
 	reg = SETFIELD(NPU2_CQ_CTL_MISC_MMIOPA_ADDR, 0ull, dev->ntl_bar.base >> 16);
 	reg = SETFIELD(NPU2_CQ_CTL_MISC_MMIOPA_SIZE, reg, ilog2(dev->ntl_bar.size >> 16));
@@ -788,20 +754,12 @@ static void setup_afu_mmio_bars(uint32_t gcid, uint32_t scom_base,
 }
 
 /* Procedure 13.1.3.9 - AFU Config BARs */
-static void setup_afu_config_bars(uint32_t gcid, uint32_t scom_base,
-				  struct npu2_dev *dev)
+static void setup_afu_config_bars(struct npu2_dev *dev)
 {
-	uint64_t stack = index_to_stack(dev->brick_index);
-	int stack_num = stack - NPU2_STACK_STCK_0;
-
 	prlog(PR_DEBUG, "OCAPI: %s: Setup AFU Config BARs\n", __func__);
-	dev->genid_bar.type = NPU_GENID;
-	dev->genid_bar.index = stack_num;
-	dev->genid_bar.reg = NPU2_REG_OFFSET(stack, 0, NPU2_GENID_BAR);
 	dev->genid_bar.enabled = true;
-	npu2_get_bar(gcid, &dev->genid_bar);
-	prlog(PR_DEBUG, "OCAPI: Assigning GENID BAR: %016llx\n", dev->genid_bar.base);
-	npu2_write_bar(NULL, &dev->genid_bar, gcid, scom_base);
+	npu2_write_bar(dev->npu, &dev->genid_bar, dev->npu->chip_id,
+		       dev->npu->xscom_base);
 }
 
 static void otl_enabletx(uint32_t gcid, uint32_t scom_base,
@@ -1580,8 +1538,8 @@ static void setup_device(struct npu2_dev *dev)
 	uint64_t mm_win[2];
 
 	/* Populate PHB device node */
-	phys_map_get(dev->npu->chip_id, NPU_OCAPI_MMIO, dev->brick_index, &mm_win[0],
-		     &mm_win[1]);
+	mm_win[0] = dev->ntl_bar.base;
+	mm_win[1] = dev->ntl_bar.size;
 	prlog(PR_DEBUG, "OCAPI: Setting MMIO window to %016llx + %016llx\n",
 	      mm_win[0], mm_win[1]);
 	dn_phb = dt_new_addr(dt_root, "pciex", mm_win[0]);
@@ -1627,7 +1585,7 @@ static void setup_device(struct npu2_dev *dev)
 	/* Procedure 13.1.3.8 - AFU MMIO Range BARs */
 	setup_afu_mmio_bars(dev->npu->chip_id, dev->npu->xscom_base, dev);
 	/* Procedure 13.1.3.9 - AFU Config BARs */
-	setup_afu_config_bars(dev->npu->chip_id, dev->npu->xscom_base, dev);
+	setup_afu_config_bars(dev);
 
 	set_fence_control(dev->npu->chip_id, dev->npu->xscom_base, dev->brick_index, 0b00);
 
@@ -1668,7 +1626,6 @@ static void read_nvram_training_state(void)
 int npu2_opencapi_init_npu(struct npu2 *npu)
 {
 	struct npu2_dev *dev;
-	uint64_t reg[2];
 	int rc;
 
 	assert(platform.ocapi);
@@ -1677,10 +1634,6 @@ int npu2_opencapi_init_npu(struct npu2 *npu)
 	/* TODO: Test OpenCAPI with fast reboot and make it work */
 	disable_fast_reboot("OpenCAPI device enabled");
 
-	setup_global_mmio_bar(npu->chip_id, npu->xscom_base, reg);
-
-	npu->regs = (void *)reg[0];
-
 	for (int i = 0; i < npu->total_devices; i++) {
 		dev = &npu->devices[i];
 		if (dev->type != NPU2_DEV_TYPE_OPENCAPI)
diff --git a/hw/npu2.c b/hw/npu2.c
index 60619f381301..b920edef3265 100644
--- a/hw/npu2.c
+++ b/hw/npu2.c
@@ -139,7 +139,7 @@ static int64_t npu2_cfg_write_cmd(void *dev,
 	 * track the enables separately.
 	 */
 	if (NPU2DEV_BRICK(ndev))
-		ndev->genid_bar.enabled1 = enabled;
+		ndev->genid_bar.enabled1 = enabled; // TODO FIXME TABS GAHHHHH
 	else
 		ndev->genid_bar.enabled0 = enabled;
 
@@ -1314,59 +1314,6 @@ static const struct phb_ops npu_ops = {
 	.tce_kill		= npu2_tce_kill,
 };
 
-static void assign_mmio_bars(uint64_t gcid, uint32_t scom, uint64_t reg[2], uint64_t mm_win[2])
-{
-	uint32_t i;
-	struct npu2_bar *bar;
-	struct npu2_bar npu2_bars[] = {
-		/* NPU_REGS must be first in this list */
-		{ .type = NPU_REGS, .index = 0,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR),
-		  .enabled = true },
-		{ .type = NPU_PHY, .index = 0,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR),
-		  .enabled = true },
-		{ .type = NPU_PHY, .index = 1,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR),
-		  .enabled = true },
-		{ .type = NPU_NTL, .index = 0,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_NTL0_BAR) },
-		{ .type = NPU_NTL, .index = 1,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_NTL1_BAR) },
-		{ .type = NPU_NTL, .index = 2,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_NTL0_BAR) },
-		{ .type = NPU_NTL, .index = 3,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_NTL1_BAR) },
-		{ .type = NPU_NTL, .index = 4,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_NTL0_BAR) },
-		{ .type = NPU_NTL, .index = 5,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_NTL1_BAR) },
-		{ .type = NPU_GENID, .index = 0,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_GENID_BAR) },
-		{ .type = NPU_GENID, .index = 1,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_GENID_BAR) },
-		{ .type = NPU_GENID, .index = 2,
-		  .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_GENID_BAR) },
-	};
-
-	for (i = 0; i < ARRAY_SIZE(npu2_bars); i++) {
-		bar = &npu2_bars[i];
-		npu2_get_bar(gcid, bar);
-		npu2_write_bar(NULL, bar, gcid, scom);
-	}
-
-	/* Global MMIO BAR */
-	reg[0] = npu2_bars[0].base;
-	reg[1] = npu2_bars[0].size;
-
-	/* NTL and GENID BARs are exposed to kernel via the mm
-	 * window */
-	mm_win[0] = npu2_bars[3].base;
-	mm_win[1] = npu2_bars[ARRAY_SIZE(npu2_bars) - 1].base +
-		    npu2_bars[ARRAY_SIZE(npu2_bars) - 1].size -
-		    mm_win[0];
-}
-
 /*
  * Set up NPU for NVLink and create PCI root device node
  * accordingly.
@@ -1428,19 +1375,9 @@ int npu2_nvlink_init_npu(struct npu2 *npu)
 	xscom_write_mask(npu->chip_id, 0x5011469, val, PPC_BITMASK(6,11));
 	xscom_write_mask(npu->chip_id, 0x5011499, val, PPC_BITMASK(6,11));
 
-	/* Reassign the BARs */
-	assign_mmio_bars(npu->chip_id, npu->xscom_base, reg, mm_win);
-	npu->regs = (void *)reg[0];
-	npu->mm_base = mm_win[0];
-	npu->mm_size = mm_win[1];
-
-	if (reg[0] && reg[1])
-		prlog(PR_INFO, "   Global MMIO BAR:  %016llx (%lldMB)\n",
-		      reg[0], reg[1] >> 20);
-	else
-		prlog(PR_ERR, "    Global MMIO BAR: Disabled\n");
-
 	/* Populate PCI root device node */
+	reg[0] = (uint64_t)npu->regs;
+	reg[1] = npu->regs_size;
 	np = dt_new_addr(dt_root, "pciex", reg[0]);
 	assert(np);
 	dt_add_property_strings(np,
@@ -1455,6 +1392,8 @@ int npu2_nvlink_init_npu(struct npu2 *npu)
 	dt_add_property_cells(np, "ibm,xscom-base", npu->xscom_base);
 	dt_add_property_cells(np, "ibm,npcq", npu->dt_node->phandle);
 	dt_add_property_cells(np, "ibm,links", npu->total_devices);
+	mm_win[0] = npu->mm_base;
+	mm_win[1] = npu->mm_size;
 	dt_add_property(np, "ibm,mmio-window", mm_win, sizeof(mm_win));
 	dt_add_property_cells(np, "ibm,phb-diag-data-size", 0);
 
@@ -1679,7 +1618,6 @@ static void npu2_populate_devices(struct npu2 *p,
 	struct npu2_dev *dev;
 	struct dt_node *npu2_dn, *link;
 	uint32_t npu_phandle, index = 0;
-	int stack;
 
 	/*
 	 * Get the npu node which has the links which we expand here
@@ -1713,32 +1651,6 @@ static void npu2_populate_devices(struct npu2 *p,
 		dev->pl_xscom_base = dt_prop_get_u64(link, "ibm,npu-phy");
 		dev->lane_mask = dt_prop_get_u32(link, "ibm,npu-lane-mask");
 
-		/* Populate BARs. BAR0/1 is the NTL bar. */
-		stack = NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev);
-		dev->ntl_bar.type = NPU_NTL;
-		dev->ntl_bar.index = dev->brick_index;
-		dev->ntl_bar.reg = NPU2_REG_OFFSET(stack, 0,
-						   NPU2DEV_BRICK(dev) == 0 ?
-						   NPU2_NTL0_BAR :
-						   NPU2_NTL1_BAR);
-		npu2_get_bar(p->chip_id, &dev->ntl_bar);
-
-		dev->ntl_bar.flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
-
-		/* BAR2/3 is the GENID bar. */
-		dev->genid_bar.type = NPU_GENID;
-		dev->genid_bar.index = NPU2DEV_STACK(dev);
-		dev->genid_bar.reg = NPU2_REG_OFFSET(stack, 0,
-						     NPU2_GENID_BAR);
-		npu2_get_bar(p->chip_id, &dev->genid_bar);
-
-		/* The GENID is a single physical BAR that we split
-		 * for each emulated device */
-		dev->genid_bar.size = 0x10000;
-		if (NPU2DEV_BRICK(dev))
-			dev->genid_bar.base += 0x10000;
-		dev->genid_bar.flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
-
 		/* Initialize PCI virtual device */
 		dev->nvlink.pvd = pci_virt_add_device(&p->phb_nvlink, dev->bdfn, 0x100, dev);
 		if (dev->nvlink.pvd)
diff --git a/include/npu2.h b/include/npu2.h
index c2ba4370c063..31d9a4a1053f 100644
--- a/include/npu2.h
+++ b/include/npu2.h
@@ -147,6 +147,7 @@ struct npu2 {
 	uint32_t	chip_id;
 	uint64_t	xscom_base;
 	void		*regs;
+	uint64_t	regs_size;
 	uint64_t	mm_base;
 	uint64_t	mm_size;
 	uint32_t	base_lsi;
-- 
git-series 0.9.1



More information about the Skiboot mailing list