[Skiboot] [PATCH v2 04/15] hw/npu2: Move PHY/NTL/GENID BAR assignment to common code
Andrew Donnellan
andrew.donnellan at au1.ibm.com
Fri Jan 11 12:09:24 AEDT 2019
Assignment of PHY/NTL/GENID BARs is currently duplicated between NVLink
and OpenCAPI. This is going to cause us particular issues later on when we
implement support for mixed-mode setups with NVLink and OpenCAPI on the
same NPU.
Centralise the assignment of PHY/NTL/GENID BARs in common code.
Signed-off-by: Andrew Donnellan <andrew.donnellan at au1.ibm.com>
---
v1->v2:
- keep config bar enablement in separate function (Fred)
- fix GENID BAR assignment for OpenCAPI OTL1
- split patch (Alexey)
---
hw/npu2-common.c | 78 +++++++++++++++++++++++++++++++++++++-
hw/npu2-opencapi.c | 61 +++--------------------------
hw/npu2.c | 98 ++---------------------------------------------
include/npu2.h | 1 +-
4 files changed, 91 insertions(+), 147 deletions(-)
diff --git a/hw/npu2-common.c b/hw/npu2-common.c
index 3446acb45bea..944719d2489a 100644
--- a/hw/npu2-common.c
+++ b/hw/npu2-common.c
@@ -17,6 +17,7 @@
#include <skiboot.h>
#include <xscom.h>
#include <pci.h>
+#include <pci-cfg.h>
#include <npu2.h>
#include <npu2-regs.h>
#include <bitutils.h>
@@ -174,6 +175,81 @@ void npu2_write_bar(struct npu2 *p, struct npu2_bar *bar, uint32_t gcid,
}
}
+static void assign_bars(struct npu2 *npu)
+{
+ uint32_t i;
+ struct npu2_bar *bar;
+ struct npu2_dev *dev;
+ struct npu2_bar phy_bars[] = {
+ /* NPU_REGS must be first in this list */
+ { .type = NPU_REGS, .index = 0,
+ .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR),
+ .enabled = true },
+ { .type = NPU_PHY, .index = 0,
+ .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR),
+ .enabled = true },
+ { .type = NPU_PHY, .index = 1,
+ .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR),
+ .enabled = true },
+ };
+ struct npu2_bar last_bar =
+ { .type = NPU_GENID, .index = 2,
+ .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_GENID_BAR) };
+
+ /* Set common PHY BARs */
+ for (i = 0; i < ARRAY_SIZE(phy_bars); i++) {
+ bar = &phy_bars[i];
+ npu2_get_bar(npu->chip_id, bar);
+ npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base);
+ }
+
+ /* Device BARs */
+ for (i = 0; i < npu->total_devices; i++) {
+ dev = &npu->devices[i];
+ if (dev->type == NPU2_DEV_TYPE_UNKNOWN)
+ continue;
+
+ /* NTL / OCAPI_MMIO BAR */
+ bar = &dev->ntl_bar;
+ if (dev->type == NPU2_DEV_TYPE_NVLINK)
+ bar->type = NPU_NTL;
+ else if (dev->type == NPU2_DEV_TYPE_OPENCAPI)
+ bar->type = NPU_OCAPI_MMIO;
+ bar->index = dev->brick_index;
+ bar->reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev),
+ 0, NPU2DEV_BRICK(dev) == 0 ?
+ NPU2_NTL0_BAR : NPU2_NTL1_BAR);
+ bar->flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
+ npu2_get_bar(npu->chip_id, bar);
+ npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base);
+
+ /* GENID BAR */
+ bar = &dev->genid_bar;
+ bar->type = NPU_GENID;
+ bar->index = NPU2DEV_STACK(dev);
+ bar->reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev),
+ 0, NPU2_GENID_BAR);
+ bar->flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
+ npu2_get_bar(npu->chip_id, bar);
+ if (dev->type == NPU2_DEV_TYPE_NVLINK) {
+ bar->size = 0x10000;
+ if (NPU2DEV_BRICK(dev))
+ bar->base += 0x10000;
+ }
+ npu2_write_bar(npu, bar, npu->chip_id, npu->xscom_base);
+ };
+
+ /* Global MMIO BAR */
+ npu->regs = (uint64_t *)phy_bars[0].base;
+ npu->regs_size = phy_bars[0].size;
+
+ /* NTL and GENID BARs are exposed to kernel via the mm
+ * window */
+ npu2_get_bar(npu->chip_id, &last_bar);
+ npu->mm_base = phy_bars[2].base + phy_bars[2].size;
+ npu->mm_size = last_bar.base + last_bar.size - npu->mm_base;
+}
+
static bool _i2c_presence_detect(struct npu2_dev *dev)
{
uint8_t state, data;
@@ -350,6 +426,8 @@ static void setup_devices(struct npu2 *npu)
return;
}
+ assign_bars(npu);
+
if (nvlink_detected)
npu2_nvlink_init_npu(npu);
else if (ocapi_detected)
diff --git a/hw/npu2-opencapi.c b/hw/npu2-opencapi.c
index 8dac552adea8..17a16be00eba 100644
--- a/hw/npu2-opencapi.c
+++ b/hw/npu2-opencapi.c
@@ -730,53 +730,19 @@ static void address_translation_config(uint32_t gcid, uint32_t scom_base,
}
}
-static void setup_global_mmio_bar(uint32_t gcid, uint32_t scom_base,
- uint64_t reg[])
-{
- struct npu2_bar *bar;
- struct npu2_bar phy_bars[] = {
- { .type = NPU_PHY, .index = 0,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR),
- .enabled = true },
- { .type = NPU_PHY, .index = 1,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR),
- .enabled = true },
- { .type = NPU_REGS, .index = 0,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR),
- .enabled = true },
- };
-
- for (int i = 0; i < ARRAY_SIZE(phy_bars); i++) {
- bar = &phy_bars[i];
- npu2_get_bar(gcid, bar);
- npu2_write_bar(NULL, bar, gcid, scom_base);
- }
- reg[0] = phy_bars[2].base;
- reg[1] = phy_bars[2].size;
-}
-
/* Procedure 13.1.3.8 - AFU MMIO Range BARs */
static void setup_afu_mmio_bars(uint32_t gcid, uint32_t scom_base,
struct npu2_dev *dev)
{
uint64_t stack = index_to_stack(dev->brick_index);
- uint64_t offset = index_to_block(dev->brick_index) == NPU2_BLOCK_OTL0 ?
- NPU2_NTL0_BAR : NPU2_NTL1_BAR;
uint64_t pa_offset = index_to_block(dev->brick_index) == NPU2_BLOCK_OTL0 ?
NPU2_CQ_CTL_MISC_MMIOPA0_CONFIG :
NPU2_CQ_CTL_MISC_MMIOPA1_CONFIG;
uint64_t reg;
prlog(PR_DEBUG, "OCAPI: %s: Setup AFU MMIO BARs\n", __func__);
- dev->ntl_bar.type = NPU_OCAPI_MMIO;
- dev->ntl_bar.index = dev->brick_index;
- dev->ntl_bar.reg = NPU2_REG_OFFSET(stack, 0, offset);
dev->ntl_bar.enabled = true;
- npu2_get_bar(gcid, &dev->ntl_bar);
-
- prlog(PR_DEBUG, "OCAPI: AFU MMIO set to %llx, size %llx\n",
- dev->ntl_bar.base, dev->ntl_bar.size);
- npu2_write_bar(NULL, &dev->ntl_bar, gcid, scom_base);
+ npu2_write_bar(dev->npu, &dev->ntl_bar, gcid, scom_base);
reg = SETFIELD(NPU2_CQ_CTL_MISC_MMIOPA_ADDR, 0ull, dev->ntl_bar.base >> 16);
reg = SETFIELD(NPU2_CQ_CTL_MISC_MMIOPA_SIZE, reg, ilog2(dev->ntl_bar.size >> 16));
@@ -788,20 +754,12 @@ static void setup_afu_mmio_bars(uint32_t gcid, uint32_t scom_base,
}
/* Procedure 13.1.3.9 - AFU Config BARs */
-static void setup_afu_config_bars(uint32_t gcid, uint32_t scom_base,
- struct npu2_dev *dev)
+static void setup_afu_config_bars(struct npu2_dev *dev)
{
- uint64_t stack = index_to_stack(dev->brick_index);
- int stack_num = stack - NPU2_STACK_STCK_0;
-
prlog(PR_DEBUG, "OCAPI: %s: Setup AFU Config BARs\n", __func__);
- dev->genid_bar.type = NPU_GENID;
- dev->genid_bar.index = stack_num;
- dev->genid_bar.reg = NPU2_REG_OFFSET(stack, 0, NPU2_GENID_BAR);
dev->genid_bar.enabled = true;
- npu2_get_bar(gcid, &dev->genid_bar);
- prlog(PR_DEBUG, "OCAPI: Assigning GENID BAR: %016llx\n", dev->genid_bar.base);
- npu2_write_bar(NULL, &dev->genid_bar, gcid, scom_base);
+ npu2_write_bar(dev->npu, &dev->genid_bar, dev->npu->chip_id,
+ dev->npu->xscom_base);
}
static void otl_enabletx(uint32_t gcid, uint32_t scom_base,
@@ -1580,8 +1538,8 @@ static void setup_device(struct npu2_dev *dev)
uint64_t mm_win[2];
/* Populate PHB device node */
- phys_map_get(dev->npu->chip_id, NPU_OCAPI_MMIO, dev->brick_index, &mm_win[0],
- &mm_win[1]);
+ mm_win[0] = dev->ntl_bar.base;
+ mm_win[1] = dev->ntl_bar.size;
prlog(PR_DEBUG, "OCAPI: Setting MMIO window to %016llx + %016llx\n",
mm_win[0], mm_win[1]);
dn_phb = dt_new_addr(dt_root, "pciex", mm_win[0]);
@@ -1627,7 +1585,7 @@ static void setup_device(struct npu2_dev *dev)
/* Procedure 13.1.3.8 - AFU MMIO Range BARs */
setup_afu_mmio_bars(dev->npu->chip_id, dev->npu->xscom_base, dev);
/* Procedure 13.1.3.9 - AFU Config BARs */
- setup_afu_config_bars(dev->npu->chip_id, dev->npu->xscom_base, dev);
+ setup_afu_config_bars(dev);
set_fence_control(dev->npu->chip_id, dev->npu->xscom_base, dev->brick_index, 0b00);
@@ -1668,7 +1626,6 @@ static void read_nvram_training_state(void)
int npu2_opencapi_init_npu(struct npu2 *npu)
{
struct npu2_dev *dev;
- uint64_t reg[2];
int rc;
assert(platform.ocapi);
@@ -1677,10 +1634,6 @@ int npu2_opencapi_init_npu(struct npu2 *npu)
/* TODO: Test OpenCAPI with fast reboot and make it work */
disable_fast_reboot("OpenCAPI device enabled");
- setup_global_mmio_bar(npu->chip_id, npu->xscom_base, reg);
-
- npu->regs = (void *)reg[0];
-
for (int i = 0; i < npu->total_devices; i++) {
dev = &npu->devices[i];
if (dev->type != NPU2_DEV_TYPE_OPENCAPI)
diff --git a/hw/npu2.c b/hw/npu2.c
index 60619f381301..b920edef3265 100644
--- a/hw/npu2.c
+++ b/hw/npu2.c
@@ -139,7 +139,7 @@ static int64_t npu2_cfg_write_cmd(void *dev,
* track the enables separately.
*/
if (NPU2DEV_BRICK(ndev))
- ndev->genid_bar.enabled1 = enabled;
+ ndev->genid_bar.enabled1 = enabled; // TODO FIXME TABS GAHHHHH
else
ndev->genid_bar.enabled0 = enabled;
@@ -1314,59 +1314,6 @@ static const struct phb_ops npu_ops = {
.tce_kill = npu2_tce_kill,
};
-static void assign_mmio_bars(uint64_t gcid, uint32_t scom, uint64_t reg[2], uint64_t mm_win[2])
-{
- uint32_t i;
- struct npu2_bar *bar;
- struct npu2_bar npu2_bars[] = {
- /* NPU_REGS must be first in this list */
- { .type = NPU_REGS, .index = 0,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR),
- .enabled = true },
- { .type = NPU_PHY, .index = 0,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR),
- .enabled = true },
- { .type = NPU_PHY, .index = 1,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR),
- .enabled = true },
- { .type = NPU_NTL, .index = 0,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_NTL0_BAR) },
- { .type = NPU_NTL, .index = 1,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_NTL1_BAR) },
- { .type = NPU_NTL, .index = 2,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_NTL0_BAR) },
- { .type = NPU_NTL, .index = 3,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_NTL1_BAR) },
- { .type = NPU_NTL, .index = 4,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_NTL0_BAR) },
- { .type = NPU_NTL, .index = 5,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_NTL1_BAR) },
- { .type = NPU_GENID, .index = 0,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_GENID_BAR) },
- { .type = NPU_GENID, .index = 1,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_GENID_BAR) },
- { .type = NPU_GENID, .index = 2,
- .reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_GENID_BAR) },
- };
-
- for (i = 0; i < ARRAY_SIZE(npu2_bars); i++) {
- bar = &npu2_bars[i];
- npu2_get_bar(gcid, bar);
- npu2_write_bar(NULL, bar, gcid, scom);
- }
-
- /* Global MMIO BAR */
- reg[0] = npu2_bars[0].base;
- reg[1] = npu2_bars[0].size;
-
- /* NTL and GENID BARs are exposed to kernel via the mm
- * window */
- mm_win[0] = npu2_bars[3].base;
- mm_win[1] = npu2_bars[ARRAY_SIZE(npu2_bars) - 1].base +
- npu2_bars[ARRAY_SIZE(npu2_bars) - 1].size -
- mm_win[0];
-}
-
/*
* Set up NPU for NVLink and create PCI root device node
* accordingly.
@@ -1428,19 +1375,9 @@ int npu2_nvlink_init_npu(struct npu2 *npu)
xscom_write_mask(npu->chip_id, 0x5011469, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011499, val, PPC_BITMASK(6,11));
- /* Reassign the BARs */
- assign_mmio_bars(npu->chip_id, npu->xscom_base, reg, mm_win);
- npu->regs = (void *)reg[0];
- npu->mm_base = mm_win[0];
- npu->mm_size = mm_win[1];
-
- if (reg[0] && reg[1])
- prlog(PR_INFO, " Global MMIO BAR: %016llx (%lldMB)\n",
- reg[0], reg[1] >> 20);
- else
- prlog(PR_ERR, " Global MMIO BAR: Disabled\n");
-
/* Populate PCI root device node */
+ reg[0] = (uint64_t)npu->regs;
+ reg[1] = npu->regs_size;
np = dt_new_addr(dt_root, "pciex", reg[0]);
assert(np);
dt_add_property_strings(np,
@@ -1455,6 +1392,8 @@ int npu2_nvlink_init_npu(struct npu2 *npu)
dt_add_property_cells(np, "ibm,xscom-base", npu->xscom_base);
dt_add_property_cells(np, "ibm,npcq", npu->dt_node->phandle);
dt_add_property_cells(np, "ibm,links", npu->total_devices);
+ mm_win[0] = npu->mm_base;
+ mm_win[1] = npu->mm_size;
dt_add_property(np, "ibm,mmio-window", mm_win, sizeof(mm_win));
dt_add_property_cells(np, "ibm,phb-diag-data-size", 0);
@@ -1679,7 +1618,6 @@ static void npu2_populate_devices(struct npu2 *p,
struct npu2_dev *dev;
struct dt_node *npu2_dn, *link;
uint32_t npu_phandle, index = 0;
- int stack;
/*
* Get the npu node which has the links which we expand here
@@ -1713,32 +1651,6 @@ static void npu2_populate_devices(struct npu2 *p,
dev->pl_xscom_base = dt_prop_get_u64(link, "ibm,npu-phy");
dev->lane_mask = dt_prop_get_u32(link, "ibm,npu-lane-mask");
- /* Populate BARs. BAR0/1 is the NTL bar. */
- stack = NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev);
- dev->ntl_bar.type = NPU_NTL;
- dev->ntl_bar.index = dev->brick_index;
- dev->ntl_bar.reg = NPU2_REG_OFFSET(stack, 0,
- NPU2DEV_BRICK(dev) == 0 ?
- NPU2_NTL0_BAR :
- NPU2_NTL1_BAR);
- npu2_get_bar(p->chip_id, &dev->ntl_bar);
-
- dev->ntl_bar.flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
-
- /* BAR2/3 is the GENID bar. */
- dev->genid_bar.type = NPU_GENID;
- dev->genid_bar.index = NPU2DEV_STACK(dev);
- dev->genid_bar.reg = NPU2_REG_OFFSET(stack, 0,
- NPU2_GENID_BAR);
- npu2_get_bar(p->chip_id, &dev->genid_bar);
-
- /* The GENID is a single physical BAR that we split
- * for each emulated device */
- dev->genid_bar.size = 0x10000;
- if (NPU2DEV_BRICK(dev))
- dev->genid_bar.base += 0x10000;
- dev->genid_bar.flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
-
/* Initialize PCI virtual device */
dev->nvlink.pvd = pci_virt_add_device(&p->phb_nvlink, dev->bdfn, 0x100, dev);
if (dev->nvlink.pvd)
diff --git a/include/npu2.h b/include/npu2.h
index c2ba4370c063..31d9a4a1053f 100644
--- a/include/npu2.h
+++ b/include/npu2.h
@@ -147,6 +147,7 @@ struct npu2 {
uint32_t chip_id;
uint64_t xscom_base;
void *regs;
+ uint64_t regs_size;
uint64_t mm_base;
uint64_t mm_size;
uint32_t base_lsi;
--
git-series 0.9.1
More information about the Skiboot
mailing list