[Skiboot] [PATCH] phb4: Enable PCI peer-to-peer
Frederic Barrat
fbarrat at linux.vnet.ibm.com
Tue May 30 23:54:11 AEST 2017
P9 supports PCI peer-to-peer: a PCI device can write directly to the
mmio space of another PCI device. It completely by-passes the
CPU. Reads are not supported for the time being.
It requires some configuration on the PHBs involved:
1. on the sending side, the address used is in the mmio space of the
receiver, i.e. well outside the range normally allowed. So when
setting the TVT entry in bypass mode, we disable range-checking.
2. on the receiving side, we need to explicitly enable p2p by setting a
bit in a configuration register. It has the side-effect of reserving
an outbound (as seen from the CPU) store queue for p2p. Therefore we only
enable p2p on the PHBs using it, as we don't want to waste the
resource if we don't have to.
The patch adds a new OPAL call to allow the OS to configure the PHBs.
Signed-off-by: Frederic Barrat <fbarrat at linux.vnet.ibm.com>
---
Stewart: this patch goes hand-in-hand with a linux patch to be posted
shortly (will add reference to it). It shouldn't be merged until the
linux patch is agreed upon, most notably around the APIs.
core/pci-opal.c | 17 ++++++++++
doc/opal-api/opal-pci-set-p2p-148.rst | 39 +++++++++++++++++++++++
hw/phb4.c | 59 +++++++++++++++++++++++++++++------
include/opal-api.h | 9 +++++-
include/pci.h | 3 ++
include/phb4-regs.h | 1 +
include/phb4.h | 1 +
7 files changed, 118 insertions(+), 11 deletions(-)
create mode 100644 doc/opal-api/opal-pci-set-p2p-148.rst
diff --git a/core/pci-opal.c b/core/pci-opal.c
index 27872aa2..0389e4d9 100644
--- a/core/pci-opal.c
+++ b/core/pci-opal.c
@@ -977,3 +977,20 @@ static int64_t opal_pci_set_phb_capi_mode(uint64_t phb_id, uint64_t mode, uint64
return rc;
}
opal_call(OPAL_PCI_SET_PHB_CAPI_MODE, opal_pci_set_phb_capi_mode, 3);
+
+static int64_t opal_pci_set_p2p(uint64_t phb_id, uint64_t mode)
+{
+ struct phb *phb = pci_get_phb(phb_id);
+ int64_t rc;
+
+ if (!phb)
+ return OPAL_PARAMETER;
+ if (!phb->ops->set_p2p)
+ return OPAL_UNSUPPORTED;
+
+ phb_lock(phb);
+ rc = phb->ops->set_p2p(phb, mode);
+ phb_unlock(phb);
+ return rc;
+}
+opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 2);
diff --git a/doc/opal-api/opal-pci-set-p2p-148.rst b/doc/opal-api/opal-pci-set-p2p-148.rst
new file mode 100644
index 00000000..8c613810
--- /dev/null
+++ b/doc/opal-api/opal-pci-set-p2p-148.rst
@@ -0,0 +1,39 @@
+OPAL_PCI_SET_P2P
+================
+::
+
+ #define OPAL_PCI_SET_P2P 149
+
+ int64_t opal_pci_set_p2p(uint64_t phb_id, uint64_t mode)
+
+ enum {
+ OPAL_PCI_P2P_SENDER = 0,
+ OPAL_PCI_P2P_RECEIVER = 1
+ };
+
+The host calls this function to enable PCI peer-to-peer on the PHBs.
+
+Parameters
+----------
+::
+
+ uint64_t phb_id
+ uint64_t mode
+
+``phb_id``
+ is the value from the PHB node ibm,opal-phbid property.
+
+``mode``
+ tells whether the PHB should be configured as a receiver (OPAL_PCI_P2P_RECEIVER) or sender (OPAL_PCI_P2P_SENDER)
+
+Return Values
+-------------
+
+``OPAL_SUCCESS``
+ Configuration was successful
+
+``OPAL_PARAMETER``
+ Invalid PHB or mode parameter
+
+``OPAL_UNSUPPORTED``
+ Not supported by hardware
diff --git a/hw/phb4.c b/hw/phb4.c
index 318d934d..6894a981 100644
--- a/hw/phb4.c
+++ b/hw/phb4.c
@@ -1343,16 +1343,27 @@ static int64_t phb4_map_pe_dma_window_real(struct phb *phb,
if (end > 0x0003ffffffffffffull)
return OPAL_PARAMETER;
- /*
- * Put start address bits 49:24 into TVE[52:53]||[0:23]
- * and end address bits 49:24 into TVE[54:55]||[24:47]
- * and set TVE[51]
- */
- tve = (pci_start_addr << 16) & (0xffffffull << 40);
- tve |= (pci_start_addr >> 38) & (3ull << 10);
- tve |= (end >> 8) & (0xfffffful << 16);
- tve |= (end >> 40) & (3ull << 8);
- tve |= PPC_BIT(51) | IODA3_TVT_NON_TRANSLATE_50;
+ if (p->flags & PHB4_P2P_SENDER) {
+ /*
+ * for PCI peer-to-peer, we need to let go any valid
+ * memory address + any MMIO address of the receiver.
+ * That's pretty much everything.
+ * => 56 bit addr, no translate, no range checking
+ */
+ tve = PPC_BIT(51);
+ PHBDBG(p, "p2p is enabled, overriding TVE setting\n");
+ } else {
+ /*
+ * Put start address bits 49:24 into TVE[52:53]||[0:23]
+ * and end address bits 49:24 into TVE[54:55]||[24:47]
+ * and set TVE[51]
+ */
+ tve = (pci_start_addr << 16) & (0xffffffull << 40);
+ tve |= (pci_start_addr >> 38) & (3ull << 10);
+ tve |= (end >> 8) & (0xfffffful << 16);
+ tve |= (end >> 40) & (3ull << 8);
+ tve |= PPC_BIT(51) | IODA3_TVT_NON_TRANSLATE_50;
+ }
} else {
/* Disable */
tve = 0;
@@ -2135,6 +2146,7 @@ static int64_t phb4_creset(struct pci_slot *slot)
case PHB4_SLOT_CRESET_REINIT:
p->flags &= ~PHB4_AIB_FENCED;
p->flags &= ~PHB4_CAPP_RECOVERY;
+ p->flags &= ~PHB4_P2P_SENDER;
phb4_init_hw(p, false);
pci_slot_set_state(slot, PHB4_SLOT_CRESET_FRESET);
return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
@@ -2507,6 +2519,32 @@ static int64_t phb4_get_diag_data(struct phb *phb,
return OPAL_SUCCESS;
}
+static void enable_p2p_receiver(struct phb4 *p)
+{
+ uint64_t val;
+
+ xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE, &val);
+ val |= XPEC_NEST_STK_PBCQ_MODE_P2P;
+ xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE, val);
+}
+
+static int64_t phb4_set_p2p(struct phb *phb, uint64_t mode)
+{
+ struct phb4 *p = phb_to_phb4(phb);
+
+ switch (mode) {
+ case OPAL_PCI_P2P_SENDER:
+ p->flags |= PHB4_P2P_SENDER;
+ break;
+ case OPAL_PCI_P2P_RECEIVER:
+ enable_p2p_receiver(p);
+ break;
+ default:
+ return OPAL_PARAMETER;
+ }
+ return OPAL_SUCCESS;
+}
+
static const struct phb_ops phb4_ops = {
.cfg_read8 = phb4_pcicfg_read8,
.cfg_read16 = phb4_pcicfg_read16,
@@ -2539,6 +2577,7 @@ static const struct phb_ops phb4_ops = {
.get_diag_data = NULL,
.get_diag_data2 = phb4_get_diag_data,
.tce_kill = phb4_tce_kill,
+ .set_p2p = phb4_set_p2p,
};
static void phb4_init_ioda3(struct phb4 *p)
diff --git a/include/opal-api.h b/include/opal-api.h
index 80033c6f..8511d7de 100644
--- a/include/opal-api.h
+++ b/include/opal-api.h
@@ -204,7 +204,8 @@
#define OPAL_NPU_INIT_CONTEXT 146
#define OPAL_NPU_DESTROY_CONTEXT 147
#define OPAL_NPU_MAP_LPAR 148
-#define OPAL_LAST 148
+#define OPAL_PCI_SET_P2P 149
+#define OPAL_LAST 149
/* Device tree flags */
@@ -1215,6 +1216,12 @@ enum {
XIVE_DUMP_EMU_STATE = 5,
};
+/* PCI p2p modes */
+enum {
+ OPAL_PCI_P2P_SENDER = 0,
+ OPAL_PCI_P2P_RECEIVER = 1,
+};
+
#endif /* __ASSEMBLY__ */
#endif /* __OPAL_API_H */
diff --git a/include/pci.h b/include/pci.h
index dc418a90..98a7491d 100644
--- a/include/pci.h
+++ b/include/pci.h
@@ -319,6 +319,9 @@ struct phb_ops {
uint64_t pe_number);
int64_t (*set_capp_recovery)(struct phb *phb);
+
+ /* PCI peer-to-peer setup */
+ int64_t (*set_p2p)(struct phb *phb, uint64_t mode);
};
enum phb_type {
diff --git a/include/phb4-regs.h b/include/phb4-regs.h
index 92bee88f..cdf1bb94 100644
--- a/include/phb4-regs.h
+++ b/include/phb4-regs.h
@@ -321,6 +321,7 @@
#define XPEC_NEST_STK_ERR_RPT1 0xb
#define XPEC_NEST_STK_PBCQ_STAT 0xc
#define XPEC_NEST_STK_PBCQ_MODE 0xd
+#define XPEC_NEST_STK_PBCQ_MODE_P2P PPC_BIT(0)
#define XPEC_NEST_STK_MMIO_BAR0 0xe
#define XPEC_NEST_STK_MMIO_BAR0_MASK 0xf
#define XPEC_NEST_STK_MMIO_BAR1 0x10
diff --git a/include/phb4.h b/include/phb4.h
index bed6eef0..fdfde267 100644
--- a/include/phb4.h
+++ b/include/phb4.h
@@ -222,6 +222,7 @@ struct phb4_err {
#define PHB4_CFG_USE_ASB 0x00000002
#define PHB4_CFG_BLOCKED 0x00000004
#define PHB4_CAPP_RECOVERY 0x00000008
+#define PHB4_P2P_SENDER 0x00000010
struct phb4 {
unsigned int index; /* 0..5 index inside p9 */
--
2.11.0
More information about the Skiboot
mailing list