[Skiboot] [PATCH] phb4: Enable PCI peer-to-peer
Michael Neuling
mikey at neuling.org
Wed May 31 16:11:45 AEST 2017
On Tue, 2017-05-30 at 15:54 +0200, Frederic Barrat wrote:
> P9 supports PCI peer-to-peer: a PCI device can write directly to the
> mmio space of another PCI device. It completely by-passes the
> CPU. Reads are not supported for the time being.
>
> It requires some configuration on the PHBs involved:
> 1. on the sending side, the address used is in the mmio space of the
> receiver, i.e. well outside the range normally allowed. So when
> setting the TVT entry in bypass mode, we disable range-checking.
The way we have it here, makes the OPAL_PCI_MAP_PE_DMA_WINDOW_REAL modal
depending of if this new call has been done or not. I'm wondering if it would be
better to extend the existing call to add a parameter/flag that enables the p2p
sending mode, rather than making the existing call now change what it does based
on this new call being done or not.
Also, can phb4 mix tve entries which have P2P enabled vs disabled? If not, we
should check the tve cache for existing entires to make sure there is nothing
already in there which has this in the other mode.
Also, as benh alluded to in the linux side, we need a disable call for this (and
the receive call)
Mikey
> 2. on the receiving side, we need to explicitly enable p2p by setting a
> bit in a configuration register. It has the side-effect of reserving
> an outbound (as seen from the CPU) store queue for p2p. Therefore we only
> enable p2p on the PHBs using it, as we don't want to waste the
> resource if we don't have to.
>
> The patch adds a new OPAL call to allow the OS to configure the PHBs.
>
> Signed-off-by: Frederic Barrat <fbarrat at linux.vnet.ibm.com>
> ---
>
> Stewart: this patch goes hand-in-hand with a linux patch to be posted
> shortly (will add reference to it). It shouldn't be merged until the
> linux patch is agreed upon, most notably around the APIs.
>
> core/pci-opal.c | 17 ++++++++++
> doc/opal-api/opal-pci-set-p2p-148.rst | 39 +++++++++++++++++++++++
> hw/phb4.c | 59 +++++++++++++++++++++++++++++-----
> -
> include/opal-api.h | 9 +++++-
> include/pci.h | 3 ++
> include/phb4-regs.h | 1 +
> include/phb4.h | 1 +
> 7 files changed, 118 insertions(+), 11 deletions(-)
> create mode 100644 doc/opal-api/opal-pci-set-p2p-148.rst
>
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index 27872aa2..0389e4d9 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -977,3 +977,20 @@ static int64_t opal_pci_set_phb_capi_mode(uint64_t
> phb_id, uint64_t mode, uint64
> return rc;
> }
> opal_call(OPAL_PCI_SET_PHB_CAPI_MODE, opal_pci_set_phb_capi_mode, 3);
> +
> +static int64_t opal_pci_set_p2p(uint64_t phb_id, uint64_t mode)
> +{
> + struct phb *phb = pci_get_phb(phb_id);
> + int64_t rc;
> +
> + if (!phb)
> + return OPAL_PARAMETER;
> + if (!phb->ops->set_p2p)
> + return OPAL_UNSUPPORTED;
> +
> + phb_lock(phb);
> + rc = phb->ops->set_p2p(phb, mode);
> + phb_unlock(phb);
> + return rc;
> +}
> +opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 2);
> diff --git a/doc/opal-api/opal-pci-set-p2p-148.rst b/doc/opal-api/opal-pci-
> set-p2p-148.rst
> new file mode 100644
> index 00000000..8c613810
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-set-p2p-148.rst
> @@ -0,0 +1,39 @@
> +OPAL_PCI_SET_P2P
> +================
> +::
> +
> + #define OPAL_PCI_SET_P2P 149
> +
> + int64_t opal_pci_set_p2p(uint64_t phb_id, uint64_t mode)
> +
> + enum {
> + OPAL_PCI_P2P_SENDER = 0,
> + OPAL_PCI_P2P_RECEIVER = 1
> + };
> +
> +The host calls this function to enable PCI peer-to-peer on the PHBs.
> +
> +Parameters
> +----------
> +::
> +
> + uint64_t phb_id
> + uint64_t mode
> +
> +``phb_id``
> + is the value from the PHB node ibm,opal-phbid property.
> +
> +``mode``
> + tells whether the PHB should be configured as a receiver
> (OPAL_PCI_P2P_RECEIVER) or sender (OPAL_PCI_P2P_SENDER)
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> + Configuration was successful
> +
> +``OPAL_PARAMETER``
> + Invalid PHB or mode parameter
> +
> +``OPAL_UNSUPPORTED``
> + Not supported by hardware
> diff --git a/hw/phb4.c b/hw/phb4.c
> index 318d934d..6894a981 100644
> --- a/hw/phb4.c
> +++ b/hw/phb4.c
> @@ -1343,16 +1343,27 @@ static int64_t phb4_map_pe_dma_window_real(struct phb
> *phb,
> if (end > 0x0003ffffffffffffull)
> return OPAL_PARAMETER;
>
> - /*
> - * Put start address bits 49:24 into TVE[52:53]||[0:23]
> - * and end address bits 49:24 into TVE[54:55]||[24:47]
> - * and set TVE[51]
> - */
> - tve = (pci_start_addr << 16) & (0xffffffull << 40);
> - tve |= (pci_start_addr >> 38) & (3ull << 10);
> - tve |= (end >> 8) & (0xfffffful << 16);
> - tve |= (end >> 40) & (3ull << 8);
> - tve |= PPC_BIT(51) | IODA3_TVT_NON_TRANSLATE_50;
> + if (p->flags & PHB4_P2P_SENDER) {
> + /*
> + * for PCI peer-to-peer, we need to let go any valid
> + * memory address + any MMIO address of the receiver.
> + * That's pretty much everything.
> + * => 56 bit addr, no translate, no range checking
> + */
> + tve = PPC_BIT(51);
> + PHBDBG(p, "p2p is enabled, overriding TVE
> setting\n");
> + } else {
> + /*
> + * Put start address bits 49:24 into
> TVE[52:53]||[0:23]
> + * and end address bits 49:24 into
> TVE[54:55]||[24:47]
> + * and set TVE[51]
> + */
> + tve = (pci_start_addr << 16) & (0xffffffull << 40);
> + tve |= (pci_start_addr >> 38) & (3ull << 10);
> + tve |= (end >> 8) & (0xfffffful << 16);
> + tve |= (end >> 40) & (3ull << 8);
> + tve |= PPC_BIT(51) | IODA3_TVT_NON_TRANSLATE_50;
> + }
> } else {
> /* Disable */
> tve = 0;
> @@ -2135,6 +2146,7 @@ static int64_t phb4_creset(struct pci_slot *slot)
> case PHB4_SLOT_CRESET_REINIT:
> p->flags &= ~PHB4_AIB_FENCED;
> p->flags &= ~PHB4_CAPP_RECOVERY;
> + p->flags &= ~PHB4_P2P_SENDER;
> phb4_init_hw(p, false);
> pci_slot_set_state(slot, PHB4_SLOT_CRESET_FRESET);
> return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
> @@ -2507,6 +2519,32 @@ static int64_t phb4_get_diag_data(struct phb *phb,
> return OPAL_SUCCESS;
> }
>
> +static void enable_p2p_receiver(struct phb4 *p)
> +{
> + uint64_t val;
> +
> + xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE,
> &val);
> + val |= XPEC_NEST_STK_PBCQ_MODE_P2P;
> + xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE,
> val);
> +}
> +
> +static int64_t phb4_set_p2p(struct phb *phb, uint64_t mode)
> +{
> + struct phb4 *p = phb_to_phb4(phb);
> +
> + switch (mode) {
> + case OPAL_PCI_P2P_SENDER:
> + p->flags |= PHB4_P2P_SENDER;
> + break;
> + case OPAL_PCI_P2P_RECEIVER:
> + enable_p2p_receiver(p);
> + break;
> + default:
> + return OPAL_PARAMETER;
> + }
> + return OPAL_SUCCESS;
> +}
> +
> static const struct phb_ops phb4_ops = {
> .cfg_read8 = phb4_pcicfg_read8,
> .cfg_read16 = phb4_pcicfg_read16,
> @@ -2539,6 +2577,7 @@ static const struct phb_ops phb4_ops = {
> .get_diag_data = NULL,
> .get_diag_data2 = phb4_get_diag_data,
> .tce_kill = phb4_tce_kill,
> + .set_p2p = phb4_set_p2p,
> };
>
> static void phb4_init_ioda3(struct phb4 *p)
> diff --git a/include/opal-api.h b/include/opal-api.h
> index 80033c6f..8511d7de 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -204,7 +204,8 @@
> #define OPAL_NPU_INIT_CONTEXT 146
> #define OPAL_NPU_DESTROY_CONTEXT 147
> #define OPAL_NPU_MAP_LPAR 148
> -#define OPAL_LAST 148
> +#define OPAL_PCI_SET_P2P 149
> +#define OPAL_LAST 149
>
> /* Device tree flags */
>
> @@ -1215,6 +1216,12 @@ enum {
> XIVE_DUMP_EMU_STATE = 5,
> };
>
> +/* PCI p2p modes */
> +enum {
> + OPAL_PCI_P2P_SENDER = 0,
> + OPAL_PCI_P2P_RECEIVER = 1,
> +};
> +
> #endif /* __ASSEMBLY__ */
>
> #endif /* __OPAL_API_H */
> diff --git a/include/pci.h b/include/pci.h
> index dc418a90..98a7491d 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -319,6 +319,9 @@ struct phb_ops {
> uint64_t pe_number);
>
> int64_t (*set_capp_recovery)(struct phb *phb);
> +
> + /* PCI peer-to-peer setup */
> + int64_t (*set_p2p)(struct phb *phb, uint64_t mode);
> };
>
> enum phb_type {
> diff --git a/include/phb4-regs.h b/include/phb4-regs.h
> index 92bee88f..cdf1bb94 100644
> --- a/include/phb4-regs.h
> +++ b/include/phb4-regs.h
> @@ -321,6 +321,7 @@
> #define XPEC_NEST_STK_ERR_RPT1 0xb
> #define XPEC_NEST_STK_PBCQ_STAT 0xc
> #define XPEC_NEST_STK_PBCQ_MODE 0xd
> +#define XPEC_NEST_STK_PBCQ_MODE_P2P PPC_BIT(0)
> #define XPEC_NEST_STK_MMIO_BAR0 0xe
> #define XPEC_NEST_STK_MMIO_BAR0_MASK 0xf
> #define XPEC_NEST_STK_MMIO_BAR1 0x10
> diff --git a/include/phb4.h b/include/phb4.h
> index bed6eef0..fdfde267 100644
> --- a/include/phb4.h
> +++ b/include/phb4.h
> @@ -222,6 +222,7 @@ struct phb4_err {
> #define PHB4_CFG_USE_ASB 0x00000002
> #define PHB4_CFG_BLOCKED 0x00000004
> #define PHB4_CAPP_RECOVERY 0x00000008
> +#define PHB4_P2P_SENDER 0x00000010
>
> struct phb4 {
> unsigned int index; /* 0..5 index inside p9
> */
More information about the Skiboot
mailing list