[Skiboot] [PATCH v3] phb4: Enable PCI peer-to-peer
Frederic Barrat
fbarrat at linux.vnet.ibm.com
Tue Jul 18 09:56:48 AEST 2017
Related linux patch:
http://patchwork.ozlabs.org/patch/789773/
Le 17/07/2017 à 18:39, Frederic Barrat a écrit :
> P9 supports PCI peer-to-peer: a PCI device can write directly to the
> mmio space of another PCI device. It completely by-passes the CPU.
>
> It requires some configuration on the PHBs involved:
>
> 1. on the initiating side, the address for the read/write operation is
> in the mmio space of the target, i.e. well outside the range normally
> allowed. So we disable range-checking on the TVT entry in bypass mode.
>
> 2. on the target side, we need to explicitly enable p2p by setting a
> bit in a configuration register. It has the side-effect of reserving
> an outbound (as seen from the CPU) store queue for p2p. Therefore we
> only enable p2p on the PHBs using it, as we don't want to waste the
> resource if we don't have to.
>
> P9 supports p2p mmio writes. Reads are currently only supported if the
> two devices are under the same PHB but that is expected to change in
> the future, and it raises questions about intermediate switches
> configuration, so we report an error for the time being.
>
> The patch adds a new OPAL call to allow the OS to declare a p2p
> (initiator, target) pair.
>
> Signed-off-by: Frederic Barrat <fbarrat at linux.vnet.ibm.com>
> ---
>
> Changelog:
> v3:
> - move target reference count to linux
> - disable any configuration with the 2 devices under the same PHB
> v2:
> - change of API
> - allow disabling of p2p config
>
> core/pci-opal.c | 39 ++++++++++++++++++++++
> doc/opal-api/opal-pci-set-p2p-152.rst | 50 ++++++++++++++++++++++++++++
> hw/phb4.c | 62 +++++++++++++++++++++++++++++++++++
> include/opal-api.h | 13 +++++++-
> include/pci.h | 4 +++
> include/phb4-regs.h | 1 +
> 6 files changed, 168 insertions(+), 1 deletion(-)
> create mode 100644 doc/opal-api/opal-pci-set-p2p-152.rst
>
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index 5d58a88f..b8aec941 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -977,3 +977,42 @@ static int64_t opal_pci_set_phb_capi_mode(uint64_t phb_id, uint64_t mode, uint64
> return rc;
> }
> opal_call(OPAL_PCI_SET_PHB_CAPI_MODE, opal_pci_set_phb_capi_mode, 3);
> +
> +static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
> + uint64_t desc, uint16_t pe_number)
> +{
> + struct phb *phb_init = pci_get_phb(phbid_init);
> + struct phb *phb_target = pci_get_phb(phbid_target);
> +
> + if (!phb_init || !phb_target)
> + return OPAL_PARAMETER;
> + /*
> + * Having the 2 devices under the same PHB may require tuning
> + * the configuration of intermediate switch(es), more easily
> + * done from linux. And it shouldn't require a PHB config
> + * change.
> + * Return an error for the time being.
> + */
> + if (phb_init == phb_target)
> + return OPAL_UNSUPPORTED;
> + if (!phb_init->ops->set_p2p || !phb_target->ops->set_p2p)
> + return OPAL_UNSUPPORTED;
> + /*
> + * Loads would be supported on p9 if the 2 devices are under
> + * the same PHB, but we ruled it out above.
> + */
> + if (desc & OPAL_PCI_P2P_LOAD)
> + return OPAL_UNSUPPORTED;
> +
> + phb_lock(phb_init);
> + phb_init->ops->set_p2p(phb_init, OPAL_PCI_P2P_INITIATOR, desc,
> + pe_number);
> + phb_unlock(phb_init);
> +
> + phb_lock(phb_target);
> + phb_target->ops->set_p2p(phb_target, OPAL_PCI_P2P_TARGET, desc,
> + pe_number);
> + phb_unlock(phb_target);
> + return OPAL_SUCCESS;
> +}
> +opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
> diff --git a/doc/opal-api/opal-pci-set-p2p-152.rst b/doc/opal-api/opal-pci-set-p2p-152.rst
> new file mode 100644
> index 00000000..5b66dd4f
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-set-p2p-152.rst
> @@ -0,0 +1,50 @@
> +OPAL_PCI_SET_P2P
> +================
> +::
> +
> + #define OPAL_PCI_SET_P2P 152
> +
> + int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
> + uint64_t desc, uint16_t pe_number)
> +
> + /* PCI p2p descriptor */
> + #define OPAL_PCI_P2P_ENABLE 0x1
> + #define OPAL_PCI_P2P_LOAD 0x2
> + #define OPAL_PCI_P2P_STORE 0x4
> +
> +The host calls this function to enable PCI peer-to-peer on the PHBs.
> +
> +Parameters
> +----------
> +::
> +
> + uint64_t phbid_init
> + uint64_t phbid_target
> + uint64_t desc
> + uint16_t pe_number
> +
> +
> +``phbid_init``
> + is the value from the PHB node ibm,opal-phbid property for the device initiating the p2p operation
> +
> +``phbid_target``
> + is the value from the PHB node ibm,opal-phbid property for the device targeted by the p2p operation
> +
> +``desc``
> + tells whether the p2p operation is a store (OPAL_PCI_P2P_STORE) or load (OPAL_PCI_P2P_LOAD). Can be both.
> + OPAL_PCI_P2P_ENABLE enables/disables the setting
> +
> +``pe_number``
> + PE number for the initiating device
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> + Configuration was successful
> +
> +``OPAL_PARAMETER``
> + Invalid PHB or mode parameter
> +
> +``OPAL_UNSUPPORTED``
> + Not supported by hardware
> diff --git a/hw/phb4.c b/hw/phb4.c
> index 2f3af3ae..2fcee343 100644
> --- a/hw/phb4.c
> +++ b/hw/phb4.c
> @@ -3394,6 +3394,67 @@ static int64_t phb4_set_capi_mode(struct phb *phb, uint64_t mode,
> return OPAL_UNSUPPORTED;
> }
>
> +static void phb4_p2p_set_initiator(struct phb4 *p, uint16_t pe_number)
> +{
> + uint64_t tve;
> + uint16_t window_id = (pe_number << 1) + 1;
> +
> + /*
> + * Initiator needs access to the MMIO space of the target,
> + * which is well beyond the 'normal' memory area. Set its TVE
> + * with no range checking.
> + */
> + PHBDBG(p, "Setting TVE#1 for peer-to-peer for pe %d\n", pe_number);
> + tve = PPC_BIT(51);
> + phb4_ioda_sel(p, IODA3_TBL_TVT, window_id, false);
> + out_be64(p->regs + PHB_IODA_DATA0, tve);
> + p->tve_cache[window_id] = tve;
> +}
> +
> +static void phb4_p2p_set_target(struct phb4 *p, bool enable)
> +{
> + uint64_t val;
> +
> + /*
> + * Enabling p2p on a target PHB reserves an outbound (as seen
> + * from the CPU) store queue for p2p
> + */
> + PHBDBG(p, "%s peer-to-peer\n", (enable ? "Enabling" : "Disabling"));
> + xscom_read(p->chip_id,
> + p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE, &val);
> + if (enable)
> + val |= XPEC_NEST_STK_PBCQ_MODE_P2P;
> + else
> + val &= ~XPEC_NEST_STK_PBCQ_MODE_P2P;
> + xscom_write(p->chip_id,
> + p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE, val);
> +}
> +
> +static void phb4_set_p2p(struct phb *phb, uint64_t mode, uint64_t flags,
> + uint16_t pe_number)
> +{
> + struct phb4 *p = phb_to_phb4(phb);
> +
> + switch (mode) {
> + case OPAL_PCI_P2P_INITIATOR:
> + if (flags & OPAL_PCI_P2P_ENABLE)
> + phb4_p2p_set_initiator(p, pe_number);
> + /*
> + * When disabling p2p on the initiator, we should
> + * reset the TVE to its default bypass setting, but it
> + * is more easily done from the OS, as it knows the
> + * the start and end address and there's already an
> + * opal call for it, so let linux handle it.
> + */
> + break;
> + case OPAL_PCI_P2P_TARGET:
> + phb4_p2p_set_target(p, !!(flags & OPAL_PCI_P2P_ENABLE));
> + break;
> + default:
> + assert(0);
> + }
> +}
> +
> static const struct phb_ops phb4_ops = {
> .cfg_read8 = phb4_pcicfg_read8,
> .cfg_read16 = phb4_pcicfg_read16,
> @@ -3427,6 +3488,7 @@ static const struct phb_ops phb4_ops = {
> .get_diag_data2 = phb4_get_diag_data,
> .tce_kill = phb4_tce_kill,
> .set_capi_mode = phb4_set_capi_mode,
> + .set_p2p = phb4_set_p2p,
> };
>
> static void phb4_init_ioda3(struct phb4 *p)
> diff --git a/include/opal-api.h b/include/opal-api.h
> index edae0693..58b8c8b9 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -207,7 +207,8 @@
> #define OPAL_IMC_COUNTERS_INIT 149
> #define OPAL_IMC_COUNTERS_START 150
> #define OPAL_IMC_COUNTERS_STOP 151
> -#define OPAL_LAST 151
> +#define OPAL_PCI_SET_P2P 152
> +#define OPAL_LAST 152
>
> /* Device tree flags */
>
> @@ -1255,6 +1256,16 @@ enum {
> };
>
>
> +/* PCI p2p descriptor */
> +#define OPAL_PCI_P2P_ENABLE 0x1
> +#define OPAL_PCI_P2P_LOAD 0x2
> +#define OPAL_PCI_P2P_STORE 0x4
> +
> +enum {
> + OPAL_PCI_P2P_INITIATOR = 0,
> + OPAL_PCI_P2P_TARGET = 1,
> +};
> +
> #endif /* __ASSEMBLY__ */
>
> #endif /* __OPAL_API_H */
> diff --git a/include/pci.h b/include/pci.h
> index f216594c..54a62fd1 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -323,6 +323,10 @@ struct phb_ops {
> uint64_t pe_number);
>
> int64_t (*set_capp_recovery)(struct phb *phb);
> +
> + /* PCI peer-to-peer setup */
> + void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
> + uint16_t pe_number);
> };
>
> enum phb_type {
> diff --git a/include/phb4-regs.h b/include/phb4-regs.h
> index 14e93511..a7842a36 100644
> --- a/include/phb4-regs.h
> +++ b/include/phb4-regs.h
> @@ -334,6 +334,7 @@
> #define XPEC_NEST_STK_ERR_RPT1 0xb
> #define XPEC_NEST_STK_PBCQ_STAT 0xc
> #define XPEC_NEST_STK_PBCQ_MODE 0xd
> +#define XPEC_NEST_STK_PBCQ_MODE_P2P PPC_BIT(0)
> #define XPEC_NEST_STK_MMIO_BAR0 0xe
> #define XPEC_NEST_STK_MMIO_BAR0_MASK 0xf
> #define XPEC_NEST_STK_MMIO_BAR1 0x10
>
More information about the Skiboot
mailing list