[Skiboot] [PATCH] phb4: Enable PCI peer-to-peer
Frederic Barrat
fbarrat at linux.vnet.ibm.com
Wed May 31 00:00:39 AEST 2017
Le 30/05/2017 à 15:54, Frederic Barrat a écrit :
> P9 supports PCI peer-to-peer: a PCI device can write directly to the
> mmio space of another PCI device. It completely by-passes the
> CPU. Reads are not supported for the time being.
>
> It requires some configuration on the PHBs involved:
> 1. on the sending side, the address used is in the mmio space of the
> receiver, i.e. well outside the range normally allowed. So when
> setting the TVT entry in bypass mode, we disable range-checking.
> 2. on the receiving side, we need to explicitly enable p2p by setting a
> bit in a configuration register. It has the side-effect of reserving
> an outbound (as seen from the CPU) store queue for p2p. Therefore we only
> enable p2p on the PHBs using it, as we don't want to waste the
> resource if we don't have to.
>
> The patch adds a new OPAL call to allow the OS to configure the PHBs.
>
> Signed-off-by: Frederic Barrat <fbarrat at linux.vnet.ibm.com>
> ---
>
> Stewart: this patch goes hand-in-hand with a linux patch to be posted
> shortly (will add reference to it). It shouldn't be merged until the
> linux patch is agreed upon, most notably around the APIs.
Linux patch:
http://patchwork.ozlabs.org/patch/768617/
>
> core/pci-opal.c | 17 ++++++++++
> doc/opal-api/opal-pci-set-p2p-148.rst | 39 +++++++++++++++++++++++
> hw/phb4.c | 59 +++++++++++++++++++++++++++++------
> include/opal-api.h | 9 +++++-
> include/pci.h | 3 ++
> include/phb4-regs.h | 1 +
> include/phb4.h | 1 +
> 7 files changed, 118 insertions(+), 11 deletions(-)
> create mode 100644 doc/opal-api/opal-pci-set-p2p-148.rst
>
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index 27872aa2..0389e4d9 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -977,3 +977,20 @@ static int64_t opal_pci_set_phb_capi_mode(uint64_t phb_id, uint64_t mode, uint64
> return rc;
> }
> opal_call(OPAL_PCI_SET_PHB_CAPI_MODE, opal_pci_set_phb_capi_mode, 3);
> +
> +static int64_t opal_pci_set_p2p(uint64_t phb_id, uint64_t mode)
> +{
> + struct phb *phb = pci_get_phb(phb_id);
> + int64_t rc;
> +
> + if (!phb)
> + return OPAL_PARAMETER;
> + if (!phb->ops->set_p2p)
> + return OPAL_UNSUPPORTED;
> +
> + phb_lock(phb);
> + rc = phb->ops->set_p2p(phb, mode);
> + phb_unlock(phb);
> + return rc;
> +}
> +opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 2);
> diff --git a/doc/opal-api/opal-pci-set-p2p-148.rst b/doc/opal-api/opal-pci-set-p2p-148.rst
> new file mode 100644
> index 00000000..8c613810
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-set-p2p-148.rst
> @@ -0,0 +1,39 @@
> +OPAL_PCI_SET_P2P
> +================
> +::
> +
> + #define OPAL_PCI_SET_P2P 149
> +
> + int64_t opal_pci_set_p2p(uint64_t phb_id, uint64_t mode)
> +
> + enum {
> + OPAL_PCI_P2P_SENDER = 0,
> + OPAL_PCI_P2P_RECEIVER = 1
> + };
> +
> +The host calls this function to enable PCI peer-to-peer on the PHBs.
> +
> +Parameters
> +----------
> +::
> +
> + uint64_t phb_id
> + uint64_t mode
> +
> +``phb_id``
> + is the value from the PHB node ibm,opal-phbid property.
> +
> +``mode``
> + tells whether the PHB should be configured as a receiver (OPAL_PCI_P2P_RECEIVER) or sender (OPAL_PCI_P2P_SENDER)
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> + Configuration was successful
> +
> +``OPAL_PARAMETER``
> + Invalid PHB or mode parameter
> +
> +``OPAL_UNSUPPORTED``
> + Not supported by hardware
> diff --git a/hw/phb4.c b/hw/phb4.c
> index 318d934d..6894a981 100644
> --- a/hw/phb4.c
> +++ b/hw/phb4.c
> @@ -1343,16 +1343,27 @@ static int64_t phb4_map_pe_dma_window_real(struct phb *phb,
> if (end > 0x0003ffffffffffffull)
> return OPAL_PARAMETER;
>
> - /*
> - * Put start address bits 49:24 into TVE[52:53]||[0:23]
> - * and end address bits 49:24 into TVE[54:55]||[24:47]
> - * and set TVE[51]
> - */
> - tve = (pci_start_addr << 16) & (0xffffffull << 40);
> - tve |= (pci_start_addr >> 38) & (3ull << 10);
> - tve |= (end >> 8) & (0xfffffful << 16);
> - tve |= (end >> 40) & (3ull << 8);
> - tve |= PPC_BIT(51) | IODA3_TVT_NON_TRANSLATE_50;
> + if (p->flags & PHB4_P2P_SENDER) {
> + /*
> + * for PCI peer-to-peer, we need to let go any valid
> + * memory address + any MMIO address of the receiver.
> + * That's pretty much everything.
> + * => 56 bit addr, no translate, no range checking
> + */
> + tve = PPC_BIT(51);
> + PHBDBG(p, "p2p is enabled, overriding TVE setting\n");
> + } else {
> + /*
> + * Put start address bits 49:24 into TVE[52:53]||[0:23]
> + * and end address bits 49:24 into TVE[54:55]||[24:47]
> + * and set TVE[51]
> + */
> + tve = (pci_start_addr << 16) & (0xffffffull << 40);
> + tve |= (pci_start_addr >> 38) & (3ull << 10);
> + tve |= (end >> 8) & (0xfffffful << 16);
> + tve |= (end >> 40) & (3ull << 8);
> + tve |= PPC_BIT(51) | IODA3_TVT_NON_TRANSLATE_50;
> + }
> } else {
> /* Disable */
> tve = 0;
> @@ -2135,6 +2146,7 @@ static int64_t phb4_creset(struct pci_slot *slot)
> case PHB4_SLOT_CRESET_REINIT:
> p->flags &= ~PHB4_AIB_FENCED;
> p->flags &= ~PHB4_CAPP_RECOVERY;
> + p->flags &= ~PHB4_P2P_SENDER;
> phb4_init_hw(p, false);
> pci_slot_set_state(slot, PHB4_SLOT_CRESET_FRESET);
> return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
> @@ -2507,6 +2519,32 @@ static int64_t phb4_get_diag_data(struct phb *phb,
> return OPAL_SUCCESS;
> }
>
> +static void enable_p2p_receiver(struct phb4 *p)
> +{
> + uint64_t val;
> +
> + xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE, &val);
> + val |= XPEC_NEST_STK_PBCQ_MODE_P2P;
> + xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE, val);
> +}
> +
> +static int64_t phb4_set_p2p(struct phb *phb, uint64_t mode)
> +{
> + struct phb4 *p = phb_to_phb4(phb);
> +
> + switch (mode) {
> + case OPAL_PCI_P2P_SENDER:
> + p->flags |= PHB4_P2P_SENDER;
> + break;
> + case OPAL_PCI_P2P_RECEIVER:
> + enable_p2p_receiver(p);
> + break;
> + default:
> + return OPAL_PARAMETER;
> + }
> + return OPAL_SUCCESS;
> +}
> +
> static const struct phb_ops phb4_ops = {
> .cfg_read8 = phb4_pcicfg_read8,
> .cfg_read16 = phb4_pcicfg_read16,
> @@ -2539,6 +2577,7 @@ static const struct phb_ops phb4_ops = {
> .get_diag_data = NULL,
> .get_diag_data2 = phb4_get_diag_data,
> .tce_kill = phb4_tce_kill,
> + .set_p2p = phb4_set_p2p,
> };
>
> static void phb4_init_ioda3(struct phb4 *p)
> diff --git a/include/opal-api.h b/include/opal-api.h
> index 80033c6f..8511d7de 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -204,7 +204,8 @@
> #define OPAL_NPU_INIT_CONTEXT 146
> #define OPAL_NPU_DESTROY_CONTEXT 147
> #define OPAL_NPU_MAP_LPAR 148
> -#define OPAL_LAST 148
> +#define OPAL_PCI_SET_P2P 149
> +#define OPAL_LAST 149
>
> /* Device tree flags */
>
> @@ -1215,6 +1216,12 @@ enum {
> XIVE_DUMP_EMU_STATE = 5,
> };
>
> +/* PCI p2p modes */
> +enum {
> + OPAL_PCI_P2P_SENDER = 0,
> + OPAL_PCI_P2P_RECEIVER = 1,
> +};
> +
> #endif /* __ASSEMBLY__ */
>
> #endif /* __OPAL_API_H */
> diff --git a/include/pci.h b/include/pci.h
> index dc418a90..98a7491d 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -319,6 +319,9 @@ struct phb_ops {
> uint64_t pe_number);
>
> int64_t (*set_capp_recovery)(struct phb *phb);
> +
> + /* PCI peer-to-peer setup */
> + int64_t (*set_p2p)(struct phb *phb, uint64_t mode);
> };
>
> enum phb_type {
> diff --git a/include/phb4-regs.h b/include/phb4-regs.h
> index 92bee88f..cdf1bb94 100644
> --- a/include/phb4-regs.h
> +++ b/include/phb4-regs.h
> @@ -321,6 +321,7 @@
> #define XPEC_NEST_STK_ERR_RPT1 0xb
> #define XPEC_NEST_STK_PBCQ_STAT 0xc
> #define XPEC_NEST_STK_PBCQ_MODE 0xd
> +#define XPEC_NEST_STK_PBCQ_MODE_P2P PPC_BIT(0)
> #define XPEC_NEST_STK_MMIO_BAR0 0xe
> #define XPEC_NEST_STK_MMIO_BAR0_MASK 0xf
> #define XPEC_NEST_STK_MMIO_BAR1 0x10
> diff --git a/include/phb4.h b/include/phb4.h
> index bed6eef0..fdfde267 100644
> --- a/include/phb4.h
> +++ b/include/phb4.h
> @@ -222,6 +222,7 @@ struct phb4_err {
> #define PHB4_CFG_USE_ASB 0x00000002
> #define PHB4_CFG_BLOCKED 0x00000004
> #define PHB4_CAPP_RECOVERY 0x00000008
> +#define PHB4_P2P_SENDER 0x00000010
>
> struct phb4 {
> unsigned int index; /* 0..5 index inside p9 */
>
More information about the Skiboot
mailing list