[Skiboot] [PATCH] phb4: Enable PCI peer-to-peer

Frederic Barrat fbarrat at linux.vnet.ibm.com
Wed May 31 00:00:39 AEST 2017



Le 30/05/2017 à 15:54, Frederic Barrat a écrit :
> P9 supports PCI peer-to-peer: a PCI device can write directly to the
> mmio space of another PCI device. It completely by-passes the
> CPU. Reads are not supported for the time being.
> 
> It requires some configuration on the PHBs involved:
> 1. on the sending side, the address used is in the mmio space of the
> receiver, i.e. well outside the range normally allowed. So when
> setting the TVT entry in bypass mode, we disable range-checking.
> 2. on the receiving side, we need to explicitly enable p2p by setting a
> bit in a configuration register. It has the side-effect of reserving
> an outbound (as seen from the CPU) store queue for p2p. Therefore we only
> enable p2p on the PHBs using it, as we don't want to waste the
> resource if we don't have to.
> 
> The patch adds a new OPAL call to allow the OS to configure the PHBs.
> 
> Signed-off-by: Frederic Barrat <fbarrat at linux.vnet.ibm.com>
> ---
> 
> Stewart: this patch goes hand-in-hand with a linux patch to be posted
> shortly (will add reference to it). It shouldn't be merged until the
> linux patch is agreed upon, most notably around the APIs.

Linux patch:
http://patchwork.ozlabs.org/patch/768617/


> 
>   core/pci-opal.c                       | 17 ++++++++++
>   doc/opal-api/opal-pci-set-p2p-148.rst | 39 +++++++++++++++++++++++
>   hw/phb4.c                             | 59 +++++++++++++++++++++++++++++------
>   include/opal-api.h                    |  9 +++++-
>   include/pci.h                         |  3 ++
>   include/phb4-regs.h                   |  1 +
>   include/phb4.h                        |  1 +
>   7 files changed, 118 insertions(+), 11 deletions(-)
>   create mode 100644 doc/opal-api/opal-pci-set-p2p-148.rst
> 
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index 27872aa2..0389e4d9 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -977,3 +977,20 @@ static int64_t opal_pci_set_phb_capi_mode(uint64_t phb_id, uint64_t mode, uint64
>   	return rc;
>   }
>   opal_call(OPAL_PCI_SET_PHB_CAPI_MODE, opal_pci_set_phb_capi_mode, 3);
> +
> +static int64_t opal_pci_set_p2p(uint64_t phb_id, uint64_t mode)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +	int64_t rc;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->set_p2p)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	rc = phb->ops->set_p2p(phb, mode);
> +	phb_unlock(phb);
> +	return rc;
> +}
> +opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 2);
> diff --git a/doc/opal-api/opal-pci-set-p2p-148.rst b/doc/opal-api/opal-pci-set-p2p-148.rst
> new file mode 100644
> index 00000000..8c613810
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-set-p2p-148.rst
> @@ -0,0 +1,39 @@
> +OPAL_PCI_SET_P2P
> +================
> +::
> +
> +   #define OPAL_PCI_SET_P2P			149
> +
> +   int64_t opal_pci_set_p2p(uint64_t phb_id, uint64_t mode)
> +
> +   enum {
> +	OPAL_PCI_P2P_SENDER	= 0,
> +	OPAL_PCI_P2P_RECEIVER	= 1
> +   };
> +
> +The host calls this function to enable PCI peer-to-peer on the PHBs.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t mode
> +
> +``phb_id``
> +  is the value from the PHB node ibm,opal-phbid property.
> +
> +``mode``
> +  tells whether the PHB should be configured as a receiver (OPAL_PCI_P2P_RECEIVER) or sender (OPAL_PCI_P2P_SENDER)
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Configuration was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or mode parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> diff --git a/hw/phb4.c b/hw/phb4.c
> index 318d934d..6894a981 100644
> --- a/hw/phb4.c
> +++ b/hw/phb4.c
> @@ -1343,16 +1343,27 @@ static int64_t phb4_map_pe_dma_window_real(struct phb *phb,
>   		if (end > 0x0003ffffffffffffull)
>   			return OPAL_PARAMETER;
>   
> -		/*
> -		 * Put start address bits 49:24 into TVE[52:53]||[0:23]
> -		 * and end address bits 49:24 into TVE[54:55]||[24:47]
> -		 * and set TVE[51]
> -		 */
> -		tve  = (pci_start_addr << 16) & (0xffffffull << 40);
> -		tve |= (pci_start_addr >> 38) & (3ull << 10);
> -		tve |= (end >>  8) & (0xfffffful << 16);
> -		tve |= (end >> 40) & (3ull << 8);
> -		tve |= PPC_BIT(51) | IODA3_TVT_NON_TRANSLATE_50;
> +		if (p->flags & PHB4_P2P_SENDER) {
> +			/*
> +			 * for PCI peer-to-peer, we need to let go any valid
> +			 * memory address + any MMIO address of the receiver.
> +			 * That's pretty much everything.
> +			 * => 56 bit addr, no translate, no range checking
> +			 */
> +			tve = PPC_BIT(51);
> +			PHBDBG(p, "p2p is enabled, overriding TVE setting\n");
> +		} else {
> +			/*
> +			 * Put start address bits 49:24 into TVE[52:53]||[0:23]
> +			 * and end address bits 49:24 into TVE[54:55]||[24:47]
> +			 * and set TVE[51]
> +			 */
> +			tve  = (pci_start_addr << 16) & (0xffffffull << 40);
> +			tve |= (pci_start_addr >> 38) & (3ull << 10);
> +			tve |= (end >>  8) & (0xfffffful << 16);
> +			tve |= (end >> 40) & (3ull << 8);
> +			tve |= PPC_BIT(51) | IODA3_TVT_NON_TRANSLATE_50;
> +		}
>   	} else {
>   		/* Disable */
>   		tve = 0;
> @@ -2135,6 +2146,7 @@ static int64_t phb4_creset(struct pci_slot *slot)
>   	case PHB4_SLOT_CRESET_REINIT:
>   		p->flags &= ~PHB4_AIB_FENCED;
>   		p->flags &= ~PHB4_CAPP_RECOVERY;
> +		p->flags &= ~PHB4_P2P_SENDER;
>   		phb4_init_hw(p, false);
>   		pci_slot_set_state(slot, PHB4_SLOT_CRESET_FRESET);
>   		return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
> @@ -2507,6 +2519,32 @@ static int64_t phb4_get_diag_data(struct phb *phb,
>   	return OPAL_SUCCESS;
>   }
>   
> +static void enable_p2p_receiver(struct phb4 *p)
> +{
> +	uint64_t val;
> +
> +	xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE, &val);
> +	val |= XPEC_NEST_STK_PBCQ_MODE_P2P;
> +	xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_PBCQ_MODE, val);
> +}
> +
> +static int64_t phb4_set_p2p(struct phb *phb, uint64_t mode)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +
> +	switch (mode) {
> +	case OPAL_PCI_P2P_SENDER:
> +		p->flags |= PHB4_P2P_SENDER;
> +		break;
> +	case OPAL_PCI_P2P_RECEIVER:
> +		enable_p2p_receiver(p);
> +		break;
> +	default:
> +		return OPAL_PARAMETER;
> +	}
> +	return OPAL_SUCCESS;
> +}
> +
>   static const struct phb_ops phb4_ops = {
>   	.cfg_read8		= phb4_pcicfg_read8,
>   	.cfg_read16		= phb4_pcicfg_read16,
> @@ -2539,6 +2577,7 @@ static const struct phb_ops phb4_ops = {
>   	.get_diag_data		= NULL,
>   	.get_diag_data2		= phb4_get_diag_data,
>   	.tce_kill		= phb4_tce_kill,
> +	.set_p2p		= phb4_set_p2p,
>   };
>   
>   static void phb4_init_ioda3(struct phb4 *p)
> diff --git a/include/opal-api.h b/include/opal-api.h
> index 80033c6f..8511d7de 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -204,7 +204,8 @@
>   #define OPAL_NPU_INIT_CONTEXT			146
>   #define OPAL_NPU_DESTROY_CONTEXT		147
>   #define OPAL_NPU_MAP_LPAR			148
> -#define OPAL_LAST				148
> +#define OPAL_PCI_SET_P2P			149
> +#define OPAL_LAST				149
>   
>   /* Device tree flags */
>   
> @@ -1215,6 +1216,12 @@ enum {
>   	XIVE_DUMP_EMU_STATE	= 5,
>   };
>   
> +/* PCI p2p modes */
> +enum {
> +	OPAL_PCI_P2P_SENDER	= 0,
> +	OPAL_PCI_P2P_RECEIVER	= 1,
> +};
> +
>   #endif /* __ASSEMBLY__ */
>   
>   #endif /* __OPAL_API_H */
> diff --git a/include/pci.h b/include/pci.h
> index dc418a90..98a7491d 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -319,6 +319,9 @@ struct phb_ops {
>   				 uint64_t pe_number);
>   
>   	int64_t (*set_capp_recovery)(struct phb *phb);
> +
> +	/* PCI peer-to-peer setup */
> +	int64_t (*set_p2p)(struct phb *phb, uint64_t mode);
>   };
>   
>   enum phb_type {
> diff --git a/include/phb4-regs.h b/include/phb4-regs.h
> index 92bee88f..cdf1bb94 100644
> --- a/include/phb4-regs.h
> +++ b/include/phb4-regs.h
> @@ -321,6 +321,7 @@
>   #define XPEC_NEST_STK_ERR_RPT1			0xb
>   #define XPEC_NEST_STK_PBCQ_STAT			0xc
>   #define XPEC_NEST_STK_PBCQ_MODE			0xd
> +#define   XPEC_NEST_STK_PBCQ_MODE_P2P		PPC_BIT(0)
>   #define XPEC_NEST_STK_MMIO_BAR0			0xe
>   #define XPEC_NEST_STK_MMIO_BAR0_MASK		0xf
>   #define XPEC_NEST_STK_MMIO_BAR1			0x10
> diff --git a/include/phb4.h b/include/phb4.h
> index bed6eef0..fdfde267 100644
> --- a/include/phb4.h
> +++ b/include/phb4.h
> @@ -222,6 +222,7 @@ struct phb4_err {
>   #define PHB4_CFG_USE_ASB	0x00000002
>   #define PHB4_CFG_BLOCKED	0x00000004
>   #define PHB4_CAPP_RECOVERY	0x00000008
> +#define PHB4_P2P_SENDER		0x00000010
>   
>   struct phb4 {
>   	unsigned int		index;	    /* 0..5 index inside p9 */
> 



More information about the Skiboot mailing list