[Skiboot] [PATCH v7 2/2] phb4: set PBCQ Tunnel BAR for tunneled operations

Frederic Barrat fbarrat at linux.vnet.ibm.com
Tue Feb 20 01:15:30 AEDT 2018



Le 16/02/2018 à 14:13, Philippe Bergheaud a écrit :
> P9 supports PCI tunneled operations (atomics and as_notify) that are
> initiated by devices.
> 
> A subset of the tunneled operations require a response, that must be
> sent back from the host to the device. For example, an atomic compare
> and swap will return the compare status, as swap will only performed
> in case of success.  Similarly, as_notify reports if the target thread
> has been woken up or not, because the operation may fail.
> 
> To enable tunneled operations, a device driver must tell the host where
> it expects tunneled operation responses, by setting the PBCQ Tunnel BAR
> Response register with a specific value within the range of its BARs.
> 
> This register is currently initialized by enable_capi_mode(). But, as
> tunneled operations may also operate in PCI mode, a new API is required
> to set the PBCQ Tunnel BAR Response register, without switching to CAPI
> mode.
> 
> This patch provides two new OPAL calls to get/set the PBCQ Tunnel
> BAR Response register.
> 
> Note: as there is only one PBCQ Tunnel BAR register, shared between
> all the devices connected to the same PHB, only one of these devices
> will be able to use tunneled operations, at any time.
> 
> Signed-off-by: Philippe Bergheaud <felix at linux.vnet.ibm.com>
> ---

Reviewed-by: Frederic Barrat <fbarrat at linux.vnet.ibm.com>


> Changelog:
> 
> v2: Adjust new OPAL call numbers.
> 
> v3: Rebase on skiboot 5.9.6.
>      Document new OPAL calls in doc/opal-api.
> 
> v4: opal_pci_get_pbcq_tunnel_bar():
>        - check that addr parameter is valid
>        - update doc/opal-api
>      enable_capi_mode():
>        - explain why we do not touch an existing tunnel bar value
>        - reuse psl tnr addr value verbatim, and shift it by 8
> 
> v5: No change.
> 
> v6: Rebased on top of Andrew Donnellan's patch
>        https://patchwork.ozlabs.org/patch/870252/
>      that adds three opal calls, and will probably be accepted
>      before this one.
> 
> v7: No change.
> ---
>   core/pci-opal.c                                    | 36 ++++++++++
>   .../opal-pci-get-set-pbcq-tunnel-bar-162-163.rst   | 76 ++++++++++++++++++++++
>   hw/phb4.c                                          | 71 ++++++++++++++++++--
>   include/opal-api.h                                 |  7 +-
>   include/pci.h                                      |  4 ++
>   5 files changed, 188 insertions(+), 6 deletions(-)
>   create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-162-163.rst
> 
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index b8aec941..77e965cc 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -1016,3 +1016,39 @@ static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
>   	return OPAL_SUCCESS;
>   }
>   opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
> +
> +static int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +
> +	if (!opal_addr_valid(addr))
> +		return OPAL_PARAMETER;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->get_tunnel_bar)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	phb->ops->get_tunnel_bar(phb, addr);
> +	phb_unlock(phb);
> +	return OPAL_SUCCESS;
> +}
> +opal_call(OPAL_PCI_GET_PBCQ_TUNNEL_BAR, opal_pci_get_pbcq_tunnel_bar, 2);
> +
> +static int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +	int64_t rc;
> +
> +	if (!phb)
> +		return OPAL_PARAMETER;
> +	if (!phb->ops->set_tunnel_bar)
> +		return OPAL_UNSUPPORTED;
> +
> +	phb_lock(phb);
> +	rc = phb->ops->set_tunnel_bar(phb, addr);
> +	phb_unlock(phb);
> +	return rc;
> +}
> +opal_call(OPAL_PCI_SET_PBCQ_TUNNEL_BAR, opal_pci_set_pbcq_tunnel_bar, 2);
> diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-162-163.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-162-163.rst
> new file mode 100644
> index 00000000..bbe2264a
> --- /dev/null
> +++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-162-163.rst
> @@ -0,0 +1,76 @@
> +OPAL_PCI_GET_PBCQ_TUNNEL_BAR
> +============================
> +::
> +
> +   #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 162
> +
> +   int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *addr)
> +
> +The host calls this function to read the address out of the PBCQ Tunnel
> +Bar register.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t *addr
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``addr``
> +  A pointer to where the address stored in the PBCQ Tunnel Bar register
> +  will be copied.
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or addr parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> +
> +OPAL_PCI_SET_PBCQ_TUNNEL_BAR
> +============================
> +::
> +
> +   #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 163
> +
> +   int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> +
> +The host calls this function to set the PBCQ Tunnel Bar register.
> +
> +Parameters
> +----------
> +::
> +
> +   uint64_t phb_id
> +   uint64_t addr
> +
> +``phb_id``
> +  The value from the PHB node ibm,opal-phbid property for the device.
> +
> +``addr``
> +  The value of the address chosen for the PBCQ Tunnel Bar register.
> +  If the address is 0, then the PBCQ Tunnel Bar register will be reset.
> +  It the address is non-zero, then the PBCQ Tunnel Bar register will be
> +  set with ::
> +
> +   Bit[0:42]      Bit[8:50] of the address
> +
> +Return Values
> +-------------
> +
> +``OPAL_SUCCESS``
> +  Operation was successful
> +
> +``OPAL_PARAMETER``
> +  Invalid PHB or addr parameter
> +
> +``OPAL_UNSUPPORTED``
> +  Not supported by hardware
> diff --git a/hw/phb4.c b/hw/phb4.c
> index 56716995..de713125 100644
> --- a/hw/phb4.c
> +++ b/hw/phb4.c
> @@ -3873,12 +3873,24 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
>   		 ((u64)CAPIMASK << 32) | PHB_CAPI_CMPM_ENABLE);
> 
>   	if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
> -		/* PBCQ Tunnel Bar Register
> -		 * Write Tunnel register to match PSL TNR register
> +		/*
> +		 * PBCQ Tunnel Bar Register
> +		 *
> +		 * If set, for example by a driver that may already have
> +		 * tweaked the tunnel bar, then we do not touch it when
> +		 * entering capi mode. It's up to the driver to handle it.
> +		 *
> +		 * If unset, then we use the PSL_TNR_ADDR[TNR_Addr] reset
> +		 * value. For fpga/cxl, this code will define the tunnel bar.
>   		 */
> -		xscom_write(p->chip_id,
> -			    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> -			    0x020000E000000000);
> +		xscom_read(p->chip_id,
> +			   p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, &reg);
> +		if (!reg) {
> +			reg = 0x00020000E0000000ull << 8;
> +			xscom_write(p->chip_id,
> +				    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +				    reg);
> +		}
> 
>   		/* PB AIB Hardware Control Register
>   		 * Wait 32 PCI clocks for a credit to become available
> @@ -4125,6 +4137,53 @@ static int64_t phb4_set_capp_recovery(struct phb *phb)
>   	return 0;
>   }
> 
> +/*
> + * Return the address out of a PBCQ Tunnel Bar register.
> + */
> +static void phb4_get_tunnel_bar(struct phb *phb, uint64_t *addr)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +	uint64_t val;
> +
> +	xscom_read(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +		   &val);
> +	*addr = val >> 8;
> +}
> +
> +/*
> + * Set PBCQ Tunnel Bar register.
> + * Store addr bits [8:50] in PBCQ Tunnel Bar register bits [0:42].
> + * Note that addr bits [8:50] must also match PSL_TNR_ADDR[8:50].
> + * Reset register if val == 0.
> + *
> + * This interface is required to let device drivers set the Tunnel Bar
> + * value of their choice.
> + *
> + * Compatibility with older versions of linux, that do not set the
> + * Tunnel Bar with phb4_set_tunnel_bar(), is ensured by enable_capi_mode(),
> + * that will set the default value that used to be assumed.
> + */
> +static int64_t phb4_set_tunnel_bar(struct phb *phb, uint64_t addr)
> +{
> +	struct phb4 *p = phb_to_phb4(phb);
> +	uint64_t mask = 0x00FFFFFFFFFFE000ULL;
> +
> +	if (!addr) {
> +		/* Reset register */
> +		xscom_write(p->chip_id,
> +			    p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR, addr);
> +		return OPAL_SUCCESS;
> +	}
> +	if ((addr & ~mask))
> +		return OPAL_PARAMETER;
> +	if (!(addr & mask))
> +		return OPAL_PARAMETER;
> +
> +	xscom_write(p->chip_id, p->pe_stk_xscom + XPEC_NEST_STK_TUNNEL_BAR,
> +		    (addr & mask) << 8);
> +	return OPAL_SUCCESS;
> +}
> +
>   static const struct phb_ops phb4_ops = {
>   	.cfg_read8		= phb4_pcicfg_read8,
>   	.cfg_read16		= phb4_pcicfg_read16,
> @@ -4160,6 +4219,8 @@ static const struct phb_ops phb4_ops = {
>   	.set_capi_mode		= phb4_set_capi_mode,
>   	.set_p2p		= phb4_set_p2p,
>   	.set_capp_recovery	= phb4_set_capp_recovery,
> +	.get_tunnel_bar         = phb4_get_tunnel_bar,
> +	.set_tunnel_bar         = phb4_set_tunnel_bar,
>   };
> 
>   static void phb4_init_ioda3(struct phb4 *p)
> diff --git a/include/opal-api.h b/include/opal-api.h
> index 0bd5f179..ec85ea1d 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -215,7 +215,12 @@
>   #define OPAL_SENSOR_GROUP_CLEAR			156
>   #define OPAL_PCI_SET_P2P			157
>   #define OPAL_QUIESCE				158
> -#define OPAL_LAST				158
> +#define OPAL_NPU_SPA_SETUP			159
> +#define OPAL_NPU_SPA_CLEAR_CACHE		160
> +#define OPAL_NPU_TL_SET				161
> +#define OPAL_PCI_GET_PBCQ_TUNNEL_BAR		162
> +#define OPAL_PCI_SET_PBCQ_TUNNEL_BAR		163
> +#define OPAL_LAST				163
> 
>   #define QUIESCE_HOLD			1 /* Spin all calls at entry */
>   #define QUIESCE_REJECT			2 /* Fail all calls with OPAL_BUSY */
> diff --git a/include/pci.h b/include/pci.h
> index 508ebf45..cc890118 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -333,6 +333,10 @@ struct phb_ops {
>   	/* PCI peer-to-peer setup */
>   	void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
>   			uint16_t pe_number);
> +
> +	/* Get/set PBCQ Tunnel BAR register */
> +	void (*get_tunnel_bar)(struct phb *phb, uint64_t *addr);
> +	int64_t (*set_tunnel_bar)(struct phb *phb, uint64_t addr);
>   };
> 
>   enum phb_type {
> 



More information about the Skiboot mailing list