[Skiboot] [PATCH 1/2] phb4: set PHB CMPM registers for tunneled operations
Benjamin Herrenschmidt
benh at au1.ibm.com
Thu Nov 2 15:03:00 AEDT 2017
On Thu, 2017-10-26 at 15:43 +0200, Philippe Bergheaud wrote:
> On 23/10/2017 15:08, Philippe Bergheaud wrote:
> > P9 supports PCI tunneled operations (atomics and as_notify) that require
> > setting the PHB ASN Compare/Mask register with a 16-bit indication.
> >
> > This register is currently initialized by enable_capi_mode(). As tunneled
> > operations may also operate in PCI mode, a new API is required to set the
> > PHB ASN register without switching to CAPI mode.
> >
> > This patch provides two new OPAL calls to get/set any of the three PHB
> > CMPM registers that obey the same logic (ASN, NBW, and CAPI).
> >
> > This new API aims at letting the operating system choose indications, that
> > may change in the future. Compatibility with older kernel versions is made
> > by enable_capi_mode().
> >
> > Signed-off-by: Philippe Bergheaud <felix at linux.vnet.ibm.com>
> > ---
>
> Linux patch:
> https://patchwork.ozlabs.org/patch/830231/
Any reason why the BAR value is under Linux control ? Can't we just
pick a range that's outside of the MMIO, DMA and MSI windows (some
top bits set) and be done with it ? (ie. just *tell* Linux where we put
it).
> > core/pci-opal.c | 36 ++++++
> > .../opal-pci-get-set-pbcq-tunnel-bar-160-161.rst | 76 +++++++++++++
> > doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst | 100 ++++++++++++++++
> > hw/phb4.c | 126 +++++++++++++++++++--
> > include/opal-api.h | 11 +-
> > include/pci.h | 4 +
> > include/phb4-regs.h | 8 +-
> > 7 files changed, 345 insertions(+), 16 deletions(-)
> > create mode 100644 doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
> > create mode 100644 doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
> >
> > diff --git a/core/pci-opal.c b/core/pci-opal.c
> > index b8aec941..bbbbcd5d 100644
> > --- a/core/pci-opal.c
> > +++ b/core/pci-opal.c
> > @@ -1016,3 +1016,39 @@ static int64_t opal_pci_set_p2p(uint64_t phbid_init, uint64_t phbid_target,
> > return OPAL_SUCCESS;
> > }
> > opal_call(OPAL_PCI_SET_P2P, opal_pci_set_p2p, 4);
> > +
> > +static int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> > + uint64_t *ind)
> > +{
> > + struct phb *phb = pci_get_phb(phb_id);
> > + int64_t rc;
> > +
> > + if (!phb)
> > + return OPAL_PARAMETER;
> > + if (!phb->ops->get_cmpm)
> > + return OPAL_UNSUPPORTED;
> > +
> > + phb_lock(phb);
> > + rc = phb->ops->get_cmpm(phb, phb_reg, ind);
> > + phb_unlock(phb);
> > + return rc;
> > +}
> > +opal_call(OPAL_PCI_GET_PHB_CMPM, opal_pci_get_phb_cmpm, 3);
> > +
> > +static int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> > + uint64_t ind)
> > +{
> > + struct phb *phb = pci_get_phb(phb_id);
> > + int64_t rc;
> > +
> > + if (!phb)
> > + return OPAL_PARAMETER;
> > + if (!phb->ops->set_cmpm)
> > + return OPAL_UNSUPPORTED;
> > +
> > + phb_lock(phb);
> > + rc = phb->ops->set_cmpm(phb, phb_reg, ind);
> > + phb_unlock(phb);
> > + return rc;
> > +}
> > +opal_call(OPAL_PCI_SET_PHB_CMPM, opal_pci_set_phb_cmpm, 3);
> > diff --git a/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
> > new file mode 100644
> > index 00000000..0af06774
> > --- /dev/null
> > +++ b/doc/opal-api/opal-pci-get-set-pbcq-tunnel-bar-160-161.rst
> > @@ -0,0 +1,76 @@
> > +OPAL_PCI_GET_PBCQ_TUNNEL_BAR
> > +============================
> > +::
> > +
> > + #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 160
> > +
> > + int64_t opal_pci_get_pbcq_tunnel_bar(uint64_t phb_id, uint64_t *val)
> > +
> > +The host calls this function to read the address out of the PBCQ Tunnel
> > +Bar register.
> > +
> > +Parameters
> > +----------
> > +::
> > +
> > + uint64_t phb_id
> > + uint64_t *val
> > +
> > +``phb_id``
> > + The value from the PHB node ibm,opal-phbid property for the device.
> > +
> > +``val``
> > + A pointer to the address where the address store in the PBCQ Tunnel Bar
> > + register will be copied.
> > +
> > +Return Values
> > +-------------
> > +
> > +``OPAL_SUCCESS``
> > + Operation was successful
> > +
> > +``OPAL_PARAMETER``
> > + Invalid PHB
> > +
> > +``OPAL_UNSUPPORTED``
> > + Not supported by hardware
> > +
> > +OPAL_PCI_SET_PBCQ_TUNNEL_BAR
> > +============================
> > +::
> > +
> > + #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 161
> > +
> > + int64_t opal_pci_set_pbcq_tunnel_bar(uint64_t phb_id, uint64_t addr)
> > +
> > +The host calls this function to set the PBCQ Tunnel Bar register.
> > +
> > +Parameters
> > +----------
> > +::
> > +
> > + uint64_t phb_id
> > + uint64_t addr
> > +
> > +``phb_id``
> > + The value from the PHB node ibm,opal-phbid property for the device.
> > +
> > +``addr``
> > + The value of the address chosen for the PBCQ Tunnel Bar register.
> > + If the address is 0, then the PBCQ Tunnel Bar register will be reset.
> > + It the address is non-zero, then the PBCQ Tunnel Bar register will be
> > + set with ::
> > +
> > + Bit[0:42] Bit[8:50] of the address
> > +
> > +Return Values
> > +-------------
> > +
> > +``OPAL_SUCCESS``
> > + Operation was successful
> > +
> > +``OPAL_PARAMETER``
> > + Invalid PHB or addr parameter
> > +
> > +``OPAL_UNSUPPORTED``
> > + Not supported by hardware
> > diff --git a/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
> > new file mode 100644
> > index 00000000..7eabdde3
> > --- /dev/null
> > +++ b/doc/opal-api/opal-pci-get-set-phb-cmpm-158-159.rst
> > @@ -0,0 +1,100 @@
> > +OPAL_PCI_GET_PHB_CMPM
> > +=====================
> > +::
> > +
> > + #define OPAL_PCI_GET_PHB_CMPM 158
> > +
> > + int64_t opal_pci_get_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> > + uint64_t *val)
> > +
> > +The host calls this function to read the indication out of a CMPM register
> > +of the PHB.
> > +
> > +Parameters
> > +----------
> > +::
> > +
> > + uint64_t phb_id
> > + uint64_t phb_reg
> > + uint64_t *val
> > +
> > +``phb_id``
> > + The value from the PHB node ibm,opal-phbid property for the device.
> > +
> > +``phb_reg``
> > + Which CMPM register should be accessed, i.e. one of ::
> > +
> > + /* PHB Compare/Mask registers */
> > + enum {
> > + OPAL_PHB_ASN_CMPM = 0,
> > + OPAL_PHB_CAPI_CMPM = 1,
> > + OPAL_PHB_PBL_NBW_CMPM = 2,
> > + };
> > +
> > +``val``
> > + A pointer to the address where the indication of the PHB CMPM register
> > + will be copied.
> > +
> > +Return Values
> > +-------------
> > +
> > +``OPAL_SUCCESS``
> > + Operation was successful
> > +
> > +``OPAL_PARAMETER``
> > + Invalid PHB or phb_reg parameter
> > +
> > +``OPAL_UNSUPPORTED``
> > + Not supported by hardware
> > +
> > +OPAL_PCI_SET_PHB_CMPM
> > +=====================
> > +::
> > +
> > + #define OPAL_PCI_SET_PHB_CMPM 159
> > +
> > + int64_t opal_pci_set_phb_cmpm(uint64_t phb_id, uint64_t phb_reg,
> > + uint64_t ind)
> > +
> > +The host calls this function to set a CMPM register of the PHB.
> > +
> > +Parameters
> > +----------
> > +::
> > +
> > + uint64_t phb_id
> > + uint64_t phb_reg
> > + uint64_t ind
> > +
> > +``phb_id``
> > + The value from the PHB node ibm,opal-phbid property for the device.
> > +
> > +``phb_reg``
> > + Which CMPM register should be accessed, i.e. one of ::
> > +
> > + /* PHB Compare/Mask registers */
> > + enum {
> > + OPAL_PHB_ASN_CMPM = 0,
> > + OPAL_PHB_CAPI_CMPM = 1,
> > + OPAL_PHB_PBL_NBW_CMPM = 2,
> > + };
> > +
> > +``ind``
> > + The value of the indication chosen for the PHB CMPM register.
> > + The PHB CMPM register will be set with ::
> > +
> > + Bit[0:15] the indication value
> > + Bit[16:31] the (hard-coded) mask value
> > + Bit[63] the enable bit
> > +
> > +Return Values
> > +-------------
> > +
> > +``OPAL_SUCCESS``
> > + Operation was successful
> > +
> > +``OPAL_PARAMETER``
> > + Invalid PHB, phb_reg, or ind parameter
> > +
> > +``OPAL_UNSUPPORTED``
> > + Not supported by hardware
> > diff --git a/hw/phb4.c b/hw/phb4.c
> > index c64ded92..1add8a75 100644
> > --- a/hw/phb4.c
> > +++ b/hw/phb4.c
> > @@ -3731,16 +3731,25 @@ static int64_t enable_capi_mode(struct phb4 *p, uint64_t pe_number,
> > */
> >
> > /*
> > - * Bit [0:7] XSL_DSNCTL[capiind]
> > - * Init_25 - CAPI Compare/Mask
> > + * Init_26 - CAPI Compare/Mask
> > + * If unset, then capiind=0x0200.
> > */
> > - out_be64(p->regs + PHB_CAPI_CMPM,
> > - 0x0200FE0000000000Ull | PHB_CAPI_CMPM_ENABLE);
> > + reg = in_be64(p->regs + PHB_CAPI_CMPM);
> > + if (!reg) {
> > + reg = 0x0200fe0000000000ull | PHB_CAPI_CMPM_ENABLE;
> > + out_be64(p->regs + PHB_CAPI_CMPM, reg);
> > + }
> >
> > if (!(p->rev == PHB4_REV_NIMBUS_DD10)) {
> > - /* Init_24 - ASN Compare/Mask */
> > - out_be64(p->regs + PHB_PBL_ASN_CMPM,
> > - 0x0400FF0000000000Ull | PHB_PBL_ASN_ENABLE);
> > + /*
> > + * Init_25 - ASN Compare/Mask
> > + * If unset, then asnind=0x0400.
> > + */
> > + reg = in_be64(p->regs + PHB_ASN_CMPM);
> > + if (!reg) {
> > + reg = 0x0400ff0000000000ull | PHB_ASN_CMPM_ENABLE;
> > + out_be64(p->regs + PHB_ASN_CMPM, reg);
> > + }
> >
> > /* PBCQ Tunnel Bar Register
> > * Write Tunnel register to match PSL TNR register
> > @@ -3990,6 +3999,99 @@ static int64_t phb4_set_capp_recovery(struct phb *phb)
> > return 0;
> > }
> >
> > +/*
> > + * Return the indication of a PHB CMPM register.
> > + */
> > +static int64_t phb4_get_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t *ind)
> > +{
> > + struct phb4 *p = phb_to_phb4(phb);
> > + uint64_t offset;
> > +
> > + switch (phb_reg) {
> > + case OPAL_PHB_ASN_CMPM:
> > + offset = PHB_ASN_CMPM;
> > + break;
> > + case OPAL_PHB_CAPI_CMPM:
> > + offset = PHB_CAPI_CMPM;
> > + break;
> > + case OPAL_PHB_PBL_NBW_CMPM:
> > + offset = PHB_PBL_NBW_CMPM;
> > + break;
> > + default:
> > + return OPAL_UNSUPPORTED;
> > + }
> > +
> > + *ind = in_be64(p->regs + offset) >> 48;
> > + return OPAL_SUCCESS;
> > +}
> > +
> > +/*
> > + * Set the 16-bit indication of a PHB CMPM register.
> > + * Set (hard-coded) mask value, and enable bit.
> > + *
> > + * This interface aims at letting linux choose the indications,
> > + * if they need to be changed in the future.
> > + *
> > + * Compatibility with older versions of linux, that do not set CMPM
> > + * indications with phb4_set_cmpm(), is ensured by enable_capi_mode(),
> > + * that will set the default hard-coded values that were assumed then.
> > + */
> > +static int64_t phb4_set_cmpm(struct phb *phb, uint64_t phb_reg, uint64_t ind)
> > +{
> > + struct phb4 *p = phb_to_phb4(phb);
> > + uint64_t mask, offset, enable;
> > +
> > + /* Indication is a 16-bit value */
> > + if (ind >> 16)
> > + return OPAL_PARAMETER;
> > + /*
> > + * The following bits of the PCI address are reserved:
> > + * Bit 59 indicates TVE#1
> > + * Bit 60 indicates MSI-X
> > + */
> > + if (ind & 0x1800ull)
> > + return OPAL_PARAMETER;
> > +
> > + switch (phb_reg) {
> > + case OPAL_PHB_ASN_CMPM:
> > + /*
> > + * Init_25 - ASN Compare/Mask
> > + * matches XSL_DSNCTL[ASB_Addr]
> > + */
> > + offset = PHB_ASN_CMPM;
> > + enable = PHB_ASN_CMPM_ENABLE;
> > + mask = 0xff00ull;
> > + break;
> > + case OPAL_PHB_CAPI_CMPM:
> > + /*
> > + * Init_26 - CAPI Compare/Mask
> > + * matches XSL_DSNCTL[capiind]
> > + */
> > + offset = PHB_CAPI_CMPM;
> > + enable = PHB_CAPI_CMPM_ENABLE;
> > + mask = 0xfe00ull;
> > + break;
> > + case OPAL_PHB_PBL_NBW_CMPM:
> > + /*
> > + * Init_124 - NBW Compare/Mask
> > + * matches XSL_DSNCTL[nbwind]
> > + */
> > + offset = PHB_PBL_NBW_CMPM;
> > + enable = PHB_PBL_NBW_CMPM_ENABLE;
> > + mask = 0xff00ull;
> > + break;
> > + default:
> > + return OPAL_UNSUPPORTED;
> > + }
> > +
> > + /* Masked indication should not be null */
> > + if (!(ind & mask))
> > + return OPAL_PARAMETER;
> > +
> > + out_be64(p->regs + offset, (ind << 48) | (mask << 32) | enable);
> > + return OPAL_SUCCESS;
> > +}
> > +
> > static const struct phb_ops phb4_ops = {
> > .cfg_read8 = phb4_pcicfg_read8,
> > .cfg_read16 = phb4_pcicfg_read16,
> > @@ -4025,6 +4127,8 @@ static const struct phb_ops phb4_ops = {
> > .set_capi_mode = phb4_set_capi_mode,
> > .set_p2p = phb4_set_p2p,
> > .set_capp_recovery = phb4_set_capp_recovery,
> > + .get_cmpm = phb4_get_cmpm,
> > + .set_cmpm = phb4_set_cmpm,
> > };
> >
> > static void phb4_init_ioda3(struct phb4 *p)
> > @@ -4059,10 +4163,10 @@ static void phb4_init_ioda3(struct phb4 *p)
> > /* See enable_capi_mode() */
> >
> > /* Init_25 - ASN Compare/Mask */
> > - /* See enable_capi_mode() */
> > + /* See phb4_set_cmpm() and enable_capi_mode() */
> >
> > /* Init_26 - CAPI Compare/Mask */
> > - /* See enable_capi_mode() */
> > + /* See phb4_set_cmpm() and enable_capi_mode() */
> >
> > /* Init_27 - PCIE Outbound upper address */
> > out_be64(p->regs + PHB_M64_UPPER_BITS, 0);
> > @@ -4412,8 +4516,8 @@ static void phb4_init_hw(struct phb4 *p, bool first_init)
> > * don't bother as we are doing a PERST soon.
> > */
> >
> > - /* Init_124 : NBW. XXX TODO */
> > - /* See enable_capi_mode() */
> > + /* Init_124 - NBW Compare/Mask */
> > + /* See phb4_set_cmpm() and enable_capi_mode() */
> >
> > /* Init_125 : Setup PCI command/status on root complex
> > * I don't know why the spec does this now and not earlier, so
> > diff --git a/include/opal-api.h b/include/opal-api.h
> > index 0bc036ed..ce948136 100644
> > --- a/include/opal-api.h
> > +++ b/include/opal-api.h
> > @@ -214,7 +214,9 @@
> > #define OPAL_SET_POWER_SHIFT_RATIO 155
> > #define OPAL_SENSOR_GROUP_CLEAR 156
> > #define OPAL_PCI_SET_P2P 157
> > -#define OPAL_LAST 157
> > +#define OPAL_PCI_GET_PHB_CMPM 158
> > +#define OPAL_PCI_SET_PHB_CMPM 159
> > +#define OPAL_LAST 159
> >
> > /* Device tree flags */
> >
> > @@ -1274,6 +1276,13 @@ enum {
> > OPAL_PCI_P2P_TARGET = 1,
> > };
> >
> > +/* PHB Compare/Mask registers */
> > +enum {
> > + OPAL_PHB_ASN_CMPM = 0,
> > + OPAL_PHB_CAPI_CMPM = 1,
> > + OPAL_PHB_PBL_NBW_CMPM = 2,
> > +};
> > +
> > #endif /* __ASSEMBLY__ */
> >
> > #endif /* __OPAL_API_H */
> > diff --git a/include/pci.h b/include/pci.h
> > index c085b6b8..cdf82ee8 100644
> > --- a/include/pci.h
> > +++ b/include/pci.h
> > @@ -333,6 +333,10 @@ struct phb_ops {
> > /* PCI peer-to-peer setup */
> > void (*set_p2p)(struct phb *phb, uint64_t mode, uint64_t flags,
> > uint16_t pe_number);
> > +
> > + /* Get/set PHB Compare/Mask registers */
> > + int64_t (*get_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t *ind);
> > + int64_t (*set_cmpm)(struct phb *phb, uint64_t phb_reg, uint64_t ind);
> > };
> >
> > enum phb_type {
> > diff --git a/include/phb4-regs.h b/include/phb4-regs.h
> > index e83c8c39..b33185ed 100644
> > --- a/include/phb4-regs.h
> > +++ b/include/phb4-regs.h
> > @@ -71,8 +71,8 @@
> > #define PHB_PEST_BAR 0x1a8
> > #define PHB_PEST_BAR_ENABLE PPC_BIT(0)
> > #define PHB_PEST_BASE_ADDRESS PPC_BITMASK(8,51)
> > -#define PHB_PBL_ASN_CMPM 0x1C0
> > -#define PHB_PBL_ASN_ENABLE PPC_BIT(63)
> > +#define PHB_ASN_CMPM 0x1C0
> > +#define PHB_ASN_CMPM_ENABLE PPC_BIT(63)
> > #define PHB_CAPI_CMPM 0x1C8
> > #define PHB_CAPI_CMPM_ENABLE PPC_BIT(63)
> > #define PHB_M64_UPPER_BITS 0x1f0
> > @@ -250,8 +250,8 @@
> > #define PHB_PBL_CONTROL 0x1800
> > #define PHB_PBL_TIMEOUT_CTRL 0x1810
> > #define PHB_PBL_NPTAG_ENABLE 0x1820
> > -#define PHB_PBL_NBW_CMP_MASK 0x1830
> > -#define PHB_PBL_NBW_MASK_ENABLE PPC_BIT(63)
> > +#define PHB_PBL_NBW_CMPM 0x1830
> > +#define PHB_PBL_NBW_CMPM_ENABLE PPC_BIT(63)
> > #define PHB_PBL_SYS_LINK_INIT 0x1838
> > #define PHB_PBL_BUF_STATUS 0x1840
> > #define PHB_PBL_ERR_STATUS 0x1900
>
> _______________________________________________
> Skiboot mailing list
> Skiboot at lists.ozlabs.org
> https://lists.ozlabs.org/listinfo/skiboot
More information about the Skiboot
mailing list