[Skiboot] [PATCH 06/16] [PATCH 06/16] opencapi5: create phb
Frederic Barrat
fbarrat at linux.ibm.com
Wed Sep 8 22:52:50 AEST 2021
On 20/08/2021 11:45, Christophe Lombard wrote:
> Implement the necessary operations for the OpenCAPI PHB type and
> inform the device-tree properties associated.
>
> The OpenCapi PCI config Addr/Data registers are reachable through
> the Generation-ID Registers MMIO BARS.
> The Config Address and Data registers are located at the following offsets
> from the AFU Config BAR plus 320 KB.
> • Config Address for Brick 0 – Offset 0
> • Config Data for Brick 0 – Offsets:
> ◦ 128 – 4-byte config register
>
> • Config Address for Brick 1 – Offset 256
> • Config Data for Brick 1 – Offsets:
> ◦ 384 – 4-byte config register
>
> Signed-off-by: Christophe Lombard <clombard at linux.vnet.ibm.com>
> ---
Looks ok to me.
Reviewed-by: Frederic Barrat <fbarrat at linux.ibm.com>
> core/pci-opal.c | 9 +-
> core/pci.c | 4 +-
> hw/pau.c | 234 ++++++++++++++++++++++++++++++++++++++++++++-
> include/pau-regs.h | 8 ++
> include/pau.h | 20 ++++
> include/pci.h | 1 +
> 6 files changed, 271 insertions(+), 5 deletions(-)
>
> diff --git a/core/pci-opal.c b/core/pci-opal.c
> index aa375c6a..acbcd2a5 100644
> --- a/core/pci-opal.c
> +++ b/core/pci-opal.c
> @@ -748,7 +748,8 @@ static void rescan_slot_devices(struct pci_slot *slot)
> * prepare_link_change() is called (if needed) by the state
> * machine during the slot reset or link polling
> */
> - if (phb->phb_type != phb_type_npu_v2_opencapi) {
> + if ((phb->phb_type != phb_type_npu_v2_opencapi) &&
> + (phb->phb_type != phb_type_pau_opencapi)) {
> pci_scan_bus(phb, pd->secondary_bus,
> pd->subordinate_bus, &pd->children, pd, true);
> pci_add_device_nodes(phb, &pd->children, pd->dn,
> @@ -766,7 +767,8 @@ static void remove_slot_devices(struct pci_slot *slot)
> struct phb *phb = slot->phb;
> struct pci_device *pd = slot->pd;
>
> - if (phb->phb_type != phb_type_npu_v2_opencapi)
> + if ((phb->phb_type != phb_type_npu_v2_opencapi) &&
> + (phb->phb_type != phb_type_pau_opencapi))
> pci_remove_bus(phb, &pd->children);
> else
> pci_remove_bus(phb, &phb->devices);
> @@ -817,7 +819,8 @@ static bool training_needed(struct pci_slot *slot)
> struct pci_device *pd = slot->pd;
>
> /* only for opencapi slots for now */
> - if (!pd && phb->phb_type == phb_type_npu_v2_opencapi)
> + if (!pd && ((phb->phb_type == phb_type_npu_v2_opencapi) ||
> + (phb->phb_type == phb_type_pau_opencapi)))
> return true;
> return false;
> }
> diff --git a/core/pci.c b/core/pci.c
> index e195ecbf..0a146c83 100644
> --- a/core/pci.c
> +++ b/core/pci.c
> @@ -1517,7 +1517,9 @@ static void __noinline pci_add_one_device_node(struct phb *phb,
> * device has a 4KB config space. It's got nothing to do with the
> * standard Type 0/1 config spaces defined by PCI.
> */
> - if (is_pcie || phb->phb_type == phb_type_npu_v2_opencapi) {
> + if (is_pcie ||
> + (phb->phb_type == phb_type_npu_v2_opencapi) ||
> + (phb->phb_type == phb_type_pau_opencapi)) {
> snprintf(compat, MAX_NAME, "pciex%x,%x",
> PCI_VENDOR_ID(pd->vdid), PCI_DEVICE_ID(pd->vdid));
> dt_add_property_cells(np, "ibm,pci-config-space-type", 1);
> diff --git a/hw/pau.c b/hw/pau.c
> index fb6a175e..5caafe6b 100644
> --- a/hw/pau.c
> +++ b/hw/pau.c
> @@ -3,12 +3,18 @@
> * Copyright 2020 IBM Corp.
> */
>
> +#include <interrupts.h>
> +#include <pci-slot.h>
> #include <phys-map.h>
> #include <pau.h>
> #include <pau-regs.h>
>
> +/* Number of PEs supported */
> +#define PAU_MAX_PE_NUM 16
> +#define PAU_RESERVED_PE_NUM 15
> +
> struct pau_dev *pau_next_dev(struct pau *pau, struct pau_dev *dev,
> - enum pau_dev_type type)
> + enum pau_dev_type type)
> {
> uint32_t i = 0;
>
> @@ -251,9 +257,235 @@ static void pau_opencapi_assign_bars(struct pau *pau)
> }
> }
>
> +static void pau_opencapi_create_phb_slot(struct pau_dev *dev)
> +{
> + struct pci_slot *slot;
> +
> + slot = pci_slot_alloc(&dev->phb, NULL);
> + if (!slot) {
> + /**
> + * @fwts-label OCAPICannotCreatePHBSlot
> + * @fwts-advice Firmware probably ran out of memory creating
> + * PAU slot. OpenCAPI functionality could be broken.
> + */
> + PAUDEVERR(dev, "Cannot create PHB slot\n");
> + }
> +}
> +
> +static int64_t pau_opencapi_pcicfg_check(struct pau_dev *dev,
> + uint32_t offset,
> + uint32_t size)
> +{
> + if (!dev || offset > 0xfff || (offset & (size - 1)))
> + return OPAL_PARAMETER;
> +
> + return OPAL_SUCCESS;
> +}
> +
> +static int64_t pau_opencapi_pcicfg_read(struct phb *phb, uint32_t bdfn,
> + uint32_t offset, uint32_t size,
> + void *data)
> +{
> + struct pau_dev *dev = pau_phb_to_opencapi_dev(phb);
> + uint64_t cfg_addr, genid_base;
> + int64_t rc;
> +
> + rc = pau_opencapi_pcicfg_check(dev, offset, size);
> + if (rc)
> + return rc;
> +
> + /* Config Address for Brick 0 – Offset 0
> + * Config Address for Brick 1 – Offset 256
> + */
> + genid_base = dev->genid_bar.cfg + (dev->index << 8);
> +
> + cfg_addr = PAU_CTL_MISC_CFG_ADDR_ENABLE;
> + cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_BUS_NBR |
> + PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR |
> + PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR,
> + cfg_addr, bdfn);
> + cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR,
> + cfg_addr, offset & ~3u);
> +
> + out_be64((uint64_t *)genid_base, cfg_addr);
> + sync();
> +
> + switch (size) {
> + case 1:
> + *((uint8_t *)data) =
> + in_8((uint8_t *)(genid_base + 128 + (offset & 3)));
> + break;
> + case 2:
> + *((uint16_t *)data) =
> + in_le16((uint16_t *)(genid_base + 128 + (offset & 2)));
> + break;
> + case 4:
> + *((uint32_t *)data) = in_le32((uint32_t *)(genid_base + 128));
> + break;
> + default:
> + return OPAL_PARAMETER;
> + }
> +
> + return OPAL_SUCCESS;
> +}
> +
> +#define PAU_OPENCAPI_PCI_CFG_READ(size, type) \
> +static int64_t pau_opencapi_pcicfg_read##size(struct phb *phb, uint32_t bdfn, \
> + uint32_t offset, type * data) \
> +{ \
> + /* Initialize data in case of error */ \
> + *data = (type)0xffffffff; \
> + return pau_opencapi_pcicfg_read(phb, bdfn, offset, sizeof(type), data); \
> +}
> +
> +static int64_t pau_opencapi_pcicfg_write(struct phb *phb, uint32_t bdfn,
> + uint32_t offset, uint32_t size,
> + uint32_t data)
> +{
> + struct pau_dev *dev = pau_phb_to_opencapi_dev(phb);
> + uint64_t genid_base, cfg_addr;
> + int64_t rc;
> +
> + rc = pau_opencapi_pcicfg_check(dev, offset, size);
> + if (rc)
> + return rc;
> +
> + /* Config Address for Brick 0 – Offset 0
> + * Config Address for Brick 1 – Offset 256
> + */
> + genid_base = dev->genid_bar.cfg + (dev->index << 8);
> +
> + cfg_addr = PAU_CTL_MISC_CFG_ADDR_ENABLE;
> + cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_BUS_NBR |
> + PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR |
> + PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR,
> + cfg_addr, bdfn);
> + cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR,
> + cfg_addr, offset & ~3u);
> +
> + out_be64((uint64_t *)genid_base, cfg_addr);
> + sync();
> +
> + switch (size) {
> + case 1:
> + out_8((uint8_t *)(genid_base + 128 + (offset & 3)), data);
> + break;
> + case 2:
> + out_le16((uint16_t *)(genid_base + 128 + (offset & 2)), data);
> + break;
> + case 4:
> + out_le32((uint32_t *)(genid_base + 128), data);
> + break;
> + default:
> + return OPAL_PARAMETER;
> + }
> +
> + return OPAL_SUCCESS;
> +}
> +
> +#define PAU_OPENCAPI_PCI_CFG_WRITE(size, type) \
> +static int64_t pau_opencapi_pcicfg_write##size(struct phb *phb, uint32_t bdfn, \
> + uint32_t offset, type data) \
> +{ \
> + return pau_opencapi_pcicfg_write(phb, bdfn, offset, sizeof(type), data);\
> +}
> +
> +PAU_OPENCAPI_PCI_CFG_READ(8, u8)
> +PAU_OPENCAPI_PCI_CFG_READ(16, u16)
> +PAU_OPENCAPI_PCI_CFG_READ(32, u32)
> +PAU_OPENCAPI_PCI_CFG_WRITE(8, u8)
> +PAU_OPENCAPI_PCI_CFG_WRITE(16, u16)
> +PAU_OPENCAPI_PCI_CFG_WRITE(32, u32)
> +
> +static const struct phb_ops pau_opencapi_ops = {
> + .cfg_read8 = pau_opencapi_pcicfg_read8,
> + .cfg_read16 = pau_opencapi_pcicfg_read16,
> + .cfg_read32 = pau_opencapi_pcicfg_read32,
> + .cfg_write8 = pau_opencapi_pcicfg_write8,
> + .cfg_write16 = pau_opencapi_pcicfg_write16,
> + .cfg_write32 = pau_opencapi_pcicfg_write32,
> +};
> +
> +static void pau_opencapi_create_phb(struct pau_dev *dev)
> +{
> + struct phb *phb = &dev->phb;
> + uint64_t mm_win[2];
> +
> + mm_win[0] = dev->ntl_bar.addr;
> + mm_win[1] = dev->ntl_bar.size;
> +
> + phb->phb_type = phb_type_pau_opencapi;
> + phb->scan_map = 0;
> +
> + phb->ops = &pau_opencapi_ops;
> + phb->dt_node = dt_new_addr(dt_root, "pciex", mm_win[0]);
> + assert(phb->dt_node);
> +
> + pci_register_phb(phb, pau_get_opal_id(dev->pau->chip_id,
> + pau_get_phb_index(dev->pau->index, dev->index)));
> + pau_opencapi_create_phb_slot(dev);
> +}
> +
> +static void pau_opencapi_dt_add_mmio_window(struct pau_dev *dev)
> +{
> + struct dt_node *dn = dev->phb.dt_node;
> + uint64_t mm_win[2];
> +
> + mm_win[0] = dev->ntl_bar.addr;
> + mm_win[1] = dev->ntl_bar.size;
> + PAUDEVDBG(dev, "Setting AFU MMIO window to %016llx %016llx\n",
> + mm_win[0], mm_win[1]);
> +
> + dt_add_property(dn, "reg", mm_win, sizeof(mm_win));
> + dt_add_property(dn, "ibm,mmio-window", mm_win, sizeof(mm_win));
> + dt_add_property_cells(dn, "ranges", 0x02000000,
> + hi32(mm_win[0]), lo32(mm_win[0]),
> + hi32(mm_win[0]), lo32(mm_win[0]),
> + hi32(mm_win[1]), lo32(mm_win[1]));
> +}
> +
> +static void pau_opencapi_dt_add_props(struct pau_dev *dev)
> +{
> + struct dt_node *dn = dev->phb.dt_node;
> + struct pau *pau = dev->pau;
> +
> + dt_add_property_strings(dn,
> + "compatible",
> + "ibm,power10-pau-opencapi-pciex",
> + "ibm,ioda3-pau-opencapi-phb",
> + "ibm,ioda2-npu2-opencapi-phb");
> +
> + dt_add_property_cells(dn, "#address-cells", 3);
> + dt_add_property_cells(dn, "#size-cells", 2);
> + dt_add_property_cells(dn, "#interrupt-cells", 1);
> + dt_add_property_cells(dn, "bus-range", 0, 0xff);
> + dt_add_property_cells(dn, "clock-frequency", 0x200, 0);
> + dt_add_property_cells(dn, "interrupt-parent", get_ics_phandle());
> +
> + dt_add_property_strings(dn, "device_type", "pciex");
> + dt_add_property_cells(dn, "ibm,pau-index", pau->index);
> + dt_add_property_cells(dn, "ibm,chip-id", pau->chip_id);
> + dt_add_property_cells(dn, "ibm,xscom-base", pau->xscom_base);
> + dt_add_property_cells(dn, "ibm,npcq", pau->dt_node->phandle);
> + dt_add_property_cells(dn, "ibm,links", 1);
> + dt_add_property_cells(dn, "ibm,phb-diag-data-size", 0);
> + dt_add_property_cells(dn, "ibm,opal-num-pes", PAU_MAX_PE_NUM);
> + dt_add_property_cells(dn, "ibm,opal-reserved-pe", PAU_RESERVED_PE_NUM);
> +
> + pau_opencapi_dt_add_mmio_window(dev);
> +}
> +
> static void pau_opencapi_init_hw(struct pau *pau)
> {
> + struct pau_dev *dev = NULL;
> +
> pau_opencapi_assign_bars(pau);
> +
> + /* Create phb */
> + pau_for_each_opencapi_dev(dev, pau) {
> + pau_opencapi_create_phb(dev);
> + pau_opencapi_dt_add_props(dev);
> + }
> }
>
> static void pau_opencapi_init(struct pau *pau)
> diff --git a/include/pau-regs.h b/include/pau-regs.h
> index afe6f958..57796920 100644
> --- a/include/pau-regs.h
> +++ b/include/pau-regs.h
> @@ -52,5 +52,13 @@
> #define PAU_CTL_MISC_MMIOPA_CONFIG(brk) (PAU_BLOCK_CQ_CTL + 0x098 + (brk) * 8)
> #define PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR PPC_BITMASK(1, 35)
> #define PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE PPC_BITMASK(39, 43)
> +#define PAU_CTL_MISC_CFG_ADDR(brk) (PAU_BLOCK_CQ_CTL + 0x250 + (brk) * 8)
> +#define PAU_CTL_MISC_CFG_ADDR_ENABLE PPC_BIT(0)
> +#define PAU_CTL_MISC_CFG_ADDR_STATUS PPC_BITMASK(1, 3)
> +#define PAU_CTL_MISC_CFG_ADDR_BUS_NBR PPC_BITMASK(4, 11)
> +#define PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR PPC_BITMASK(12, 16)
> +#define PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR PPC_BITMASK(17, 19)
> +#define PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR PPC_BITMASK(20, 31)
> +#define PAU_CTL_MISC_CFG_ADDR_TYPE PPC_BIT(32)
>
> #endif /* __PAU_REGS_H */
> diff --git a/include/pau.h b/include/pau.h
> index 05b27196..6f2eef6e 100644
> --- a/include/pau.h
> +++ b/include/pau.h
> @@ -8,6 +8,7 @@
> #include <io.h>
> #include <pci.h>
> #include <xscom.h>
> +#include <phb4.h>
> #include <pau-regs.h>
>
> #define PAU_NBR 6
> @@ -30,6 +31,7 @@ struct pau_dev {
> enum pau_dev_type type;
> uint32_t index;
> struct dt_node *dn;
> + struct phb phb;
>
> struct pau_bar ntl_bar;
> struct pau_bar genid_bar;
> @@ -86,6 +88,12 @@ static inline uint32_t pau_dev_index(struct pau_dev *dev, int links)
> return dev->pau->index * links + dev->index;
> }
>
> +static inline struct pau_dev *pau_phb_to_opencapi_dev(struct phb *phb)
> +{
> + assert(phb->phb_type == phb_type_pau_opencapi);
> + return container_of(phb, struct pau_dev, phb);
> +}
> +
> struct pau_dev *pau_next_dev(struct pau *pau, struct pau_dev *dev,
> enum pau_dev_type type);
>
> @@ -98,6 +106,18 @@ struct pau_dev *pau_next_dev(struct pau *pau, struct pau_dev *dev,
> #define pau_for_each_dev(dev, pau) \
> pau_for_each_dev_type(dev, pau, PAU_DEV_TYPE_ANY)
>
> +#define PAU_PHB_INDEX_BASE 6 /* immediately after real PHBs */
> +static inline int pau_get_phb_index(unsigned int pau_index,
> + unsigned int link_index)
> +{
> + return PAU_PHB_INDEX_BASE + pau_index * 2 + link_index;
> +}
> +
> +static inline int pau_get_opal_id(unsigned int chip_id, unsigned int index)
> +{
> + return phb4_get_opal_id(chip_id, index);
> +}
> +
> /*
> * We use the indirect method because it uses the same addresses as
> * the MMIO offsets (PAU RING)
> diff --git a/include/pci.h b/include/pci.h
> index eb23a6d9..cb8e7741 100644
> --- a/include/pci.h
> +++ b/include/pci.h
> @@ -353,6 +353,7 @@ enum phb_type {
> phb_type_npu_v2,
> phb_type_npu_v2_opencapi,
> phb_type_npu_v3,
> + phb_type_pau_opencapi,
> };
>
> /* Generic PCI NVRAM flags */
>
More information about the Skiboot
mailing list