[Skiboot] [RFC PATCH 2/3] hw/phb4: Restore bus numbers with a state machine
Jordan Niethe
jniethe5 at gmail.com
Tue Oct 22 14:32:01 AEDT 2019
After performing a fundamental reset bus numbers need to be restored. This
is done by pci_restore_slot_bus_configs() which is called after the
phb's link is active. Restoring the bus numbers can require waiting for
certain delays, which are currently performed using time_wait_ms(). This
leads to excessive times spent waiting in skiboot during the
OPAL_PCI_RESET opal call.
If a device is a behind a switch, it needs to wait for the switch's
downstream port link to become active before the buses can be restored.
Additionally all devices potentially need to wait for CRS.
After waiting, bridges need to have their bus settings restored.
To stop blocking in skiboot, we rearrange this process into a state
machine like the ones used to poll_link() and creset(). The state
machine is entered where pci_restore_slot_bus_configs() was called.
Signed-off-by: Jordan Niethe <jniethe5 at gmail.com>
---
core/pci-slot.c | 3 ++
core/pci.c | 7 +++
hw/phb4.c | 129 +++++++++++++++++++++++++++++++++++++++++++--
include/pci-slot.h | 2 +
include/pci.h | 1 +
include/phb4.h | 9 ++++
6 files changed, 148 insertions(+), 3 deletions(-)
diff --git a/core/pci-slot.c b/core/pci-slot.c
index ffb54cbb7a49..4321f30f98aa 100644
--- a/core/pci-slot.c
+++ b/core/pci-slot.c
@@ -79,6 +79,9 @@ static int64_t pci_slot_run_sm(struct pci_slot *slot)
slot->delay_tgt_tb = 0;
switch (slot->state & PCI_SLOT_STATE_MASK) {
+ case PCI_SLOT_STATE_BUSES:
+ ret = slot->ops.restore_buses(slot);
+ break;
case PCI_SLOT_STATE_LINK:
ret = slot->ops.poll_link(slot);
break;
diff --git a/core/pci.c b/core/pci.c
index 547c8e32272b..732ba68d5d23 100644
--- a/core/pci.c
+++ b/core/pci.c
@@ -1981,6 +1981,13 @@ void pci_restore_slot_bus_configs(struct pci_slot *slot)
slot->phb->ops->device_init, NULL);
}
+void pci_init_all_devices(struct pci_slot *slot)
+{
+ if (slot->phb->ops->device_init)
+ pci_walk_dev(slot->phb, slot->pd,
+ slot->phb->ops->device_init, NULL);
+}
+
struct pci_cfg_reg_filter *pci_find_cfg_reg_filter(struct pci_device *pd,
uint32_t start, uint32_t len)
{
diff --git a/hw/phb4.c b/hw/phb4.c
index 3c71427aef2f..487965c15036 100644
--- a/hw/phb4.c
+++ b/hw/phb4.c
@@ -2760,6 +2760,129 @@ static int64_t phb4_slot_sm_run_completed(struct pci_slot *slot, uint64_t err)
return OPAL_SUCCESS;
}
+static int64_t phb4_restore_buses(struct pci_slot *slot)
+{
+ struct phb4 *p = phb_to_phb4(slot->phb);
+ struct pci_device *pd, *parent;
+ uint32_t link_cap = 0;
+ uint16_t link_sts = 0;
+ int32_t ecap = 0;
+ struct phb *phb;
+ uint32_t vdid;
+ int64_t rc;
+
+ phb = slot->phb;
+ pd = slot->phb->current_pd;
+ parent = pd ? pd->parent : NULL;
+
+ switch (slot->state) {
+ case PHB4_SLOT_NORMAL:
+ case PHB4_SLOT_BUSES_START:
+ PHBDBG(p, "BUSES: Starts\n");
+ pci_device_iter_reset(slot->phb, NULL);
+ slot->phb->current_pd = pci_device_iter_next(slot->phb, NULL);
+ pci_slot_set_state(slot, PHB4_SLOT_BUSES_PD_START);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(1));
+ case PHB4_SLOT_BUSES_PD_START:
+ if (slot->phb->current_pd == NULL) {
+ /* No more devices left */
+ PHBDBG(p, "BUSES: All PCI Devices seen\n");
+ pci_slot_set_state(slot, PHB4_SLOT_NORMAL);
+ pci_init_all_devices(slot);
+ return OPAL_SUCCESS;
+ }
+ PCIDBG(phb, pd->bdfn, "BUSES: PCI Device starts\n");
+
+ /* Need to check if the device is behind a switch */
+ if (!pd->is_vf && !(pd->bdfn & 7) && pd->parent != NULL &&
+ pd->parent->dev_type == PCIE_TYPE_SWITCH_DNPORT) {
+ PCIDBG(phb, pd->bdfn, "BUSES: Behind a switch\n");
+ slot->retries = PHB4_BUS_SWITCH_LINK_RETRIES;
+ pci_slot_set_state(slot,
+ PHB4_SLOT_BUSES_PD_WAIT_SWITCH_LINK);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(1));
+ } else {
+ PCIDBG(phb, pd->bdfn, "BUSES: Not behind a switch\n");
+ slot->retries = PHB4_BUS_CRS_RETRIES;
+ pci_slot_set_state(slot, PHB4_SLOT_BUSES_PD_WAIT_CRS);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(1));
+ }
+ case PHB4_SLOT_BUSES_PD_WAIT_SWITCH_LINK:
+ PCIDBG(phb, pd->bdfn, "BUSES: Wait for switch link\n");
+
+ if (pci_has_cap(parent, PCI_CFG_CAP_ID_EXP, false)) {
+ ecap = pci_cap(parent, PCI_CFG_CAP_ID_EXP, false);
+ pci_cfg_read32(phb, parent->bdfn, ecap + PCICAP_EXP_LCAP, &link_cap);
+ }
+
+ if (!(link_cap & PCICAP_EXP_LCAP_DL_ACT_REP)) {
+ PCIDBG(phb, parent->bdfn, "BUSES: No link state reporting\n");
+ slot->retries = PHB4_BUS_CRS_RETRIES;
+ pci_slot_set_state(slot, PHB4_SLOT_BUSES_PD_WAIT_CRS);
+ return pci_slot_set_sm_timeout(slot, secs_to_tb(1));
+ }
+
+ pci_cfg_read16(phb, parent->bdfn,
+ ecap + PCICAP_EXP_LSTAT, &link_sts);
+
+ if (link_sts & PCICAP_EXP_LSTAT_DLLL_ACT) {
+ /* Have to wait 100ms before touch config space */
+ slot->retries = PHB4_BUS_CRS_RETRIES;
+ pci_slot_set_state(slot, PHB4_SLOT_BUSES_PD_WAIT_CRS);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
+ }
+
+ if (slot->retries-- == 0) {
+ PCIDBG(phb, pd->bdfn, "BUSES: Timeout waiting downstream link\n");
+ pci_slot_set_state(slot, PHB4_SLOT_NORMAL);
+ return OPAL_HARDWARE;
+ }
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
+ case PHB4_SLOT_BUSES_PD_WAIT_CRS:
+ PCIDBG(phb, pd->bdfn, "BUSES: Wait for CRS\n");
+
+ rc = pci_cfg_read32(phb, pd->bdfn, PCI_CFG_VENDOR_ID, &vdid);
+
+ if (rc || vdid == 0xffffffff || vdid == 0x00000000) {
+ PCIERR(phb, pd->bdfn, "Error reading VENDOR ID\n");
+ pci_slot_set_state(slot, PHB4_SLOT_NORMAL);
+ return OPAL_HARDWARE;
+ }
+
+ if (vdid == 0xffff0001) {
+ PCIDBG(phb, pd->bdfn, "BUSES: Got a CRS\n");
+ if (slot->retries-- == 0) {
+ PCIERR(phb, pd->bdfn, "Timeout waiting CRS\n");
+ pci_slot_set_state(slot, PHB4_SLOT_NORMAL);
+ return OPAL_HARDWARE;
+ }
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
+ }
+
+ /* Actual Vendor ID */
+ if (slot->retries != PHB4_BUS_CRS_RETRIES)
+ PCIDBG(phb, pd->bdfn, "BUSES: Probe success after CRS\n");
+
+ /* Make devices below a bridge "re-capture" the bdfn */
+ pci_cfg_write32(phb, pd->bdfn, PCI_CFG_VENDOR_ID, vdid);
+
+ if (pd->is_bridge) {
+ PCIDBG(phb, pd->bdfn, "BUSES: Restoring buses\n");
+ pci_cfg_write8(phb, pd->bdfn, PCI_CFG_PRIMARY_BUS,
+ pd->primary_bus);
+ pci_cfg_write8(phb, pd->bdfn, PCI_CFG_SECONDARY_BUS,
+ pd->secondary_bus);
+ pci_cfg_write8(phb, pd->bdfn, PCI_CFG_SUBORDINATE_BUS,
+ pd->subordinate_bus);
+ }
+
+ slot->phb->current_pd = pci_device_iter_next(slot->phb, NULL);
+ pci_slot_set_state(slot, PHB4_SLOT_BUSES_PD_START);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(1));
+ }
+ return 0;
+}
+
static int64_t phb4_poll_link(struct pci_slot *slot)
{
struct phb4 *p = phb_to_phb4(slot->phb);
@@ -2851,9 +2974,8 @@ static int64_t phb4_poll_link(struct pci_slot *slot)
*/
PHBERR(p, "LINK: Degraded but no more retries\n");
}
- pci_restore_slot_bus_configs(slot);
- pci_slot_set_state(slot, PHB4_SLOT_NORMAL);
- return OPAL_SUCCESS;
+ pci_slot_set_state(slot, PHB4_SLOT_BUSES_START);
+ return phb4_restore_buses(slot);
}
PHBERR(p, "LINK: Went down waiting for stabilty\n");
PHBDBG(p, "LINK: DLP train control: 0x%016llx\n", reg);
@@ -3431,6 +3553,7 @@ static struct pci_slot *phb4_slot_create(struct phb *phb)
* individual platforms.
*/
slot->ops.prepare_link_change = phb4_prepare_link_change;
+ slot->ops.restore_buses = phb4_restore_buses;
slot->ops.poll_link = phb4_poll_link;
slot->ops.hreset = phb4_hreset;
slot->ops.freset = phb4_freset;
diff --git a/include/pci-slot.h b/include/pci-slot.h
index 6e7dc1c06970..dd20bb454a8a 100644
--- a/include/pci-slot.h
+++ b/include/pci-slot.h
@@ -92,6 +92,7 @@ struct pci_slot_ops {
/* SM based functions for reset */
void (*prepare_link_change)(struct pci_slot *slot, bool is_up);
+ int64_t (*restore_buses)(struct pci_slot *slot);
int64_t (*poll_link)(struct pci_slot *slot);
int64_t (*creset)(struct pci_slot *slot);
int64_t (*freset)(struct pci_slot *slot);
@@ -129,6 +130,7 @@ struct pci_slot_ops {
#define PCI_SLOT_STATE_SPOWER_DONE (PCI_SLOT_STATE_SPOWER + 2)
#define PCI_SLOT_STATE_GPRESENCE 0x00000700
#define PCI_SLOT_STATE_GPRESENCE_START (PCI_SLOT_STATE_GPRESENCE + 1)
+#define PCI_SLOT_STATE_BUSES 0x00000800
struct pci_slot {
diff --git a/include/pci.h b/include/pci.h
index 9672a2d342f6..9bb82cc30cb4 100644
--- a/include/pci.h
+++ b/include/pci.h
@@ -454,6 +454,7 @@ extern int64_t pci_find_ecap(struct phb *phb, uint16_t bdfn, uint16_t cap,
extern void pci_init_capabilities(struct phb *phb, struct pci_device *pd);
extern bool pci_wait_crs(struct phb *phb, uint16_t bdfn, uint32_t *out_vdid);
extern void pci_restore_slot_bus_configs(struct pci_slot *slot);
+extern void pci_init_all_devices(struct pci_slot *slot);
extern void pci_device_init(struct phb *phb, struct pci_device *pd);
extern void pci_device_iter_reset(struct phb *phb, struct pci_device *pd);
extern struct pci_device *pci_device_iter_next(struct phb *phb, struct pci_device *pd);
diff --git a/include/phb4.h b/include/phb4.h
index 1c68ec2e2ff7..d30733a02ce5 100644
--- a/include/phb4.h
+++ b/include/phb4.h
@@ -115,6 +115,12 @@
#define PHB4_SLOT_CRESET_WAIT_CQ (PHB4_SLOT_CRESET + 2)
#define PHB4_SLOT_CRESET_REINIT (PHB4_SLOT_CRESET + 3)
#define PHB4_SLOT_CRESET_FRESET (PHB4_SLOT_CRESET + 4)
+#define PHB4_SLOT_BUSES PCI_SLOT_STATE_BUSES
+#define PHB4_SLOT_BUSES_START (PHB4_SLOT_BUSES + 1)
+#define PHB4_SLOT_BUSES_PD_START (PHB4_SLOT_BUSES + 2)
+#define PHB4_SLOT_BUSES_PD_WAIT_SWITCH_LINK (PHB4_SLOT_BUSES + 3)
+#define PHB4_SLOT_BUSES_PD_WAIT_CRS (PHB4_SLOT_BUSES + 4)
+#define PHB4_SLOT_BUSES_PD_END (PHB4_SLOT_BUSES + 5)
/*
* PHB4 error descriptor. Errors from all components (PBCQ, PHB)
@@ -143,6 +149,9 @@ struct phb4_err {
#define PHB4_LINK_ELECTRICAL_RETRIES 100
#define PHB4_LINK_WAIT_RETRIES 200
+#define PHB4_BUS_CRS_RETRIES 40
+#define PHB4_BUS_SWITCH_LINK_RETRIES 100
+
#define PHB4_RX_ERR_MAX 8
/* PHB4 flags */
--
2.20.1
More information about the Skiboot
mailing list