[Skiboot] [RFC PATCH v2 2/3] core/pcie-slot: Extend slot state machine to restore bus numbers
Jordan Niethe
jniethe5 at gmail.com
Tue Oct 29 13:11:05 AEDT 2019
After performing a fundamental reset bus numbers need to be restored. This
is done by pci_restore_slot_bus_configs() which is called after the
PHB's link is active. Restoring the bus numbers can require waiting for
certain delays, which are currently performed using time_wait_ms(). This
leads to excessive times spent waiting in skiboot during the
OPAL_PCI_RESET opal call.
If a device is a behind a switch, it needs to wait for the switch's
downstream port link to become active before the buses can be restored.
Additionally all devices potentially need to wait for CRS.
Slots already have state machine based functions that are used for
reset. Add a new state machine function, restore_buses(), with
associated states for restoring buses. Call restore_buses() instead of
pci_restore_slot_bus_configs().
The actually process for this is not specific to the PHB so we only
implement a generic PCIe slot function. pci_restore_slot_bus_configs()
is only called by PHB4, so for now only it calls restore_buses().
Signed-off-by: Jordan Niethe <jniethe5 at gmail.com>
---
v2: - Move out from PHB4 and into PCIe generic slot
- Changes to debug print messages
core/pci-slot.c | 3 ++
core/pci.c | 7 +++
core/pcie-slot.c | 126 +++++++++++++++++++++++++++++++++++++++++++++
hw/phb4.c | 6 +--
include/pci-slot.h | 13 +++++
include/pci.h | 1 +
6 files changed, 153 insertions(+), 3 deletions(-)
diff --git a/core/pci-slot.c b/core/pci-slot.c
index ffb54cbb7a49..4321f30f98aa 100644
--- a/core/pci-slot.c
+++ b/core/pci-slot.c
@@ -79,6 +79,9 @@ static int64_t pci_slot_run_sm(struct pci_slot *slot)
slot->delay_tgt_tb = 0;
switch (slot->state & PCI_SLOT_STATE_MASK) {
+ case PCI_SLOT_STATE_BUSES:
+ ret = slot->ops.restore_buses(slot);
+ break;
case PCI_SLOT_STATE_LINK:
ret = slot->ops.poll_link(slot);
break;
diff --git a/core/pci.c b/core/pci.c
index 547c8e32272b..732ba68d5d23 100644
--- a/core/pci.c
+++ b/core/pci.c
@@ -1981,6 +1981,13 @@ void pci_restore_slot_bus_configs(struct pci_slot *slot)
slot->phb->ops->device_init, NULL);
}
+void pci_init_all_devices(struct pci_slot *slot)
+{
+ if (slot->phb->ops->device_init)
+ pci_walk_dev(slot->phb, slot->pd,
+ slot->phb->ops->device_init, NULL);
+}
+
struct pci_cfg_reg_filter *pci_find_cfg_reg_filter(struct pci_device *pd,
uint32_t start, uint32_t len)
{
diff --git a/core/pcie-slot.c b/core/pcie-slot.c
index 894fd40513c3..048a8b279c71 100644
--- a/core/pcie-slot.c
+++ b/core/pcie-slot.c
@@ -262,6 +262,131 @@ static int64_t pcie_slot_set_power_state(struct pci_slot *slot, uint8_t val)
return pcie_slot_set_power_state_ext(slot, val, true);
}
+int64_t pcie_slot_sm_restore_buses(struct pci_slot *slot)
+{
+ struct pci_device *pd, *parent;
+ uint32_t link_cap = 0;
+ uint16_t link_sts = 0;
+ int32_t ecap = 0;
+ struct phb *phb;
+ uint32_t vdid;
+ int64_t rc;
+
+ phb = slot->phb;
+ pd = slot->phb->current_pd;
+ parent = pd ? pd->parent : NULL;
+
+ switch (slot->state) {
+ case PCI_SLOT_STATE_NORMAL:
+ case PCI_SLOT_BUSES_START:
+ PCIE_SLOT_DBG(slot, "BUSES: Starts\n");
+ pci_device_iter_reset(slot->phb, NULL);
+ slot->phb->current_pd = pci_device_iter_next(slot->phb, NULL);
+ pci_slot_set_state(slot, PCI_SLOT_BUSES_PD_START);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(1));
+ case PCI_SLOT_BUSES_PD_START:
+ if (slot->phb->current_pd == NULL) {
+ PCIE_SLOT_DBG(slot, "BUSES: Finished all PCI devices\n");
+ pci_slot_set_state(slot, PCI_SLOT_STATE_NORMAL);
+ pci_init_all_devices(slot);
+ return OPAL_SUCCESS;
+ }
+
+ PCIDBG(phb, pd->bdfn, "BUSES: PCI Device Starts\n");
+ if (!pd->is_vf && !(pd->bdfn & 7) && pd->parent != NULL &&
+ pd->parent->dev_type == PCIE_TYPE_SWITCH_DNPORT) {
+ PCIDBG(phb, pd->bdfn, "BUSES: Behind a switch\n");
+ slot->retries = PCI_BUS_SWITCH_LINK_RETRIES;
+ pci_slot_set_state(slot,
+ PCI_SLOT_BUSES_PD_WAIT_SWITCH_LINK);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(1));
+ } else {
+ PCIDBG(phb, pd->bdfn, "BUSES: Not behind a switch\n");
+ slot->retries = PCI_BUS_CRS_RETRIES;
+ pci_slot_set_state(slot, PCI_SLOT_BUSES_PD_WAIT_CRS);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(1));
+ }
+ case PCI_SLOT_BUSES_PD_WAIT_SWITCH_LINK:
+ PCIDBG(phb, pd->bdfn, "BUSES: Wait for Switch Link\n");
+
+ if (pci_has_cap(parent, PCI_CFG_CAP_ID_EXP, false)) {
+ ecap = pci_cap(parent, PCI_CFG_CAP_ID_EXP, false);
+ pci_cfg_read32(phb, parent->bdfn,
+ ecap + PCICAP_EXP_LCAP, &link_cap);
+ }
+
+ if (!(link_cap & PCICAP_EXP_LCAP_DL_ACT_REP)) {
+ PCIDBG(phb, parent->bdfn,
+ "BUSES: Parent: No link state reporting\n");
+ slot->retries = PCI_BUS_CRS_RETRIES;
+ pci_slot_set_state(slot, PCI_SLOT_BUSES_PD_WAIT_CRS);
+ return pci_slot_set_sm_timeout(slot, secs_to_tb(1));
+ }
+
+ pci_cfg_read16(phb, parent->bdfn,
+ ecap + PCICAP_EXP_LSTAT, &link_sts);
+
+ if (link_sts & PCICAP_EXP_LSTAT_DLLL_ACT) {
+ /* Have to wait 100ms before touch config space */
+ PCIDBG(phb, parent->bdfn, "BUSES: Parent: Link active\n");
+ slot->retries = PCI_BUS_CRS_RETRIES;
+ pci_slot_set_state(slot, PCI_SLOT_BUSES_PD_WAIT_CRS);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
+ }
+
+ if (slot->retries-- == 0) {
+ PCIDBG(phb, pd->bdfn,
+ "BUSES: Timeout waiting downstream link\n");
+ pci_slot_set_state(slot, PCI_SLOT_STATE_NORMAL);
+ return OPAL_HARDWARE;
+ }
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
+ case PCI_SLOT_BUSES_PD_WAIT_CRS:
+ rc = pci_cfg_read32(phb, pd->bdfn, PCI_CFG_VENDOR_ID, &vdid);
+
+ if (rc || vdid == 0xffffffff || vdid == 0x00000000) {
+ PCIERR(phb, pd->bdfn, "Error reading VENDOR ID\n");
+ pci_slot_set_state(slot, PCI_SLOT_STATE_NORMAL);
+ return OPAL_HARDWARE;
+ }
+
+ if (vdid == 0xffff0001) {
+ PCIDBG(phb, pd->bdfn, "BUSES: Got a CRS\n");
+ if (slot->retries-- == 0) {
+ PCIERR(phb, pd->bdfn, "Timeout waiting for CRS\n");
+ pci_slot_set_state(slot, PCI_SLOT_STATE_NORMAL);
+ return OPAL_HARDWARE;
+ }
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
+ }
+
+ /* Actual Vendor ID */
+ if (slot->retries != PCI_BUS_CRS_RETRIES)
+ PCIDBG(phb, pd->bdfn, "BUSES: Probe success after CRS\n");
+
+ /* Make devices below a bridge "re-capture" the bdfn */
+ pci_cfg_write32(phb, pd->bdfn, PCI_CFG_VENDOR_ID, vdid);
+
+ if (pd->is_bridge) {
+ PCIDBG(phb, pd->bdfn,
+ "BUSES: Device is a bridge. Restoring buses\n");
+ pci_cfg_write8(phb, pd->bdfn, PCI_CFG_PRIMARY_BUS,
+ pd->primary_bus);
+ pci_cfg_write8(phb, pd->bdfn, PCI_CFG_SECONDARY_BUS,
+ pd->secondary_bus);
+ pci_cfg_write8(phb, pd->bdfn, PCI_CFG_SUBORDINATE_BUS,
+ pd->subordinate_bus);
+ }
+
+ PCIDBG(phb, pd->bdfn, "BUSES: PCI Device Finished\n");
+ slot->phb->current_pd = pci_device_iter_next(slot->phb, NULL);
+ pci_slot_set_state(slot, PCI_SLOT_BUSES_PD_START);
+ return pci_slot_set_sm_timeout(slot, msecs_to_tb(1));
+ }
+ return 0;
+}
+
+
static int64_t pcie_slot_sm_poll_link(struct pci_slot *slot)
{
struct phb *phb = slot->phb;
@@ -511,6 +636,7 @@ struct pci_slot *pcie_slot_create(struct phb *phb, struct pci_device *pd)
* State machine (SM) based reset stuff. The poll function is always
* unified for all cases.
*/
+ slot->ops.restore_buses = pcie_slot_sm_restore_buses;
slot->ops.poll_link = pcie_slot_sm_poll_link;
slot->ops.hreset = pcie_slot_sm_hreset;
slot->ops.freset = pcie_slot_sm_freset;
diff --git a/hw/phb4.c b/hw/phb4.c
index 3c71427aef2f..fd13abee4099 100644
--- a/hw/phb4.c
+++ b/hw/phb4.c
@@ -2851,9 +2851,8 @@ static int64_t phb4_poll_link(struct pci_slot *slot)
*/
PHBERR(p, "LINK: Degraded but no more retries\n");
}
- pci_restore_slot_bus_configs(slot);
- pci_slot_set_state(slot, PHB4_SLOT_NORMAL);
- return OPAL_SUCCESS;
+ pci_slot_set_state(slot, PCI_SLOT_BUSES_START);
+ return slot->ops.restore_buses(slot);
}
PHBERR(p, "LINK: Went down waiting for stabilty\n");
PHBDBG(p, "LINK: DLP train control: 0x%016llx\n", reg);
@@ -3431,6 +3430,7 @@ static struct pci_slot *phb4_slot_create(struct phb *phb)
* individual platforms.
*/
slot->ops.prepare_link_change = phb4_prepare_link_change;
+ slot->ops.restore_buses = pcie_slot_sm_restore_buses;
slot->ops.poll_link = phb4_poll_link;
slot->ops.hreset = phb4_hreset;
slot->ops.freset = phb4_freset;
diff --git a/include/pci-slot.h b/include/pci-slot.h
index 6e7dc1c06970..353efc7e96e1 100644
--- a/include/pci-slot.h
+++ b/include/pci-slot.h
@@ -92,6 +92,7 @@ struct pci_slot_ops {
/* SM based functions for reset */
void (*prepare_link_change)(struct pci_slot *slot, bool is_up);
+ int64_t (*restore_buses)(struct pci_slot *slot);
int64_t (*poll_link)(struct pci_slot *slot);
int64_t (*creset)(struct pci_slot *slot);
int64_t (*freset)(struct pci_slot *slot);
@@ -129,6 +130,17 @@ struct pci_slot_ops {
#define PCI_SLOT_STATE_SPOWER_DONE (PCI_SLOT_STATE_SPOWER + 2)
#define PCI_SLOT_STATE_GPRESENCE 0x00000700
#define PCI_SLOT_STATE_GPRESENCE_START (PCI_SLOT_STATE_GPRESENCE + 1)
+#define PCI_SLOT_STATE_BUSES 0x00000800
+#define PCI_SLOT_BUSES PCI_SLOT_STATE_BUSES
+#define PCI_SLOT_BUSES_START (PCI_SLOT_BUSES + 1)
+#define PCI_SLOT_BUSES_PD_START (PCI_SLOT_BUSES + 2)
+#define PCI_SLOT_BUSES_PD_WAIT_SWITCH_LINK (PCI_SLOT_BUSES + 3)
+#define PCI_SLOT_BUSES_PD_WAIT_CRS (PCI_SLOT_BUSES + 4)
+#define PCI_SLOT_BUSES_PD_END (PCI_SLOT_BUSES + 5)
+
+#define PCI_BUS_CRS_RETRIES 40
+#define PCI_BUS_SWITCH_LINK_RETRIES 100
+
struct pci_slot {
@@ -244,6 +256,7 @@ extern struct pci_slot *pcie_slot_create(struct phb *phb,
struct pci_device *pd);
extern struct pci_slot *pcie_slot_create_dynamic(struct phb *phb,
struct pci_device *pd);
+extern int64_t pcie_slot_sm_restore_buses(struct pci_slot *slot);
extern void pci_slot_add_dt_properties(struct pci_slot *slot,
struct dt_node *np);
diff --git a/include/pci.h b/include/pci.h
index 9672a2d342f6..9bb82cc30cb4 100644
--- a/include/pci.h
+++ b/include/pci.h
@@ -454,6 +454,7 @@ extern int64_t pci_find_ecap(struct phb *phb, uint16_t bdfn, uint16_t cap,
extern void pci_init_capabilities(struct phb *phb, struct pci_device *pd);
extern bool pci_wait_crs(struct phb *phb, uint16_t bdfn, uint32_t *out_vdid);
extern void pci_restore_slot_bus_configs(struct pci_slot *slot);
+extern void pci_init_all_devices(struct pci_slot *slot);
extern void pci_device_init(struct phb *phb, struct pci_device *pd);
extern void pci_device_iter_reset(struct phb *phb, struct pci_device *pd);
extern struct pci_device *pci_device_iter_next(struct phb *phb, struct pci_device *pd);
--
2.20.1
More information about the Skiboot
mailing list