[Skiboot] [PATCH v2 3/5] hw/phb3: disable CAPI mode during complete reset
Gavin Shan
gwshan at linux.vnet.ibm.com
Mon Jan 16 10:26:49 AEDT 2017
On Fri, Jan 13, 2017 at 04:09:39PM +1100, Andrew Donnellan wrote:
>When fast rebooting or kexec-ing a system with a PHB in CAPI mode, we need
>to return the PHB to regular PCIe mode.
>
>In order to do this, we have to reset a bunch of registers to their
>pre-CAPI-mode state. However, doing this while there is traffic going over
>the PCI link is dangerous and will generally cause a checkstop.
>
>As such, we want to do this while the PHB is fenced. Conveniently, during a
>complete reset we force a PHB fence, so this is a good opportunity to
>disable CAPI mode.
>
>When doing a complete reset, if the PHB is in CAPI mode, execute a sequence
>of SCOMs to reset PHB-related registers back to their regular, PCIe mode
>values.
>
>Signed-off-by: Andrew Donnellan <andrew.donnellan at au1.ibm.com>
>
>---
>
>v1->v2:
>* add comments to remind people to update disable_capi_mode()
>---
> hw/phb3.c | 140 ++++++++++++++++++++++++++++++++++++++++++++++++--
> include/phb3.h | 9 +--
> 2 files changed, 140 insertions(+), 9 deletions(-)
>
>diff --git a/hw/phb3.c b/hw/phb3.c
>index e246e46..87dc87f 100644
>--- a/hw/phb3.c
>+++ b/hw/phb3.c
>@@ -47,6 +47,11 @@ static void phb3_init_hw(struct phb3 *p, bool first_init);
> #define PHBERR(p, fmt, a...) prlog(PR_ERR, "PHB#%04x: " fmt, \
> (p)->phb.opal_id, ## a)
>
>+#define PE_CAPP_EN 0x9013c03
>+
>+#define PE_REG_OFFSET(p) \
>+ ((PHB3_IS_NAPLES(p) && (p)->index) ? 0x40 : 0x0)
>+
> /* Helper to select an IODA table entry */
> static inline void phb3_ioda_sel(struct phb3 *p, uint32_t table,
> uint32_t addr, bool autoinc)
>@@ -2571,6 +2576,121 @@ static void do_capp_recovery_scoms(struct phb3 *p)
> xscom_write(p->chip_id, CAPP_ERR_STATUS_CTRL + offset, reg);
> }
>
>+/*
>+ * Disable CAPI mode on a PHB.
>+ *
>+ * Must be done while PHB is fenced and in recovery. Leaves CAPP in recovery -
>+ * we can't come out of recovery until the PHB has been reinitialised.
>+ *
>+ * We don't reset generic error registers here - we rely on phb3_init_hw() to
>+ * do that.
>+ *
>+ * Sets PHB3_CAPP_DISABLING flag when complete.
>+ */
>+static void disable_capi_mode(struct phb3 *p)
>+{
>+ struct proc_chip *chip = get_chip(p->chip_id);
>+ uint64_t reg;
>+ uint32_t offset = PHB3_CAPP_REG_OFFSET(p);
>+
>+ lock(&capi_lock);
>+
>+ xscom_read(p->chip_id, PE_CAPP_EN + PE_REG_OFFSET(p), ®);
>+ if (!(reg & PPC_BIT(0))) {
>+ /* Not in CAPI mode, no action required */
>+ unlock(&capi_lock);
>+ return;
>+ }
>+
>+ PHBDBG(p, "CAPP: Disabling CAPI mode\n");
>+ if (!(chip->capp_phb3_attached_mask & (1 << p->index)))
>+ PHBERR(p, "CAPP: CAPP attached mask not set!\n");
>+
>+ xscom_read(p->chip_id, CAPP_ERR_STATUS_CTRL + offset, ®);
>+ if (!(reg & PPC_BIT(0))) {
>+ PHBERR(p, "CAPP: not in recovery, can't disable CAPI mode!\n");
>+ return;
I have no knowledge to understand the logic, but @capi_lock is missed
to be released?
>+ }
>+
>+ /* Snoop CAPI Configuration Register - disable snooping */
>+ xscom_write(p->chip_id, SNOOP_CAPI_CONFIG + offset, 0ull);
>+
>+ /* APC Master PB Control Register - disable examining cResps */
>+ xscom_read(p->chip_id, APC_MASTER_PB_CTRL + offset, ®);
>+ reg &= ~PPC_BIT(3);
>+ xscom_write(p->chip_id, APC_MASTER_PB_CTRL + offset, reg);
>+
>+ /* APC Master Config Register - de-select PHBs */
>+ xscom_read(p->chip_id, APC_MASTER_CAPI_CTRL + offset, ®);
>+ reg &= ~PPC_BITMASK(1, 3);
>+ xscom_write(p->chip_id, APC_MASTER_CAPI_CTRL + offset, reg);
>+
>+ /* PE Bus AIB Mode Bits */
>+ xscom_read(p->chip_id, p->pci_xscom + 0xf, ®);
>+ reg |= PPC_BITMASK(7, 8); /* Ch2 command credit */
>+ reg &= ~PPC_BITMASK(40, 42); /* Disable HOL blocking */
>+ xscom_write(p->chip_id, p->pci_xscom + 0xf, reg);
>+
>+ /* PCI Hardware Configuration 0 Register - all store queues free */
>+ xscom_read(p->chip_id, p->pe_xscom + 0x18, ®);
>+ reg &= ~PPC_BIT(14);
>+ reg |= PPC_BIT(15);
>+ xscom_write(p->chip_id, p->pe_xscom + 0x18, reg);
>+
>+ /*
>+ * PCI Hardware Configuration 1 Register - enable read response
>+ * arrival/address request ordering
>+ */
>+ xscom_read(p->chip_id, p->pe_xscom + 0x19, ®);
>+ reg |= PPC_BITMASK(17,18);
>+ xscom_write(p->chip_id, p->pe_xscom + 0x19, reg);
>+
>+ /*
>+ * AIB TX Command Credit Register - set AIB credit values back to
>+ * normal
>+ */
>+ xscom_read(p->chip_id, p->pci_xscom + 0xd, ®);
>+ reg |= PPC_BIT(42);
>+ reg &= ~PPC_BITMASK(43, 47);
>+ xscom_write(p->chip_id, p->pci_xscom + 0xd, reg);
>+
>+ /* AIB TX Credit Init Timer - reset timer */
>+ xscom_write(p->chip_id, p->pci_xscom + 0xc, 0xff00000000000000);
>+
>+ /*
>+ * PBCQ Mode Control Register - set dcache handling to normal, not CAPP
>+ * mode
>+ */
>+ xscom_read(p->chip_id, p->pe_xscom + 0xb, ®);
>+ reg &= ~PPC_BIT(25);
>+ xscom_write(p->chip_id, p->pe_xscom + 0xb, reg);
>+
>+ /* Registers touched by phb3_init_capp_regs() */
>+
>+ /* CAPP Transport Control Register */
>+ xscom_write(p->chip_id, TRANSPORT_CONTROL + offset, 0x0001000000000000);
>+
>+ /* Canned pResp Map Register 0/1/2 */
>+ xscom_write(p->chip_id, CANNED_PRESP_MAP0 + offset, 0);
>+ xscom_write(p->chip_id, CANNED_PRESP_MAP1 + offset, 0);
>+ xscom_write(p->chip_id, CANNED_PRESP_MAP2 + offset, 0);
>+
>+ /* Flush SUE State Map Register */
>+ xscom_write(p->chip_id, FLUSH_SUE_STATE_MAP + offset, 0);
>+
>+ /* CAPP Epoch and Recovery Timers Control Register */
>+ xscom_write(p->chip_id, CAPP_EPOCH_TIMER_CTRL + offset, 0);
>+
>+ /* PE Secure CAPP Enable Register - we're all done! Disable CAPP mode! */
>+ xscom_write(p->chip_id, PE_CAPP_EN + PE_REG_OFFSET(p), 0ull);
>+
>+ /* Trigger CAPP recovery scoms after reinit */
>+ p->flags |= PHB3_CAPP_DISABLING;
>+
>+ chip->capp_phb3_attached_mask &= ~(1 << p->index);
>+ unlock(&capi_lock);
>+}
>+
> static int64_t phb3_creset(struct pci_slot *slot)
> {
> struct phb3 *p = phb_to_phb3(slot->phb);
>@@ -2602,6 +2722,10 @@ static int64_t phb3_creset(struct pci_slot *slot)
> if (!phb3_fenced(p))
> xscom_write(p->chip_id, p->pe_xscom + 0x2, 0x000000f000000000ull);
>
>+ /* Now that we're guaranteed to be fenced, disable CAPI mode */
>+ if (!(p->flags & PHB3_CAPP_RECOVERY))
>+ disable_capi_mode(p);
>+
> /* Clear errors in NFIR and raise ETU reset */
> xscom_read(p->chip_id, p->pe_xscom + 0x0, &p->nfir_cache);
>
>@@ -2641,6 +2765,11 @@ static int64_t phb3_creset(struct pci_slot *slot)
> p->flags &= ~PHB3_CAPP_RECOVERY;
> phb3_init_hw(p, false);
>
>+ if (p->flags & PHB3_CAPP_DISABLING) {
>+ do_capp_recovery_scoms(p);
>+ p->flags &= ~PHB3_CAPP_DISABLING;
>+ }
>+
> pci_slot_set_state(slot, PHB3_SLOT_CRESET_FRESET);
> return pci_slot_set_sm_timeout(slot, msecs_to_tb(100));
> case PHB3_SLOT_CRESET_FRESET:
>@@ -3459,11 +3588,11 @@ static void phb3_init_capp_errors(struct phb3 *p)
> out_be64(p->regs + PHB_LEM_ERROR_MASK, 0x40018e2400022482);
> }
>
>-#define PE_CAPP_EN 0x9013c03
>-
>-#define PE_REG_OFFSET(p) \
>- ((PHB3_IS_NAPLES(p) && (p)->index) ? 0x40 : 0x0)
>-
>+/*
>+ * Enable CAPI mode on a PHB
>+ *
>+ * Changes to this init sequence may require updating disable_capi_mode().
>+ */
> static int64_t enable_capi_mode(struct phb3 *p, uint64_t pe_number, bool dma_mode)
> {
> uint64_t reg;
>@@ -3634,6 +3763,7 @@ static int64_t phb3_set_capi_mode(struct phb *phb, uint64_t mode,
>
> switch (mode) {
> case OPAL_PHB_CAPI_MODE_PCIE:
>+ /* Switching back to PCIe mode requires a creset */
> return OPAL_UNSUPPORTED;
>
> case OPAL_PHB_CAPI_MODE_CAPI:
>diff --git a/include/phb3.h b/include/phb3.h
>index 9bedc3a..cb36515 100644
>--- a/include/phb3.h
>+++ b/include/phb3.h
>@@ -257,10 +257,11 @@ struct phb3_err {
> #define PHB3_LINK_ELECTRICAL_RETRIES 20
>
> /* PHB3 flags */
>-#define PHB3_AIB_FENCED 0x00000001
>-#define PHB3_CFG_USE_ASB 0x00000002
>-#define PHB3_CFG_BLOCKED 0x00000004
>-#define PHB3_CAPP_RECOVERY 0x00000008
>+#define PHB3_AIB_FENCED (1 << 0)
>+#define PHB3_CFG_USE_ASB (1 << 1)
>+#define PHB3_CFG_BLOCKED (1 << 2)
>+#define PHB3_CAPP_RECOVERY (1 << 3)
>+#define PHB3_CAPP_DISABLING (1 << 4)
>
> struct phb3 {
> unsigned int index; /* 0..2 index inside P8 */
>--
>git-series 0.9.1
>
>_______________________________________________
>Skiboot mailing list
>Skiboot at lists.ozlabs.org
>https://lists.ozlabs.org/listinfo/skiboot
More information about the Skiboot
mailing list