[RFC PATCH 1/2] powerpc/powernv: Add support for CXL mode switch that need PHB reset

[Vaibhav Jain]

Recent updates to OPAL [1] have provided support for new CXL modes on
PHB that need to force a cold reset on the bridge (CRESET). However
PHB CRESET is a multi step process and cannot be completed
synchronously as expected by current kernel implementation that issues
opal call opal_pci_set_phb_cxl_mode().

Hence this patch updates pnv_phb_to_cxl_mode() to implement a polling
loop that handles specific error codes (OPAL_BUSY) returned from
opal_pci_set_phb_cxl_mode() and drive the OPAL pci-state machine, if the
requested CXL mode needs a CRESET.

The patch also updates pnv_phb_to_cxl_mode() to convert and return
OPAL error codes into kernel error codes. This removes a previous
issue where callers to this function would have to include
'opal-api.h' to check for specific OPAL error codes.

References:
[1]: https://lists.ozlabs.org/pipermail/skiboot/2019-January/013063.html

Signed-off-by: Vaibhav Jain <vaibhav at linux.ibm.com>
---
 arch/powerpc/platforms/powernv/pci-cxl.c | 71 +++++++++++++++++++++---
 1 file changed, 63 insertions(+), 8 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c
index 1b18111453d7..d33d662c6212 100644
--- a/arch/powerpc/platforms/powernv/pci-cxl.c
+++ b/arch/powerpc/platforms/powernv/pci-cxl.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <asm/pnv-pci.h>
 #include <asm/opal.h>
+#include <linux/delay.h>
 
 #include "pci.h"
 
@@ -18,21 +19,75 @@ int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
 	struct pci_controller *hose = pci_bus_to_host(dev->bus);
 	struct pnv_phb *phb = hose->private_data;
 	struct pnv_ioda_pe *pe;
+	unsigned long starttime, endtime;
 	int rc;
 
 	pe = pnv_ioda_get_pe(dev);
 	if (!pe)
-		return -ENODEV;
+		return -ENOENT;
+
+	pe_info(pe, "Switching PHB to CXL mode=%d\n", mode);
+
+	/*
+	 * Use a 15 second timeout for mode switch. Value arrived after
+	 * limited testing and may need more tweaking.
+	 */
+	starttime = jiffies;
+	endtime = starttime + HZ * 15;
+
+	do {
+		rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode,
+					       pe->pe_number);
+
+		/* Wait until mode transistion done */
+		if (rc != OPAL_BUSY && rc != OPAL_BUSY_EVENT)
+			break;
+
+		/* Check if we timedout */
+		if (time_after(jiffies, endtime)) {
+			rc = OPAL_TIMEOUT;
+			break;
+		}
 
-	pe_info(pe, "Switching PHB to CXL\n");
+		/* Opal Busy with mode switch. Run pci state-machine */
+		rc = opal_pci_poll(phb->opal_id);
+		if (rc >= 0) {
+			/* wait for some time */
+			if (rc > 0)
+				msleep(rc);
+			opal_poll_events(NULL);
+			rc = OPAL_BUSY;
+			/* Continue with the mode switch */
+		}
+	} while (rc == OPAL_BUSY || rc == OPAL_BUSY_EVENT);
+
+	pe_level_printk(pe, KERN_DEBUG, "CXL mode switch finished in %u-msecs.",
+			jiffies_to_msecs(jiffies - starttime));
 
-	rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
-	if (rc == OPAL_UNSUPPORTED)
-		dev_err(&dev->dev, "Required cxl mode not supported by firmware - update skiboot\n");
-	else if (rc)
-		dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
+	/* Check OPAL errors and convert them to kernel error codes */
+	switch (rc) {
+	case OPAL_SUCCESS:
+		return 0;
 
-	return rc;
+	case OPAL_PARAMETER:
+		dev_err(&dev->dev, "CXL not supported on this PHB\n");
+		return -ENOENT;
+
+	case OPAL_UNSUPPORTED:
+		dev_err(&dev->dev,
+			"Required cxl mode not supported by firmware"
+			" - update skiboot\n");
+		return -ENODEV;
+
+	case OPAL_TIMEOUT:
+		dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode Timedout\n");
+		return -ETIME;
+
+	default:
+		dev_err(&dev->dev,
+			"opal_pci_set_phb_cxl_mode failed: %i\n", rc);
+		return -EIO;
+	};
 }
 EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
 
-- 
2.20.1