[Skiboot] [PATCH 1/2] xscom: Trace XSCOMs that take more than 10us

Benjamin Herrenschmidt benh at kernel.crashing.org
Thu Sep 8 16:05:52 AEST 2016


If a XSCOM takes more than 10us, this will display an entry in the OPAL
log, with the PCB address, the time, and a number which is either an
error code (negative: the access failed) or a number of retries
(positive: the number of times the HMER status returned "1").

We need to suspend that testing while the timebase is being resynchronized
otherwise we will get bogus values.

Signed-off-by: Benjamin Herrenschmidt <benh at kernel.crashing.org>
---
 hw/chiptod.c    | 12 ++++++---
 hw/xscom.c      | 81 ++++++++++++++++++++++++++++++++++++++++++++++-----------
 include/xscom.h |  5 ++++
 3 files changed, 80 insertions(+), 18 deletions(-)

diff --git a/hw/chiptod.c b/hw/chiptod.c
index f647830..e0fe397 100644
--- a/hw/chiptod.c
+++ b/hw/chiptod.c
@@ -635,13 +635,15 @@ static bool chiptod_to_tb(void)
 		return false;
 	}
 
+	xscom_set_tb_unreliable(true);
+
 	/* Make us ready to get the TB from the chipTOD */
 	mtspr(SPR_TFMR, base_tfmr | SPR_TFMR_MOVE_CHIP_TOD_TO_TB);
 
 	/* Tell the ChipTOD to send it */
 	if (xscom_writeme(TOD_CHIPTOD_TO_TB, PPC_BIT(0))) {
 		prerror("XSCOM error writing CHIPTOD_TO_TB\n");
-		return false;
+		goto fail;
 	}
 
 	/* Wait for it to complete */
@@ -649,16 +651,20 @@ static bool chiptod_to_tb(void)
 	do {
 		if (++timeout >= TIMEOUT_LOOPS) {
 			prerror("Chip to TB timeout\n");
-			return false;
+			goto fail;
 		}
 		tfmr = mfspr(SPR_TFMR);
 		if (tfmr & SPR_TFMR_TFMR_CORRUPT) {
 			prerror("MoveToTB: corrupt TFMR !\n");
-			return false;
+			goto fail;
 		}
 	} while(tfmr & SPR_TFMR_MOVE_CHIP_TOD_TO_TB);
 
+	xscom_set_tb_unreliable(false);
 	return true;
+ fail:
+	xscom_set_tb_unreliable(false);
+	return false;
 }
 
 static bool chiptod_check_tb_running(void)
diff --git a/hw/xscom.c b/hw/xscom.c
index 9e9dcee..1b7e896 100644
--- a/hw/xscom.c
+++ b/hw/xscom.c
@@ -25,6 +25,9 @@
 #include <opal-api.h>
 #include <timebase.h>
 
+/* Delay after which we warn that a XSCOM is taking a long time */
+#define XSCOM_DELAY_WARN_THRESHOLD	(usecs_to_tb(10))
+
 /* Mask of bits to clear in HMER before an access */
 #define HMER_CLR_MASK	(~(SPR_HMER_XSCOM_FAIL | \
 			   SPR_HMER_XSCOM_DONE | \
@@ -61,6 +64,7 @@ static struct {
  * use a global lock instead
  */
 static struct lock xscom_lock = LOCK_UNLOCKED;
+static int64_t xscom_tb_unreliable;
 
 static inline void *xscom_addr(uint32_t gcid, uint32_t pcb_addr)
 {
@@ -220,7 +224,7 @@ static bool xscom_gcid_ok(uint32_t gcid)
  * Low level XSCOM access functions, perform a single direct xscom
  * access via MMIO
  */
-static int __xscom_read(uint32_t gcid, uint32_t pcb_addr, uint64_t *val)
+static int64_t __xscom_read(uint32_t gcid, uint32_t pcb_addr, uint64_t *val)
 {
 	uint64_t hmer;
 	int64_t ret, retries;
@@ -244,7 +248,7 @@ static int __xscom_read(uint32_t gcid, uint32_t pcb_addr, uint64_t *val)
 
 		/* Check for error */
 		if (!(hmer & SPR_HMER_XSCOM_FAIL))
-			return OPAL_SUCCESS;
+			return retries;
 
 		/* Handle error and possibly eventually retry */
 		ret = xscom_handle_error(hmer, gcid, pcb_addr, false, retries);
@@ -256,7 +260,7 @@ static int __xscom_read(uint32_t gcid, uint32_t pcb_addr, uint64_t *val)
 	return ret;
 }
 
-static int __xscom_write(uint32_t gcid, uint32_t pcb_addr, uint64_t val)
+static int64_t __xscom_write(uint32_t gcid, uint32_t pcb_addr, uint64_t val)
 {
 	uint64_t hmer;
 	int64_t ret, retries = 0;
@@ -280,7 +284,7 @@ static int __xscom_write(uint32_t gcid, uint32_t pcb_addr, uint64_t val)
 
 		/* Check for error */
 		if (!(hmer & SPR_HMER_XSCOM_FAIL))
-			return OPAL_SUCCESS;
+			return retries;
 
 		/* Handle error and possibly eventually retry */
 		ret = xscom_handle_error(hmer, gcid, pcb_addr, true, retries);
@@ -295,11 +299,12 @@ static int __xscom_write(uint32_t gcid, uint32_t pcb_addr, uint64_t val)
 /*
  * Indirect XSCOM access functions
  */
-static int xscom_indirect_read(uint32_t gcid, uint64_t pcb_addr, uint64_t *val)
+static int64_t xscom_indirect_read(uint32_t gcid, uint64_t pcb_addr, uint64_t *val)
 {
 	uint32_t addr;
 	uint64_t data;
 	int rc, retries;
+	int64_t comp_retries = 0;
 
 	if (proc_gen < proc_gen_p8) {
 		*val = (uint64_t)-1;
@@ -311,14 +316,16 @@ static int xscom_indirect_read(uint32_t gcid, uint64_t pcb_addr, uint64_t *val)
 	data = XSCOM_DATA_IND_READ |
 		(pcb_addr & XSCOM_ADDR_IND_ADDR);
 	rc = __xscom_write(gcid, addr, data);
-	if (rc)
+	if (rc < 0)
 		goto bail;
+	comp_retries = rc;
 
 	/* Wait for completion */
 	for (retries = 0; retries < XSCOM_IND_MAX_RETRIES; retries++) {
 		rc = __xscom_read(gcid, addr, &data);
-		if (rc)
+		if (rc < 0)
 			goto bail;
+		comp_retries += rc;
 		if ((data & XSCOM_DATA_IND_COMPLETE) &&
 		    ((data & XSCOM_DATA_IND_ERR) == 0)) {
 			*val = data & XSCOM_DATA_IND_DATA;
@@ -333,16 +340,19 @@ static int xscom_indirect_read(uint32_t gcid, uint64_t pcb_addr, uint64_t *val)
 		}
 	}
  bail:
-	if (rc)
+	if (rc < 0) {
 		*val = (uint64_t)-1;
-	return rc;
+		return rc;
+	}
+	return comp_retries;
 }
 
-static int xscom_indirect_write(uint32_t gcid, uint64_t pcb_addr, uint64_t val)
+static int64_t xscom_indirect_write(uint32_t gcid, uint64_t pcb_addr, uint64_t val)
 {
 	uint32_t addr;
 	uint64_t data;
 	int rc, retries;
+	int64_t comp_retries = 0;
 
 	if (proc_gen < proc_gen_p8)
 		return OPAL_UNSUPPORTED;
@@ -353,14 +363,16 @@ static int xscom_indirect_write(uint32_t gcid, uint64_t pcb_addr, uint64_t val)
 	data |= val & XSCOM_ADDR_IND_DATA;
 
 	rc = __xscom_write(gcid, addr, data);
-	if (rc)
+	if (rc < 0)
 		goto bail;
+	comp_retries = rc;
 
 	/* Wait for completion */
 	for (retries = 0; retries < XSCOM_IND_MAX_RETRIES; retries++) {
 		rc = __xscom_read(gcid, addr, &data);
-		if (rc)
+		if (rc < 0)
 			goto bail;
+		comp_retries += rc;
 		if ((data & XSCOM_DATA_IND_COMPLETE) &&
 		    ((data & XSCOM_DATA_IND_ERR) == 0))
 			break;
@@ -373,7 +385,9 @@ static int xscom_indirect_write(uint32_t gcid, uint64_t pcb_addr, uint64_t val)
 		}
 	}
  bail:
-	return rc;
+	if (rc < 0)
+		return rc;
+	return comp_retries;
 }
 
 static uint32_t xscom_decode_chiplet(uint32_t partid, uint64_t *pcb_addr)
@@ -397,8 +411,9 @@ static uint32_t xscom_decode_chiplet(uint32_t partid, uint64_t *pcb_addr)
  */
 int xscom_read(uint32_t partid, uint64_t pcb_addr, uint64_t *val)
 {
+	uint64_t t_begin, t_end;
 	uint32_t gcid;
-	int rc;
+	int64_t rc;
 
 	/* Handle part ID decoding */
 	switch(partid >> 28) {
@@ -426,15 +441,27 @@ int xscom_read(uint32_t partid, uint64_t pcb_addr, uint64_t *val)
 
 	/* HW822317 requires us to do global locking */
 	lock(&xscom_lock);
+	t_begin = mftb();
 
 	/* Direct vs indirect access */
 	if (pcb_addr & XSCOM_ADDR_IND_FLAG)
 		rc = xscom_indirect_read(gcid, pcb_addr, val);
 	else
 		rc = __xscom_read(gcid, pcb_addr & 0x7fffffff, val);
+	t_end = mftb();
+
+	/* Check if we are told not to bother about warning on long delays... */
+	if (xscom_tb_unreliable)
+		t_end = t_begin;
 
 	/* Unlock it */
 	unlock(&xscom_lock);
+
+	if (tb_compare(t_begin + XSCOM_DELAY_WARN_THRESHOLD, t_end) == TB_ABEFOREB)
+		printf("XSCOM Read from %08llx took %ld us with %lld (err/retries)\n",
+		       pcb_addr, tb_to_usecs(t_end - t_begin), rc);
+	if (rc > 0)
+		rc = 0;
 	return rc;
 }
 
@@ -442,8 +469,9 @@ opal_call(OPAL_XSCOM_READ, xscom_read, 3);
 
 int xscom_write(uint32_t partid, uint64_t pcb_addr, uint64_t val)
 {
+	uint64_t t_begin, t_end;
 	uint32_t gcid;
-	int rc;
+	int64_t rc;
 
 	/* Handle part ID decoding */
 	switch(partid >> 28) {
@@ -469,19 +497,42 @@ int xscom_write(uint32_t partid, uint64_t pcb_addr, uint64_t val)
 
 	/* HW822317 requires us to do global locking */
 	lock(&xscom_lock);
+	t_begin = mftb();
 
 	/* Direct vs indirect access */
 	if (pcb_addr & XSCOM_ADDR_IND_FLAG)
 		rc = xscom_indirect_write(gcid, pcb_addr, val);
 	else
 		rc = __xscom_write(gcid, pcb_addr & 0x7fffffff, val);
+	t_end = mftb();
+
+	/* Check if we are told not to bother about warning on long delays... */
+	if (xscom_tb_unreliable)
+		t_end = t_begin;
 
 	/* Unlock it */
 	unlock(&xscom_lock);
+
+	if (tb_compare(t_begin + XSCOM_DELAY_WARN_THRESHOLD, t_end) == TB_ABEFOREB)
+		printf("XSCOM Write to %08llx took %ld us with %lld (err/retries)\n",
+		       pcb_addr, tb_to_usecs(t_end - t_begin), rc);
+	if (rc > 0)
+		rc = 0;
+
 	return rc;
 }
 opal_call(OPAL_XSCOM_WRITE, xscom_write, 3);
 
+void xscom_set_tb_unreliable(bool unreliable)
+{
+	lock(&xscom_lock);
+	if (unreliable)
+		xscom_tb_unreliable++;
+	else if (xscom_tb_unreliable)
+		xscom_tb_unreliable--;
+	unlock(&xscom_lock);
+}
+
 int xscom_readme(uint64_t pcb_addr, uint64_t *val)
 {
 	return xscom_read(this_cpu()->chip_id, pcb_addr, val);
diff --git a/include/xscom.h b/include/xscom.h
index f955021..4457b82 100644
--- a/include/xscom.h
+++ b/include/xscom.h
@@ -220,6 +220,11 @@ extern void xscom_init(void);
 /* Mark XSCOM lock as being in console path */
 extern void xscom_used_by_console(void);
 
+/* Mark XSCOM timebase warnings unreliable (used by the chiptod
+ * code when TB is being resynchronized
+ */
+extern void xscom_set_tb_unreliable(bool unreliable);
+
 /* Returns true if XSCOM can be used. Typically this returns false if
  * the current CPU holds the XSCOM lock (to avoid re-entrancy from error path).
  */
-- 
2.7.4



More information about the Skiboot mailing list