[Skiboot] [PATCH v3] xive: disable store EOI support

[Cédric Le Goater]

Hardware has limitations which would require to put a sync after each
store EOI to make sure the MMIO operations that change the ESB state
are ordered. This is a killer for performance and the PHBs do not
support the sync. So remove the store EOI for the moment, until
hardware is improved.

Also, while we are at changing the XIVE source flags, let's fix the
settings for the PHB4s which should follow these rules :

  - SHIFT_BUG    for DD10
  - STORE_EOI    for DD20 and if enabled
  - TRIGGER_PAGE for DDx0 and if not STORE_EOI

Signed-off-by: Cédric Le Goater <clg at kaod.org>
---

 Tested on a Witherspoon system.

 Changes since v3 :

 - fix the XIVE source flags settings for the PHB4s

 hw/phb4.c      | 18 ++++++++++++++----
 hw/xive.c      | 13 ++++++++++---
 include/xive.h |  3 +++
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/hw/phb4.c b/hw/phb4.c
index e45be01fe094..721fc85bede8 100644
--- a/hw/phb4.c
+++ b/hw/phb4.c
@@ -145,6 +145,9 @@ static void phb4_init_hw(struct phb4 *p, bool first_init);
 #define PHBLOGCFG(p, fmt, a...) do {} while (0)
 #endif
 
+#define PHB4_CAN_STORE_EOI(p) \
+	(XIVE_STORE_EOI_ENABLED && ((p)->rev >= PHB4_REV_NIMBUS_DD20))
+
 static bool verbose_eeh;
 static bool pci_tracing;
 static bool pci_eeh_mmio;
@@ -4609,7 +4612,8 @@ static void phb4_init_hw(struct phb4 *p, bool first_init)
 		val |= SETFIELD(PHB_CTRLR_TVT_ADDR_SEL, 0ull, TVT_DD1_2_PER_PE);
 	} else {
 		val |= SETFIELD(PHB_CTRLR_TVT_ADDR_SEL, 0ull, TVT_2_PER_PE);
-		val |= PHB_CTRLR_IRQ_STORE_EOI;
+		if (PHB4_CAN_STORE_EOI(p))
+			val |= PHB_CTRLR_IRQ_STORE_EOI;
 	}
 
 	if (!pci_eeh_mmio)
@@ -5204,10 +5208,16 @@ static void phb4_create(struct dt_node *np)
 	/* Load capp microcode into capp unit */
 	load_capp_ucode(p);
 
-	/* Register all interrupt sources with XIVE */
-	irq_flags = XIVE_SRC_SHIFT_BUG | XIVE_SRC_TRIGGER_PAGE;
-	if (p->rev >= PHB4_REV_NIMBUS_DD20)
+	/* Compute XIVE source flags depending on PHB revision */
+	irq_flags = 0;
+	if (p->rev == PHB4_REV_NIMBUS_DD10)
+		irq_flags |= XIVE_SRC_SHIFT_BUG;
+	if (PHB4_CAN_STORE_EOI(p))
 		irq_flags |= XIVE_SRC_STORE_EOI;
+	else
+		irq_flags |= XIVE_SRC_TRIGGER_PAGE;
+
+	/* Register all interrupt sources with XIVE */
 	xive_register_hw_source(p->base_msi, p->num_irqs - 8, 16,
 				p->int_mmio, irq_flags, NULL, NULL);
 
diff --git a/hw/xive.c b/hw/xive.c
index f9c80cc7177d..17e091f4648b 100644
--- a/hw/xive.c
+++ b/hw/xive.c
@@ -487,6 +487,9 @@ struct xive {
 	void		*q_ovf;
 };
 
+#define XIVE_CAN_STORE_EOI(x) \
+	(XIVE_STORE_EOI_ENABLED && ((x)->rev >= XIVE_REV_2))
+
 /* Global DT node */
 static struct dt_node *xive_dt_node;
 
@@ -1761,7 +1764,11 @@ static bool xive_config_init(struct xive *x)
 
 	/* Enable StoreEOI */
 	val = xive_regr(x, VC_SBC_CONFIG);
-	val |= VC_SBC_CONF_CPLX_CIST | VC_SBC_CONF_CIST_BOTH;
+	if (XIVE_CAN_STORE_EOI(x))
+		val |= VC_SBC_CONF_CPLX_CIST | VC_SBC_CONF_CIST_BOTH;
+	else
+		xive_dbg(x, "store EOI is disabled\n");
+
 	val |= VC_SBC_CONF_NO_UPD_PRF;
 	xive_regw(x, VC_SBC_CONFIG, val);
 
@@ -2807,7 +2814,7 @@ void xive_register_ipi_source(uint32_t base, uint32_t count, void *data,
 	assert(s);
 
 	/* Store EOI supported on DD2.0 */
-	if (x->rev >= XIVE_REV_2)
+	if (XIVE_CAN_STORE_EOI(x))
 		flags |= XIVE_SRC_STORE_EOI;
 
 	/* Callbacks assume the MMIO base corresponds to the first
@@ -2915,7 +2922,7 @@ static struct xive *init_one_xive(struct dt_node *np)
 
 	/* Register built-in source controllers (aka IPIs) */
 	flags = XIVE_SRC_EOI_PAGE1 | XIVE_SRC_TRIGGER_PAGE;
-	if (x->rev >= XIVE_REV_2)
+	if (XIVE_CAN_STORE_EOI(x))
 		flags |= XIVE_SRC_STORE_EOI;
 	__xive_register_source(x, &x->ipis, x->int_base,
 			       x->int_hw_bot - x->int_base, IPI_ESB_SHIFT,
diff --git a/include/xive.h b/include/xive.h
index 5b4454bd3455..0c728a44670e 100644
--- a/include/xive.h
+++ b/include/xive.h
@@ -493,6 +493,9 @@ uint32_t xive_alloc_ipi_irqs(uint32_t chip_id, uint32_t count, uint32_t align);
 uint64_t xive_get_notify_port(uint32_t chip_id, uint32_t ent);
 uint32_t xive_get_notify_base(uint32_t girq);
 
+/* XIVE feature flag to de/activate store EOI */
+#define XIVE_STORE_EOI_ENABLED 0
+
 /* Internal IRQ flags */
 #define XIVE_SRC_TRIGGER_PAGE	0x01 /* Trigger page exist (either separate
 				      * or not, so different from the OPAL
-- 
2.13.6