[Skiboot] [PATCH v2 09/15] platforms/astbmc/witherspoon: Rework NPU presence detection

[Andrew Donnellan]

Rework NPU presence detection in preparation for supporting both NVLink and
OpenCAPI devices operating simultaneously on the same NPU.

If an OpenCAPI card is connected to GPU#0, and an NVLink GPU is connected
to GPU#1, the GPU will only receive 2 links rather than the usual 3.

The reason for this is that without the OpenCAPI card, the GPU would be
use links 3-5, connected to NPU bricks 3-5, which needs both stacks 1 and 2
to be in NVLink mode.

However, with an OpenCAPI card in the GPU#0 slot that uses links 0-1, we
need to use NPU bricks 2-3, which means stack 1 must be set in OpenCAPI
mode. As such, the GPU will be restricted to using links 4 and 5.

Signed-off-by: Andrew Donnellan <andrew.donnellan at au1.ibm.com>
Reviewed-by: Frederic Barrat <fbarrat at linux.ibm.com>
---
 platforms/astbmc/witherspoon.c | 62 ++++++++++++++++++++++++++---------
 1 file changed, 47 insertions(+), 15 deletions(-)

diff --git a/platforms/astbmc/witherspoon.c b/platforms/astbmc/witherspoon.c
index fe138991696f..c41f0c5b1971 100644
--- a/platforms/astbmc/witherspoon.c
+++ b/platforms/astbmc/witherspoon.c
@@ -233,6 +233,8 @@ static void witherspoon_npu2_device_detect(struct npu2 *npu)
 	int rc;
 
 	bool gpu0_present, gpu1_present;
+	enum npu2_dev_type gpu0_type = NPU2_DEV_TYPE_UNKNOWN;
+	enum npu2_dev_type gpu1_type = NPU2_DEV_TYPE_UNKNOWN;
 
 	if (witherspoon_type != WITHERSPOON_TYPE_REDBUD) {
 		prlog(PR_DEBUG, "PLAT: Setting all NPU links to NVLink, OpenCAPI only supported on Redbud\n");
@@ -298,19 +300,11 @@ static void witherspoon_npu2_device_detect(struct npu2 *npu)
 		if (state & (1 << 0)) {
 			prlog(PR_DEBUG, "PLAT: Chip %d GPU#0 is OpenCAPI\n",
 			      chip->id);
-			/*
-			 * On witherspoon, bricks 2 and 3 are connected to
-			 * the lanes matching links 0 and 1 in OpenCAPI mode.
-			 */
-			set_link_details(npu, 1, 3, NPU2_DEV_TYPE_OPENCAPI);
-			/* We current don't support using the second link */
-			set_link_details(npu, 0, 2, NPU2_DEV_TYPE_UNKNOWN);
+			gpu0_type = NPU2_DEV_TYPE_OPENCAPI;
 		} else {
 			prlog(PR_DEBUG, "PLAT: Chip %d GPU#0 is NVLink\n",
 			      chip->id);
-			set_link_details(npu, 0, 0, NPU2_DEV_TYPE_NVLINK);
-			set_link_details(npu, 1, 1, NPU2_DEV_TYPE_NVLINK);
-			set_link_details(npu, 2, 2, NPU2_DEV_TYPE_NVLINK);
+			gpu0_type = NPU2_DEV_TYPE_NVLINK;
 		}
 	}
 
@@ -318,16 +312,54 @@ static void witherspoon_npu2_device_detect(struct npu2 *npu)
 		if (state & (1 << 1)) {
 			prlog(PR_DEBUG, "PLAT: Chip %d GPU#1 is OpenCAPI\n",
 			      chip->id);
-			set_link_details(npu, 4, 4, NPU2_DEV_TYPE_OPENCAPI);
-			/* We current don't support using the second link */
-			set_link_details(npu, 5, 5, NPU2_DEV_TYPE_UNKNOWN);
+			gpu1_type = NPU2_DEV_TYPE_OPENCAPI;
 		} else {
 			prlog(PR_DEBUG, "PLAT: Chip %d GPU#1 is NVLink\n",
 			      chip->id);
+			gpu1_type = NPU2_DEV_TYPE_NVLINK;
+		}
+	}
+
+	if (gpu0_type == NPU2_DEV_TYPE_OPENCAPI) {
+		set_link_details(npu, 1, 3, NPU2_DEV_TYPE_OPENCAPI);
+		/* We currently don't support using the second link */
+		set_link_details(npu, 0, 2, NPU2_DEV_TYPE_UNKNOWN);
+	}
+
+	if (gpu0_type == NPU2_DEV_TYPE_NVLINK) {
+		set_link_details(npu, 0, 0, NPU2_DEV_TYPE_NVLINK);
+		set_link_details(npu, 1, 1, NPU2_DEV_TYPE_NVLINK);
+		set_link_details(npu, 2, 2, NPU2_DEV_TYPE_NVLINK);
+	}
+
+	if (gpu1_type == NPU2_DEV_TYPE_OPENCAPI) {
+		set_link_details(npu, 4, 4, NPU2_DEV_TYPE_OPENCAPI);
+		/* We currently don't support using the second link */
+		set_link_details(npu, 5, 5, NPU2_DEV_TYPE_UNKNOWN);
+	}
+
+	/*
+	 * If an OpenCAPI card is connected to GPU#0, and an NVLink GPU is
+	 * connected to GPU#1, the GPU will only receive 2 links rather than the
+	 * usual 3.
+	 *
+	 * The reason for this is that without the OpenCAPI card, the GPU would
+	 * be use links 3-5, connected to NPU bricks 3-5, which needs both
+	 * stacks 1 and 2 to be in NVLink mode.
+	 *
+	 * However, with an OpenCAPI card in the GPU#0 slot that uses links 0-1,
+	 * we need to use NPU bricks 2-3, which means stack 1 must be set in
+	 * OpenCAPI mode. As such, the GPU will be restricted to using links 4
+	 * and 5.
+	 */
+	if (gpu1_type == NPU2_DEV_TYPE_NVLINK) {
+		if (gpu0_type == NPU2_DEV_TYPE_OPENCAPI) {
+			prlog(PR_WARNING, "PLAT: Chip %d GPU#1 will operate at reduced performance due to presence of OpenCAPI device. For optimal performance, swap device locations\n", chip->id);
+		} else {
 			set_link_details(npu, 3, 3, NPU2_DEV_TYPE_NVLINK);
-			set_link_details(npu, 4, 4, NPU2_DEV_TYPE_NVLINK);
-			set_link_details(npu, 5, 5, NPU2_DEV_TYPE_NVLINK);
 		}
+		set_link_details(npu, 4, 4, NPU2_DEV_TYPE_NVLINK);
+		set_link_details(npu, 5, 5, NPU2_DEV_TYPE_NVLINK);
 	}
 
 	return;
-- 
git-series 0.9.1