[kvm-unit-tests PATCH v9 02/31] report: Add known failure reporting option

[Nicholas Piggin]

There are times we would like to test a function that is known to fail
in some conditions due to a bug in implementation (QEMU, KVM, or even
hardware). It would be nice to count these as known failures and not
report a summary failure.

xfail is not the same thing, xfail means failure is required and a pass
causes the test to fail. So add kfail for known failures.

Mark the failing ppc64 h_cede_tm and spapr_vpa tests as kfail.

Signed-off-by: Nicholas Piggin <npiggin at gmail.com>
---
 lib/libcflat.h |  2 ++
 lib/report.c   | 33 +++++++++++++++++++++++++--------
 2 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/lib/libcflat.h b/lib/libcflat.h
index 700f43527..ae3c2c6d0 100644
--- a/lib/libcflat.h
+++ b/lib/libcflat.h
@@ -100,6 +100,8 @@ extern void report(bool pass, const char *msg_fmt, ...)
 		__attribute__((format(printf, 2, 3), nonnull(2)));
 extern void report_xfail(bool xfail, bool pass, const char *msg_fmt, ...)
 		__attribute__((format(printf, 3, 4), nonnull(3)));
+extern void report_kfail(bool kfail, bool pass, const char *msg_fmt, ...)
+		__attribute__((format(printf, 3, 4), nonnull(3)));
 extern void report_abort(const char *msg_fmt, ...)
 					__attribute__((format(printf, 1, 2)))
 					__attribute__((noreturn));
diff --git a/lib/report.c b/lib/report.c
index 8e9bff5b8..7f3c4f059 100644
--- a/lib/report.c
+++ b/lib/report.c
@@ -13,7 +13,7 @@
 #include "libcflat.h"
 #include "asm/spinlock.h"
 
-static unsigned int tests, failures, xfailures, skipped;
+static unsigned int tests, failures, xfailures, kfailures, skipped;
 static char prefixes[256];
 static struct spinlock lock;
 
@@ -81,11 +81,12 @@ void report_prefix_pop(void)
 }
 
 static void va_report(const char *msg_fmt,
-		bool pass, bool xfail, bool skip, va_list va)
+		bool pass, bool xfail, bool kfail, bool skip, va_list va)
 {
 	const char *prefix = skip ? "SKIP"
 				  : xfail ? (pass ? "XPASS" : "XFAIL")
-					  : (pass ? "PASS"  : "FAIL");
+				          : kfail ? (pass ? "PASS" : "KFAIL")
+					          : (pass ? "PASS"  : "FAIL");
 
 	spin_lock(&lock);
 
@@ -98,6 +99,8 @@ static void va_report(const char *msg_fmt,
 		skipped++;
 	else if (xfail && !pass)
 		xfailures++;
+	else if (kfail && !pass)
+		kfailures++;
 	else if (xfail || !pass)
 		failures++;
 
@@ -108,7 +111,7 @@ void report(bool pass, const char *msg_fmt, ...)
 {
 	va_list va;
 	va_start(va, msg_fmt);
-	va_report(msg_fmt, pass, false, false, va);
+	va_report(msg_fmt, pass, false, false, false, va);
 	va_end(va);
 }
 
@@ -117,7 +120,7 @@ void report_pass(const char *msg_fmt, ...)
 	va_list va;
 
 	va_start(va, msg_fmt);
-	va_report(msg_fmt, true, false, false, va);
+	va_report(msg_fmt, true, false, false, false, va);
 	va_end(va);
 }
 
@@ -126,7 +129,7 @@ void report_fail(const char *msg_fmt, ...)
 	va_list va;
 
 	va_start(va, msg_fmt);
-	va_report(msg_fmt, false, false, false, va);
+	va_report(msg_fmt, false, false, false, false, va);
 	va_end(va);
 }
 
@@ -134,7 +137,19 @@ void report_xfail(bool xfail, bool pass, const char *msg_fmt, ...)
 {
 	va_list va;
 	va_start(va, msg_fmt);
-	va_report(msg_fmt, pass, xfail, false, va);
+	va_report(msg_fmt, pass, xfail, false, false, va);
+	va_end(va);
+}
+
+/*
+ * kfail is known failure. If kfail is true then test will succeed
+ * regardless of pass.
+ */
+void report_kfail(bool kfail, bool pass, const char *msg_fmt, ...)
+{
+	va_list va;
+	va_start(va, msg_fmt);
+	va_report(msg_fmt, pass, false, kfail, false, va);
 	va_end(va);
 }
 
@@ -142,7 +157,7 @@ void report_skip(const char *msg_fmt, ...)
 {
 	va_list va;
 	va_start(va, msg_fmt);
-	va_report(msg_fmt, false, false, true, va);
+	va_report(msg_fmt, false, false, false, true, va);
 	va_end(va);
 }
 
@@ -168,6 +183,8 @@ int report_summary(void)
 	printf("SUMMARY: %d tests", tests);
 	if (failures)
 		printf(", %d unexpected failures", failures);
+	if (kfailures)
+		printf(", %d known failures", kfailures);
 	if (xfailures)
 		printf(", %d expected failures", xfailures);
 	if (skipped)
-- 
2.43.0