[RFC PATCH 14/14] powerpc/qspinlock: use spin_begin/end API

[Nicholas Piggin]

Use the spin_begin/spin_cpu_relax/spin_end APIs in qspinlock, which helps
to prevent threads issuing a lot of expensive priority nops which may not
have much effect due to immediately executing low then medium priority.
---
 arch/powerpc/lib/qspinlock.c | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c
index d67b923e4f98..486423d566d3 100644
--- a/arch/powerpc/lib/qspinlock.c
+++ b/arch/powerpc/lib/qspinlock.c
@@ -243,7 +243,7 @@ static void __yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravi
 		return;
 	}
 relax:
-	cpu_relax();
+	spin_cpu_relax();
 }
 
 static void yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt)
@@ -331,7 +331,7 @@ static void yield_to_prev(struct qspinlock *lock, struct qnode *node, int prev_c
 	}
 
 relax:
-	cpu_relax();
+	spin_cpu_relax();
 }
 
 
@@ -340,6 +340,7 @@ static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool parav
 	int iters;
 
 	/* Attempt to steal the lock */
+	spin_begin();
 	for (iters = 0; iters < STEAL_SPINS; iters++) {
 		u32 val = READ_ONCE(lock->val);
 
@@ -354,6 +355,7 @@ static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool parav
 		if (trylock_with_tail_cpu(lock, val))
 			return true;
 	}
+	spin_end();
 
 	return false;
 }
@@ -404,8 +406,10 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 		WRITE_ONCE(prev->next, node);
 
 		/* Wait for mcs node lock to be released */
+		spin_begin();
 		while (!node->locked)
 			yield_to_prev(lock, node, prev_cpu, paravirt);
+		spin_end();
 
 		/* Clear out stale propagated yield_cpu */
 		if (paravirt && pv_yield_propagate_owner && node->yield_cpu != -1)
@@ -418,10 +422,12 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 		int set_yield_cpu = -1;
 
 		/* We're at the head of the waitqueue, wait for the lock. */
+		spin_begin();
 		while ((val = READ_ONCE(lock->val)) & _Q_LOCKED_VAL) {
 			propagate_yield_cpu(node, val, &set_yield_cpu, paravirt);
 			yield_head_to_locked_owner(lock, val, paravirt, false);
 		}
+		spin_end();
 
 		/* If we're the last queued, must clean up the tail. */
 		if ((val & _Q_TAIL_CPU_MASK) == tail) {
@@ -436,6 +442,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 		int set_yield_cpu = -1;
 		int iters = 0;
 again:
+		spin_begin();
 		/* We're at the head of the waitqueue, wait for the lock. */
 		while ((val = READ_ONCE(lock->val)) & _Q_LOCKED_VAL) {
 			if (iters++ == HEAD_SPINS) {
@@ -446,6 +453,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 			yield_head_to_locked_owner(lock, val, paravirt,
 				pv_yield_allow_steal && (iters > HEAD_SPINS));
 		}
+		spin_end();
 
 		/* If we're the last queued, must clean up the tail. */
 		if ((val & _Q_TAIL_CPU_MASK) == tail) {
@@ -461,8 +469,13 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 
 unlock_next:
 	/* contended path; must wait for next != NULL (MCS protocol) */
-	while (!(next = READ_ONCE(node->next)))
-		cpu_relax();
+	next = READ_ONCE(node->next);
+	if (!next) {
+		spin_begin();
+		while (!(next = READ_ONCE(node->next)))
+			cpu_relax();
+		spin_end();
+	}
 
 	/*
 	 * Unlock the next mcs waiter node. Release barrier is not required
-- 
2.35.1