[PATCH v2 2/2] powerpc/32s: Save content of sr0 to avoid 'mfsr'
Christophe Leroy
christophe.leroy at csgroup.eu
Fri Jul 9 23:16:15 AEST 2021
Calling 'mfsr' to get the content of segment registers is heavy,
in addition it requires clearing of the 'reserved' bits.
In order to avoid this operation, save it in mm context and in
thread struct.
The saved sr0 is the one used by kernel, this means that on
interrupt/syscall entry it can be used as is.
In interrupt/syscall exit, the only thing to do is to clear SR_NX.
This improves null_syscall selftest by 12 cycles, ie 4%.
Capability to deactive KUEP at boot time is re-enabled by this patch.
Signed-off-by: Christophe Leroy <christophe.leroy at csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 3 ++
arch/powerpc/include/asm/book3s/32/mmu-hash.h | 1 +
arch/powerpc/include/asm/processor.h | 1 +
arch/powerpc/kernel/entry_32.S | 24 ++++++++-------
arch/powerpc/mm/book3s32/kuap.c | 5 +++-
arch/powerpc/mm/book3s32/kuep.c | 30 ++++++++++++++-----
arch/powerpc/mm/book3s32/mmu_context.c | 15 +++++-----
arch/powerpc/mm/mmu_context.c | 3 ++
8 files changed, 55 insertions(+), 27 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 2e0e87cf7d7a..05659fd01557 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -12,6 +12,9 @@
extern struct static_key_false disable_kuap_key;
extern struct static_key_false disable_kuep_key;
+extern s32 patch__kuep_interrupt_entry, patch__kuep_interrupt_exit;
+extern s32 patch__kuep_syscall_entry, patch__kuep_syscall_exit;
+
static __always_inline bool kuap_is_disabled(void)
{
return !IS_ENABLED(CONFIG_PPC_KUAP) || static_branch_unlikely(&disable_kuap_key);
diff --git a/arch/powerpc/include/asm/book3s/32/mmu-hash.h b/arch/powerpc/include/asm/book3s/32/mmu-hash.h
index e6c90802de03..fa613693949d 100644
--- a/arch/powerpc/include/asm/book3s/32/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/32/mmu-hash.h
@@ -196,6 +196,7 @@ struct hash_pte {
typedef struct {
unsigned long id;
+ unsigned long sr0;
void __user *vdso;
} mm_context_t;
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index f348e564f7dd..4b13f94a4f42 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -157,6 +157,7 @@ struct thread_struct {
#ifdef CONFIG_PPC_BOOK3S_32
unsigned long r0, r3, r4, r5, r6, r8, r9, r11;
unsigned long lr, ctr;
+ unsigned long sr0;
#endif
#endif /* CONFIG_PPC32 */
/* Debug Registers */
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index e382c7fc952e..773ce90df360 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -33,6 +33,7 @@
#include <asm/kup.h>
#include <asm/bug.h>
#include <asm/interrupt.h>
+#include <asm/code-patching-asm.h>
#include "head_32.h"
@@ -55,11 +56,10 @@ prepare_transfer_to_handler:
#ifdef CONFIG_PPC_KUEP
beq 1f
- mfsr r4,0
- rlwinm r4,r4,0,8,3
- oris r4,r4,SR_NX at h
+0: blr /* lwz r4, current->thread.sr0(r2) */
update_user_segments_by_6 r4, r5, r6, r7, r8, r9
blr
+ patch_site 0b, patch__kuep_interrupt_entry
1:
#endif
/* if from kernel, check interrupted DOZE/NAP mode */
@@ -104,10 +104,10 @@ transfer_to_syscall:
SAVE_2GPRS(7, r1)
addi r2,r10,-THREAD
#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
- mfsr r9,0
- rlwinm r9,r9,0,8,3
- oris r9,r9,SR_NX at h
+0: b 1f /* lwz r9, &thread.sr0(r10) */
update_user_segments_by_4 r9, r10, r11, r12
+ patch_site 0b, patch__kuep_syscall_entry
+1:
#endif
SAVE_NVGPRS(r1)
@@ -127,9 +127,11 @@ ret_from_syscall:
bne- 2f
#endif /* CONFIG_PPC_47x */
#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
- mfsr r7,0
- rlwinm r7,r7,0,8,2
+0: b 1f /* lwz r7, current->thread.sr0(r2) */
+ rlwinm r7,r7,0,~SR_NX
update_user_segments_by_6 r7, r8, r9, r10, r11, r12
+ patch_site 0b, patch__kuep_syscall_exit
+1:
#endif
lwz r4,_LINK(r1)
lwz r5,_CCR(r1)
@@ -295,9 +297,11 @@ interrupt_return:
bl interrupt_exit_user_prepare
cmpwi r3,0
#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
- mfsr r7,0
- rlwinm r7,r7,0,8,2
+0: b 1f /* lwz r7, current->thread.sr0(r2) */
+ rlwinm r7,r7,0,~SR_NX
update_user_segments_by_6 r7, r8, r9, r10, r11, r12
+ patch_site 0b, patch__kuep_interrupt_exit
+1:
#endif
bne- .Lrestore_nvgprs
diff --git a/arch/powerpc/mm/book3s32/kuap.c b/arch/powerpc/mm/book3s32/kuap.c
index 0f920f09af57..28676cabb005 100644
--- a/arch/powerpc/mm/book3s32/kuap.c
+++ b/arch/powerpc/mm/book3s32/kuap.c
@@ -20,8 +20,11 @@ EXPORT_SYMBOL(kuap_unlock_all_ool);
void setup_kuap(bool disabled)
{
- if (!disabled)
+ if (!disabled) {
kuap_lock_all_ool();
+ init_mm.context.sr0 |= SR_KS;
+ current->thread.sr0 |= SR_KS;
+ }
if (smp_processor_id() != boot_cpuid)
return;
diff --git a/arch/powerpc/mm/book3s32/kuep.c b/arch/powerpc/mm/book3s32/kuep.c
index 45c9967f9aef..409d170bc52f 100644
--- a/arch/powerpc/mm/book3s32/kuep.c
+++ b/arch/powerpc/mm/book3s32/kuep.c
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <asm/code-patching.h>
#include <asm/kup.h>
#include <asm/smp.h>
@@ -7,19 +8,32 @@ struct static_key_false disable_kuep_key;
void setup_kuep(bool disabled)
{
- if (disabled) {
- pr_info("KUEP cannot be disabled for the time being\n");
- disabled = false;
- }
+ u32 insn;
- if (!disabled)
- update_user_segments(mfsr(0) | SR_NX);
+ if (!disabled) {
+ init_mm.context.sr0 |= SR_NX;
+ current->thread.sr0 |= SR_NX;
+ update_user_segments(init_mm.context.sr0);
+ }
if (smp_processor_id() != boot_cpuid)
return;
if (disabled)
static_branch_enable(&disable_kuep_key);
- else
- pr_info("Activating Kernel Userspace Execution Prevention\n");
+
+ if (disabled)
+ return;
+
+ insn = PPC_RAW_LWZ(_R4, _R2, offsetof(struct task_struct, thread.sr0));
+ patch_instruction_site(&patch__kuep_interrupt_entry, ppc_inst(insn));
+
+ insn = PPC_RAW_LWZ(_R9, _R10, offsetof(struct thread_struct, sr0));
+ patch_instruction_site(&patch__kuep_syscall_entry, ppc_inst(insn));
+
+ insn = PPC_RAW_LWZ(_R7, _R2, offsetof(struct task_struct, thread.sr0));
+ patch_instruction_site(&patch__kuep_syscall_exit, ppc_inst(insn));
+ patch_instruction_site(&patch__kuep_interrupt_exit, ppc_inst(insn));
+
+ pr_info("Activating Kernel Userspace Execution Prevention\n");
}
diff --git a/arch/powerpc/mm/book3s32/mmu_context.c b/arch/powerpc/mm/book3s32/mmu_context.c
index e2708e387dc3..269a3eb25a73 100644
--- a/arch/powerpc/mm/book3s32/mmu_context.c
+++ b/arch/powerpc/mm/book3s32/mmu_context.c
@@ -69,6 +69,12 @@ EXPORT_SYMBOL_GPL(__init_new_context);
int init_new_context(struct task_struct *t, struct mm_struct *mm)
{
mm->context.id = __init_new_context();
+ mm->context.sr0 = CTX_TO_VSID(mm->context.id, 0);
+
+ if (!kuep_is_disabled())
+ mm->context.sr0 |= SR_NX;
+ if (!kuap_is_disabled())
+ mm->context.sr0 |= SR_KS;
return 0;
}
@@ -108,20 +114,13 @@ void __init mmu_context_init(void)
void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk)
{
long id = next->context.id;
- unsigned long val;
if (id < 0)
panic("mm_struct %p has no context ID", next);
isync();
- val = CTX_TO_VSID(id, 0);
- if (!kuep_is_disabled())
- val |= SR_NX;
- if (!kuap_is_disabled())
- val |= SR_KS;
-
- update_user_segments(val);
+ update_user_segments(next->context.sr0);
if (IS_ENABLED(CONFIG_BDI_SWITCH))
abatron_pteptrs[1] = next->pgd;
diff --git a/arch/powerpc/mm/mmu_context.c b/arch/powerpc/mm/mmu_context.c
index 74246536b832..e618d5442a28 100644
--- a/arch/powerpc/mm/mmu_context.c
+++ b/arch/powerpc/mm/mmu_context.c
@@ -18,6 +18,9 @@ static inline void switch_mm_pgdir(struct task_struct *tsk,
{
/* 32-bit keeps track of the current PGDIR in the thread struct */
tsk->thread.pgdir = mm->pgd;
+#ifdef CONFIG_PPC_BOOK3S_32
+ tsk->thread.sr0 = mm->context.sr0;
+#endif
}
#elif defined(CONFIG_PPC_BOOK3E_64)
static inline void switch_mm_pgdir(struct task_struct *tsk,
--
2.25.0
More information about the Linuxppc-dev
mailing list