[PATCH 04/14] powerpc/64s: avoid reloading (H)SRR registers if they are still valid

[Nicholas Piggin]

Excerpts from Michael Ellerman's message of April 3, 2021 8:39 am:
> Nicholas Piggin <npiggin at gmail.com> writes:
>> When an interrupt is taken, the SRR registers are set to return to
>> where it left off. Unless they are modified in the meantime, or the
>> return address or MSR are modified, there is no need to reload these
>> registers when returning from interrupt.
>>
>> Introduce per-CPU flags that track the validity of SRR and HSRR
>> registers, clear them when returning from interrupt, using the registers
>> for something else (e.g., OPAL calls), or adjusting return address or MSR.
>>
>> This improves the performance of interrupt returns.
>>
>> XXX: may not need to invalidate both hsrr and srr all the time
>>
>> Signed-off-by: Nicholas Piggin <npiggin at gmail.com>
>> ---
> 
> I needed something like below to get 32-bit building.

That looks much better.

Thanks,
Nick

> 
> cheers
> 
> 
> diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h
> index 6d6237e0cbd7..7f9bbd19db10 100644
> --- a/arch/powerpc/include/asm/ptrace.h
> +++ b/arch/powerpc/include/asm/ptrace.h
> @@ -153,15 +153,21 @@ static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
>  	regs->gpr[3] = rc;
>  }
>  
> -static inline void regs_set_return_ip(struct pt_regs *regs, unsigned long ip)
> +static inline void invalidate_srrs(void)
>  {
> -	regs->nip = ip;
>  #ifdef CONFIG_PPC_BOOK3S_64
> +	// XXX: We may not need to invalidate both hsrr and srr all the time
>  	local_paca->hsrr_valid = 0;
>  	local_paca->srr_valid = 0;
>  #endif
>  }
>  
> +static inline void regs_set_return_ip(struct pt_regs *regs, unsigned long ip)
> +{
> +	regs->nip = ip;
> +	invalidate_srrs();
> +}
> +
>  static inline void regs_add_return_ip(struct pt_regs *regs, long offset)
>  {
>  	regs_set_return_ip(regs, regs->nip + offset);
> diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
> index 200b4805f999..82623b57e2d6 100644
> --- a/arch/powerpc/kernel/process.c
> +++ b/arch/powerpc/kernel/process.c
> @@ -98,8 +98,7 @@ static void check_if_tm_restore_required(struct task_struct *tsk)
>  	    !test_thread_flag(TIF_RESTORE_TM)) {
>  		tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr;
>  		set_thread_flag(TIF_RESTORE_TM);
> -		local_paca->hsrr_valid = 0;
> -		local_paca->srr_valid = 0;
> +		invalidate_srrs();
>  	}
>  }
>  
> @@ -164,8 +163,7 @@ static void __giveup_fpu(struct task_struct *tsk)
>  	if (cpu_has_feature(CPU_FTR_VSX))
>  		msr &= ~MSR_VSX;
>  	tsk->thread.regs->msr = msr;
> -	local_paca->hsrr_valid = 0;
> -	local_paca->srr_valid = 0;
> +	invalidate_srrs();
>  }
>  
>  void giveup_fpu(struct task_struct *tsk)
> @@ -249,8 +247,7 @@ static void __giveup_altivec(struct task_struct *tsk)
>  	if (cpu_has_feature(CPU_FTR_VSX))
>  		msr &= ~MSR_VSX;
>  	tsk->thread.regs->msr = msr;
> -	local_paca->hsrr_valid = 0;
> -	local_paca->srr_valid = 0;
> +	invalidate_srrs();
>  }
>  
>  void giveup_altivec(struct task_struct *tsk)
> @@ -566,8 +563,7 @@ void notrace restore_math(struct pt_regs *regs)
>  		msr_check_and_clear(new_msr);
>  
>  		regs->msr |= new_msr | fpexc_mode;
> -		local_paca->hsrr_valid = 0;
> -		local_paca->srr_valid = 0;
> +		invalidate_srrs();
>  	}
>  }
>  #endif /* CONFIG_PPC_BOOK3S_64 */
> @@ -1293,8 +1289,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
>  			atomic_read(&current->mm->context.vas_windows)))
>  			asm volatile(PPC_CP_ABORT);
>  	}
> -	local_paca->hsrr_valid = 0;
> -	local_paca->srr_valid = 0;
> +	invalidate_srrs();
>  #endif /* CONFIG_PPC_BOOK3S_64 */
>  
>  	return last;
> @@ -1884,8 +1879,7 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
>  	current->thread.load_tm = 0;
>  #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
>  
> -	local_paca->hsrr_valid = 0;
> -	local_paca->srr_valid = 0;
> +	invalidate_srrs();
>  }
>  EXPORT_SYMBOL(start_thread);
>  
> @@ -1936,8 +1930,7 @@ int set_fpexc_mode(struct task_struct *tsk, unsigned int val)
>  	if (regs != NULL && (regs->msr & MSR_FP) != 0) {
>  		regs->msr = (regs->msr & ~(MSR_FE0|MSR_FE1))
>  			| tsk->thread.fpexc_mode;
> -		local_paca->hsrr_valid = 0;
> -		local_paca->srr_valid = 0;
> +		invalidate_srrs();
>  	}
>  	return 0;
>  }
> @@ -1990,8 +1983,7 @@ int set_endian(struct task_struct *tsk, unsigned int val)
>  	else
>  		return -EINVAL;
>  
> -	local_paca->hsrr_valid = 0;
> -	local_paca->srr_valid = 0;
> +	invalidate_srrs();
>  
>  	return 0;
>  }
> diff --git a/arch/powerpc/kernel/syscalls.c b/arch/powerpc/kernel/syscalls.c
> index 4cb38afa28a8..9d1d6070a516 100644
> --- a/arch/powerpc/kernel/syscalls.c
> +++ b/arch/powerpc/kernel/syscalls.c
> @@ -115,8 +115,8 @@ SYSCALL_DEFINE0(switch_endian)
>  	struct thread_info *ti;
>  
>  	current->thread.regs->msr ^= MSR_LE;
> -	local_paca->hsrr_valid = 0;
> -	local_paca->srr_valid = 0;
> +
> +	invalidate_srrs();
>  
>  	/*
>  	 * Set TIF_RESTOREALL so that r3 isn't clobbered on return to
> diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
> index 96505d4bba1c..2b94bf21d6ae 100644
> --- a/arch/powerpc/lib/sstep.c
> +++ b/arch/powerpc/lib/sstep.c
> @@ -3480,8 +3480,7 @@ int emulate_step(struct pt_regs *regs, struct ppc_inst instr)
>  	unsigned long val;
>  	unsigned long ea;
>  
> -	local_paca->hsrr_valid = 0;
> -	local_paca->srr_valid = 0;
> +	invalidate_srrs();
>  
>  	r = analyse_instr(&op, regs, instr);
>  	if (r < 0)
>