TechPanelGM/kernel_xiaomi_sm8250
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ARCv2: entry: rewrite to enable use of double load/stores LDD/STD

Browse Source 
[ Upstream commit a4880801a72ecc2dcdfa432f81a754f3e7438567 ]

 - the motivation was to be remove blatent copy-paste due to hasty support
   of CONFIG_ARC_IRQ_NO_AUTOSAVE support

 - but with refactoring we could use LDD/STD to greatly optimize the code

Signed-off-by: Vineet Gupta <vgupta@synopsys.com>
Stable-dep-of: 92e2921eeafd ("ARC: define ASM_NL and __ALIGN(_STR) outside #ifdef __ASSEMBLY__ guard")
Signed-off-by: Sasha Levin <sashal@kernel.org>
This commit is contained in:
Vineet Gupta
2019-05-15 15:36:46 -07:00
committed by Greg Kroah-Hartman
parent baea6f9bd6
commit 4ac3a2220d
4 changed files with 191 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

345
arch/arc/include/asm/entry-arcv2.h
View File

@@ -46,7 +46,8 @@
*/ */
/*------------------------------------------------------------------------*/ /*------------------------------------------------------------------------*/
.macro INTERRUPT_PROLOGUE called_from .macro INTERRUPT_PROLOGUE
; (A) Before jumping to Interrupt Vector, hardware micro-ops did following: ; (A) Before jumping to Interrupt Vector, hardware micro-ops did following:
; 1. SP auto-switched to kernel mode stack ; 1. SP auto-switched to kernel mode stack
; 2. STATUS32.Z flag set if in U mode at time of interrupt (U:1,K:0) ; 2. STATUS32.Z flag set if in U mode at time of interrupt (U:1,K:0)
@@ -57,133 +58,16 @@
; (B) Manually saved some regs: r12,r25,r30, sp,fp,gp, ACCL pair ; (B) Manually saved some regs: r12,r25,r30, sp,fp,gp, ACCL pair
#ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE #ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE
.ifnc \called_from, exception ; carve pt_regs on stack (case #3), PC/STAT32 already on stack
st.as r9, [sp, -10] ; save r9 in it's final stack slot sub sp, sp, SZ_PT_REGS - 8
sub sp, sp, 12 ; skip JLI, LDI, EI
PUSH lp_count __SAVE_REGFILE_HARD
PUSHAX lp_start
PUSHAX lp_end
PUSH blink
PUSH r11
PUSH r10
sub sp, sp, 4 ; skip r9
PUSH r8
PUSH r7
PUSH r6
PUSH r5
PUSH r4
PUSH r3
PUSH r2
PUSH r1
PUSH r0
.endif
#endif
#ifdef CONFIG_ARC_HAS_ACCL_REGS
PUSH r59
PUSH r58
#endif
PUSH r30
PUSH r12
; Saving pt_regs->sp correctly requires some extra work due to the way
; Auto stack switch works
; - U mode: retrieve it from AUX_USER_SP
; - K mode: add the offset from current SP where H/w starts auto push
;
; 1. Utilize the fact that Z bit is set if Intr taken in U mode
; 2. Upon entry SP is always saved (for any inspection, unwinding etc),
; but on return, restored only if U mode
lr r9, [AUX_USER_SP] ; U mode SP
mov.nz r9, sp
add.nz r9, r9, SZ_PT_REGS - PT_sp - 4 ; K mode SP
PUSH r9 ; SP (pt_regs->sp)
PUSH fp
PUSH gp
#ifdef CONFIG_ARC_CURR_IN_REG
PUSH r25 ; user_r25
GET_CURR_TASK_ON_CPU r25
#else #else
sub sp, sp, 4 ; carve pt_regs on stack (case #4), which grew partially already
#endif sub sp, sp, PT_r0
.ifnc \called_from, exception
sub sp, sp, 12 ; BTA/ECR/orig_r0 placeholder per pt_regs
.endif
.endm
/*------------------------------------------------------------------------*/
.macro INTERRUPT_EPILOGUE called_from
; INPUT: r0 has STAT32 of calling context
; INPUT: Z flag set if returning to K mode
.ifnc \called_from, exception
add sp, sp, 12 ; skip BTA/ECR/orig_r0 placeholderss
.endif
#ifdef CONFIG_ARC_CURR_IN_REG
POP r25
#else
add sp, sp, 4
#endif
POP gp
POP fp
; Restore SP (into AUX_USER_SP) only if returning to U mode
; - for K mode, it will be implicitly restored as stack is unwound
; - Z flag set on K is inverse of what hardware does on interrupt entry
; but that doesn't really matter
bz 1f
POPAX AUX_USER_SP
1:
POP r12
POP r30
#ifdef CONFIG_ARC_HAS_ACCL_REGS
POP r58
POP r59
#endif
#ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE
.ifnc \called_from, exception
POP r0
POP r1
POP r2
POP r3
POP r4
POP r5
POP r6
POP r7
POP r8
POP r9
POP r10
POP r11
POP blink
POPAX lp_end
POPAX lp_start
POP r9
mov lp_count, r9
add sp, sp, 12 ; skip JLI, LDI, EI
ld.as r9, [sp, -10] ; reload r9 which got clobbered
.endif
#endif #endif
__SAVE_REGFILE_SOFT
.endm .endm
/*------------------------------------------------------------------------*/ /*------------------------------------------------------------------------*/
@@ -195,81 +79,180 @@
; ;
; (B) Manually save the complete reg file below ; (B) Manually save the complete reg file below
PUSH r9 ; freeup a register: slot of erstatus sub sp, sp, SZ_PT_REGS ; carve pt_regs
PUSHAX eret ; _HARD saves r10 clobbered by _SOFT as scratch hence comes first
sub sp, sp, 12 ; skip JLI, LDI, EI
PUSH lp_count
PUSHAX lp_start
PUSHAX lp_end
PUSH blink
PUSH r11 __SAVE_REGFILE_HARD
PUSH r10 __SAVE_REGFILE_SOFT
ld.as r9, [sp, 10] ; load stashed r9 (status32 stack slot) st r0, [sp] ; orig_r0
lr r10, [erstatus]
st.as r10, [sp, 10] ; save status32 at it's right stack slot
PUSH r9 lr r10, [eret]
PUSH r8 lr r11, [erstatus]
PUSH r7 ST2 r10, r11, PT_ret
PUSH r6
PUSH r5
PUSH r4
PUSH r3
PUSH r2
PUSH r1
PUSH r0
; -- for interrupts, regs above are auto-saved by h/w in that order -- lr r10, [ecr]
; Now do what ISR prologue does (manually save r12, sp, fp, gp, r25) lr r11, [erbta]
ST2 r10, r11, PT_event
mov r9, r10
INTERRUPT_PROLOGUE exception
PUSHAX erbta
PUSHAX ecr ; r9 contains ECR, expected by EV_Trap
PUSH r0 ; orig_r0
; OUTPUT: r9 has ECR ; OUTPUT: r9 has ECR
.endm .endm
/*------------------------------------------------------------------------
* This macro saves the registers manually which would normally be autosaved
* by hardware on taken interrupts. It is used by
* - exception handlers (which don't have autosave)
* - interrupt autosave disabled due to CONFIG_ARC_IRQ_NO_AUTOSAVE
*/
.macro __SAVE_REGFILE_HARD
ST2 r0, r1, PT_r0
ST2 r2, r3, PT_r2
ST2 r4, r5, PT_r4
ST2 r6, r7, PT_r6
ST2 r8, r9, PT_r8
ST2 r10, r11, PT_r10
st blink, [sp, PT_blink]
lr r10, [lp_end]
lr r11, [lp_start]
ST2 r10, r11, PT_lpe
st lp_count, [sp, PT_lpc]
; skip JLI, LDI, EI for now
.endm
/*------------------------------------------------------------------------
* This macros saves a bunch of other registers which can't be autosaved for
* various reasons:
* - r12: the last caller saved scratch reg since hardware saves in pairs so r0-r11
* - r30: free reg, used by gcc as scratch
* - ACCL/ACCH pair when they exist
*/
.macro __SAVE_REGFILE_SOFT
ST2 gp, fp, PT_r26 ; gp (r26), fp (r27)
st r12, [sp, PT_sp + 4]
st r30, [sp, PT_sp + 8]
; Saving pt_regs->sp correctly requires some extra work due to the way
; Auto stack switch works
; - U mode: retrieve it from AUX_USER_SP
; - K mode: add the offset from current SP where H/w starts auto push
;
; 1. Utilize the fact that Z bit is set if Intr taken in U mode
; 2. Upon entry SP is always saved (for any inspection, unwinding etc),
; but on return, restored only if U mode
lr r10, [AUX_USER_SP] ; U mode SP
; ISA requires ADD.nz to have same dest and src reg operands
mov.nz r10, sp
add.nz r10, r10, SZ_PT_REGS ; K mode SP
st r10, [sp, PT_sp] ; SP (pt_regs->sp)
#ifdef CONFIG_ARC_CURR_IN_REG
st r25, [sp, PT_user_r25]
GET_CURR_TASK_ON_CPU r25
#endif
#ifdef CONFIG_ARC_HAS_ACCL_REGS
ST2 r58, r59, PT_sp + 12
#endif
.endm
/*------------------------------------------------------------------------*/
.macro __RESTORE_REGFILE_SOFT
LD2 gp, fp, PT_r26 ; gp (r26), fp (r27)
ld r12, [sp, PT_sp + 4]
ld r30, [sp, PT_sp + 8]
; Restore SP (into AUX_USER_SP) only if returning to U mode
; - for K mode, it will be implicitly restored as stack is unwound
; - Z flag set on K is inverse of what hardware does on interrupt entry
; but that doesn't really matter
bz 1f
ld r10, [sp, PT_sp] ; SP (pt_regs->sp)
sr r10, [AUX_USER_SP]
1:
#ifdef CONFIG_ARC_CURR_IN_REG
ld r25, [sp, PT_user_r25]
#endif
#ifdef CONFIG_ARC_HAS_ACCL_REGS
LD2 r58, r59, PT_sp + 12
#endif
.endm
/*------------------------------------------------------------------------*/
.macro __RESTORE_REGFILE_HARD
ld blink, [sp, PT_blink]
LD2 r10, r11, PT_lpe
sr r10, [lp_end]
sr r11, [lp_start]
ld r10, [sp, PT_lpc] ; lp_count can't be target of LD
mov lp_count, r10
LD2 r0, r1, PT_r0
LD2 r2, r3, PT_r2
LD2 r4, r5, PT_r4
LD2 r6, r7, PT_r6
LD2 r8, r9, PT_r8
LD2 r10, r11, PT_r10
.endm
/*------------------------------------------------------------------------*/
.macro INTERRUPT_EPILOGUE
; INPUT: r0 has STAT32 of calling context
; INPUT: Z flag set if returning to K mode
; _SOFT clobbers r10 restored by _HARD hence the order
__RESTORE_REGFILE_SOFT
#ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE
__RESTORE_REGFILE_HARD
add sp, sp, SZ_PT_REGS - 8
#else
add sp, sp, PT_r0
#endif
.endm
/*------------------------------------------------------------------------*/ /*------------------------------------------------------------------------*/
.macro EXCEPTION_EPILOGUE .macro EXCEPTION_EPILOGUE
; INPUT: r0 has STAT32 of calling context ; INPUT: r0 has STAT32 of calling context
btst r0, STATUS_U_BIT ; Z flag set if K, used in INTERRUPT_EPILOGUE
add sp, sp, 8 ; orig_r0/ECR don't need restoring btst r0, STATUS_U_BIT ; Z flag set if K, used in restoring SP
POPAX erbta
INTERRUPT_EPILOGUE exception ld r10, [sp, PT_event + 4]
sr r10, [erbta]
POP r0 LD2 r10, r11, PT_ret
POP r1 sr r10, [eret]
POP r2 sr r11, [erstatus]
POP r3
POP r4
POP r5
POP r6
POP r7
POP r8
POP r9
POP r10
POP r11
POP blink __RESTORE_REGFILE_SOFT
POPAX lp_end __RESTORE_REGFILE_HARD
POPAX lp_start
POP r9 add sp, sp, SZ_PT_REGS
mov lp_count, r9
add sp, sp, 12 ; skip JLI, LDI, EI
POPAX eret
POPAX erstatus
ld.as r9, [sp, -12] ; reload r9 which got clobbered
.endm .endm
.macro FAKE_RET_FROM_EXCPN .macro FAKE_RET_FROM_EXCPN

18
arch/arc/include/asm/linkage.h
View File

@@ -13,6 +13,24 @@
#ifdef __ASSEMBLY__ #ifdef __ASSEMBLY__
.macro ST2 e, o, off
#ifdef CONFIG_ARC_HAS_LL64
std \e, [sp, \off]
#else
st \e, [sp, \off]
st \o, [sp, \off+4]
#endif
.endm
.macro LD2 e, o, off
#ifdef CONFIG_ARC_HAS_LL64
ldd \e, [sp, \off]
#else
ld \e, [sp, \off]
ld \o, [sp, \off+4]
#endif
.endm
#define ASM_NL ` /* use '`' to mark new line in macro */ #define ASM_NL ` /* use '`' to mark new line in macro */
#define __ALIGN .align 4 #define __ALIGN .align 4
#define __ALIGN_STR __stringify(__ALIGN) #define __ALIGN_STR __stringify(__ALIGN)

7
arch/arc/kernel/asm-offsets.c
View File

@@ -58,7 +58,14 @@ int main(void)
DEFINE(PT_r5, offsetof(struct pt_regs, r5)); DEFINE(PT_r5, offsetof(struct pt_regs, r5));
DEFINE(PT_r6, offsetof(struct pt_regs, r6)); DEFINE(PT_r6, offsetof(struct pt_regs, r6));
DEFINE(PT_r7, offsetof(struct pt_regs, r7)); DEFINE(PT_r7, offsetof(struct pt_regs, r7));
DEFINE(PT_r8, offsetof(struct pt_regs, r8));
DEFINE(PT_r10, offsetof(struct pt_regs, r10));
DEFINE(PT_r26, offsetof(struct pt_regs, r26));
DEFINE(PT_ret, offsetof(struct pt_regs, ret)); DEFINE(PT_ret, offsetof(struct pt_regs, ret));
DEFINE(PT_blink, offsetof(struct pt_regs, blink));
DEFINE(PT_lpe, offsetof(struct pt_regs, lp_end));
DEFINE(PT_lpc, offsetof(struct pt_regs, lp_count));
DEFINE(PT_user_r25, offsetof(struct pt_regs, user_r25));
DEFINE(SZ_CALLEE_REGS, sizeof(struct callee_regs)); DEFINE(SZ_CALLEE_REGS, sizeof(struct callee_regs));
DEFINE(SZ_PT_REGS, sizeof(struct pt_regs)); DEFINE(SZ_PT_REGS, sizeof(struct pt_regs));

4
arch/arc/kernel/entry-arcv2.S
View File

@@ -70,7 +70,7 @@ reserved:
ENTRY(handle_interrupt) ENTRY(handle_interrupt)
INTERRUPT_PROLOGUE irq INTERRUPT_PROLOGUE
# irq control APIs local_irq_save/restore/disable/enable fiddle with # irq control APIs local_irq_save/restore/disable/enable fiddle with
# global interrupt enable bits in STATUS32 (.IE for 1 prio, .E[] for 2 prio) # global interrupt enable bits in STATUS32 (.IE for 1 prio, .E[] for 2 prio)
@@ -226,7 +226,7 @@ debug_marker_l1:
bset.nz r11, r11, AUX_IRQ_ACT_BIT_U ; NZ means U bset.nz r11, r11, AUX_IRQ_ACT_BIT_U ; NZ means U
sr r11, [AUX_IRQ_ACT] sr r11, [AUX_IRQ_ACT]
INTERRUPT_EPILOGUE irq INTERRUPT_EPILOGUE
rtie rtie
;####### Return from Exception / pure kernel mode ####### ;####### Return from Exception / pure kernel mode #######