mirror of
https://github.com/torvalds/linux.git
synced 2024-11-17 01:22:07 +00:00
x86: 64 bits: shrink and align IRQ stubs
Move the IRQ stub generation to assembly to simplify it and for consistency with 32 bits. Doing it in a C file with asm() statements doesn't help clarity, and it prevents some optimizations. Shrink the IRQ stubs down to just over four bytes per (we fit seven into a 32-byte chunk.) This shrinks the total icache consumption of the IRQ stubs down to an even kilobyte, if all of them are in active use. The downside is that we end up with a double jump, which could have a negative effect on some pipelines. The double jump is always inside the same cacheline on any modern chips. To get the most effect, cache-align the IRQ stubs. This makes the 64-bit code match changes already done to the 32-bit code, and should open up irqinit*.c for unification. Signed-off-by: H. Peter Anvin <hpa@zytor.com>
This commit is contained in:
H. Peter Anvin
parent
b7c6244f13
commit
939b787130
@ -109,9 +109,7 @@ extern asmlinkage void smp_invalidate_interrupt(struct pt_regs *);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_X86_32
|
||||
extern void (*__initconst interrupt[NR_VECTORS-FIRST_EXTERNAL_VECTOR])(void);
|
||||
#endif
|
||||
|
||||
typedef int vector_irq_t[NR_VECTORS];
|
||||
DECLARE_PER_CPU(vector_irq_t, vector_irq);
|
||||
|
||||
@ -627,6 +627,46 @@ END(stub_rt_sigreturn)
|
||||
vector already pushed) */
|
||||
#define XCPT_FRAME _frame ORIG_RAX
|
||||
|
||||
/*
|
||||
* Build the entry stubs and pointer table with some assembler magic.
|
||||
* We pack 7 stubs into a single 32-byte chunk, which will fit in a
|
||||
* single cache line on all modern x86 implementations.
|
||||
*/
|
||||
.section .init.rodata,"a"
|
||||
ENTRY(interrupt)
|
||||
.text
|
||||
.p2align 5
|
||||
.p2align CONFIG_X86_L1_CACHE_SHIFT
|
||||
ENTRY(irq_entries_start)
|
||||
INTR_FRAME
|
||||
vector=FIRST_EXTERNAL_VECTOR
|
||||
.rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
|
||||
.balign 32
|
||||
.rept 7
|
||||
.if vector < NR_VECTORS
|
||||
.if vector != FIRST_EXTERNAL_VECTOR
|
||||
CFI_ADJUST_CFA_OFFSET -8
|
||||
.endif
|
||||
1: pushq $(~vector+0x80) /* Note: always in signed byte range */
|
||||
CFI_ADJUST_CFA_OFFSET 8
|
||||
.if ((vector-FIRST_EXTERNAL_VECTOR)%7) != 6
|
||||
jmp 2f
|
||||
.endif
|
||||
.previous
|
||||
.quad 1b
|
||||
.text
|
||||
vector=vector+1
|
||||
.endif
|
||||
.endr
|
||||
2: jmp common_interrupt
|
||||
.endr
|
||||
CFI_ENDPROC
|
||||
END(irq_entries_start)
|
||||
|
||||
.previous
|
||||
END(interrupt)
|
||||
.previous
|
||||
|
||||
/*
|
||||
* Interrupt entry/exit.
|
||||
*
|
||||
@ -635,11 +675,12 @@ END(stub_rt_sigreturn)
|
||||
* Entry runs with interrupts off.
|
||||
*/
|
||||
|
||||
/* 0(%rsp): interrupt number */
|
||||
/* 0(%rsp): ~(interrupt number)+0x80 */
|
||||
.macro interrupt func
|
||||
addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
|
||||
cld
|
||||
SAVE_ARGS
|
||||
leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
|
||||
leaq -ARGOFFSET(%rsp),%rdi /* arg1 for handler */
|
||||
pushq %rbp
|
||||
/*
|
||||
* Save rbp twice: One is for marking the stack frame, as usual, and the
|
||||
@ -670,7 +711,8 @@ END(stub_rt_sigreturn)
|
||||
call \func
|
||||
.endm
|
||||
|
||||
ENTRY(common_interrupt)
|
||||
.p2align CONFIG_X86_L1_CACHE_SHIFT
|
||||
common_interrupt:
|
||||
XCPT_FRAME
|
||||
interrupt do_IRQ
|
||||
/* 0(%rsp): oldrsp-ARGOFFSET */
|
||||
|
||||
@ -23,41 +23,6 @@
|
||||
#include <asm/apic.h>
|
||||
#include <asm/i8259.h>
|
||||
|
||||
/*
|
||||
* Common place to define all x86 IRQ vectors
|
||||
*
|
||||
* This builds up the IRQ handler stubs using some ugly macros in irq.h
|
||||
*
|
||||
* These macros create the low-level assembly IRQ routines that save
|
||||
* register context and call do_IRQ(). do_IRQ() then does all the
|
||||
* operations that are needed to keep the AT (or SMP IOAPIC)
|
||||
* interrupt-controller happy.
|
||||
*/
|
||||
|
||||
#define IRQ_NAME2(nr) nr##_interrupt(void)
|
||||
#define IRQ_NAME(nr) IRQ_NAME2(IRQ##nr)
|
||||
|
||||
/*
|
||||
* SMP has a few special interrupts for IPI messages
|
||||
*/
|
||||
|
||||
#define BUILD_IRQ(nr) \
|
||||
asmlinkage void IRQ_NAME(nr); \
|
||||
asm("\n.text\n.p2align\n" \
|
||||
"IRQ" #nr "_interrupt:\n\t" \
|
||||
"push $~(" #nr ") ; " \
|
||||
"jmp common_interrupt\n" \
|
||||
".previous");
|
||||
|
||||
#define BI(x,y) \
|
||||
BUILD_IRQ(x##y)
|
||||
|
||||
#define BUILD_16_IRQS(x) \
|
||||
BI(x,0) BI(x,1) BI(x,2) BI(x,3) \
|
||||
BI(x,4) BI(x,5) BI(x,6) BI(x,7) \
|
||||
BI(x,8) BI(x,9) BI(x,a) BI(x,b) \
|
||||
BI(x,c) BI(x,d) BI(x,e) BI(x,f)
|
||||
|
||||
/*
|
||||
* ISA PIC or low IO-APIC triggered (INTA-cycle or APIC) interrupts:
|
||||
* (these are usually mapped to vectors 0x30-0x3f)
|
||||
@ -73,37 +38,6 @@
|
||||
*
|
||||
* (these are usually mapped into the 0x30-0xff vector range)
|
||||
*/
|
||||
BUILD_16_IRQS(0x2) BUILD_16_IRQS(0x3)
|
||||
BUILD_16_IRQS(0x4) BUILD_16_IRQS(0x5) BUILD_16_IRQS(0x6) BUILD_16_IRQS(0x7)
|
||||
BUILD_16_IRQS(0x8) BUILD_16_IRQS(0x9) BUILD_16_IRQS(0xa) BUILD_16_IRQS(0xb)
|
||||
BUILD_16_IRQS(0xc) BUILD_16_IRQS(0xd) BUILD_16_IRQS(0xe) BUILD_16_IRQS(0xf)
|
||||
|
||||
#undef BUILD_16_IRQS
|
||||
#undef BI
|
||||
|
||||
|
||||
#define IRQ(x,y) \
|
||||
IRQ##x##y##_interrupt
|
||||
|
||||
#define IRQLIST_16(x) \
|
||||
IRQ(x,0), IRQ(x,1), IRQ(x,2), IRQ(x,3), \
|
||||
IRQ(x,4), IRQ(x,5), IRQ(x,6), IRQ(x,7), \
|
||||
IRQ(x,8), IRQ(x,9), IRQ(x,a), IRQ(x,b), \
|
||||
IRQ(x,c), IRQ(x,d), IRQ(x,e), IRQ(x,f)
|
||||
|
||||
/* for the irq vectors */
|
||||
static void (*__initdata interrupt[NR_VECTORS - FIRST_EXTERNAL_VECTOR])(void) = {
|
||||
IRQLIST_16(0x2), IRQLIST_16(0x3),
|
||||
IRQLIST_16(0x4), IRQLIST_16(0x5), IRQLIST_16(0x6), IRQLIST_16(0x7),
|
||||
IRQLIST_16(0x8), IRQLIST_16(0x9), IRQLIST_16(0xa), IRQLIST_16(0xb),
|
||||
IRQLIST_16(0xc), IRQLIST_16(0xd), IRQLIST_16(0xe), IRQLIST_16(0xf)
|
||||
};
|
||||
|
||||
#undef IRQ
|
||||
#undef IRQLIST_16
|
||||
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* IRQ2 is cascade interrupt to second interrupt controller
|
||||
|
||||
Loading…
Reference in New Issue
Block a user