From 3f18b1bf599d3d13cd81fdf6bf869c458772adfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 16 Dec 2013 10:24:46 +0100 Subject: [PATCH 01/89] ARM: make isa_mode macro more robust and fix for v7-M MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The definition of isa_mode hardcodes the values to shift PSR_J_BIT and PSR_T_BIT to move them to bits 1 and 0 respectively. Instead use __ffs to calculate the shift from the #define already used for masking. This is relevant on v7-M as there PSR_T_BIT is 0x01000000 instead of 0x00000020 for V7-[AR] and earlier. Because of that isa_mode produced values >= 0x80000 which are unsuitable to index into isa_modes[4] there and so made __show_regs read from undefined memory which resulted in hangs and crashes. Moreover isa_mode is wrong for v7-M even after this robustness fix as there is no J-bit in the PSR register. So hardcode isa_mode to "Thumb" for v7-M. Signed-off-by: Uwe Kleine-König --- arch/arm/include/asm/ptrace.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/ptrace.h b/arch/arm/include/asm/ptrace.h index 04c99f36ff7f..627a03ebb987 100644 --- a/arch/arm/include/asm/ptrace.h +++ b/arch/arm/include/asm/ptrace.h @@ -27,9 +27,13 @@ struct pt_regs { #define thumb_mode(regs) (0) #endif +#ifndef CONFIG_CPU_V7M #define isa_mode(regs) \ - ((((regs)->ARM_cpsr & PSR_J_BIT) >> 23) | \ - (((regs)->ARM_cpsr & PSR_T_BIT) >> 5)) + ((((regs)->ARM_cpsr & PSR_J_BIT) >> (__ffs(PSR_J_BIT) - 1)) | \ + (((regs)->ARM_cpsr & PSR_T_BIT) >> (__ffs(PSR_T_BIT)))) +#else +#define isa_mode(regs) 1 /* Thumb */ +#endif #define processor_mode(regs) \ ((regs)->ARM_cpsr & MODE_MASK) From e2e55fde3f2eaeb1d341508108a51a61dd0fb470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 16 Dec 2013 10:38:57 +0100 Subject: [PATCH 02/89] ARM: show_regs: on v7-M there are no FIQs, different processor modes, ... MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no indication about irqs in PSR and only a single ISA. So skip the whole decoding and just print the xPSR on v7-M. Also mark two static variables as __maybe_unused to prevent the compiler from emitting: arch/arm/kernel/process.c:51:20: warning: 'processor_modes' defined but not used [-Wunused-variable] arch/arm/kernel/process.c:58:20: warning: 'isa_modes' defined but not used [-Wunused-variable] Signed-off-by: Uwe Kleine-König --- arch/arm/kernel/process.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 92f7b15dd221..204f7d273319 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -48,14 +48,14 @@ unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); #endif -static const char *processor_modes[] = { +static const char *processor_modes[] __maybe_unused = { "USER_26", "FIQ_26" , "IRQ_26" , "SVC_26" , "UK4_26" , "UK5_26" , "UK6_26" , "UK7_26" , "UK8_26" , "UK9_26" , "UK10_26", "UK11_26", "UK12_26", "UK13_26", "UK14_26", "UK15_26", "USER_32", "FIQ_32" , "IRQ_32" , "SVC_32" , "UK4_32" , "UK5_32" , "UK6_32" , "ABT_32" , "UK8_32" , "UK9_32" , "UK10_32", "UND_32" , "UK12_32", "UK13_32", "UK14_32", "SYS_32" }; -static const char *isa_modes[] = { +static const char *isa_modes[] __maybe_unused = { "ARM" , "Thumb" , "Jazelle", "ThumbEE" }; @@ -276,12 +276,17 @@ void __show_regs(struct pt_regs *regs) buf[3] = flags & PSR_V_BIT ? 'V' : 'v'; buf[4] = '\0'; +#ifndef CONFIG_CPU_V7M printk("Flags: %s IRQs o%s FIQs o%s Mode %s ISA %s Segment %s\n", buf, interrupts_enabled(regs) ? "n" : "ff", fast_interrupts_enabled(regs) ? "n" : "ff", processor_modes[processor_mode(regs)], isa_modes[isa_mode(regs)], get_fs() == get_ds() ? "kernel" : "user"); +#else + printk("xPSR: %08lx\n", regs->ARM_cpsr); +#endif + #ifdef CONFIG_CPU_CP15 { unsigned int ctrl; From f0d7515372ff0ea53fd66099533d3c8e0464cbbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 1 Feb 2012 10:00:00 +0100 Subject: [PATCH 03/89] ARM: v7m: add trivial suspend support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uwe Kleine-König --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e25419817791..41266af5dfc8 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -2270,7 +2270,7 @@ source "kernel/power/Kconfig" config ARCH_SUSPEND_POSSIBLE depends on !ARCH_S5PC100 depends on CPU_ARM920T || CPU_ARM926T || CPU_FEROCEON || CPU_SA1100 || \ - CPU_V6 || CPU_V6K || CPU_V7 || CPU_XSC3 || CPU_XSCALE || CPU_MOHAWK + CPU_V6 || CPU_V6K || CPU_V7 || CPU_V7M || CPU_XSC3 || CPU_XSCALE || CPU_MOHAWK def_bool y config ARM_CPU_SUSPEND From ddb2ff731b53ae28ec3a2af0da96a108b8bad814 Mon Sep 17 00:00:00 2001 From: Jonathan Austin Date: Mon, 13 Jan 2014 12:10:57 +0100 Subject: [PATCH 04/89] ARM: 7940/1: add support for the Cortex-A12 processor The A12 behaves as the A7/A15 does with respect to setting the SMP bit, and doesn't require TLB ops broadcasting to be explicitly enabled like the A9 does. Note that as the ACTLR cannot (usually) be written from non-secure, it is the responsibility of the bootloader/firmware to set this bit per core - it is done here in Linux as last resort in case of bad firmware. Acked-by: Catalin Marinas Acked-by: Will Deacon Signed-off-by: Jonathan Austin Signed-off-by: Russell King --- arch/arm/include/asm/cputype.h | 1 + arch/arm/mm/proc-v7.S | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h index acdde76b39bb..42f0889f0584 100644 --- a/arch/arm/include/asm/cputype.h +++ b/arch/arm/include/asm/cputype.h @@ -71,6 +71,7 @@ #define ARM_CPU_PART_CORTEX_A5 0xC050 #define ARM_CPU_PART_CORTEX_A15 0xC0F0 #define ARM_CPU_PART_CORTEX_A7 0xC070 +#define ARM_CPU_PART_CORTEX_A12 0xC0D0 #define ARM_CPU_XSCALE_ARCH_MASK 0xe000 #define ARM_CPU_XSCALE_ARCH_V1 0x2000 diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index bd1781979a39..7f9de7e88cd3 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -192,6 +192,7 @@ __v7_cr7mp_setup: mov r10, #(1 << 0) @ Cache/TLB ops broadcasting b 1f __v7_ca7mp_setup: +__v7_ca12mp_setup: __v7_ca15mp_setup: mov r10, #0 1: @@ -483,6 +484,16 @@ __v7_ca7mp_proc_info: __v7_proc __v7_ca7mp_setup .size __v7_ca7mp_proc_info, . - __v7_ca7mp_proc_info + /* + * ARM Ltd. Cortex A12 processor. + */ + .type __v7_ca12mp_proc_info, #object +__v7_ca12mp_proc_info: + .long 0x410fc0d0 + .long 0xff0ffff0 + __v7_proc __v7_ca12mp_setup + .size __v7_ca12mp_proc_info, . - __v7_ca12mp_proc_info + /* * ARM Ltd. Cortex A15 processor. */ From 889f172d920b6f275408199ec836df62979bcd52 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Fri, 17 Jan 2014 20:42:48 +0100 Subject: [PATCH 05/89] ARM: 7945/1: footbridge: Switch to sched_clock_register() The 32 bit sched_clock interface supports 64 bits since 3.13-rc1. Upgrade to the 64 bit function to allow us to remove the 32 bit registration interface. Signed-off-by: Stephen Boyd Signed-off-by: Russell King --- arch/arm/mach-footbridge/dc21285-timer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/mach-footbridge/dc21285-timer.c b/arch/arm/mach-footbridge/dc21285-timer.c index 3971104d32d4..5d2725fd878c 100644 --- a/arch/arm/mach-footbridge/dc21285-timer.c +++ b/arch/arm/mach-footbridge/dc21285-timer.c @@ -125,7 +125,7 @@ void __init footbridge_timer_init(void) clockevents_config_and_register(ce, rate, 0x4, 0xffffff); } -static u32 notrace footbridge_read_sched_clock(void) +static u64 notrace footbridge_read_sched_clock(void) { return ~*CSR_TIMER3_VALUE; } @@ -138,5 +138,5 @@ void __init footbridge_sched_clock(void) *CSR_TIMER3_CLR = 0; *CSR_TIMER3_CNTL = TIMER_CNTL_ENABLE | TIMER_CNTL_DIV16; - setup_sched_clock(footbridge_read_sched_clock, 24, rate); + sched_clock_register(footbridge_read_sched_clock, 24, rate); } From 5b61d4a5d6676b5bb4c3c101683d3c7fd0df2a38 Mon Sep 17 00:00:00 2001 From: Christopher Covington Date: Wed, 29 Jan 2014 22:01:31 +0100 Subject: [PATCH 06/89] ARM: 7948/1: hw_breakpoint: Add ARMv8 support Add the trivial support necessary to get hardware breakpoints working for GDB on ARMv8 simulators running in AArch32 mode. Acked-by: Will Deacon Signed-off-by: Christopher Covington Signed-off-by: Russell King --- arch/arm/include/asm/hw_breakpoint.h | 1 + arch/arm/kernel/hw_breakpoint.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/hw_breakpoint.h b/arch/arm/include/asm/hw_breakpoint.h index eef55ea9ef00..8e427c7b4425 100644 --- a/arch/arm/include/asm/hw_breakpoint.h +++ b/arch/arm/include/asm/hw_breakpoint.h @@ -51,6 +51,7 @@ static inline void decode_ctrl_reg(u32 reg, #define ARM_DEBUG_ARCH_V7_ECP14 3 #define ARM_DEBUG_ARCH_V7_MM 4 #define ARM_DEBUG_ARCH_V7_1 5 +#define ARM_DEBUG_ARCH_V8 6 /* Breakpoint */ #define ARM_BREAKPOINT_EXECUTE 0 diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 3d446605cbf8..9da35c6d3411 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -167,7 +167,7 @@ static int debug_arch_supported(void) /* Can we determine the watchpoint access type from the fsr? */ static int debug_exception_updates_fsr(void) { - return 0; + return get_debug_arch() >= ARM_DEBUG_ARCH_V8; } /* Determine number of WRP registers available. */ @@ -257,6 +257,7 @@ static int enable_monitor_mode(void) break; case ARM_DEBUG_ARCH_V7_ECP14: case ARM_DEBUG_ARCH_V7_1: + case ARM_DEBUG_ARCH_V8: ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN)); isb(); break; From 0f054e3ceab33a7fd3b41a9708286bce420d570d Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 31 Jan 2014 19:49:24 +0100 Subject: [PATCH 07/89] ARM: 7949/1: feroceon: Log a FW_BUG if the L2 cache is turned on at boot Booting on feroceon CPUS requires the L2 cache to be turned off. With some kernel configurations (notably CONFIG_ARM_PATCH_PHYS_VIRT disabled) the kernel will boot even if the L2 is turned on. However there may be subtle breakage, and when PATCH_PHYS_VIRT is enabled it is very likely that booting with L2 will crash at early boot before any kernel diagnostic output. The diagnostic message is intended to discourage people from shipping bootloaders that leave the L2 turned on. The issue on feroceon is that the L2 is bypassed when the L1 caches are disabled. So the decompressor will place parts of the kernel image into the L2 and the early cache-off boot code in head.S will write to parts of the kernel image, bypassing the L2 and creating inconsistency. Tested on ARM Kirkwood. Signed-off-by: Jason Gunthorpe Acked-by: Jason Cooper Signed-off-by: Russell King --- arch/arm/mm/cache-feroceon-l2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm/mm/cache-feroceon-l2.c b/arch/arm/mm/cache-feroceon-l2.c index 48bc3c0a87ce..aae891820f8f 100644 --- a/arch/arm/mm/cache-feroceon-l2.c +++ b/arch/arm/mm/cache-feroceon-l2.c @@ -331,7 +331,9 @@ static void __init enable_l2(void) enable_icache(); if (d) enable_dcache(); - } + } else + pr_err(FW_BUG + "Feroceon L2: bootloader left the L2 cache on!\n"); } void __init feroceon_l2_init(int __l2_wt_override) From afdd3bba3ca18e6b0515b2e60101a932f5fa9afa Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 4 Feb 2014 13:23:48 +0100 Subject: [PATCH 08/89] ARM: 7951/1: uaccess: use CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS Now that we select HAVE_EFFICIENT_UNALIGNED_ACCESS for ARMv6+ CPUs, replace the __LINUX_ARM_ARCH__ check in uaccess.h with the new symbol. Signed-off-by: Nicolas Pitre Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/include/asm/uaccess.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 72abdc541f38..12c3a5decc60 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -19,7 +19,7 @@ #include #include -#if __LINUX_ARM_ARCH__ < 6 +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS #include #else #define __get_user_unaligned __get_user From b6ccb9803e90c16b212cf4ed62913a7591e79a39 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 7 Feb 2014 19:12:27 +0100 Subject: [PATCH 09/89] ARM: 7954/1: mm: remove remaining domain support from ARMv6 CPU_32v6 currently selects CPU_USE_DOMAINS if CPU_V6 and MMU. This is because ARM 1136 r0pX CPUs lack the v6k extensions, and therefore do not have hardware thread registers. The lack of these registers requires the kernel to update the vectors page at each context switch in order to write a new TLS pointer. This write must be done via the userspace mapping, since aliasing caches can lead to expensive flushing when using kmap. Finally, this requires the vectors page to be mapped r/w for kernel and r/o for user, which has implications for things like put_user which must trigger CoW appropriately when targetting user pages. The upshot of all this is that a v6/v7 kernel makes use of domains to segregate kernel and user memory accesses. This has the nasty side-effect of making device mappings executable, which has been observed to cause subtle bugs on recent cores (e.g. Cortex-A15 performing a speculative instruction fetch from the GIC and acking an interrupt in the process). This patch solves this problem by removing the remaining domain support from ARMv6. A new memory type is added specifically for the vectors page which allows that page (and only that page) to be mapped as user r/o, kernel r/w. All other user r/o pages are mapped also as kernel r/o. Patch co-developed with Russell King. Cc: Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/include/asm/futex.h | 6 ------ arch/arm/include/asm/pgtable-2level.h | 1 + arch/arm/mm/Kconfig | 3 +-- arch/arm/mm/mmu.c | 10 ++++++++++ arch/arm/mm/proc-macros.S | 19 ++++++------------- arch/arm/mm/proc-v7-2level.S | 7 ------- 6 files changed, 18 insertions(+), 28 deletions(-) diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index e42cf597f6e6..2aff798fbef4 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -3,11 +3,6 @@ #ifdef __KERNEL__ -#if defined(CONFIG_CPU_USE_DOMAINS) && defined(CONFIG_SMP) -/* ARM doesn't provide unprivileged exclusive memory accessors */ -#include -#else - #include #include #include @@ -164,6 +159,5 @@ futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) return ret; } -#endif /* !(CPU_USE_DOMAINS && SMP) */ #endif /* __KERNEL__ */ #endif /* _ASM_ARM_FUTEX_H */ diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index dfff709fda3c..219ac88a9542 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -140,6 +140,7 @@ #define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 0x0c) << 2) /* 1100 */ #define L_PTE_MT_DEV_WC (_AT(pteval_t, 0x09) << 2) /* 1001 */ #define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 0x0b) << 2) /* 1011 */ +#define L_PTE_MT_VECTORS (_AT(pteval_t, 0x0f) << 2) /* 1111 */ #define L_PTE_MT_MASK (_AT(pteval_t, 0x0f) << 2) #ifndef __ASSEMBLY__ diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 1f8fed94c2a4..ca8ecdee47d8 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -446,7 +446,6 @@ config CPU_32v5 config CPU_32v6 bool - select CPU_USE_DOMAINS if CPU_V6 && MMU select TLS_REG_EMUL if !CPU_32v6K && !MMU config CPU_32v6K @@ -671,7 +670,7 @@ config ARM_VIRT_EXT config SWP_EMULATE bool "Emulate SWP/SWPB instructions" - depends on !CPU_USE_DOMAINS && CPU_V7 + depends on CPU_V7 default y if SMP select HAVE_PROC_CPU if PROC_FS help diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 4f08c133cc25..6ec07a84f759 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -510,6 +510,16 @@ static void __init build_mem_type_table(void) s2_pgprot = cp->pte_s2; hyp_device_pgprot = s2_device_pgprot = mem_types[MT_DEVICE].prot_pte; + /* + * We don't use domains on ARMv6 (since this causes problems with + * v6/v7 kernels), so we must use a separate memory type for user + * r/o, kernel r/w to map the vectors page. + */ +#ifndef CONFIG_ARM_LPAE + if (cpu_arch == CPU_ARCH_ARMv6) + vecs_pgprot |= L_PTE_MT_VECTORS; +#endif + /* * ARMv6 and above have extended page tables. */ diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S index e3c48a3fe063..ee1d80593958 100644 --- a/arch/arm/mm/proc-macros.S +++ b/arch/arm/mm/proc-macros.S @@ -112,13 +112,9 @@ * 100x 1 0 1 r/o no acc * 10x0 1 0 1 r/o no acc * 1011 0 0 1 r/w no acc - * 110x 0 1 0 r/w r/o - * 11x0 0 1 0 r/w r/o - * 1111 0 1 1 r/w r/w - * - * If !CONFIG_CPU_USE_DOMAINS, the following permissions are changed: * 110x 1 1 1 r/o r/o * 11x0 1 1 1 r/o r/o + * 1111 0 1 1 r/w r/w */ .macro armv6_mt_table pfx \pfx\()_mt_table: @@ -137,7 +133,7 @@ .long PTE_EXT_TEX(2) @ L_PTE_MT_DEV_NONSHARED .long 0x00 @ unused .long 0x00 @ unused - .long 0x00 @ unused + .long PTE_CACHEABLE | PTE_BUFFERABLE | PTE_EXT_APX @ L_PTE_MT_VECTORS .endm .macro armv6_set_pte_ext pfx @@ -158,24 +154,21 @@ tst r1, #L_PTE_USER orrne r3, r3, #PTE_EXT_AP1 -#ifdef CONFIG_CPU_USE_DOMAINS - @ allow kernel read/write access to read-only user pages tstne r3, #PTE_EXT_APX - bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0 -#endif + + @ user read-only -> kernel read-only + bicne r3, r3, #PTE_EXT_AP0 tst r1, #L_PTE_XN orrne r3, r3, #PTE_EXT_XN - orr r3, r3, r2 + eor r3, r3, r2 tst r1, #L_PTE_YOUNG tstne r1, #L_PTE_PRESENT moveq r3, #0 -#ifndef CONFIG_CPU_USE_DOMAINS tstne r1, #L_PTE_NONE movne r3, #0 -#endif str r3, [r0] mcr p15, 0, r0, c7, c10, 1 @ flush_pte diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S index bdd3be4be77a..1f52915f2b28 100644 --- a/arch/arm/mm/proc-v7-2level.S +++ b/arch/arm/mm/proc-v7-2level.S @@ -90,21 +90,14 @@ ENTRY(cpu_v7_set_pte_ext) tst r1, #L_PTE_USER orrne r3, r3, #PTE_EXT_AP1 -#ifdef CONFIG_CPU_USE_DOMAINS - @ allow kernel read/write access to read-only user pages - tstne r3, #PTE_EXT_APX - bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0 -#endif tst r1, #L_PTE_XN orrne r3, r3, #PTE_EXT_XN tst r1, #L_PTE_YOUNG tstne r1, #L_PTE_VALID -#ifndef CONFIG_CPU_USE_DOMAINS eorne r1, r1, #L_PTE_NONE tstne r1, #L_PTE_NONE -#endif moveq r3, #0 ARM( str r3, [r0, #2048]! ) From eb02313d2c96e1793cf37693d432fd2c129ae170 Mon Sep 17 00:00:00 2001 From: Dmitry Eremin-Solenikov Date: Tue, 26 Nov 2013 11:11:00 +0100 Subject: [PATCH 10/89] ARM: 7902/1: sa1100: h3xxx: move serial port GPIO handling to common place Both h3100 and h3600 request UART gpios during init_machine time. As sa1100 gpio driver is going to become proper machine driver, move gpio handling to UART port functions. Request all gpios using gpio_request array once and then guard them from rerequesting with bool variable. Signed-off-by: Dmitry Eremin-Solenikov Acked-by: Linus Walleij Signed-off-by: Russell King --- arch/arm/mach-sa1100/h3100.c | 7 ------- arch/arm/mach-sa1100/h3600.c | 7 ------- arch/arm/mach-sa1100/h3xxx.c | 27 +++++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/arch/arm/mach-sa1100/h3100.c b/arch/arm/mach-sa1100/h3100.c index daa27c474c13..3c43219bc881 100644 --- a/arch/arm/mach-sa1100/h3100.c +++ b/arch/arm/mach-sa1100/h3100.c @@ -122,15 +122,8 @@ static struct irda_platform_data h3100_irda_data = { .shutdown = h3100_irda_shutdown, }; -static struct gpio_default_state h3100_default_gpio[] = { - { H3XXX_GPIO_COM_DCD, GPIO_MODE_IN, "COM DCD" }, - { H3XXX_GPIO_COM_CTS, GPIO_MODE_IN, "COM CTS" }, - { H3XXX_GPIO_COM_RTS, GPIO_MODE_OUT0, "COM RTS" }, -}; - static void __init h3100_mach_init(void) { - h3xxx_init_gpio(h3100_default_gpio, ARRAY_SIZE(h3100_default_gpio)); h3xxx_mach_init(); sa11x0_register_lcd(&h3100_lcd_info); diff --git a/arch/arm/mach-sa1100/h3600.c b/arch/arm/mach-sa1100/h3600.c index a663e7230141..5be54c214c7c 100644 --- a/arch/arm/mach-sa1100/h3600.c +++ b/arch/arm/mach-sa1100/h3600.c @@ -130,15 +130,8 @@ static struct irda_platform_data h3600_irda_data = { .shutdown = h3600_irda_shutdown, }; -static struct gpio_default_state h3600_default_gpio[] = { - { H3XXX_GPIO_COM_DCD, GPIO_MODE_IN, "COM DCD" }, - { H3XXX_GPIO_COM_CTS, GPIO_MODE_IN, "COM CTS" }, - { H3XXX_GPIO_COM_RTS, GPIO_MODE_OUT0, "COM RTS" }, -}; - static void __init h3600_mach_init(void) { - h3xxx_init_gpio(h3600_default_gpio, ARRAY_SIZE(h3600_default_gpio)); h3xxx_mach_init(); sa11x0_register_lcd(&h3600_lcd_info); diff --git a/arch/arm/mach-sa1100/h3xxx.c b/arch/arm/mach-sa1100/h3xxx.c index f17e7382242a..82e87024b1a6 100644 --- a/arch/arm/mach-sa1100/h3xxx.c +++ b/arch/arm/mach-sa1100/h3xxx.c @@ -116,9 +116,34 @@ static struct resource h3xxx_flash_resource = /* * H3xxx uart support */ +static struct gpio h3xxx_uart_gpio[] = { + { H3XXX_GPIO_COM_DCD, GPIOF_IN, "COM DCD" }, + { H3XXX_GPIO_COM_CTS, GPIOF_IN, "COM CTS" }, + { H3XXX_GPIO_COM_RTS, GPIOF_OUT_INIT_LOW, "COM RTS" }, +}; + +static bool h3xxx_uart_request_gpios(void) +{ + static bool h3xxx_uart_gpio_ok; + int rc; + + if (h3xxx_uart_gpio_ok) + return true; + + rc = gpio_request_array(h3xxx_uart_gpio, ARRAY_SIZE(h3xxx_uart_gpio)); + if (rc) + pr_err("h3xxx_uart_request_gpios: error %d\n", rc); + else + h3xxx_uart_gpio_ok = true; + + return h3xxx_uart_gpio_ok; +} + static void h3xxx_uart_set_mctrl(struct uart_port *port, u_int mctrl) { if (port->mapbase == _Ser3UTCR0) { + if (!h3xxx_uart_request_gpios()) + return; gpio_set_value(H3XXX_GPIO_COM_RTS, !(mctrl & TIOCM_RTS)); } } @@ -128,6 +153,8 @@ static u_int h3xxx_uart_get_mctrl(struct uart_port *port) u_int ret = TIOCM_CD | TIOCM_CTS | TIOCM_DSR; if (port->mapbase == _Ser3UTCR0) { + if (!h3xxx_uart_request_gpios()) + return ret; /* * DCD and CTS bits are inverted in GPLR by RS232 transceiver */ From 75916d5549e4c134f69068de175c90c170c17a7a Mon Sep 17 00:00:00 2001 From: Dmitry Eremin-Solenikov Date: Tue, 26 Nov 2013 11:11:21 +0100 Subject: [PATCH 11/89] ARM: 7903/1: sa1100: h3xxx: drop hand-coded gpio_request_array analogue h3xxx_init_gpio() behaves alsmost like gpio_request_array. Also after all GPIO refactoring it is unused. Drop it now. Signed-off-by: Dmitry Eremin-Solenikov Acked-by: Linus Walleij Signed-off-by: Russell King --- arch/arm/mach-sa1100/h3xxx.c | 31 ----------------------- arch/arm/mach-sa1100/include/mach/h3xxx.h | 11 -------- 2 files changed, 42 deletions(-) diff --git a/arch/arm/mach-sa1100/h3xxx.c b/arch/arm/mach-sa1100/h3xxx.c index 82e87024b1a6..c79bf467fb7f 100644 --- a/arch/arm/mach-sa1100/h3xxx.c +++ b/arch/arm/mach-sa1100/h3xxx.c @@ -28,37 +28,6 @@ #include "generic.h" -void h3xxx_init_gpio(struct gpio_default_state *s, size_t n) -{ - while (n--) { - const char *name = s->name; - int err; - - if (!name) - name = "[init]"; - err = gpio_request(s->gpio, name); - if (err) { - printk(KERN_ERR "gpio%u: unable to request: %d\n", - s->gpio, err); - continue; - } - if (s->mode >= 0) { - err = gpio_direction_output(s->gpio, s->mode); - } else { - err = gpio_direction_input(s->gpio); - } - if (err) { - printk(KERN_ERR "gpio%u: unable to set direction: %d\n", - s->gpio, err); - continue; - } - if (!s->name) - gpio_free(s->gpio); - s++; - } -} - - /* * H3xxx flash support */ diff --git a/arch/arm/mach-sa1100/include/mach/h3xxx.h b/arch/arm/mach-sa1100/include/mach/h3xxx.h index c810620db53d..603d4343f7f6 100644 --- a/arch/arm/mach-sa1100/include/mach/h3xxx.h +++ b/arch/arm/mach-sa1100/include/mach/h3xxx.h @@ -79,17 +79,6 @@ #define H3600_EGPIO_LCD_5V_ON (H3XXX_EGPIO_BASE + 14) /* enable 5V to LCD. active high. */ #define H3600_EGPIO_LVDD_ON (H3XXX_EGPIO_BASE + 15) /* enable 9V and -6.5V to LCD. */ -struct gpio_default_state { - int gpio; - int mode; - const char *name; -}; - -#define GPIO_MODE_IN -1 -#define GPIO_MODE_OUT0 0 -#define GPIO_MODE_OUT1 1 - -void h3xxx_init_gpio(struct gpio_default_state *s, size_t n); void __init h3xxx_map_io(void); void __init h3xxx_mach_init(void); From 287d4d51e6bb1cd43cb77352bdec5bac1b2e1226 Mon Sep 17 00:00:00 2001 From: Dmitry Eremin-Solenikov Date: Thu, 9 Jan 2014 13:32:18 +0100 Subject: [PATCH 12/89] ARM: 7936/1: sa1100: collie: add support for IrDA transceiver Collie has onboard IrDA transceiver controlled via active-low gpio. Add corresponding platform data. Signed-off-by: Dmitry Eremin-Solenikov Signed-off-by: Russell King --- arch/arm/mach-sa1100/collie.c | 33 ++++++++++++++++++++++ arch/arm/mach-sa1100/include/mach/collie.h | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/arch/arm/mach-sa1100/collie.c b/arch/arm/mach-sa1100/collie.c index 831a15824ec8..f9874ba60cc8 100644 --- a/arch/arm/mach-sa1100/collie.c +++ b/arch/arm/mach-sa1100/collie.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -96,6 +97,37 @@ static struct mcp_plat_data collie_mcp_data = { .codec_pdata = &collie_ucb1x00_data, }; +static int collie_ir_startup(struct device *dev) +{ + int rc = gpio_request(COLLIE_GPIO_IR_ON, "IrDA"); + if (rc) + return rc; + rc = gpio_direction_output(COLLIE_GPIO_IR_ON, 1); + + if (!rc) + return 0; + + gpio_free(COLLIE_GPIO_IR_ON); + return rc; +} + +static void collie_ir_shutdown(struct device *dev) +{ + gpio_free(COLLIE_GPIO_IR_ON); +} + +static int collie_ir_set_power(struct device *dev, unsigned int state) +{ + gpio_set_value(COLLIE_GPIO_IR_ON, !state); + return 0; +} + +static struct irda_platform_data collie_ir_data = { + .startup = collie_ir_startup, + .shutdown = collie_ir_shutdown, + .set_power = collie_ir_set_power, +}; + /* * Collie AC IN */ @@ -400,6 +432,7 @@ static void __init collie_init(void) sa11x0_register_mtd(&collie_flash_data, collie_flash_resources, ARRAY_SIZE(collie_flash_resources)); sa11x0_register_mcp(&collie_mcp_data); + sa11x0_register_irda(&collie_ir_data); sharpsl_save_param(); } diff --git a/arch/arm/mach-sa1100/include/mach/collie.h b/arch/arm/mach-sa1100/include/mach/collie.h index f33679d2d3ee..0ef22f9cda95 100644 --- a/arch/arm/mach-sa1100/include/mach/collie.h +++ b/arch/arm/mach-sa1100/include/mach/collie.h @@ -78,7 +78,7 @@ extern void locomolcd_power(int on); #define COLLIE_TC35143_GPIO_VERSION0 UCB_IO_0 #define COLLIE_TC35143_GPIO_TBL_CHK UCB_IO_1 #define COLLIE_TC35143_GPIO_VPEN_ON UCB_IO_2 -#define COLLIE_TC35143_GPIO_IR_ON UCB_IO_3 +#define COLLIE_GPIO_IR_ON (COLLIE_TC35143_GPIO_BASE + 3) #define COLLIE_TC35143_GPIO_AMP_ON UCB_IO_4 #define COLLIE_TC35143_GPIO_VERSION1 UCB_IO_5 #define COLLIE_TC35143_GPIO_FS8KLPF UCB_IO_5 From dfdf5f63b4380b9e719145c7cb2cb450a3d06ac4 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 11 Feb 2014 09:28:09 +0100 Subject: [PATCH 13/89] ARM: 7956/1: mmci: rename some extended flags These four (so far unused) flags are only found in the ST Micro versions of MMCI, so infix them properly with the _ST_ infix. Signed-off-by: Linus Walleij Signed-off-by: Russell King --- drivers/mmc/host/mmci.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/mmc/host/mmci.h b/drivers/mmc/host/mmci.h index 168bc72f7a94..84c0e59b792a 100644 --- a/drivers/mmc/host/mmci.h +++ b/drivers/mmc/host/mmci.h @@ -38,10 +38,11 @@ #define MCI_CPSM_INTERRUPT (1 << 8) #define MCI_CPSM_PENDING (1 << 9) #define MCI_CPSM_ENABLE (1 << 10) -#define MCI_SDIO_SUSP (1 << 11) -#define MCI_ENCMD_COMPL (1 << 12) -#define MCI_NIEN (1 << 13) -#define MCI_CE_ATACMD (1 << 14) +/* Argument flag extenstions in the ST Micro versions */ +#define MCI_ST_SDIO_SUSP (1 << 11) +#define MCI_ST_ENCMD_COMPL (1 << 12) +#define MCI_ST_NIEN (1 << 13) +#define MCI_ST_CE_ATACMD (1 << 14) #define MMCIRESPCMD 0x010 #define MMCIRESPONSE0 0x014 From f210c53a82ab9ba10574f8121a300d78e618febd Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Wed, 12 Feb 2014 14:06:43 +0100 Subject: [PATCH 14/89] ARM: 7958/1: amba: Let runtime PM callbacks be available for CONFIG_PM Convert to the SET_PM_RUNTIME_PM macro while defining the runtime PM callbacks. This means the callbacks becomes available for both CONFIG_PM_SLEEP and CONFIG_PM_RUNTIME, which is needed by drivers and power domains. Signed-off-by: Ulf Hansson Signed-off-by: Russell King --- drivers/amba/bus.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index 9e6029105607..3cf61a127ee5 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -83,7 +83,7 @@ static struct device_attribute amba_dev_attrs[] = { __ATTR_NULL, }; -#ifdef CONFIG_PM_RUNTIME +#ifdef CONFIG_PM /* * Hooks to provide runtime PM of the pclk (bus clock). It is safe to * enable/disable the bus clock at runtime PM suspend/resume as this @@ -123,7 +123,7 @@ static const struct dev_pm_ops amba_pm = { .thaw = pm_generic_thaw, .poweroff = pm_generic_poweroff, .restore = pm_generic_restore, - SET_RUNTIME_PM_OPS( + SET_PM_RUNTIME_PM_OPS( amba_pm_runtime_suspend, amba_pm_runtime_resume, NULL From ea36d2ab1a04d38bb24145b2b8fa8c8109cbe64d Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Fri, 14 Feb 2014 12:46:02 +0100 Subject: [PATCH 15/89] ARM: 7962/2: Make all mcpm functions notrace The functions in mcpm_entry.c are mostly intended for use during scary cache and coherency disabling sequences, or do other things which confuse trace ... like powering a CPU down and not returning. Similarly for the backend code. For simplicity, this patch just makes whole files notrace. There should be more than enough traceable points on the paths to these functions, but we can be more fine-grained later if there is a need for it. Jon Medhurst: Also added spc.o to the list of files as it contains functions used by MCPM code which have comments comments like: "might be used in code paths where normal cacheable locks are not working" Signed-off-by: Dave Martin Signed-off-by: Jon Medhurst Acked-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/common/Makefile | 1 + arch/arm/mach-vexpress/Makefile | 3 +++ 2 files changed, 4 insertions(+) diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index 4bdc41622c36..70b1eff477b3 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_SHARP_SCOOP) += scoop.o obj-$(CONFIG_PCI_HOST_ITE8152) += it8152.o obj-$(CONFIG_ARM_TIMER_SP804) += timer-sp.o obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o mcpm_platsmp.o vlock.o +CFLAGS_REMOVE_mcpm_entry.o = -pg AFLAGS_mcpm_head.o := -march=armv7-a AFLAGS_vlock.o := -march=armv7-a obj-$(CONFIG_TI_PRIV_EDMA) += edma.o diff --git a/arch/arm/mach-vexpress/Makefile b/arch/arm/mach-vexpress/Makefile index 0997e0b7494c..fc649bc09d0c 100644 --- a/arch/arm/mach-vexpress/Makefile +++ b/arch/arm/mach-vexpress/Makefile @@ -8,8 +8,11 @@ obj-y := v2m.o obj-$(CONFIG_ARCH_VEXPRESS_CA9X4) += ct-ca9x4.o obj-$(CONFIG_ARCH_VEXPRESS_DCSCB) += dcscb.o dcscb_setup.o CFLAGS_dcscb.o += -march=armv7-a +CFLAGS_REMOVE_dcscb.o = -pg obj-$(CONFIG_ARCH_VEXPRESS_SPC) += spc.o +CFLAGS_REMOVE_spc.o = -pg obj-$(CONFIG_ARCH_VEXPRESS_TC2_PM) += tc2_pm.o CFLAGS_tc2_pm.o += -march=armv7-a +CFLAGS_REMOVE_tc2_pm.o = -pg obj-$(CONFIG_SMP) += platsmp.o obj-$(CONFIG_HOTPLUG_CPU) += hotplug.o From 6ea41c80115f49e7d8b80312ffc99973d283471f Mon Sep 17 00:00:00 2001 From: Steven Capper Date: Tue, 18 Feb 2014 16:56:55 +0100 Subject: [PATCH 16/89] ARM: 7979/1: mm: Remove hugetlb warning from Coherent DMA allocator The Coherant DMA allocator allocates pages of high order then splits them up into smaller pages. This splitting logic would run into problems if the allocator was given compound pages. Thus the Coherant DMA allocator was originally incompatible with compound pages existing and, by extension, huge pages. A compile #error was put in place whenever huge pages were enabled. Compatibility with compound pages has since been introduced by the following commit (which merely excludes GFP_COMP pages from being requested by the coherant DMA allocator): ea2e705 ARM: 7172/1: dma: Drop GFP_COMP for DMA memory allocations When huge page support was introduced to ARM, the compile #error in dma-mapping.c was replaced by a #warning when it should have been removed instead. This patch removes the compile #warning in dma-mapping.c when huge pages are enabled. Signed-off-by: Steve Capper Signed-off-by: Russell King --- arch/arm/mm/dma-mapping.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 1a77450e728a..5bb4e00de9f8 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -284,9 +284,6 @@ static void __dma_free_buffer(struct page *page, size_t size) } #ifdef CONFIG_MMU -#ifdef CONFIG_HUGETLB_PAGE -#warning ARM Coherent DMA allocator does not (yet) support huge TLB -#endif static void *__alloc_from_contiguous(struct device *dev, size_t size, pgprot_t prot, struct page **ret_page, From bbd64559376fa25732994c4181c8ec493fa57871 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Fri, 7 Feb 2014 21:01:19 +0000 Subject: [PATCH 17/89] ARM: perf: support percpu irqs for the CPU PMU Some CPU PMUs are wired up with one PPI for all the CPUs instead of with a different SPI for each CPU. Add support for these devices. Signed-off-by: Stephen Boyd Signed-off-by: Will Deacon --- arch/arm/kernel/perf_event.c | 14 ++++- arch/arm/kernel/perf_event_cpu.c | 95 +++++++++++++++++++++++--------- 2 files changed, 79 insertions(+), 30 deletions(-) diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index 789d846a9184..b0c8489018d3 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include @@ -295,9 +297,15 @@ validate_group(struct perf_event *event) static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) { - struct arm_pmu *armpmu = (struct arm_pmu *) dev; - struct platform_device *plat_device = armpmu->plat_device; - struct arm_pmu_platdata *plat = dev_get_platdata(&plat_device->dev); + struct arm_pmu *armpmu; + struct platform_device *plat_device; + struct arm_pmu_platdata *plat; + + if (irq_is_percpu(irq)) + dev = *(void **)dev; + armpmu = dev; + plat_device = armpmu->plat_device; + plat = dev_get_platdata(&plat_device->dev); if (plat && plat->handle_irq) return plat->handle_irq(irq, dev, armpmu->handle_irq); diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index 20d553c9f5e2..6efd8aab15df 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include @@ -33,6 +35,7 @@ /* Set at runtime when we know what CPU type we are. */ static struct arm_pmu *cpu_pmu; +static DEFINE_PER_CPU(struct arm_pmu *, percpu_pmu); static DEFINE_PER_CPU(struct perf_event * [ARMPMU_MAX_HWEVENTS], hw_events); static DEFINE_PER_CPU(unsigned long [BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)], used_mask); static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events); @@ -71,6 +74,26 @@ static struct pmu_hw_events *cpu_pmu_get_cpu_events(void) return this_cpu_ptr(&cpu_hw_events); } +static void cpu_pmu_enable_percpu_irq(void *data) +{ + struct arm_pmu *cpu_pmu = data; + struct platform_device *pmu_device = cpu_pmu->plat_device; + int irq = platform_get_irq(pmu_device, 0); + + enable_percpu_irq(irq, IRQ_TYPE_NONE); + cpumask_set_cpu(smp_processor_id(), &cpu_pmu->active_irqs); +} + +static void cpu_pmu_disable_percpu_irq(void *data) +{ + struct arm_pmu *cpu_pmu = data; + struct platform_device *pmu_device = cpu_pmu->plat_device; + int irq = platform_get_irq(pmu_device, 0); + + cpumask_clear_cpu(smp_processor_id(), &cpu_pmu->active_irqs); + disable_percpu_irq(irq); +} + static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu) { int i, irq, irqs; @@ -78,12 +101,18 @@ static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu) irqs = min(pmu_device->num_resources, num_possible_cpus()); - for (i = 0; i < irqs; ++i) { - if (!cpumask_test_and_clear_cpu(i, &cpu_pmu->active_irqs)) - continue; - irq = platform_get_irq(pmu_device, i); - if (irq >= 0) - free_irq(irq, cpu_pmu); + irq = platform_get_irq(pmu_device, 0); + if (irq >= 0 && irq_is_percpu(irq)) { + on_each_cpu(cpu_pmu_disable_percpu_irq, cpu_pmu, 1); + free_percpu_irq(irq, &percpu_pmu); + } else { + for (i = 0; i < irqs; ++i) { + if (!cpumask_test_and_clear_cpu(i, &cpu_pmu->active_irqs)) + continue; + irq = platform_get_irq(pmu_device, i); + if (irq >= 0) + free_irq(irq, cpu_pmu); + } } } @@ -101,33 +130,44 @@ static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler) return -ENODEV; } - for (i = 0; i < irqs; ++i) { - err = 0; - irq = platform_get_irq(pmu_device, i); - if (irq < 0) - continue; - - /* - * If we have a single PMU interrupt that we can't shift, - * assume that we're running on a uniprocessor machine and - * continue. Otherwise, continue without this interrupt. - */ - if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { - pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", - irq, i); - continue; - } - - err = request_irq(irq, handler, - IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu", - cpu_pmu); + irq = platform_get_irq(pmu_device, 0); + if (irq >= 0 && irq_is_percpu(irq)) { + err = request_percpu_irq(irq, handler, "arm-pmu", &percpu_pmu); if (err) { pr_err("unable to request IRQ%d for ARM PMU counters\n", irq); return err; } + on_each_cpu(cpu_pmu_enable_percpu_irq, cpu_pmu, 1); + } else { + for (i = 0; i < irqs; ++i) { + err = 0; + irq = platform_get_irq(pmu_device, i); + if (irq < 0) + continue; - cpumask_set_cpu(i, &cpu_pmu->active_irqs); + /* + * If we have a single PMU interrupt that we can't shift, + * assume that we're running on a uniprocessor machine and + * continue. Otherwise, continue without this interrupt. + */ + if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { + pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", + irq, i); + continue; + } + + err = request_irq(irq, handler, + IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu", + cpu_pmu); + if (err) { + pr_err("unable to request IRQ%d for ARM PMU counters\n", + irq); + return err; + } + + cpumask_set_cpu(i, &cpu_pmu->active_irqs); + } } return 0; @@ -141,6 +181,7 @@ static void cpu_pmu_init(struct arm_pmu *cpu_pmu) events->events = per_cpu(hw_events, cpu); events->used_mask = per_cpu(used_mask, cpu); raw_spin_lock_init(&events->pmu_lock); + per_cpu(percpu_pmu, cpu) = cpu_pmu; } cpu_pmu->get_hw_events = cpu_pmu_get_cpu_events; From 3a3967ed9135f931f3848b0c946dcf3c10ca9f9a Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Fri, 7 Feb 2014 21:01:20 +0000 Subject: [PATCH 18/89] ARM: perf: assign pdev pointer earlier for CPU PMUs We want to inspect the of_node that the pdev is pointing to in the Krait CPU specific PMU initialization function. Assign it earlier so that we don't crash with a NULL pointer dereference. Signed-off-by: Stephen Boyd Signed-off-by: Will Deacon --- arch/arm/kernel/perf_event_cpu.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index 6efd8aab15df..68d02ca0ca1b 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -311,6 +311,9 @@ static int cpu_pmu_device_probe(struct platform_device *pdev) return -ENOMEM; } + cpu_pmu = pmu; + cpu_pmu->plat_device = pdev; + if (node && (of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node))) { init_fn = of_id->data; ret = init_fn(pmu); @@ -323,8 +326,6 @@ static int cpu_pmu_device_probe(struct platform_device *pdev) goto out_free; } - cpu_pmu = pmu; - cpu_pmu->plat_device = pdev; cpu_pmu_init(cpu_pmu); ret = armpmu_register(cpu_pmu, PERF_TYPE_RAW); From 2a3391cdb3c7bbea3be5cb39279764843a15ef25 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Fri, 7 Feb 2014 21:01:21 +0000 Subject: [PATCH 19/89] ARM: perf: add basic support for Krait CPU PMUs Add basic support for the Krait CPU PMU. This allows us to use the architected functionality of the PMU. This is based on code originally written by Ashwin Chaugule and Neil Leeder [1]. [1] https://www.codeaurora.org/cgit/quic/la/kernel/msm/tree/arch/arm/kernel/perf_event_msm_krait.c?h=msm-3.4 Cc: Neil Leeder Cc: Ashwin Chaugule Signed-off-by: Stephen Boyd Signed-off-by: Will Deacon --- arch/arm/kernel/perf_event_cpu.c | 1 + arch/arm/kernel/perf_event_v7.c | 164 +++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index 68d02ca0ca1b..ed571d386c0b 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -229,6 +229,7 @@ static struct of_device_id cpu_pmu_of_device_ids[] = { {.compatible = "arm,arm11mpcore-pmu", .data = armv6mpcore_pmu_init}, {.compatible = "arm,arm1176-pmu", .data = armv6pmu_init}, {.compatible = "arm,arm1136-pmu", .data = armv6pmu_init}, + {.compatible = "qcom,krait-pmu", .data = krait_pmu_init}, {}, }; diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index 039cffb053a7..16386b1d27a8 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -731,6 +731,138 @@ static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] }, }; +/* + * Krait HW events mapping + */ +static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, + [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, +}; + +static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, + [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, +}; + +static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + /* + * The performance counters don't differentiate between read + * and write accesses/misses so this isn't strictly correct, + * but it's the best we can do. Writes and reads get + * combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + /* * Perf Events' indices */ @@ -1212,6 +1344,18 @@ static int armv7_a7_map_event(struct perf_event *event) &armv7_a7_perf_cache_map, 0xFF); } +static int krait_map_event(struct perf_event *event) +{ + return armpmu_map_event(event, &krait_perf_map, + &krait_perf_cache_map, 0xFFFFF); +} + +static int krait_map_event_no_branch(struct perf_event *event) +{ + return armpmu_map_event(event, &krait_perf_map_no_branch, + &krait_perf_cache_map, 0xFFFFF); +} + static void armv7pmu_init(struct arm_pmu *cpu_pmu) { cpu_pmu->handle_irq = armv7pmu_handle_irq; @@ -1283,6 +1427,21 @@ static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) cpu_pmu->set_event_filter = armv7pmu_set_event_filter; return 0; } + +static int krait_pmu_init(struct arm_pmu *cpu_pmu) +{ + armv7pmu_init(cpu_pmu); + cpu_pmu->name = "ARMv7 Krait"; + /* Some early versions of Krait don't support PC write events */ + if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node, + "qcom,no-pc-write")) + cpu_pmu->map_event = krait_map_event_no_branch; + else + cpu_pmu->map_event = krait_map_event; + cpu_pmu->num_events = armv7_read_num_pmnc_events(); + cpu_pmu->set_event_filter = armv7pmu_set_event_filter; + return 0; +} #else static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) { @@ -1308,4 +1467,9 @@ static inline int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) { return -ENODEV; } + +static inline int krait_pmu_init(struct arm_pmu *cpu_pmu) +{ + return -ENODEV; +} #endif /* CONFIG_CPU_V7 */ From eab443ef391d18772710dc2c156f7ee05e51f754 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Fri, 7 Feb 2014 21:01:22 +0000 Subject: [PATCH 20/89] ARM: perf: add hook for event index clearing On Krait processors we have a many-to-one relationship between raw CPU events and the event programmed into the PMNx counter. Two raw CPU events could map to the same value programmed in the PMNx counter. To avoid this problem, we check for collisions during the get_event_idx() callback by setting a bit in a bitmap whenever a certain event is used in a PMNx counter (see the next patch). Unfortunately, we don't have a hook to clear this bit in the bitmap when the event is deleted so let's add an optional clear_event_idx() callback for this purpose. Signed-off-by: Stephen Boyd Signed-off-by: Will Deacon --- arch/arm/include/asm/pmu.h | 2 ++ arch/arm/kernel/perf_event.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/arm/include/asm/pmu.h b/arch/arm/include/asm/pmu.h index f24edad26c70..ae1919be8f98 100644 --- a/arch/arm/include/asm/pmu.h +++ b/arch/arm/include/asm/pmu.h @@ -71,6 +71,8 @@ struct arm_pmu { void (*disable)(struct perf_event *event); int (*get_event_idx)(struct pmu_hw_events *hw_events, struct perf_event *event); + void (*clear_event_idx)(struct pmu_hw_events *hw_events, + struct perf_event *event); int (*set_event_filter)(struct hw_perf_event *evt, struct perf_event_attr *attr); u32 (*read_counter)(struct perf_event *event); diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index b0c8489018d3..361a1aaee7c8 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c @@ -207,6 +207,8 @@ armpmu_del(struct perf_event *event, int flags) armpmu_stop(event, PERF_EF_UPDATE); hw_events->events[idx] = NULL; clear_bit(idx, hw_events->used_mask); + if (armpmu->clear_event_idx) + armpmu->clear_event_idx(hw_events, event); perf_event_update_userpage(event); } From b7aafe9928beda2ed2c1a3035a1bed878f0cb2ab Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Fri, 7 Feb 2014 13:01:23 -0800 Subject: [PATCH 21/89] ARM: perf: fully support Krait CPU PMU events Krait supports a set of performance monitor region event selection registers (PMRESR) sitting behind a cp15 based interface that extend the architected PMU events to include Krait CPU and Venum VFP specific events. To use these events the user is expected to program the region register (PMRESRn) with the event code shifted into the group they care about and then point the PMNx event at that region+group combo by writing a PMRESRn_GROUPx event. Add support for this hardware. Note: the raw event number is a pure software construct that allows us to map the multi-dimensional number space of regions, groups, and event codes into a flat event number space suitable for use by the perf framework. This is based on code originally written by Ashwin Chaugule and Neil Leeder [1]. [1] https://www.codeaurora.org/cgit/quic/la/kernel/msm/tree/arch/arm/kernel/perf_event_msm_krait.c?h=msm-3.4 Cc: Neil Leeder Cc: Ashwin Chaugule Signed-off-by: Stephen Boyd Signed-off-by: Will Deacon --- arch/arm/kernel/perf_event_v7.c | 407 +++++++++++++++++++++++++++++++- 1 file changed, 401 insertions(+), 6 deletions(-) diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index 16386b1d27a8..2fce4751f4c0 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -18,6 +18,10 @@ #ifdef CONFIG_CPU_V7 +#include +#include +#include "../vfp/vfpinstr.h" + /* * Common ARMv7 event types * @@ -109,6 +113,20 @@ enum armv7_a15_perf_types { ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76, }; +/* ARMv7 Krait specific event types */ +enum krait_perf_types { + KRAIT_PMRESR0_GROUP0 = 0xcc, + KRAIT_PMRESR1_GROUP0 = 0xd0, + KRAIT_PMRESR2_GROUP0 = 0xd4, + KRAIT_VPMRESR0_GROUP0 = 0xd8, + + KRAIT_PERFCTR_L1_ICACHE_ACCESS = 0x10011, + KRAIT_PERFCTR_L1_ICACHE_MISS = 0x10010, + + KRAIT_PERFCTR_L1_ITLB_ACCESS = 0x12222, + KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210, +}; + /* * Cortex-A8 HW events mapping * @@ -779,8 +797,8 @@ static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] }, [C(L1I)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS, + [C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, @@ -807,11 +825,11 @@ static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] }, [C(DTLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, [C(OP_PREFETCH)] = { @@ -821,11 +839,11 @@ static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] }, [C(ITLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, [C(OP_PREFETCH)] = { @@ -1428,6 +1446,378 @@ static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) return 0; } +/* + * Krait Performance Monitor Region Event Selection Register (PMRESRn) + * + * 31 30 24 16 8 0 + * +--------------------------------+ + * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0 + * +--------------------------------+ + * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1 + * +--------------------------------+ + * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2 + * +--------------------------------+ + * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ? + * +--------------------------------+ + * EN | G=3 | G=2 | G=1 | G=0 + * + * Event Encoding: + * + * hwc->config_base = 0xNRCCG + * + * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR) + * R = region register + * CC = class of events the group G is choosing from + * G = group or particular event + * + * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2 + * + * A region (R) corresponds to a piece of the CPU (execution unit, instruction + * unit, etc.) while the event code (CC) corresponds to a particular class of + * events (interrupts for example). An event code is broken down into + * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for + * example). + */ + +#define KRAIT_EVENT (1 << 16) +#define VENUM_EVENT (2 << 16) +#define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT) +#define PMRESRn_EN BIT(31) + +static u32 krait_read_pmresrn(int n) +{ + u32 val; + + switch (n) { + case 0: + asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val)); + break; + case 1: + asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val)); + break; + case 2: + asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val)); + break; + default: + BUG(); /* Should be validated in krait_pmu_get_event_idx() */ + } + + return val; +} + +static void krait_write_pmresrn(int n, u32 val) +{ + switch (n) { + case 0: + asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val)); + break; + case 1: + asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val)); + break; + case 2: + asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val)); + break; + default: + BUG(); /* Should be validated in krait_pmu_get_event_idx() */ + } +} + +static u32 krait_read_vpmresr0(void) +{ + u32 val; + asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); + return val; +} + +static void krait_write_vpmresr0(u32 val) +{ + asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); +} + +static void krait_pre_vpmresr0(u32 *venum_orig_val, u32 *fp_orig_val) +{ + u32 venum_new_val; + u32 fp_new_val; + + BUG_ON(preemptible()); + /* CPACR Enable CP10 and CP11 access */ + *venum_orig_val = get_copro_access(); + venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11); + set_copro_access(venum_new_val); + + /* Enable FPEXC */ + *fp_orig_val = fmrx(FPEXC); + fp_new_val = *fp_orig_val | FPEXC_EN; + fmxr(FPEXC, fp_new_val); +} + +static void krait_post_vpmresr0(u32 venum_orig_val, u32 fp_orig_val) +{ + BUG_ON(preemptible()); + /* Restore FPEXC */ + fmxr(FPEXC, fp_orig_val); + isb(); + /* Restore CPACR */ + set_copro_access(venum_orig_val); +} + +static u32 krait_get_pmresrn_event(unsigned int region) +{ + static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0, + KRAIT_PMRESR1_GROUP0, + KRAIT_PMRESR2_GROUP0 }; + return pmresrn_table[region]; +} + +static void krait_evt_setup(int idx, u32 config_base) +{ + u32 val; + u32 mask; + u32 vval, fval; + unsigned int region; + unsigned int group; + unsigned int code; + unsigned int group_shift; + bool venum_event; + + venum_event = !!(config_base & VENUM_EVENT); + region = (config_base >> 12) & 0xf; + code = (config_base >> 4) & 0xff; + group = (config_base >> 0) & 0xf; + + group_shift = group * 8; + mask = 0xff << group_shift; + + /* Configure evtsel for the region and group */ + if (venum_event) + val = KRAIT_VPMRESR0_GROUP0; + else + val = krait_get_pmresrn_event(region); + val += group; + /* Mix in mode-exclusion bits */ + val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); + armv7_pmnc_write_evtsel(idx, val); + + asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); + + if (venum_event) { + krait_pre_vpmresr0(&vval, &fval); + val = krait_read_vpmresr0(); + val &= ~mask; + val |= code << group_shift; + val |= PMRESRn_EN; + krait_write_vpmresr0(val); + krait_post_vpmresr0(vval, fval); + } else { + val = krait_read_pmresrn(region); + val &= ~mask; + val |= code << group_shift; + val |= PMRESRn_EN; + krait_write_pmresrn(region, val); + } +} + +static u32 krait_clear_pmresrn_group(u32 val, int group) +{ + u32 mask; + int group_shift; + + group_shift = group * 8; + mask = 0xff << group_shift; + val &= ~mask; + + /* Don't clear enable bit if entire region isn't disabled */ + if (val & ~PMRESRn_EN) + return val |= PMRESRn_EN; + + return 0; +} + +static void krait_clearpmu(u32 config_base) +{ + u32 val; + u32 vval, fval; + unsigned int region; + unsigned int group; + bool venum_event; + + venum_event = !!(config_base & VENUM_EVENT); + region = (config_base >> 12) & 0xf; + group = (config_base >> 0) & 0xf; + + if (venum_event) { + krait_pre_vpmresr0(&vval, &fval); + val = krait_read_vpmresr0(); + val = krait_clear_pmresrn_group(val, group); + krait_write_vpmresr0(val); + krait_post_vpmresr0(vval, fval); + } else { + val = krait_read_pmresrn(region); + val = krait_clear_pmresrn_group(val, group); + krait_write_pmresrn(region, val); + } +} + +static void krait_pmu_disable_event(struct perf_event *event) +{ + unsigned long flags; + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + struct pmu_hw_events *events = cpu_pmu->get_hw_events(); + + /* Disable counter and interrupt */ + raw_spin_lock_irqsave(&events->pmu_lock, flags); + + /* Disable counter */ + armv7_pmnc_disable_counter(idx); + + /* + * Clear pmresr code (if destined for PMNx counters) + */ + if (hwc->config_base & KRAIT_EVENT_MASK) + krait_clearpmu(hwc->config_base); + + /* Disable interrupt for this counter */ + armv7_pmnc_disable_intens(idx); + + raw_spin_unlock_irqrestore(&events->pmu_lock, flags); +} + +static void krait_pmu_enable_event(struct perf_event *event) +{ + unsigned long flags; + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + struct pmu_hw_events *events = cpu_pmu->get_hw_events(); + + /* + * Enable counter and interrupt, and set the counter to count + * the event that we're interested in. + */ + raw_spin_lock_irqsave(&events->pmu_lock, flags); + + /* Disable counter */ + armv7_pmnc_disable_counter(idx); + + /* + * Set event (if destined for PMNx counters) + * We set the event for the cycle counter because we + * have the ability to perform event filtering. + */ + if (hwc->config_base & KRAIT_EVENT_MASK) + krait_evt_setup(idx, hwc->config_base); + else + armv7_pmnc_write_evtsel(idx, hwc->config_base); + + /* Enable interrupt for this counter */ + armv7_pmnc_enable_intens(idx); + + /* Enable counter */ + armv7_pmnc_enable_counter(idx); + + raw_spin_unlock_irqrestore(&events->pmu_lock, flags); +} + +static void krait_pmu_reset(void *info) +{ + u32 vval, fval; + + armv7pmu_reset(info); + + /* Clear all pmresrs */ + krait_write_pmresrn(0, 0); + krait_write_pmresrn(1, 0); + krait_write_pmresrn(2, 0); + + krait_pre_vpmresr0(&vval, &fval); + krait_write_vpmresr0(0); + krait_post_vpmresr0(vval, fval); +} + +static int krait_event_to_bit(struct perf_event *event, unsigned int region, + unsigned int group) +{ + int bit; + struct hw_perf_event *hwc = &event->hw; + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); + + if (hwc->config_base & VENUM_EVENT) + bit = KRAIT_VPMRESR0_GROUP0; + else + bit = krait_get_pmresrn_event(region); + bit -= krait_get_pmresrn_event(0); + bit += group; + /* + * Lower bits are reserved for use by the counters (see + * armv7pmu_get_event_idx() for more info) + */ + bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; + + return bit; +} + +/* + * We check for column exclusion constraints here. + * Two events cant use the same group within a pmresr register. + */ +static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc, + struct perf_event *event) +{ + int idx; + int bit; + unsigned int prefix; + unsigned int region; + unsigned int code; + unsigned int group; + bool krait_event; + struct hw_perf_event *hwc = &event->hw; + + region = (hwc->config_base >> 12) & 0xf; + code = (hwc->config_base >> 4) & 0xff; + group = (hwc->config_base >> 0) & 0xf; + krait_event = !!(hwc->config_base & KRAIT_EVENT_MASK); + + if (krait_event) { + /* Ignore invalid events */ + if (group > 3 || region > 2) + return -EINVAL; + prefix = hwc->config_base & KRAIT_EVENT_MASK; + if (prefix != KRAIT_EVENT && prefix != VENUM_EVENT) + return -EINVAL; + if (prefix == VENUM_EVENT && (code & 0xe0)) + return -EINVAL; + + bit = krait_event_to_bit(event, region, group); + if (test_and_set_bit(bit, cpuc->used_mask)) + return -EAGAIN; + } + + idx = armv7pmu_get_event_idx(cpuc, event); + if (idx < 0 && krait_event) + clear_bit(bit, cpuc->used_mask); + + return idx; +} + +static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc, + struct perf_event *event) +{ + int bit; + struct hw_perf_event *hwc = &event->hw; + unsigned int region; + unsigned int group; + bool krait_event; + + region = (hwc->config_base >> 12) & 0xf; + group = (hwc->config_base >> 0) & 0xf; + krait_event = !!(hwc->config_base & KRAIT_EVENT_MASK); + + if (krait_event) { + bit = krait_event_to_bit(event, region, group); + clear_bit(bit, cpuc->used_mask); + } +} + static int krait_pmu_init(struct arm_pmu *cpu_pmu) { armv7pmu_init(cpu_pmu); @@ -1440,6 +1830,11 @@ static int krait_pmu_init(struct arm_pmu *cpu_pmu) cpu_pmu->map_event = krait_map_event; cpu_pmu->num_events = armv7_read_num_pmnc_events(); cpu_pmu->set_event_filter = armv7pmu_set_event_filter; + cpu_pmu->reset = krait_pmu_reset; + cpu_pmu->enable = krait_pmu_enable_event; + cpu_pmu->disable = krait_pmu_disable_event; + cpu_pmu->get_event_idx = krait_pmu_get_event_idx; + cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx; return 0; } #else From d84c47837d800c0db1c9f2015a1565504417f812 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Fri, 7 Feb 2014 21:01:24 +0000 Subject: [PATCH 22/89] devicetree: bindings: Document Krait performance monitor units (PMU) Document the Krait PMU compatible string. Cc: Signed-off-by: Stephen Boyd Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/arm/pmu.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/arm/pmu.txt b/Documentation/devicetree/bindings/arm/pmu.txt index 3e1e498fea96..ce731441e64f 100644 --- a/Documentation/devicetree/bindings/arm/pmu.txt +++ b/Documentation/devicetree/bindings/arm/pmu.txt @@ -16,7 +16,14 @@ Required properties: "arm,arm11mpcore-pmu" "arm,arm1176-pmu" "arm,arm1136-pmu" -- interrupts : 1 combined interrupt or 1 per core. + "qcom,krait-pmu" +- interrupts : 1 combined interrupt or 1 per core. If the interrupt is a per-cpu + interrupt (PPI) then 1 interrupt should be specified. + +Optional properties: + +- qcom,no-pc-write : Indicates that this PMU doesn't support the 0xc and 0xd + events. Example: From 5f5092e72cc25a6a5785308270e0085b2b2772cc Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 11 Feb 2014 18:08:41 +0000 Subject: [PATCH 23/89] ARM: perf: hook up perf_sample_event_took around pmu irq handling Since we indirect all of our PMU IRQ handling through a dispatcher, it's trivial to hook up perf_sample_event_took to prevent applications such as oprofile from generating interrupt storms due to an unrealisticly low sample period. Reported-by: Robert Richter Signed-off-by: Will Deacon --- arch/arm/kernel/perf_event.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index 361a1aaee7c8..a6bc431cde70 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c @@ -302,6 +302,8 @@ static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) struct arm_pmu *armpmu; struct platform_device *plat_device; struct arm_pmu_platdata *plat; + int ret; + u64 start_clock, finish_clock; if (irq_is_percpu(irq)) dev = *(void **)dev; @@ -309,10 +311,15 @@ static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) plat_device = armpmu->plat_device; plat = dev_get_platdata(&plat_device->dev); + start_clock = sched_clock(); if (plat && plat->handle_irq) - return plat->handle_irq(irq, dev, armpmu->handle_irq); + ret = plat->handle_irq(irq, dev, armpmu->handle_irq); else - return armpmu->handle_irq(irq, dev); + ret = armpmu->handle_irq(irq, dev); + finish_clock = sched_clock(); + + perf_sample_event_took(finish_clock - start_clock); + return ret; } static void From 51e5fef62059e4074054ee731385e0dc7ca561ff Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 18 Feb 2014 10:31:10 +0000 Subject: [PATCH 24/89] ARM: perf: remove unused PMU probing code The ARM perf backend can discover the type of PMU it needs to drive either from DT or by probing a CPU it is running on. For Cortex-A{5,7,15} there are no platforms in mainline not using dt, and this probing won't work well for big.LITTLE systems with heterogeneous PMUs. This patch drops the probing for those CPUs, relying on information from dt instead. Future platforms should describe their PMU(s) with dt. Suggested-by: Will Deacon Signed-off-by: Mark Rutland Signed-off-by: Will Deacon --- arch/arm/kernel/perf_event_cpu.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index ed571d386c0b..326cb58de2f8 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -267,15 +267,6 @@ static int probe_current_pmu(struct arm_pmu *pmu) case ARM_CPU_PART_CORTEX_A9: ret = armv7_a9_pmu_init(pmu); break; - case ARM_CPU_PART_CORTEX_A5: - ret = armv7_a5_pmu_init(pmu); - break; - case ARM_CPU_PART_CORTEX_A15: - ret = armv7_a15_pmu_init(pmu); - break; - case ARM_CPU_PART_CORTEX_A7: - ret = armv7_a7_pmu_init(pmu); - break; } /* Intel CPUs [xscale]. */ } else if (implementor == ARM_CPU_IMP_INTEL) { From 8e781f65423c2e8e65a56972ba996b6c01a5ef3e Mon Sep 17 00:00:00 2001 From: Albin Tonnerre Date: Wed, 29 Jan 2014 14:28:57 +0000 Subject: [PATCH 25/89] ARM: perf: add support for the Cortex-A12 PMU Cortex-A12 implements Performance Monitors compliant with the PMUv2 architecture. This patch adds support for the Cortex-A12 PMU to the ARM perf backend. Signed-off-by: Albin Tonnerre Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/arm/pmu.txt | 1 + arch/arm/kernel/perf_event_cpu.c | 1 + arch/arm/kernel/perf_event_v7.c | 158 ++++++++++++++++++ 3 files changed, 160 insertions(+) diff --git a/Documentation/devicetree/bindings/arm/pmu.txt b/Documentation/devicetree/bindings/arm/pmu.txt index ce731441e64f..fe5cef8976cb 100644 --- a/Documentation/devicetree/bindings/arm/pmu.txt +++ b/Documentation/devicetree/bindings/arm/pmu.txt @@ -9,6 +9,7 @@ Required properties: - compatible : should be one of "arm,armv8-pmuv3" "arm,cortex-a15-pmu" + "arm,cortex-a12-pmu" "arm,cortex-a9-pmu" "arm,cortex-a8-pmu" "arm,cortex-a7-pmu" diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index 326cb58de2f8..51798d7854ac 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -222,6 +222,7 @@ static struct notifier_block cpu_pmu_hotplug_notifier = { */ static struct of_device_id cpu_pmu_of_device_ids[] = { {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init}, + {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init}, {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init}, {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init}, {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init}, diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index 2fce4751f4c0..f4ef3981ed02 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -113,6 +113,19 @@ enum armv7_a15_perf_types { ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76, }; +/* ARMv7 Cortex-A12 specific event types */ +enum armv7_a12_perf_types { + ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40, + ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41, + + ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ = 0x50, + ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51, + + ARMV7_A12_PERFCTR_PC_WRITE_SPEC = 0x76, + + ARMV7_A12_PERFCTR_PF_TLB_REFILL = 0xe7, +}; + /* ARMv7 Krait specific event types */ enum krait_perf_types { KRAIT_PMRESR0_GROUP0 = 0xcc, @@ -749,6 +762,130 @@ static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] }, }; +/* + * Cortex-A12 HW events mapping + */ +static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, + [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, + [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, + [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED, +}; + +static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + /* + * Not all performance counters differentiate between read + * and write accesses/misses so we're not always strictly + * correct, but it's the best we can do. Writes and reads get + * combined in these cases. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + /* * Krait HW events mapping */ @@ -1362,6 +1499,12 @@ static int armv7_a7_map_event(struct perf_event *event) &armv7_a7_perf_cache_map, 0xFF); } +static int armv7_a12_map_event(struct perf_event *event) +{ + return armpmu_map_event(event, &armv7_a12_perf_map, + &armv7_a12_perf_cache_map, 0xFF); +} + static int krait_map_event(struct perf_event *event) { return armpmu_map_event(event, &krait_perf_map, @@ -1446,6 +1589,16 @@ static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) return 0; } +static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) +{ + armv7pmu_init(cpu_pmu); + cpu_pmu->name = "ARMv7 Cortex-A12"; + cpu_pmu->map_event = armv7_a12_map_event; + cpu_pmu->num_events = armv7_read_num_pmnc_events(); + cpu_pmu->set_event_filter = armv7pmu_set_event_filter; + return 0; +} + /* * Krait Performance Monitor Region Event Selection Register (PMRESRn) * @@ -1863,6 +2016,11 @@ static inline int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) return -ENODEV; } +static inline int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) +{ + return -ENODEV; +} + static inline int krait_pmu_init(struct arm_pmu *cpu_pmu) { return -ENODEV; From c32ffce0f66e5d1d4856254516e24f5ef275cd00 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 21 Feb 2014 17:01:48 +0100 Subject: [PATCH 26/89] ARM: 7984/1: prefetch: add prefetchw invocations for barriered atomics After a bunch of benchmarking on the interaction between dmb and pldw, it turns out that issuing the pldw *after* the dmb instruction can give modest performance gains (~3% atomic_add_return improvement on a dual A15). This patch adds prefetchw invocations to our barriered atomic operations including cmpxchg, test_and_xxx and futexes. Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/include/asm/atomic.h | 9 +++++++++ arch/arm/include/asm/cmpxchg.h | 6 ++++++ arch/arm/include/asm/futex.h | 3 +++ arch/arm/lib/bitops.h | 5 +++++ 4 files changed, 23 insertions(+) diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 62d2cb53b069..6e410090896e 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -60,6 +60,7 @@ static inline int atomic_add_return(int i, atomic_t *v) int result; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic_add_return\n" "1: ldrex %0, [%3]\n" @@ -99,6 +100,7 @@ static inline int atomic_sub_return(int i, atomic_t *v) int result; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic_sub_return\n" "1: ldrex %0, [%3]\n" @@ -121,6 +123,7 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) unsigned long res; smp_mb(); + prefetchw(&ptr->counter); do { __asm__ __volatile__("@ atomic_cmpxchg\n" @@ -299,6 +302,7 @@ static inline long long atomic64_add_return(long long i, atomic64_t *v) unsigned long tmp; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic64_add_return\n" "1: ldrexd %0, %H0, [%3]\n" @@ -340,6 +344,7 @@ static inline long long atomic64_sub_return(long long i, atomic64_t *v) unsigned long tmp; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic64_sub_return\n" "1: ldrexd %0, %H0, [%3]\n" @@ -364,6 +369,7 @@ static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old, unsigned long res; smp_mb(); + prefetchw(&ptr->counter); do { __asm__ __volatile__("@ atomic64_cmpxchg\n" @@ -388,6 +394,7 @@ static inline long long atomic64_xchg(atomic64_t *ptr, long long new) unsigned long tmp; smp_mb(); + prefetchw(&ptr->counter); __asm__ __volatile__("@ atomic64_xchg\n" "1: ldrexd %0, %H0, [%3]\n" @@ -409,6 +416,7 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) unsigned long tmp; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic64_dec_if_positive\n" "1: ldrexd %0, %H0, [%3]\n" @@ -436,6 +444,7 @@ static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) int ret = 1; smp_mb(); + prefetchw(&v->counter); __asm__ __volatile__("@ atomic64_add_unless\n" "1: ldrexd %0, %H0, [%4]\n" diff --git a/arch/arm/include/asm/cmpxchg.h b/arch/arm/include/asm/cmpxchg.h index df2fbba7efc8..abb2c3769b01 100644 --- a/arch/arm/include/asm/cmpxchg.h +++ b/arch/arm/include/asm/cmpxchg.h @@ -2,6 +2,7 @@ #define __ASM_ARM_CMPXCHG_H #include +#include #include #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) @@ -35,6 +36,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size #endif smp_mb(); + prefetchw((const void *)ptr); switch (size) { #if __LINUX_ARM_ARCH__ >= 6 @@ -138,6 +140,8 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, { unsigned long oldval, res; + prefetchw((const void *)ptr); + switch (size) { #ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ case 1: @@ -230,6 +234,8 @@ static inline unsigned long long __cmpxchg64(unsigned long long *ptr, unsigned long long oldval; unsigned long res; + prefetchw(ptr); + __asm__ __volatile__( "1: ldrexd %1, %H1, [%3]\n" " teq %1, %4\n" diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 2aff798fbef4..53e69dae796f 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -23,6 +23,7 @@ #define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \ smp_mb(); \ + prefetchw(uaddr); \ __asm__ __volatile__( \ "1: ldrex %1, [%3]\n" \ " " insn "\n" \ @@ -46,6 +47,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return -EFAULT; smp_mb(); + /* Prefetching cannot fault */ + prefetchw(uaddr); __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" "1: ldrex %1, [%4]\n" " teq %1, %2\n" diff --git a/arch/arm/lib/bitops.h b/arch/arm/lib/bitops.h index 52886b89706c..9f12ed1eea86 100644 --- a/arch/arm/lib/bitops.h +++ b/arch/arm/lib/bitops.h @@ -37,6 +37,11 @@ UNWIND( .fnstart ) add r1, r1, r0, lsl #2 @ Get word offset mov r3, r2, lsl r3 @ create mask smp_dmb +#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP) + .arch_extension mp + ALT_SMP(W(pldw) [r1]) + ALT_UP(W(nop)) +#endif 1: ldrex r2, [r1] ands r0, r2, r3 @ save old value of bit \instr r2, r2, r3 @ toggle bit From 1971188aa19651d8f447211c6535fb68661d77c5 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 21 Feb 2014 17:01:48 +0100 Subject: [PATCH 27/89] ARM: 7985/1: mm: implement pte_accessible for faulting mappings The pte_accessible macro can be used to identify page table entries capable of being cached by a TLB. In principle, this differs from pte_present, since PROT_NONE mappings are mapped using invalid entries identified as present and ptes designated as `old' can use either invalid entries or those with the access flag cleared (guaranteed not to be in the TLB). However, there is a race to take care of, as described in 20841405940e ("mm: fix TLB flush race between migration, and change_protection_range"), between a page being migrated and mprotected at the same time. In this case, we can check whether a TLB invalidation is pending for the mm and if so, temporarily consider PROT_NONE mappings as valid. This patch implements a quick pte_accessible macro for ARM by simply checking if the pte is valid/present depending on the mm. For classic MMU, these checks are identical and will generate some false positives for PROT_NONE mappings, but this is better than the current asm-generic definition of ((void)(pte),1). Finally, pte_present_user is moved to use pte_valid (and renamed appropriately) since we don't care about cache flushing for faulting mappings. Acked-by: Steve Capper Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/include/asm/pgtable.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 7d59b524f2af..5478e5d6ad89 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -216,13 +216,16 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pte_none(pte) (!pte_val(pte)) #define pte_present(pte) (pte_val(pte) & L_PTE_PRESENT) +#define pte_valid(pte) (pte_val(pte) & L_PTE_VALID) +#define pte_accessible(mm, pte) (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte)) #define pte_write(pte) (!(pte_val(pte) & L_PTE_RDONLY)) #define pte_dirty(pte) (pte_val(pte) & L_PTE_DIRTY) #define pte_young(pte) (pte_val(pte) & L_PTE_YOUNG) #define pte_exec(pte) (!(pte_val(pte) & L_PTE_XN)) #define pte_special(pte) (0) -#define pte_present_user(pte) (pte_present(pte) && (pte_val(pte) & L_PTE_USER)) +#define pte_valid_user(pte) \ + (pte_valid(pte) && (pte_val(pte) & L_PTE_USER) && pte_young(pte)) #if __LINUX_ARM_ARCH__ < 6 static inline void __sync_icache_dcache(pte_t pteval) @@ -237,7 +240,7 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, { unsigned long ext = 0; - if (addr < TASK_SIZE && pte_present_user(pteval)) { + if (addr < TASK_SIZE && pte_valid_user(pteval)) { __sync_icache_dcache(pteval); ext |= PTE_EXT_NG; } From 74c4137b2a9c73e7e6887ea1bac93d7012827012 Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 24 Feb 2014 16:48:26 +0100 Subject: [PATCH 28/89] ARM: 7989/1: Delete asm/system.h Delete ARM's asm/system.h. It's the last holdout and should be got rid of. This builds for defconfig, lpc32xx_defconfig, exynos_defconfig + XEN, the previous changed to a Gemini system and an omap3 config with TI_DAVINCI_EMAC. Signed-off-by: David Howells Acked-by: Arnd Bergmann Cc: linux-arm-kernel@lists.infradead.org Signed-off-by: Russell King --- arch/arm/include/asm/jump_label.h | 1 - arch/arm/include/asm/sync_bitops.h | 1 - arch/arm/include/asm/system.h | 7 ------- arch/arm/mach-gemini/idle.c | 2 +- arch/arm/mach-omap2/am35xx-emac.c | 1 - drivers/usb/gadget/lpc32xx_udc.c | 1 - 6 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 arch/arm/include/asm/system.h diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h index 863c892b4aaa..70f9b9bfb1f9 100644 --- a/arch/arm/include/asm/jump_label.h +++ b/arch/arm/include/asm/jump_label.h @@ -4,7 +4,6 @@ #ifdef __KERNEL__ #include -#include #define JUMP_LABEL_NOP_SIZE 4 diff --git a/arch/arm/include/asm/sync_bitops.h b/arch/arm/include/asm/sync_bitops.h index 63479eecbf76..9732b8e11e63 100644 --- a/arch/arm/include/asm/sync_bitops.h +++ b/arch/arm/include/asm/sync_bitops.h @@ -2,7 +2,6 @@ #define __ASM_SYNC_BITOPS_H__ #include -#include /* sync_bitops functions are equivalent to the SMP implementation of the * original functions, independently from CONFIG_SMP being defined. diff --git a/arch/arm/include/asm/system.h b/arch/arm/include/asm/system.h deleted file mode 100644 index 368165e33c1c..000000000000 --- a/arch/arm/include/asm/system.h +++ /dev/null @@ -1,7 +0,0 @@ -/* FILE TO BE DELETED. DO NOT ADD STUFF HERE! */ -#include -#include -#include -#include -#include -#include diff --git a/arch/arm/mach-gemini/idle.c b/arch/arm/mach-gemini/idle.c index 87dff4f5059e..ddf8ec9d203b 100644 --- a/arch/arm/mach-gemini/idle.c +++ b/arch/arm/mach-gemini/idle.c @@ -3,7 +3,7 @@ */ #include -#include +#include #include static void gemini_idle(void) diff --git a/arch/arm/mach-omap2/am35xx-emac.c b/arch/arm/mach-omap2/am35xx-emac.c index 25b79a297365..6a6935caac1e 100644 --- a/arch/arm/mach-omap2/am35xx-emac.c +++ b/arch/arm/mach-omap2/am35xx-emac.c @@ -17,7 +17,6 @@ #include #include -#include #include "omap_device.h" #include "am35xx.h" #include "control.h" diff --git a/drivers/usb/gadget/lpc32xx_udc.c b/drivers/usb/gadget/lpc32xx_udc.c index 049ebab0d360..a94bb10eeb03 100644 --- a/drivers/usb/gadget/lpc32xx_udc.c +++ b/drivers/usb/gadget/lpc32xx_udc.c @@ -55,7 +55,6 @@ #include #include #include -#include #include #include From d98b90ea22b0a28d9d787769704a9cf1ea5a513a Mon Sep 17 00:00:00 2001 From: Victor Kamensky Date: Tue, 25 Feb 2014 08:41:09 +0100 Subject: [PATCH 29/89] ARM: 7990/1: asm: rename logical shift macros push pull into lspush lspull Renames logical shift macros, 'push' and 'pull', defined in arch/arm/include/asm/assembler.h, into 'lspush' and 'lspull'. That eliminates name conflict between 'push' logical shift macro and 'push' instruction mnemonic. That allows assembler.h to be included in .S files that use 'push' instruction. Suggested-by: Will Deacon Signed-off-by: Victor Kamensky Acked-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/include/asm/assembler.h | 8 +- arch/arm/lib/copy_template.S | 36 ++--- arch/arm/lib/csumpartialcopygeneric.S | 96 ++++++------- arch/arm/lib/io-readsl.S | 12 +- arch/arm/lib/io-writesl.S | 12 +- arch/arm/lib/memmove.S | 36 ++--- arch/arm/lib/uaccess.S | 192 +++++++++++++------------- 7 files changed, 196 insertions(+), 196 deletions(-) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 5c2285160575..380ac4f20000 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -30,8 +30,8 @@ * Endian independent macros for shifting bytes within registers. */ #ifndef __ARMEB__ -#define pull lsr -#define push lsl +#define lspull lsr +#define lspush lsl #define get_byte_0 lsl #0 #define get_byte_1 lsr #8 #define get_byte_2 lsr #16 @@ -41,8 +41,8 @@ #define put_byte_2 lsl #16 #define put_byte_3 lsl #24 #else -#define pull lsl -#define push lsr +#define lspull lsl +#define lspush lsr #define get_byte_0 lsr #24 #define get_byte_1 lsr #16 #define get_byte_2 lsr #8 diff --git a/arch/arm/lib/copy_template.S b/arch/arm/lib/copy_template.S index 805e3f8fb007..3bc8eb811a73 100644 --- a/arch/arm/lib/copy_template.S +++ b/arch/arm/lib/copy_template.S @@ -197,24 +197,24 @@ 12: PLD( pld [r1, #124] ) 13: ldr4w r1, r4, r5, r6, r7, abort=19f - mov r3, lr, pull #\pull + mov r3, lr, lspull #\pull subs r2, r2, #32 ldr4w r1, r8, r9, ip, lr, abort=19f - orr r3, r3, r4, push #\push - mov r4, r4, pull #\pull - orr r4, r4, r5, push #\push - mov r5, r5, pull #\pull - orr r5, r5, r6, push #\push - mov r6, r6, pull #\pull - orr r6, r6, r7, push #\push - mov r7, r7, pull #\pull - orr r7, r7, r8, push #\push - mov r8, r8, pull #\pull - orr r8, r8, r9, push #\push - mov r9, r9, pull #\pull - orr r9, r9, ip, push #\push - mov ip, ip, pull #\pull - orr ip, ip, lr, push #\push + orr r3, r3, r4, lspush #\push + mov r4, r4, lspull #\pull + orr r4, r4, r5, lspush #\push + mov r5, r5, lspull #\pull + orr r5, r5, r6, lspush #\push + mov r6, r6, lspull #\pull + orr r6, r6, r7, lspush #\push + mov r7, r7, lspull #\pull + orr r7, r7, r8, lspush #\push + mov r8, r8, lspull #\pull + orr r8, r8, r9, lspush #\push + mov r9, r9, lspull #\pull + orr r9, r9, ip, lspush #\push + mov ip, ip, lspull #\pull + orr ip, ip, lr, lspush #\push str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f bge 12b PLD( cmn r2, #96 ) @@ -225,10 +225,10 @@ 14: ands ip, r2, #28 beq 16f -15: mov r3, lr, pull #\pull +15: mov r3, lr, lspull #\pull ldr1w r1, lr, abort=21f subs ip, ip, #4 - orr r3, r3, lr, push #\push + orr r3, r3, lr, lspush #\push str1w r0, r3, abort=21f bgt 15b CALGN( cmp r2, #0 ) diff --git a/arch/arm/lib/csumpartialcopygeneric.S b/arch/arm/lib/csumpartialcopygeneric.S index d620a5f22a09..d6e742d24007 100644 --- a/arch/arm/lib/csumpartialcopygeneric.S +++ b/arch/arm/lib/csumpartialcopygeneric.S @@ -141,7 +141,7 @@ FN_ENTRY tst len, #2 mov r5, r4, get_byte_0 beq .Lexit - adcs sum, sum, r4, push #16 + adcs sum, sum, r4, lspush #16 strb r5, [dst], #1 mov r5, r4, get_byte_1 strb r5, [dst], #1 @@ -171,23 +171,23 @@ FN_ENTRY cmp ip, #2 beq .Lsrc2_aligned bhi .Lsrc3_aligned - mov r4, r5, pull #8 @ C = 0 + mov r4, r5, lspull #8 @ C = 0 bics ip, len, #15 beq 2f 1: load4l r5, r6, r7, r8 - orr r4, r4, r5, push #24 - mov r5, r5, pull #8 - orr r5, r5, r6, push #24 - mov r6, r6, pull #8 - orr r6, r6, r7, push #24 - mov r7, r7, pull #8 - orr r7, r7, r8, push #24 + orr r4, r4, r5, lspush #24 + mov r5, r5, lspull #8 + orr r5, r5, r6, lspush #24 + mov r6, r6, lspull #8 + orr r6, r6, r7, lspush #24 + mov r7, r7, lspull #8 + orr r7, r7, r8, lspush #24 stmia dst!, {r4, r5, r6, r7} adcs sum, sum, r4 adcs sum, sum, r5 adcs sum, sum, r6 adcs sum, sum, r7 - mov r4, r8, pull #8 + mov r4, r8, lspull #8 sub ip, ip, #16 teq ip, #0 bne 1b @@ -196,50 +196,50 @@ FN_ENTRY tst ip, #8 beq 3f load2l r5, r6 - orr r4, r4, r5, push #24 - mov r5, r5, pull #8 - orr r5, r5, r6, push #24 + orr r4, r4, r5, lspush #24 + mov r5, r5, lspull #8 + orr r5, r5, r6, lspush #24 stmia dst!, {r4, r5} adcs sum, sum, r4 adcs sum, sum, r5 - mov r4, r6, pull #8 + mov r4, r6, lspull #8 tst ip, #4 beq 4f 3: load1l r5 - orr r4, r4, r5, push #24 + orr r4, r4, r5, lspush #24 str r4, [dst], #4 adcs sum, sum, r4 - mov r4, r5, pull #8 + mov r4, r5, lspull #8 4: ands len, len, #3 beq .Ldone mov r5, r4, get_byte_0 tst len, #2 beq .Lexit - adcs sum, sum, r4, push #16 + adcs sum, sum, r4, lspush #16 strb r5, [dst], #1 mov r5, r4, get_byte_1 strb r5, [dst], #1 mov r5, r4, get_byte_2 b .Lexit -.Lsrc2_aligned: mov r4, r5, pull #16 +.Lsrc2_aligned: mov r4, r5, lspull #16 adds sum, sum, #0 bics ip, len, #15 beq 2f 1: load4l r5, r6, r7, r8 - orr r4, r4, r5, push #16 - mov r5, r5, pull #16 - orr r5, r5, r6, push #16 - mov r6, r6, pull #16 - orr r6, r6, r7, push #16 - mov r7, r7, pull #16 - orr r7, r7, r8, push #16 + orr r4, r4, r5, lspush #16 + mov r5, r5, lspull #16 + orr r5, r5, r6, lspush #16 + mov r6, r6, lspull #16 + orr r6, r6, r7, lspush #16 + mov r7, r7, lspull #16 + orr r7, r7, r8, lspush #16 stmia dst!, {r4, r5, r6, r7} adcs sum, sum, r4 adcs sum, sum, r5 adcs sum, sum, r6 adcs sum, sum, r7 - mov r4, r8, pull #16 + mov r4, r8, lspull #16 sub ip, ip, #16 teq ip, #0 bne 1b @@ -248,20 +248,20 @@ FN_ENTRY tst ip, #8 beq 3f load2l r5, r6 - orr r4, r4, r5, push #16 - mov r5, r5, pull #16 - orr r5, r5, r6, push #16 + orr r4, r4, r5, lspush #16 + mov r5, r5, lspull #16 + orr r5, r5, r6, lspush #16 stmia dst!, {r4, r5} adcs sum, sum, r4 adcs sum, sum, r5 - mov r4, r6, pull #16 + mov r4, r6, lspull #16 tst ip, #4 beq 4f 3: load1l r5 - orr r4, r4, r5, push #16 + orr r4, r4, r5, lspush #16 str r4, [dst], #4 adcs sum, sum, r4 - mov r4, r5, pull #16 + mov r4, r5, lspull #16 4: ands len, len, #3 beq .Ldone mov r5, r4, get_byte_0 @@ -276,24 +276,24 @@ FN_ENTRY load1b r5 b .Lexit -.Lsrc3_aligned: mov r4, r5, pull #24 +.Lsrc3_aligned: mov r4, r5, lspull #24 adds sum, sum, #0 bics ip, len, #15 beq 2f 1: load4l r5, r6, r7, r8 - orr r4, r4, r5, push #8 - mov r5, r5, pull #24 - orr r5, r5, r6, push #8 - mov r6, r6, pull #24 - orr r6, r6, r7, push #8 - mov r7, r7, pull #24 - orr r7, r7, r8, push #8 + orr r4, r4, r5, lspush #8 + mov r5, r5, lspull #24 + orr r5, r5, r6, lspush #8 + mov r6, r6, lspull #24 + orr r6, r6, r7, lspush #8 + mov r7, r7, lspull #24 + orr r7, r7, r8, lspush #8 stmia dst!, {r4, r5, r6, r7} adcs sum, sum, r4 adcs sum, sum, r5 adcs sum, sum, r6 adcs sum, sum, r7 - mov r4, r8, pull #24 + mov r4, r8, lspull #24 sub ip, ip, #16 teq ip, #0 bne 1b @@ -302,20 +302,20 @@ FN_ENTRY tst ip, #8 beq 3f load2l r5, r6 - orr r4, r4, r5, push #8 - mov r5, r5, pull #24 - orr r5, r5, r6, push #8 + orr r4, r4, r5, lspush #8 + mov r5, r5, lspull #24 + orr r5, r5, r6, lspush #8 stmia dst!, {r4, r5} adcs sum, sum, r4 adcs sum, sum, r5 - mov r4, r6, pull #24 + mov r4, r6, lspull #24 tst ip, #4 beq 4f 3: load1l r5 - orr r4, r4, r5, push #8 + orr r4, r4, r5, lspush #8 str r4, [dst], #4 adcs sum, sum, r4 - mov r4, r5, pull #24 + mov r4, r5, lspull #24 4: ands len, len, #3 beq .Ldone mov r5, r4, get_byte_0 @@ -326,7 +326,7 @@ FN_ENTRY load1l r4 mov r5, r4, get_byte_0 strb r5, [dst], #1 - adcs sum, sum, r4, push #24 + adcs sum, sum, r4, lspush #24 mov r5, r4, get_byte_1 b .Lexit FN_EXIT diff --git a/arch/arm/lib/io-readsl.S b/arch/arm/lib/io-readsl.S index 5fb97e7f9f4b..7a7430950c79 100644 --- a/arch/arm/lib/io-readsl.S +++ b/arch/arm/lib/io-readsl.S @@ -47,25 +47,25 @@ ENTRY(__raw_readsl) strb ip, [r1], #1 4: subs r2, r2, #1 - mov ip, r3, pull #24 + mov ip, r3, lspull #24 ldrne r3, [r0] - orrne ip, ip, r3, push #8 + orrne ip, ip, r3, lspush #8 strne ip, [r1], #4 bne 4b b 8f 5: subs r2, r2, #1 - mov ip, r3, pull #16 + mov ip, r3, lspull #16 ldrne r3, [r0] - orrne ip, ip, r3, push #16 + orrne ip, ip, r3, lspush #16 strne ip, [r1], #4 bne 5b b 7f 6: subs r2, r2, #1 - mov ip, r3, pull #8 + mov ip, r3, lspull #8 ldrne r3, [r0] - orrne ip, ip, r3, push #24 + orrne ip, ip, r3, lspush #24 strne ip, [r1], #4 bne 6b diff --git a/arch/arm/lib/io-writesl.S b/arch/arm/lib/io-writesl.S index 8d3b7813725c..d0d104a0dd11 100644 --- a/arch/arm/lib/io-writesl.S +++ b/arch/arm/lib/io-writesl.S @@ -41,26 +41,26 @@ ENTRY(__raw_writesl) blt 5f bgt 6f -4: mov ip, r3, pull #16 +4: mov ip, r3, lspull #16 ldr r3, [r1], #4 subs r2, r2, #1 - orr ip, ip, r3, push #16 + orr ip, ip, r3, lspush #16 str ip, [r0] bne 4b mov pc, lr -5: mov ip, r3, pull #8 +5: mov ip, r3, lspull #8 ldr r3, [r1], #4 subs r2, r2, #1 - orr ip, ip, r3, push #24 + orr ip, ip, r3, lspush #24 str ip, [r0] bne 5b mov pc, lr -6: mov ip, r3, pull #24 +6: mov ip, r3, lspull #24 ldr r3, [r1], #4 subs r2, r2, #1 - orr ip, ip, r3, push #8 + orr ip, ip, r3, lspush #8 str ip, [r0] bne 6b mov pc, lr diff --git a/arch/arm/lib/memmove.S b/arch/arm/lib/memmove.S index 938fc14f962d..d1fc0c0c342c 100644 --- a/arch/arm/lib/memmove.S +++ b/arch/arm/lib/memmove.S @@ -147,24 +147,24 @@ ENTRY(memmove) 12: PLD( pld [r1, #-128] ) 13: ldmdb r1!, {r7, r8, r9, ip} - mov lr, r3, push #\push + mov lr, r3, lspush #\push subs r2, r2, #32 ldmdb r1!, {r3, r4, r5, r6} - orr lr, lr, ip, pull #\pull - mov ip, ip, push #\push - orr ip, ip, r9, pull #\pull - mov r9, r9, push #\push - orr r9, r9, r8, pull #\pull - mov r8, r8, push #\push - orr r8, r8, r7, pull #\pull - mov r7, r7, push #\push - orr r7, r7, r6, pull #\pull - mov r6, r6, push #\push - orr r6, r6, r5, pull #\pull - mov r5, r5, push #\push - orr r5, r5, r4, pull #\pull - mov r4, r4, push #\push - orr r4, r4, r3, pull #\pull + orr lr, lr, ip, lspull #\pull + mov ip, ip, lspush #\push + orr ip, ip, r9, lspull #\pull + mov r9, r9, lspush #\push + orr r9, r9, r8, lspull #\pull + mov r8, r8, lspush #\push + orr r8, r8, r7, lspull #\pull + mov r7, r7, lspush #\push + orr r7, r7, r6, lspull #\pull + mov r6, r6, lspush #\push + orr r6, r6, r5, lspull #\pull + mov r5, r5, lspush #\push + orr r5, r5, r4, lspull #\pull + mov r4, r4, lspush #\push + orr r4, r4, r3, lspull #\pull stmdb r0!, {r4 - r9, ip, lr} bge 12b PLD( cmn r2, #96 ) @@ -175,10 +175,10 @@ ENTRY(memmove) 14: ands ip, r2, #28 beq 16f -15: mov lr, r3, push #\push +15: mov lr, r3, lspush #\push ldr r3, [r1, #-4]! subs ip, ip, #4 - orr lr, lr, r3, pull #\pull + orr lr, lr, r3, lspull #\pull str lr, [r0, #-4]! bgt 15b CALGN( cmp r2, #0 ) diff --git a/arch/arm/lib/uaccess.S b/arch/arm/lib/uaccess.S index 5c908b1cb8ed..e50520904b76 100644 --- a/arch/arm/lib/uaccess.S +++ b/arch/arm/lib/uaccess.S @@ -117,9 +117,9 @@ USER( TUSER( strgtb) r3, [r0], #1) @ May fault .Lc2u_1fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lc2u_1nowords - mov r3, r7, pull #8 + mov r3, r7, lspull #8 ldr r7, [r1], #4 - orr r3, r3, r7, push #24 + orr r3, r3, r7, lspush #24 USER( TUSER( str) r3, [r0], #4) @ May fault mov ip, r0, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -131,30 +131,30 @@ USER( TUSER( str) r3, [r0], #4) @ May fault subs ip, ip, #16 blt .Lc2u_1rem8lp -.Lc2u_1cpy8lp: mov r3, r7, pull #8 +.Lc2u_1cpy8lp: mov r3, r7, lspull #8 ldmia r1!, {r4 - r7} subs ip, ip, #16 - orr r3, r3, r4, push #24 - mov r4, r4, pull #8 - orr r4, r4, r5, push #24 - mov r5, r5, pull #8 - orr r5, r5, r6, push #24 - mov r6, r6, pull #8 - orr r6, r6, r7, push #24 + orr r3, r3, r4, lspush #24 + mov r4, r4, lspull #8 + orr r4, r4, r5, lspush #24 + mov r5, r5, lspull #8 + orr r5, r5, r6, lspush #24 + mov r6, r6, lspull #8 + orr r6, r6, r7, lspush #24 stmia r0!, {r3 - r6} @ Shouldnt fault bpl .Lc2u_1cpy8lp .Lc2u_1rem8lp: tst ip, #8 - movne r3, r7, pull #8 + movne r3, r7, lspull #8 ldmneia r1!, {r4, r7} - orrne r3, r3, r4, push #24 - movne r4, r4, pull #8 - orrne r4, r4, r7, push #24 + orrne r3, r3, r4, lspush #24 + movne r4, r4, lspull #8 + orrne r4, r4, r7, lspush #24 stmneia r0!, {r3 - r4} @ Shouldnt fault tst ip, #4 - movne r3, r7, pull #8 + movne r3, r7, lspull #8 ldrne r7, [r1], #4 - orrne r3, r3, r7, push #24 + orrne r3, r3, r7, lspush #24 TUSER( strne) r3, [r0], #4 @ Shouldnt fault ands ip, ip, #3 beq .Lc2u_1fupi @@ -172,9 +172,9 @@ USER( TUSER( strgtb) r3, [r0], #1) @ May fault .Lc2u_2fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lc2u_2nowords - mov r3, r7, pull #16 + mov r3, r7, lspull #16 ldr r7, [r1], #4 - orr r3, r3, r7, push #16 + orr r3, r3, r7, lspush #16 USER( TUSER( str) r3, [r0], #4) @ May fault mov ip, r0, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -186,30 +186,30 @@ USER( TUSER( str) r3, [r0], #4) @ May fault subs ip, ip, #16 blt .Lc2u_2rem8lp -.Lc2u_2cpy8lp: mov r3, r7, pull #16 +.Lc2u_2cpy8lp: mov r3, r7, lspull #16 ldmia r1!, {r4 - r7} subs ip, ip, #16 - orr r3, r3, r4, push #16 - mov r4, r4, pull #16 - orr r4, r4, r5, push #16 - mov r5, r5, pull #16 - orr r5, r5, r6, push #16 - mov r6, r6, pull #16 - orr r6, r6, r7, push #16 + orr r3, r3, r4, lspush #16 + mov r4, r4, lspull #16 + orr r4, r4, r5, lspush #16 + mov r5, r5, lspull #16 + orr r5, r5, r6, lspush #16 + mov r6, r6, lspull #16 + orr r6, r6, r7, lspush #16 stmia r0!, {r3 - r6} @ Shouldnt fault bpl .Lc2u_2cpy8lp .Lc2u_2rem8lp: tst ip, #8 - movne r3, r7, pull #16 + movne r3, r7, lspull #16 ldmneia r1!, {r4, r7} - orrne r3, r3, r4, push #16 - movne r4, r4, pull #16 - orrne r4, r4, r7, push #16 + orrne r3, r3, r4, lspush #16 + movne r4, r4, lspull #16 + orrne r4, r4, r7, lspush #16 stmneia r0!, {r3 - r4} @ Shouldnt fault tst ip, #4 - movne r3, r7, pull #16 + movne r3, r7, lspull #16 ldrne r7, [r1], #4 - orrne r3, r3, r7, push #16 + orrne r3, r3, r7, lspush #16 TUSER( strne) r3, [r0], #4 @ Shouldnt fault ands ip, ip, #3 beq .Lc2u_2fupi @@ -227,9 +227,9 @@ USER( TUSER( strgtb) r3, [r0], #1) @ May fault .Lc2u_3fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lc2u_3nowords - mov r3, r7, pull #24 + mov r3, r7, lspull #24 ldr r7, [r1], #4 - orr r3, r3, r7, push #8 + orr r3, r3, r7, lspush #8 USER( TUSER( str) r3, [r0], #4) @ May fault mov ip, r0, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -241,30 +241,30 @@ USER( TUSER( str) r3, [r0], #4) @ May fault subs ip, ip, #16 blt .Lc2u_3rem8lp -.Lc2u_3cpy8lp: mov r3, r7, pull #24 +.Lc2u_3cpy8lp: mov r3, r7, lspull #24 ldmia r1!, {r4 - r7} subs ip, ip, #16 - orr r3, r3, r4, push #8 - mov r4, r4, pull #24 - orr r4, r4, r5, push #8 - mov r5, r5, pull #24 - orr r5, r5, r6, push #8 - mov r6, r6, pull #24 - orr r6, r6, r7, push #8 + orr r3, r3, r4, lspush #8 + mov r4, r4, lspull #24 + orr r4, r4, r5, lspush #8 + mov r5, r5, lspull #24 + orr r5, r5, r6, lspush #8 + mov r6, r6, lspull #24 + orr r6, r6, r7, lspush #8 stmia r0!, {r3 - r6} @ Shouldnt fault bpl .Lc2u_3cpy8lp .Lc2u_3rem8lp: tst ip, #8 - movne r3, r7, pull #24 + movne r3, r7, lspull #24 ldmneia r1!, {r4, r7} - orrne r3, r3, r4, push #8 - movne r4, r4, pull #24 - orrne r4, r4, r7, push #8 + orrne r3, r3, r4, lspush #8 + movne r4, r4, lspull #24 + orrne r4, r4, r7, lspush #8 stmneia r0!, {r3 - r4} @ Shouldnt fault tst ip, #4 - movne r3, r7, pull #24 + movne r3, r7, lspull #24 ldrne r7, [r1], #4 - orrne r3, r3, r7, push #8 + orrne r3, r3, r7, lspush #8 TUSER( strne) r3, [r0], #4 @ Shouldnt fault ands ip, ip, #3 beq .Lc2u_3fupi @@ -382,9 +382,9 @@ USER( TUSER( ldr) r7, [r1], #4) @ May fault .Lcfu_1fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lcfu_1nowords - mov r3, r7, pull #8 + mov r3, r7, lspull #8 USER( TUSER( ldr) r7, [r1], #4) @ May fault - orr r3, r3, r7, push #24 + orr r3, r3, r7, lspush #24 str r3, [r0], #4 mov ip, r1, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -396,30 +396,30 @@ USER( TUSER( ldr) r7, [r1], #4) @ May fault subs ip, ip, #16 blt .Lcfu_1rem8lp -.Lcfu_1cpy8lp: mov r3, r7, pull #8 +.Lcfu_1cpy8lp: mov r3, r7, lspull #8 ldmia r1!, {r4 - r7} @ Shouldnt fault subs ip, ip, #16 - orr r3, r3, r4, push #24 - mov r4, r4, pull #8 - orr r4, r4, r5, push #24 - mov r5, r5, pull #8 - orr r5, r5, r6, push #24 - mov r6, r6, pull #8 - orr r6, r6, r7, push #24 + orr r3, r3, r4, lspush #24 + mov r4, r4, lspull #8 + orr r4, r4, r5, lspush #24 + mov r5, r5, lspull #8 + orr r5, r5, r6, lspush #24 + mov r6, r6, lspull #8 + orr r6, r6, r7, lspush #24 stmia r0!, {r3 - r6} bpl .Lcfu_1cpy8lp .Lcfu_1rem8lp: tst ip, #8 - movne r3, r7, pull #8 + movne r3, r7, lspull #8 ldmneia r1!, {r4, r7} @ Shouldnt fault - orrne r3, r3, r4, push #24 - movne r4, r4, pull #8 - orrne r4, r4, r7, push #24 + orrne r3, r3, r4, lspush #24 + movne r4, r4, lspull #8 + orrne r4, r4, r7, lspush #24 stmneia r0!, {r3 - r4} tst ip, #4 - movne r3, r7, pull #8 + movne r3, r7, lspull #8 USER( TUSER( ldrne) r7, [r1], #4) @ May fault - orrne r3, r3, r7, push #24 + orrne r3, r3, r7, lspush #24 strne r3, [r0], #4 ands ip, ip, #3 beq .Lcfu_1fupi @@ -437,9 +437,9 @@ USER( TUSER( ldrne) r7, [r1], #4) @ May fault .Lcfu_2fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lcfu_2nowords - mov r3, r7, pull #16 + mov r3, r7, lspull #16 USER( TUSER( ldr) r7, [r1], #4) @ May fault - orr r3, r3, r7, push #16 + orr r3, r3, r7, lspush #16 str r3, [r0], #4 mov ip, r1, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -452,30 +452,30 @@ USER( TUSER( ldr) r7, [r1], #4) @ May fault blt .Lcfu_2rem8lp -.Lcfu_2cpy8lp: mov r3, r7, pull #16 +.Lcfu_2cpy8lp: mov r3, r7, lspull #16 ldmia r1!, {r4 - r7} @ Shouldnt fault subs ip, ip, #16 - orr r3, r3, r4, push #16 - mov r4, r4, pull #16 - orr r4, r4, r5, push #16 - mov r5, r5, pull #16 - orr r5, r5, r6, push #16 - mov r6, r6, pull #16 - orr r6, r6, r7, push #16 + orr r3, r3, r4, lspush #16 + mov r4, r4, lspull #16 + orr r4, r4, r5, lspush #16 + mov r5, r5, lspull #16 + orr r5, r5, r6, lspush #16 + mov r6, r6, lspull #16 + orr r6, r6, r7, lspush #16 stmia r0!, {r3 - r6} bpl .Lcfu_2cpy8lp .Lcfu_2rem8lp: tst ip, #8 - movne r3, r7, pull #16 + movne r3, r7, lspull #16 ldmneia r1!, {r4, r7} @ Shouldnt fault - orrne r3, r3, r4, push #16 - movne r4, r4, pull #16 - orrne r4, r4, r7, push #16 + orrne r3, r3, r4, lspush #16 + movne r4, r4, lspull #16 + orrne r4, r4, r7, lspush #16 stmneia r0!, {r3 - r4} tst ip, #4 - movne r3, r7, pull #16 + movne r3, r7, lspull #16 USER( TUSER( ldrne) r7, [r1], #4) @ May fault - orrne r3, r3, r7, push #16 + orrne r3, r3, r7, lspush #16 strne r3, [r0], #4 ands ip, ip, #3 beq .Lcfu_2fupi @@ -493,9 +493,9 @@ USER( TUSER( ldrgtb) r3, [r1], #0) @ May fault .Lcfu_3fupi: subs r2, r2, #4 addmi ip, r2, #4 bmi .Lcfu_3nowords - mov r3, r7, pull #24 + mov r3, r7, lspull #24 USER( TUSER( ldr) r7, [r1], #4) @ May fault - orr r3, r3, r7, push #8 + orr r3, r3, r7, lspush #8 str r3, [r0], #4 mov ip, r1, lsl #32 - PAGE_SHIFT rsb ip, ip, #0 @@ -507,30 +507,30 @@ USER( TUSER( ldr) r7, [r1], #4) @ May fault subs ip, ip, #16 blt .Lcfu_3rem8lp -.Lcfu_3cpy8lp: mov r3, r7, pull #24 +.Lcfu_3cpy8lp: mov r3, r7, lspull #24 ldmia r1!, {r4 - r7} @ Shouldnt fault - orr r3, r3, r4, push #8 - mov r4, r4, pull #24 - orr r4, r4, r5, push #8 - mov r5, r5, pull #24 - orr r5, r5, r6, push #8 - mov r6, r6, pull #24 - orr r6, r6, r7, push #8 + orr r3, r3, r4, lspush #8 + mov r4, r4, lspull #24 + orr r4, r4, r5, lspush #8 + mov r5, r5, lspull #24 + orr r5, r5, r6, lspush #8 + mov r6, r6, lspull #24 + orr r6, r6, r7, lspush #8 stmia r0!, {r3 - r6} subs ip, ip, #16 bpl .Lcfu_3cpy8lp .Lcfu_3rem8lp: tst ip, #8 - movne r3, r7, pull #24 + movne r3, r7, lspull #24 ldmneia r1!, {r4, r7} @ Shouldnt fault - orrne r3, r3, r4, push #8 - movne r4, r4, pull #24 - orrne r4, r4, r7, push #8 + orrne r3, r3, r4, lspush #8 + movne r4, r4, lspull #24 + orrne r4, r4, r7, lspush #8 stmneia r0!, {r3 - r4} tst ip, #4 - movne r3, r7, pull #24 + movne r3, r7, lspull #24 USER( TUSER( ldrne) r7, [r1], #4) @ May fault - orrne r3, r3, r7, push #8 + orrne r3, r3, r7, lspush #8 strne r3, [r0], #4 ands ip, ip, #3 beq .Lcfu_3fupi From db38ee874c48713d0723221d08332242e0088970 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 21 Feb 2014 17:01:48 +0100 Subject: [PATCH 30/89] ARM: 7983/1: atomics: implement a better __atomic_add_unless for v6+ Looking at perf profiles of multi-threaded hackbench runs, a significant performance hit appears to manifest from the cmpxchg loop used to implement the 32-bit atomic_add_unless function. This can be mitigated by writing a direct implementation of __atomic_add_unless which doesn't require iteration outside of the atomic operation. Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/include/asm/atomic.h | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 6e410090896e..9a92fd7864a8 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -141,6 +141,33 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) return oldval; } +static inline int __atomic_add_unless(atomic_t *v, int a, int u) +{ + int oldval, newval; + unsigned long tmp; + + smp_mb(); + prefetchw(&v->counter); + + __asm__ __volatile__ ("@ atomic_add_unless\n" +"1: ldrex %0, [%4]\n" +" teq %0, %5\n" +" beq 2f\n" +" add %1, %0, %6\n" +" strex %2, %1, [%4]\n" +" teq %2, #0\n" +" bne 1b\n" +"2:" + : "=&r" (oldval), "=&r" (newval), "=&r" (tmp), "+Qo" (v->counter) + : "r" (&v->counter), "r" (u), "r" (a) + : "cc"); + + if (oldval != u) + smp_mb(); + + return oldval; +} + #else /* ARM_ARCH_6 */ #ifdef CONFIG_SMP @@ -189,10 +216,6 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return ret; } -#endif /* __LINUX_ARM_ARCH__ */ - -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) - static inline int __atomic_add_unless(atomic_t *v, int a, int u) { int c, old; @@ -203,6 +226,10 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return c; } +#endif /* __LINUX_ARM_ARCH__ */ + +#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) + #define atomic_inc(v) atomic_add(1, v) #define atomic_dec(v) atomic_sub(1, v) From 27e8efdbf1aecb65dd712fd93766f1a04b86ac22 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 21 Feb 2014 18:48:01 +0100 Subject: [PATCH 31/89] ARM: 7986/1: bios32: use pci_enable_resource to enable PCI resources This patch moves bios32 over to using the generic code for enabling PCI resources. Since the core code takes care of bridge resources too, we can also drop the explicit IO and MEMORY enabling for them in the arch code. A side-effect of this change is that we no longer explicitly enable devices when running in PCI_PROBE_ONLY mode. This stays closer to the meaning of the option and prevents us from trying to enable devices without any assigned resources (the core code refuses to enable resources without parents). Tested-by: Jason Gunthorpe Tested-by: Jingoo Han Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/kernel/bios32.c | 37 +++---------------------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c index 317da88ae65b..91f48804e3bb 100644 --- a/arch/arm/kernel/bios32.c +++ b/arch/arm/kernel/bios32.c @@ -608,41 +608,10 @@ resource_size_t pcibios_align_resource(void *data, const struct resource *res, */ int pcibios_enable_device(struct pci_dev *dev, int mask) { - u16 cmd, old_cmd; - int idx; - struct resource *r; + if (pci_has_flag(PCI_PROBE_ONLY)) + return 0; - pci_read_config_word(dev, PCI_COMMAND, &cmd); - old_cmd = cmd; - for (idx = 0; idx < 6; idx++) { - /* Only set up the requested stuff */ - if (!(mask & (1 << idx))) - continue; - - r = dev->resource + idx; - if (!r->start && r->end) { - printk(KERN_ERR "PCI: Device %s not available because" - " of resource collisions\n", pci_name(dev)); - return -EINVAL; - } - if (r->flags & IORESOURCE_IO) - cmd |= PCI_COMMAND_IO; - if (r->flags & IORESOURCE_MEM) - cmd |= PCI_COMMAND_MEMORY; - } - - /* - * Bridges (eg, cardbus bridges) need to be fully enabled - */ - if ((dev->class >> 16) == PCI_BASE_CLASS_BRIDGE) - cmd |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY; - - if (cmd != old_cmd) { - printk("PCI: enabling device %s (%04x -> %04x)\n", - pci_name(dev), old_cmd, cmd); - pci_write_config_word(dev, PCI_COMMAND, cmd); - } - return 0; + return pci_enable_resources(dev, mask); } int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, From b342ea4e4ff970518264c81eefd05f637e3f193a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 19 Feb 2014 22:28:40 +0100 Subject: [PATCH 32/89] ARM: 7981/1: add support for AT_HWCAP2 ELF auxv entry This enables AT_HWCAP2 for ARM. The generic support for this new ELF auxv entry was added in commit 2171364d1a9 (powerpc: Add HWCAP2 aux entry) Signed-off-by: Ard Biesheuvel Acked-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/hwcap.h | 3 ++- arch/arm/include/uapi/asm/hwcap.h | 4 ++++ arch/arm/kernel/setup.c | 11 +++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/hwcap.h b/arch/arm/include/asm/hwcap.h index 6ff56eca3f1f..6e183fd269fb 100644 --- a/arch/arm/include/asm/hwcap.h +++ b/arch/arm/include/asm/hwcap.h @@ -9,6 +9,7 @@ * instruction set this cpu supports. */ #define ELF_HWCAP (elf_hwcap) -extern unsigned int elf_hwcap; +#define ELF_HWCAP2 (elf_hwcap2) +extern unsigned int elf_hwcap, elf_hwcap2; #endif #endif diff --git a/arch/arm/include/uapi/asm/hwcap.h b/arch/arm/include/uapi/asm/hwcap.h index 7dcc10d67253..87768b5cffd1 100644 --- a/arch/arm/include/uapi/asm/hwcap.h +++ b/arch/arm/include/uapi/asm/hwcap.h @@ -28,4 +28,8 @@ #define HWCAP_LPAE (1 << 20) #define HWCAP_EVTSTRM (1 << 21) +/* + * HWCAP2 flags - for elf_hwcap2 (in kernel) and AT_HWCAP2 + */ + #endif /* _UAPI__ASMARM_HWCAP_H */ diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index b0df9761de6d..dbb5449d6b79 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -100,6 +100,9 @@ EXPORT_SYMBOL(system_serial_high); unsigned int elf_hwcap __read_mostly; EXPORT_SYMBOL(elf_hwcap); +unsigned int elf_hwcap2 __read_mostly; +EXPORT_SYMBOL(elf_hwcap2); + #ifdef MULTI_CPU struct processor processor __read_mostly; @@ -1005,6 +1008,10 @@ static const char *hwcap_str[] = { NULL }; +static const char *hwcap2_str[] = { + NULL +}; + static int c_show(struct seq_file *m, void *v) { int i, j; @@ -1028,6 +1035,10 @@ static int c_show(struct seq_file *m, void *v) if (elf_hwcap & (1 << j)) seq_printf(m, "%s ", hwcap_str[j]); + for (j = 0; hwcap2_str[j]; j++) + if (elf_hwcap2 & (1 << j)) + seq_printf(m, "%s ", hwcap2_str[j]); + seq_printf(m, "\nCPU implementer\t: 0x%02x\n", cpuid >> 24); seq_printf(m, "CPU architecture: %s\n", proc_arch[cpu_architecture()]); From 8258a9895c99cdaacad8edc4748c0a624c710961 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 19 Feb 2014 22:29:40 +0100 Subject: [PATCH 33/89] ARM: 7982/1: introduce HWCAP2 feature bits for ARMv8 Crypto Extensions This allocates feature bits 0-4 in HWCAP2 for the crypto and CRC extensions introduced in ARMv8. Signed-off-by: Ard Biesheuvel Acked-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/uapi/asm/hwcap.h | 5 +++++ arch/arm/kernel/setup.c | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/arch/arm/include/uapi/asm/hwcap.h b/arch/arm/include/uapi/asm/hwcap.h index 87768b5cffd1..20d12f230a2f 100644 --- a/arch/arm/include/uapi/asm/hwcap.h +++ b/arch/arm/include/uapi/asm/hwcap.h @@ -31,5 +31,10 @@ /* * HWCAP2 flags - for elf_hwcap2 (in kernel) and AT_HWCAP2 */ +#define HWCAP2_AES (1 << 0) +#define HWCAP2_PMULL (1 << 1) +#define HWCAP2_SHA1 (1 << 2) +#define HWCAP2_SHA2 (1 << 3) +#define HWCAP2_CRC32 (1 << 4) #endif /* _UAPI__ASMARM_HWCAP_H */ diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index dbb5449d6b79..0a6c70b0b0f9 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -1009,6 +1009,11 @@ static const char *hwcap_str[] = { }; static const char *hwcap2_str[] = { + "aes", + "pmull", + "sha1", + "sha2", + "crc32", NULL }; From a51345770e519552e749ff457a2a9f83171a67b5 Mon Sep 17 00:00:00 2001 From: Anurag Aggarwal Date: Mon, 24 Feb 2014 11:17:36 +0100 Subject: [PATCH 34/89] ARM: 7987/1: ARM : unwinder : Prevent data abort due to stack overflow While unwinding backtrace, stack overflow is possible. This stack overflow can sometimes lead to data abort in system if the area after stack is not mapped to physical memory. To prevent this problem from happening, execute the instructions that can cause a data abort in separate helper functions, where a check for feasibility is made before reading each word from the stack. Signed-off-by: Anurag Aggarwal Reviewed-by: Dave Martin Signed-off-by: Russell King --- arch/arm/kernel/unwind.c | 137 ++++++++++++++++++++++++++++----------- 1 file changed, 100 insertions(+), 37 deletions(-) diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 00df012c4678..3c217694ebec 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -68,6 +68,12 @@ EXPORT_SYMBOL(__aeabi_unwind_cpp_pr2); struct unwind_ctrl_block { unsigned long vrs[16]; /* virtual register set */ const unsigned long *insn; /* pointer to the current instructions word */ + unsigned long sp_high; /* highest value of sp allowed */ + /* + * 1 : check for stack overflow for each register pop. + * 0 : save overhead if there is plenty of stack remaining. + */ + int check_each_pop; int entries; /* number of entries left to interpret */ int byte; /* current byte number in the instructions word */ }; @@ -235,12 +241,85 @@ static unsigned long unwind_get_byte(struct unwind_ctrl_block *ctrl) return ret; } +/* Before poping a register check whether it is feasible or not */ +static int unwind_pop_register(struct unwind_ctrl_block *ctrl, + unsigned long **vsp, unsigned int reg) +{ + if (unlikely(ctrl->check_each_pop)) + if (*vsp >= (unsigned long *)ctrl->sp_high) + return -URC_FAILURE; + + ctrl->vrs[reg] = *(*vsp)++; + return URC_OK; +} + +/* Helper functions to execute the instructions */ +static int unwind_exec_pop_subset_r4_to_r13(struct unwind_ctrl_block *ctrl, + unsigned long mask) +{ + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int load_sp, reg = 4; + + load_sp = mask & (1 << (13 - 4)); + while (mask) { + if (mask & 1) + if (unwind_pop_register(ctrl, &vsp, reg)) + return -URC_FAILURE; + mask >>= 1; + reg++; + } + if (!load_sp) + ctrl->vrs[SP] = (unsigned long)vsp; + + return URC_OK; +} + +static int unwind_exec_pop_r4_to_rN(struct unwind_ctrl_block *ctrl, + unsigned long insn) +{ + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int reg; + + /* pop R4-R[4+bbb] */ + for (reg = 4; reg <= 4 + (insn & 7); reg++) + if (unwind_pop_register(ctrl, &vsp, reg)) + return -URC_FAILURE; + + if (insn & 0x80) + if (unwind_pop_register(ctrl, &vsp, 14)) + return -URC_FAILURE; + + ctrl->vrs[SP] = (unsigned long)vsp; + + return URC_OK; +} + +static int unwind_exec_pop_subset_r0_to_r3(struct unwind_ctrl_block *ctrl, + unsigned long mask) +{ + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int reg = 0; + + /* pop R0-R3 according to mask */ + while (mask) { + if (mask & 1) + if (unwind_pop_register(ctrl, &vsp, reg)) + return -URC_FAILURE; + mask >>= 1; + reg++; + } + ctrl->vrs[SP] = (unsigned long)vsp; + + return URC_OK; +} + /* * Execute the current unwind instruction. */ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) { unsigned long insn = unwind_get_byte(ctrl); + int ret = URC_OK; pr_debug("%s: insn = %08lx\n", __func__, insn); @@ -250,8 +329,6 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) ctrl->vrs[SP] -= ((insn & 0x3f) << 2) + 4; else if ((insn & 0xf0) == 0x80) { unsigned long mask; - unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; - int load_sp, reg = 4; insn = (insn << 8) | unwind_get_byte(ctrl); mask = insn & 0x0fff; @@ -261,29 +338,16 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) return -URC_FAILURE; } - /* pop R4-R15 according to mask */ - load_sp = mask & (1 << (13 - 4)); - while (mask) { - if (mask & 1) - ctrl->vrs[reg] = *vsp++; - mask >>= 1; - reg++; - } - if (!load_sp) - ctrl->vrs[SP] = (unsigned long)vsp; + ret = unwind_exec_pop_subset_r4_to_r13(ctrl, mask); + if (ret) + goto error; } else if ((insn & 0xf0) == 0x90 && (insn & 0x0d) != 0x0d) ctrl->vrs[SP] = ctrl->vrs[insn & 0x0f]; else if ((insn & 0xf0) == 0xa0) { - unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; - int reg; - - /* pop R4-R[4+bbb] */ - for (reg = 4; reg <= 4 + (insn & 7); reg++) - ctrl->vrs[reg] = *vsp++; - if (insn & 0x80) - ctrl->vrs[14] = *vsp++; - ctrl->vrs[SP] = (unsigned long)vsp; + ret = unwind_exec_pop_r4_to_rN(ctrl, insn); + if (ret) + goto error; } else if (insn == 0xb0) { if (ctrl->vrs[PC] == 0) ctrl->vrs[PC] = ctrl->vrs[LR]; @@ -291,8 +355,6 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) ctrl->entries = 0; } else if (insn == 0xb1) { unsigned long mask = unwind_get_byte(ctrl); - unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; - int reg = 0; if (mask == 0 || mask & 0xf0) { pr_warning("unwind: Spare encoding %04lx\n", @@ -300,14 +362,9 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) return -URC_FAILURE; } - /* pop R0-R3 according to mask */ - while (mask) { - if (mask & 1) - ctrl->vrs[reg] = *vsp++; - mask >>= 1; - reg++; - } - ctrl->vrs[SP] = (unsigned long)vsp; + ret = unwind_exec_pop_subset_r0_to_r3(ctrl, mask); + if (ret) + goto error; } else if (insn == 0xb2) { unsigned long uleb128 = unwind_get_byte(ctrl); @@ -320,7 +377,8 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) pr_debug("%s: fp = %08lx sp = %08lx lr = %08lx pc = %08lx\n", __func__, ctrl->vrs[FP], ctrl->vrs[SP], ctrl->vrs[LR], ctrl->vrs[PC]); - return URC_OK; +error: + return ret; } /* @@ -329,13 +387,13 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) */ int unwind_frame(struct stackframe *frame) { - unsigned long high, low; + unsigned long low; const struct unwind_idx *idx; struct unwind_ctrl_block ctrl; - /* only go to a higher address on the stack */ + /* store the highest address on the stack to avoid crossing it*/ low = frame->sp; - high = ALIGN(low, THREAD_SIZE); + ctrl.sp_high = ALIGN(low, THREAD_SIZE); pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, frame->pc, frame->lr, frame->sp); @@ -382,11 +440,16 @@ int unwind_frame(struct stackframe *frame) return -URC_FAILURE; } + ctrl.check_each_pop = 0; + while (ctrl.entries > 0) { - int urc = unwind_exec_insn(&ctrl); + int urc; + if ((ctrl.sp_high - ctrl.vrs[SP]) < sizeof(ctrl.vrs)) + ctrl.check_each_pop = 1; + urc = unwind_exec_insn(&ctrl); if (urc < 0) return urc; - if (ctrl.vrs[SP] < low || ctrl.vrs[SP] >= high) + if (ctrl.vrs[SP] < low || ctrl.vrs[SP] >= ctrl.sp_high) return -URC_FAILURE; } From fec510141088ca1f15d1b79f9f6838810d668b77 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Thu, 27 Feb 2014 01:23:43 +0100 Subject: [PATCH 35/89] ARM: 7993/1: mm/memblock: add memblock_get_current_limit Apart from setting the limit of memblock, it's also useful to be able to get the limit to avoid recalculating it every time. Add the function to do so. Acked-by: Catalin Marinas Acked-by: Santosh Shilimkar Acked-by: Andrew Morton Acked-by: Nicolas Pitre Signed-off-by: Laura Abbott Signed-off-by: Russell King --- include/linux/memblock.h | 2 ++ mm/memblock.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 1ef66360f0b0..8a20a51ed42d 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -252,6 +252,8 @@ static inline void memblock_dump_all(void) void memblock_set_current_limit(phys_addr_t limit); +phys_addr_t memblock_get_current_limit(void); + /* * pfn conversion functions * diff --git a/mm/memblock.c b/mm/memblock.c index 39a31e7f0045..7fe5354e7552 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1407,6 +1407,11 @@ void __init_memblock memblock_set_current_limit(phys_addr_t limit) memblock.current_limit = limit; } +phys_addr_t __init_memblock memblock_get_current_limit(void) +{ + return memblock.current_limit; +} + static void __init_memblock memblock_dump(struct memblock_type *type, char *name) { unsigned long long base, size; From 26632becc3c126599567d7fec69c9d91cf3dc124 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 21:45:48 +0100 Subject: [PATCH 36/89] ARM: 7995/1: footbridge: remove obsolete IRQF_DISABLED This patch removes the IRQF_DISABLED flag from footbridge code. It's a NOOP since 2.6.35. Signed-off-by: Michael Opdenacker Signed-off-by: Russell King --- arch/arm/mach-footbridge/dc21285-timer.c | 2 +- arch/arm/mach-footbridge/dc21285.c | 10 +++++----- arch/arm/mach-footbridge/isa-timer.c | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/arm/mach-footbridge/dc21285-timer.c b/arch/arm/mach-footbridge/dc21285-timer.c index 5d2725fd878c..bf7aa7d298e7 100644 --- a/arch/arm/mach-footbridge/dc21285-timer.c +++ b/arch/arm/mach-footbridge/dc21285-timer.c @@ -105,7 +105,7 @@ static irqreturn_t timer1_interrupt(int irq, void *dev_id) static struct irqaction footbridge_timer_irq = { .name = "dc21285_timer1", .handler = timer1_interrupt, - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .dev_id = &ckevt_dc21285, }; diff --git a/arch/arm/mach-footbridge/dc21285.c b/arch/arm/mach-footbridge/dc21285.c index 7c2fdae9a38b..96a3d73ef4bf 100644 --- a/arch/arm/mach-footbridge/dc21285.c +++ b/arch/arm/mach-footbridge/dc21285.c @@ -334,15 +334,15 @@ void __init dc21285_preinit(void) /* * We don't care if these fail. */ - dc21285_request_irq(IRQ_PCI_SERR, dc21285_serr_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_PCI_SERR, dc21285_serr_irq, 0, "PCI system error", &serr_timer); - dc21285_request_irq(IRQ_PCI_PERR, dc21285_parity_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_PCI_PERR, dc21285_parity_irq, 0, "PCI parity error", &perr_timer); - dc21285_request_irq(IRQ_PCI_ABORT, dc21285_abort_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_PCI_ABORT, dc21285_abort_irq, 0, "PCI abort", NULL); - dc21285_request_irq(IRQ_DISCARD_TIMER, dc21285_discard_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_DISCARD_TIMER, dc21285_discard_irq, 0, "Discard timer", NULL); - dc21285_request_irq(IRQ_PCI_DPERR, dc21285_dparity_irq, IRQF_DISABLED, + dc21285_request_irq(IRQ_PCI_DPERR, dc21285_dparity_irq, 0, "PCI data parity", NULL); if (cfn_mode) { diff --git a/arch/arm/mach-footbridge/isa-timer.c b/arch/arm/mach-footbridge/isa-timer.c index d9301dd56354..b73f52e196b9 100644 --- a/arch/arm/mach-footbridge/isa-timer.c +++ b/arch/arm/mach-footbridge/isa-timer.c @@ -27,7 +27,7 @@ static irqreturn_t pit_timer_interrupt(int irq, void *dev_id) static struct irqaction pit_timer_irq = { .name = "pit", .handler = pit_timer_interrupt, - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .dev_id = &i8253_clockevent, }; From 57c06a8ed7db71e14a4388590ad258c5b557e557 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 21:51:44 +0100 Subject: [PATCH 37/89] ARM: 7996/1: floppy.h: remove deprecated IRQF_DISABLED This patch removes the use of the IRQF_DISABLED flag in arch/arm/include/asm/floppy.h It's a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Signed-off-by: Russell King --- arch/arm/include/asm/floppy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/include/asm/floppy.h b/arch/arm/include/asm/floppy.h index c9f03eccc9d8..f4882553fbb0 100644 --- a/arch/arm/include/asm/floppy.h +++ b/arch/arm/include/asm/floppy.h @@ -25,7 +25,7 @@ #define fd_inb(port) inb((port)) #define fd_request_irq() request_irq(IRQ_FLOPPYDISK,floppy_interrupt,\ - IRQF_DISABLED,"floppy",NULL) + 0,"floppy",NULL) #define fd_free_irq() free_irq(IRQ_FLOPPYDISK,NULL) #define fd_disable_irq() disable_irq(IRQ_FLOPPYDISK) #define fd_enable_irq() enable_irq(IRQ_FLOPPYDISK) From 19bd9b286da6f28f4cb04757dd8509a38f9f34d7 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 21:56:10 +0100 Subject: [PATCH 38/89] ARM: 7997/1: cns3xxx: remove deprecated IRQF_DISABLED This patch removes the use of the IRQF_DISABLED flag from arch/arm/mach-cns3xxx/core.c It's a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Signed-off-by: Russell King --- arch/arm/mach-cns3xxx/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/mach-cns3xxx/core.c b/arch/arm/mach-cns3xxx/core.c index e38b279f402c..384dc859e6c6 100644 --- a/arch/arm/mach-cns3xxx/core.c +++ b/arch/arm/mach-cns3xxx/core.c @@ -155,7 +155,7 @@ static irqreturn_t cns3xxx_timer_interrupt(int irq, void *dev_id) static struct irqaction cns3xxx_timer_irq = { .name = "timer", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = cns3xxx_timer_interrupt, }; From 1ee6564d72ce718bf4e50d5684aa98d9d895f859 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 21:59:03 +0100 Subject: [PATCH 39/89] ARM: 7998/1: IXP4xx: remove deprecated IRQF_DISABLED This patch removes the use of the IRQF_DISABLED flag from code in arch/arm/mach-ixp4xx It's a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Signed-off-by: Russell King --- arch/arm/mach-ixp4xx/common.c | 2 +- arch/arm/mach-ixp4xx/dsmg600-setup.c | 3 +-- arch/arm/mach-ixp4xx/fsg-setup.c | 6 ++---- arch/arm/mach-ixp4xx/nas100d-setup.c | 3 +-- arch/arm/mach-ixp4xx/nslu2-setup.c | 6 ++---- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index 6d68aed6548a..a465f27bc263 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -312,7 +312,7 @@ static irqreturn_t ixp4xx_timer_interrupt(int irq, void *dev_id) static struct irqaction ixp4xx_timer_irq = { .name = "timer1", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = ixp4xx_timer_interrupt, .dev_id = &clockevent_ixp4xx, }; diff --git a/arch/arm/mach-ixp4xx/dsmg600-setup.c b/arch/arm/mach-ixp4xx/dsmg600-setup.c index 736dc692d540..43ee06d3abe5 100644 --- a/arch/arm/mach-ixp4xx/dsmg600-setup.c +++ b/arch/arm/mach-ixp4xx/dsmg600-setup.c @@ -233,8 +233,7 @@ static int __init dsmg600_gpio_init(void) gpio_request(DSMG600_RB_GPIO, "reset button"); if (request_irq(gpio_to_irq(DSMG600_RB_GPIO), &dsmg600_reset_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "DSM-G600 reset button", NULL) < 0) { + IRQF_TRIGGER_LOW, "DSM-G600 reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", gpio_to_irq(DSMG600_RB_GPIO)); diff --git a/arch/arm/mach-ixp4xx/fsg-setup.c b/arch/arm/mach-ixp4xx/fsg-setup.c index 429966b756ed..5c4b0c4a1b37 100644 --- a/arch/arm/mach-ixp4xx/fsg-setup.c +++ b/arch/arm/mach-ixp4xx/fsg-setup.c @@ -208,16 +208,14 @@ static void __init fsg_init(void) platform_add_devices(fsg_devices, ARRAY_SIZE(fsg_devices)); if (request_irq(gpio_to_irq(FSG_RB_GPIO), &fsg_reset_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "FSG reset button", NULL) < 0) { + IRQF_TRIGGER_LOW, "FSG reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", gpio_to_irq(FSG_RB_GPIO)); } if (request_irq(gpio_to_irq(FSG_SB_GPIO), &fsg_power_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "FSG power button", NULL) < 0) { + IRQF_TRIGGER_LOW, "FSG power button", NULL) < 0) { printk(KERN_DEBUG "Power Button IRQ %d not available\n", gpio_to_irq(FSG_SB_GPIO)); diff --git a/arch/arm/mach-ixp4xx/nas100d-setup.c b/arch/arm/mach-ixp4xx/nas100d-setup.c index 507cb5233537..4e0f762bc651 100644 --- a/arch/arm/mach-ixp4xx/nas100d-setup.c +++ b/arch/arm/mach-ixp4xx/nas100d-setup.c @@ -295,8 +295,7 @@ static void __init nas100d_init(void) pm_power_off = nas100d_power_off; if (request_irq(gpio_to_irq(NAS100D_RB_GPIO), &nas100d_reset_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "NAS100D reset button", NULL) < 0) { + IRQF_TRIGGER_LOW, "NAS100D reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", gpio_to_irq(NAS100D_RB_GPIO)); diff --git a/arch/arm/mach-ixp4xx/nslu2-setup.c b/arch/arm/mach-ixp4xx/nslu2-setup.c index ba5f1cda2a9d..88c025f52d8d 100644 --- a/arch/arm/mach-ixp4xx/nslu2-setup.c +++ b/arch/arm/mach-ixp4xx/nslu2-setup.c @@ -265,16 +265,14 @@ static void __init nslu2_init(void) pm_power_off = nslu2_power_off; if (request_irq(gpio_to_irq(NSLU2_RB_GPIO), &nslu2_reset_handler, - IRQF_DISABLED | IRQF_TRIGGER_LOW, - "NSLU2 reset button", NULL) < 0) { + IRQF_TRIGGER_LOW, "NSLU2 reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", gpio_to_irq(NSLU2_RB_GPIO)); } if (request_irq(gpio_to_irq(NSLU2_PB_GPIO), &nslu2_power_handler, - IRQF_DISABLED | IRQF_TRIGGER_HIGH, - "NSLU2 power button", NULL) < 0) { + IRQF_TRIGGER_HIGH, "NSLU2 power button", NULL) < 0) { printk(KERN_DEBUG "Power Button IRQ %d not available\n", gpio_to_irq(NSLU2_PB_GPIO)); From a09df10585d3b3b7a2f640b11fabea262e751091 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 22:02:14 +0100 Subject: [PATCH 40/89] ARM: 7999/1: arch/arm/mach-lpc32xx-remove-irqf-disabled This patch removes the use of the IRQF_DISABLED flag from arch/arm/mach-lpc32xx/timer.c It's a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Signed-off-by: Russell King --- arch/arm/mach-lpc32xx/timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/mach-lpc32xx/timer.c b/arch/arm/mach-lpc32xx/timer.c index 20eab63d10ba..4e5837299c04 100644 --- a/arch/arm/mach-lpc32xx/timer.c +++ b/arch/arm/mach-lpc32xx/timer.c @@ -90,7 +90,7 @@ static irqreturn_t lpc32xx_timer_interrupt(int irq, void *dev_id) static struct irqaction lpc32xx_timer_irq = { .name = "LPC32XX Timer Tick", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = lpc32xx_timer_interrupt, }; From 78f6db99522bbdd2f2a90c963744bd51c8602990 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 22:04:50 +0100 Subject: [PATCH 41/89] ARM: 8000/1: misc: remove deprecated IRQF_DISABLED This patch removes the use of the IRQF_DISABLED flag from miscellaneous code in mach-xxx and plat-xxx This flag is a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Acked-by: Linus Walleij Acked-by: Greg Ungerer Signed-off-by: Russell King --- arch/arm/mach-ebsa110/core.c | 2 +- arch/arm/mach-integrator/integrator_ap.c | 2 +- arch/arm/mach-ks8695/time.c | 2 +- arch/arm/mach-netx/time.c | 2 +- arch/arm/mach-rpc/dma.c | 2 +- arch/arm/mach-rpc/time.c | 1 - arch/arm/mach-sa1100/time.c | 2 +- arch/arm/mach-u300/timer.c | 2 +- arch/arm/plat-iop/time.c | 2 +- 9 files changed, 8 insertions(+), 9 deletions(-) diff --git a/arch/arm/mach-ebsa110/core.c b/arch/arm/mach-ebsa110/core.c index 68ac934d4565..8254e716b095 100644 --- a/arch/arm/mach-ebsa110/core.c +++ b/arch/arm/mach-ebsa110/core.c @@ -206,7 +206,7 @@ ebsa110_timer_interrupt(int irq, void *dev_id) static struct irqaction ebsa110_timer_irq = { .name = "EBSA110 Timer Tick", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = ebsa110_timer_interrupt, }; diff --git a/arch/arm/mach-integrator/integrator_ap.c b/arch/arm/mach-integrator/integrator_ap.c index 17c0fe627435..e4f27f0e56ac 100644 --- a/arch/arm/mach-integrator/integrator_ap.c +++ b/arch/arm/mach-integrator/integrator_ap.c @@ -358,7 +358,7 @@ static struct clock_event_device integrator_clockevent = { static struct irqaction integrator_timer_irq = { .name = "timer", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = integrator_timer_interrupt, .dev_id = &integrator_clockevent, }; diff --git a/arch/arm/mach-ks8695/time.c b/arch/arm/mach-ks8695/time.c index 426c97662f5b..a197874bf382 100644 --- a/arch/arm/mach-ks8695/time.c +++ b/arch/arm/mach-ks8695/time.c @@ -122,7 +122,7 @@ static irqreturn_t ks8695_timer_interrupt(int irq, void *dev_id) static struct irqaction ks8695_timer_irq = { .name = "ks8695_tick", - .flags = IRQF_DISABLED | IRQF_TIMER, + .flags = IRQF_TIMER, .handler = ks8695_timer_interrupt, }; diff --git a/arch/arm/mach-netx/time.c b/arch/arm/mach-netx/time.c index 6df42e643031..3177c7a40930 100644 --- a/arch/arm/mach-netx/time.c +++ b/arch/arm/mach-netx/time.c @@ -99,7 +99,7 @@ netx_timer_interrupt(int irq, void *dev_id) static struct irqaction netx_timer_irq = { .name = "NetX Timer Tick", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = netx_timer_interrupt, }; diff --git a/arch/arm/mach-rpc/dma.c b/arch/arm/mach-rpc/dma.c index 85883b2e0e49..6d3517dc4772 100644 --- a/arch/arm/mach-rpc/dma.c +++ b/arch/arm/mach-rpc/dma.c @@ -141,7 +141,7 @@ static int iomd_request_dma(unsigned int chan, dma_t *dma) struct iomd_dma *idma = container_of(dma, struct iomd_dma, dma); return request_irq(idma->irq, iomd_dma_handle, - IRQF_DISABLED, idma->dma.device_id, idma); + 0, idma->dma.device_id, idma); } static void iomd_free_dma(unsigned int chan, dma_t *dma) diff --git a/arch/arm/mach-rpc/time.c b/arch/arm/mach-rpc/time.c index 9a6def14df01..9a5158861ca9 100644 --- a/arch/arm/mach-rpc/time.c +++ b/arch/arm/mach-rpc/time.c @@ -75,7 +75,6 @@ ioc_timer_interrupt(int irq, void *dev_id) static struct irqaction ioc_timer_irq = { .name = "timer", - .flags = IRQF_DISABLED, .handler = ioc_timer_interrupt }; diff --git a/arch/arm/mach-sa1100/time.c b/arch/arm/mach-sa1100/time.c index 6fd4acb8f187..4852c08cb526 100644 --- a/arch/arm/mach-sa1100/time.c +++ b/arch/arm/mach-sa1100/time.c @@ -112,7 +112,7 @@ static struct clock_event_device ckevt_sa1100_osmr0 = { static struct irqaction sa1100_timer_irq = { .name = "ost0", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = sa1100_ost0_interrupt, .dev_id = &ckevt_sa1100_osmr0, }; diff --git a/arch/arm/mach-u300/timer.c b/arch/arm/mach-u300/timer.c index fe08fd34c0ce..de52cb37c479 100644 --- a/arch/arm/mach-u300/timer.c +++ b/arch/arm/mach-u300/timer.c @@ -337,7 +337,7 @@ static irqreturn_t u300_timer_interrupt(int irq, void *dev_id) static struct irqaction u300_timer_irq = { .name = "U300 Timer Tick", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = u300_timer_interrupt, }; diff --git a/arch/arm/plat-iop/time.c b/arch/arm/plat-iop/time.c index d70b73364a3f..6ad65d8ae237 100644 --- a/arch/arm/plat-iop/time.c +++ b/arch/arm/plat-iop/time.c @@ -127,7 +127,7 @@ iop_timer_interrupt(int irq, void *dev_id) static struct irqaction iop_timer_irq = { .name = "IOP Timer Tick", .handler = iop_timer_interrupt, - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .dev_id = &iop_clockevent, }; From 9929eedc0c3495105018f3c632ee73b7fb4c1f72 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 22:07:26 +0100 Subject: [PATCH 42/89] ARM: 8001/1: mmp: remove deprecated IRQF_DISABLED This patch removes the use of the IRQF_DISABLED flag from arch/arm/mach-mmp/time.c It's a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Signed-off-by: Russell King --- arch/arm/mach-mmp/time.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/mach-mmp/time.c b/arch/arm/mach-mmp/time.c index 024022d91fe3..bbcd2322fd27 100644 --- a/arch/arm/mach-mmp/time.c +++ b/arch/arm/mach-mmp/time.c @@ -186,7 +186,7 @@ static void __init timer_config(void) static struct irqaction timer_irq = { .name = "timer", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = timer_interrupt, .dev_id = &ckevt, }; From 49710fa4d20cf4e210fa1dc59d7dd39181eeea11 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 22:09:18 +0100 Subject: [PATCH 43/89] ARM: 8002/1: spear: remove deprecated IRQF_DISABLED This patch removes the use of the IRQF_DISABLED flag from arch/arm/mach-spear/time.c It's a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Signed-off-by: Russell King --- arch/arm/mach-spear/time.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/mach-spear/time.c b/arch/arm/mach-spear/time.c index d449673e40f7..218ba5b67d92 100644 --- a/arch/arm/mach-spear/time.c +++ b/arch/arm/mach-spear/time.c @@ -172,7 +172,7 @@ static irqreturn_t spear_timer_interrupt(int irq, void *dev_id) static struct irqaction spear_timer_irq = { .name = "timer", - .flags = IRQF_DISABLED | IRQF_TIMER, + .flags = IRQF_TIMER, .handler = spear_timer_interrupt }; From 2ed71e7531878b74c0d018891a55ca190a2f1f1b Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Tue, 4 Mar 2014 22:11:56 +0100 Subject: [PATCH 44/89] ARM: 8003/1: w90x900: remove deprecated IRQF_DISABLED This patch removes the use of the IRQF_DISABLED flag from arch/arm/mach-w90x900/time.c It's a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Acked-by: Wan zongshun Signed-off-by: Russell King --- arch/arm/mach-w90x900/time.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/mach-w90x900/time.c b/arch/arm/mach-w90x900/time.c index 30fbca844575..9230d3725599 100644 --- a/arch/arm/mach-w90x900/time.c +++ b/arch/arm/mach-w90x900/time.c @@ -111,7 +111,7 @@ static irqreturn_t nuc900_timer0_interrupt(int irq, void *dev_id) static struct irqaction nuc900_timer0_irq = { .name = "nuc900-timer0", - .flags = IRQF_DISABLED | IRQF_TIMER | IRQF_IRQPOLL, + .flags = IRQF_TIMER | IRQF_IRQPOLL, .handler = nuc900_timer0_interrupt, }; From 83b3f64d46d1d9bf1e0be1d16f805ccdd52d31c6 Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Wed, 5 Mar 2014 06:23:13 +0100 Subject: [PATCH 45/89] ARM: 8004/1: [SCSI]: remove deprecated IRQF_DISABLED This patch removes the use of the IRQF_DISABLED flag from drivers/scsi/arm It's a NOOP since 2.6.35 and it will be removed one day. Signed-off-by: Michael Opdenacker Signed-off-by: Russell King --- drivers/scsi/arm/acornscsi.c | 2 +- drivers/scsi/arm/cumana_1.c | 2 +- drivers/scsi/arm/cumana_2.c | 2 +- drivers/scsi/arm/powertec.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/scsi/arm/acornscsi.c b/drivers/scsi/arm/acornscsi.c index 09ba1869d366..059ff477a398 100644 --- a/drivers/scsi/arm/acornscsi.c +++ b/drivers/scsi/arm/acornscsi.c @@ -2971,7 +2971,7 @@ static int acornscsi_probe(struct expansion_card *ec, const struct ecard_id *id) ec->irqaddr = ashost->fast + INT_REG; ec->irqmask = 0x0a; - ret = request_irq(host->irq, acornscsi_intr, IRQF_DISABLED, "acornscsi", ashost); + ret = request_irq(host->irq, acornscsi_intr, 0, "acornscsi", ashost); if (ret) { printk(KERN_CRIT "scsi%d: IRQ%d not free: %d\n", host->host_no, ashost->scsi.irq, ret); diff --git a/drivers/scsi/arm/cumana_1.c b/drivers/scsi/arm/cumana_1.c index b679778376c5..f8e060900052 100644 --- a/drivers/scsi/arm/cumana_1.c +++ b/drivers/scsi/arm/cumana_1.c @@ -262,7 +262,7 @@ static int cumanascsi1_probe(struct expansion_card *ec, goto out_unmap; } - ret = request_irq(host->irq, cumanascsi_intr, IRQF_DISABLED, + ret = request_irq(host->irq, cumanascsi_intr, 0, "CumanaSCSI-1", host); if (ret) { printk("scsi%d: IRQ%d not free: %d\n", diff --git a/drivers/scsi/arm/cumana_2.c b/drivers/scsi/arm/cumana_2.c index 58915f29055b..abc66f5263ec 100644 --- a/drivers/scsi/arm/cumana_2.c +++ b/drivers/scsi/arm/cumana_2.c @@ -431,7 +431,7 @@ static int cumanascsi2_probe(struct expansion_card *ec, goto out_free; ret = request_irq(ec->irq, cumanascsi_2_intr, - IRQF_DISABLED, "cumanascsi2", info); + 0, "cumanascsi2", info); if (ret) { printk("scsi%d: IRQ%d not free: %d\n", host->host_no, ec->irq, ret); diff --git a/drivers/scsi/arm/powertec.c b/drivers/scsi/arm/powertec.c index abc9593615e9..5e1b73e1b743 100644 --- a/drivers/scsi/arm/powertec.c +++ b/drivers/scsi/arm/powertec.c @@ -358,7 +358,7 @@ static int powertecscsi_probe(struct expansion_card *ec, goto out_free; ret = request_irq(ec->irq, powertecscsi_intr, - IRQF_DISABLED, "powertec", info); + 0, "powertec", info); if (ret) { printk("scsi%d: IRQ%d not free: %d\n", host->host_no, ec->irq, ret); From 09294e31b1779dda22f420c195994a0db54c9a92 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Fri, 7 Mar 2014 10:32:22 -0500 Subject: [PATCH 46/89] uprobes: Kconfig dependency fix Suggested change from Oleg Nesterov. Fixes incomplete dependencies for uprobes feature. Signed-off-by: David A. Long Acked-by: Oleg Nesterov --- arch/Kconfig | 6 +----- kernel/trace/Kconfig | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 80bbb8ccd0d1..97ff872c7acc 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -86,9 +86,7 @@ config KPROBES_ON_FTRACE optimize on top of function tracing. config UPROBES - bool "Transparent user-space probes (EXPERIMENTAL)" - depends on UPROBE_EVENT && PERF_EVENTS - default n + def_bool n select PERCPU_RWSEM help Uprobes is the user-space counterpart to kprobes: they @@ -101,8 +99,6 @@ config UPROBES managed by the kernel and kept transparent to the probed application. ) - If in doubt, say "N". - config HAVE_64BIT_ALIGNED_ACCESS def_bool 64BIT && !HAVE_EFFICIENT_UNALIGNED_ACCESS help diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 015f85aaca08..8639819f6cef 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -424,6 +424,7 @@ config UPROBE_EVENT bool "Enable uprobes-based dynamic events" depends on ARCH_SUPPORTS_UPROBES depends on MMU + depends on PERF_EVENTS select UPROBES select PROBE_EVENTS select TRACING From 21254ebc9e509967317ad8c6922797e21137ad53 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Thu, 6 Mar 2014 18:17:52 -0500 Subject: [PATCH 47/89] ARM: Fix missing includes in kprobes sources Make sure includes in ARM kprobes sources are done explicitly. Do not rely on includes from other includes. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/include/asm/kprobes.h | 2 +- arch/arm/kernel/kprobes-common.c | 3 +++ arch/arm/kernel/kprobes-test-arm.c | 1 + arch/arm/kernel/kprobes-test.c | 4 +++- arch/arm/kernel/kprobes.c | 2 ++ 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/kprobes.h b/arch/arm/include/asm/kprobes.h index f82ec22eeb11..fd2e5caec6ed 100644 --- a/arch/arm/include/asm/kprobes.h +++ b/arch/arm/include/asm/kprobes.h @@ -18,7 +18,7 @@ #include #include -#include +#include #define __ARCH_WANT_KPROBES_INSN_SLOT #define MAX_INSN_SIZE 2 diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index 18a76282970e..455c8003bffb 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include "kprobes.h" diff --git a/arch/arm/kernel/kprobes-test-arm.c b/arch/arm/kernel/kprobes-test-arm.c index 839312905067..87839de77e5f 100644 --- a/arch/arm/kernel/kprobes-test-arm.c +++ b/arch/arm/kernel/kprobes-test-arm.c @@ -10,6 +10,7 @@ #include #include +#include #include "kprobes-test.h" diff --git a/arch/arm/kernel/kprobes-test.c b/arch/arm/kernel/kprobes-test.c index 0cd63d080c7b..4a774d40c946 100644 --- a/arch/arm/kernel/kprobes-test.c +++ b/arch/arm/kernel/kprobes-test.c @@ -201,7 +201,9 @@ #include #include #include - +#include +#include +#include #include #include "kprobes.h" diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c index a7b621ece23d..54e7b46a3295 100644 --- a/arch/arm/kernel/kprobes.c +++ b/arch/arm/kernel/kprobes.c @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include "kprobes.h" #include "patch.h" From 6fe50a28ba6e5fafb4a549dea666dd15297dd8bd Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Mon, 3 Feb 2014 14:25:49 -0500 Subject: [PATCH 48/89] uprobes: allow ignoring of probe hits Allow arches to decided to ignore a probe hit. ARM will use this to only call handlers if the conditions to execute a conditionally executed instruction are satisfied. Signed-off-by: David A. Long Acked-by: Oleg Nesterov --- include/linux/uprobes.h | 1 + kernel/events/uprobes.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index e32251e00e62..edff2b97b864 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -126,6 +126,7 @@ extern bool arch_uprobe_xol_was_trapped(struct task_struct *tsk); extern int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data); extern void arch_uprobe_abort_xol(struct arch_uprobe *aup, struct pt_regs *regs); extern unsigned long arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs *regs); +extern bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs); #else /* !CONFIG_UPROBES */ struct uprobes_state { }; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 307d87c0991a..04709b66369d 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1804,6 +1804,11 @@ static bool handle_trampoline(struct pt_regs *regs) return true; } +bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs) +{ + return false; +} + /* * Run handler and ask thread to singlestep. * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. @@ -1858,7 +1863,11 @@ static void handle_swbp(struct pt_regs *regs) if (!get_utask()) goto out; + if (arch_uprobe_ignore(&uprobe->arch, regs)) + goto out; + handler_chain(uprobe, regs); + if (can_skip_sstep(uprobe, regs)) goto out; From b2531dd5e5f19ea01d67aed82d81c5f778ec0fb7 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Thu, 6 Mar 2014 17:53:34 -0500 Subject: [PATCH 49/89] ARM: move shared uprobe/kprobe definitions into new include file Separate the kprobe-only definitions from the definitions needed by both kprobes and uprobes. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/include/asm/kprobes.h | 15 +------------- arch/arm/include/asm/probes.h | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 14 deletions(-) create mode 100644 arch/arm/include/asm/probes.h diff --git a/arch/arm/include/asm/kprobes.h b/arch/arm/include/asm/kprobes.h index fd2e5caec6ed..6e1046661f07 100644 --- a/arch/arm/include/asm/kprobes.h +++ b/arch/arm/include/asm/kprobes.h @@ -28,21 +28,8 @@ #define kretprobe_blacklist_size 0 typedef u32 kprobe_opcode_t; - struct kprobe; -typedef void (kprobe_insn_handler_t)(struct kprobe *, struct pt_regs *); -typedef unsigned long (kprobe_check_cc)(unsigned long); -typedef void (kprobe_insn_singlestep_t)(struct kprobe *, struct pt_regs *); -typedef void (kprobe_insn_fn_t)(void); - -/* Architecture specific copy of original instruction. */ -struct arch_specific_insn { - kprobe_opcode_t *insn; - kprobe_insn_handler_t *insn_handler; - kprobe_check_cc *insn_check_cc; - kprobe_insn_singlestep_t *insn_singlestep; - kprobe_insn_fn_t *insn_fn; -}; +#include struct prev_kprobe { struct kprobe *kp; diff --git a/arch/arm/include/asm/probes.h b/arch/arm/include/asm/probes.h new file mode 100644 index 000000000000..90c5f5485202 --- /dev/null +++ b/arch/arm/include/asm/probes.h @@ -0,0 +1,36 @@ +/* + * arch/arm/include/asm/probes.h + * + * Original contents copied from arch/arm/include/asm/kprobes.h + * which contains the following notice... + * + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#ifndef _ASM_PROBES_H +#define _ASM_PROBES_H + +typedef void (kprobe_insn_handler_t)(struct kprobe *, struct pt_regs *); +typedef unsigned long (kprobe_check_cc)(unsigned long); +typedef void (kprobe_insn_singlestep_t)(struct kprobe *, struct pt_regs *); +typedef void (kprobe_insn_fn_t)(void); + +/* Architecture specific copy of original instruction. */ +struct arch_specific_insn { + kprobe_opcode_t *insn; + kprobe_insn_handler_t *insn_handler; + kprobe_check_cc *insn_check_cc; + kprobe_insn_singlestep_t *insn_singlestep; + kprobe_insn_fn_t *insn_fn; +}; + +#endif From c18377c303787ded44b7decd7dee694db0f205e9 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Fri, 7 Mar 2014 11:16:10 -0500 Subject: [PATCH 50/89] ARM: Move generic arm instruction parsing code to new files for sharing between features Move the arm version of the kprobes instruction parsing code into more generic files from where it can be used by uprobes and possibly other subsystems. The symbol names will be made more generic in a subsequent part of this patchset. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/include/asm/probes.h | 2 + arch/arm/kernel/Makefile | 4 +- arch/arm/kernel/kprobes-arm.c | 724 +----------------------------- arch/arm/kernel/kprobes-common.c | 425 ------------------ arch/arm/kernel/kprobes.h | 373 +--------------- arch/arm/kernel/probes-arm.c | 731 +++++++++++++++++++++++++++++++ arch/arm/kernel/probes-arm.h | 38 ++ arch/arm/kernel/probes.c | 443 +++++++++++++++++++ arch/arm/kernel/probes.h | 397 +++++++++++++++++ 9 files changed, 1624 insertions(+), 1513 deletions(-) create mode 100644 arch/arm/kernel/probes-arm.c create mode 100644 arch/arm/kernel/probes-arm.h create mode 100644 arch/arm/kernel/probes.c create mode 100644 arch/arm/kernel/probes.h diff --git a/arch/arm/include/asm/probes.h b/arch/arm/include/asm/probes.h index 90c5f5485202..737a9b310efc 100644 --- a/arch/arm/include/asm/probes.h +++ b/arch/arm/include/asm/probes.h @@ -19,6 +19,8 @@ #ifndef _ASM_PROBES_H #define _ASM_PROBES_H +struct kprobe; + typedef void (kprobe_insn_handler_t)(struct kprobe *, struct pt_regs *); typedef unsigned long (kprobe_check_cc)(unsigned long); typedef void (kprobe_insn_singlestep_t)(struct kprobe *, struct pt_regs *); diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index a30fc9be9e9e..4c8b13e64280 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -50,11 +50,11 @@ obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o insn.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o insn.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o insn.o patch.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o -obj-$(CONFIG_KPROBES) += kprobes.o kprobes-common.o patch.o +obj-$(CONFIG_KPROBES) += probes.o kprobes.o kprobes-common.o patch.o ifdef CONFIG_THUMB2_KERNEL obj-$(CONFIG_KPROBES) += kprobes-thumb.o else -obj-$(CONFIG_KPROBES) += kprobes-arm.o +obj-$(CONFIG_KPROBES) += kprobes-arm.o probes-arm.o endif obj-$(CONFIG_ARM_KPROBES_TEST) += test-kprobes.o test-kprobes-objs := kprobes-test.o diff --git a/arch/arm/kernel/kprobes-arm.c b/arch/arm/kernel/kprobes-arm.c index 8a30c89da70e..a1d0a8f00f9e 100644 --- a/arch/arm/kernel/kprobes-arm.c +++ b/arch/arm/kernel/kprobes-arm.c @@ -60,13 +60,10 @@ #include #include -#include +#include #include "kprobes.h" - -#define sign_extend(x, signbit) ((x) | (0 - ((x) & (1 << (signbit))))) - -#define branch_displacement(insn) sign_extend(((insn) & 0xffffff) << 2, 25) +#include "probes-arm.h" #if __LINUX_ARM_ARCH__ >= 6 #define BLX(reg) "blx "reg" \n\t" @@ -75,88 +72,8 @@ "mov pc, "reg" \n\t" #endif -/* - * To avoid the complications of mimicing single-stepping on a - * processor without a Next-PC or a single-step mode, and to - * avoid having to deal with the side-effects of boosting, we - * simulate or emulate (almost) all ARM instructions. - * - * "Simulation" is where the instruction's behavior is duplicated in - * C code. "Emulation" is where the original instruction is rewritten - * and executed, often by altering its registers. - * - * By having all behavior of the kprobe'd instruction completed before - * returning from the kprobe_handler(), all locks (scheduler and - * interrupt) can safely be released. There is no need for secondary - * breakpoints, no race with MP or preemptable kernels, nor having to - * clean up resources counts at a later time impacting overall system - * performance. By rewriting the instruction, only the minimum registers - * need to be loaded and saved back optimizing performance. - * - * Calling the insnslot_*_rwflags version of a function doesn't hurt - * anything even when the CPSR flags aren't updated by the - * instruction. It's just a little slower in return for saving - * a little space by not having a duplicate function that doesn't - * update the flags. (The same optimization can be said for - * instructions that do or don't perform register writeback) - * Also, instructions can either read the flags, only write the - * flags, or read and write the flags. To save combinations - * rather than for sheer performance, flag functions just assume - * read and write of flags. - */ -static void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs) -{ - kprobe_opcode_t insn = p->opcode; - long iaddr = (long)p->addr; - int disp = branch_displacement(insn); - - if (insn & (1 << 24)) - regs->ARM_lr = iaddr + 4; - - regs->ARM_pc = iaddr + 8 + disp; -} - -static void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs) -{ - kprobe_opcode_t insn = p->opcode; - long iaddr = (long)p->addr; - int disp = branch_displacement(insn); - - regs->ARM_lr = iaddr + 4; - regs->ARM_pc = iaddr + 8 + disp + ((insn >> 23) & 0x2); - regs->ARM_cpsr |= PSR_T_BIT; -} - -static void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs) -{ - kprobe_opcode_t insn = p->opcode; - int rm = insn & 0xf; - long rmv = regs->uregs[rm]; - - if (insn & (1 << 5)) - regs->ARM_lr = (long)p->addr + 4; - - regs->ARM_pc = rmv & ~0x1; - regs->ARM_cpsr &= ~PSR_T_BIT; - if (rmv & 0x1) - regs->ARM_cpsr |= PSR_T_BIT; -} - -static void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs) -{ - kprobe_opcode_t insn = p->opcode; - int rd = (insn >> 12) & 0xf; - unsigned long mask = 0xf8ff03df; /* Mask out execution state */ - regs->uregs[rd] = regs->ARM_cpsr & mask; -} - -static void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs) -{ - regs->uregs[12] = regs->uregs[13]; -} - -static void __kprobes +void __kprobes emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -185,7 +102,7 @@ emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) regs->uregs[rn] = rnv; } -static void __kprobes +void __kprobes emulate_ldr(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -215,7 +132,7 @@ emulate_ldr(struct kprobe *p, struct pt_regs *regs) regs->uregs[rn] = rnv; } -static void __kprobes +void __kprobes emulate_str(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -242,7 +159,7 @@ emulate_str(struct kprobe *p, struct pt_regs *regs) regs->uregs[rn] = rnv; } -static void __kprobes +void __kprobes emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -277,7 +194,7 @@ emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -static void __kprobes +void __kprobes emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -304,7 +221,7 @@ emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -static void __kprobes +void __kprobes emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -333,7 +250,7 @@ emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -static void __kprobes +void __kprobes emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -353,7 +270,7 @@ emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->uregs[rd] = rdv; } -static void __kprobes +void __kprobes emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -382,624 +299,3 @@ emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->uregs[rdhi] = rdhiv; regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } - -/* - * For the instruction masking and comparisons in all the "space_*" - * functions below, Do _not_ rearrange the order of tests unless - * you're very, very sure of what you are doing. For the sake of - * efficiency, the masks for some tests sometimes assume other test - * have been done prior to them so the number of patterns to test - * for an instruction set can be as broad as possible to reduce the - * number of tests needed. - */ - -static const union decode_item arm_1111_table[] = { - /* Unconditional instructions */ - - /* memory hint 1111 0100 x001 xxxx xxxx xxxx xxxx xxxx */ - /* PLDI (immediate) 1111 0100 x101 xxxx xxxx xxxx xxxx xxxx */ - /* PLDW (immediate) 1111 0101 x001 xxxx xxxx xxxx xxxx xxxx */ - /* PLD (immediate) 1111 0101 x101 xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe300000, 0xf4100000, kprobe_simulate_nop), - - /* memory hint 1111 0110 x001 xxxx xxxx xxxx xxx0 xxxx */ - /* PLDI (register) 1111 0110 x101 xxxx xxxx xxxx xxx0 xxxx */ - /* PLDW (register) 1111 0111 x001 xxxx xxxx xxxx xxx0 xxxx */ - /* PLD (register) 1111 0111 x101 xxxx xxxx xxxx xxx0 xxxx */ - DECODE_SIMULATE (0xfe300010, 0xf6100000, kprobe_simulate_nop), - - /* BLX (immediate) 1111 101x xxxx xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe000000, 0xfa000000, simulate_blx1), - - /* CPS 1111 0001 0000 xxx0 xxxx xxxx xx0x xxxx */ - /* SETEND 1111 0001 0000 0001 xxxx xxxx 0000 xxxx */ - /* SRS 1111 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ - /* RFE 1111 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ - - /* Coprocessor instructions... */ - /* MCRR2 1111 1100 0100 xxxx xxxx xxxx xxxx xxxx */ - /* MRRC2 1111 1100 0101 xxxx xxxx xxxx xxxx xxxx */ - /* LDC2 1111 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ - /* STC2 1111 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ - /* CDP2 1111 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ - /* MCR2 1111 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ - /* MRC2 1111 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item arm_cccc_0001_0xx0____0xxx_table[] = { - /* Miscellaneous instructions */ - - /* MRS cpsr cccc 0001 0000 xxxx xxxx xxxx 0000 xxxx */ - DECODE_SIMULATEX(0x0ff000f0, 0x01000000, simulate_mrs, - REGS(0, NOPC, 0, 0, 0)), - - /* BX cccc 0001 0010 xxxx xxxx xxxx 0001 xxxx */ - DECODE_SIMULATE (0x0ff000f0, 0x01200010, simulate_blx2bx), - - /* BLX (register) cccc 0001 0010 xxxx xxxx xxxx 0011 xxxx */ - DECODE_SIMULATEX(0x0ff000f0, 0x01200030, simulate_blx2bx, - REGS(0, 0, 0, 0, NOPC)), - - /* CLZ cccc 0001 0110 xxxx xxxx xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x01600010, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* QADD cccc 0001 0000 xxxx xxxx xxxx 0101 xxxx */ - /* QSUB cccc 0001 0010 xxxx xxxx xxxx 0101 xxxx */ - /* QDADD cccc 0001 0100 xxxx xxxx xxxx 0101 xxxx */ - /* QDSUB cccc 0001 0110 xxxx xxxx xxxx 0101 xxxx */ - DECODE_EMULATEX (0x0f9000f0, 0x01000050, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), - - /* BXJ cccc 0001 0010 xxxx xxxx xxxx 0010 xxxx */ - /* MSR cccc 0001 0x10 xxxx xxxx xxxx 0000 xxxx */ - /* MRS spsr cccc 0001 0100 xxxx xxxx xxxx 0000 xxxx */ - /* BKPT 1110 0001 0010 xxxx xxxx xxxx 0111 xxxx */ - /* SMC cccc 0001 0110 xxxx xxxx xxxx 0111 xxxx */ - /* And unallocated instructions... */ - DECODE_END -}; - -static const union decode_item arm_cccc_0001_0xx0____1xx0_table[] = { - /* Halfword multiply and multiply-accumulate */ - - /* SMLALxy cccc 0001 0100 xxxx xxxx xxxx 1xx0 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x01400080, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - /* SMULWy cccc 0001 0010 xxxx xxxx xxxx 1x10 xxxx */ - DECODE_OR (0x0ff000b0, 0x012000a0), - /* SMULxy cccc 0001 0110 xxxx xxxx xxxx 1xx0 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x01600080, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, 0, NOPC, 0, NOPC)), - - /* SMLAxy cccc 0001 0000 xxxx xxxx xxxx 1xx0 xxxx */ - DECODE_OR (0x0ff00090, 0x01000080), - /* SMLAWy cccc 0001 0010 xxxx xxxx xxxx 1x00 xxxx */ - DECODE_EMULATEX (0x0ff000b0, 0x01200080, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - DECODE_END -}; - -static const union decode_item arm_cccc_0000_____1001_table[] = { - /* Multiply and multiply-accumulate */ - - /* MUL cccc 0000 0000 xxxx xxxx xxxx 1001 xxxx */ - /* MULS cccc 0000 0001 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0fe000f0, 0x00000090, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, 0, NOPC, 0, NOPC)), - - /* MLA cccc 0000 0010 xxxx xxxx xxxx 1001 xxxx */ - /* MLAS cccc 0000 0011 xxxx xxxx xxxx 1001 xxxx */ - DECODE_OR (0x0fe000f0, 0x00200090), - /* MLS cccc 0000 0110 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x00600090, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - /* UMAAL cccc 0000 0100 xxxx xxxx xxxx 1001 xxxx */ - DECODE_OR (0x0ff000f0, 0x00400090), - /* UMULL cccc 0000 1000 xxxx xxxx xxxx 1001 xxxx */ - /* UMULLS cccc 0000 1001 xxxx xxxx xxxx 1001 xxxx */ - /* UMLAL cccc 0000 1010 xxxx xxxx xxxx 1001 xxxx */ - /* UMLALS cccc 0000 1011 xxxx xxxx xxxx 1001 xxxx */ - /* SMULL cccc 0000 1100 xxxx xxxx xxxx 1001 xxxx */ - /* SMULLS cccc 0000 1101 xxxx xxxx xxxx 1001 xxxx */ - /* SMLAL cccc 0000 1110 xxxx xxxx xxxx 1001 xxxx */ - /* SMLALS cccc 0000 1111 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0f8000f0, 0x00800090, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - DECODE_END -}; - -static const union decode_item arm_cccc_0001_____1001_table[] = { - /* Synchronization primitives */ - -#if __LINUX_ARM_ARCH__ < 6 - /* Deprecated on ARMv6 and may be UNDEFINED on v7 */ - /* SMP/SWPB cccc 0001 0x00 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0fb000f0, 0x01000090, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), -#endif - /* LDREX/STREX{,D,B,H} cccc 0001 1xxx xxxx xxxx xxxx 1001 xxxx */ - /* And unallocated instructions... */ - DECODE_END -}; - -static const union decode_item arm_cccc_000x_____1xx1_table[] = { - /* Extra load/store instructions */ - - /* STRHT cccc 0000 xx10 xxxx xxxx xxxx 1011 xxxx */ - /* ??? cccc 0000 xx10 xxxx xxxx xxxx 11x1 xxxx */ - /* LDRHT cccc 0000 xx11 xxxx xxxx xxxx 1011 xxxx */ - /* LDRSBT cccc 0000 xx11 xxxx xxxx xxxx 1101 xxxx */ - /* LDRSHT cccc 0000 xx11 xxxx xxxx xxxx 1111 xxxx */ - DECODE_REJECT (0x0f200090, 0x00200090), - - /* LDRD/STRD lr,pc,{... cccc 000x x0x0 xxxx 111x xxxx 1101 xxxx */ - DECODE_REJECT (0x0e10e0d0, 0x0000e0d0), - - /* LDRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1101 xxxx */ - /* STRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e5000d0, 0x000000d0, emulate_ldrdstrd, - REGS(NOPCWB, NOPCX, 0, 0, NOPC)), - - /* LDRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1101 xxxx */ - /* STRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e5000d0, 0x004000d0, emulate_ldrdstrd, - REGS(NOPCWB, NOPCX, 0, 0, 0)), - - /* STRH (register) cccc 000x x0x0 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0e5000f0, 0x000000b0, emulate_str, - REGS(NOPCWB, NOPC, 0, 0, NOPC)), - - /* LDRH (register) cccc 000x x0x1 xxxx xxxx xxxx 1011 xxxx */ - /* LDRSB (register) cccc 000x x0x1 xxxx xxxx xxxx 1101 xxxx */ - /* LDRSH (register) cccc 000x x0x1 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e500090, 0x00100090, emulate_ldr, - REGS(NOPCWB, NOPC, 0, 0, NOPC)), - - /* STRH (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0e5000f0, 0x004000b0, emulate_str, - REGS(NOPCWB, NOPC, 0, 0, 0)), - - /* LDRH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1011 xxxx */ - /* LDRSB (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1101 xxxx */ - /* LDRSH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e500090, 0x00500090, emulate_ldr, - REGS(NOPCWB, NOPC, 0, 0, 0)), - - DECODE_END -}; - -static const union decode_item arm_cccc_000x_table[] = { - /* Data-processing (register) */ - - /* S PC, ... cccc 000x xxx1 xxxx 1111 xxxx xxxx xxxx */ - DECODE_REJECT (0x0e10f000, 0x0010f000), - - /* MOV IP, SP 1110 0001 1010 0000 1100 0000 0000 1101 */ - DECODE_SIMULATE (0xffffffff, 0xe1a0c00d, simulate_mov_ipsp), - - /* TST (register) cccc 0001 0001 xxxx xxxx xxxx xxx0 xxxx */ - /* TEQ (register) cccc 0001 0011 xxxx xxxx xxxx xxx0 xxxx */ - /* CMP (register) cccc 0001 0101 xxxx xxxx xxxx xxx0 xxxx */ - /* CMN (register) cccc 0001 0111 xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0f900010, 0x01100000, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, 0, 0, 0, ANY)), - - /* MOV (register) cccc 0001 101x xxxx xxxx xxxx xxx0 xxxx */ - /* MVN (register) cccc 0001 111x xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0fa00010, 0x01a00000, emulate_rd12rn16rm0rs8_rwflags, - REGS(0, ANY, 0, 0, ANY)), - - /* AND (register) cccc 0000 000x xxxx xxxx xxxx xxx0 xxxx */ - /* EOR (register) cccc 0000 001x xxxx xxxx xxxx xxx0 xxxx */ - /* SUB (register) cccc 0000 010x xxxx xxxx xxxx xxx0 xxxx */ - /* RSB (register) cccc 0000 011x xxxx xxxx xxxx xxx0 xxxx */ - /* ADD (register) cccc 0000 100x xxxx xxxx xxxx xxx0 xxxx */ - /* ADC (register) cccc 0000 101x xxxx xxxx xxxx xxx0 xxxx */ - /* SBC (register) cccc 0000 110x xxxx xxxx xxxx xxx0 xxxx */ - /* RSC (register) cccc 0000 111x xxxx xxxx xxxx xxx0 xxxx */ - /* ORR (register) cccc 0001 100x xxxx xxxx xxxx xxx0 xxxx */ - /* BIC (register) cccc 0001 110x xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0e000010, 0x00000000, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, ANY, 0, 0, ANY)), - - /* TST (reg-shift reg) cccc 0001 0001 xxxx xxxx xxxx 0xx1 xxxx */ - /* TEQ (reg-shift reg) cccc 0001 0011 xxxx xxxx xxxx 0xx1 xxxx */ - /* CMP (reg-shift reg) cccc 0001 0101 xxxx xxxx xxxx 0xx1 xxxx */ - /* CMN (reg-shift reg) cccc 0001 0111 xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0f900090, 0x01100010, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, 0, NOPC, 0, ANY)), - - /* MOV (reg-shift reg) cccc 0001 101x xxxx xxxx xxxx 0xx1 xxxx */ - /* MVN (reg-shift reg) cccc 0001 111x xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0fa00090, 0x01a00010, emulate_rd12rn16rm0rs8_rwflags, - REGS(0, ANY, NOPC, 0, ANY)), - - /* AND (reg-shift reg) cccc 0000 000x xxxx xxxx xxxx 0xx1 xxxx */ - /* EOR (reg-shift reg) cccc 0000 001x xxxx xxxx xxxx 0xx1 xxxx */ - /* SUB (reg-shift reg) cccc 0000 010x xxxx xxxx xxxx 0xx1 xxxx */ - /* RSB (reg-shift reg) cccc 0000 011x xxxx xxxx xxxx 0xx1 xxxx */ - /* ADD (reg-shift reg) cccc 0000 100x xxxx xxxx xxxx 0xx1 xxxx */ - /* ADC (reg-shift reg) cccc 0000 101x xxxx xxxx xxxx 0xx1 xxxx */ - /* SBC (reg-shift reg) cccc 0000 110x xxxx xxxx xxxx 0xx1 xxxx */ - /* RSC (reg-shift reg) cccc 0000 111x xxxx xxxx xxxx 0xx1 xxxx */ - /* ORR (reg-shift reg) cccc 0001 100x xxxx xxxx xxxx 0xx1 xxxx */ - /* BIC (reg-shift reg) cccc 0001 110x xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0e000090, 0x00000010, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, ANY, NOPC, 0, ANY)), - - DECODE_END -}; - -static const union decode_item arm_cccc_001x_table[] = { - /* Data-processing (immediate) */ - - /* MOVW cccc 0011 0000 xxxx xxxx xxxx xxxx xxxx */ - /* MOVT cccc 0011 0100 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0fb00000, 0x03000000, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, 0)), - - /* YIELD cccc 0011 0010 0000 xxxx xxxx 0000 0001 */ - DECODE_OR (0x0fff00ff, 0x03200001), - /* SEV cccc 0011 0010 0000 xxxx xxxx 0000 0100 */ - DECODE_EMULATE (0x0fff00ff, 0x03200004, kprobe_emulate_none), - /* NOP cccc 0011 0010 0000 xxxx xxxx 0000 0000 */ - /* WFE cccc 0011 0010 0000 xxxx xxxx 0000 0010 */ - /* WFI cccc 0011 0010 0000 xxxx xxxx 0000 0011 */ - DECODE_SIMULATE (0x0fff00fc, 0x03200000, kprobe_simulate_nop), - /* DBG cccc 0011 0010 0000 xxxx xxxx ffff xxxx */ - /* unallocated hints cccc 0011 0010 0000 xxxx xxxx xxxx xxxx */ - /* MSR (immediate) cccc 0011 0x10 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0x0fb00000, 0x03200000), - - /* S PC, ... cccc 001x xxx1 xxxx 1111 xxxx xxxx xxxx */ - DECODE_REJECT (0x0e10f000, 0x0210f000), - - /* TST (immediate) cccc 0011 0001 xxxx xxxx xxxx xxxx xxxx */ - /* TEQ (immediate) cccc 0011 0011 xxxx xxxx xxxx xxxx xxxx */ - /* CMP (immediate) cccc 0011 0101 xxxx xxxx xxxx xxxx xxxx */ - /* CMN (immediate) cccc 0011 0111 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0f900000, 0x03100000, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, 0, 0, 0, 0)), - - /* MOV (immediate) cccc 0011 101x xxxx xxxx xxxx xxxx xxxx */ - /* MVN (immediate) cccc 0011 111x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0fa00000, 0x03a00000, emulate_rd12rn16rm0rs8_rwflags, - REGS(0, ANY, 0, 0, 0)), - - /* AND (immediate) cccc 0010 000x xxxx xxxx xxxx xxxx xxxx */ - /* EOR (immediate) cccc 0010 001x xxxx xxxx xxxx xxxx xxxx */ - /* SUB (immediate) cccc 0010 010x xxxx xxxx xxxx xxxx xxxx */ - /* RSB (immediate) cccc 0010 011x xxxx xxxx xxxx xxxx xxxx */ - /* ADD (immediate) cccc 0010 100x xxxx xxxx xxxx xxxx xxxx */ - /* ADC (immediate) cccc 0010 101x xxxx xxxx xxxx xxxx xxxx */ - /* SBC (immediate) cccc 0010 110x xxxx xxxx xxxx xxxx xxxx */ - /* RSC (immediate) cccc 0010 111x xxxx xxxx xxxx xxxx xxxx */ - /* ORR (immediate) cccc 0011 100x xxxx xxxx xxxx xxxx xxxx */ - /* BIC (immediate) cccc 0011 110x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e000000, 0x02000000, emulate_rd12rn16rm0rs8_rwflags, - REGS(ANY, ANY, 0, 0, 0)), - - DECODE_END -}; - -static const union decode_item arm_cccc_0110_____xxx1_table[] = { - /* Media instructions */ - - /* SEL cccc 0110 1000 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x068000b0, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), - - /* SSAT cccc 0110 101x xxxx xxxx xxxx xx01 xxxx */ - /* USAT cccc 0110 111x xxxx xxxx xxxx xx01 xxxx */ - DECODE_OR(0x0fa00030, 0x06a00010), - /* SSAT16 cccc 0110 1010 xxxx xxxx xxxx 0011 xxxx */ - /* USAT16 cccc 0110 1110 xxxx xxxx xxxx 0011 xxxx */ - DECODE_EMULATEX (0x0fb000f0, 0x06a00030, emulate_rd12rn16rm0_rwflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* REV cccc 0110 1011 xxxx xxxx xxxx 0011 xxxx */ - /* REV16 cccc 0110 1011 xxxx xxxx xxxx 1011 xxxx */ - /* RBIT cccc 0110 1111 xxxx xxxx xxxx 0011 xxxx */ - /* REVSH cccc 0110 1111 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0fb00070, 0x06b00030, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* ??? cccc 0110 0x00 xxxx xxxx xxxx xxx1 xxxx */ - DECODE_REJECT (0x0fb00010, 0x06000010), - /* ??? cccc 0110 0xxx xxxx xxxx xxxx 1011 xxxx */ - DECODE_REJECT (0x0f8000f0, 0x060000b0), - /* ??? cccc 0110 0xxx xxxx xxxx xxxx 1101 xxxx */ - DECODE_REJECT (0x0f8000f0, 0x060000d0), - /* SADD16 cccc 0110 0001 xxxx xxxx xxxx 0001 xxxx */ - /* SADDSUBX cccc 0110 0001 xxxx xxxx xxxx 0011 xxxx */ - /* SSUBADDX cccc 0110 0001 xxxx xxxx xxxx 0101 xxxx */ - /* SSUB16 cccc 0110 0001 xxxx xxxx xxxx 0111 xxxx */ - /* SADD8 cccc 0110 0001 xxxx xxxx xxxx 1001 xxxx */ - /* SSUB8 cccc 0110 0001 xxxx xxxx xxxx 1111 xxxx */ - /* QADD16 cccc 0110 0010 xxxx xxxx xxxx 0001 xxxx */ - /* QADDSUBX cccc 0110 0010 xxxx xxxx xxxx 0011 xxxx */ - /* QSUBADDX cccc 0110 0010 xxxx xxxx xxxx 0101 xxxx */ - /* QSUB16 cccc 0110 0010 xxxx xxxx xxxx 0111 xxxx */ - /* QADD8 cccc 0110 0010 xxxx xxxx xxxx 1001 xxxx */ - /* QSUB8 cccc 0110 0010 xxxx xxxx xxxx 1111 xxxx */ - /* SHADD16 cccc 0110 0011 xxxx xxxx xxxx 0001 xxxx */ - /* SHADDSUBX cccc 0110 0011 xxxx xxxx xxxx 0011 xxxx */ - /* SHSUBADDX cccc 0110 0011 xxxx xxxx xxxx 0101 xxxx */ - /* SHSUB16 cccc 0110 0011 xxxx xxxx xxxx 0111 xxxx */ - /* SHADD8 cccc 0110 0011 xxxx xxxx xxxx 1001 xxxx */ - /* SHSUB8 cccc 0110 0011 xxxx xxxx xxxx 1111 xxxx */ - /* UADD16 cccc 0110 0101 xxxx xxxx xxxx 0001 xxxx */ - /* UADDSUBX cccc 0110 0101 xxxx xxxx xxxx 0011 xxxx */ - /* USUBADDX cccc 0110 0101 xxxx xxxx xxxx 0101 xxxx */ - /* USUB16 cccc 0110 0101 xxxx xxxx xxxx 0111 xxxx */ - /* UADD8 cccc 0110 0101 xxxx xxxx xxxx 1001 xxxx */ - /* USUB8 cccc 0110 0101 xxxx xxxx xxxx 1111 xxxx */ - /* UQADD16 cccc 0110 0110 xxxx xxxx xxxx 0001 xxxx */ - /* UQADDSUBX cccc 0110 0110 xxxx xxxx xxxx 0011 xxxx */ - /* UQSUBADDX cccc 0110 0110 xxxx xxxx xxxx 0101 xxxx */ - /* UQSUB16 cccc 0110 0110 xxxx xxxx xxxx 0111 xxxx */ - /* UQADD8 cccc 0110 0110 xxxx xxxx xxxx 1001 xxxx */ - /* UQSUB8 cccc 0110 0110 xxxx xxxx xxxx 1111 xxxx */ - /* UHADD16 cccc 0110 0111 xxxx xxxx xxxx 0001 xxxx */ - /* UHADDSUBX cccc 0110 0111 xxxx xxxx xxxx 0011 xxxx */ - /* UHSUBADDX cccc 0110 0111 xxxx xxxx xxxx 0101 xxxx */ - /* UHSUB16 cccc 0110 0111 xxxx xxxx xxxx 0111 xxxx */ - /* UHADD8 cccc 0110 0111 xxxx xxxx xxxx 1001 xxxx */ - /* UHSUB8 cccc 0110 0111 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0f800010, 0x06000010, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), - - /* PKHBT cccc 0110 1000 xxxx xxxx xxxx x001 xxxx */ - /* PKHTB cccc 0110 1000 xxxx xxxx xxxx x101 xxxx */ - DECODE_EMULATEX (0x0ff00030, 0x06800010, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPC, NOPC, 0, 0, NOPC)), - - /* ??? cccc 0110 1001 xxxx xxxx xxxx 0111 xxxx */ - /* ??? cccc 0110 1101 xxxx xxxx xxxx 0111 xxxx */ - DECODE_REJECT (0x0fb000f0, 0x06900070), - - /* SXTB16 cccc 0110 1000 1111 xxxx xxxx 0111 xxxx */ - /* SXTB cccc 0110 1010 1111 xxxx xxxx 0111 xxxx */ - /* SXTH cccc 0110 1011 1111 xxxx xxxx 0111 xxxx */ - /* UXTB16 cccc 0110 1100 1111 xxxx xxxx 0111 xxxx */ - /* UXTB cccc 0110 1110 1111 xxxx xxxx 0111 xxxx */ - /* UXTH cccc 0110 1111 1111 xxxx xxxx 0111 xxxx */ - DECODE_EMULATEX (0x0f8f00f0, 0x068f0070, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* SXTAB16 cccc 0110 1000 xxxx xxxx xxxx 0111 xxxx */ - /* SXTAB cccc 0110 1010 xxxx xxxx xxxx 0111 xxxx */ - /* SXTAH cccc 0110 1011 xxxx xxxx xxxx 0111 xxxx */ - /* UXTAB16 cccc 0110 1100 xxxx xxxx xxxx 0111 xxxx */ - /* UXTAB cccc 0110 1110 xxxx xxxx xxxx 0111 xxxx */ - /* UXTAH cccc 0110 1111 xxxx xxxx xxxx 0111 xxxx */ - DECODE_EMULATEX (0x0f8000f0, 0x06800070, emulate_rd12rn16rm0_rwflags_nopc, - REGS(NOPCX, NOPC, 0, 0, NOPC)), - - DECODE_END -}; - -static const union decode_item arm_cccc_0111_____xxx1_table[] = { - /* Media instructions */ - - /* UNDEFINED cccc 0111 1111 xxxx xxxx xxxx 1111 xxxx */ - DECODE_REJECT (0x0ff000f0, 0x07f000f0), - - /* SMLALD cccc 0111 0100 xxxx xxxx xxxx 00x1 xxxx */ - /* SMLSLD cccc 0111 0100 xxxx xxxx xxxx 01x1 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x07400010, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - /* SMUAD cccc 0111 0000 xxxx 1111 xxxx 00x1 xxxx */ - /* SMUSD cccc 0111 0000 xxxx 1111 xxxx 01x1 xxxx */ - DECODE_OR (0x0ff0f090, 0x0700f010), - /* SMMUL cccc 0111 0101 xxxx 1111 xxxx 00x1 xxxx */ - DECODE_OR (0x0ff0f0d0, 0x0750f010), - /* USAD8 cccc 0111 1000 xxxx 1111 xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff0f0f0, 0x0780f010, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, 0, NOPC, 0, NOPC)), - - /* SMLAD cccc 0111 0000 xxxx xxxx xxxx 00x1 xxxx */ - /* SMLSD cccc 0111 0000 xxxx xxxx xxxx 01x1 xxxx */ - DECODE_OR (0x0ff00090, 0x07000010), - /* SMMLA cccc 0111 0101 xxxx xxxx xxxx 00x1 xxxx */ - DECODE_OR (0x0ff000d0, 0x07500010), - /* USADA8 cccc 0111 1000 xxxx xxxx xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x07800010, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, NOPCX, NOPC, 0, NOPC)), - - /* SMMLS cccc 0111 0101 xxxx xxxx xxxx 11x1 xxxx */ - DECODE_EMULATEX (0x0ff000d0, 0x075000d0, emulate_rd16rn12rm0rs8_rwflags_nopc, - REGS(NOPC, NOPC, NOPC, 0, NOPC)), - - /* SBFX cccc 0111 101x xxxx xxxx xxxx x101 xxxx */ - /* UBFX cccc 0111 111x xxxx xxxx xxxx x101 xxxx */ - DECODE_EMULATEX (0x0fa00070, 0x07a00050, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPC)), - - /* BFC cccc 0111 110x xxxx xxxx xxxx x001 1111 */ - DECODE_EMULATEX (0x0fe0007f, 0x07c0001f, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, 0)), - - /* BFI cccc 0111 110x xxxx xxxx xxxx x001 xxxx */ - DECODE_EMULATEX (0x0fe00070, 0x07c00010, emulate_rd12rm0_noflags_nopc, - REGS(0, NOPC, 0, 0, NOPCX)), - - DECODE_END -}; - -static const union decode_item arm_cccc_01xx_table[] = { - /* Load/store word and unsigned byte */ - - /* LDRB/STRB pc,[...] cccc 01xx x0xx xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0x0c40f000, 0x0440f000), - - /* STRT cccc 01x0 x010 xxxx xxxx xxxx xxxx xxxx */ - /* LDRT cccc 01x0 x011 xxxx xxxx xxxx xxxx xxxx */ - /* STRBT cccc 01x0 x110 xxxx xxxx xxxx xxxx xxxx */ - /* LDRBT cccc 01x0 x111 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0x0d200000, 0x04200000), - - /* STR (immediate) cccc 010x x0x0 xxxx xxxx xxxx xxxx xxxx */ - /* STRB (immediate) cccc 010x x1x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x04000000, emulate_str, - REGS(NOPCWB, ANY, 0, 0, 0)), - - /* LDR (immediate) cccc 010x x0x1 xxxx xxxx xxxx xxxx xxxx */ - /* LDRB (immediate) cccc 010x x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x04100000, emulate_ldr, - REGS(NOPCWB, ANY, 0, 0, 0)), - - /* STR (register) cccc 011x x0x0 xxxx xxxx xxxx xxxx xxxx */ - /* STRB (register) cccc 011x x1x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x06000000, emulate_str, - REGS(NOPCWB, ANY, 0, 0, NOPC)), - - /* LDR (register) cccc 011x x0x1 xxxx xxxx xxxx xxxx xxxx */ - /* LDRB (register) cccc 011x x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x06100000, emulate_ldr, - REGS(NOPCWB, ANY, 0, 0, NOPC)), - - DECODE_END -}; - -static const union decode_item arm_cccc_100x_table[] = { - /* Block data transfer instructions */ - - /* LDM cccc 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ - /* STM cccc 100x x0x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_CUSTOM (0x0e400000, 0x08000000, kprobe_decode_ldmstm), - - /* STM (user registers) cccc 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ - /* LDM (user registers) cccc 100x x1x1 xxxx 0xxx xxxx xxxx xxxx */ - /* LDM (exception ret) cccc 100x x1x1 xxxx 1xxx xxxx xxxx xxxx */ - DECODE_END -}; - -const union decode_item kprobe_decode_arm_table[] = { - /* - * Unconditional instructions - * 1111 xxxx xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xf0000000, 0xf0000000, arm_1111_table), - - /* - * Miscellaneous instructions - * cccc 0001 0xx0 xxxx xxxx xxxx 0xxx xxxx - */ - DECODE_TABLE (0x0f900080, 0x01000000, arm_cccc_0001_0xx0____0xxx_table), - - /* - * Halfword multiply and multiply-accumulate - * cccc 0001 0xx0 xxxx xxxx xxxx 1xx0 xxxx - */ - DECODE_TABLE (0x0f900090, 0x01000080, arm_cccc_0001_0xx0____1xx0_table), - - /* - * Multiply and multiply-accumulate - * cccc 0000 xxxx xxxx xxxx xxxx 1001 xxxx - */ - DECODE_TABLE (0x0f0000f0, 0x00000090, arm_cccc_0000_____1001_table), - - /* - * Synchronization primitives - * cccc 0001 xxxx xxxx xxxx xxxx 1001 xxxx - */ - DECODE_TABLE (0x0f0000f0, 0x01000090, arm_cccc_0001_____1001_table), - - /* - * Extra load/store instructions - * cccc 000x xxxx xxxx xxxx xxxx 1xx1 xxxx - */ - DECODE_TABLE (0x0e000090, 0x00000090, arm_cccc_000x_____1xx1_table), - - /* - * Data-processing (register) - * cccc 000x xxxx xxxx xxxx xxxx xxx0 xxxx - * Data-processing (register-shifted register) - * cccc 000x xxxx xxxx xxxx xxxx 0xx1 xxxx - */ - DECODE_TABLE (0x0e000000, 0x00000000, arm_cccc_000x_table), - - /* - * Data-processing (immediate) - * cccc 001x xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0x0e000000, 0x02000000, arm_cccc_001x_table), - - /* - * Media instructions - * cccc 011x xxxx xxxx xxxx xxxx xxx1 xxxx - */ - DECODE_TABLE (0x0f000010, 0x06000010, arm_cccc_0110_____xxx1_table), - DECODE_TABLE (0x0f000010, 0x07000010, arm_cccc_0111_____xxx1_table), - - /* - * Load/store word and unsigned byte - * cccc 01xx xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0x0c000000, 0x04000000, arm_cccc_01xx_table), - - /* - * Block data transfer instructions - * cccc 100x xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0x0e000000, 0x08000000, arm_cccc_100x_table), - - /* B cccc 1010 xxxx xxxx xxxx xxxx xxxx xxxx */ - /* BL cccc 1011 xxxx xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0x0e000000, 0x0a000000, simulate_bbl), - - /* - * Supervisor Call, and coprocessor instructions - */ - - /* MCRR cccc 1100 0100 xxxx xxxx xxxx xxxx xxxx */ - /* MRRC cccc 1100 0101 xxxx xxxx xxxx xxxx xxxx */ - /* LDC cccc 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ - /* STC cccc 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ - /* CDP cccc 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ - /* MCR cccc 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ - /* MRC cccc 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ - /* SVC cccc 1111 xxxx xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0x0c000000, 0x0c000000), - - DECODE_END -}; -#ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_arm_table); -#endif - -static void __kprobes arm_singlestep(struct kprobe *p, struct pt_regs *regs) -{ - regs->ARM_pc += 4; - p->ainsn.insn_handler(p, regs); -} - -/* Return: - * INSN_REJECTED If instruction is one not allowed to kprobe, - * INSN_GOOD If instruction is supported and uses instruction slot, - * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. - * - * For instructions we don't want to kprobe (INSN_REJECTED return result): - * These are generally ones that modify the processor state making - * them "hard" to simulate such as switches processor modes or - * make accesses in alternate modes. Any of these could be simulated - * if the work was put into it, but low return considering they - * should also be very rare. - */ -enum kprobe_insn __kprobes -arm_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) -{ - asi->insn_singlestep = arm_singlestep; - asi->insn_check_cc = kprobe_condition_checks[insn>>28]; - return kprobe_decode_insn(insn, asi, kprobe_decode_arm_table, false); -} diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index 455c8003bffb..f02c038059c3 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -13,178 +13,10 @@ #include #include -#include -#include -#include -#include #include "kprobes.h" -#ifndef find_str_pc_offset - -/* - * For STR and STM instructions, an ARM core may choose to use either - * a +8 or a +12 displacement from the current instruction's address. - * Whichever value is chosen for a given core, it must be the same for - * both instructions and may not change. This function measures it. - */ - -int str_pc_offset; - -void __init find_str_pc_offset(void) -{ - int addr, scratch, ret; - - __asm__ ( - "sub %[ret], pc, #4 \n\t" - "str pc, %[addr] \n\t" - "ldr %[scr], %[addr] \n\t" - "sub %[ret], %[scr], %[ret] \n\t" - : [ret] "=r" (ret), [scr] "=r" (scratch), [addr] "+m" (addr)); - - str_pc_offset = ret; -} - -#endif /* !find_str_pc_offset */ - - -#ifndef test_load_write_pc_interworking - -bool load_write_pc_interworks; - -void __init test_load_write_pc_interworking(void) -{ - int arch = cpu_architecture(); - BUG_ON(arch == CPU_ARCH_UNKNOWN); - load_write_pc_interworks = arch >= CPU_ARCH_ARMv5T; -} - -#endif /* !test_load_write_pc_interworking */ - - -#ifndef test_alu_write_pc_interworking - -bool alu_write_pc_interworks; - -void __init test_alu_write_pc_interworking(void) -{ - int arch = cpu_architecture(); - BUG_ON(arch == CPU_ARCH_UNKNOWN); - alu_write_pc_interworks = arch >= CPU_ARCH_ARMv7; -} - -#endif /* !test_alu_write_pc_interworking */ - - -void __init arm_kprobe_decode_init(void) -{ - find_str_pc_offset(); - test_load_write_pc_interworking(); - test_alu_write_pc_interworking(); -} - - -static unsigned long __kprobes __check_eq(unsigned long cpsr) -{ - return cpsr & PSR_Z_BIT; -} - -static unsigned long __kprobes __check_ne(unsigned long cpsr) -{ - return (~cpsr) & PSR_Z_BIT; -} - -static unsigned long __kprobes __check_cs(unsigned long cpsr) -{ - return cpsr & PSR_C_BIT; -} - -static unsigned long __kprobes __check_cc(unsigned long cpsr) -{ - return (~cpsr) & PSR_C_BIT; -} - -static unsigned long __kprobes __check_mi(unsigned long cpsr) -{ - return cpsr & PSR_N_BIT; -} - -static unsigned long __kprobes __check_pl(unsigned long cpsr) -{ - return (~cpsr) & PSR_N_BIT; -} - -static unsigned long __kprobes __check_vs(unsigned long cpsr) -{ - return cpsr & PSR_V_BIT; -} - -static unsigned long __kprobes __check_vc(unsigned long cpsr) -{ - return (~cpsr) & PSR_V_BIT; -} - -static unsigned long __kprobes __check_hi(unsigned long cpsr) -{ - cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ - return cpsr & PSR_C_BIT; -} - -static unsigned long __kprobes __check_ls(unsigned long cpsr) -{ - cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ - return (~cpsr) & PSR_C_BIT; -} - -static unsigned long __kprobes __check_ge(unsigned long cpsr) -{ - cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - return (~cpsr) & PSR_N_BIT; -} - -static unsigned long __kprobes __check_lt(unsigned long cpsr) -{ - cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - return cpsr & PSR_N_BIT; -} - -static unsigned long __kprobes __check_gt(unsigned long cpsr) -{ - unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ - return (~temp) & PSR_N_BIT; -} - -static unsigned long __kprobes __check_le(unsigned long cpsr) -{ - unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ - return temp & PSR_N_BIT; -} - -static unsigned long __kprobes __check_al(unsigned long cpsr) -{ - return true; -} - -kprobe_check_cc * const kprobe_condition_checks[16] = { - &__check_eq, &__check_ne, &__check_cs, &__check_cc, - &__check_mi, &__check_pl, &__check_vs, &__check_vc, - &__check_hi, &__check_ls, &__check_ge, &__check_lt, - &__check_gt, &__check_le, &__check_al, &__check_al -}; - - -void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs) -{ -} - -void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs) -{ - p->ainsn.insn_fn(); -} - static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -322,260 +154,3 @@ kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD_NO_SLOT; } - -/* - * Prepare an instruction slot to receive an instruction for emulating. - * This is done by placing a subroutine return after the location where the - * instruction will be placed. We also modify ARM instructions to be - * unconditional as the condition code will already be checked before any - * emulation handler is called. - */ -static kprobe_opcode_t __kprobes -prepare_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - bool thumb) -{ -#ifdef CONFIG_THUMB2_KERNEL - if (thumb) { - u16 *thumb_insn = (u16 *)asi->insn; - thumb_insn[1] = 0x4770; /* Thumb bx lr */ - thumb_insn[2] = 0x4770; /* Thumb bx lr */ - return insn; - } - asi->insn[1] = 0xe12fff1e; /* ARM bx lr */ -#else - asi->insn[1] = 0xe1a0f00e; /* mov pc, lr */ -#endif - /* Make an ARM instruction unconditional */ - if (insn < 0xe0000000) - insn = (insn | 0xe0000000) & ~0x10000000; - return insn; -} - -/* - * Write a (probably modified) instruction into the slot previously prepared by - * prepare_emulated_insn - */ -static void __kprobes -set_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - bool thumb) -{ -#ifdef CONFIG_THUMB2_KERNEL - if (thumb) { - u16 *ip = (u16 *)asi->insn; - if (is_wide_instruction(insn)) - *ip++ = insn >> 16; - *ip++ = insn; - return; - } -#endif - asi->insn[0] = insn; -} - -/* - * When we modify the register numbers encoded in an instruction to be emulated, - * the new values come from this define. For ARM and 32-bit Thumb instructions - * this gives... - * - * bit position 16 12 8 4 0 - * ---------------+---+---+---+---+---+ - * register r2 r0 r1 -- r3 - */ -#define INSN_NEW_BITS 0x00020103 - -/* Each nibble has same value as that at INSN_NEW_BITS bit 16 */ -#define INSN_SAMEAS16_BITS 0x22222222 - -/* - * Validate and modify each of the registers encoded in an instruction. - * - * Each nibble in regs contains a value from enum decode_reg_type. For each - * non-zero value, the corresponding nibble in pinsn is validated and modified - * according to the type. - */ -static bool __kprobes decode_regs(kprobe_opcode_t* pinsn, u32 regs) -{ - kprobe_opcode_t insn = *pinsn; - kprobe_opcode_t mask = 0xf; /* Start at least significant nibble */ - - for (; regs != 0; regs >>= 4, mask <<= 4) { - - kprobe_opcode_t new_bits = INSN_NEW_BITS; - - switch (regs & 0xf) { - - case REG_TYPE_NONE: - /* Nibble not a register, skip to next */ - continue; - - case REG_TYPE_ANY: - /* Any register is allowed */ - break; - - case REG_TYPE_SAMEAS16: - /* Replace register with same as at bit position 16 */ - new_bits = INSN_SAMEAS16_BITS; - break; - - case REG_TYPE_SP: - /* Only allow SP (R13) */ - if ((insn ^ 0xdddddddd) & mask) - goto reject; - break; - - case REG_TYPE_PC: - /* Only allow PC (R15) */ - if ((insn ^ 0xffffffff) & mask) - goto reject; - break; - - case REG_TYPE_NOSP: - /* Reject SP (R13) */ - if (((insn ^ 0xdddddddd) & mask) == 0) - goto reject; - break; - - case REG_TYPE_NOSPPC: - case REG_TYPE_NOSPPCX: - /* Reject SP and PC (R13 and R15) */ - if (((insn ^ 0xdddddddd) & 0xdddddddd & mask) == 0) - goto reject; - break; - - case REG_TYPE_NOPCWB: - if (!is_writeback(insn)) - break; /* No writeback, so any register is OK */ - /* fall through... */ - case REG_TYPE_NOPC: - case REG_TYPE_NOPCX: - /* Reject PC (R15) */ - if (((insn ^ 0xffffffff) & mask) == 0) - goto reject; - break; - } - - /* Replace value of nibble with new register number... */ - insn &= ~mask; - insn |= new_bits & mask; - } - - *pinsn = insn; - return true; - -reject: - return false; -} - -static const int decode_struct_sizes[NUM_DECODE_TYPES] = { - [DECODE_TYPE_TABLE] = sizeof(struct decode_table), - [DECODE_TYPE_CUSTOM] = sizeof(struct decode_custom), - [DECODE_TYPE_SIMULATE] = sizeof(struct decode_simulate), - [DECODE_TYPE_EMULATE] = sizeof(struct decode_emulate), - [DECODE_TYPE_OR] = sizeof(struct decode_or), - [DECODE_TYPE_REJECT] = sizeof(struct decode_reject) -}; - -/* - * kprobe_decode_insn operates on data tables in order to decode an ARM - * architecture instruction onto which a kprobe has been placed. - * - * These instruction decoding tables are a concatenation of entries each - * of which consist of one of the following structs: - * - * decode_table - * decode_custom - * decode_simulate - * decode_emulate - * decode_or - * decode_reject - * - * Each of these starts with a struct decode_header which has the following - * fields: - * - * type_regs - * mask - * value - * - * The least significant DECODE_TYPE_BITS of type_regs contains a value - * from enum decode_type, this indicates which of the decode_* structs - * the entry contains. The value DECODE_TYPE_END indicates the end of the - * table. - * - * When the table is parsed, each entry is checked in turn to see if it - * matches the instruction to be decoded using the test: - * - * (insn & mask) == value - * - * If no match is found before the end of the table is reached then decoding - * fails with INSN_REJECTED. - * - * When a match is found, decode_regs() is called to validate and modify each - * of the registers encoded in the instruction; the data it uses to do this - * is (type_regs >> DECODE_TYPE_BITS). A validation failure will cause decoding - * to fail with INSN_REJECTED. - * - * Once the instruction has passed the above tests, further processing - * depends on the type of the table entry's decode struct. - * - */ -int __kprobes -kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - const union decode_item *table, bool thumb) -{ - const struct decode_header *h = (struct decode_header *)table; - const struct decode_header *next; - bool matched = false; - - insn = prepare_emulated_insn(insn, asi, thumb); - - for (;; h = next) { - enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK; - u32 regs = h->type_regs.bits >> DECODE_TYPE_BITS; - - if (type == DECODE_TYPE_END) - return INSN_REJECTED; - - next = (struct decode_header *) - ((uintptr_t)h + decode_struct_sizes[type]); - - if (!matched && (insn & h->mask.bits) != h->value.bits) - continue; - - if (!decode_regs(&insn, regs)) - return INSN_REJECTED; - - switch (type) { - - case DECODE_TYPE_TABLE: { - struct decode_table *d = (struct decode_table *)h; - next = (struct decode_header *)d->table.table; - break; - } - - case DECODE_TYPE_CUSTOM: { - struct decode_custom *d = (struct decode_custom *)h; - return (*d->decoder.decoder)(insn, asi); - } - - case DECODE_TYPE_SIMULATE: { - struct decode_simulate *d = (struct decode_simulate *)h; - asi->insn_handler = d->handler.handler; - return INSN_GOOD_NO_SLOT; - } - - case DECODE_TYPE_EMULATE: { - struct decode_emulate *d = (struct decode_emulate *)h; - asi->insn_handler = d->handler.handler; - set_emulated_insn(insn, asi, thumb); - return INSN_GOOD; - } - - case DECODE_TYPE_OR: - matched = true; - break; - - case DECODE_TYPE_REJECT: - default: - return INSN_REJECTED; - } - } - } diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index 38945f78f9f1..aa68c0ea1a0b 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -52,377 +52,6 @@ enum kprobe_insn arm_kprobe_decode_insn(kprobe_opcode_t, void __init arm_kprobe_decode_init(void); -extern kprobe_check_cc * const kprobe_condition_checks[16]; - - -#if __LINUX_ARM_ARCH__ >= 7 - -/* str_pc_offset is architecturally defined from ARMv7 onwards */ -#define str_pc_offset 8 -#define find_str_pc_offset() - -#else /* __LINUX_ARM_ARCH__ < 7 */ - -/* We need a run-time check to determine str_pc_offset */ -extern int str_pc_offset; -void __init find_str_pc_offset(void); - -#endif - - -/* - * Update ITSTATE after normal execution of an IT block instruction. - * - * The 8 IT state bits are split into two parts in CPSR: - * ITSTATE<1:0> are in CPSR<26:25> - * ITSTATE<7:2> are in CPSR<15:10> - */ -static inline unsigned long it_advance(unsigned long cpsr) - { - if ((cpsr & 0x06000400) == 0) { - /* ITSTATE<2:0> == 0 means end of IT block, so clear IT state */ - cpsr &= ~PSR_IT_MASK; - } else { - /* We need to shift left ITSTATE<4:0> */ - const unsigned long mask = 0x06001c00; /* Mask ITSTATE<4:0> */ - unsigned long it = cpsr & mask; - it <<= 1; - it |= it >> (27 - 10); /* Carry ITSTATE<2> to correct place */ - it &= mask; - cpsr &= ~mask; - cpsr |= it; - } - return cpsr; -} - -static inline void __kprobes bx_write_pc(long pcv, struct pt_regs *regs) -{ - long cpsr = regs->ARM_cpsr; - if (pcv & 0x1) { - cpsr |= PSR_T_BIT; - pcv &= ~0x1; - } else { - cpsr &= ~PSR_T_BIT; - pcv &= ~0x2; /* Avoid UNPREDICTABLE address allignment */ - } - regs->ARM_cpsr = cpsr; - regs->ARM_pc = pcv; -} - - -#if __LINUX_ARM_ARCH__ >= 6 - -/* Kernels built for >= ARMv6 should never run on <= ARMv5 hardware, so... */ -#define load_write_pc_interworks true -#define test_load_write_pc_interworking() - -#else /* __LINUX_ARM_ARCH__ < 6 */ - -/* We need run-time testing to determine if load_write_pc() should interwork. */ -extern bool load_write_pc_interworks; -void __init test_load_write_pc_interworking(void); - -#endif - -static inline void __kprobes load_write_pc(long pcv, struct pt_regs *regs) -{ - if (load_write_pc_interworks) - bx_write_pc(pcv, regs); - else - regs->ARM_pc = pcv; -} - - -#if __LINUX_ARM_ARCH__ >= 7 - -#define alu_write_pc_interworks true -#define test_alu_write_pc_interworking() - -#elif __LINUX_ARM_ARCH__ <= 5 - -/* Kernels built for <= ARMv5 should never run on >= ARMv6 hardware, so... */ -#define alu_write_pc_interworks false -#define test_alu_write_pc_interworking() - -#else /* __LINUX_ARM_ARCH__ == 6 */ - -/* We could be an ARMv6 binary on ARMv7 hardware so we need a run-time check. */ -extern bool alu_write_pc_interworks; -void __init test_alu_write_pc_interworking(void); - -#endif /* __LINUX_ARM_ARCH__ == 6 */ - -static inline void __kprobes alu_write_pc(long pcv, struct pt_regs *regs) -{ - if (alu_write_pc_interworks) - bx_write_pc(pcv, regs); - else - regs->ARM_pc = pcv; -} - - -void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs); -void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs); - -enum kprobe_insn __kprobes -kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi); - -/* - * Test if load/store instructions writeback the address register. - * if P (bit 24) == 0 or W (bit 21) == 1 - */ -#define is_writeback(insn) ((insn ^ 0x01000000) & 0x01200000) - -/* - * The following definitions and macros are used to build instruction - * decoding tables for use by kprobe_decode_insn. - * - * These tables are a concatenation of entries each of which consist of one of - * the decode_* structs. All of the fields in every type of decode structure - * are of the union type decode_item, therefore the entire decode table can be - * viewed as an array of these and declared like: - * - * static const union decode_item table_name[] = {}; - * - * In order to construct each entry in the table, macros are used to - * initialise a number of sequential decode_item values in a layout which - * matches the relevant struct. E.g. DECODE_SIMULATE initialise a struct - * decode_simulate by initialising four decode_item objects like this... - * - * {.bits = _type}, - * {.bits = _mask}, - * {.bits = _value}, - * {.handler = _handler}, - * - * Initialising a specified member of the union means that the compiler - * will produce a warning if the argument is of an incorrect type. - * - * Below is a list of each of the macros used to initialise entries and a - * description of the action performed when that entry is matched to an - * instruction. A match is found when (instruction & mask) == value. - * - * DECODE_TABLE(mask, value, table) - * Instruction decoding jumps to parsing the new sub-table 'table'. - * - * DECODE_CUSTOM(mask, value, decoder) - * The custom function 'decoder' is called to the complete decoding - * of an instruction. - * - * DECODE_SIMULATE(mask, value, handler) - * Set the probes instruction handler to 'handler', this will be used - * to simulate the instruction when the probe is hit. Decoding returns - * with INSN_GOOD_NO_SLOT. - * - * DECODE_EMULATE(mask, value, handler) - * Set the probes instruction handler to 'handler', this will be used - * to emulate the instruction when the probe is hit. The modified - * instruction (see below) is placed in the probes instruction slot so it - * may be called by the emulation code. Decoding returns with INSN_GOOD. - * - * DECODE_REJECT(mask, value) - * Instruction decoding fails with INSN_REJECTED - * - * DECODE_OR(mask, value) - * This allows the mask/value test of multiple table entries to be - * logically ORed. Once an 'or' entry is matched the decoding action to - * be performed is that of the next entry which isn't an 'or'. E.g. - * - * DECODE_OR (mask1, value1) - * DECODE_OR (mask2, value2) - * DECODE_SIMULATE (mask3, value3, simulation_handler) - * - * This means that if any of the three mask/value pairs match the - * instruction being decoded, then 'simulation_handler' will be used - * for it. - * - * Both the SIMULATE and EMULATE macros have a second form which take an - * additional 'regs' argument. - * - * DECODE_SIMULATEX(mask, value, handler, regs) - * DECODE_EMULATEX (mask, value, handler, regs) - * - * These are used to specify what kind of CPU register is encoded in each of the - * least significant 5 nibbles of the instruction being decoded. The regs value - * is specified using the REGS macro, this takes any of the REG_TYPE_* values - * from enum decode_reg_type as arguments; only the '*' part of the name is - * given. E.g. - * - * REGS(0, ANY, NOPC, 0, ANY) - * - * This indicates an instruction is encoded like: - * - * bits 19..16 ignore - * bits 15..12 any register allowed here - * bits 11.. 8 any register except PC allowed here - * bits 7.. 4 ignore - * bits 3.. 0 any register allowed here - * - * This register specification is checked after a decode table entry is found to - * match an instruction (through the mask/value test). Any invalid register then - * found in the instruction will cause decoding to fail with INSN_REJECTED. In - * the above example this would happen if bits 11..8 of the instruction were - * 1111, indicating R15 or PC. - * - * As well as checking for legal combinations of registers, this data is also - * used to modify the registers encoded in the instructions so that an - * emulation routines can use it. (See decode_regs() and INSN_NEW_BITS.) - * - * Here is a real example which matches ARM instructions of the form - * "AND ,,, " - * - * DECODE_EMULATEX (0x0e000090, 0x00000010, emulate_rd12rn16rm0rs8_rwflags, - * REGS(ANY, ANY, NOPC, 0, ANY)), - * ^ ^ ^ ^ - * Rn Rd Rs Rm - * - * Decoding the instruction "AND R4, R5, R6, ASL R15" will be rejected because - * Rs == R15 - * - * Decoding the instruction "AND R4, R5, R6, ASL R7" will be accepted and the - * instruction will be modified to "AND R0, R2, R3, ASL R1" and then placed into - * the kprobes instruction slot. This can then be called later by the handler - * function emulate_rd12rn16rm0rs8_rwflags in order to simulate the instruction. - */ - -enum decode_type { - DECODE_TYPE_END, - DECODE_TYPE_TABLE, - DECODE_TYPE_CUSTOM, - DECODE_TYPE_SIMULATE, - DECODE_TYPE_EMULATE, - DECODE_TYPE_OR, - DECODE_TYPE_REJECT, - NUM_DECODE_TYPES /* Must be last enum */ -}; - -#define DECODE_TYPE_BITS 4 -#define DECODE_TYPE_MASK ((1 << DECODE_TYPE_BITS) - 1) - -enum decode_reg_type { - REG_TYPE_NONE = 0, /* Not a register, ignore */ - REG_TYPE_ANY, /* Any register allowed */ - REG_TYPE_SAMEAS16, /* Register should be same as that at bits 19..16 */ - REG_TYPE_SP, /* Register must be SP */ - REG_TYPE_PC, /* Register must be PC */ - REG_TYPE_NOSP, /* Register must not be SP */ - REG_TYPE_NOSPPC, /* Register must not be SP or PC */ - REG_TYPE_NOPC, /* Register must not be PC */ - REG_TYPE_NOPCWB, /* No PC if load/store write-back flag also set */ - - /* The following types are used when the encoding for PC indicates - * another instruction form. This distiction only matters for test - * case coverage checks. - */ - REG_TYPE_NOPCX, /* Register must not be PC */ - REG_TYPE_NOSPPCX, /* Register must not be SP or PC */ - - /* Alias to allow '0' arg to be used in REGS macro. */ - REG_TYPE_0 = REG_TYPE_NONE -}; - -#define REGS(r16, r12, r8, r4, r0) \ - ((REG_TYPE_##r16) << 16) + \ - ((REG_TYPE_##r12) << 12) + \ - ((REG_TYPE_##r8) << 8) + \ - ((REG_TYPE_##r4) << 4) + \ - (REG_TYPE_##r0) - -union decode_item { - u32 bits; - const union decode_item *table; - kprobe_insn_handler_t *handler; - kprobe_decode_insn_t *decoder; -}; - - -#define DECODE_END \ - {.bits = DECODE_TYPE_END} - - -struct decode_header { - union decode_item type_regs; - union decode_item mask; - union decode_item value; -}; - -#define DECODE_HEADER(_type, _mask, _value, _regs) \ - {.bits = (_type) | ((_regs) << DECODE_TYPE_BITS)}, \ - {.bits = (_mask)}, \ - {.bits = (_value)} - - -struct decode_table { - struct decode_header header; - union decode_item table; -}; - -#define DECODE_TABLE(_mask, _value, _table) \ - DECODE_HEADER(DECODE_TYPE_TABLE, _mask, _value, 0), \ - {.table = (_table)} - - -struct decode_custom { - struct decode_header header; - union decode_item decoder; -}; - -#define DECODE_CUSTOM(_mask, _value, _decoder) \ - DECODE_HEADER(DECODE_TYPE_CUSTOM, _mask, _value, 0), \ - {.decoder = (_decoder)} - - -struct decode_simulate { - struct decode_header header; - union decode_item handler; -}; - -#define DECODE_SIMULATEX(_mask, _value, _handler, _regs) \ - DECODE_HEADER(DECODE_TYPE_SIMULATE, _mask, _value, _regs), \ - {.handler = (_handler)} - -#define DECODE_SIMULATE(_mask, _value, _handler) \ - DECODE_SIMULATEX(_mask, _value, _handler, 0) - - -struct decode_emulate { - struct decode_header header; - union decode_item handler; -}; - -#define DECODE_EMULATEX(_mask, _value, _handler, _regs) \ - DECODE_HEADER(DECODE_TYPE_EMULATE, _mask, _value, _regs), \ - {.handler = (_handler)} - -#define DECODE_EMULATE(_mask, _value, _handler) \ - DECODE_EMULATEX(_mask, _value, _handler, 0) - - -struct decode_or { - struct decode_header header; -}; - -#define DECODE_OR(_mask, _value) \ - DECODE_HEADER(DECODE_TYPE_OR, _mask, _value, 0) - - -struct decode_reject { - struct decode_header header; -}; - -#define DECODE_REJECT(_mask, _value) \ - DECODE_HEADER(DECODE_TYPE_REJECT, _mask, _value, 0) - - -#ifdef CONFIG_THUMB2_KERNEL -extern const union decode_item kprobe_decode_thumb16_table[]; -extern const union decode_item kprobe_decode_thumb32_table[]; -#else -extern const union decode_item kprobe_decode_arm_table[]; -#endif - - -int kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - const union decode_item *table, bool thumb16); - +#include "probes.h" #endif /* _ARM_KERNEL_KPROBES_H */ diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c new file mode 100644 index 000000000000..57e08b28e87f --- /dev/null +++ b/arch/arm/kernel/probes-arm.c @@ -0,0 +1,731 @@ +/* + * arch/arm/kernel/probes-arm.c + * + * Some code moved here from arch/arm/kernel/kprobes-arm.c + * + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include + +#include "kprobes.h" +#include "probes-arm.h" + +#define sign_extend(x, signbit) ((x) | (0 - ((x) & (1 << (signbit))))) + +#define branch_displacement(insn) sign_extend(((insn) & 0xffffff) << 2, 25) + +/* + * To avoid the complications of mimicing single-stepping on a + * processor without a Next-PC or a single-step mode, and to + * avoid having to deal with the side-effects of boosting, we + * simulate or emulate (almost) all ARM instructions. + * + * "Simulation" is where the instruction's behavior is duplicated in + * C code. "Emulation" is where the original instruction is rewritten + * and executed, often by altering its registers. + * + * By having all behavior of the kprobe'd instruction completed before + * returning from the kprobe_handler(), all locks (scheduler and + * interrupt) can safely be released. There is no need for secondary + * breakpoints, no race with MP or preemptable kernels, nor having to + * clean up resources counts at a later time impacting overall system + * performance. By rewriting the instruction, only the minimum registers + * need to be loaded and saved back optimizing performance. + * + * Calling the insnslot_*_rwflags version of a function doesn't hurt + * anything even when the CPSR flags aren't updated by the + * instruction. It's just a little slower in return for saving + * a little space by not having a duplicate function that doesn't + * update the flags. (The same optimization can be said for + * instructions that do or don't perform register writeback) + * Also, instructions can either read the flags, only write the + * flags, or read and write the flags. To save combinations + * rather than for sheer performance, flag functions just assume + * read and write of flags. + */ + +void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + long iaddr = (long)p->addr; + int disp = branch_displacement(insn); + + if (insn & (1 << 24)) + regs->ARM_lr = iaddr + 4; + + regs->ARM_pc = iaddr + 8 + disp; +} + +void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + long iaddr = (long)p->addr; + int disp = branch_displacement(insn); + + regs->ARM_lr = iaddr + 4; + regs->ARM_pc = iaddr + 8 + disp + ((insn >> 23) & 0x2); + regs->ARM_cpsr |= PSR_T_BIT; +} + +void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + int rm = insn & 0xf; + long rmv = regs->uregs[rm]; + + if (insn & (1 << 5)) + regs->ARM_lr = (long)p->addr + 4; + + regs->ARM_pc = rmv & ~0x1; + regs->ARM_cpsr &= ~PSR_T_BIT; + if (rmv & 0x1) + regs->ARM_cpsr |= PSR_T_BIT; +} + +void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + unsigned long mask = 0xf8ff03df; /* Mask out execution state */ + regs->uregs[rd] = regs->ARM_cpsr & mask; +} + +void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs) +{ + regs->uregs[12] = regs->uregs[13]; +} + +/* + * For the instruction masking and comparisons in all the "space_*" + * functions below, Do _not_ rearrange the order of tests unless + * you're very, very sure of what you are doing. For the sake of + * efficiency, the masks for some tests sometimes assume other test + * have been done prior to them so the number of patterns to test + * for an instruction set can be as broad as possible to reduce the + * number of tests needed. + */ + +static const union decode_item arm_1111_table[] = { + /* Unconditional instructions */ + + /* memory hint 1111 0100 x001 xxxx xxxx xxxx xxxx xxxx */ + /* PLDI (immediate) 1111 0100 x101 xxxx xxxx xxxx xxxx xxxx */ + /* PLDW (immediate) 1111 0101 x001 xxxx xxxx xxxx xxxx xxxx */ + /* PLD (immediate) 1111 0101 x101 xxxx xxxx xxxx xxxx xxxx */ + DECODE_SIMULATE (0xfe300000, 0xf4100000, kprobe_simulate_nop), + + /* memory hint 1111 0110 x001 xxxx xxxx xxxx xxx0 xxxx */ + /* PLDI (register) 1111 0110 x101 xxxx xxxx xxxx xxx0 xxxx */ + /* PLDW (register) 1111 0111 x001 xxxx xxxx xxxx xxx0 xxxx */ + /* PLD (register) 1111 0111 x101 xxxx xxxx xxxx xxx0 xxxx */ + DECODE_SIMULATE (0xfe300010, 0xf6100000, kprobe_simulate_nop), + + /* BLX (immediate) 1111 101x xxxx xxxx xxxx xxxx xxxx xxxx */ + DECODE_SIMULATE (0xfe000000, 0xfa000000, simulate_blx1), + + /* CPS 1111 0001 0000 xxx0 xxxx xxxx xx0x xxxx */ + /* SETEND 1111 0001 0000 0001 xxxx xxxx 0000 xxxx */ + /* SRS 1111 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ + /* RFE 1111 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ + + /* Coprocessor instructions... */ + /* MCRR2 1111 1100 0100 xxxx xxxx xxxx xxxx xxxx */ + /* MRRC2 1111 1100 0101 xxxx xxxx xxxx xxxx xxxx */ + /* LDC2 1111 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ + /* STC2 1111 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ + /* CDP2 1111 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ + /* MCR2 1111 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ + /* MRC2 1111 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item arm_cccc_0001_0xx0____0xxx_table[] = { + /* Miscellaneous instructions */ + + /* MRS cpsr cccc 0001 0000 xxxx xxxx xxxx 0000 xxxx */ + DECODE_SIMULATEX(0x0ff000f0, 0x01000000, simulate_mrs, + REGS(0, NOPC, 0, 0, 0)), + + /* BX cccc 0001 0010 xxxx xxxx xxxx 0001 xxxx */ + DECODE_SIMULATE (0x0ff000f0, 0x01200010, simulate_blx2bx), + + /* BLX (register) cccc 0001 0010 xxxx xxxx xxxx 0011 xxxx */ + DECODE_SIMULATEX(0x0ff000f0, 0x01200030, simulate_blx2bx, + REGS(0, 0, 0, 0, NOPC)), + + /* CLZ cccc 0001 0110 xxxx xxxx xxxx 0001 xxxx */ + DECODE_EMULATEX (0x0ff000f0, 0x01600010, emulate_rd12rm0_noflags_nopc, + REGS(0, NOPC, 0, 0, NOPC)), + + /* QADD cccc 0001 0000 xxxx xxxx xxxx 0101 xxxx */ + /* QSUB cccc 0001 0010 xxxx xxxx xxxx 0101 xxxx */ + /* QDADD cccc 0001 0100 xxxx xxxx xxxx 0101 xxxx */ + /* QDSUB cccc 0001 0110 xxxx xxxx xxxx 0101 xxxx */ + DECODE_EMULATEX (0x0f9000f0, 0x01000050, emulate_rd12rn16rm0_rwflags_nopc, + REGS(NOPC, NOPC, 0, 0, NOPC)), + + /* BXJ cccc 0001 0010 xxxx xxxx xxxx 0010 xxxx */ + /* MSR cccc 0001 0x10 xxxx xxxx xxxx 0000 xxxx */ + /* MRS spsr cccc 0001 0100 xxxx xxxx xxxx 0000 xxxx */ + /* BKPT 1110 0001 0010 xxxx xxxx xxxx 0111 xxxx */ + /* SMC cccc 0001 0110 xxxx xxxx xxxx 0111 xxxx */ + /* And unallocated instructions... */ + DECODE_END +}; + +static const union decode_item arm_cccc_0001_0xx0____1xx0_table[] = { + /* Halfword multiply and multiply-accumulate */ + + /* SMLALxy cccc 0001 0100 xxxx xxxx xxxx 1xx0 xxxx */ + DECODE_EMULATEX (0x0ff00090, 0x01400080, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + /* SMULWy cccc 0001 0010 xxxx xxxx xxxx 1x10 xxxx */ + DECODE_OR (0x0ff000b0, 0x012000a0), + /* SMULxy cccc 0001 0110 xxxx xxxx xxxx 1xx0 xxxx */ + DECODE_EMULATEX (0x0ff00090, 0x01600080, emulate_rd16rn12rm0rs8_rwflags_nopc, + REGS(NOPC, 0, NOPC, 0, NOPC)), + + /* SMLAxy cccc 0001 0000 xxxx xxxx xxxx 1xx0 xxxx */ + DECODE_OR (0x0ff00090, 0x01000080), + /* SMLAWy cccc 0001 0010 xxxx xxxx xxxx 1x00 xxxx */ + DECODE_EMULATEX (0x0ff000b0, 0x01200080, emulate_rd16rn12rm0rs8_rwflags_nopc, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + DECODE_END +}; + +static const union decode_item arm_cccc_0000_____1001_table[] = { + /* Multiply and multiply-accumulate */ + + /* MUL cccc 0000 0000 xxxx xxxx xxxx 1001 xxxx */ + /* MULS cccc 0000 0001 xxxx xxxx xxxx 1001 xxxx */ + DECODE_EMULATEX (0x0fe000f0, 0x00000090, emulate_rd16rn12rm0rs8_rwflags_nopc, + REGS(NOPC, 0, NOPC, 0, NOPC)), + + /* MLA cccc 0000 0010 xxxx xxxx xxxx 1001 xxxx */ + /* MLAS cccc 0000 0011 xxxx xxxx xxxx 1001 xxxx */ + DECODE_OR (0x0fe000f0, 0x00200090), + /* MLS cccc 0000 0110 xxxx xxxx xxxx 1001 xxxx */ + DECODE_EMULATEX (0x0ff000f0, 0x00600090, emulate_rd16rn12rm0rs8_rwflags_nopc, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + /* UMAAL cccc 0000 0100 xxxx xxxx xxxx 1001 xxxx */ + DECODE_OR (0x0ff000f0, 0x00400090), + /* UMULL cccc 0000 1000 xxxx xxxx xxxx 1001 xxxx */ + /* UMULLS cccc 0000 1001 xxxx xxxx xxxx 1001 xxxx */ + /* UMLAL cccc 0000 1010 xxxx xxxx xxxx 1001 xxxx */ + /* UMLALS cccc 0000 1011 xxxx xxxx xxxx 1001 xxxx */ + /* SMULL cccc 0000 1100 xxxx xxxx xxxx 1001 xxxx */ + /* SMULLS cccc 0000 1101 xxxx xxxx xxxx 1001 xxxx */ + /* SMLAL cccc 0000 1110 xxxx xxxx xxxx 1001 xxxx */ + /* SMLALS cccc 0000 1111 xxxx xxxx xxxx 1001 xxxx */ + DECODE_EMULATEX (0x0f8000f0, 0x00800090, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + DECODE_END +}; + +static const union decode_item arm_cccc_0001_____1001_table[] = { + /* Synchronization primitives */ + +#if __LINUX_ARM_ARCH__ < 6 + /* Deprecated on ARMv6 and may be UNDEFINED on v7 */ + /* SMP/SWPB cccc 0001 0x00 xxxx xxxx xxxx 1001 xxxx */ + DECODE_EMULATEX (0x0fb000f0, 0x01000090, emulate_rd12rn16rm0_rwflags_nopc, + REGS(NOPC, NOPC, 0, 0, NOPC)), +#endif + /* LDREX/STREX{,D,B,H} cccc 0001 1xxx xxxx xxxx xxxx 1001 xxxx */ + /* And unallocated instructions... */ + DECODE_END +}; + +static const union decode_item arm_cccc_000x_____1xx1_table[] = { + /* Extra load/store instructions */ + + /* STRHT cccc 0000 xx10 xxxx xxxx xxxx 1011 xxxx */ + /* ??? cccc 0000 xx10 xxxx xxxx xxxx 11x1 xxxx */ + /* LDRHT cccc 0000 xx11 xxxx xxxx xxxx 1011 xxxx */ + /* LDRSBT cccc 0000 xx11 xxxx xxxx xxxx 1101 xxxx */ + /* LDRSHT cccc 0000 xx11 xxxx xxxx xxxx 1111 xxxx */ + DECODE_REJECT (0x0f200090, 0x00200090), + + /* LDRD/STRD lr,pc,{... cccc 000x x0x0 xxxx 111x xxxx 1101 xxxx */ + DECODE_REJECT (0x0e10e0d0, 0x0000e0d0), + + /* LDRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1101 xxxx */ + /* STRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0e5000d0, 0x000000d0, emulate_ldrdstrd, + REGS(NOPCWB, NOPCX, 0, 0, NOPC)), + + /* LDRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1101 xxxx */ + /* STRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0e5000d0, 0x004000d0, emulate_ldrdstrd, + REGS(NOPCWB, NOPCX, 0, 0, 0)), + + /* STRH (register) cccc 000x x0x0 xxxx xxxx xxxx 1011 xxxx */ + DECODE_EMULATEX (0x0e5000f0, 0x000000b0, emulate_str, + REGS(NOPCWB, NOPC, 0, 0, NOPC)), + + /* LDRH (register) cccc 000x x0x1 xxxx xxxx xxxx 1011 xxxx */ + /* LDRSB (register) cccc 000x x0x1 xxxx xxxx xxxx 1101 xxxx */ + /* LDRSH (register) cccc 000x x0x1 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0e500090, 0x00100090, emulate_ldr, + REGS(NOPCWB, NOPC, 0, 0, NOPC)), + + /* STRH (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1011 xxxx */ + DECODE_EMULATEX (0x0e5000f0, 0x004000b0, emulate_str, + REGS(NOPCWB, NOPC, 0, 0, 0)), + + /* LDRH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1011 xxxx */ + /* LDRSB (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1101 xxxx */ + /* LDRSH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0e500090, 0x00500090, emulate_ldr, + REGS(NOPCWB, NOPC, 0, 0, 0)), + + DECODE_END +}; + +static const union decode_item arm_cccc_000x_table[] = { + /* Data-processing (register) */ + + /* S PC, ... cccc 000x xxx1 xxxx 1111 xxxx xxxx xxxx */ + DECODE_REJECT (0x0e10f000, 0x0010f000), + + /* MOV IP, SP 1110 0001 1010 0000 1100 0000 0000 1101 */ + DECODE_SIMULATE (0xffffffff, 0xe1a0c00d, simulate_mov_ipsp), + + /* TST (register) cccc 0001 0001 xxxx xxxx xxxx xxx0 xxxx */ + /* TEQ (register) cccc 0001 0011 xxxx xxxx xxxx xxx0 xxxx */ + /* CMP (register) cccc 0001 0101 xxxx xxxx xxxx xxx0 xxxx */ + /* CMN (register) cccc 0001 0111 xxxx xxxx xxxx xxx0 xxxx */ + DECODE_EMULATEX (0x0f900010, 0x01100000, emulate_rd12rn16rm0rs8_rwflags, + REGS(ANY, 0, 0, 0, ANY)), + + /* MOV (register) cccc 0001 101x xxxx xxxx xxxx xxx0 xxxx */ + /* MVN (register) cccc 0001 111x xxxx xxxx xxxx xxx0 xxxx */ + DECODE_EMULATEX (0x0fa00010, 0x01a00000, emulate_rd12rn16rm0rs8_rwflags, + REGS(0, ANY, 0, 0, ANY)), + + /* AND (register) cccc 0000 000x xxxx xxxx xxxx xxx0 xxxx */ + /* EOR (register) cccc 0000 001x xxxx xxxx xxxx xxx0 xxxx */ + /* SUB (register) cccc 0000 010x xxxx xxxx xxxx xxx0 xxxx */ + /* RSB (register) cccc 0000 011x xxxx xxxx xxxx xxx0 xxxx */ + /* ADD (register) cccc 0000 100x xxxx xxxx xxxx xxx0 xxxx */ + /* ADC (register) cccc 0000 101x xxxx xxxx xxxx xxx0 xxxx */ + /* SBC (register) cccc 0000 110x xxxx xxxx xxxx xxx0 xxxx */ + /* RSC (register) cccc 0000 111x xxxx xxxx xxxx xxx0 xxxx */ + /* ORR (register) cccc 0001 100x xxxx xxxx xxxx xxx0 xxxx */ + /* BIC (register) cccc 0001 110x xxxx xxxx xxxx xxx0 xxxx */ + DECODE_EMULATEX (0x0e000010, 0x00000000, emulate_rd12rn16rm0rs8_rwflags, + REGS(ANY, ANY, 0, 0, ANY)), + + /* TST (reg-shift reg) cccc 0001 0001 xxxx xxxx xxxx 0xx1 xxxx */ + /* TEQ (reg-shift reg) cccc 0001 0011 xxxx xxxx xxxx 0xx1 xxxx */ + /* CMP (reg-shift reg) cccc 0001 0101 xxxx xxxx xxxx 0xx1 xxxx */ + /* CMN (reg-shift reg) cccc 0001 0111 xxxx xxxx xxxx 0xx1 xxxx */ + DECODE_EMULATEX (0x0f900090, 0x01100010, emulate_rd12rn16rm0rs8_rwflags, + REGS(ANY, 0, NOPC, 0, ANY)), + + /* MOV (reg-shift reg) cccc 0001 101x xxxx xxxx xxxx 0xx1 xxxx */ + /* MVN (reg-shift reg) cccc 0001 111x xxxx xxxx xxxx 0xx1 xxxx */ + DECODE_EMULATEX (0x0fa00090, 0x01a00010, emulate_rd12rn16rm0rs8_rwflags, + REGS(0, ANY, NOPC, 0, ANY)), + + /* AND (reg-shift reg) cccc 0000 000x xxxx xxxx xxxx 0xx1 xxxx */ + /* EOR (reg-shift reg) cccc 0000 001x xxxx xxxx xxxx 0xx1 xxxx */ + /* SUB (reg-shift reg) cccc 0000 010x xxxx xxxx xxxx 0xx1 xxxx */ + /* RSB (reg-shift reg) cccc 0000 011x xxxx xxxx xxxx 0xx1 xxxx */ + /* ADD (reg-shift reg) cccc 0000 100x xxxx xxxx xxxx 0xx1 xxxx */ + /* ADC (reg-shift reg) cccc 0000 101x xxxx xxxx xxxx 0xx1 xxxx */ + /* SBC (reg-shift reg) cccc 0000 110x xxxx xxxx xxxx 0xx1 xxxx */ + /* RSC (reg-shift reg) cccc 0000 111x xxxx xxxx xxxx 0xx1 xxxx */ + /* ORR (reg-shift reg) cccc 0001 100x xxxx xxxx xxxx 0xx1 xxxx */ + /* BIC (reg-shift reg) cccc 0001 110x xxxx xxxx xxxx 0xx1 xxxx */ + DECODE_EMULATEX (0x0e000090, 0x00000010, emulate_rd12rn16rm0rs8_rwflags, + REGS(ANY, ANY, NOPC, 0, ANY)), + + DECODE_END +}; + +static const union decode_item arm_cccc_001x_table[] = { + /* Data-processing (immediate) */ + + /* MOVW cccc 0011 0000 xxxx xxxx xxxx xxxx xxxx */ + /* MOVT cccc 0011 0100 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0fb00000, 0x03000000, emulate_rd12rm0_noflags_nopc, + REGS(0, NOPC, 0, 0, 0)), + + /* YIELD cccc 0011 0010 0000 xxxx xxxx 0000 0001 */ + DECODE_OR (0x0fff00ff, 0x03200001), + /* SEV cccc 0011 0010 0000 xxxx xxxx 0000 0100 */ + DECODE_EMULATE (0x0fff00ff, 0x03200004, kprobe_emulate_none), + /* NOP cccc 0011 0010 0000 xxxx xxxx 0000 0000 */ + /* WFE cccc 0011 0010 0000 xxxx xxxx 0000 0010 */ + /* WFI cccc 0011 0010 0000 xxxx xxxx 0000 0011 */ + DECODE_SIMULATE (0x0fff00fc, 0x03200000, kprobe_simulate_nop), + /* DBG cccc 0011 0010 0000 xxxx xxxx ffff xxxx */ + /* unallocated hints cccc 0011 0010 0000 xxxx xxxx xxxx xxxx */ + /* MSR (immediate) cccc 0011 0x10 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0x0fb00000, 0x03200000), + + /* S PC, ... cccc 001x xxx1 xxxx 1111 xxxx xxxx xxxx */ + DECODE_REJECT (0x0e10f000, 0x0210f000), + + /* TST (immediate) cccc 0011 0001 xxxx xxxx xxxx xxxx xxxx */ + /* TEQ (immediate) cccc 0011 0011 xxxx xxxx xxxx xxxx xxxx */ + /* CMP (immediate) cccc 0011 0101 xxxx xxxx xxxx xxxx xxxx */ + /* CMN (immediate) cccc 0011 0111 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0f900000, 0x03100000, emulate_rd12rn16rm0rs8_rwflags, + REGS(ANY, 0, 0, 0, 0)), + + /* MOV (immediate) cccc 0011 101x xxxx xxxx xxxx xxxx xxxx */ + /* MVN (immediate) cccc 0011 111x xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0fa00000, 0x03a00000, emulate_rd12rn16rm0rs8_rwflags, + REGS(0, ANY, 0, 0, 0)), + + /* AND (immediate) cccc 0010 000x xxxx xxxx xxxx xxxx xxxx */ + /* EOR (immediate) cccc 0010 001x xxxx xxxx xxxx xxxx xxxx */ + /* SUB (immediate) cccc 0010 010x xxxx xxxx xxxx xxxx xxxx */ + /* RSB (immediate) cccc 0010 011x xxxx xxxx xxxx xxxx xxxx */ + /* ADD (immediate) cccc 0010 100x xxxx xxxx xxxx xxxx xxxx */ + /* ADC (immediate) cccc 0010 101x xxxx xxxx xxxx xxxx xxxx */ + /* SBC (immediate) cccc 0010 110x xxxx xxxx xxxx xxxx xxxx */ + /* RSC (immediate) cccc 0010 111x xxxx xxxx xxxx xxxx xxxx */ + /* ORR (immediate) cccc 0011 100x xxxx xxxx xxxx xxxx xxxx */ + /* BIC (immediate) cccc 0011 110x xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e000000, 0x02000000, emulate_rd12rn16rm0rs8_rwflags, + REGS(ANY, ANY, 0, 0, 0)), + + DECODE_END +}; + +static const union decode_item arm_cccc_0110_____xxx1_table[] = { + /* Media instructions */ + + /* SEL cccc 0110 1000 xxxx xxxx xxxx 1011 xxxx */ + DECODE_EMULATEX (0x0ff000f0, 0x068000b0, emulate_rd12rn16rm0_rwflags_nopc, + REGS(NOPC, NOPC, 0, 0, NOPC)), + + /* SSAT cccc 0110 101x xxxx xxxx xxxx xx01 xxxx */ + /* USAT cccc 0110 111x xxxx xxxx xxxx xx01 xxxx */ + DECODE_OR(0x0fa00030, 0x06a00010), + /* SSAT16 cccc 0110 1010 xxxx xxxx xxxx 0011 xxxx */ + /* USAT16 cccc 0110 1110 xxxx xxxx xxxx 0011 xxxx */ + DECODE_EMULATEX (0x0fb000f0, 0x06a00030, emulate_rd12rn16rm0_rwflags_nopc, + REGS(0, NOPC, 0, 0, NOPC)), + + /* REV cccc 0110 1011 xxxx xxxx xxxx 0011 xxxx */ + /* REV16 cccc 0110 1011 xxxx xxxx xxxx 1011 xxxx */ + /* RBIT cccc 0110 1111 xxxx xxxx xxxx 0011 xxxx */ + /* REVSH cccc 0110 1111 xxxx xxxx xxxx 1011 xxxx */ + DECODE_EMULATEX (0x0fb00070, 0x06b00030, emulate_rd12rm0_noflags_nopc, + REGS(0, NOPC, 0, 0, NOPC)), + + /* ??? cccc 0110 0x00 xxxx xxxx xxxx xxx1 xxxx */ + DECODE_REJECT (0x0fb00010, 0x06000010), + /* ??? cccc 0110 0xxx xxxx xxxx xxxx 1011 xxxx */ + DECODE_REJECT (0x0f8000f0, 0x060000b0), + /* ??? cccc 0110 0xxx xxxx xxxx xxxx 1101 xxxx */ + DECODE_REJECT (0x0f8000f0, 0x060000d0), + /* SADD16 cccc 0110 0001 xxxx xxxx xxxx 0001 xxxx */ + /* SADDSUBX cccc 0110 0001 xxxx xxxx xxxx 0011 xxxx */ + /* SSUBADDX cccc 0110 0001 xxxx xxxx xxxx 0101 xxxx */ + /* SSUB16 cccc 0110 0001 xxxx xxxx xxxx 0111 xxxx */ + /* SADD8 cccc 0110 0001 xxxx xxxx xxxx 1001 xxxx */ + /* SSUB8 cccc 0110 0001 xxxx xxxx xxxx 1111 xxxx */ + /* QADD16 cccc 0110 0010 xxxx xxxx xxxx 0001 xxxx */ + /* QADDSUBX cccc 0110 0010 xxxx xxxx xxxx 0011 xxxx */ + /* QSUBADDX cccc 0110 0010 xxxx xxxx xxxx 0101 xxxx */ + /* QSUB16 cccc 0110 0010 xxxx xxxx xxxx 0111 xxxx */ + /* QADD8 cccc 0110 0010 xxxx xxxx xxxx 1001 xxxx */ + /* QSUB8 cccc 0110 0010 xxxx xxxx xxxx 1111 xxxx */ + /* SHADD16 cccc 0110 0011 xxxx xxxx xxxx 0001 xxxx */ + /* SHADDSUBX cccc 0110 0011 xxxx xxxx xxxx 0011 xxxx */ + /* SHSUBADDX cccc 0110 0011 xxxx xxxx xxxx 0101 xxxx */ + /* SHSUB16 cccc 0110 0011 xxxx xxxx xxxx 0111 xxxx */ + /* SHADD8 cccc 0110 0011 xxxx xxxx xxxx 1001 xxxx */ + /* SHSUB8 cccc 0110 0011 xxxx xxxx xxxx 1111 xxxx */ + /* UADD16 cccc 0110 0101 xxxx xxxx xxxx 0001 xxxx */ + /* UADDSUBX cccc 0110 0101 xxxx xxxx xxxx 0011 xxxx */ + /* USUBADDX cccc 0110 0101 xxxx xxxx xxxx 0101 xxxx */ + /* USUB16 cccc 0110 0101 xxxx xxxx xxxx 0111 xxxx */ + /* UADD8 cccc 0110 0101 xxxx xxxx xxxx 1001 xxxx */ + /* USUB8 cccc 0110 0101 xxxx xxxx xxxx 1111 xxxx */ + /* UQADD16 cccc 0110 0110 xxxx xxxx xxxx 0001 xxxx */ + /* UQADDSUBX cccc 0110 0110 xxxx xxxx xxxx 0011 xxxx */ + /* UQSUBADDX cccc 0110 0110 xxxx xxxx xxxx 0101 xxxx */ + /* UQSUB16 cccc 0110 0110 xxxx xxxx xxxx 0111 xxxx */ + /* UQADD8 cccc 0110 0110 xxxx xxxx xxxx 1001 xxxx */ + /* UQSUB8 cccc 0110 0110 xxxx xxxx xxxx 1111 xxxx */ + /* UHADD16 cccc 0110 0111 xxxx xxxx xxxx 0001 xxxx */ + /* UHADDSUBX cccc 0110 0111 xxxx xxxx xxxx 0011 xxxx */ + /* UHSUBADDX cccc 0110 0111 xxxx xxxx xxxx 0101 xxxx */ + /* UHSUB16 cccc 0110 0111 xxxx xxxx xxxx 0111 xxxx */ + /* UHADD8 cccc 0110 0111 xxxx xxxx xxxx 1001 xxxx */ + /* UHSUB8 cccc 0110 0111 xxxx xxxx xxxx 1111 xxxx */ + DECODE_EMULATEX (0x0f800010, 0x06000010, emulate_rd12rn16rm0_rwflags_nopc, + REGS(NOPC, NOPC, 0, 0, NOPC)), + + /* PKHBT cccc 0110 1000 xxxx xxxx xxxx x001 xxxx */ + /* PKHTB cccc 0110 1000 xxxx xxxx xxxx x101 xxxx */ + DECODE_EMULATEX (0x0ff00030, 0x06800010, emulate_rd12rn16rm0_rwflags_nopc, + REGS(NOPC, NOPC, 0, 0, NOPC)), + + /* ??? cccc 0110 1001 xxxx xxxx xxxx 0111 xxxx */ + /* ??? cccc 0110 1101 xxxx xxxx xxxx 0111 xxxx */ + DECODE_REJECT (0x0fb000f0, 0x06900070), + + /* SXTB16 cccc 0110 1000 1111 xxxx xxxx 0111 xxxx */ + /* SXTB cccc 0110 1010 1111 xxxx xxxx 0111 xxxx */ + /* SXTH cccc 0110 1011 1111 xxxx xxxx 0111 xxxx */ + /* UXTB16 cccc 0110 1100 1111 xxxx xxxx 0111 xxxx */ + /* UXTB cccc 0110 1110 1111 xxxx xxxx 0111 xxxx */ + /* UXTH cccc 0110 1111 1111 xxxx xxxx 0111 xxxx */ + DECODE_EMULATEX (0x0f8f00f0, 0x068f0070, emulate_rd12rm0_noflags_nopc, + REGS(0, NOPC, 0, 0, NOPC)), + + /* SXTAB16 cccc 0110 1000 xxxx xxxx xxxx 0111 xxxx */ + /* SXTAB cccc 0110 1010 xxxx xxxx xxxx 0111 xxxx */ + /* SXTAH cccc 0110 1011 xxxx xxxx xxxx 0111 xxxx */ + /* UXTAB16 cccc 0110 1100 xxxx xxxx xxxx 0111 xxxx */ + /* UXTAB cccc 0110 1110 xxxx xxxx xxxx 0111 xxxx */ + /* UXTAH cccc 0110 1111 xxxx xxxx xxxx 0111 xxxx */ + DECODE_EMULATEX (0x0f8000f0, 0x06800070, emulate_rd12rn16rm0_rwflags_nopc, + REGS(NOPCX, NOPC, 0, 0, NOPC)), + + DECODE_END +}; + +static const union decode_item arm_cccc_0111_____xxx1_table[] = { + /* Media instructions */ + + /* UNDEFINED cccc 0111 1111 xxxx xxxx xxxx 1111 xxxx */ + DECODE_REJECT (0x0ff000f0, 0x07f000f0), + + /* SMLALD cccc 0111 0100 xxxx xxxx xxxx 00x1 xxxx */ + /* SMLSLD cccc 0111 0100 xxxx xxxx xxxx 01x1 xxxx */ + DECODE_EMULATEX (0x0ff00090, 0x07400010, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + /* SMUAD cccc 0111 0000 xxxx 1111 xxxx 00x1 xxxx */ + /* SMUSD cccc 0111 0000 xxxx 1111 xxxx 01x1 xxxx */ + DECODE_OR (0x0ff0f090, 0x0700f010), + /* SMMUL cccc 0111 0101 xxxx 1111 xxxx 00x1 xxxx */ + DECODE_OR (0x0ff0f0d0, 0x0750f010), + /* USAD8 cccc 0111 1000 xxxx 1111 xxxx 0001 xxxx */ + DECODE_EMULATEX (0x0ff0f0f0, 0x0780f010, emulate_rd16rn12rm0rs8_rwflags_nopc, + REGS(NOPC, 0, NOPC, 0, NOPC)), + + /* SMLAD cccc 0111 0000 xxxx xxxx xxxx 00x1 xxxx */ + /* SMLSD cccc 0111 0000 xxxx xxxx xxxx 01x1 xxxx */ + DECODE_OR (0x0ff00090, 0x07000010), + /* SMMLA cccc 0111 0101 xxxx xxxx xxxx 00x1 xxxx */ + DECODE_OR (0x0ff000d0, 0x07500010), + /* USADA8 cccc 0111 1000 xxxx xxxx xxxx 0001 xxxx */ + DECODE_EMULATEX (0x0ff000f0, 0x07800010, emulate_rd16rn12rm0rs8_rwflags_nopc, + REGS(NOPC, NOPCX, NOPC, 0, NOPC)), + + /* SMMLS cccc 0111 0101 xxxx xxxx xxxx 11x1 xxxx */ + DECODE_EMULATEX (0x0ff000d0, 0x075000d0, emulate_rd16rn12rm0rs8_rwflags_nopc, + REGS(NOPC, NOPC, NOPC, 0, NOPC)), + + /* SBFX cccc 0111 101x xxxx xxxx xxxx x101 xxxx */ + /* UBFX cccc 0111 111x xxxx xxxx xxxx x101 xxxx */ + DECODE_EMULATEX (0x0fa00070, 0x07a00050, emulate_rd12rm0_noflags_nopc, + REGS(0, NOPC, 0, 0, NOPC)), + + /* BFC cccc 0111 110x xxxx xxxx xxxx x001 1111 */ + DECODE_EMULATEX (0x0fe0007f, 0x07c0001f, emulate_rd12rm0_noflags_nopc, + REGS(0, NOPC, 0, 0, 0)), + + /* BFI cccc 0111 110x xxxx xxxx xxxx x001 xxxx */ + DECODE_EMULATEX (0x0fe00070, 0x07c00010, emulate_rd12rm0_noflags_nopc, + REGS(0, NOPC, 0, 0, NOPCX)), + + DECODE_END +}; + +static const union decode_item arm_cccc_01xx_table[] = { + /* Load/store word and unsigned byte */ + + /* LDRB/STRB pc,[...] cccc 01xx x0xx xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0x0c40f000, 0x0440f000), + + /* STRT cccc 01x0 x010 xxxx xxxx xxxx xxxx xxxx */ + /* LDRT cccc 01x0 x011 xxxx xxxx xxxx xxxx xxxx */ + /* STRBT cccc 01x0 x110 xxxx xxxx xxxx xxxx xxxx */ + /* LDRBT cccc 01x0 x111 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0x0d200000, 0x04200000), + + /* STR (immediate) cccc 010x x0x0 xxxx xxxx xxxx xxxx xxxx */ + /* STRB (immediate) cccc 010x x1x0 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e100000, 0x04000000, emulate_str, + REGS(NOPCWB, ANY, 0, 0, 0)), + + /* LDR (immediate) cccc 010x x0x1 xxxx xxxx xxxx xxxx xxxx */ + /* LDRB (immediate) cccc 010x x1x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e100000, 0x04100000, emulate_ldr, + REGS(NOPCWB, ANY, 0, 0, 0)), + + /* STR (register) cccc 011x x0x0 xxxx xxxx xxxx xxxx xxxx */ + /* STRB (register) cccc 011x x1x0 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e100000, 0x06000000, emulate_str, + REGS(NOPCWB, ANY, 0, 0, NOPC)), + + /* LDR (register) cccc 011x x0x1 xxxx xxxx xxxx xxxx xxxx */ + /* LDRB (register) cccc 011x x1x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0x0e100000, 0x06100000, emulate_ldr, + REGS(NOPCWB, ANY, 0, 0, NOPC)), + + DECODE_END +}; + +static const union decode_item arm_cccc_100x_table[] = { + /* Block data transfer instructions */ + + /* LDM cccc 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ + /* STM cccc 100x x0x0 xxxx xxxx xxxx xxxx xxxx */ + DECODE_CUSTOM (0x0e400000, 0x08000000, kprobe_decode_ldmstm), + + /* STM (user registers) cccc 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ + /* LDM (user registers) cccc 100x x1x1 xxxx 0xxx xxxx xxxx xxxx */ + /* LDM (exception ret) cccc 100x x1x1 xxxx 1xxx xxxx xxxx xxxx */ + DECODE_END +}; + +const union decode_item kprobe_decode_arm_table[] = { + /* + * Unconditional instructions + * 1111 xxxx xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xf0000000, 0xf0000000, arm_1111_table), + + /* + * Miscellaneous instructions + * cccc 0001 0xx0 xxxx xxxx xxxx 0xxx xxxx + */ + DECODE_TABLE (0x0f900080, 0x01000000, arm_cccc_0001_0xx0____0xxx_table), + + /* + * Halfword multiply and multiply-accumulate + * cccc 0001 0xx0 xxxx xxxx xxxx 1xx0 xxxx + */ + DECODE_TABLE (0x0f900090, 0x01000080, arm_cccc_0001_0xx0____1xx0_table), + + /* + * Multiply and multiply-accumulate + * cccc 0000 xxxx xxxx xxxx xxxx 1001 xxxx + */ + DECODE_TABLE (0x0f0000f0, 0x00000090, arm_cccc_0000_____1001_table), + + /* + * Synchronization primitives + * cccc 0001 xxxx xxxx xxxx xxxx 1001 xxxx + */ + DECODE_TABLE (0x0f0000f0, 0x01000090, arm_cccc_0001_____1001_table), + + /* + * Extra load/store instructions + * cccc 000x xxxx xxxx xxxx xxxx 1xx1 xxxx + */ + DECODE_TABLE (0x0e000090, 0x00000090, arm_cccc_000x_____1xx1_table), + + /* + * Data-processing (register) + * cccc 000x xxxx xxxx xxxx xxxx xxx0 xxxx + * Data-processing (register-shifted register) + * cccc 000x xxxx xxxx xxxx xxxx 0xx1 xxxx + */ + DECODE_TABLE (0x0e000000, 0x00000000, arm_cccc_000x_table), + + /* + * Data-processing (immediate) + * cccc 001x xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0x0e000000, 0x02000000, arm_cccc_001x_table), + + /* + * Media instructions + * cccc 011x xxxx xxxx xxxx xxxx xxx1 xxxx + */ + DECODE_TABLE (0x0f000010, 0x06000010, arm_cccc_0110_____xxx1_table), + DECODE_TABLE (0x0f000010, 0x07000010, arm_cccc_0111_____xxx1_table), + + /* + * Load/store word and unsigned byte + * cccc 01xx xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0x0c000000, 0x04000000, arm_cccc_01xx_table), + + /* + * Block data transfer instructions + * cccc 100x xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0x0e000000, 0x08000000, arm_cccc_100x_table), + + /* B cccc 1010 xxxx xxxx xxxx xxxx xxxx xxxx */ + /* BL cccc 1011 xxxx xxxx xxxx xxxx xxxx xxxx */ + DECODE_SIMULATE (0x0e000000, 0x0a000000, simulate_bbl), + + /* + * Supervisor Call, and coprocessor instructions + */ + + /* MCRR cccc 1100 0100 xxxx xxxx xxxx xxxx xxxx */ + /* MRRC cccc 1100 0101 xxxx xxxx xxxx xxxx xxxx */ + /* LDC cccc 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ + /* STC cccc 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ + /* CDP cccc 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ + /* MCR cccc 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ + /* MRC cccc 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ + /* SVC cccc 1111 xxxx xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0x0c000000, 0x0c000000), + + DECODE_END +}; +#ifdef CONFIG_ARM_KPROBES_TEST_MODULE +EXPORT_SYMBOL_GPL(kprobe_decode_arm_table); +#endif + +static void __kprobes arm_singlestep(struct kprobe *p, struct pt_regs *regs) +{ + regs->ARM_pc += 4; + p->ainsn.insn_handler(p, regs); +} + +/* Return: + * INSN_REJECTED If instruction is one not allowed to kprobe, + * INSN_GOOD If instruction is supported and uses instruction slot, + * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. + * + * For instructions we don't want to kprobe (INSN_REJECTED return result): + * These are generally ones that modify the processor state making + * them "hard" to simulate such as switches processor modes or + * make accesses in alternate modes. Any of these could be simulated + * if the work was put into it, but low return considering they + * should also be very rare. + */ +enum kprobe_insn __kprobes +arm_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + asi->insn_singlestep = arm_singlestep; + asi->insn_check_cc = kprobe_condition_checks[insn>>28]; + return kprobe_decode_insn(insn, asi, kprobe_decode_arm_table, false); +} diff --git a/arch/arm/kernel/probes-arm.h b/arch/arm/kernel/probes-arm.h new file mode 100644 index 000000000000..86084727d36d --- /dev/null +++ b/arch/arm/kernel/probes-arm.h @@ -0,0 +1,38 @@ +/* + * arch/arm/kernel/probes-arm.h + * + * Copyright 2013 Linaro Ltd. + * Written by: David A. Long + * + * The code contained herein is licensed under the GNU General Public + * License. You may obtain a copy of the GNU General Public License + * Version 2 or later at the following locations: + * + * http://www.opensource.org/licenses/gpl-license.html + * http://www.gnu.org/copyleft/gpl.html + */ + +#ifndef _ARM_KERNEL_PROBES_ARM_H +#define _ARM_KERNEL_PROBES_ARM_H + +void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs); +void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs); +void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs); +void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs); +void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs); + +void __kprobes emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs); +void __kprobes emulate_ldr(struct kprobe *p, struct pt_regs *regs); +void __kprobes emulate_str(struct kprobe *p, struct pt_regs *regs); +void __kprobes emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, + struct pt_regs *regs); +void __kprobes emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, + struct pt_regs *regs); +void __kprobes emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, + struct pt_regs *regs); +void __kprobes emulate_rd12rm0_noflags_nopc(struct kprobe *p, + struct pt_regs *regs); +void __kprobes emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, + struct pt_regs *regs); + +#endif diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c new file mode 100644 index 000000000000..3a63f8f83cf8 --- /dev/null +++ b/arch/arm/kernel/probes.c @@ -0,0 +1,443 @@ +/* + * arch/arm/kernel/probes.c + * + * Copyright (C) 2011 Jon Medhurst . + * + * Some contents moved here from arch/arm/include/asm/kprobes-arm.c which is + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include + +#include "kprobes.h" + + +#ifndef find_str_pc_offset + +/* + * For STR and STM instructions, an ARM core may choose to use either + * a +8 or a +12 displacement from the current instruction's address. + * Whichever value is chosen for a given core, it must be the same for + * both instructions and may not change. This function measures it. + */ + +int str_pc_offset; + +void __init find_str_pc_offset(void) +{ + int addr, scratch, ret; + + __asm__ ( + "sub %[ret], pc, #4 \n\t" + "str pc, %[addr] \n\t" + "ldr %[scr], %[addr] \n\t" + "sub %[ret], %[scr], %[ret] \n\t" + : [ret] "=r" (ret), [scr] "=r" (scratch), [addr] "+m" (addr)); + + str_pc_offset = ret; +} + +#endif /* !find_str_pc_offset */ + + +#ifndef test_load_write_pc_interworking + +bool load_write_pc_interworks; + +void __init test_load_write_pc_interworking(void) +{ + int arch = cpu_architecture(); + BUG_ON(arch == CPU_ARCH_UNKNOWN); + load_write_pc_interworks = arch >= CPU_ARCH_ARMv5T; +} + +#endif /* !test_load_write_pc_interworking */ + + +#ifndef test_alu_write_pc_interworking + +bool alu_write_pc_interworks; + +void __init test_alu_write_pc_interworking(void) +{ + int arch = cpu_architecture(); + BUG_ON(arch == CPU_ARCH_UNKNOWN); + alu_write_pc_interworks = arch >= CPU_ARCH_ARMv7; +} + +#endif /* !test_alu_write_pc_interworking */ + + +void __init arm_kprobe_decode_init(void) +{ + find_str_pc_offset(); + test_load_write_pc_interworking(); + test_alu_write_pc_interworking(); +} + + +static unsigned long __kprobes __check_eq(unsigned long cpsr) +{ + return cpsr & PSR_Z_BIT; +} + +static unsigned long __kprobes __check_ne(unsigned long cpsr) +{ + return (~cpsr) & PSR_Z_BIT; +} + +static unsigned long __kprobes __check_cs(unsigned long cpsr) +{ + return cpsr & PSR_C_BIT; +} + +static unsigned long __kprobes __check_cc(unsigned long cpsr) +{ + return (~cpsr) & PSR_C_BIT; +} + +static unsigned long __kprobes __check_mi(unsigned long cpsr) +{ + return cpsr & PSR_N_BIT; +} + +static unsigned long __kprobes __check_pl(unsigned long cpsr) +{ + return (~cpsr) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_vs(unsigned long cpsr) +{ + return cpsr & PSR_V_BIT; +} + +static unsigned long __kprobes __check_vc(unsigned long cpsr) +{ + return (~cpsr) & PSR_V_BIT; +} + +static unsigned long __kprobes __check_hi(unsigned long cpsr) +{ + cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return cpsr & PSR_C_BIT; +} + +static unsigned long __kprobes __check_ls(unsigned long cpsr) +{ + cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (~cpsr) & PSR_C_BIT; +} + +static unsigned long __kprobes __check_ge(unsigned long cpsr) +{ + cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (~cpsr) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_lt(unsigned long cpsr) +{ + cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return cpsr & PSR_N_BIT; +} + +static unsigned long __kprobes __check_gt(unsigned long cpsr) +{ + unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ + return (~temp) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_le(unsigned long cpsr) +{ + unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ + return temp & PSR_N_BIT; +} + +static unsigned long __kprobes __check_al(unsigned long cpsr) +{ + return true; +} + +kprobe_check_cc * const kprobe_condition_checks[16] = { + &__check_eq, &__check_ne, &__check_cs, &__check_cc, + &__check_mi, &__check_pl, &__check_vs, &__check_vc, + &__check_hi, &__check_ls, &__check_ge, &__check_lt, + &__check_gt, &__check_le, &__check_al, &__check_al +}; + + +void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs) +{ +} + +void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs) +{ + p->ainsn.insn_fn(); +} + +/* + * Prepare an instruction slot to receive an instruction for emulating. + * This is done by placing a subroutine return after the location where the + * instruction will be placed. We also modify ARM instructions to be + * unconditional as the condition code will already be checked before any + * emulation handler is called. + */ +static kprobe_opcode_t __kprobes +prepare_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, + bool thumb) +{ +#ifdef CONFIG_THUMB2_KERNEL + if (thumb) { + u16 *thumb_insn = (u16 *)asi->insn; + thumb_insn[1] = 0x4770; /* Thumb bx lr */ + thumb_insn[2] = 0x4770; /* Thumb bx lr */ + return insn; + } + asi->insn[1] = 0xe12fff1e; /* ARM bx lr */ +#else + asi->insn[1] = 0xe1a0f00e; /* mov pc, lr */ +#endif + /* Make an ARM instruction unconditional */ + if (insn < 0xe0000000) + insn = (insn | 0xe0000000) & ~0x10000000; + return insn; +} + +/* + * Write a (probably modified) instruction into the slot previously prepared by + * prepare_emulated_insn + */ +static void __kprobes +set_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, + bool thumb) +{ +#ifdef CONFIG_THUMB2_KERNEL + if (thumb) { + u16 *ip = (u16 *)asi->insn; + if (is_wide_instruction(insn)) + *ip++ = insn >> 16; + *ip++ = insn; + return; + } +#endif + asi->insn[0] = insn; +} + +/* + * When we modify the register numbers encoded in an instruction to be emulated, + * the new values come from this define. For ARM and 32-bit Thumb instructions + * this gives... + * + * bit position 16 12 8 4 0 + * ---------------+---+---+---+---+---+ + * register r2 r0 r1 -- r3 + */ +#define INSN_NEW_BITS 0x00020103 + +/* Each nibble has same value as that at INSN_NEW_BITS bit 16 */ +#define INSN_SAMEAS16_BITS 0x22222222 + +/* + * Validate and modify each of the registers encoded in an instruction. + * + * Each nibble in regs contains a value from enum decode_reg_type. For each + * non-zero value, the corresponding nibble in pinsn is validated and modified + * according to the type. + */ +static bool __kprobes decode_regs(kprobe_opcode_t *pinsn, u32 regs) +{ + kprobe_opcode_t insn = *pinsn; + kprobe_opcode_t mask = 0xf; /* Start at least significant nibble */ + + for (; regs != 0; regs >>= 4, mask <<= 4) { + + kprobe_opcode_t new_bits = INSN_NEW_BITS; + + switch (regs & 0xf) { + + case REG_TYPE_NONE: + /* Nibble not a register, skip to next */ + continue; + + case REG_TYPE_ANY: + /* Any register is allowed */ + break; + + case REG_TYPE_SAMEAS16: + /* Replace register with same as at bit position 16 */ + new_bits = INSN_SAMEAS16_BITS; + break; + + case REG_TYPE_SP: + /* Only allow SP (R13) */ + if ((insn ^ 0xdddddddd) & mask) + goto reject; + break; + + case REG_TYPE_PC: + /* Only allow PC (R15) */ + if ((insn ^ 0xffffffff) & mask) + goto reject; + break; + + case REG_TYPE_NOSP: + /* Reject SP (R13) */ + if (((insn ^ 0xdddddddd) & mask) == 0) + goto reject; + break; + + case REG_TYPE_NOSPPC: + case REG_TYPE_NOSPPCX: + /* Reject SP and PC (R13 and R15) */ + if (((insn ^ 0xdddddddd) & 0xdddddddd & mask) == 0) + goto reject; + break; + + case REG_TYPE_NOPCWB: + if (!is_writeback(insn)) + break; /* No writeback, so any register is OK */ + /* fall through... */ + case REG_TYPE_NOPC: + case REG_TYPE_NOPCX: + /* Reject PC (R15) */ + if (((insn ^ 0xffffffff) & mask) == 0) + goto reject; + break; + } + + /* Replace value of nibble with new register number... */ + insn &= ~mask; + insn |= new_bits & mask; + } + + *pinsn = insn; + return true; + +reject: + return false; +} + +static const int decode_struct_sizes[NUM_DECODE_TYPES] = { + [DECODE_TYPE_TABLE] = sizeof(struct decode_table), + [DECODE_TYPE_CUSTOM] = sizeof(struct decode_custom), + [DECODE_TYPE_SIMULATE] = sizeof(struct decode_simulate), + [DECODE_TYPE_EMULATE] = sizeof(struct decode_emulate), + [DECODE_TYPE_OR] = sizeof(struct decode_or), + [DECODE_TYPE_REJECT] = sizeof(struct decode_reject) +}; + +/* + * kprobe_decode_insn operates on data tables in order to decode an ARM + * architecture instruction onto which a kprobe has been placed. + * + * These instruction decoding tables are a concatenation of entries each + * of which consist of one of the following structs: + * + * decode_table + * decode_custom + * decode_simulate + * decode_emulate + * decode_or + * decode_reject + * + * Each of these starts with a struct decode_header which has the following + * fields: + * + * type_regs + * mask + * value + * + * The least significant DECODE_TYPE_BITS of type_regs contains a value + * from enum decode_type, this indicates which of the decode_* structs + * the entry contains. The value DECODE_TYPE_END indicates the end of the + * table. + * + * When the table is parsed, each entry is checked in turn to see if it + * matches the instruction to be decoded using the test: + * + * (insn & mask) == value + * + * If no match is found before the end of the table is reached then decoding + * fails with INSN_REJECTED. + * + * When a match is found, decode_regs() is called to validate and modify each + * of the registers encoded in the instruction; the data it uses to do this + * is (type_regs >> DECODE_TYPE_BITS). A validation failure will cause decoding + * to fail with INSN_REJECTED. + * + * Once the instruction has passed the above tests, further processing + * depends on the type of the table entry's decode struct. + * + */ +int __kprobes +kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const union decode_item *table, bool thumb) +{ + const struct decode_header *h = (struct decode_header *)table; + const struct decode_header *next; + bool matched = false; + + insn = prepare_emulated_insn(insn, asi, thumb); + + for (;; h = next) { + enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK; + u32 regs = h->type_regs.bits >> DECODE_TYPE_BITS; + + if (type == DECODE_TYPE_END) + return INSN_REJECTED; + + next = (struct decode_header *) + ((uintptr_t)h + decode_struct_sizes[type]); + + if (!matched && (insn & h->mask.bits) != h->value.bits) + continue; + + if (!decode_regs(&insn, regs)) + return INSN_REJECTED; + + switch (type) { + + case DECODE_TYPE_TABLE: { + struct decode_table *d = (struct decode_table *)h; + next = (struct decode_header *)d->table.table; + break; + } + + case DECODE_TYPE_CUSTOM: { + struct decode_custom *d = (struct decode_custom *)h; + return (*d->decoder.decoder)(insn, asi); + } + + case DECODE_TYPE_SIMULATE: { + struct decode_simulate *d = (struct decode_simulate *)h; + asi->insn_handler = d->handler.handler; + return INSN_GOOD_NO_SLOT; + } + + case DECODE_TYPE_EMULATE: { + struct decode_emulate *d = (struct decode_emulate *)h; + asi->insn_handler = d->handler.handler; + set_emulated_insn(insn, asi, thumb); + return INSN_GOOD; + } + + case DECODE_TYPE_OR: + matched = true; + break; + + case DECODE_TYPE_REJECT: + default: + return INSN_REJECTED; + } + } +} diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h new file mode 100644 index 000000000000..17f656011aa3 --- /dev/null +++ b/arch/arm/kernel/probes.h @@ -0,0 +1,397 @@ +/* + * arch/arm/kernel/probes.h + * + * Copyright (C) 2011 Jon Medhurst . + * + * Some contents moved here from arch/arm/include/asm/kprobes.h which is + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#ifndef _ARM_KERNEL_PROBES_H +#define _ARM_KERNEL_PROBES_H + +#include +#include +#include + +#if __LINUX_ARM_ARCH__ >= 7 + +/* str_pc_offset is architecturally defined from ARMv7 onwards */ +#define str_pc_offset 8 +#define find_str_pc_offset() + +#else /* __LINUX_ARM_ARCH__ < 7 */ + +/* We need a run-time check to determine str_pc_offset */ +extern int str_pc_offset; +void __init find_str_pc_offset(void); + +#endif + + +/* + * Update ITSTATE after normal execution of an IT block instruction. + * + * The 8 IT state bits are split into two parts in CPSR: + * ITSTATE<1:0> are in CPSR<26:25> + * ITSTATE<7:2> are in CPSR<15:10> + */ +static inline unsigned long it_advance(unsigned long cpsr) + { + if ((cpsr & 0x06000400) == 0) { + /* ITSTATE<2:0> == 0 means end of IT block, so clear IT state */ + cpsr &= ~PSR_IT_MASK; + } else { + /* We need to shift left ITSTATE<4:0> */ + const unsigned long mask = 0x06001c00; /* Mask ITSTATE<4:0> */ + unsigned long it = cpsr & mask; + it <<= 1; + it |= it >> (27 - 10); /* Carry ITSTATE<2> to correct place */ + it &= mask; + cpsr &= ~mask; + cpsr |= it; + } + return cpsr; +} + +static inline void __kprobes bx_write_pc(long pcv, struct pt_regs *regs) +{ + long cpsr = regs->ARM_cpsr; + if (pcv & 0x1) { + cpsr |= PSR_T_BIT; + pcv &= ~0x1; + } else { + cpsr &= ~PSR_T_BIT; + pcv &= ~0x2; /* Avoid UNPREDICTABLE address allignment */ + } + regs->ARM_cpsr = cpsr; + regs->ARM_pc = pcv; +} + + +#if __LINUX_ARM_ARCH__ >= 6 + +/* Kernels built for >= ARMv6 should never run on <= ARMv5 hardware, so... */ +#define load_write_pc_interworks true +#define test_load_write_pc_interworking() + +#else /* __LINUX_ARM_ARCH__ < 6 */ + +/* We need run-time testing to determine if load_write_pc() should interwork. */ +extern bool load_write_pc_interworks; +void __init test_load_write_pc_interworking(void); + +#endif + +static inline void __kprobes load_write_pc(long pcv, struct pt_regs *regs) +{ + if (load_write_pc_interworks) + bx_write_pc(pcv, regs); + else + regs->ARM_pc = pcv; +} + + +#if __LINUX_ARM_ARCH__ >= 7 + +#define alu_write_pc_interworks true +#define test_alu_write_pc_interworking() + +#elif __LINUX_ARM_ARCH__ <= 5 + +/* Kernels built for <= ARMv5 should never run on >= ARMv6 hardware, so... */ +#define alu_write_pc_interworks false +#define test_alu_write_pc_interworking() + +#else /* __LINUX_ARM_ARCH__ == 6 */ + +/* We could be an ARMv6 binary on ARMv7 hardware so we need a run-time check. */ +extern bool alu_write_pc_interworks; +void __init test_alu_write_pc_interworking(void); + +#endif /* __LINUX_ARM_ARCH__ == 6 */ + +static inline void __kprobes alu_write_pc(long pcv, struct pt_regs *regs) +{ + if (alu_write_pc_interworks) + bx_write_pc(pcv, regs); + else + regs->ARM_pc = pcv; +} + + +void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs); +void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs); + +enum kprobe_insn __kprobes +kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi); + +/* + * Test if load/store instructions writeback the address register. + * if P (bit 24) == 0 or W (bit 21) == 1 + */ +#define is_writeback(insn) ((insn ^ 0x01000000) & 0x01200000) + +/* + * The following definitions and macros are used to build instruction + * decoding tables for use by kprobe_decode_insn. + * + * These tables are a concatenation of entries each of which consist of one of + * the decode_* structs. All of the fields in every type of decode structure + * are of the union type decode_item, therefore the entire decode table can be + * viewed as an array of these and declared like: + * + * static const union decode_item table_name[] = {}; + * + * In order to construct each entry in the table, macros are used to + * initialise a number of sequential decode_item values in a layout which + * matches the relevant struct. E.g. DECODE_SIMULATE initialise a struct + * decode_simulate by initialising four decode_item objects like this... + * + * {.bits = _type}, + * {.bits = _mask}, + * {.bits = _value}, + * {.handler = _handler}, + * + * Initialising a specified member of the union means that the compiler + * will produce a warning if the argument is of an incorrect type. + * + * Below is a list of each of the macros used to initialise entries and a + * description of the action performed when that entry is matched to an + * instruction. A match is found when (instruction & mask) == value. + * + * DECODE_TABLE(mask, value, table) + * Instruction decoding jumps to parsing the new sub-table 'table'. + * + * DECODE_CUSTOM(mask, value, decoder) + * The custom function 'decoder' is called to the complete decoding + * of an instruction. + * + * DECODE_SIMULATE(mask, value, handler) + * Set the probes instruction handler to 'handler', this will be used + * to simulate the instruction when the probe is hit. Decoding returns + * with INSN_GOOD_NO_SLOT. + * + * DECODE_EMULATE(mask, value, handler) + * Set the probes instruction handler to 'handler', this will be used + * to emulate the instruction when the probe is hit. The modified + * instruction (see below) is placed in the probes instruction slot so it + * may be called by the emulation code. Decoding returns with INSN_GOOD. + * + * DECODE_REJECT(mask, value) + * Instruction decoding fails with INSN_REJECTED + * + * DECODE_OR(mask, value) + * This allows the mask/value test of multiple table entries to be + * logically ORed. Once an 'or' entry is matched the decoding action to + * be performed is that of the next entry which isn't an 'or'. E.g. + * + * DECODE_OR (mask1, value1) + * DECODE_OR (mask2, value2) + * DECODE_SIMULATE (mask3, value3, simulation_handler) + * + * This means that if any of the three mask/value pairs match the + * instruction being decoded, then 'simulation_handler' will be used + * for it. + * + * Both the SIMULATE and EMULATE macros have a second form which take an + * additional 'regs' argument. + * + * DECODE_SIMULATEX(mask, value, handler, regs) + * DECODE_EMULATEX (mask, value, handler, regs) + * + * These are used to specify what kind of CPU register is encoded in each of the + * least significant 5 nibbles of the instruction being decoded. The regs value + * is specified using the REGS macro, this takes any of the REG_TYPE_* values + * from enum decode_reg_type as arguments; only the '*' part of the name is + * given. E.g. + * + * REGS(0, ANY, NOPC, 0, ANY) + * + * This indicates an instruction is encoded like: + * + * bits 19..16 ignore + * bits 15..12 any register allowed here + * bits 11.. 8 any register except PC allowed here + * bits 7.. 4 ignore + * bits 3.. 0 any register allowed here + * + * This register specification is checked after a decode table entry is found to + * match an instruction (through the mask/value test). Any invalid register then + * found in the instruction will cause decoding to fail with INSN_REJECTED. In + * the above example this would happen if bits 11..8 of the instruction were + * 1111, indicating R15 or PC. + * + * As well as checking for legal combinations of registers, this data is also + * used to modify the registers encoded in the instructions so that an + * emulation routines can use it. (See decode_regs() and INSN_NEW_BITS.) + * + * Here is a real example which matches ARM instructions of the form + * "AND ,,, " + * + * DECODE_EMULATEX (0x0e000090, 0x00000010, emulate_rd12rn16rm0rs8_rwflags, + * REGS(ANY, ANY, NOPC, 0, ANY)), + * ^ ^ ^ ^ + * Rn Rd Rs Rm + * + * Decoding the instruction "AND R4, R5, R6, ASL R15" will be rejected because + * Rs == R15 + * + * Decoding the instruction "AND R4, R5, R6, ASL R7" will be accepted and the + * instruction will be modified to "AND R0, R2, R3, ASL R1" and then placed into + * the kprobes instruction slot. This can then be called later by the handler + * function emulate_rd12rn16rm0rs8_rwflags in order to simulate the instruction. + */ + +enum decode_type { + DECODE_TYPE_END, + DECODE_TYPE_TABLE, + DECODE_TYPE_CUSTOM, + DECODE_TYPE_SIMULATE, + DECODE_TYPE_EMULATE, + DECODE_TYPE_OR, + DECODE_TYPE_REJECT, + NUM_DECODE_TYPES /* Must be last enum */ +}; + +#define DECODE_TYPE_BITS 4 +#define DECODE_TYPE_MASK ((1 << DECODE_TYPE_BITS) - 1) + +enum decode_reg_type { + REG_TYPE_NONE = 0, /* Not a register, ignore */ + REG_TYPE_ANY, /* Any register allowed */ + REG_TYPE_SAMEAS16, /* Register should be same as that at bits 19..16 */ + REG_TYPE_SP, /* Register must be SP */ + REG_TYPE_PC, /* Register must be PC */ + REG_TYPE_NOSP, /* Register must not be SP */ + REG_TYPE_NOSPPC, /* Register must not be SP or PC */ + REG_TYPE_NOPC, /* Register must not be PC */ + REG_TYPE_NOPCWB, /* No PC if load/store write-back flag also set */ + + /* The following types are used when the encoding for PC indicates + * another instruction form. This distiction only matters for test + * case coverage checks. + */ + REG_TYPE_NOPCX, /* Register must not be PC */ + REG_TYPE_NOSPPCX, /* Register must not be SP or PC */ + + /* Alias to allow '0' arg to be used in REGS macro. */ + REG_TYPE_0 = REG_TYPE_NONE +}; + +#define REGS(r16, r12, r8, r4, r0) \ + (((REG_TYPE_##r16) << 16) + \ + ((REG_TYPE_##r12) << 12) + \ + ((REG_TYPE_##r8) << 8) + \ + ((REG_TYPE_##r4) << 4) + \ + (REG_TYPE_##r0)) + +union decode_item { + u32 bits; + const union decode_item *table; + kprobe_insn_handler_t *handler; + kprobe_decode_insn_t *decoder; +}; + + +#define DECODE_END \ + {.bits = DECODE_TYPE_END} + + +struct decode_header { + union decode_item type_regs; + union decode_item mask; + union decode_item value; +}; + +#define DECODE_HEADER(_type, _mask, _value, _regs) \ + {.bits = (_type) | ((_regs) << DECODE_TYPE_BITS)}, \ + {.bits = (_mask)}, \ + {.bits = (_value)} + + +struct decode_table { + struct decode_header header; + union decode_item table; +}; + +#define DECODE_TABLE(_mask, _value, _table) \ + DECODE_HEADER(DECODE_TYPE_TABLE, _mask, _value, 0), \ + {.table = (_table)} + + +struct decode_custom { + struct decode_header header; + union decode_item decoder; +}; + +#define DECODE_CUSTOM(_mask, _value, _decoder) \ + DECODE_HEADER(DECODE_TYPE_CUSTOM, _mask, _value, 0), \ + {.decoder = (_decoder)} + + +struct decode_simulate { + struct decode_header header; + union decode_item handler; +}; + +#define DECODE_SIMULATEX(_mask, _value, _handler, _regs) \ + DECODE_HEADER(DECODE_TYPE_SIMULATE, _mask, _value, _regs), \ + {.handler = (_handler)} + +#define DECODE_SIMULATE(_mask, _value, _handler) \ + DECODE_SIMULATEX(_mask, _value, _handler, 0) + + +struct decode_emulate { + struct decode_header header; + union decode_item handler; +}; + +#define DECODE_EMULATEX(_mask, _value, _handler, _regs) \ + DECODE_HEADER(DECODE_TYPE_EMULATE, _mask, _value, _regs), \ + {.handler = (_handler)} + +#define DECODE_EMULATE(_mask, _value, _handler) \ + DECODE_EMULATEX(_mask, _value, _handler, 0) + + +struct decode_or { + struct decode_header header; +}; + +#define DECODE_OR(_mask, _value) \ + DECODE_HEADER(DECODE_TYPE_OR, _mask, _value, 0) + + +struct decode_reject { + struct decode_header header; +}; + +#define DECODE_REJECT(_mask, _value) \ + DECODE_HEADER(DECODE_TYPE_REJECT, _mask, _value, 0) + + +#ifdef CONFIG_THUMB2_KERNEL +extern const union decode_item kprobe_decode_thumb16_table[]; +extern const union decode_item kprobe_decode_thumb32_table[]; +#else +extern const union decode_item kprobe_decode_arm_table[]; +#endif + +extern kprobe_check_cc * const kprobe_condition_checks[16]; + + +int kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const union decode_item *table, bool thumb16); + +#endif From 87abef63ead5ac9e2c67f0c07c461eda6be16aeb Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Wed, 5 Mar 2014 21:06:29 -0500 Subject: [PATCH 51/89] ARM: move generic thumb instruction parsing code to new files for use by other feature Move the thumb version of the kprobes instruction parsing code into more generic files from where it can be used by uprobes and possibly other subsystems. The symbol names will be made more generic in a subsequent part of this patchset. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/kernel/Makefile | 2 +- arch/arm/kernel/kprobes-thumb.c | 949 ++------------------------------ arch/arm/kernel/probes-thumb.c | 878 +++++++++++++++++++++++++++++ arch/arm/kernel/probes-thumb.h | 81 +++ 4 files changed, 1000 insertions(+), 910 deletions(-) create mode 100644 arch/arm/kernel/probes-thumb.c create mode 100644 arch/arm/kernel/probes-thumb.h diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 4c8b13e64280..bb739f28dd80 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -52,7 +52,7 @@ obj-$(CONFIG_JUMP_LABEL) += jump_label.o insn.o patch.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o obj-$(CONFIG_KPROBES) += probes.o kprobes.o kprobes-common.o patch.o ifdef CONFIG_THUMB2_KERNEL -obj-$(CONFIG_KPROBES) += kprobes-thumb.o +obj-$(CONFIG_KPROBES) += kprobes-thumb.o probes-thumb.o else obj-$(CONFIG_KPROBES) += kprobes-arm.o probes-arm.o endif diff --git a/arch/arm/kernel/kprobes-thumb.c b/arch/arm/kernel/kprobes-thumb.c index 6123daf397a7..977f21723a9c 100644 --- a/arch/arm/kernel/kprobes-thumb.c +++ b/arch/arm/kernel/kprobes-thumb.c @@ -8,24 +8,13 @@ * published by the Free Software Foundation. */ +#include #include +#include #include -#include #include "kprobes.h" - - -/* - * True if current instruction is in an IT block. - */ -#define in_it_block(cpsr) ((cpsr & 0x06000c00) != 0x00000000) - -/* - * Return the condition code to check for the currently executing instruction. - * This is in ITSTATE<7:4> which is in CPSR<15:12> but is only valid if - * in_it_block returns true. - */ -#define current_cond(cpsr) ((cpsr >> 12) & 0xf) +#include "probes-thumb.h" /* * Return the PC value for a probe in thumb code. @@ -38,7 +27,9 @@ static inline unsigned long __kprobes thumb_probe_pc(struct kprobe *p) return (unsigned long)p->addr - 1 + 4; } -static void __kprobes +/* t32 thumb actions */ + +void __kprobes t32_simulate_table_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -58,7 +49,7 @@ t32_simulate_table_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + 2 * halfwords; } -static void __kprobes +void __kprobes t32_simulate_mrs(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -67,7 +58,7 @@ t32_simulate_mrs(struct kprobe *p, struct pt_regs *regs) regs->uregs[rd] = regs->ARM_cpsr & mask; } -static void __kprobes +void __kprobes t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -82,7 +73,7 @@ t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + (offset * 2); } -static enum kprobe_insn __kprobes +enum kprobe_insn __kprobes t32_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) { int cc = (insn >> 22) & 0xf; @@ -91,7 +82,7 @@ t32_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD_NO_SLOT; } -static void __kprobes +void __kprobes t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -119,7 +110,7 @@ t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + (offset * 2); } -static void __kprobes +void __kprobes t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -157,7 +148,7 @@ t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = rtv; } -static enum kprobe_insn __kprobes +enum kprobe_insn __kprobes t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) { enum kprobe_insn ret = kprobe_decode_ldmstm(insn, asi); @@ -170,7 +161,7 @@ t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) return ret; } -static void __kprobes +void __kprobes t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -197,7 +188,7 @@ t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt2] = rt2v; } -static void __kprobes +void __kprobes t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -223,7 +214,7 @@ t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = rtv; } -static void __kprobes +void __kprobes t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -250,7 +241,7 @@ t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -static void __kprobes +void __kprobes t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -270,7 +261,7 @@ t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) regs->uregs[rd] = rdv; } -static void __kprobes +void __kprobes t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -290,7 +281,7 @@ t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) regs->uregs[rd] = rdv; } -static void __kprobes +void __kprobes t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -315,640 +306,9 @@ t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) regs->uregs[rdlo] = rdlov; regs->uregs[rdhi] = rdhiv; } +/* t16 thumb actions */ -/* These emulation encodings are functionally equivalent... */ -#define t32_emulate_rd8rn16rm0ra12_noflags \ - t32_emulate_rdlo12rdhi8rn16rm0_noflags - -static const union decode_item t32_table_1110_100x_x0xx[] = { - /* Load/store multiple instructions */ - - /* Rn is PC 1110 100x x0xx 1111 xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfe4f0000, 0xe80f0000), - - /* SRS 1110 1000 00x0 xxxx xxxx xxxx xxxx xxxx */ - /* RFE 1110 1000 00x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffc00000, 0xe8000000), - /* SRS 1110 1001 10x0 xxxx xxxx xxxx xxxx xxxx */ - /* RFE 1110 1001 10x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffc00000, 0xe9800000), - - /* STM Rn, {...pc} 1110 100x x0x0 xxxx 1xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfe508000, 0xe8008000), - /* LDM Rn, {...lr,pc} 1110 100x x0x1 xxxx 11xx xxxx xxxx xxxx */ - DECODE_REJECT (0xfe50c000, 0xe810c000), - /* LDM/STM Rn, {...sp} 1110 100x x0xx xxxx xx1x xxxx xxxx xxxx */ - DECODE_REJECT (0xfe402000, 0xe8002000), - - /* STMIA 1110 1000 10x0 xxxx xxxx xxxx xxxx xxxx */ - /* LDMIA 1110 1000 10x1 xxxx xxxx xxxx xxxx xxxx */ - /* STMDB 1110 1001 00x0 xxxx xxxx xxxx xxxx xxxx */ - /* LDMDB 1110 1001 00x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_CUSTOM (0xfe400000, 0xe8000000, t32_decode_ldmstm), - - DECODE_END -}; - -static const union decode_item t32_table_1110_100x_x1xx[] = { - /* Load/store dual, load/store exclusive, table branch */ - - /* STRD (immediate) 1110 1000 x110 xxxx xxxx xxxx xxxx xxxx */ - /* LDRD (immediate) 1110 1000 x111 xxxx xxxx xxxx xxxx xxxx */ - DECODE_OR (0xff600000, 0xe8600000), - /* STRD (immediate) 1110 1001 x1x0 xxxx xxxx xxxx xxxx xxxx */ - /* LDRD (immediate) 1110 1001 x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xff400000, 0xe9400000, t32_emulate_ldrdstrd, - REGS(NOPCWB, NOSPPC, NOSPPC, 0, 0)), - - /* TBB 1110 1000 1101 xxxx xxxx xxxx 0000 xxxx */ - /* TBH 1110 1000 1101 xxxx xxxx xxxx 0001 xxxx */ - DECODE_SIMULATEX(0xfff000e0, 0xe8d00000, t32_simulate_table_branch, - REGS(NOSP, 0, 0, 0, NOSPPC)), - - /* STREX 1110 1000 0100 xxxx xxxx xxxx xxxx xxxx */ - /* LDREX 1110 1000 0101 xxxx xxxx xxxx xxxx xxxx */ - /* STREXB 1110 1000 1100 xxxx xxxx xxxx 0100 xxxx */ - /* STREXH 1110 1000 1100 xxxx xxxx xxxx 0101 xxxx */ - /* STREXD 1110 1000 1100 xxxx xxxx xxxx 0111 xxxx */ - /* LDREXB 1110 1000 1101 xxxx xxxx xxxx 0100 xxxx */ - /* LDREXH 1110 1000 1101 xxxx xxxx xxxx 0101 xxxx */ - /* LDREXD 1110 1000 1101 xxxx xxxx xxxx 0111 xxxx */ - /* And unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1110_101x[] = { - /* Data-processing (shifted register) */ - - /* TST 1110 1010 0001 xxxx xxxx 1111 xxxx xxxx */ - /* TEQ 1110 1010 1001 xxxx xxxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xff700f00, 0xea100f00, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, 0, 0, NOSPPC)), - - /* CMN 1110 1011 0001 xxxx xxxx 1111 xxxx xxxx */ - DECODE_OR (0xfff00f00, 0xeb100f00), - /* CMP 1110 1011 1011 xxxx xxxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfff00f00, 0xebb00f00, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOPC, 0, 0, 0, NOSPPC)), - - /* MOV 1110 1010 010x 1111 xxxx xxxx xxxx xxxx */ - /* MVN 1110 1010 011x 1111 xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xffcf0000, 0xea4f0000, t32_emulate_rd8rn16rm0_rwflags, - REGS(0, 0, NOSPPC, 0, NOSPPC)), - - /* ??? 1110 1010 101x xxxx xxxx xxxx xxxx xxxx */ - /* ??? 1110 1010 111x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffa00000, 0xeaa00000), - /* ??? 1110 1011 001x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffe00000, 0xeb200000), - /* ??? 1110 1011 100x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffe00000, 0xeb800000), - /* ??? 1110 1011 111x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xffe00000, 0xebe00000), - - /* ADD/SUB SP, SP, Rm, LSL #0..3 */ - /* 1110 1011 x0xx 1101 x000 1101 xx00 xxxx */ - DECODE_EMULATEX (0xff4f7f30, 0xeb0d0d00, t32_emulate_rd8rn16rm0_rwflags, - REGS(SP, 0, SP, 0, NOSPPC)), - - /* ADD/SUB SP, SP, Rm, shift */ - /* 1110 1011 x0xx 1101 xxxx 1101 xxxx xxxx */ - DECODE_REJECT (0xff4f0f00, 0xeb0d0d00), - - /* ADD/SUB Rd, SP, Rm, shift */ - /* 1110 1011 x0xx 1101 xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xff4f0000, 0xeb0d0000, t32_emulate_rd8rn16rm0_rwflags, - REGS(SP, 0, NOPC, 0, NOSPPC)), - - /* AND 1110 1010 000x xxxx xxxx xxxx xxxx xxxx */ - /* BIC 1110 1010 001x xxxx xxxx xxxx xxxx xxxx */ - /* ORR 1110 1010 010x xxxx xxxx xxxx xxxx xxxx */ - /* ORN 1110 1010 011x xxxx xxxx xxxx xxxx xxxx */ - /* EOR 1110 1010 100x xxxx xxxx xxxx xxxx xxxx */ - /* PKH 1110 1010 110x xxxx xxxx xxxx xxxx xxxx */ - /* ADD 1110 1011 000x xxxx xxxx xxxx xxxx xxxx */ - /* ADC 1110 1011 010x xxxx xxxx xxxx xxxx xxxx */ - /* SBC 1110 1011 011x xxxx xxxx xxxx xxxx xxxx */ - /* SUB 1110 1011 101x xxxx xxxx xxxx xxxx xxxx */ - /* RSB 1110 1011 110x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfe000000, 0xea000000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), - - DECODE_END -}; - -static const union decode_item t32_table_1111_0x0x___0[] = { - /* Data-processing (modified immediate) */ - - /* TST 1111 0x00 0001 xxxx 0xxx 1111 xxxx xxxx */ - /* TEQ 1111 0x00 1001 xxxx 0xxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfb708f00, 0xf0100f00, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, 0, 0, 0)), - - /* CMN 1111 0x01 0001 xxxx 0xxx 1111 xxxx xxxx */ - DECODE_OR (0xfbf08f00, 0xf1100f00), - /* CMP 1111 0x01 1011 xxxx 0xxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfbf08f00, 0xf1b00f00, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOPC, 0, 0, 0, 0)), - - /* MOV 1111 0x00 010x 1111 0xxx xxxx xxxx xxxx */ - /* MVN 1111 0x00 011x 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbcf8000, 0xf04f0000, t32_emulate_rd8rn16rm0_rwflags, - REGS(0, 0, NOSPPC, 0, 0)), - - /* ??? 1111 0x00 101x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbe08000, 0xf0a00000), - /* ??? 1111 0x00 110x xxxx 0xxx xxxx xxxx xxxx */ - /* ??? 1111 0x00 111x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbc08000, 0xf0c00000), - /* ??? 1111 0x01 001x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbe08000, 0xf1200000), - /* ??? 1111 0x01 100x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbe08000, 0xf1800000), - /* ??? 1111 0x01 111x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfbe08000, 0xf1e00000), - - /* ADD Rd, SP, #imm 1111 0x01 000x 1101 0xxx xxxx xxxx xxxx */ - /* SUB Rd, SP, #imm 1111 0x01 101x 1101 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb4f8000, 0xf10d0000, t32_emulate_rd8rn16rm0_rwflags, - REGS(SP, 0, NOPC, 0, 0)), - - /* AND 1111 0x00 000x xxxx 0xxx xxxx xxxx xxxx */ - /* BIC 1111 0x00 001x xxxx 0xxx xxxx xxxx xxxx */ - /* ORR 1111 0x00 010x xxxx 0xxx xxxx xxxx xxxx */ - /* ORN 1111 0x00 011x xxxx 0xxx xxxx xxxx xxxx */ - /* EOR 1111 0x00 100x xxxx 0xxx xxxx xxxx xxxx */ - /* ADD 1111 0x01 000x xxxx 0xxx xxxx xxxx xxxx */ - /* ADC 1111 0x01 010x xxxx 0xxx xxxx xxxx xxxx */ - /* SBC 1111 0x01 011x xxxx 0xxx xxxx xxxx xxxx */ - /* SUB 1111 0x01 101x xxxx 0xxx xxxx xxxx xxxx */ - /* RSB 1111 0x01 110x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfa008000, 0xf0000000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, 0)), - - DECODE_END -}; - -static const union decode_item t32_table_1111_0x1x___0[] = { - /* Data-processing (plain binary immediate) */ - - /* ADDW Rd, PC, #imm 1111 0x10 0000 1111 0xxx xxxx xxxx xxxx */ - DECODE_OR (0xfbff8000, 0xf20f0000), - /* SUBW Rd, PC, #imm 1111 0x10 1010 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbff8000, 0xf2af0000, t32_emulate_rd8pc16_noflags, - REGS(PC, 0, NOSPPC, 0, 0)), - - /* ADDW SP, SP, #imm 1111 0x10 0000 1101 0xxx 1101 xxxx xxxx */ - DECODE_OR (0xfbff8f00, 0xf20d0d00), - /* SUBW SP, SP, #imm 1111 0x10 1010 1101 0xxx 1101 xxxx xxxx */ - DECODE_EMULATEX (0xfbff8f00, 0xf2ad0d00, t32_emulate_rd8rn16_noflags, - REGS(SP, 0, SP, 0, 0)), - - /* ADDW 1111 0x10 0000 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_OR (0xfbf08000, 0xf2000000), - /* SUBW 1111 0x10 1010 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbf08000, 0xf2a00000, t32_emulate_rd8rn16_noflags, - REGS(NOPCX, 0, NOSPPC, 0, 0)), - - /* MOVW 1111 0x10 0100 xxxx 0xxx xxxx xxxx xxxx */ - /* MOVT 1111 0x10 1100 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb708000, 0xf2400000, t32_emulate_rd8rn16_noflags, - REGS(0, 0, NOSPPC, 0, 0)), - - /* SSAT16 1111 0x11 0010 xxxx 0000 xxxx 00xx xxxx */ - /* SSAT 1111 0x11 00x0 xxxx 0xxx xxxx xxxx xxxx */ - /* USAT16 1111 0x11 1010 xxxx 0000 xxxx 00xx xxxx */ - /* USAT 1111 0x11 10x0 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb508000, 0xf3000000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, 0)), - - /* SFBX 1111 0x11 0100 xxxx 0xxx xxxx xxxx xxxx */ - /* UFBX 1111 0x11 1100 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb708000, 0xf3400000, t32_emulate_rd8rn16_noflags, - REGS(NOSPPC, 0, NOSPPC, 0, 0)), - - /* BFC 1111 0x11 0110 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbff8000, 0xf36f0000, t32_emulate_rd8rn16_noflags, - REGS(0, 0, NOSPPC, 0, 0)), - - /* BFI 1111 0x11 0110 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbf08000, 0xf3600000, t32_emulate_rd8rn16_noflags, - REGS(NOSPPCX, 0, NOSPPC, 0, 0)), - - DECODE_END -}; - -static const union decode_item t32_table_1111_0xxx___1[] = { - /* Branches and miscellaneous control */ - - /* YIELD 1111 0011 1010 xxxx 10x0 x000 0000 0001 */ - DECODE_OR (0xfff0d7ff, 0xf3a08001), - /* SEV 1111 0011 1010 xxxx 10x0 x000 0000 0100 */ - DECODE_EMULATE (0xfff0d7ff, 0xf3a08004, kprobe_emulate_none), - /* NOP 1111 0011 1010 xxxx 10x0 x000 0000 0000 */ - /* WFE 1111 0011 1010 xxxx 10x0 x000 0000 0010 */ - /* WFI 1111 0011 1010 xxxx 10x0 x000 0000 0011 */ - DECODE_SIMULATE (0xfff0d7fc, 0xf3a08000, kprobe_simulate_nop), - - /* MRS Rd, CPSR 1111 0011 1110 xxxx 10x0 xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xfff0d000, 0xf3e08000, t32_simulate_mrs, - REGS(0, 0, NOSPPC, 0, 0)), - - /* - * Unsupported instructions - * 1111 0x11 1xxx xxxx 10x0 xxxx xxxx xxxx - * - * MSR 1111 0011 100x xxxx 10x0 xxxx xxxx xxxx - * DBG hint 1111 0011 1010 xxxx 10x0 x000 1111 xxxx - * Unallocated hints 1111 0011 1010 xxxx 10x0 x000 xxxx xxxx - * CPS 1111 0011 1010 xxxx 10x0 xxxx xxxx xxxx - * CLREX/DSB/DMB/ISB 1111 0011 1011 xxxx 10x0 xxxx xxxx xxxx - * BXJ 1111 0011 1100 xxxx 10x0 xxxx xxxx xxxx - * SUBS PC,LR,# 1111 0011 1101 xxxx 10x0 xxxx xxxx xxxx - * MRS Rd, SPSR 1111 0011 1111 xxxx 10x0 xxxx xxxx xxxx - * SMC 1111 0111 1111 xxxx 1000 xxxx xxxx xxxx - * UNDEFINED 1111 0111 1111 xxxx 1010 xxxx xxxx xxxx - * ??? 1111 0111 1xxx xxxx 1010 xxxx xxxx xxxx - */ - DECODE_REJECT (0xfb80d000, 0xf3808000), - - /* Bcc 1111 0xxx xxxx xxxx 10x0 xxxx xxxx xxxx */ - DECODE_CUSTOM (0xf800d000, 0xf0008000, t32_decode_cond_branch), - - /* BLX 1111 0xxx xxxx xxxx 11x0 xxxx xxxx xxx0 */ - DECODE_OR (0xf800d001, 0xf000c000), - /* B 1111 0xxx xxxx xxxx 10x1 xxxx xxxx xxxx */ - /* BL 1111 0xxx xxxx xxxx 11x1 xxxx xxxx xxxx */ - DECODE_SIMULATE (0xf8009000, 0xf0009000, t32_simulate_branch), - - DECODE_END -}; - -static const union decode_item t32_table_1111_100x_x0x1__1111[] = { - /* Memory hints */ - - /* PLD (literal) 1111 1000 x001 1111 1111 xxxx xxxx xxxx */ - /* PLI (literal) 1111 1001 x001 1111 1111 xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe7ff000, 0xf81ff000, kprobe_simulate_nop), - - /* PLD{W} (immediate) 1111 1000 10x1 xxxx 1111 xxxx xxxx xxxx */ - DECODE_OR (0xffd0f000, 0xf890f000), - /* PLD{W} (immediate) 1111 1000 00x1 xxxx 1111 1100 xxxx xxxx */ - DECODE_OR (0xffd0ff00, 0xf810fc00), - /* PLI (immediate) 1111 1001 1001 xxxx 1111 xxxx xxxx xxxx */ - DECODE_OR (0xfff0f000, 0xf990f000), - /* PLI (immediate) 1111 1001 0001 xxxx 1111 1100 xxxx xxxx */ - DECODE_SIMULATEX(0xfff0ff00, 0xf910fc00, kprobe_simulate_nop, - REGS(NOPCX, 0, 0, 0, 0)), - - /* PLD{W} (register) 1111 1000 00x1 xxxx 1111 0000 00xx xxxx */ - DECODE_OR (0xffd0ffc0, 0xf810f000), - /* PLI (register) 1111 1001 0001 xxxx 1111 0000 00xx xxxx */ - DECODE_SIMULATEX(0xfff0ffc0, 0xf910f000, kprobe_simulate_nop, - REGS(NOPCX, 0, 0, 0, NOSPPC)), - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1111_100x[] = { - /* Store/Load single data item */ - - /* ??? 1111 100x x11x xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfe600000, 0xf8600000), - - /* ??? 1111 1001 0101 xxxx xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xfff00000, 0xf9500000), - - /* ??? 1111 100x 0xxx xxxx xxxx 10x0 xxxx xxxx */ - DECODE_REJECT (0xfe800d00, 0xf8000800), - - /* STRBT 1111 1000 0000 xxxx xxxx 1110 xxxx xxxx */ - /* STRHT 1111 1000 0010 xxxx xxxx 1110 xxxx xxxx */ - /* STRT 1111 1000 0100 xxxx xxxx 1110 xxxx xxxx */ - /* LDRBT 1111 1000 0001 xxxx xxxx 1110 xxxx xxxx */ - /* LDRSBT 1111 1001 0001 xxxx xxxx 1110 xxxx xxxx */ - /* LDRHT 1111 1000 0011 xxxx xxxx 1110 xxxx xxxx */ - /* LDRSHT 1111 1001 0011 xxxx xxxx 1110 xxxx xxxx */ - /* LDRT 1111 1000 0101 xxxx xxxx 1110 xxxx xxxx */ - DECODE_REJECT (0xfe800f00, 0xf8000e00), - - /* STR{,B,H} Rn,[PC...] 1111 1000 xxx0 1111 xxxx xxxx xxxx xxxx */ - DECODE_REJECT (0xff1f0000, 0xf80f0000), - - /* STR{,B,H} PC,[Rn...] 1111 1000 xxx0 xxxx 1111 xxxx xxxx xxxx */ - DECODE_REJECT (0xff10f000, 0xf800f000), - - /* LDR (literal) 1111 1000 x101 1111 xxxx xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xff7f0000, 0xf85f0000, t32_simulate_ldr_literal, - REGS(PC, ANY, 0, 0, 0)), - - /* STR (immediate) 1111 1000 0100 xxxx xxxx 1xxx xxxx xxxx */ - /* LDR (immediate) 1111 1000 0101 xxxx xxxx 1xxx xxxx xxxx */ - DECODE_OR (0xffe00800, 0xf8400800), - /* STR (immediate) 1111 1000 1100 xxxx xxxx xxxx xxxx xxxx */ - /* LDR (immediate) 1111 1000 1101 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xffe00000, 0xf8c00000, t32_emulate_ldrstr, - REGS(NOPCX, ANY, 0, 0, 0)), - - /* STR (register) 1111 1000 0100 xxxx xxxx 0000 00xx xxxx */ - /* LDR (register) 1111 1000 0101 xxxx xxxx 0000 00xx xxxx */ - DECODE_EMULATEX (0xffe00fc0, 0xf8400000, t32_emulate_ldrstr, - REGS(NOPCX, ANY, 0, 0, NOSPPC)), - - /* LDRB (literal) 1111 1000 x001 1111 xxxx xxxx xxxx xxxx */ - /* LDRSB (literal) 1111 1001 x001 1111 xxxx xxxx xxxx xxxx */ - /* LDRH (literal) 1111 1000 x011 1111 xxxx xxxx xxxx xxxx */ - /* LDRSH (literal) 1111 1001 x011 1111 xxxx xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xfe5f0000, 0xf81f0000, t32_simulate_ldr_literal, - REGS(PC, NOSPPCX, 0, 0, 0)), - - /* STRB (immediate) 1111 1000 0000 xxxx xxxx 1xxx xxxx xxxx */ - /* STRH (immediate) 1111 1000 0010 xxxx xxxx 1xxx xxxx xxxx */ - /* LDRB (immediate) 1111 1000 0001 xxxx xxxx 1xxx xxxx xxxx */ - /* LDRSB (immediate) 1111 1001 0001 xxxx xxxx 1xxx xxxx xxxx */ - /* LDRH (immediate) 1111 1000 0011 xxxx xxxx 1xxx xxxx xxxx */ - /* LDRSH (immediate) 1111 1001 0011 xxxx xxxx 1xxx xxxx xxxx */ - DECODE_OR (0xfec00800, 0xf8000800), - /* STRB (immediate) 1111 1000 1000 xxxx xxxx xxxx xxxx xxxx */ - /* STRH (immediate) 1111 1000 1010 xxxx xxxx xxxx xxxx xxxx */ - /* LDRB (immediate) 1111 1000 1001 xxxx xxxx xxxx xxxx xxxx */ - /* LDRSB (immediate) 1111 1001 1001 xxxx xxxx xxxx xxxx xxxx */ - /* LDRH (immediate) 1111 1000 1011 xxxx xxxx xxxx xxxx xxxx */ - /* LDRSH (immediate) 1111 1001 1011 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfec00000, 0xf8800000, t32_emulate_ldrstr, - REGS(NOPCX, NOSPPCX, 0, 0, 0)), - - /* STRB (register) 1111 1000 0000 xxxx xxxx 0000 00xx xxxx */ - /* STRH (register) 1111 1000 0010 xxxx xxxx 0000 00xx xxxx */ - /* LDRB (register) 1111 1000 0001 xxxx xxxx 0000 00xx xxxx */ - /* LDRSB (register) 1111 1001 0001 xxxx xxxx 0000 00xx xxxx */ - /* LDRH (register) 1111 1000 0011 xxxx xxxx 0000 00xx xxxx */ - /* LDRSH (register) 1111 1001 0011 xxxx xxxx 0000 00xx xxxx */ - DECODE_EMULATEX (0xfe800fc0, 0xf8000000, t32_emulate_ldrstr, - REGS(NOPCX, NOSPPCX, 0, 0, NOSPPC)), - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1111_1010___1111[] = { - /* Data-processing (register) */ - - /* ??? 1111 1010 011x xxxx 1111 xxxx 1xxx xxxx */ - DECODE_REJECT (0xffe0f080, 0xfa60f080), - - /* SXTH 1111 1010 0000 1111 1111 xxxx 1xxx xxxx */ - /* UXTH 1111 1010 0001 1111 1111 xxxx 1xxx xxxx */ - /* SXTB16 1111 1010 0010 1111 1111 xxxx 1xxx xxxx */ - /* UXTB16 1111 1010 0011 1111 1111 xxxx 1xxx xxxx */ - /* SXTB 1111 1010 0100 1111 1111 xxxx 1xxx xxxx */ - /* UXTB 1111 1010 0101 1111 1111 xxxx 1xxx xxxx */ - DECODE_EMULATEX (0xff8ff080, 0xfa0ff080, t32_emulate_rd8rn16rm0_rwflags, - REGS(0, 0, NOSPPC, 0, NOSPPC)), - - - /* ??? 1111 1010 1xxx xxxx 1111 xxxx 0x11 xxxx */ - DECODE_REJECT (0xff80f0b0, 0xfa80f030), - /* ??? 1111 1010 1x11 xxxx 1111 xxxx 0xxx xxxx */ - DECODE_REJECT (0xffb0f080, 0xfab0f000), - - /* SADD16 1111 1010 1001 xxxx 1111 xxxx 0000 xxxx */ - /* SASX 1111 1010 1010 xxxx 1111 xxxx 0000 xxxx */ - /* SSAX 1111 1010 1110 xxxx 1111 xxxx 0000 xxxx */ - /* SSUB16 1111 1010 1101 xxxx 1111 xxxx 0000 xxxx */ - /* SADD8 1111 1010 1000 xxxx 1111 xxxx 0000 xxxx */ - /* SSUB8 1111 1010 1100 xxxx 1111 xxxx 0000 xxxx */ - - /* QADD16 1111 1010 1001 xxxx 1111 xxxx 0001 xxxx */ - /* QASX 1111 1010 1010 xxxx 1111 xxxx 0001 xxxx */ - /* QSAX 1111 1010 1110 xxxx 1111 xxxx 0001 xxxx */ - /* QSUB16 1111 1010 1101 xxxx 1111 xxxx 0001 xxxx */ - /* QADD8 1111 1010 1000 xxxx 1111 xxxx 0001 xxxx */ - /* QSUB8 1111 1010 1100 xxxx 1111 xxxx 0001 xxxx */ - - /* SHADD16 1111 1010 1001 xxxx 1111 xxxx 0010 xxxx */ - /* SHASX 1111 1010 1010 xxxx 1111 xxxx 0010 xxxx */ - /* SHSAX 1111 1010 1110 xxxx 1111 xxxx 0010 xxxx */ - /* SHSUB16 1111 1010 1101 xxxx 1111 xxxx 0010 xxxx */ - /* SHADD8 1111 1010 1000 xxxx 1111 xxxx 0010 xxxx */ - /* SHSUB8 1111 1010 1100 xxxx 1111 xxxx 0010 xxxx */ - - /* UADD16 1111 1010 1001 xxxx 1111 xxxx 0100 xxxx */ - /* UASX 1111 1010 1010 xxxx 1111 xxxx 0100 xxxx */ - /* USAX 1111 1010 1110 xxxx 1111 xxxx 0100 xxxx */ - /* USUB16 1111 1010 1101 xxxx 1111 xxxx 0100 xxxx */ - /* UADD8 1111 1010 1000 xxxx 1111 xxxx 0100 xxxx */ - /* USUB8 1111 1010 1100 xxxx 1111 xxxx 0100 xxxx */ - - /* UQADD16 1111 1010 1001 xxxx 1111 xxxx 0101 xxxx */ - /* UQASX 1111 1010 1010 xxxx 1111 xxxx 0101 xxxx */ - /* UQSAX 1111 1010 1110 xxxx 1111 xxxx 0101 xxxx */ - /* UQSUB16 1111 1010 1101 xxxx 1111 xxxx 0101 xxxx */ - /* UQADD8 1111 1010 1000 xxxx 1111 xxxx 0101 xxxx */ - /* UQSUB8 1111 1010 1100 xxxx 1111 xxxx 0101 xxxx */ - - /* UHADD16 1111 1010 1001 xxxx 1111 xxxx 0110 xxxx */ - /* UHASX 1111 1010 1010 xxxx 1111 xxxx 0110 xxxx */ - /* UHSAX 1111 1010 1110 xxxx 1111 xxxx 0110 xxxx */ - /* UHSUB16 1111 1010 1101 xxxx 1111 xxxx 0110 xxxx */ - /* UHADD8 1111 1010 1000 xxxx 1111 xxxx 0110 xxxx */ - /* UHSUB8 1111 1010 1100 xxxx 1111 xxxx 0110 xxxx */ - DECODE_OR (0xff80f080, 0xfa80f000), - - /* SXTAH 1111 1010 0000 xxxx 1111 xxxx 1xxx xxxx */ - /* UXTAH 1111 1010 0001 xxxx 1111 xxxx 1xxx xxxx */ - /* SXTAB16 1111 1010 0010 xxxx 1111 xxxx 1xxx xxxx */ - /* UXTAB16 1111 1010 0011 xxxx 1111 xxxx 1xxx xxxx */ - /* SXTAB 1111 1010 0100 xxxx 1111 xxxx 1xxx xxxx */ - /* UXTAB 1111 1010 0101 xxxx 1111 xxxx 1xxx xxxx */ - DECODE_OR (0xff80f080, 0xfa00f080), - - /* QADD 1111 1010 1000 xxxx 1111 xxxx 1000 xxxx */ - /* QDADD 1111 1010 1000 xxxx 1111 xxxx 1001 xxxx */ - /* QSUB 1111 1010 1000 xxxx 1111 xxxx 1010 xxxx */ - /* QDSUB 1111 1010 1000 xxxx 1111 xxxx 1011 xxxx */ - DECODE_OR (0xfff0f0c0, 0xfa80f080), - - /* SEL 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ - DECODE_OR (0xfff0f0f0, 0xfaa0f080), - - /* LSL 1111 1010 000x xxxx 1111 xxxx 0000 xxxx */ - /* LSR 1111 1010 001x xxxx 1111 xxxx 0000 xxxx */ - /* ASR 1111 1010 010x xxxx 1111 xxxx 0000 xxxx */ - /* ROR 1111 1010 011x xxxx 1111 xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff80f0f0, 0xfa00f000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), - - /* CLZ 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ - DECODE_OR (0xfff0f0f0, 0xfab0f080), - - /* REV 1111 1010 1001 xxxx 1111 xxxx 1000 xxxx */ - /* REV16 1111 1010 1001 xxxx 1111 xxxx 1001 xxxx */ - /* RBIT 1111 1010 1001 xxxx 1111 xxxx 1010 xxxx */ - /* REVSH 1111 1010 1001 xxxx 1111 xxxx 1011 xxxx */ - DECODE_EMULATEX (0xfff0f0c0, 0xfa90f080, t32_emulate_rd8rn16_noflags, - REGS(NOSPPC, 0, NOSPPC, 0, SAMEAS16)), - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1111_1011_0[] = { - /* Multiply, multiply accumulate, and absolute difference */ - - /* ??? 1111 1011 0000 xxxx 1111 xxxx 0001 xxxx */ - DECODE_REJECT (0xfff0f0f0, 0xfb00f010), - /* ??? 1111 1011 0111 xxxx 1111 xxxx 0001 xxxx */ - DECODE_REJECT (0xfff0f0f0, 0xfb70f010), - - /* SMULxy 1111 1011 0001 xxxx 1111 xxxx 00xx xxxx */ - DECODE_OR (0xfff0f0c0, 0xfb10f000), - /* MUL 1111 1011 0000 xxxx 1111 xxxx 0000 xxxx */ - /* SMUAD{X} 1111 1011 0010 xxxx 1111 xxxx 000x xxxx */ - /* SMULWy 1111 1011 0011 xxxx 1111 xxxx 000x xxxx */ - /* SMUSD{X} 1111 1011 0100 xxxx 1111 xxxx 000x xxxx */ - /* SMMUL{R} 1111 1011 0101 xxxx 1111 xxxx 000x xxxx */ - /* USAD8 1111 1011 0111 xxxx 1111 xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff80f0e0, 0xfb00f000, t32_emulate_rd8rn16rm0_rwflags, - REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), - - /* ??? 1111 1011 0111 xxxx xxxx xxxx 0001 xxxx */ - DECODE_REJECT (0xfff000f0, 0xfb700010), - - /* SMLAxy 1111 1011 0001 xxxx xxxx xxxx 00xx xxxx */ - DECODE_OR (0xfff000c0, 0xfb100000), - /* MLA 1111 1011 0000 xxxx xxxx xxxx 0000 xxxx */ - /* MLS 1111 1011 0000 xxxx xxxx xxxx 0001 xxxx */ - /* SMLAD{X} 1111 1011 0010 xxxx xxxx xxxx 000x xxxx */ - /* SMLAWy 1111 1011 0011 xxxx xxxx xxxx 000x xxxx */ - /* SMLSD{X} 1111 1011 0100 xxxx xxxx xxxx 000x xxxx */ - /* SMMLA{R} 1111 1011 0101 xxxx xxxx xxxx 000x xxxx */ - /* SMMLS{R} 1111 1011 0110 xxxx xxxx xxxx 000x xxxx */ - /* USADA8 1111 1011 0111 xxxx xxxx xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff8000c0, 0xfb000000, t32_emulate_rd8rn16rm0ra12_noflags, - REGS(NOSPPC, NOSPPCX, NOSPPC, 0, NOSPPC)), - - /* Other unallocated instructions... */ - DECODE_END -}; - -static const union decode_item t32_table_1111_1011_1[] = { - /* Long multiply, long multiply accumulate, and divide */ - - /* UMAAL 1111 1011 1110 xxxx xxxx xxxx 0110 xxxx */ - DECODE_OR (0xfff000f0, 0xfbe00060), - /* SMLALxy 1111 1011 1100 xxxx xxxx xxxx 10xx xxxx */ - DECODE_OR (0xfff000c0, 0xfbc00080), - /* SMLALD{X} 1111 1011 1100 xxxx xxxx xxxx 110x xxxx */ - /* SMLSLD{X} 1111 1011 1101 xxxx xxxx xxxx 110x xxxx */ - DECODE_OR (0xffe000e0, 0xfbc000c0), - /* SMULL 1111 1011 1000 xxxx xxxx xxxx 0000 xxxx */ - /* UMULL 1111 1011 1010 xxxx xxxx xxxx 0000 xxxx */ - /* SMLAL 1111 1011 1100 xxxx xxxx xxxx 0000 xxxx */ - /* UMLAL 1111 1011 1110 xxxx xxxx xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff9000f0, 0xfb800000, t32_emulate_rdlo12rdhi8rn16rm0_noflags, - REGS(NOSPPC, NOSPPC, NOSPPC, 0, NOSPPC)), - - /* SDIV 1111 1011 1001 xxxx xxxx xxxx 1111 xxxx */ - /* UDIV 1111 1011 1011 xxxx xxxx xxxx 1111 xxxx */ - /* Other unallocated instructions... */ - DECODE_END -}; - -const union decode_item kprobe_decode_thumb32_table[] = { - - /* - * Load/store multiple instructions - * 1110 100x x0xx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe400000, 0xe8000000, t32_table_1110_100x_x0xx), - - /* - * Load/store dual, load/store exclusive, table branch - * 1110 100x x1xx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe400000, 0xe8400000, t32_table_1110_100x_x1xx), - - /* - * Data-processing (shifted register) - * 1110 101x xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe000000, 0xea000000, t32_table_1110_101x), - - /* - * Coprocessor instructions - * 1110 11xx xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_REJECT (0xfc000000, 0xec000000), - - /* - * Data-processing (modified immediate) - * 1111 0x0x xxxx xxxx 0xxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfa008000, 0xf0000000, t32_table_1111_0x0x___0), - - /* - * Data-processing (plain binary immediate) - * 1111 0x1x xxxx xxxx 0xxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfa008000, 0xf2000000, t32_table_1111_0x1x___0), - - /* - * Branches and miscellaneous control - * 1111 0xxx xxxx xxxx 1xxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xf8008000, 0xf0008000, t32_table_1111_0xxx___1), - - /* - * Advanced SIMD element or structure load/store instructions - * 1111 1001 xxx0 xxxx xxxx xxxx xxxx xxxx - */ - DECODE_REJECT (0xff100000, 0xf9000000), - - /* - * Memory hints - * 1111 100x x0x1 xxxx 1111 xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe50f000, 0xf810f000, t32_table_1111_100x_x0x1__1111), - - /* - * Store single data item - * 1111 1000 xxx0 xxxx xxxx xxxx xxxx xxxx - * Load single data items - * 1111 100x xxx1 xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xfe000000, 0xf8000000, t32_table_1111_100x), - - /* - * Data-processing (register) - * 1111 1010 xxxx xxxx 1111 xxxx xxxx xxxx - */ - DECODE_TABLE (0xff00f000, 0xfa00f000, t32_table_1111_1010___1111), - - /* - * Multiply, multiply accumulate, and absolute difference - * 1111 1011 0xxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xff800000, 0xfb000000, t32_table_1111_1011_0), - - /* - * Long multiply, long multiply accumulate, and divide - * 1111 1011 1xxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_TABLE (0xff800000, 0xfb800000, t32_table_1111_1011_1), - - /* - * Coprocessor instructions - * 1111 11xx xxxx xxxx xxxx xxxx xxxx xxxx - */ - DECODE_END -}; -#ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_thumb32_table); -#endif - -static void __kprobes +void __kprobes t16_simulate_bxblx(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -962,7 +322,7 @@ t16_simulate_bxblx(struct kprobe *p, struct pt_regs *regs) bx_write_pc(rmv, regs); } -static void __kprobes +void __kprobes t16_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -972,7 +332,7 @@ t16_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = base[index]; } -static void __kprobes +void __kprobes t16_simulate_ldrstr_sp_relative(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -985,7 +345,7 @@ t16_simulate_ldrstr_sp_relative(struct kprobe *p, struct pt_regs *regs) base[index] = regs->uregs[rt]; } -static void __kprobes +void __kprobes t16_simulate_reladr(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -996,7 +356,7 @@ t16_simulate_reladr(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = base + offset * 4; } -static void __kprobes +void __kprobes t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -1007,7 +367,7 @@ t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs) regs->ARM_sp += imm * 4; } -static void __kprobes +void __kprobes t16_simulate_cbz(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -1021,7 +381,7 @@ t16_simulate_cbz(struct kprobe *p, struct pt_regs *regs) } } -static void __kprobes +void __kprobes t16_simulate_it(struct kprobe *p, struct pt_regs *regs) { /* @@ -1038,21 +398,21 @@ t16_simulate_it(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = cpsr; } -static void __kprobes +void __kprobes t16_singlestep_it(struct kprobe *p, struct pt_regs *regs) { regs->ARM_pc += 2; t16_simulate_it(p, regs); } -static enum kprobe_insn __kprobes +enum kprobe_insn __kprobes t16_decode_it(kprobe_opcode_t insn, struct arch_specific_insn *asi) { asi->insn_singlestep = t16_singlestep_it; return INSN_GOOD_NO_SLOT; } -static void __kprobes +void __kprobes t16_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -1062,7 +422,7 @@ t16_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + (offset * 2); } -static enum kprobe_insn __kprobes +enum kprobe_insn __kprobes t16_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) { int cc = (insn >> 8) & 0xf; @@ -1071,7 +431,7 @@ t16_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD_NO_SLOT; } -static void __kprobes +void __kprobes t16_simulate_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -1103,13 +463,13 @@ t16_emulate_loregs(struct kprobe *p, struct pt_regs *regs) return (oldcpsr & ~APSR_MASK) | (newcpsr & APSR_MASK); } -static void __kprobes +void __kprobes t16_emulate_loregs_rwflags(struct kprobe *p, struct pt_regs *regs) { regs->ARM_cpsr = t16_emulate_loregs(p, regs); } -static void __kprobes +void __kprobes t16_emulate_loregs_noitrwflags(struct kprobe *p, struct pt_regs *regs) { unsigned long cpsr = t16_emulate_loregs(p, regs); @@ -1117,7 +477,7 @@ t16_emulate_loregs_noitrwflags(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = cpsr; } -static void __kprobes +void __kprobes t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -1148,7 +508,7 @@ t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -static enum kprobe_insn __kprobes +enum kprobe_insn __kprobes t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi) { insn &= ~0x00ff; @@ -1158,7 +518,7 @@ t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD; } -static void __kprobes +void __kprobes t16_emulate_push(struct kprobe *p, struct pt_regs *regs) { __asm__ __volatile__ ( @@ -1174,7 +534,7 @@ t16_emulate_push(struct kprobe *p, struct pt_regs *regs) ); } -static enum kprobe_insn __kprobes +enum kprobe_insn __kprobes t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi) { /* @@ -1188,7 +548,7 @@ t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD; } -static void __kprobes +void __kprobes t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs) { __asm__ __volatile__ ( @@ -1204,7 +564,7 @@ t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs) ); } -static void __kprobes +void __kprobes t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) { register unsigned long pc asm("r8"); @@ -1224,7 +584,7 @@ t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) bx_write_pc(pc, regs); } -static enum kprobe_insn __kprobes +enum kprobe_insn __kprobes t16_decode_pop(kprobe_opcode_t insn, struct arch_specific_insn *asi) { /* @@ -1238,232 +598,3 @@ t16_decode_pop(kprobe_opcode_t insn, struct arch_specific_insn *asi) : t16_emulate_pop_nopc; return INSN_GOOD; } - -static const union decode_item t16_table_1011[] = { - /* Miscellaneous 16-bit instructions */ - - /* ADD (SP plus immediate) 1011 0000 0xxx xxxx */ - /* SUB (SP minus immediate) 1011 0000 1xxx xxxx */ - DECODE_SIMULATE (0xff00, 0xb000, t16_simulate_add_sp_imm), - - /* CBZ 1011 00x1 xxxx xxxx */ - /* CBNZ 1011 10x1 xxxx xxxx */ - DECODE_SIMULATE (0xf500, 0xb100, t16_simulate_cbz), - - /* SXTH 1011 0010 00xx xxxx */ - /* SXTB 1011 0010 01xx xxxx */ - /* UXTH 1011 0010 10xx xxxx */ - /* UXTB 1011 0010 11xx xxxx */ - /* REV 1011 1010 00xx xxxx */ - /* REV16 1011 1010 01xx xxxx */ - /* ??? 1011 1010 10xx xxxx */ - /* REVSH 1011 1010 11xx xxxx */ - DECODE_REJECT (0xffc0, 0xba80), - DECODE_EMULATE (0xf500, 0xb000, t16_emulate_loregs_rwflags), - - /* PUSH 1011 010x xxxx xxxx */ - DECODE_CUSTOM (0xfe00, 0xb400, t16_decode_push), - /* POP 1011 110x xxxx xxxx */ - DECODE_CUSTOM (0xfe00, 0xbc00, t16_decode_pop), - - /* - * If-Then, and hints - * 1011 1111 xxxx xxxx - */ - - /* YIELD 1011 1111 0001 0000 */ - DECODE_OR (0xffff, 0xbf10), - /* SEV 1011 1111 0100 0000 */ - DECODE_EMULATE (0xffff, 0xbf40, kprobe_emulate_none), - /* NOP 1011 1111 0000 0000 */ - /* WFE 1011 1111 0010 0000 */ - /* WFI 1011 1111 0011 0000 */ - DECODE_SIMULATE (0xffcf, 0xbf00, kprobe_simulate_nop), - /* Unassigned hints 1011 1111 xxxx 0000 */ - DECODE_REJECT (0xff0f, 0xbf00), - /* IT 1011 1111 xxxx xxxx */ - DECODE_CUSTOM (0xff00, 0xbf00, t16_decode_it), - - /* SETEND 1011 0110 010x xxxx */ - /* CPS 1011 0110 011x xxxx */ - /* BKPT 1011 1110 xxxx xxxx */ - /* And unallocated instructions... */ - DECODE_END -}; - -const union decode_item kprobe_decode_thumb16_table[] = { - - /* - * Shift (immediate), add, subtract, move, and compare - * 00xx xxxx xxxx xxxx - */ - - /* CMP (immediate) 0010 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf800, 0x2800, t16_emulate_loregs_rwflags), - - /* ADD (register) 0001 100x xxxx xxxx */ - /* SUB (register) 0001 101x xxxx xxxx */ - /* LSL (immediate) 0000 0xxx xxxx xxxx */ - /* LSR (immediate) 0000 1xxx xxxx xxxx */ - /* ASR (immediate) 0001 0xxx xxxx xxxx */ - /* ADD (immediate, Thumb) 0001 110x xxxx xxxx */ - /* SUB (immediate, Thumb) 0001 111x xxxx xxxx */ - /* MOV (immediate) 0010 0xxx xxxx xxxx */ - /* ADD (immediate, Thumb) 0011 0xxx xxxx xxxx */ - /* SUB (immediate, Thumb) 0011 1xxx xxxx xxxx */ - DECODE_EMULATE (0xc000, 0x0000, t16_emulate_loregs_noitrwflags), - - /* - * 16-bit Thumb data-processing instructions - * 0100 00xx xxxx xxxx - */ - - /* TST (register) 0100 0010 00xx xxxx */ - DECODE_EMULATE (0xffc0, 0x4200, t16_emulate_loregs_rwflags), - /* CMP (register) 0100 0010 10xx xxxx */ - /* CMN (register) 0100 0010 11xx xxxx */ - DECODE_EMULATE (0xff80, 0x4280, t16_emulate_loregs_rwflags), - /* AND (register) 0100 0000 00xx xxxx */ - /* EOR (register) 0100 0000 01xx xxxx */ - /* LSL (register) 0100 0000 10xx xxxx */ - /* LSR (register) 0100 0000 11xx xxxx */ - /* ASR (register) 0100 0001 00xx xxxx */ - /* ADC (register) 0100 0001 01xx xxxx */ - /* SBC (register) 0100 0001 10xx xxxx */ - /* ROR (register) 0100 0001 11xx xxxx */ - /* RSB (immediate) 0100 0010 01xx xxxx */ - /* ORR (register) 0100 0011 00xx xxxx */ - /* MUL 0100 0011 00xx xxxx */ - /* BIC (register) 0100 0011 10xx xxxx */ - /* MVN (register) 0100 0011 10xx xxxx */ - DECODE_EMULATE (0xfc00, 0x4000, t16_emulate_loregs_noitrwflags), - - /* - * Special data instructions and branch and exchange - * 0100 01xx xxxx xxxx - */ - - /* BLX pc 0100 0111 1111 1xxx */ - DECODE_REJECT (0xfff8, 0x47f8), - - /* BX (register) 0100 0111 0xxx xxxx */ - /* BLX (register) 0100 0111 1xxx xxxx */ - DECODE_SIMULATE (0xff00, 0x4700, t16_simulate_bxblx), - - /* ADD pc, pc 0100 0100 1111 1111 */ - DECODE_REJECT (0xffff, 0x44ff), - - /* ADD (register) 0100 0100 xxxx xxxx */ - /* CMP (register) 0100 0101 xxxx xxxx */ - /* MOV (register) 0100 0110 xxxx xxxx */ - DECODE_CUSTOM (0xfc00, 0x4400, t16_decode_hiregs), - - /* - * Load from Literal Pool - * LDR (literal) 0100 1xxx xxxx xxxx - */ - DECODE_SIMULATE (0xf800, 0x4800, t16_simulate_ldr_literal), - - /* - * 16-bit Thumb Load/store instructions - * 0101 xxxx xxxx xxxx - * 011x xxxx xxxx xxxx - * 100x xxxx xxxx xxxx - */ - - /* STR (register) 0101 000x xxxx xxxx */ - /* STRH (register) 0101 001x xxxx xxxx */ - /* STRB (register) 0101 010x xxxx xxxx */ - /* LDRSB (register) 0101 011x xxxx xxxx */ - /* LDR (register) 0101 100x xxxx xxxx */ - /* LDRH (register) 0101 101x xxxx xxxx */ - /* LDRB (register) 0101 110x xxxx xxxx */ - /* LDRSH (register) 0101 111x xxxx xxxx */ - /* STR (immediate, Thumb) 0110 0xxx xxxx xxxx */ - /* LDR (immediate, Thumb) 0110 1xxx xxxx xxxx */ - /* STRB (immediate, Thumb) 0111 0xxx xxxx xxxx */ - /* LDRB (immediate, Thumb) 0111 1xxx xxxx xxxx */ - DECODE_EMULATE (0xc000, 0x4000, t16_emulate_loregs_rwflags), - /* STRH (immediate, Thumb) 1000 0xxx xxxx xxxx */ - /* LDRH (immediate, Thumb) 1000 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf000, 0x8000, t16_emulate_loregs_rwflags), - /* STR (immediate, Thumb) 1001 0xxx xxxx xxxx */ - /* LDR (immediate, Thumb) 1001 1xxx xxxx xxxx */ - DECODE_SIMULATE (0xf000, 0x9000, t16_simulate_ldrstr_sp_relative), - - /* - * Generate PC-/SP-relative address - * ADR (literal) 1010 0xxx xxxx xxxx - * ADD (SP plus immediate) 1010 1xxx xxxx xxxx - */ - DECODE_SIMULATE (0xf000, 0xa000, t16_simulate_reladr), - - /* - * Miscellaneous 16-bit instructions - * 1011 xxxx xxxx xxxx - */ - DECODE_TABLE (0xf000, 0xb000, t16_table_1011), - - /* STM 1100 0xxx xxxx xxxx */ - /* LDM 1100 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf000, 0xc000, t16_emulate_loregs_rwflags), - - /* - * Conditional branch, and Supervisor Call - */ - - /* Permanently UNDEFINED 1101 1110 xxxx xxxx */ - /* SVC 1101 1111 xxxx xxxx */ - DECODE_REJECT (0xfe00, 0xde00), - - /* Conditional branch 1101 xxxx xxxx xxxx */ - DECODE_CUSTOM (0xf000, 0xd000, t16_decode_cond_branch), - - /* - * Unconditional branch - * B 1110 0xxx xxxx xxxx - */ - DECODE_SIMULATE (0xf800, 0xe000, t16_simulate_branch), - - DECODE_END -}; -#ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_thumb16_table); -#endif - -static unsigned long __kprobes thumb_check_cc(unsigned long cpsr) -{ - if (unlikely(in_it_block(cpsr))) - return kprobe_condition_checks[current_cond(cpsr)](cpsr); - return true; -} - -static void __kprobes thumb16_singlestep(struct kprobe *p, struct pt_regs *regs) -{ - regs->ARM_pc += 2; - p->ainsn.insn_handler(p, regs); - regs->ARM_cpsr = it_advance(regs->ARM_cpsr); -} - -static void __kprobes thumb32_singlestep(struct kprobe *p, struct pt_regs *regs) -{ - regs->ARM_pc += 4; - p->ainsn.insn_handler(p, regs); - regs->ARM_cpsr = it_advance(regs->ARM_cpsr); -} - -enum kprobe_insn __kprobes -thumb16_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) -{ - asi->insn_singlestep = thumb16_singlestep; - asi->insn_check_cc = thumb_check_cc; - return kprobe_decode_insn(insn, asi, kprobe_decode_thumb16_table, true); -} - -enum kprobe_insn __kprobes -thumb32_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) -{ - asi->insn_singlestep = thumb32_singlestep; - asi->insn_check_cc = thumb_check_cc; - return kprobe_decode_insn(insn, asi, kprobe_decode_thumb32_table, true); -} diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c new file mode 100644 index 000000000000..a1f24777a41a --- /dev/null +++ b/arch/arm/kernel/probes-thumb.c @@ -0,0 +1,878 @@ +/* + * arch/arm/kernel/kprobes-thumb.c + * + * Copyright (C) 2011 Jon Medhurst . + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include + +#include "kprobes.h" +#include "probes-thumb.h" + +/* These emulation encodings are functionally equivalent... */ +#define t32_emulate_rd8rn16rm0ra12_noflags \ + t32_emulate_rdlo12rdhi8rn16rm0_noflags + +static const union decode_item t32_table_1110_100x_x0xx[] = { + /* Load/store multiple instructions */ + + /* Rn is PC 1110 100x x0xx 1111 xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfe4f0000, 0xe80f0000), + + /* SRS 1110 1000 00x0 xxxx xxxx xxxx xxxx xxxx */ + /* RFE 1110 1000 00x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffc00000, 0xe8000000), + /* SRS 1110 1001 10x0 xxxx xxxx xxxx xxxx xxxx */ + /* RFE 1110 1001 10x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffc00000, 0xe9800000), + + /* STM Rn, {...pc} 1110 100x x0x0 xxxx 1xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfe508000, 0xe8008000), + /* LDM Rn, {...lr,pc} 1110 100x x0x1 xxxx 11xx xxxx xxxx xxxx */ + DECODE_REJECT (0xfe50c000, 0xe810c000), + /* LDM/STM Rn, {...sp} 1110 100x x0xx xxxx xx1x xxxx xxxx xxxx */ + DECODE_REJECT (0xfe402000, 0xe8002000), + + /* STMIA 1110 1000 10x0 xxxx xxxx xxxx xxxx xxxx */ + /* LDMIA 1110 1000 10x1 xxxx xxxx xxxx xxxx xxxx */ + /* STMDB 1110 1001 00x0 xxxx xxxx xxxx xxxx xxxx */ + /* LDMDB 1110 1001 00x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_CUSTOM (0xfe400000, 0xe8000000, t32_decode_ldmstm), + + DECODE_END +}; + +static const union decode_item t32_table_1110_100x_x1xx[] = { + /* Load/store dual, load/store exclusive, table branch */ + + /* STRD (immediate) 1110 1000 x110 xxxx xxxx xxxx xxxx xxxx */ + /* LDRD (immediate) 1110 1000 x111 xxxx xxxx xxxx xxxx xxxx */ + DECODE_OR (0xff600000, 0xe8600000), + /* STRD (immediate) 1110 1001 x1x0 xxxx xxxx xxxx xxxx xxxx */ + /* LDRD (immediate) 1110 1001 x1x1 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xff400000, 0xe9400000, t32_emulate_ldrdstrd, + REGS(NOPCWB, NOSPPC, NOSPPC, 0, 0)), + + /* TBB 1110 1000 1101 xxxx xxxx xxxx 0000 xxxx */ + /* TBH 1110 1000 1101 xxxx xxxx xxxx 0001 xxxx */ + DECODE_SIMULATEX(0xfff000e0, 0xe8d00000, t32_simulate_table_branch, + REGS(NOSP, 0, 0, 0, NOSPPC)), + + /* STREX 1110 1000 0100 xxxx xxxx xxxx xxxx xxxx */ + /* LDREX 1110 1000 0101 xxxx xxxx xxxx xxxx xxxx */ + /* STREXB 1110 1000 1100 xxxx xxxx xxxx 0100 xxxx */ + /* STREXH 1110 1000 1100 xxxx xxxx xxxx 0101 xxxx */ + /* STREXD 1110 1000 1100 xxxx xxxx xxxx 0111 xxxx */ + /* LDREXB 1110 1000 1101 xxxx xxxx xxxx 0100 xxxx */ + /* LDREXH 1110 1000 1101 xxxx xxxx xxxx 0101 xxxx */ + /* LDREXD 1110 1000 1101 xxxx xxxx xxxx 0111 xxxx */ + /* And unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1110_101x[] = { + /* Data-processing (shifted register) */ + + /* TST 1110 1010 0001 xxxx xxxx 1111 xxxx xxxx */ + /* TEQ 1110 1010 1001 xxxx xxxx 1111 xxxx xxxx */ + DECODE_EMULATEX (0xff700f00, 0xea100f00, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOSPPC, 0, 0, 0, NOSPPC)), + + /* CMN 1110 1011 0001 xxxx xxxx 1111 xxxx xxxx */ + DECODE_OR (0xfff00f00, 0xeb100f00), + /* CMP 1110 1011 1011 xxxx xxxx 1111 xxxx xxxx */ + DECODE_EMULATEX (0xfff00f00, 0xebb00f00, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOPC, 0, 0, 0, NOSPPC)), + + /* MOV 1110 1010 010x 1111 xxxx xxxx xxxx xxxx */ + /* MVN 1110 1010 011x 1111 xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xffcf0000, 0xea4f0000, t32_emulate_rd8rn16rm0_rwflags, + REGS(0, 0, NOSPPC, 0, NOSPPC)), + + /* ??? 1110 1010 101x xxxx xxxx xxxx xxxx xxxx */ + /* ??? 1110 1010 111x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffa00000, 0xeaa00000), + /* ??? 1110 1011 001x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffe00000, 0xeb200000), + /* ??? 1110 1011 100x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffe00000, 0xeb800000), + /* ??? 1110 1011 111x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xffe00000, 0xebe00000), + + /* ADD/SUB SP, SP, Rm, LSL #0..3 */ + /* 1110 1011 x0xx 1101 x000 1101 xx00 xxxx */ + DECODE_EMULATEX (0xff4f7f30, 0xeb0d0d00, t32_emulate_rd8rn16rm0_rwflags, + REGS(SP, 0, SP, 0, NOSPPC)), + + /* ADD/SUB SP, SP, Rm, shift */ + /* 1110 1011 x0xx 1101 xxxx 1101 xxxx xxxx */ + DECODE_REJECT (0xff4f0f00, 0xeb0d0d00), + + /* ADD/SUB Rd, SP, Rm, shift */ + /* 1110 1011 x0xx 1101 xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xff4f0000, 0xeb0d0000, t32_emulate_rd8rn16rm0_rwflags, + REGS(SP, 0, NOPC, 0, NOSPPC)), + + /* AND 1110 1010 000x xxxx xxxx xxxx xxxx xxxx */ + /* BIC 1110 1010 001x xxxx xxxx xxxx xxxx xxxx */ + /* ORR 1110 1010 010x xxxx xxxx xxxx xxxx xxxx */ + /* ORN 1110 1010 011x xxxx xxxx xxxx xxxx xxxx */ + /* EOR 1110 1010 100x xxxx xxxx xxxx xxxx xxxx */ + /* PKH 1110 1010 110x xxxx xxxx xxxx xxxx xxxx */ + /* ADD 1110 1011 000x xxxx xxxx xxxx xxxx xxxx */ + /* ADC 1110 1011 010x xxxx xxxx xxxx xxxx xxxx */ + /* SBC 1110 1011 011x xxxx xxxx xxxx xxxx xxxx */ + /* SUB 1110 1011 101x xxxx xxxx xxxx xxxx xxxx */ + /* RSB 1110 1011 110x xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfe000000, 0xea000000, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), + + DECODE_END +}; + +static const union decode_item t32_table_1111_0x0x___0[] = { + /* Data-processing (modified immediate) */ + + /* TST 1111 0x00 0001 xxxx 0xxx 1111 xxxx xxxx */ + /* TEQ 1111 0x00 1001 xxxx 0xxx 1111 xxxx xxxx */ + DECODE_EMULATEX (0xfb708f00, 0xf0100f00, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOSPPC, 0, 0, 0, 0)), + + /* CMN 1111 0x01 0001 xxxx 0xxx 1111 xxxx xxxx */ + DECODE_OR (0xfbf08f00, 0xf1100f00), + /* CMP 1111 0x01 1011 xxxx 0xxx 1111 xxxx xxxx */ + DECODE_EMULATEX (0xfbf08f00, 0xf1b00f00, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOPC, 0, 0, 0, 0)), + + /* MOV 1111 0x00 010x 1111 0xxx xxxx xxxx xxxx */ + /* MVN 1111 0x00 011x 1111 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbcf8000, 0xf04f0000, t32_emulate_rd8rn16rm0_rwflags, + REGS(0, 0, NOSPPC, 0, 0)), + + /* ??? 1111 0x00 101x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbe08000, 0xf0a00000), + /* ??? 1111 0x00 110x xxxx 0xxx xxxx xxxx xxxx */ + /* ??? 1111 0x00 111x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbc08000, 0xf0c00000), + /* ??? 1111 0x01 001x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbe08000, 0xf1200000), + /* ??? 1111 0x01 100x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbe08000, 0xf1800000), + /* ??? 1111 0x01 111x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfbe08000, 0xf1e00000), + + /* ADD Rd, SP, #imm 1111 0x01 000x 1101 0xxx xxxx xxxx xxxx */ + /* SUB Rd, SP, #imm 1111 0x01 101x 1101 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfb4f8000, 0xf10d0000, t32_emulate_rd8rn16rm0_rwflags, + REGS(SP, 0, NOPC, 0, 0)), + + /* AND 1111 0x00 000x xxxx 0xxx xxxx xxxx xxxx */ + /* BIC 1111 0x00 001x xxxx 0xxx xxxx xxxx xxxx */ + /* ORR 1111 0x00 010x xxxx 0xxx xxxx xxxx xxxx */ + /* ORN 1111 0x00 011x xxxx 0xxx xxxx xxxx xxxx */ + /* EOR 1111 0x00 100x xxxx 0xxx xxxx xxxx xxxx */ + /* ADD 1111 0x01 000x xxxx 0xxx xxxx xxxx xxxx */ + /* ADC 1111 0x01 010x xxxx 0xxx xxxx xxxx xxxx */ + /* SBC 1111 0x01 011x xxxx 0xxx xxxx xxxx xxxx */ + /* SUB 1111 0x01 101x xxxx 0xxx xxxx xxxx xxxx */ + /* RSB 1111 0x01 110x xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfa008000, 0xf0000000, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOSPPC, 0, NOSPPC, 0, 0)), + + DECODE_END +}; + +static const union decode_item t32_table_1111_0x1x___0[] = { + /* Data-processing (plain binary immediate) */ + + /* ADDW Rd, PC, #imm 1111 0x10 0000 1111 0xxx xxxx xxxx xxxx */ + DECODE_OR (0xfbff8000, 0xf20f0000), + /* SUBW Rd, PC, #imm 1111 0x10 1010 1111 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbff8000, 0xf2af0000, t32_emulate_rd8pc16_noflags, + REGS(PC, 0, NOSPPC, 0, 0)), + + /* ADDW SP, SP, #imm 1111 0x10 0000 1101 0xxx 1101 xxxx xxxx */ + DECODE_OR (0xfbff8f00, 0xf20d0d00), + /* SUBW SP, SP, #imm 1111 0x10 1010 1101 0xxx 1101 xxxx xxxx */ + DECODE_EMULATEX (0xfbff8f00, 0xf2ad0d00, t32_emulate_rd8rn16_noflags, + REGS(SP, 0, SP, 0, 0)), + + /* ADDW 1111 0x10 0000 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_OR (0xfbf08000, 0xf2000000), + /* SUBW 1111 0x10 1010 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbf08000, 0xf2a00000, t32_emulate_rd8rn16_noflags, + REGS(NOPCX, 0, NOSPPC, 0, 0)), + + /* MOVW 1111 0x10 0100 xxxx 0xxx xxxx xxxx xxxx */ + /* MOVT 1111 0x10 1100 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfb708000, 0xf2400000, t32_emulate_rd8rn16_noflags, + REGS(0, 0, NOSPPC, 0, 0)), + + /* SSAT16 1111 0x11 0010 xxxx 0000 xxxx 00xx xxxx */ + /* SSAT 1111 0x11 00x0 xxxx 0xxx xxxx xxxx xxxx */ + /* USAT16 1111 0x11 1010 xxxx 0000 xxxx 00xx xxxx */ + /* USAT 1111 0x11 10x0 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfb508000, 0xf3000000, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOSPPC, 0, NOSPPC, 0, 0)), + + /* SFBX 1111 0x11 0100 xxxx 0xxx xxxx xxxx xxxx */ + /* UFBX 1111 0x11 1100 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfb708000, 0xf3400000, t32_emulate_rd8rn16_noflags, + REGS(NOSPPC, 0, NOSPPC, 0, 0)), + + /* BFC 1111 0x11 0110 1111 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbff8000, 0xf36f0000, t32_emulate_rd8rn16_noflags, + REGS(0, 0, NOSPPC, 0, 0)), + + /* BFI 1111 0x11 0110 xxxx 0xxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfbf08000, 0xf3600000, t32_emulate_rd8rn16_noflags, + REGS(NOSPPCX, 0, NOSPPC, 0, 0)), + + DECODE_END +}; + +static const union decode_item t32_table_1111_0xxx___1[] = { + /* Branches and miscellaneous control */ + + /* YIELD 1111 0011 1010 xxxx 10x0 x000 0000 0001 */ + DECODE_OR (0xfff0d7ff, 0xf3a08001), + /* SEV 1111 0011 1010 xxxx 10x0 x000 0000 0100 */ + DECODE_EMULATE (0xfff0d7ff, 0xf3a08004, kprobe_emulate_none), + /* NOP 1111 0011 1010 xxxx 10x0 x000 0000 0000 */ + /* WFE 1111 0011 1010 xxxx 10x0 x000 0000 0010 */ + /* WFI 1111 0011 1010 xxxx 10x0 x000 0000 0011 */ + DECODE_SIMULATE (0xfff0d7fc, 0xf3a08000, kprobe_simulate_nop), + + /* MRS Rd, CPSR 1111 0011 1110 xxxx 10x0 xxxx xxxx xxxx */ + DECODE_SIMULATEX(0xfff0d000, 0xf3e08000, t32_simulate_mrs, + REGS(0, 0, NOSPPC, 0, 0)), + + /* + * Unsupported instructions + * 1111 0x11 1xxx xxxx 10x0 xxxx xxxx xxxx + * + * MSR 1111 0011 100x xxxx 10x0 xxxx xxxx xxxx + * DBG hint 1111 0011 1010 xxxx 10x0 x000 1111 xxxx + * Unallocated hints 1111 0011 1010 xxxx 10x0 x000 xxxx xxxx + * CPS 1111 0011 1010 xxxx 10x0 xxxx xxxx xxxx + * CLREX/DSB/DMB/ISB 1111 0011 1011 xxxx 10x0 xxxx xxxx xxxx + * BXJ 1111 0011 1100 xxxx 10x0 xxxx xxxx xxxx + * SUBS PC,LR,# 1111 0011 1101 xxxx 10x0 xxxx xxxx xxxx + * MRS Rd, SPSR 1111 0011 1111 xxxx 10x0 xxxx xxxx xxxx + * SMC 1111 0111 1111 xxxx 1000 xxxx xxxx xxxx + * UNDEFINED 1111 0111 1111 xxxx 1010 xxxx xxxx xxxx + * ??? 1111 0111 1xxx xxxx 1010 xxxx xxxx xxxx + */ + DECODE_REJECT (0xfb80d000, 0xf3808000), + + /* Bcc 1111 0xxx xxxx xxxx 10x0 xxxx xxxx xxxx */ + DECODE_CUSTOM (0xf800d000, 0xf0008000, t32_decode_cond_branch), + + /* BLX 1111 0xxx xxxx xxxx 11x0 xxxx xxxx xxx0 */ + DECODE_OR (0xf800d001, 0xf000c000), + /* B 1111 0xxx xxxx xxxx 10x1 xxxx xxxx xxxx */ + /* BL 1111 0xxx xxxx xxxx 11x1 xxxx xxxx xxxx */ + DECODE_SIMULATE (0xf8009000, 0xf0009000, t32_simulate_branch), + + DECODE_END +}; + +static const union decode_item t32_table_1111_100x_x0x1__1111[] = { + /* Memory hints */ + + /* PLD (literal) 1111 1000 x001 1111 1111 xxxx xxxx xxxx */ + /* PLI (literal) 1111 1001 x001 1111 1111 xxxx xxxx xxxx */ + DECODE_SIMULATE (0xfe7ff000, 0xf81ff000, kprobe_simulate_nop), + + /* PLD{W} (immediate) 1111 1000 10x1 xxxx 1111 xxxx xxxx xxxx */ + DECODE_OR (0xffd0f000, 0xf890f000), + /* PLD{W} (immediate) 1111 1000 00x1 xxxx 1111 1100 xxxx xxxx */ + DECODE_OR (0xffd0ff00, 0xf810fc00), + /* PLI (immediate) 1111 1001 1001 xxxx 1111 xxxx xxxx xxxx */ + DECODE_OR (0xfff0f000, 0xf990f000), + /* PLI (immediate) 1111 1001 0001 xxxx 1111 1100 xxxx xxxx */ + DECODE_SIMULATEX(0xfff0ff00, 0xf910fc00, kprobe_simulate_nop, + REGS(NOPCX, 0, 0, 0, 0)), + + /* PLD{W} (register) 1111 1000 00x1 xxxx 1111 0000 00xx xxxx */ + DECODE_OR (0xffd0ffc0, 0xf810f000), + /* PLI (register) 1111 1001 0001 xxxx 1111 0000 00xx xxxx */ + DECODE_SIMULATEX(0xfff0ffc0, 0xf910f000, kprobe_simulate_nop, + REGS(NOPCX, 0, 0, 0, NOSPPC)), + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1111_100x[] = { + /* Store/Load single data item */ + + /* ??? 1111 100x x11x xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfe600000, 0xf8600000), + + /* ??? 1111 1001 0101 xxxx xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xfff00000, 0xf9500000), + + /* ??? 1111 100x 0xxx xxxx xxxx 10x0 xxxx xxxx */ + DECODE_REJECT (0xfe800d00, 0xf8000800), + + /* STRBT 1111 1000 0000 xxxx xxxx 1110 xxxx xxxx */ + /* STRHT 1111 1000 0010 xxxx xxxx 1110 xxxx xxxx */ + /* STRT 1111 1000 0100 xxxx xxxx 1110 xxxx xxxx */ + /* LDRBT 1111 1000 0001 xxxx xxxx 1110 xxxx xxxx */ + /* LDRSBT 1111 1001 0001 xxxx xxxx 1110 xxxx xxxx */ + /* LDRHT 1111 1000 0011 xxxx xxxx 1110 xxxx xxxx */ + /* LDRSHT 1111 1001 0011 xxxx xxxx 1110 xxxx xxxx */ + /* LDRT 1111 1000 0101 xxxx xxxx 1110 xxxx xxxx */ + DECODE_REJECT (0xfe800f00, 0xf8000e00), + + /* STR{,B,H} Rn,[PC...] 1111 1000 xxx0 1111 xxxx xxxx xxxx xxxx */ + DECODE_REJECT (0xff1f0000, 0xf80f0000), + + /* STR{,B,H} PC,[Rn...] 1111 1000 xxx0 xxxx 1111 xxxx xxxx xxxx */ + DECODE_REJECT (0xff10f000, 0xf800f000), + + /* LDR (literal) 1111 1000 x101 1111 xxxx xxxx xxxx xxxx */ + DECODE_SIMULATEX(0xff7f0000, 0xf85f0000, t32_simulate_ldr_literal, + REGS(PC, ANY, 0, 0, 0)), + + /* STR (immediate) 1111 1000 0100 xxxx xxxx 1xxx xxxx xxxx */ + /* LDR (immediate) 1111 1000 0101 xxxx xxxx 1xxx xxxx xxxx */ + DECODE_OR (0xffe00800, 0xf8400800), + /* STR (immediate) 1111 1000 1100 xxxx xxxx xxxx xxxx xxxx */ + /* LDR (immediate) 1111 1000 1101 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xffe00000, 0xf8c00000, t32_emulate_ldrstr, + REGS(NOPCX, ANY, 0, 0, 0)), + + /* STR (register) 1111 1000 0100 xxxx xxxx 0000 00xx xxxx */ + /* LDR (register) 1111 1000 0101 xxxx xxxx 0000 00xx xxxx */ + DECODE_EMULATEX (0xffe00fc0, 0xf8400000, t32_emulate_ldrstr, + REGS(NOPCX, ANY, 0, 0, NOSPPC)), + + /* LDRB (literal) 1111 1000 x001 1111 xxxx xxxx xxxx xxxx */ + /* LDRSB (literal) 1111 1001 x001 1111 xxxx xxxx xxxx xxxx */ + /* LDRH (literal) 1111 1000 x011 1111 xxxx xxxx xxxx xxxx */ + /* LDRSH (literal) 1111 1001 x011 1111 xxxx xxxx xxxx xxxx */ + DECODE_SIMULATEX(0xfe5f0000, 0xf81f0000, t32_simulate_ldr_literal, + REGS(PC, NOSPPCX, 0, 0, 0)), + + /* STRB (immediate) 1111 1000 0000 xxxx xxxx 1xxx xxxx xxxx */ + /* STRH (immediate) 1111 1000 0010 xxxx xxxx 1xxx xxxx xxxx */ + /* LDRB (immediate) 1111 1000 0001 xxxx xxxx 1xxx xxxx xxxx */ + /* LDRSB (immediate) 1111 1001 0001 xxxx xxxx 1xxx xxxx xxxx */ + /* LDRH (immediate) 1111 1000 0011 xxxx xxxx 1xxx xxxx xxxx */ + /* LDRSH (immediate) 1111 1001 0011 xxxx xxxx 1xxx xxxx xxxx */ + DECODE_OR (0xfec00800, 0xf8000800), + /* STRB (immediate) 1111 1000 1000 xxxx xxxx xxxx xxxx xxxx */ + /* STRH (immediate) 1111 1000 1010 xxxx xxxx xxxx xxxx xxxx */ + /* LDRB (immediate) 1111 1000 1001 xxxx xxxx xxxx xxxx xxxx */ + /* LDRSB (immediate) 1111 1001 1001 xxxx xxxx xxxx xxxx xxxx */ + /* LDRH (immediate) 1111 1000 1011 xxxx xxxx xxxx xxxx xxxx */ + /* LDRSH (immediate) 1111 1001 1011 xxxx xxxx xxxx xxxx xxxx */ + DECODE_EMULATEX (0xfec00000, 0xf8800000, t32_emulate_ldrstr, + REGS(NOPCX, NOSPPCX, 0, 0, 0)), + + /* STRB (register) 1111 1000 0000 xxxx xxxx 0000 00xx xxxx */ + /* STRH (register) 1111 1000 0010 xxxx xxxx 0000 00xx xxxx */ + /* LDRB (register) 1111 1000 0001 xxxx xxxx 0000 00xx xxxx */ + /* LDRSB (register) 1111 1001 0001 xxxx xxxx 0000 00xx xxxx */ + /* LDRH (register) 1111 1000 0011 xxxx xxxx 0000 00xx xxxx */ + /* LDRSH (register) 1111 1001 0011 xxxx xxxx 0000 00xx xxxx */ + DECODE_EMULATEX (0xfe800fc0, 0xf8000000, t32_emulate_ldrstr, + REGS(NOPCX, NOSPPCX, 0, 0, NOSPPC)), + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1111_1010___1111[] = { + /* Data-processing (register) */ + + /* ??? 1111 1010 011x xxxx 1111 xxxx 1xxx xxxx */ + DECODE_REJECT (0xffe0f080, 0xfa60f080), + + /* SXTH 1111 1010 0000 1111 1111 xxxx 1xxx xxxx */ + /* UXTH 1111 1010 0001 1111 1111 xxxx 1xxx xxxx */ + /* SXTB16 1111 1010 0010 1111 1111 xxxx 1xxx xxxx */ + /* UXTB16 1111 1010 0011 1111 1111 xxxx 1xxx xxxx */ + /* SXTB 1111 1010 0100 1111 1111 xxxx 1xxx xxxx */ + /* UXTB 1111 1010 0101 1111 1111 xxxx 1xxx xxxx */ + DECODE_EMULATEX (0xff8ff080, 0xfa0ff080, t32_emulate_rd8rn16rm0_rwflags, + REGS(0, 0, NOSPPC, 0, NOSPPC)), + + + /* ??? 1111 1010 1xxx xxxx 1111 xxxx 0x11 xxxx */ + DECODE_REJECT (0xff80f0b0, 0xfa80f030), + /* ??? 1111 1010 1x11 xxxx 1111 xxxx 0xxx xxxx */ + DECODE_REJECT (0xffb0f080, 0xfab0f000), + + /* SADD16 1111 1010 1001 xxxx 1111 xxxx 0000 xxxx */ + /* SASX 1111 1010 1010 xxxx 1111 xxxx 0000 xxxx */ + /* SSAX 1111 1010 1110 xxxx 1111 xxxx 0000 xxxx */ + /* SSUB16 1111 1010 1101 xxxx 1111 xxxx 0000 xxxx */ + /* SADD8 1111 1010 1000 xxxx 1111 xxxx 0000 xxxx */ + /* SSUB8 1111 1010 1100 xxxx 1111 xxxx 0000 xxxx */ + + /* QADD16 1111 1010 1001 xxxx 1111 xxxx 0001 xxxx */ + /* QASX 1111 1010 1010 xxxx 1111 xxxx 0001 xxxx */ + /* QSAX 1111 1010 1110 xxxx 1111 xxxx 0001 xxxx */ + /* QSUB16 1111 1010 1101 xxxx 1111 xxxx 0001 xxxx */ + /* QADD8 1111 1010 1000 xxxx 1111 xxxx 0001 xxxx */ + /* QSUB8 1111 1010 1100 xxxx 1111 xxxx 0001 xxxx */ + + /* SHADD16 1111 1010 1001 xxxx 1111 xxxx 0010 xxxx */ + /* SHASX 1111 1010 1010 xxxx 1111 xxxx 0010 xxxx */ + /* SHSAX 1111 1010 1110 xxxx 1111 xxxx 0010 xxxx */ + /* SHSUB16 1111 1010 1101 xxxx 1111 xxxx 0010 xxxx */ + /* SHADD8 1111 1010 1000 xxxx 1111 xxxx 0010 xxxx */ + /* SHSUB8 1111 1010 1100 xxxx 1111 xxxx 0010 xxxx */ + + /* UADD16 1111 1010 1001 xxxx 1111 xxxx 0100 xxxx */ + /* UASX 1111 1010 1010 xxxx 1111 xxxx 0100 xxxx */ + /* USAX 1111 1010 1110 xxxx 1111 xxxx 0100 xxxx */ + /* USUB16 1111 1010 1101 xxxx 1111 xxxx 0100 xxxx */ + /* UADD8 1111 1010 1000 xxxx 1111 xxxx 0100 xxxx */ + /* USUB8 1111 1010 1100 xxxx 1111 xxxx 0100 xxxx */ + + /* UQADD16 1111 1010 1001 xxxx 1111 xxxx 0101 xxxx */ + /* UQASX 1111 1010 1010 xxxx 1111 xxxx 0101 xxxx */ + /* UQSAX 1111 1010 1110 xxxx 1111 xxxx 0101 xxxx */ + /* UQSUB16 1111 1010 1101 xxxx 1111 xxxx 0101 xxxx */ + /* UQADD8 1111 1010 1000 xxxx 1111 xxxx 0101 xxxx */ + /* UQSUB8 1111 1010 1100 xxxx 1111 xxxx 0101 xxxx */ + + /* UHADD16 1111 1010 1001 xxxx 1111 xxxx 0110 xxxx */ + /* UHASX 1111 1010 1010 xxxx 1111 xxxx 0110 xxxx */ + /* UHSAX 1111 1010 1110 xxxx 1111 xxxx 0110 xxxx */ + /* UHSUB16 1111 1010 1101 xxxx 1111 xxxx 0110 xxxx */ + /* UHADD8 1111 1010 1000 xxxx 1111 xxxx 0110 xxxx */ + /* UHSUB8 1111 1010 1100 xxxx 1111 xxxx 0110 xxxx */ + DECODE_OR (0xff80f080, 0xfa80f000), + + /* SXTAH 1111 1010 0000 xxxx 1111 xxxx 1xxx xxxx */ + /* UXTAH 1111 1010 0001 xxxx 1111 xxxx 1xxx xxxx */ + /* SXTAB16 1111 1010 0010 xxxx 1111 xxxx 1xxx xxxx */ + /* UXTAB16 1111 1010 0011 xxxx 1111 xxxx 1xxx xxxx */ + /* SXTAB 1111 1010 0100 xxxx 1111 xxxx 1xxx xxxx */ + /* UXTAB 1111 1010 0101 xxxx 1111 xxxx 1xxx xxxx */ + DECODE_OR (0xff80f080, 0xfa00f080), + + /* QADD 1111 1010 1000 xxxx 1111 xxxx 1000 xxxx */ + /* QDADD 1111 1010 1000 xxxx 1111 xxxx 1001 xxxx */ + /* QSUB 1111 1010 1000 xxxx 1111 xxxx 1010 xxxx */ + /* QDSUB 1111 1010 1000 xxxx 1111 xxxx 1011 xxxx */ + DECODE_OR (0xfff0f0c0, 0xfa80f080), + + /* SEL 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ + DECODE_OR (0xfff0f0f0, 0xfaa0f080), + + /* LSL 1111 1010 000x xxxx 1111 xxxx 0000 xxxx */ + /* LSR 1111 1010 001x xxxx 1111 xxxx 0000 xxxx */ + /* ASR 1111 1010 010x xxxx 1111 xxxx 0000 xxxx */ + /* ROR 1111 1010 011x xxxx 1111 xxxx 0000 xxxx */ + DECODE_EMULATEX (0xff80f0f0, 0xfa00f000, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), + + /* CLZ 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ + DECODE_OR (0xfff0f0f0, 0xfab0f080), + + /* REV 1111 1010 1001 xxxx 1111 xxxx 1000 xxxx */ + /* REV16 1111 1010 1001 xxxx 1111 xxxx 1001 xxxx */ + /* RBIT 1111 1010 1001 xxxx 1111 xxxx 1010 xxxx */ + /* REVSH 1111 1010 1001 xxxx 1111 xxxx 1011 xxxx */ + DECODE_EMULATEX (0xfff0f0c0, 0xfa90f080, t32_emulate_rd8rn16_noflags, + REGS(NOSPPC, 0, NOSPPC, 0, SAMEAS16)), + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1111_1011_0[] = { + /* Multiply, multiply accumulate, and absolute difference */ + + /* ??? 1111 1011 0000 xxxx 1111 xxxx 0001 xxxx */ + DECODE_REJECT (0xfff0f0f0, 0xfb00f010), + /* ??? 1111 1011 0111 xxxx 1111 xxxx 0001 xxxx */ + DECODE_REJECT (0xfff0f0f0, 0xfb70f010), + + /* SMULxy 1111 1011 0001 xxxx 1111 xxxx 00xx xxxx */ + DECODE_OR (0xfff0f0c0, 0xfb10f000), + /* MUL 1111 1011 0000 xxxx 1111 xxxx 0000 xxxx */ + /* SMUAD{X} 1111 1011 0010 xxxx 1111 xxxx 000x xxxx */ + /* SMULWy 1111 1011 0011 xxxx 1111 xxxx 000x xxxx */ + /* SMUSD{X} 1111 1011 0100 xxxx 1111 xxxx 000x xxxx */ + /* SMMUL{R} 1111 1011 0101 xxxx 1111 xxxx 000x xxxx */ + /* USAD8 1111 1011 0111 xxxx 1111 xxxx 0000 xxxx */ + DECODE_EMULATEX (0xff80f0e0, 0xfb00f000, t32_emulate_rd8rn16rm0_rwflags, + REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), + + /* ??? 1111 1011 0111 xxxx xxxx xxxx 0001 xxxx */ + DECODE_REJECT (0xfff000f0, 0xfb700010), + + /* SMLAxy 1111 1011 0001 xxxx xxxx xxxx 00xx xxxx */ + DECODE_OR (0xfff000c0, 0xfb100000), + /* MLA 1111 1011 0000 xxxx xxxx xxxx 0000 xxxx */ + /* MLS 1111 1011 0000 xxxx xxxx xxxx 0001 xxxx */ + /* SMLAD{X} 1111 1011 0010 xxxx xxxx xxxx 000x xxxx */ + /* SMLAWy 1111 1011 0011 xxxx xxxx xxxx 000x xxxx */ + /* SMLSD{X} 1111 1011 0100 xxxx xxxx xxxx 000x xxxx */ + /* SMMLA{R} 1111 1011 0101 xxxx xxxx xxxx 000x xxxx */ + /* SMMLS{R} 1111 1011 0110 xxxx xxxx xxxx 000x xxxx */ + /* USADA8 1111 1011 0111 xxxx xxxx xxxx 0000 xxxx */ + DECODE_EMULATEX (0xff8000c0, 0xfb000000, t32_emulate_rd8rn16rm0ra12_noflags, + REGS(NOSPPC, NOSPPCX, NOSPPC, 0, NOSPPC)), + + /* Other unallocated instructions... */ + DECODE_END +}; + +static const union decode_item t32_table_1111_1011_1[] = { + /* Long multiply, long multiply accumulate, and divide */ + + /* UMAAL 1111 1011 1110 xxxx xxxx xxxx 0110 xxxx */ + DECODE_OR (0xfff000f0, 0xfbe00060), + /* SMLALxy 1111 1011 1100 xxxx xxxx xxxx 10xx xxxx */ + DECODE_OR (0xfff000c0, 0xfbc00080), + /* SMLALD{X} 1111 1011 1100 xxxx xxxx xxxx 110x xxxx */ + /* SMLSLD{X} 1111 1011 1101 xxxx xxxx xxxx 110x xxxx */ + DECODE_OR (0xffe000e0, 0xfbc000c0), + /* SMULL 1111 1011 1000 xxxx xxxx xxxx 0000 xxxx */ + /* UMULL 1111 1011 1010 xxxx xxxx xxxx 0000 xxxx */ + /* SMLAL 1111 1011 1100 xxxx xxxx xxxx 0000 xxxx */ + /* UMLAL 1111 1011 1110 xxxx xxxx xxxx 0000 xxxx */ + DECODE_EMULATEX (0xff9000f0, 0xfb800000, t32_emulate_rdlo12rdhi8rn16rm0_noflags, + REGS(NOSPPC, NOSPPC, NOSPPC, 0, NOSPPC)), + + /* SDIV 1111 1011 1001 xxxx xxxx xxxx 1111 xxxx */ + /* UDIV 1111 1011 1011 xxxx xxxx xxxx 1111 xxxx */ + /* Other unallocated instructions... */ + DECODE_END +}; + +const union decode_item kprobe_decode_thumb32_table[] = { + + /* + * Load/store multiple instructions + * 1110 100x x0xx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe400000, 0xe8000000, t32_table_1110_100x_x0xx), + + /* + * Load/store dual, load/store exclusive, table branch + * 1110 100x x1xx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe400000, 0xe8400000, t32_table_1110_100x_x1xx), + + /* + * Data-processing (shifted register) + * 1110 101x xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe000000, 0xea000000, t32_table_1110_101x), + + /* + * Coprocessor instructions + * 1110 11xx xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_REJECT (0xfc000000, 0xec000000), + + /* + * Data-processing (modified immediate) + * 1111 0x0x xxxx xxxx 0xxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfa008000, 0xf0000000, t32_table_1111_0x0x___0), + + /* + * Data-processing (plain binary immediate) + * 1111 0x1x xxxx xxxx 0xxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfa008000, 0xf2000000, t32_table_1111_0x1x___0), + + /* + * Branches and miscellaneous control + * 1111 0xxx xxxx xxxx 1xxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xf8008000, 0xf0008000, t32_table_1111_0xxx___1), + + /* + * Advanced SIMD element or structure load/store instructions + * 1111 1001 xxx0 xxxx xxxx xxxx xxxx xxxx + */ + DECODE_REJECT (0xff100000, 0xf9000000), + + /* + * Memory hints + * 1111 100x x0x1 xxxx 1111 xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe50f000, 0xf810f000, t32_table_1111_100x_x0x1__1111), + + /* + * Store single data item + * 1111 1000 xxx0 xxxx xxxx xxxx xxxx xxxx + * Load single data items + * 1111 100x xxx1 xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xfe000000, 0xf8000000, t32_table_1111_100x), + + /* + * Data-processing (register) + * 1111 1010 xxxx xxxx 1111 xxxx xxxx xxxx + */ + DECODE_TABLE (0xff00f000, 0xfa00f000, t32_table_1111_1010___1111), + + /* + * Multiply, multiply accumulate, and absolute difference + * 1111 1011 0xxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xff800000, 0xfb000000, t32_table_1111_1011_0), + + /* + * Long multiply, long multiply accumulate, and divide + * 1111 1011 1xxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_TABLE (0xff800000, 0xfb800000, t32_table_1111_1011_1), + + /* + * Coprocessor instructions + * 1111 11xx xxxx xxxx xxxx xxxx xxxx xxxx + */ + DECODE_END +}; +#ifdef CONFIG_ARM_KPROBES_TEST_MODULE +EXPORT_SYMBOL_GPL(kprobe_decode_thumb32_table); +#endif + +static const union decode_item t16_table_1011[] = { + /* Miscellaneous 16-bit instructions */ + + /* ADD (SP plus immediate) 1011 0000 0xxx xxxx */ + /* SUB (SP minus immediate) 1011 0000 1xxx xxxx */ + DECODE_SIMULATE (0xff00, 0xb000, t16_simulate_add_sp_imm), + + /* CBZ 1011 00x1 xxxx xxxx */ + /* CBNZ 1011 10x1 xxxx xxxx */ + DECODE_SIMULATE (0xf500, 0xb100, t16_simulate_cbz), + + /* SXTH 1011 0010 00xx xxxx */ + /* SXTB 1011 0010 01xx xxxx */ + /* UXTH 1011 0010 10xx xxxx */ + /* UXTB 1011 0010 11xx xxxx */ + /* REV 1011 1010 00xx xxxx */ + /* REV16 1011 1010 01xx xxxx */ + /* ??? 1011 1010 10xx xxxx */ + /* REVSH 1011 1010 11xx xxxx */ + DECODE_REJECT (0xffc0, 0xba80), + DECODE_EMULATE (0xf500, 0xb000, t16_emulate_loregs_rwflags), + + /* PUSH 1011 010x xxxx xxxx */ + DECODE_CUSTOM (0xfe00, 0xb400, t16_decode_push), + /* POP 1011 110x xxxx xxxx */ + DECODE_CUSTOM (0xfe00, 0xbc00, t16_decode_pop), + + /* + * If-Then, and hints + * 1011 1111 xxxx xxxx + */ + + /* YIELD 1011 1111 0001 0000 */ + DECODE_OR (0xffff, 0xbf10), + /* SEV 1011 1111 0100 0000 */ + DECODE_EMULATE (0xffff, 0xbf40, kprobe_emulate_none), + /* NOP 1011 1111 0000 0000 */ + /* WFE 1011 1111 0010 0000 */ + /* WFI 1011 1111 0011 0000 */ + DECODE_SIMULATE (0xffcf, 0xbf00, kprobe_simulate_nop), + /* Unassigned hints 1011 1111 xxxx 0000 */ + DECODE_REJECT (0xff0f, 0xbf00), + /* IT 1011 1111 xxxx xxxx */ + DECODE_CUSTOM (0xff00, 0xbf00, t16_decode_it), + + /* SETEND 1011 0110 010x xxxx */ + /* CPS 1011 0110 011x xxxx */ + /* BKPT 1011 1110 xxxx xxxx */ + /* And unallocated instructions... */ + DECODE_END +}; + +const union decode_item kprobe_decode_thumb16_table[] = { + + /* + * Shift (immediate), add, subtract, move, and compare + * 00xx xxxx xxxx xxxx + */ + + /* CMP (immediate) 0010 1xxx xxxx xxxx */ + DECODE_EMULATE (0xf800, 0x2800, t16_emulate_loregs_rwflags), + + /* ADD (register) 0001 100x xxxx xxxx */ + /* SUB (register) 0001 101x xxxx xxxx */ + /* LSL (immediate) 0000 0xxx xxxx xxxx */ + /* LSR (immediate) 0000 1xxx xxxx xxxx */ + /* ASR (immediate) 0001 0xxx xxxx xxxx */ + /* ADD (immediate, Thumb) 0001 110x xxxx xxxx */ + /* SUB (immediate, Thumb) 0001 111x xxxx xxxx */ + /* MOV (immediate) 0010 0xxx xxxx xxxx */ + /* ADD (immediate, Thumb) 0011 0xxx xxxx xxxx */ + /* SUB (immediate, Thumb) 0011 1xxx xxxx xxxx */ + DECODE_EMULATE (0xc000, 0x0000, t16_emulate_loregs_noitrwflags), + + /* + * 16-bit Thumb data-processing instructions + * 0100 00xx xxxx xxxx + */ + + /* TST (register) 0100 0010 00xx xxxx */ + DECODE_EMULATE (0xffc0, 0x4200, t16_emulate_loregs_rwflags), + /* CMP (register) 0100 0010 10xx xxxx */ + /* CMN (register) 0100 0010 11xx xxxx */ + DECODE_EMULATE (0xff80, 0x4280, t16_emulate_loregs_rwflags), + /* AND (register) 0100 0000 00xx xxxx */ + /* EOR (register) 0100 0000 01xx xxxx */ + /* LSL (register) 0100 0000 10xx xxxx */ + /* LSR (register) 0100 0000 11xx xxxx */ + /* ASR (register) 0100 0001 00xx xxxx */ + /* ADC (register) 0100 0001 01xx xxxx */ + /* SBC (register) 0100 0001 10xx xxxx */ + /* ROR (register) 0100 0001 11xx xxxx */ + /* RSB (immediate) 0100 0010 01xx xxxx */ + /* ORR (register) 0100 0011 00xx xxxx */ + /* MUL 0100 0011 00xx xxxx */ + /* BIC (register) 0100 0011 10xx xxxx */ + /* MVN (register) 0100 0011 10xx xxxx */ + DECODE_EMULATE (0xfc00, 0x4000, t16_emulate_loregs_noitrwflags), + + /* + * Special data instructions and branch and exchange + * 0100 01xx xxxx xxxx + */ + + /* BLX pc 0100 0111 1111 1xxx */ + DECODE_REJECT (0xfff8, 0x47f8), + + /* BX (register) 0100 0111 0xxx xxxx */ + /* BLX (register) 0100 0111 1xxx xxxx */ + DECODE_SIMULATE (0xff00, 0x4700, t16_simulate_bxblx), + + /* ADD pc, pc 0100 0100 1111 1111 */ + DECODE_REJECT (0xffff, 0x44ff), + + /* ADD (register) 0100 0100 xxxx xxxx */ + /* CMP (register) 0100 0101 xxxx xxxx */ + /* MOV (register) 0100 0110 xxxx xxxx */ + DECODE_CUSTOM (0xfc00, 0x4400, t16_decode_hiregs), + + /* + * Load from Literal Pool + * LDR (literal) 0100 1xxx xxxx xxxx + */ + DECODE_SIMULATE (0xf800, 0x4800, t16_simulate_ldr_literal), + + /* + * 16-bit Thumb Load/store instructions + * 0101 xxxx xxxx xxxx + * 011x xxxx xxxx xxxx + * 100x xxxx xxxx xxxx + */ + + /* STR (register) 0101 000x xxxx xxxx */ + /* STRH (register) 0101 001x xxxx xxxx */ + /* STRB (register) 0101 010x xxxx xxxx */ + /* LDRSB (register) 0101 011x xxxx xxxx */ + /* LDR (register) 0101 100x xxxx xxxx */ + /* LDRH (register) 0101 101x xxxx xxxx */ + /* LDRB (register) 0101 110x xxxx xxxx */ + /* LDRSH (register) 0101 111x xxxx xxxx */ + /* STR (immediate, Thumb) 0110 0xxx xxxx xxxx */ + /* LDR (immediate, Thumb) 0110 1xxx xxxx xxxx */ + /* STRB (immediate, Thumb) 0111 0xxx xxxx xxxx */ + /* LDRB (immediate, Thumb) 0111 1xxx xxxx xxxx */ + DECODE_EMULATE (0xc000, 0x4000, t16_emulate_loregs_rwflags), + /* STRH (immediate, Thumb) 1000 0xxx xxxx xxxx */ + /* LDRH (immediate, Thumb) 1000 1xxx xxxx xxxx */ + DECODE_EMULATE (0xf000, 0x8000, t16_emulate_loregs_rwflags), + /* STR (immediate, Thumb) 1001 0xxx xxxx xxxx */ + /* LDR (immediate, Thumb) 1001 1xxx xxxx xxxx */ + DECODE_SIMULATE (0xf000, 0x9000, t16_simulate_ldrstr_sp_relative), + + /* + * Generate PC-/SP-relative address + * ADR (literal) 1010 0xxx xxxx xxxx + * ADD (SP plus immediate) 1010 1xxx xxxx xxxx + */ + DECODE_SIMULATE (0xf000, 0xa000, t16_simulate_reladr), + + /* + * Miscellaneous 16-bit instructions + * 1011 xxxx xxxx xxxx + */ + DECODE_TABLE (0xf000, 0xb000, t16_table_1011), + + /* STM 1100 0xxx xxxx xxxx */ + /* LDM 1100 1xxx xxxx xxxx */ + DECODE_EMULATE (0xf000, 0xc000, t16_emulate_loregs_rwflags), + + /* + * Conditional branch, and Supervisor Call + */ + + /* Permanently UNDEFINED 1101 1110 xxxx xxxx */ + /* SVC 1101 1111 xxxx xxxx */ + DECODE_REJECT (0xfe00, 0xde00), + + /* Conditional branch 1101 xxxx xxxx xxxx */ + DECODE_CUSTOM (0xf000, 0xd000, t16_decode_cond_branch), + + /* + * Unconditional branch + * B 1110 0xxx xxxx xxxx + */ + DECODE_SIMULATE (0xf800, 0xe000, t16_simulate_branch), + + DECODE_END +}; +#ifdef CONFIG_ARM_KPROBES_TEST_MODULE +EXPORT_SYMBOL_GPL(kprobe_decode_thumb16_table); +#endif + +static unsigned long __kprobes thumb_check_cc(unsigned long cpsr) +{ + if (unlikely(in_it_block(cpsr))) + return kprobe_condition_checks[current_cond(cpsr)](cpsr); + return true; +} + +static void __kprobes thumb16_singlestep(struct kprobe *p, struct pt_regs *regs) +{ + regs->ARM_pc += 2; + p->ainsn.insn_handler(p, regs); + regs->ARM_cpsr = it_advance(regs->ARM_cpsr); +} + +static void __kprobes thumb32_singlestep(struct kprobe *p, struct pt_regs *regs) +{ + regs->ARM_pc += 4; + p->ainsn.insn_handler(p, regs); + regs->ARM_cpsr = it_advance(regs->ARM_cpsr); +} + +enum kprobe_insn __kprobes +thumb16_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + asi->insn_singlestep = thumb16_singlestep; + asi->insn_check_cc = thumb_check_cc; + return kprobe_decode_insn(insn, asi, kprobe_decode_thumb16_table, true); +} + +enum kprobe_insn __kprobes +thumb32_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + asi->insn_singlestep = thumb32_singlestep; + asi->insn_check_cc = thumb_check_cc; + return kprobe_decode_insn(insn, asi, kprobe_decode_thumb32_table, true); +} diff --git a/arch/arm/kernel/probes-thumb.h b/arch/arm/kernel/probes-thumb.h new file mode 100644 index 000000000000..98709c40b659 --- /dev/null +++ b/arch/arm/kernel/probes-thumb.h @@ -0,0 +1,81 @@ +/* + * arch/arm/kernel/probes-thumb.h + * + * Copyright 2013 Linaro Ltd. + * Written by: David A. Long + * + * The code contained herein is licensed under the GNU General Public + * License. You may obtain a copy of the GNU General Public License + * Version 2 or later at the following locations: + * + * http://www.opensource.org/licenses/gpl-license.html + * http://www.gnu.org/copyleft/gpl.html + */ + +#ifndef _ARM_KERNEL_PROBES_THUMB_H +#define _ARM_KERNEL_PROBES_THUMB_H + +/* + * True if current instruction is in an IT block. + */ +#define in_it_block(cpsr) ((cpsr & 0x06000c00) != 0x00000000) + +/* + * Return the condition code to check for the currently executing instruction. + * This is in ITSTATE<7:4> which is in CPSR<15:12> but is only valid if + * in_it_block returns true. + */ +#define current_cond(cpsr) ((cpsr >> 12) & 0xf) + +void __kprobes t16_simulate_bxblx(struct kprobe *p, struct pt_regs *regs); +void __kprobes t16_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs); +void __kprobes t16_simulate_ldrstr_sp_relative(struct kprobe *p, + struct pt_regs *regs); +void __kprobes t16_simulate_reladr(struct kprobe *p, struct pt_regs *regs); +void __kprobes t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs); +void __kprobes t16_simulate_cbz(struct kprobe *p, struct pt_regs *regs); +void __kprobes t16_simulate_it(struct kprobe *p, struct pt_regs *regs); +void __kprobes t16_singlestep_it(struct kprobe *p, struct pt_regs *regs); +enum kprobe_insn __kprobes t16_decode_it(kprobe_opcode_t insn, + struct arch_specific_insn *asi); +void __kprobes t16_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs); +enum kprobe_insn __kprobes t16_decode_cond_branch(kprobe_opcode_t insn, + struct arch_specific_insn *asi); +void __kprobes t16_simulate_branch(struct kprobe *p, struct pt_regs *regs); +void __kprobes t16_emulate_loregs_rwflags(struct kprobe *p, + struct pt_regs *regs); +void __kprobes t16_emulate_loregs_noitrwflags(struct kprobe *p, + struct pt_regs *regs); +void __kprobes t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs); +enum kprobe_insn __kprobes t16_decode_hiregs(kprobe_opcode_t insn, + struct arch_specific_insn *asi); +void __kprobes t16_emulate_push(struct kprobe *p, struct pt_regs *regs); +enum kprobe_insn __kprobes t16_decode_push(kprobe_opcode_t insn, + struct arch_specific_insn *asi); +void __kprobes t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs); +void __kprobes t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs); +enum kprobe_insn __kprobes t16_decode_pop(kprobe_opcode_t insn, + struct arch_specific_insn *asi); + +void __kprobes t32_simulate_table_branch(struct kprobe *p, + struct pt_regs *regs); +void __kprobes t32_simulate_mrs(struct kprobe *p, struct pt_regs *regs); +void __kprobes t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs); +enum kprobe_insn __kprobes t32_decode_cond_branch(kprobe_opcode_t insn, + struct arch_specific_insn *asi); +void __kprobes t32_simulate_branch(struct kprobe *p, struct pt_regs *regs); +void __kprobes t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs); +enum kprobe_insn __kprobes t32_decode_ldmstm(kprobe_opcode_t insn, + struct arch_specific_insn *asi); +void __kprobes t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs); +void __kprobes t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs); +void __kprobes t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, + struct pt_regs *regs); +void __kprobes t32_emulate_rd8pc16_noflags(struct kprobe *p, + struct pt_regs *regs); +void __kprobes t32_emulate_rd8rn16_noflags(struct kprobe *p, + struct pt_regs *regs); +void __kprobes t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, + struct pt_regs *regs); + +#endif From 3e6cd394bb10c2d65322e5f5d2ff0a9074d903a1 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Thu, 6 Mar 2014 18:06:43 -0500 Subject: [PATCH 52/89] ARM: use a function table for determining instruction interpreter action Make the instruction interpreter call back to semantic action functions through a function pointer array provided by the invoker. The interpreter decodes the instructions into groups and uses the group number to index into the supplied array. kprobes and uprobes code will each supply their own array of functions. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/kernel/kprobes-arm.c | 57 ++++++++++-- arch/arm/kernel/kprobes-common.c | 3 +- arch/arm/kernel/kprobes-thumb.c | 149 ++++++++++++++++++++++--------- arch/arm/kernel/kprobes.c | 9 +- arch/arm/kernel/kprobes.h | 15 +++- arch/arm/kernel/probes-arm.c | 114 +++++++++++------------ arch/arm/kernel/probes-arm.h | 52 ++++++++--- arch/arm/kernel/probes-thumb.c | 145 +++++++++++++++--------------- arch/arm/kernel/probes-thumb.h | 104 +++++++++++---------- arch/arm/kernel/probes.c | 9 +- arch/arm/kernel/probes.h | 55 ++++++++---- 11 files changed, 441 insertions(+), 271 deletions(-) diff --git a/arch/arm/kernel/kprobes-arm.c b/arch/arm/kernel/kprobes-arm.c index a1d0a8f00f9e..8ebd84c48867 100644 --- a/arch/arm/kernel/kprobes-arm.c +++ b/arch/arm/kernel/kprobes-arm.c @@ -73,7 +73,7 @@ #endif -void __kprobes +static void __kprobes emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -102,7 +102,7 @@ emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) regs->uregs[rn] = rnv; } -void __kprobes +static void __kprobes emulate_ldr(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -132,7 +132,7 @@ emulate_ldr(struct kprobe *p, struct pt_regs *regs) regs->uregs[rn] = rnv; } -void __kprobes +static void __kprobes emulate_str(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -159,7 +159,7 @@ emulate_str(struct kprobe *p, struct pt_regs *regs) regs->uregs[rn] = rnv; } -void __kprobes +static void __kprobes emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -194,7 +194,7 @@ emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -void __kprobes +static void __kprobes emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -221,7 +221,7 @@ emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -void __kprobes +static void __kprobes emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -250,7 +250,7 @@ emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -void __kprobes +static void __kprobes emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -270,7 +270,7 @@ emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->uregs[rd] = rdv; } -void __kprobes +static void __kprobes emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -299,3 +299,44 @@ emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) regs->uregs[rdhi] = rdhiv; regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } + +const union decode_action kprobes_arm_actions[NUM_PROBES_ARM_ACTIONS] = { + [PROBES_EMULATE_NONE] = {.handler = kprobe_emulate_none}, + [PROBES_SIMULATE_NOP] = {.handler = kprobe_simulate_nop}, + [PROBES_PRELOAD_IMM] = {.handler = kprobe_simulate_nop}, + [PROBES_PRELOAD_REG] = {.handler = kprobe_simulate_nop}, + [PROBES_BRANCH_IMM] = {.handler = simulate_blx1}, + [PROBES_MRS] = {.handler = simulate_mrs}, + [PROBES_BRANCH_REG] = {.handler = simulate_blx2bx}, + [PROBES_CLZ] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_SATURATING_ARITHMETIC] = { + .handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_MUL1] = {.handler = emulate_rdlo12rdhi16rn0rm8_rwflags_nopc}, + [PROBES_MUL2] = {.handler = emulate_rd16rn12rm0rs8_rwflags_nopc}, + [PROBES_SWP] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_LDRSTRD] = {.handler = emulate_ldrdstrd}, + [PROBES_LOAD_EXTRA] = {.handler = emulate_ldr}, + [PROBES_LOAD] = {.handler = emulate_ldr}, + [PROBES_STORE_EXTRA] = {.handler = emulate_str}, + [PROBES_STORE] = {.handler = emulate_str}, + [PROBES_MOV_IP_SP] = {.handler = simulate_mov_ipsp}, + [PROBES_DATA_PROCESSING_REG] = { + .handler = emulate_rd12rn16rm0rs8_rwflags}, + [PROBES_DATA_PROCESSING_IMM] = { + .handler = emulate_rd12rn16rm0rs8_rwflags}, + [PROBES_MOV_HALFWORD] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_SEV] = {.handler = kprobe_emulate_none}, + [PROBES_WFE] = {.handler = kprobe_simulate_nop}, + [PROBES_SATURATE] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_REV] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_MMI] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_PACK] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_EXTEND] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_EXTEND_ADD] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, + [PROBES_MUL_ADD_LONG] = { + .handler = emulate_rdlo12rdhi16rn0rm8_rwflags_nopc}, + [PROBES_MUL_ADD] = {.handler = emulate_rd16rn12rm0rs8_rwflags_nopc}, + [PROBES_BITFIELD] = {.handler = emulate_rd12rm0_noflags_nopc}, + [PROBES_BRANCH] = {.handler = simulate_bbl}, + [PROBES_LDMSTM] = {.decoder = kprobe_decode_ldmstm} +}; diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index f02c038059c3..029b79c6face 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -112,7 +112,8 @@ emulate_ldm_r3_15(struct kprobe *p, struct pt_regs *regs) } enum kprobe_insn __kprobes -kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) +kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *h) { kprobe_insn_handler_t *handler = 0; unsigned reglist = insn & 0xffff; diff --git a/arch/arm/kernel/kprobes-thumb.c b/arch/arm/kernel/kprobes-thumb.c index 977f21723a9c..d83f6092920a 100644 --- a/arch/arm/kernel/kprobes-thumb.c +++ b/arch/arm/kernel/kprobes-thumb.c @@ -16,6 +16,10 @@ #include "kprobes.h" #include "probes-thumb.h" +/* These emulation encodings are functionally equivalent... */ +#define t32_emulate_rd8rn16rm0ra12_noflags \ + t32_emulate_rdlo12rdhi8rn16rm0_noflags + /* * Return the PC value for a probe in thumb code. * This is the address of the probed instruction plus 4. @@ -29,7 +33,7 @@ static inline unsigned long __kprobes thumb_probe_pc(struct kprobe *p) /* t32 thumb actions */ -void __kprobes +static void __kprobes t32_simulate_table_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -49,7 +53,7 @@ t32_simulate_table_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + 2 * halfwords; } -void __kprobes +static void __kprobes t32_simulate_mrs(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -58,7 +62,7 @@ t32_simulate_mrs(struct kprobe *p, struct pt_regs *regs) regs->uregs[rd] = regs->ARM_cpsr & mask; } -void __kprobes +static void __kprobes t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -73,8 +77,9 @@ t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + (offset * 2); } -enum kprobe_insn __kprobes -t32_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum kprobe_insn __kprobes +t32_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *d) { int cc = (insn >> 22) & 0xf; asi->insn_check_cc = kprobe_condition_checks[cc]; @@ -82,7 +87,7 @@ t32_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD_NO_SLOT; } -void __kprobes +static void __kprobes t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -110,7 +115,7 @@ t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + (offset * 2); } -void __kprobes +static void __kprobes t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -148,10 +153,11 @@ t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = rtv; } -enum kprobe_insn __kprobes -t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum kprobe_insn __kprobes +t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *d) { - enum kprobe_insn ret = kprobe_decode_ldmstm(insn, asi); + enum kprobe_insn ret = kprobe_decode_ldmstm(insn, asi, d); /* Fixup modified instruction to have halfwords in correct order...*/ insn = asi->insn[0]; @@ -161,7 +167,7 @@ t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi) return ret; } -void __kprobes +static void __kprobes t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -188,7 +194,7 @@ t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt2] = rt2v; } -void __kprobes +static void __kprobes t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -214,7 +220,7 @@ t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = rtv; } -void __kprobes +static void __kprobes t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -241,7 +247,7 @@ t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -void __kprobes +static void __kprobes t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -261,7 +267,7 @@ t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) regs->uregs[rd] = rdv; } -void __kprobes +static void __kprobes t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -281,7 +287,7 @@ t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) regs->uregs[rd] = rdv; } -void __kprobes +static void __kprobes t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -308,7 +314,7 @@ t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) } /* t16 thumb actions */ -void __kprobes +static void __kprobes t16_simulate_bxblx(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -322,7 +328,7 @@ t16_simulate_bxblx(struct kprobe *p, struct pt_regs *regs) bx_write_pc(rmv, regs); } -void __kprobes +static void __kprobes t16_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -332,7 +338,7 @@ t16_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = base[index]; } -void __kprobes +static void __kprobes t16_simulate_ldrstr_sp_relative(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -345,7 +351,7 @@ t16_simulate_ldrstr_sp_relative(struct kprobe *p, struct pt_regs *regs) base[index] = regs->uregs[rt]; } -void __kprobes +static void __kprobes t16_simulate_reladr(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -356,7 +362,7 @@ t16_simulate_reladr(struct kprobe *p, struct pt_regs *regs) regs->uregs[rt] = base + offset * 4; } -void __kprobes +static void __kprobes t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -367,7 +373,7 @@ t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs) regs->ARM_sp += imm * 4; } -void __kprobes +static void __kprobes t16_simulate_cbz(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -381,7 +387,7 @@ t16_simulate_cbz(struct kprobe *p, struct pt_regs *regs) } } -void __kprobes +static void __kprobes t16_simulate_it(struct kprobe *p, struct pt_regs *regs) { /* @@ -398,21 +404,22 @@ t16_simulate_it(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = cpsr; } -void __kprobes +static void __kprobes t16_singlestep_it(struct kprobe *p, struct pt_regs *regs) { regs->ARM_pc += 2; t16_simulate_it(p, regs); } -enum kprobe_insn __kprobes -t16_decode_it(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum kprobe_insn __kprobes +t16_decode_it(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *d) { asi->insn_singlestep = t16_singlestep_it; return INSN_GOOD_NO_SLOT; } -void __kprobes +static void __kprobes t16_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -422,8 +429,9 @@ t16_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = pc + (offset * 2); } -enum kprobe_insn __kprobes -t16_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum kprobe_insn __kprobes +t16_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *d) { int cc = (insn >> 8) & 0xf; asi->insn_check_cc = kprobe_condition_checks[cc]; @@ -431,7 +439,7 @@ t16_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD_NO_SLOT; } -void __kprobes +static void __kprobes t16_simulate_branch(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -463,13 +471,13 @@ t16_emulate_loregs(struct kprobe *p, struct pt_regs *regs) return (oldcpsr & ~APSR_MASK) | (newcpsr & APSR_MASK); } -void __kprobes +static void __kprobes t16_emulate_loregs_rwflags(struct kprobe *p, struct pt_regs *regs) { regs->ARM_cpsr = t16_emulate_loregs(p, regs); } -void __kprobes +static void __kprobes t16_emulate_loregs_noitrwflags(struct kprobe *p, struct pt_regs *regs) { unsigned long cpsr = t16_emulate_loregs(p, regs); @@ -477,7 +485,7 @@ t16_emulate_loregs_noitrwflags(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = cpsr; } -void __kprobes +static void __kprobes t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) { kprobe_opcode_t insn = p->opcode; @@ -508,8 +516,9 @@ t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -enum kprobe_insn __kprobes -t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum kprobe_insn __kprobes +t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *d) { insn &= ~0x00ff; insn |= 0x001; /* Set Rdn = R1 and Rm = R0 */ @@ -518,7 +527,7 @@ t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD; } -void __kprobes +static void __kprobes t16_emulate_push(struct kprobe *p, struct pt_regs *regs) { __asm__ __volatile__ ( @@ -534,8 +543,9 @@ t16_emulate_push(struct kprobe *p, struct pt_regs *regs) ); } -enum kprobe_insn __kprobes -t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum kprobe_insn __kprobes +t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *d) { /* * To simulate a PUSH we use a Thumb-2 "STMDB R9!, {registers}" @@ -548,7 +558,7 @@ t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi) return INSN_GOOD; } -void __kprobes +static void __kprobes t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs) { __asm__ __volatile__ ( @@ -564,7 +574,7 @@ t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs) ); } -void __kprobes +static void __kprobes t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) { register unsigned long pc asm("r8"); @@ -584,8 +594,9 @@ t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) bx_write_pc(pc, regs); } -enum kprobe_insn __kprobes -t16_decode_pop(kprobe_opcode_t insn, struct arch_specific_insn *asi) +static enum kprobe_insn __kprobes +t16_decode_pop(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *d) { /* * To simulate a POP we use a Thumb-2 "LDMDB R9!, {registers}" @@ -598,3 +609,57 @@ t16_decode_pop(kprobe_opcode_t insn, struct arch_specific_insn *asi) : t16_emulate_pop_nopc; return INSN_GOOD; } + +const union decode_action kprobes_t16_actions[NUM_PROBES_T16_ACTIONS] = { + [PROBES_T16_ADD_SP] = {.handler = t16_simulate_add_sp_imm}, + [PROBES_T16_CBZ] = {.handler = t16_simulate_cbz}, + [PROBES_T16_SIGN_EXTEND] = {.handler = t16_emulate_loregs_rwflags}, + [PROBES_T16_PUSH] = {.decoder = t16_decode_push}, + [PROBES_T16_POP] = {.decoder = t16_decode_pop}, + [PROBES_T16_SEV] = {.handler = kprobe_emulate_none}, + [PROBES_T16_WFE] = {.handler = kprobe_simulate_nop}, + [PROBES_T16_IT] = {.decoder = t16_decode_it}, + [PROBES_T16_CMP] = {.handler = t16_emulate_loregs_rwflags}, + [PROBES_T16_ADDSUB] = {.handler = t16_emulate_loregs_noitrwflags}, + [PROBES_T16_LOGICAL] = {.handler = t16_emulate_loregs_noitrwflags}, + [PROBES_T16_LDR_LIT] = {.handler = t16_simulate_ldr_literal}, + [PROBES_T16_BLX] = {.handler = t16_simulate_bxblx}, + [PROBES_T16_HIREGOPS] = {.decoder = t16_decode_hiregs}, + [PROBES_T16_LDRHSTRH] = {.handler = t16_emulate_loregs_rwflags}, + [PROBES_T16_LDRSTR] = {.handler = t16_simulate_ldrstr_sp_relative}, + [PROBES_T16_ADR] = {.handler = t16_simulate_reladr}, + [PROBES_T16_LDMSTM] = {.handler = t16_emulate_loregs_rwflags}, + [PROBES_T16_BRANCH_COND] = {.decoder = t16_decode_cond_branch}, + [PROBES_T16_BRANCH] = {.handler = t16_simulate_branch}, +}; + +const union decode_action kprobes_t32_actions[NUM_PROBES_T32_ACTIONS] = { + [PROBES_T32_LDMSTM] = {.decoder = t32_decode_ldmstm}, + [PROBES_T32_LDRDSTRD] = {.handler = t32_emulate_ldrdstrd}, + [PROBES_T32_TABLE_BRANCH] = {.handler = t32_simulate_table_branch}, + [PROBES_T32_TST] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_MOV] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_ADDSUB] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_LOGICAL] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_CMP] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_ADDWSUBW_PC] = {.handler = t32_emulate_rd8pc16_noflags,}, + [PROBES_T32_ADDWSUBW] = {.handler = t32_emulate_rd8rn16_noflags}, + [PROBES_T32_MOVW] = {.handler = t32_emulate_rd8rn16_noflags}, + [PROBES_T32_SAT] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_BITFIELD] = {.handler = t32_emulate_rd8rn16_noflags}, + [PROBES_T32_SEV] = {.handler = kprobe_emulate_none}, + [PROBES_T32_WFE] = {.handler = kprobe_simulate_nop}, + [PROBES_T32_MRS] = {.handler = t32_simulate_mrs}, + [PROBES_T32_BRANCH_COND] = {.decoder = t32_decode_cond_branch}, + [PROBES_T32_BRANCH] = {.handler = t32_simulate_branch}, + [PROBES_T32_PLDI] = {.handler = kprobe_simulate_nop}, + [PROBES_T32_LDR_LIT] = {.handler = t32_simulate_ldr_literal}, + [PROBES_T32_LDRSTR] = {.handler = t32_emulate_ldrstr}, + [PROBES_T32_SIGN_EXTEND] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_MEDIA] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_REVERSE] = {.handler = t32_emulate_rd8rn16_noflags}, + [PROBES_T32_MUL_ADD] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, + [PROBES_T32_MUL_ADD2] = {.handler = t32_emulate_rd8rn16rm0ra12_noflags}, + [PROBES_T32_MUL_ADD_LONG] = { + .handler = t32_emulate_rdlo12rdhi8rn16rm0_noflags}, +}; diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c index 54e7b46a3295..a757c3c22381 100644 --- a/arch/arm/kernel/kprobes.c +++ b/arch/arm/kernel/kprobes.c @@ -56,6 +56,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) unsigned long addr = (unsigned long)p->addr; bool thumb; kprobe_decode_insn_t *decode_insn; + const union decode_action *actions; int is; if (in_exception_text(addr)) @@ -69,20 +70,24 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) insn <<= 16; insn |= ((u16 *)addr)[1]; decode_insn = thumb32_kprobe_decode_insn; - } else + actions = kprobes_t32_actions; + } else { decode_insn = thumb16_kprobe_decode_insn; + actions = kprobes_t16_actions; + } #else /* !CONFIG_THUMB2_KERNEL */ thumb = false; if (addr & 0x3) return -EINVAL; insn = *p->addr; decode_insn = arm_kprobe_decode_insn; + actions = kprobes_arm_actions; #endif p->opcode = insn; p->ainsn.insn = tmp_insn; - switch ((*decode_insn)(insn, &p->ainsn)) { + switch ((*decode_insn)(insn, &p->ainsn, actions)) { case INSN_REJECTED: /* not supported */ return -EINVAL; diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index aa68c0ea1a0b..7798035d6003 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -27,6 +27,8 @@ #define KPROBE_THUMB16_BREAKPOINT_INSTRUCTION 0xde18 #define KPROBE_THUMB32_BREAKPOINT_INSTRUCTION 0xf7f0a018 +struct decode_header; +union decode_action; enum kprobe_insn { INSN_REJECTED, @@ -35,19 +37,24 @@ enum kprobe_insn { }; typedef enum kprobe_insn (kprobe_decode_insn_t)(kprobe_opcode_t, - struct arch_specific_insn *); + struct arch_specific_insn *, + const union decode_action *); #ifdef CONFIG_THUMB2_KERNEL enum kprobe_insn thumb16_kprobe_decode_insn(kprobe_opcode_t, - struct arch_specific_insn *); + struct arch_specific_insn *, + const union decode_action *); enum kprobe_insn thumb32_kprobe_decode_insn(kprobe_opcode_t, - struct arch_specific_insn *); + struct arch_specific_insn *, + const union decode_action *); #else /* !CONFIG_THUMB2_KERNEL */ enum kprobe_insn arm_kprobe_decode_insn(kprobe_opcode_t, - struct arch_specific_insn *); + struct arch_specific_insn *, + const union decode_action *); + #endif void __init arm_kprobe_decode_init(void); diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c index 57e08b28e87f..496e0e913fa6 100644 --- a/arch/arm/kernel/probes-arm.c +++ b/arch/arm/kernel/probes-arm.c @@ -126,16 +126,16 @@ static const union decode_item arm_1111_table[] = { /* PLDI (immediate) 1111 0100 x101 xxxx xxxx xxxx xxxx xxxx */ /* PLDW (immediate) 1111 0101 x001 xxxx xxxx xxxx xxxx xxxx */ /* PLD (immediate) 1111 0101 x101 xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe300000, 0xf4100000, kprobe_simulate_nop), + DECODE_SIMULATE (0xfe300000, 0xf4100000, PROBES_PRELOAD_IMM), /* memory hint 1111 0110 x001 xxxx xxxx xxxx xxx0 xxxx */ /* PLDI (register) 1111 0110 x101 xxxx xxxx xxxx xxx0 xxxx */ /* PLDW (register) 1111 0111 x001 xxxx xxxx xxxx xxx0 xxxx */ /* PLD (register) 1111 0111 x101 xxxx xxxx xxxx xxx0 xxxx */ - DECODE_SIMULATE (0xfe300010, 0xf6100000, kprobe_simulate_nop), + DECODE_SIMULATE (0xfe300010, 0xf6100000, PROBES_PRELOAD_REG), /* BLX (immediate) 1111 101x xxxx xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe000000, 0xfa000000, simulate_blx1), + DECODE_SIMULATE (0xfe000000, 0xfa000000, PROBES_BRANCH_IMM), /* CPS 1111 0001 0000 xxx0 xxxx xxxx xx0x xxxx */ /* SETEND 1111 0001 0000 0001 xxxx xxxx 0000 xxxx */ @@ -159,25 +159,25 @@ static const union decode_item arm_cccc_0001_0xx0____0xxx_table[] = { /* Miscellaneous instructions */ /* MRS cpsr cccc 0001 0000 xxxx xxxx xxxx 0000 xxxx */ - DECODE_SIMULATEX(0x0ff000f0, 0x01000000, simulate_mrs, + DECODE_SIMULATEX(0x0ff000f0, 0x01000000, PROBES_MRS, REGS(0, NOPC, 0, 0, 0)), /* BX cccc 0001 0010 xxxx xxxx xxxx 0001 xxxx */ - DECODE_SIMULATE (0x0ff000f0, 0x01200010, simulate_blx2bx), + DECODE_SIMULATE (0x0ff000f0, 0x01200010, PROBES_BRANCH_REG), /* BLX (register) cccc 0001 0010 xxxx xxxx xxxx 0011 xxxx */ - DECODE_SIMULATEX(0x0ff000f0, 0x01200030, simulate_blx2bx, + DECODE_SIMULATEX(0x0ff000f0, 0x01200030, PROBES_BRANCH_REG, REGS(0, 0, 0, 0, NOPC)), /* CLZ cccc 0001 0110 xxxx xxxx xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x01600010, emulate_rd12rm0_noflags_nopc, + DECODE_EMULATEX (0x0ff000f0, 0x01600010, PROBES_CLZ, REGS(0, NOPC, 0, 0, NOPC)), /* QADD cccc 0001 0000 xxxx xxxx xxxx 0101 xxxx */ /* QSUB cccc 0001 0010 xxxx xxxx xxxx 0101 xxxx */ /* QDADD cccc 0001 0100 xxxx xxxx xxxx 0101 xxxx */ /* QDSUB cccc 0001 0110 xxxx xxxx xxxx 0101 xxxx */ - DECODE_EMULATEX (0x0f9000f0, 0x01000050, emulate_rd12rn16rm0_rwflags_nopc, + DECODE_EMULATEX (0x0f9000f0, 0x01000050, PROBES_SATURATING_ARITHMETIC, REGS(NOPC, NOPC, 0, 0, NOPC)), /* BXJ cccc 0001 0010 xxxx xxxx xxxx 0010 xxxx */ @@ -193,19 +193,19 @@ static const union decode_item arm_cccc_0001_0xx0____1xx0_table[] = { /* Halfword multiply and multiply-accumulate */ /* SMLALxy cccc 0001 0100 xxxx xxxx xxxx 1xx0 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x01400080, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, + DECODE_EMULATEX (0x0ff00090, 0x01400080, PROBES_MUL1, REGS(NOPC, NOPC, NOPC, 0, NOPC)), /* SMULWy cccc 0001 0010 xxxx xxxx xxxx 1x10 xxxx */ DECODE_OR (0x0ff000b0, 0x012000a0), /* SMULxy cccc 0001 0110 xxxx xxxx xxxx 1xx0 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x01600080, emulate_rd16rn12rm0rs8_rwflags_nopc, + DECODE_EMULATEX (0x0ff00090, 0x01600080, PROBES_MUL2, REGS(NOPC, 0, NOPC, 0, NOPC)), /* SMLAxy cccc 0001 0000 xxxx xxxx xxxx 1xx0 xxxx */ DECODE_OR (0x0ff00090, 0x01000080), /* SMLAWy cccc 0001 0010 xxxx xxxx xxxx 1x00 xxxx */ - DECODE_EMULATEX (0x0ff000b0, 0x01200080, emulate_rd16rn12rm0rs8_rwflags_nopc, + DECODE_EMULATEX (0x0ff000b0, 0x01200080, PROBES_MUL2, REGS(NOPC, NOPC, NOPC, 0, NOPC)), DECODE_END @@ -216,14 +216,14 @@ static const union decode_item arm_cccc_0000_____1001_table[] = { /* MUL cccc 0000 0000 xxxx xxxx xxxx 1001 xxxx */ /* MULS cccc 0000 0001 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0fe000f0, 0x00000090, emulate_rd16rn12rm0rs8_rwflags_nopc, + DECODE_EMULATEX (0x0fe000f0, 0x00000090, PROBES_MUL2, REGS(NOPC, 0, NOPC, 0, NOPC)), /* MLA cccc 0000 0010 xxxx xxxx xxxx 1001 xxxx */ /* MLAS cccc 0000 0011 xxxx xxxx xxxx 1001 xxxx */ DECODE_OR (0x0fe000f0, 0x00200090), /* MLS cccc 0000 0110 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x00600090, emulate_rd16rn12rm0rs8_rwflags_nopc, + DECODE_EMULATEX (0x0ff000f0, 0x00600090, PROBES_MUL2, REGS(NOPC, NOPC, NOPC, 0, NOPC)), /* UMAAL cccc 0000 0100 xxxx xxxx xxxx 1001 xxxx */ @@ -236,7 +236,7 @@ static const union decode_item arm_cccc_0000_____1001_table[] = { /* SMULLS cccc 0000 1101 xxxx xxxx xxxx 1001 xxxx */ /* SMLAL cccc 0000 1110 xxxx xxxx xxxx 1001 xxxx */ /* SMLALS cccc 0000 1111 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0f8000f0, 0x00800090, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, + DECODE_EMULATEX (0x0f8000f0, 0x00800090, PROBES_MUL1, REGS(NOPC, NOPC, NOPC, 0, NOPC)), DECODE_END @@ -248,7 +248,7 @@ static const union decode_item arm_cccc_0001_____1001_table[] = { #if __LINUX_ARM_ARCH__ < 6 /* Deprecated on ARMv6 and may be UNDEFINED on v7 */ /* SMP/SWPB cccc 0001 0x00 xxxx xxxx xxxx 1001 xxxx */ - DECODE_EMULATEX (0x0fb000f0, 0x01000090, emulate_rd12rn16rm0_rwflags_nopc, + DECODE_EMULATEX (0x0fb000f0, 0x01000090, PROBES_SWP, REGS(NOPC, NOPC, 0, 0, NOPC)), #endif /* LDREX/STREX{,D,B,H} cccc 0001 1xxx xxxx xxxx xxxx 1001 xxxx */ @@ -271,32 +271,32 @@ static const union decode_item arm_cccc_000x_____1xx1_table[] = { /* LDRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1101 xxxx */ /* STRD (register) cccc 000x x0x0 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e5000d0, 0x000000d0, emulate_ldrdstrd, + DECODE_EMULATEX (0x0e5000d0, 0x000000d0, PROBES_LDRSTRD, REGS(NOPCWB, NOPCX, 0, 0, NOPC)), /* LDRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1101 xxxx */ /* STRD (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e5000d0, 0x004000d0, emulate_ldrdstrd, + DECODE_EMULATEX (0x0e5000d0, 0x004000d0, PROBES_LDRSTRD, REGS(NOPCWB, NOPCX, 0, 0, 0)), /* STRH (register) cccc 000x x0x0 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0e5000f0, 0x000000b0, emulate_str, + DECODE_EMULATEX (0x0e5000f0, 0x000000b0, PROBES_STORE_EXTRA, REGS(NOPCWB, NOPC, 0, 0, NOPC)), /* LDRH (register) cccc 000x x0x1 xxxx xxxx xxxx 1011 xxxx */ /* LDRSB (register) cccc 000x x0x1 xxxx xxxx xxxx 1101 xxxx */ /* LDRSH (register) cccc 000x x0x1 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e500090, 0x00100090, emulate_ldr, + DECODE_EMULATEX (0x0e500090, 0x00100090, PROBES_LOAD_EXTRA, REGS(NOPCWB, NOPC, 0, 0, NOPC)), /* STRH (immediate) cccc 000x x1x0 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0e5000f0, 0x004000b0, emulate_str, + DECODE_EMULATEX (0x0e5000f0, 0x004000b0, PROBES_STORE_EXTRA, REGS(NOPCWB, NOPC, 0, 0, 0)), /* LDRH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1011 xxxx */ /* LDRSB (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1101 xxxx */ /* LDRSH (immediate) cccc 000x x1x1 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0e500090, 0x00500090, emulate_ldr, + DECODE_EMULATEX (0x0e500090, 0x00500090, PROBES_LOAD_EXTRA, REGS(NOPCWB, NOPC, 0, 0, 0)), DECODE_END @@ -309,18 +309,18 @@ static const union decode_item arm_cccc_000x_table[] = { DECODE_REJECT (0x0e10f000, 0x0010f000), /* MOV IP, SP 1110 0001 1010 0000 1100 0000 0000 1101 */ - DECODE_SIMULATE (0xffffffff, 0xe1a0c00d, simulate_mov_ipsp), + DECODE_SIMULATE (0xffffffff, 0xe1a0c00d, PROBES_MOV_IP_SP), /* TST (register) cccc 0001 0001 xxxx xxxx xxxx xxx0 xxxx */ /* TEQ (register) cccc 0001 0011 xxxx xxxx xxxx xxx0 xxxx */ /* CMP (register) cccc 0001 0101 xxxx xxxx xxxx xxx0 xxxx */ /* CMN (register) cccc 0001 0111 xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0f900010, 0x01100000, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0f900010, 0x01100000, PROBES_DATA_PROCESSING_REG, REGS(ANY, 0, 0, 0, ANY)), /* MOV (register) cccc 0001 101x xxxx xxxx xxxx xxx0 xxxx */ /* MVN (register) cccc 0001 111x xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0fa00010, 0x01a00000, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0fa00010, 0x01a00000, PROBES_DATA_PROCESSING_REG, REGS(0, ANY, 0, 0, ANY)), /* AND (register) cccc 0000 000x xxxx xxxx xxxx xxx0 xxxx */ @@ -333,19 +333,19 @@ static const union decode_item arm_cccc_000x_table[] = { /* RSC (register) cccc 0000 111x xxxx xxxx xxxx xxx0 xxxx */ /* ORR (register) cccc 0001 100x xxxx xxxx xxxx xxx0 xxxx */ /* BIC (register) cccc 0001 110x xxxx xxxx xxxx xxx0 xxxx */ - DECODE_EMULATEX (0x0e000010, 0x00000000, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0e000010, 0x00000000, PROBES_DATA_PROCESSING_REG, REGS(ANY, ANY, 0, 0, ANY)), /* TST (reg-shift reg) cccc 0001 0001 xxxx xxxx xxxx 0xx1 xxxx */ /* TEQ (reg-shift reg) cccc 0001 0011 xxxx xxxx xxxx 0xx1 xxxx */ /* CMP (reg-shift reg) cccc 0001 0101 xxxx xxxx xxxx 0xx1 xxxx */ /* CMN (reg-shift reg) cccc 0001 0111 xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0f900090, 0x01100010, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0f900090, 0x01100010, PROBES_DATA_PROCESSING_REG, REGS(ANY, 0, NOPC, 0, ANY)), /* MOV (reg-shift reg) cccc 0001 101x xxxx xxxx xxxx 0xx1 xxxx */ /* MVN (reg-shift reg) cccc 0001 111x xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0fa00090, 0x01a00010, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0fa00090, 0x01a00010, PROBES_DATA_PROCESSING_REG, REGS(0, ANY, NOPC, 0, ANY)), /* AND (reg-shift reg) cccc 0000 000x xxxx xxxx xxxx 0xx1 xxxx */ @@ -358,7 +358,7 @@ static const union decode_item arm_cccc_000x_table[] = { /* RSC (reg-shift reg) cccc 0000 111x xxxx xxxx xxxx 0xx1 xxxx */ /* ORR (reg-shift reg) cccc 0001 100x xxxx xxxx xxxx 0xx1 xxxx */ /* BIC (reg-shift reg) cccc 0001 110x xxxx xxxx xxxx 0xx1 xxxx */ - DECODE_EMULATEX (0x0e000090, 0x00000010, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0e000090, 0x00000010, PROBES_DATA_PROCESSING_REG, REGS(ANY, ANY, NOPC, 0, ANY)), DECODE_END @@ -369,17 +369,17 @@ static const union decode_item arm_cccc_001x_table[] = { /* MOVW cccc 0011 0000 xxxx xxxx xxxx xxxx xxxx */ /* MOVT cccc 0011 0100 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0fb00000, 0x03000000, emulate_rd12rm0_noflags_nopc, + DECODE_EMULATEX (0x0fb00000, 0x03000000, PROBES_DATA_PROCESSING_IMM, REGS(0, NOPC, 0, 0, 0)), /* YIELD cccc 0011 0010 0000 xxxx xxxx 0000 0001 */ DECODE_OR (0x0fff00ff, 0x03200001), /* SEV cccc 0011 0010 0000 xxxx xxxx 0000 0100 */ - DECODE_EMULATE (0x0fff00ff, 0x03200004, kprobe_emulate_none), + DECODE_EMULATE (0x0fff00ff, 0x03200004, PROBES_EMULATE_NONE), /* NOP cccc 0011 0010 0000 xxxx xxxx 0000 0000 */ /* WFE cccc 0011 0010 0000 xxxx xxxx 0000 0010 */ /* WFI cccc 0011 0010 0000 xxxx xxxx 0000 0011 */ - DECODE_SIMULATE (0x0fff00fc, 0x03200000, kprobe_simulate_nop), + DECODE_SIMULATE (0x0fff00fc, 0x03200000, PROBES_SIMULATE_NOP), /* DBG cccc 0011 0010 0000 xxxx xxxx ffff xxxx */ /* unallocated hints cccc 0011 0010 0000 xxxx xxxx xxxx xxxx */ /* MSR (immediate) cccc 0011 0x10 xxxx xxxx xxxx xxxx xxxx */ @@ -392,12 +392,12 @@ static const union decode_item arm_cccc_001x_table[] = { /* TEQ (immediate) cccc 0011 0011 xxxx xxxx xxxx xxxx xxxx */ /* CMP (immediate) cccc 0011 0101 xxxx xxxx xxxx xxxx xxxx */ /* CMN (immediate) cccc 0011 0111 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0f900000, 0x03100000, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0f900000, 0x03100000, PROBES_DATA_PROCESSING_IMM, REGS(ANY, 0, 0, 0, 0)), /* MOV (immediate) cccc 0011 101x xxxx xxxx xxxx xxxx xxxx */ /* MVN (immediate) cccc 0011 111x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0fa00000, 0x03a00000, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0fa00000, 0x03a00000, PROBES_DATA_PROCESSING_IMM, REGS(0, ANY, 0, 0, 0)), /* AND (immediate) cccc 0010 000x xxxx xxxx xxxx xxxx xxxx */ @@ -410,7 +410,7 @@ static const union decode_item arm_cccc_001x_table[] = { /* RSC (immediate) cccc 0010 111x xxxx xxxx xxxx xxxx xxxx */ /* ORR (immediate) cccc 0011 100x xxxx xxxx xxxx xxxx xxxx */ /* BIC (immediate) cccc 0011 110x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e000000, 0x02000000, emulate_rd12rn16rm0rs8_rwflags, + DECODE_EMULATEX (0x0e000000, 0x02000000, PROBES_DATA_PROCESSING_IMM, REGS(ANY, ANY, 0, 0, 0)), DECODE_END @@ -420,7 +420,7 @@ static const union decode_item arm_cccc_0110_____xxx1_table[] = { /* Media instructions */ /* SEL cccc 0110 1000 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x068000b0, emulate_rd12rn16rm0_rwflags_nopc, + DECODE_EMULATEX (0x0ff000f0, 0x068000b0, PROBES_SATURATE, REGS(NOPC, NOPC, 0, 0, NOPC)), /* SSAT cccc 0110 101x xxxx xxxx xxxx xx01 xxxx */ @@ -428,14 +428,14 @@ static const union decode_item arm_cccc_0110_____xxx1_table[] = { DECODE_OR(0x0fa00030, 0x06a00010), /* SSAT16 cccc 0110 1010 xxxx xxxx xxxx 0011 xxxx */ /* USAT16 cccc 0110 1110 xxxx xxxx xxxx 0011 xxxx */ - DECODE_EMULATEX (0x0fb000f0, 0x06a00030, emulate_rd12rn16rm0_rwflags_nopc, + DECODE_EMULATEX (0x0fb000f0, 0x06a00030, PROBES_SATURATE, REGS(0, NOPC, 0, 0, NOPC)), /* REV cccc 0110 1011 xxxx xxxx xxxx 0011 xxxx */ /* REV16 cccc 0110 1011 xxxx xxxx xxxx 1011 xxxx */ /* RBIT cccc 0110 1111 xxxx xxxx xxxx 0011 xxxx */ /* REVSH cccc 0110 1111 xxxx xxxx xxxx 1011 xxxx */ - DECODE_EMULATEX (0x0fb00070, 0x06b00030, emulate_rd12rm0_noflags_nopc, + DECODE_EMULATEX (0x0fb00070, 0x06b00030, PROBES_REV, REGS(0, NOPC, 0, 0, NOPC)), /* ??? cccc 0110 0x00 xxxx xxxx xxxx xxx1 xxxx */ @@ -480,12 +480,12 @@ static const union decode_item arm_cccc_0110_____xxx1_table[] = { /* UHSUB16 cccc 0110 0111 xxxx xxxx xxxx 0111 xxxx */ /* UHADD8 cccc 0110 0111 xxxx xxxx xxxx 1001 xxxx */ /* UHSUB8 cccc 0110 0111 xxxx xxxx xxxx 1111 xxxx */ - DECODE_EMULATEX (0x0f800010, 0x06000010, emulate_rd12rn16rm0_rwflags_nopc, + DECODE_EMULATEX (0x0f800010, 0x06000010, PROBES_MMI, REGS(NOPC, NOPC, 0, 0, NOPC)), /* PKHBT cccc 0110 1000 xxxx xxxx xxxx x001 xxxx */ /* PKHTB cccc 0110 1000 xxxx xxxx xxxx x101 xxxx */ - DECODE_EMULATEX (0x0ff00030, 0x06800010, emulate_rd12rn16rm0_rwflags_nopc, + DECODE_EMULATEX (0x0ff00030, 0x06800010, PROBES_PACK, REGS(NOPC, NOPC, 0, 0, NOPC)), /* ??? cccc 0110 1001 xxxx xxxx xxxx 0111 xxxx */ @@ -498,7 +498,7 @@ static const union decode_item arm_cccc_0110_____xxx1_table[] = { /* UXTB16 cccc 0110 1100 1111 xxxx xxxx 0111 xxxx */ /* UXTB cccc 0110 1110 1111 xxxx xxxx 0111 xxxx */ /* UXTH cccc 0110 1111 1111 xxxx xxxx 0111 xxxx */ - DECODE_EMULATEX (0x0f8f00f0, 0x068f0070, emulate_rd12rm0_noflags_nopc, + DECODE_EMULATEX (0x0f8f00f0, 0x068f0070, PROBES_EXTEND, REGS(0, NOPC, 0, 0, NOPC)), /* SXTAB16 cccc 0110 1000 xxxx xxxx xxxx 0111 xxxx */ @@ -507,7 +507,7 @@ static const union decode_item arm_cccc_0110_____xxx1_table[] = { /* UXTAB16 cccc 0110 1100 xxxx xxxx xxxx 0111 xxxx */ /* UXTAB cccc 0110 1110 xxxx xxxx xxxx 0111 xxxx */ /* UXTAH cccc 0110 1111 xxxx xxxx xxxx 0111 xxxx */ - DECODE_EMULATEX (0x0f8000f0, 0x06800070, emulate_rd12rn16rm0_rwflags_nopc, + DECODE_EMULATEX (0x0f8000f0, 0x06800070, PROBES_EXTEND_ADD, REGS(NOPCX, NOPC, 0, 0, NOPC)), DECODE_END @@ -521,7 +521,7 @@ static const union decode_item arm_cccc_0111_____xxx1_table[] = { /* SMLALD cccc 0111 0100 xxxx xxxx xxxx 00x1 xxxx */ /* SMLSLD cccc 0111 0100 xxxx xxxx xxxx 01x1 xxxx */ - DECODE_EMULATEX (0x0ff00090, 0x07400010, emulate_rdlo12rdhi16rn0rm8_rwflags_nopc, + DECODE_EMULATEX (0x0ff00090, 0x07400010, PROBES_MUL_ADD_LONG, REGS(NOPC, NOPC, NOPC, 0, NOPC)), /* SMUAD cccc 0111 0000 xxxx 1111 xxxx 00x1 xxxx */ @@ -530,7 +530,7 @@ static const union decode_item arm_cccc_0111_____xxx1_table[] = { /* SMMUL cccc 0111 0101 xxxx 1111 xxxx 00x1 xxxx */ DECODE_OR (0x0ff0f0d0, 0x0750f010), /* USAD8 cccc 0111 1000 xxxx 1111 xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff0f0f0, 0x0780f010, emulate_rd16rn12rm0rs8_rwflags_nopc, + DECODE_EMULATEX (0x0ff0f0f0, 0x0780f010, PROBES_MUL_ADD, REGS(NOPC, 0, NOPC, 0, NOPC)), /* SMLAD cccc 0111 0000 xxxx xxxx xxxx 00x1 xxxx */ @@ -539,24 +539,24 @@ static const union decode_item arm_cccc_0111_____xxx1_table[] = { /* SMMLA cccc 0111 0101 xxxx xxxx xxxx 00x1 xxxx */ DECODE_OR (0x0ff000d0, 0x07500010), /* USADA8 cccc 0111 1000 xxxx xxxx xxxx 0001 xxxx */ - DECODE_EMULATEX (0x0ff000f0, 0x07800010, emulate_rd16rn12rm0rs8_rwflags_nopc, + DECODE_EMULATEX (0x0ff000f0, 0x07800010, PROBES_MUL_ADD, REGS(NOPC, NOPCX, NOPC, 0, NOPC)), /* SMMLS cccc 0111 0101 xxxx xxxx xxxx 11x1 xxxx */ - DECODE_EMULATEX (0x0ff000d0, 0x075000d0, emulate_rd16rn12rm0rs8_rwflags_nopc, + DECODE_EMULATEX (0x0ff000d0, 0x075000d0, PROBES_MUL_ADD, REGS(NOPC, NOPC, NOPC, 0, NOPC)), /* SBFX cccc 0111 101x xxxx xxxx xxxx x101 xxxx */ /* UBFX cccc 0111 111x xxxx xxxx xxxx x101 xxxx */ - DECODE_EMULATEX (0x0fa00070, 0x07a00050, emulate_rd12rm0_noflags_nopc, + DECODE_EMULATEX (0x0fa00070, 0x07a00050, PROBES_BITFIELD, REGS(0, NOPC, 0, 0, NOPC)), /* BFC cccc 0111 110x xxxx xxxx xxxx x001 1111 */ - DECODE_EMULATEX (0x0fe0007f, 0x07c0001f, emulate_rd12rm0_noflags_nopc, + DECODE_EMULATEX (0x0fe0007f, 0x07c0001f, PROBES_BITFIELD, REGS(0, NOPC, 0, 0, 0)), /* BFI cccc 0111 110x xxxx xxxx xxxx x001 xxxx */ - DECODE_EMULATEX (0x0fe00070, 0x07c00010, emulate_rd12rm0_noflags_nopc, + DECODE_EMULATEX (0x0fe00070, 0x07c00010, PROBES_BITFIELD, REGS(0, NOPC, 0, 0, NOPCX)), DECODE_END @@ -576,22 +576,22 @@ static const union decode_item arm_cccc_01xx_table[] = { /* STR (immediate) cccc 010x x0x0 xxxx xxxx xxxx xxxx xxxx */ /* STRB (immediate) cccc 010x x1x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x04000000, emulate_str, + DECODE_EMULATEX (0x0e100000, 0x04000000, PROBES_STORE, REGS(NOPCWB, ANY, 0, 0, 0)), /* LDR (immediate) cccc 010x x0x1 xxxx xxxx xxxx xxxx xxxx */ /* LDRB (immediate) cccc 010x x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x04100000, emulate_ldr, + DECODE_EMULATEX (0x0e100000, 0x04100000, PROBES_LOAD, REGS(NOPCWB, ANY, 0, 0, 0)), /* STR (register) cccc 011x x0x0 xxxx xxxx xxxx xxxx xxxx */ /* STRB (register) cccc 011x x1x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x06000000, emulate_str, + DECODE_EMULATEX (0x0e100000, 0x06000000, PROBES_STORE, REGS(NOPCWB, ANY, 0, 0, NOPC)), /* LDR (register) cccc 011x x0x1 xxxx xxxx xxxx xxxx xxxx */ /* LDRB (register) cccc 011x x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0x0e100000, 0x06100000, emulate_ldr, + DECODE_EMULATEX (0x0e100000, 0x06100000, PROBES_LOAD, REGS(NOPCWB, ANY, 0, 0, NOPC)), DECODE_END @@ -602,7 +602,7 @@ static const union decode_item arm_cccc_100x_table[] = { /* LDM cccc 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ /* STM cccc 100x x0x0 xxxx xxxx xxxx xxxx xxxx */ - DECODE_CUSTOM (0x0e400000, 0x08000000, kprobe_decode_ldmstm), + DECODE_CUSTOM (0x0e400000, 0x08000000, PROBES_LDMSTM), /* STM (user registers) cccc 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ /* LDM (user registers) cccc 100x x1x1 xxxx 0xxx xxxx xxxx xxxx */ @@ -682,7 +682,7 @@ const union decode_item kprobe_decode_arm_table[] = { /* B cccc 1010 xxxx xxxx xxxx xxxx xxxx xxxx */ /* BL cccc 1011 xxxx xxxx xxxx xxxx xxxx xxxx */ - DECODE_SIMULATE (0x0e000000, 0x0a000000, simulate_bbl), + DECODE_SIMULATE (0x0e000000, 0x0a000000, PROBES_BRANCH), /* * Supervisor Call, and coprocessor instructions @@ -723,9 +723,11 @@ static void __kprobes arm_singlestep(struct kprobe *p, struct pt_regs *regs) * should also be very rare. */ enum kprobe_insn __kprobes -arm_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +arm_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const union decode_action *actions) { asi->insn_singlestep = arm_singlestep; asi->insn_check_cc = kprobe_condition_checks[insn>>28]; - return kprobe_decode_insn(insn, asi, kprobe_decode_arm_table, false); + return kprobe_decode_insn(insn, asi, kprobe_decode_arm_table, false, + actions); } diff --git a/arch/arm/kernel/probes-arm.h b/arch/arm/kernel/probes-arm.h index 86084727d36d..ef3089419a0b 100644 --- a/arch/arm/kernel/probes-arm.h +++ b/arch/arm/kernel/probes-arm.h @@ -15,24 +15,48 @@ #ifndef _ARM_KERNEL_PROBES_ARM_H #define _ARM_KERNEL_PROBES_ARM_H +enum probes_arm_action { + PROBES_EMULATE_NONE, + PROBES_SIMULATE_NOP, + PROBES_PRELOAD_IMM, + PROBES_PRELOAD_REG, + PROBES_BRANCH_IMM, + PROBES_BRANCH_REG, + PROBES_MRS, + PROBES_CLZ, + PROBES_SATURATING_ARITHMETIC, + PROBES_MUL1, + PROBES_MUL2, + PROBES_SWP, + PROBES_LDRSTRD, + PROBES_LOAD, + PROBES_STORE, + PROBES_LOAD_EXTRA, + PROBES_STORE_EXTRA, + PROBES_MOV_IP_SP, + PROBES_DATA_PROCESSING_REG, + PROBES_DATA_PROCESSING_IMM, + PROBES_MOV_HALFWORD, + PROBES_SEV, + PROBES_WFE, + PROBES_SATURATE, + PROBES_REV, + PROBES_MMI, + PROBES_PACK, + PROBES_EXTEND, + PROBES_EXTEND_ADD, + PROBES_MUL_ADD_LONG, + PROBES_MUL_ADD, + PROBES_BITFIELD, + PROBES_BRANCH, + PROBES_LDMSTM, + NUM_PROBES_ARM_ACTIONS +}; + void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs); void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs); void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs); void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs); void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs); -void __kprobes emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs); -void __kprobes emulate_ldr(struct kprobe *p, struct pt_regs *regs); -void __kprobes emulate_str(struct kprobe *p, struct pt_regs *regs); -void __kprobes emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, - struct pt_regs *regs); -void __kprobes emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, - struct pt_regs *regs); -void __kprobes emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, - struct pt_regs *regs); -void __kprobes emulate_rd12rm0_noflags_nopc(struct kprobe *p, - struct pt_regs *regs); -void __kprobes emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, - struct pt_regs *regs); - #endif diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c index a1f24777a41a..2abe8ceeb670 100644 --- a/arch/arm/kernel/probes-thumb.c +++ b/arch/arm/kernel/probes-thumb.c @@ -16,9 +16,6 @@ #include "kprobes.h" #include "probes-thumb.h" -/* These emulation encodings are functionally equivalent... */ -#define t32_emulate_rd8rn16rm0ra12_noflags \ - t32_emulate_rdlo12rdhi8rn16rm0_noflags static const union decode_item t32_table_1110_100x_x0xx[] = { /* Load/store multiple instructions */ @@ -44,7 +41,7 @@ static const union decode_item t32_table_1110_100x_x0xx[] = { /* LDMIA 1110 1000 10x1 xxxx xxxx xxxx xxxx xxxx */ /* STMDB 1110 1001 00x0 xxxx xxxx xxxx xxxx xxxx */ /* LDMDB 1110 1001 00x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_CUSTOM (0xfe400000, 0xe8000000, t32_decode_ldmstm), + DECODE_CUSTOM (0xfe400000, 0xe8000000, PROBES_T32_LDMSTM), DECODE_END }; @@ -57,12 +54,12 @@ static const union decode_item t32_table_1110_100x_x1xx[] = { DECODE_OR (0xff600000, 0xe8600000), /* STRD (immediate) 1110 1001 x1x0 xxxx xxxx xxxx xxxx xxxx */ /* LDRD (immediate) 1110 1001 x1x1 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xff400000, 0xe9400000, t32_emulate_ldrdstrd, + DECODE_EMULATEX (0xff400000, 0xe9400000, PROBES_T32_LDRDSTRD, REGS(NOPCWB, NOSPPC, NOSPPC, 0, 0)), /* TBB 1110 1000 1101 xxxx xxxx xxxx 0000 xxxx */ /* TBH 1110 1000 1101 xxxx xxxx xxxx 0001 xxxx */ - DECODE_SIMULATEX(0xfff000e0, 0xe8d00000, t32_simulate_table_branch, + DECODE_SIMULATEX(0xfff000e0, 0xe8d00000, PROBES_T32_TABLE_BRANCH, REGS(NOSP, 0, 0, 0, NOSPPC)), /* STREX 1110 1000 0100 xxxx xxxx xxxx xxxx xxxx */ @@ -82,18 +79,18 @@ static const union decode_item t32_table_1110_101x[] = { /* TST 1110 1010 0001 xxxx xxxx 1111 xxxx xxxx */ /* TEQ 1110 1010 1001 xxxx xxxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xff700f00, 0xea100f00, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xff700f00, 0xea100f00, PROBES_T32_TST, REGS(NOSPPC, 0, 0, 0, NOSPPC)), /* CMN 1110 1011 0001 xxxx xxxx 1111 xxxx xxxx */ DECODE_OR (0xfff00f00, 0xeb100f00), /* CMP 1110 1011 1011 xxxx xxxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfff00f00, 0xebb00f00, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xfff00f00, 0xebb00f00, PROBES_T32_TST, REGS(NOPC, 0, 0, 0, NOSPPC)), /* MOV 1110 1010 010x 1111 xxxx xxxx xxxx xxxx */ /* MVN 1110 1010 011x 1111 xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xffcf0000, 0xea4f0000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xffcf0000, 0xea4f0000, PROBES_T32_MOV, REGS(0, 0, NOSPPC, 0, NOSPPC)), /* ??? 1110 1010 101x xxxx xxxx xxxx xxxx xxxx */ @@ -108,7 +105,7 @@ static const union decode_item t32_table_1110_101x[] = { /* ADD/SUB SP, SP, Rm, LSL #0..3 */ /* 1110 1011 x0xx 1101 x000 1101 xx00 xxxx */ - DECODE_EMULATEX (0xff4f7f30, 0xeb0d0d00, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xff4f7f30, 0xeb0d0d00, PROBES_T32_ADDSUB, REGS(SP, 0, SP, 0, NOSPPC)), /* ADD/SUB SP, SP, Rm, shift */ @@ -117,7 +114,7 @@ static const union decode_item t32_table_1110_101x[] = { /* ADD/SUB Rd, SP, Rm, shift */ /* 1110 1011 x0xx 1101 xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xff4f0000, 0xeb0d0000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xff4f0000, 0xeb0d0000, PROBES_T32_ADDSUB, REGS(SP, 0, NOPC, 0, NOSPPC)), /* AND 1110 1010 000x xxxx xxxx xxxx xxxx xxxx */ @@ -131,7 +128,7 @@ static const union decode_item t32_table_1110_101x[] = { /* SBC 1110 1011 011x xxxx xxxx xxxx xxxx xxxx */ /* SUB 1110 1011 101x xxxx xxxx xxxx xxxx xxxx */ /* RSB 1110 1011 110x xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfe000000, 0xea000000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xfe000000, 0xea000000, PROBES_T32_LOGICAL, REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), DECODE_END @@ -142,18 +139,18 @@ static const union decode_item t32_table_1111_0x0x___0[] = { /* TST 1111 0x00 0001 xxxx 0xxx 1111 xxxx xxxx */ /* TEQ 1111 0x00 1001 xxxx 0xxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfb708f00, 0xf0100f00, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xfb708f00, 0xf0100f00, PROBES_T32_TST, REGS(NOSPPC, 0, 0, 0, 0)), /* CMN 1111 0x01 0001 xxxx 0xxx 1111 xxxx xxxx */ DECODE_OR (0xfbf08f00, 0xf1100f00), /* CMP 1111 0x01 1011 xxxx 0xxx 1111 xxxx xxxx */ - DECODE_EMULATEX (0xfbf08f00, 0xf1b00f00, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xfbf08f00, 0xf1b00f00, PROBES_T32_CMP, REGS(NOPC, 0, 0, 0, 0)), /* MOV 1111 0x00 010x 1111 0xxx xxxx xxxx xxxx */ /* MVN 1111 0x00 011x 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbcf8000, 0xf04f0000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xfbcf8000, 0xf04f0000, PROBES_T32_MOV, REGS(0, 0, NOSPPC, 0, 0)), /* ??? 1111 0x00 101x xxxx 0xxx xxxx xxxx xxxx */ @@ -170,7 +167,7 @@ static const union decode_item t32_table_1111_0x0x___0[] = { /* ADD Rd, SP, #imm 1111 0x01 000x 1101 0xxx xxxx xxxx xxxx */ /* SUB Rd, SP, #imm 1111 0x01 101x 1101 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb4f8000, 0xf10d0000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xfb4f8000, 0xf10d0000, PROBES_T32_ADDSUB, REGS(SP, 0, NOPC, 0, 0)), /* AND 1111 0x00 000x xxxx 0xxx xxxx xxxx xxxx */ @@ -183,7 +180,7 @@ static const union decode_item t32_table_1111_0x0x___0[] = { /* SBC 1111 0x01 011x xxxx 0xxx xxxx xxxx xxxx */ /* SUB 1111 0x01 101x xxxx 0xxx xxxx xxxx xxxx */ /* RSB 1111 0x01 110x xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfa008000, 0xf0000000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xfa008000, 0xf0000000, PROBES_T32_LOGICAL, REGS(NOSPPC, 0, NOSPPC, 0, 0)), DECODE_END @@ -195,44 +192,44 @@ static const union decode_item t32_table_1111_0x1x___0[] = { /* ADDW Rd, PC, #imm 1111 0x10 0000 1111 0xxx xxxx xxxx xxxx */ DECODE_OR (0xfbff8000, 0xf20f0000), /* SUBW Rd, PC, #imm 1111 0x10 1010 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbff8000, 0xf2af0000, t32_emulate_rd8pc16_noflags, + DECODE_EMULATEX (0xfbff8000, 0xf2af0000, PROBES_T32_ADDWSUBW_PC, REGS(PC, 0, NOSPPC, 0, 0)), /* ADDW SP, SP, #imm 1111 0x10 0000 1101 0xxx 1101 xxxx xxxx */ DECODE_OR (0xfbff8f00, 0xf20d0d00), /* SUBW SP, SP, #imm 1111 0x10 1010 1101 0xxx 1101 xxxx xxxx */ - DECODE_EMULATEX (0xfbff8f00, 0xf2ad0d00, t32_emulate_rd8rn16_noflags, + DECODE_EMULATEX (0xfbff8f00, 0xf2ad0d00, PROBES_T32_ADDWSUBW, REGS(SP, 0, SP, 0, 0)), /* ADDW 1111 0x10 0000 xxxx 0xxx xxxx xxxx xxxx */ DECODE_OR (0xfbf08000, 0xf2000000), /* SUBW 1111 0x10 1010 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbf08000, 0xf2a00000, t32_emulate_rd8rn16_noflags, + DECODE_EMULATEX (0xfbf08000, 0xf2a00000, PROBES_T32_ADDWSUBW, REGS(NOPCX, 0, NOSPPC, 0, 0)), /* MOVW 1111 0x10 0100 xxxx 0xxx xxxx xxxx xxxx */ /* MOVT 1111 0x10 1100 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb708000, 0xf2400000, t32_emulate_rd8rn16_noflags, + DECODE_EMULATEX (0xfb708000, 0xf2400000, PROBES_T32_MOVW, REGS(0, 0, NOSPPC, 0, 0)), /* SSAT16 1111 0x11 0010 xxxx 0000 xxxx 00xx xxxx */ /* SSAT 1111 0x11 00x0 xxxx 0xxx xxxx xxxx xxxx */ /* USAT16 1111 0x11 1010 xxxx 0000 xxxx 00xx xxxx */ /* USAT 1111 0x11 10x0 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb508000, 0xf3000000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xfb508000, 0xf3000000, PROBES_T32_SAT, REGS(NOSPPC, 0, NOSPPC, 0, 0)), /* SFBX 1111 0x11 0100 xxxx 0xxx xxxx xxxx xxxx */ /* UFBX 1111 0x11 1100 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfb708000, 0xf3400000, t32_emulate_rd8rn16_noflags, + DECODE_EMULATEX (0xfb708000, 0xf3400000, PROBES_T32_BITFIELD, REGS(NOSPPC, 0, NOSPPC, 0, 0)), /* BFC 1111 0x11 0110 1111 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbff8000, 0xf36f0000, t32_emulate_rd8rn16_noflags, + DECODE_EMULATEX (0xfbff8000, 0xf36f0000, PROBES_T32_BITFIELD, REGS(0, 0, NOSPPC, 0, 0)), /* BFI 1111 0x11 0110 xxxx 0xxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfbf08000, 0xf3600000, t32_emulate_rd8rn16_noflags, + DECODE_EMULATEX (0xfbf08000, 0xf3600000, PROBES_T32_BITFIELD, REGS(NOSPPCX, 0, NOSPPC, 0, 0)), DECODE_END @@ -244,14 +241,14 @@ static const union decode_item t32_table_1111_0xxx___1[] = { /* YIELD 1111 0011 1010 xxxx 10x0 x000 0000 0001 */ DECODE_OR (0xfff0d7ff, 0xf3a08001), /* SEV 1111 0011 1010 xxxx 10x0 x000 0000 0100 */ - DECODE_EMULATE (0xfff0d7ff, 0xf3a08004, kprobe_emulate_none), + DECODE_EMULATE (0xfff0d7ff, 0xf3a08004, PROBES_T32_SEV), /* NOP 1111 0011 1010 xxxx 10x0 x000 0000 0000 */ /* WFE 1111 0011 1010 xxxx 10x0 x000 0000 0010 */ /* WFI 1111 0011 1010 xxxx 10x0 x000 0000 0011 */ - DECODE_SIMULATE (0xfff0d7fc, 0xf3a08000, kprobe_simulate_nop), + DECODE_SIMULATE (0xfff0d7fc, 0xf3a08000, PROBES_T32_WFE), /* MRS Rd, CPSR 1111 0011 1110 xxxx 10x0 xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xfff0d000, 0xf3e08000, t32_simulate_mrs, + DECODE_SIMULATEX(0xfff0d000, 0xf3e08000, PROBES_T32_MRS, REGS(0, 0, NOSPPC, 0, 0)), /* @@ -273,13 +270,13 @@ static const union decode_item t32_table_1111_0xxx___1[] = { DECODE_REJECT (0xfb80d000, 0xf3808000), /* Bcc 1111 0xxx xxxx xxxx 10x0 xxxx xxxx xxxx */ - DECODE_CUSTOM (0xf800d000, 0xf0008000, t32_decode_cond_branch), + DECODE_CUSTOM (0xf800d000, 0xf0008000, PROBES_T32_BRANCH_COND), /* BLX 1111 0xxx xxxx xxxx 11x0 xxxx xxxx xxx0 */ DECODE_OR (0xf800d001, 0xf000c000), /* B 1111 0xxx xxxx xxxx 10x1 xxxx xxxx xxxx */ /* BL 1111 0xxx xxxx xxxx 11x1 xxxx xxxx xxxx */ - DECODE_SIMULATE (0xf8009000, 0xf0009000, t32_simulate_branch), + DECODE_SIMULATE (0xf8009000, 0xf0009000, PROBES_T32_BRANCH), DECODE_END }; @@ -289,7 +286,7 @@ static const union decode_item t32_table_1111_100x_x0x1__1111[] = { /* PLD (literal) 1111 1000 x001 1111 1111 xxxx xxxx xxxx */ /* PLI (literal) 1111 1001 x001 1111 1111 xxxx xxxx xxxx */ - DECODE_SIMULATE (0xfe7ff000, 0xf81ff000, kprobe_simulate_nop), + DECODE_SIMULATE (0xfe7ff000, 0xf81ff000, PROBES_T32_PLDI), /* PLD{W} (immediate) 1111 1000 10x1 xxxx 1111 xxxx xxxx xxxx */ DECODE_OR (0xffd0f000, 0xf890f000), @@ -298,13 +295,13 @@ static const union decode_item t32_table_1111_100x_x0x1__1111[] = { /* PLI (immediate) 1111 1001 1001 xxxx 1111 xxxx xxxx xxxx */ DECODE_OR (0xfff0f000, 0xf990f000), /* PLI (immediate) 1111 1001 0001 xxxx 1111 1100 xxxx xxxx */ - DECODE_SIMULATEX(0xfff0ff00, 0xf910fc00, kprobe_simulate_nop, + DECODE_SIMULATEX(0xfff0ff00, 0xf910fc00, PROBES_T32_PLDI, REGS(NOPCX, 0, 0, 0, 0)), /* PLD{W} (register) 1111 1000 00x1 xxxx 1111 0000 00xx xxxx */ DECODE_OR (0xffd0ffc0, 0xf810f000), /* PLI (register) 1111 1001 0001 xxxx 1111 0000 00xx xxxx */ - DECODE_SIMULATEX(0xfff0ffc0, 0xf910f000, kprobe_simulate_nop, + DECODE_SIMULATEX(0xfff0ffc0, 0xf910f000, PROBES_T32_PLDI, REGS(NOPCX, 0, 0, 0, NOSPPC)), /* Other unallocated instructions... */ @@ -340,7 +337,7 @@ static const union decode_item t32_table_1111_100x[] = { DECODE_REJECT (0xff10f000, 0xf800f000), /* LDR (literal) 1111 1000 x101 1111 xxxx xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xff7f0000, 0xf85f0000, t32_simulate_ldr_literal, + DECODE_SIMULATEX(0xff7f0000, 0xf85f0000, PROBES_T32_LDR_LIT, REGS(PC, ANY, 0, 0, 0)), /* STR (immediate) 1111 1000 0100 xxxx xxxx 1xxx xxxx xxxx */ @@ -348,19 +345,19 @@ static const union decode_item t32_table_1111_100x[] = { DECODE_OR (0xffe00800, 0xf8400800), /* STR (immediate) 1111 1000 1100 xxxx xxxx xxxx xxxx xxxx */ /* LDR (immediate) 1111 1000 1101 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xffe00000, 0xf8c00000, t32_emulate_ldrstr, + DECODE_EMULATEX (0xffe00000, 0xf8c00000, PROBES_T32_LDRSTR, REGS(NOPCX, ANY, 0, 0, 0)), /* STR (register) 1111 1000 0100 xxxx xxxx 0000 00xx xxxx */ /* LDR (register) 1111 1000 0101 xxxx xxxx 0000 00xx xxxx */ - DECODE_EMULATEX (0xffe00fc0, 0xf8400000, t32_emulate_ldrstr, + DECODE_EMULATEX (0xffe00fc0, 0xf8400000, PROBES_T32_LDRSTR, REGS(NOPCX, ANY, 0, 0, NOSPPC)), /* LDRB (literal) 1111 1000 x001 1111 xxxx xxxx xxxx xxxx */ /* LDRSB (literal) 1111 1001 x001 1111 xxxx xxxx xxxx xxxx */ /* LDRH (literal) 1111 1000 x011 1111 xxxx xxxx xxxx xxxx */ /* LDRSH (literal) 1111 1001 x011 1111 xxxx xxxx xxxx xxxx */ - DECODE_SIMULATEX(0xfe5f0000, 0xf81f0000, t32_simulate_ldr_literal, + DECODE_SIMULATEX(0xfe5f0000, 0xf81f0000, PROBES_T32_LDR_LIT, REGS(PC, NOSPPCX, 0, 0, 0)), /* STRB (immediate) 1111 1000 0000 xxxx xxxx 1xxx xxxx xxxx */ @@ -376,7 +373,7 @@ static const union decode_item t32_table_1111_100x[] = { /* LDRSB (immediate) 1111 1001 1001 xxxx xxxx xxxx xxxx xxxx */ /* LDRH (immediate) 1111 1000 1011 xxxx xxxx xxxx xxxx xxxx */ /* LDRSH (immediate) 1111 1001 1011 xxxx xxxx xxxx xxxx xxxx */ - DECODE_EMULATEX (0xfec00000, 0xf8800000, t32_emulate_ldrstr, + DECODE_EMULATEX (0xfec00000, 0xf8800000, PROBES_T32_LDRSTR, REGS(NOPCX, NOSPPCX, 0, 0, 0)), /* STRB (register) 1111 1000 0000 xxxx xxxx 0000 00xx xxxx */ @@ -385,7 +382,7 @@ static const union decode_item t32_table_1111_100x[] = { /* LDRSB (register) 1111 1001 0001 xxxx xxxx 0000 00xx xxxx */ /* LDRH (register) 1111 1000 0011 xxxx xxxx 0000 00xx xxxx */ /* LDRSH (register) 1111 1001 0011 xxxx xxxx 0000 00xx xxxx */ - DECODE_EMULATEX (0xfe800fc0, 0xf8000000, t32_emulate_ldrstr, + DECODE_EMULATEX (0xfe800fc0, 0xf8000000, PROBES_T32_LDRSTR, REGS(NOPCX, NOSPPCX, 0, 0, NOSPPC)), /* Other unallocated instructions... */ @@ -404,7 +401,7 @@ static const union decode_item t32_table_1111_1010___1111[] = { /* UXTB16 1111 1010 0011 1111 1111 xxxx 1xxx xxxx */ /* SXTB 1111 1010 0100 1111 1111 xxxx 1xxx xxxx */ /* UXTB 1111 1010 0101 1111 1111 xxxx 1xxx xxxx */ - DECODE_EMULATEX (0xff8ff080, 0xfa0ff080, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xff8ff080, 0xfa0ff080, PROBES_T32_SIGN_EXTEND, REGS(0, 0, NOSPPC, 0, NOSPPC)), @@ -477,7 +474,7 @@ static const union decode_item t32_table_1111_1010___1111[] = { /* LSR 1111 1010 001x xxxx 1111 xxxx 0000 xxxx */ /* ASR 1111 1010 010x xxxx 1111 xxxx 0000 xxxx */ /* ROR 1111 1010 011x xxxx 1111 xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff80f0f0, 0xfa00f000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xff80f0f0, 0xfa00f000, PROBES_T32_MEDIA, REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), /* CLZ 1111 1010 1010 xxxx 1111 xxxx 1000 xxxx */ @@ -487,7 +484,7 @@ static const union decode_item t32_table_1111_1010___1111[] = { /* REV16 1111 1010 1001 xxxx 1111 xxxx 1001 xxxx */ /* RBIT 1111 1010 1001 xxxx 1111 xxxx 1010 xxxx */ /* REVSH 1111 1010 1001 xxxx 1111 xxxx 1011 xxxx */ - DECODE_EMULATEX (0xfff0f0c0, 0xfa90f080, t32_emulate_rd8rn16_noflags, + DECODE_EMULATEX (0xfff0f0c0, 0xfa90f080, PROBES_T32_REVERSE, REGS(NOSPPC, 0, NOSPPC, 0, SAMEAS16)), /* Other unallocated instructions... */ @@ -510,7 +507,7 @@ static const union decode_item t32_table_1111_1011_0[] = { /* SMUSD{X} 1111 1011 0100 xxxx 1111 xxxx 000x xxxx */ /* SMMUL{R} 1111 1011 0101 xxxx 1111 xxxx 000x xxxx */ /* USAD8 1111 1011 0111 xxxx 1111 xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff80f0e0, 0xfb00f000, t32_emulate_rd8rn16rm0_rwflags, + DECODE_EMULATEX (0xff80f0e0, 0xfb00f000, PROBES_T32_MUL_ADD, REGS(NOSPPC, 0, NOSPPC, 0, NOSPPC)), /* ??? 1111 1011 0111 xxxx xxxx xxxx 0001 xxxx */ @@ -526,7 +523,7 @@ static const union decode_item t32_table_1111_1011_0[] = { /* SMMLA{R} 1111 1011 0101 xxxx xxxx xxxx 000x xxxx */ /* SMMLS{R} 1111 1011 0110 xxxx xxxx xxxx 000x xxxx */ /* USADA8 1111 1011 0111 xxxx xxxx xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff8000c0, 0xfb000000, t32_emulate_rd8rn16rm0ra12_noflags, + DECODE_EMULATEX (0xff8000c0, 0xfb000000, PROBES_T32_MUL_ADD2, REGS(NOSPPC, NOSPPCX, NOSPPC, 0, NOSPPC)), /* Other unallocated instructions... */ @@ -547,7 +544,7 @@ static const union decode_item t32_table_1111_1011_1[] = { /* UMULL 1111 1011 1010 xxxx xxxx xxxx 0000 xxxx */ /* SMLAL 1111 1011 1100 xxxx xxxx xxxx 0000 xxxx */ /* UMLAL 1111 1011 1110 xxxx xxxx xxxx 0000 xxxx */ - DECODE_EMULATEX (0xff9000f0, 0xfb800000, t32_emulate_rdlo12rdhi8rn16rm0_noflags, + DECODE_EMULATEX (0xff9000f0, 0xfb800000, PROBES_T32_MUL_ADD_LONG, REGS(NOSPPC, NOSPPC, NOSPPC, 0, NOSPPC)), /* SDIV 1111 1011 1001 xxxx xxxx xxxx 1111 xxxx */ @@ -653,11 +650,11 @@ static const union decode_item t16_table_1011[] = { /* ADD (SP plus immediate) 1011 0000 0xxx xxxx */ /* SUB (SP minus immediate) 1011 0000 1xxx xxxx */ - DECODE_SIMULATE (0xff00, 0xb000, t16_simulate_add_sp_imm), + DECODE_SIMULATE (0xff00, 0xb000, PROBES_T16_ADD_SP), /* CBZ 1011 00x1 xxxx xxxx */ /* CBNZ 1011 10x1 xxxx xxxx */ - DECODE_SIMULATE (0xf500, 0xb100, t16_simulate_cbz), + DECODE_SIMULATE (0xf500, 0xb100, PROBES_T16_CBZ), /* SXTH 1011 0010 00xx xxxx */ /* SXTB 1011 0010 01xx xxxx */ @@ -668,12 +665,12 @@ static const union decode_item t16_table_1011[] = { /* ??? 1011 1010 10xx xxxx */ /* REVSH 1011 1010 11xx xxxx */ DECODE_REJECT (0xffc0, 0xba80), - DECODE_EMULATE (0xf500, 0xb000, t16_emulate_loregs_rwflags), + DECODE_EMULATE (0xf500, 0xb000, PROBES_T16_SIGN_EXTEND), /* PUSH 1011 010x xxxx xxxx */ - DECODE_CUSTOM (0xfe00, 0xb400, t16_decode_push), + DECODE_CUSTOM (0xfe00, 0xb400, PROBES_T16_PUSH), /* POP 1011 110x xxxx xxxx */ - DECODE_CUSTOM (0xfe00, 0xbc00, t16_decode_pop), + DECODE_CUSTOM (0xfe00, 0xbc00, PROBES_T16_POP), /* * If-Then, and hints @@ -683,15 +680,15 @@ static const union decode_item t16_table_1011[] = { /* YIELD 1011 1111 0001 0000 */ DECODE_OR (0xffff, 0xbf10), /* SEV 1011 1111 0100 0000 */ - DECODE_EMULATE (0xffff, 0xbf40, kprobe_emulate_none), + DECODE_EMULATE (0xffff, 0xbf40, PROBES_T16_SEV), /* NOP 1011 1111 0000 0000 */ /* WFE 1011 1111 0010 0000 */ /* WFI 1011 1111 0011 0000 */ - DECODE_SIMULATE (0xffcf, 0xbf00, kprobe_simulate_nop), + DECODE_SIMULATE (0xffcf, 0xbf00, PROBES_T16_WFE), /* Unassigned hints 1011 1111 xxxx 0000 */ DECODE_REJECT (0xff0f, 0xbf00), /* IT 1011 1111 xxxx xxxx */ - DECODE_CUSTOM (0xff00, 0xbf00, t16_decode_it), + DECODE_CUSTOM (0xff00, 0xbf00, PROBES_T16_IT), /* SETEND 1011 0110 010x xxxx */ /* CPS 1011 0110 011x xxxx */ @@ -708,7 +705,7 @@ const union decode_item kprobe_decode_thumb16_table[] = { */ /* CMP (immediate) 0010 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf800, 0x2800, t16_emulate_loregs_rwflags), + DECODE_EMULATE (0xf800, 0x2800, PROBES_T16_CMP), /* ADD (register) 0001 100x xxxx xxxx */ /* SUB (register) 0001 101x xxxx xxxx */ @@ -720,7 +717,7 @@ const union decode_item kprobe_decode_thumb16_table[] = { /* MOV (immediate) 0010 0xxx xxxx xxxx */ /* ADD (immediate, Thumb) 0011 0xxx xxxx xxxx */ /* SUB (immediate, Thumb) 0011 1xxx xxxx xxxx */ - DECODE_EMULATE (0xc000, 0x0000, t16_emulate_loregs_noitrwflags), + DECODE_EMULATE (0xc000, 0x0000, PROBES_T16_ADDSUB), /* * 16-bit Thumb data-processing instructions @@ -728,10 +725,10 @@ const union decode_item kprobe_decode_thumb16_table[] = { */ /* TST (register) 0100 0010 00xx xxxx */ - DECODE_EMULATE (0xffc0, 0x4200, t16_emulate_loregs_rwflags), + DECODE_EMULATE (0xffc0, 0x4200, PROBES_T16_CMP), /* CMP (register) 0100 0010 10xx xxxx */ /* CMN (register) 0100 0010 11xx xxxx */ - DECODE_EMULATE (0xff80, 0x4280, t16_emulate_loregs_rwflags), + DECODE_EMULATE (0xff80, 0x4280, PROBES_T16_CMP), /* AND (register) 0100 0000 00xx xxxx */ /* EOR (register) 0100 0000 01xx xxxx */ /* LSL (register) 0100 0000 10xx xxxx */ @@ -745,7 +742,7 @@ const union decode_item kprobe_decode_thumb16_table[] = { /* MUL 0100 0011 00xx xxxx */ /* BIC (register) 0100 0011 10xx xxxx */ /* MVN (register) 0100 0011 10xx xxxx */ - DECODE_EMULATE (0xfc00, 0x4000, t16_emulate_loregs_noitrwflags), + DECODE_EMULATE (0xfc00, 0x4000, PROBES_T16_LOGICAL), /* * Special data instructions and branch and exchange @@ -757,7 +754,7 @@ const union decode_item kprobe_decode_thumb16_table[] = { /* BX (register) 0100 0111 0xxx xxxx */ /* BLX (register) 0100 0111 1xxx xxxx */ - DECODE_SIMULATE (0xff00, 0x4700, t16_simulate_bxblx), + DECODE_SIMULATE (0xff00, 0x4700, PROBES_T16_BLX), /* ADD pc, pc 0100 0100 1111 1111 */ DECODE_REJECT (0xffff, 0x44ff), @@ -765,13 +762,13 @@ const union decode_item kprobe_decode_thumb16_table[] = { /* ADD (register) 0100 0100 xxxx xxxx */ /* CMP (register) 0100 0101 xxxx xxxx */ /* MOV (register) 0100 0110 xxxx xxxx */ - DECODE_CUSTOM (0xfc00, 0x4400, t16_decode_hiregs), + DECODE_CUSTOM (0xfc00, 0x4400, PROBES_T16_HIREGOPS), /* * Load from Literal Pool * LDR (literal) 0100 1xxx xxxx xxxx */ - DECODE_SIMULATE (0xf800, 0x4800, t16_simulate_ldr_literal), + DECODE_SIMULATE (0xf800, 0x4800, PROBES_T16_LDR_LIT), /* * 16-bit Thumb Load/store instructions @@ -792,20 +789,20 @@ const union decode_item kprobe_decode_thumb16_table[] = { /* LDR (immediate, Thumb) 0110 1xxx xxxx xxxx */ /* STRB (immediate, Thumb) 0111 0xxx xxxx xxxx */ /* LDRB (immediate, Thumb) 0111 1xxx xxxx xxxx */ - DECODE_EMULATE (0xc000, 0x4000, t16_emulate_loregs_rwflags), + DECODE_EMULATE (0xc000, 0x4000, PROBES_T16_LDRHSTRH), /* STRH (immediate, Thumb) 1000 0xxx xxxx xxxx */ /* LDRH (immediate, Thumb) 1000 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf000, 0x8000, t16_emulate_loregs_rwflags), + DECODE_EMULATE (0xf000, 0x8000, PROBES_T16_LDRHSTRH), /* STR (immediate, Thumb) 1001 0xxx xxxx xxxx */ /* LDR (immediate, Thumb) 1001 1xxx xxxx xxxx */ - DECODE_SIMULATE (0xf000, 0x9000, t16_simulate_ldrstr_sp_relative), + DECODE_SIMULATE (0xf000, 0x9000, PROBES_T16_LDRSTR), /* * Generate PC-/SP-relative address * ADR (literal) 1010 0xxx xxxx xxxx * ADD (SP plus immediate) 1010 1xxx xxxx xxxx */ - DECODE_SIMULATE (0xf000, 0xa000, t16_simulate_reladr), + DECODE_SIMULATE (0xf000, 0xa000, PROBES_T16_ADR), /* * Miscellaneous 16-bit instructions @@ -815,7 +812,7 @@ const union decode_item kprobe_decode_thumb16_table[] = { /* STM 1100 0xxx xxxx xxxx */ /* LDM 1100 1xxx xxxx xxxx */ - DECODE_EMULATE (0xf000, 0xc000, t16_emulate_loregs_rwflags), + DECODE_EMULATE (0xf000, 0xc000, PROBES_T16_LDMSTM), /* * Conditional branch, and Supervisor Call @@ -826,13 +823,13 @@ const union decode_item kprobe_decode_thumb16_table[] = { DECODE_REJECT (0xfe00, 0xde00), /* Conditional branch 1101 xxxx xxxx xxxx */ - DECODE_CUSTOM (0xf000, 0xd000, t16_decode_cond_branch), + DECODE_CUSTOM (0xf000, 0xd000, PROBES_T16_BRANCH_COND), /* * Unconditional branch * B 1110 0xxx xxxx xxxx */ - DECODE_SIMULATE (0xf800, 0xe000, t16_simulate_branch), + DECODE_SIMULATE (0xf800, 0xe000, PROBES_T16_BRANCH), DECODE_END }; @@ -862,17 +859,21 @@ static void __kprobes thumb32_singlestep(struct kprobe *p, struct pt_regs *regs) } enum kprobe_insn __kprobes -thumb16_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +thumb16_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const union decode_action *actions) { asi->insn_singlestep = thumb16_singlestep; asi->insn_check_cc = thumb_check_cc; - return kprobe_decode_insn(insn, asi, kprobe_decode_thumb16_table, true); + return kprobe_decode_insn(insn, asi, kprobe_decode_thumb16_table, true, + actions); } enum kprobe_insn __kprobes -thumb32_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +thumb32_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const union decode_action *actions) { asi->insn_singlestep = thumb32_singlestep; asi->insn_check_cc = thumb_check_cc; - return kprobe_decode_insn(insn, asi, kprobe_decode_thumb32_table, true); + return kprobe_decode_insn(insn, asi, kprobe_decode_thumb32_table, true, + actions); } diff --git a/arch/arm/kernel/probes-thumb.h b/arch/arm/kernel/probes-thumb.h index 98709c40b659..8d6b4eefa706 100644 --- a/arch/arm/kernel/probes-thumb.h +++ b/arch/arm/kernel/probes-thumb.h @@ -27,55 +27,61 @@ */ #define current_cond(cpsr) ((cpsr >> 12) & 0xf) -void __kprobes t16_simulate_bxblx(struct kprobe *p, struct pt_regs *regs); -void __kprobes t16_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs); -void __kprobes t16_simulate_ldrstr_sp_relative(struct kprobe *p, - struct pt_regs *regs); -void __kprobes t16_simulate_reladr(struct kprobe *p, struct pt_regs *regs); -void __kprobes t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs); -void __kprobes t16_simulate_cbz(struct kprobe *p, struct pt_regs *regs); -void __kprobes t16_simulate_it(struct kprobe *p, struct pt_regs *regs); -void __kprobes t16_singlestep_it(struct kprobe *p, struct pt_regs *regs); -enum kprobe_insn __kprobes t16_decode_it(kprobe_opcode_t insn, - struct arch_specific_insn *asi); -void __kprobes t16_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs); -enum kprobe_insn __kprobes t16_decode_cond_branch(kprobe_opcode_t insn, - struct arch_specific_insn *asi); -void __kprobes t16_simulate_branch(struct kprobe *p, struct pt_regs *regs); -void __kprobes t16_emulate_loregs_rwflags(struct kprobe *p, - struct pt_regs *regs); -void __kprobes t16_emulate_loregs_noitrwflags(struct kprobe *p, - struct pt_regs *regs); -void __kprobes t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs); -enum kprobe_insn __kprobes t16_decode_hiregs(kprobe_opcode_t insn, - struct arch_specific_insn *asi); -void __kprobes t16_emulate_push(struct kprobe *p, struct pt_regs *regs); -enum kprobe_insn __kprobes t16_decode_push(kprobe_opcode_t insn, - struct arch_specific_insn *asi); -void __kprobes t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs); -void __kprobes t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs); -enum kprobe_insn __kprobes t16_decode_pop(kprobe_opcode_t insn, - struct arch_specific_insn *asi); +enum probes_t32_action { + PROBES_T32_EMULATE_NONE, + PROBES_T32_SIMULATE_NOP, + PROBES_T32_LDMSTM, + PROBES_T32_LDRDSTRD, + PROBES_T32_TABLE_BRANCH, + PROBES_T32_TST, + PROBES_T32_CMP, + PROBES_T32_MOV, + PROBES_T32_ADDSUB, + PROBES_T32_LOGICAL, + PROBES_T32_ADDWSUBW_PC, + PROBES_T32_ADDWSUBW, + PROBES_T32_MOVW, + PROBES_T32_SAT, + PROBES_T32_BITFIELD, + PROBES_T32_SEV, + PROBES_T32_WFE, + PROBES_T32_MRS, + PROBES_T32_BRANCH_COND, + PROBES_T32_BRANCH, + PROBES_T32_PLDI, + PROBES_T32_LDR_LIT, + PROBES_T32_LDRSTR, + PROBES_T32_SIGN_EXTEND, + PROBES_T32_MEDIA, + PROBES_T32_REVERSE, + PROBES_T32_MUL_ADD, + PROBES_T32_MUL_ADD2, + PROBES_T32_MUL_ADD_LONG, + NUM_PROBES_T32_ACTIONS +}; -void __kprobes t32_simulate_table_branch(struct kprobe *p, - struct pt_regs *regs); -void __kprobes t32_simulate_mrs(struct kprobe *p, struct pt_regs *regs); -void __kprobes t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs); -enum kprobe_insn __kprobes t32_decode_cond_branch(kprobe_opcode_t insn, - struct arch_specific_insn *asi); -void __kprobes t32_simulate_branch(struct kprobe *p, struct pt_regs *regs); -void __kprobes t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs); -enum kprobe_insn __kprobes t32_decode_ldmstm(kprobe_opcode_t insn, - struct arch_specific_insn *asi); -void __kprobes t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs); -void __kprobes t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs); -void __kprobes t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, - struct pt_regs *regs); -void __kprobes t32_emulate_rd8pc16_noflags(struct kprobe *p, - struct pt_regs *regs); -void __kprobes t32_emulate_rd8rn16_noflags(struct kprobe *p, - struct pt_regs *regs); -void __kprobes t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, - struct pt_regs *regs); +enum probes_t16_action { + PROBES_T16_ADD_SP, + PROBES_T16_CBZ, + PROBES_T16_SIGN_EXTEND, + PROBES_T16_PUSH, + PROBES_T16_POP, + PROBES_T16_SEV, + PROBES_T16_WFE, + PROBES_T16_IT, + PROBES_T16_CMP, + PROBES_T16_ADDSUB, + PROBES_T16_LOGICAL, + PROBES_T16_BLX, + PROBES_T16_HIREGOPS, + PROBES_T16_LDR_LIT, + PROBES_T16_LDRHSTRH, + PROBES_T16_LDRSTR, + PROBES_T16_ADR, + PROBES_T16_LDMSTM, + PROBES_T16_BRANCH_COND, + PROBES_T16_BRANCH, + NUM_PROBES_T16_ACTIONS +}; #endif diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c index 3a63f8f83cf8..efd92c5b4a52 100644 --- a/arch/arm/kernel/probes.c +++ b/arch/arm/kernel/probes.c @@ -381,7 +381,8 @@ static const int decode_struct_sizes[NUM_DECODE_TYPES] = { */ int __kprobes kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - const union decode_item *table, bool thumb) + const union decode_item *table, bool thumb, + const union decode_action *actions) { const struct decode_header *h = (struct decode_header *)table; const struct decode_header *next; @@ -415,18 +416,18 @@ kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, case DECODE_TYPE_CUSTOM: { struct decode_custom *d = (struct decode_custom *)h; - return (*d->decoder.decoder)(insn, asi); + return actions[d->decoder.action].decoder(insn, asi, h); } case DECODE_TYPE_SIMULATE: { struct decode_simulate *d = (struct decode_simulate *)h; - asi->insn_handler = d->handler.handler; + asi->insn_handler = actions[d->handler.action].handler; return INSN_GOOD_NO_SLOT; } case DECODE_TYPE_EMULATE: { struct decode_emulate *d = (struct decode_emulate *)h; - asi->insn_handler = d->handler.handler; + asi->insn_handler = actions[d->handler.action].handler; set_emulated_insn(insn, asi, thumb); return INSN_GOOD; } diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h index 17f656011aa3..5554f161bdac 100644 --- a/arch/arm/kernel/probes.h +++ b/arch/arm/kernel/probes.h @@ -133,7 +133,8 @@ void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs); void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs); enum kprobe_insn __kprobes -kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi); +kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *h); /* * Test if load/store instructions writeback the address register. @@ -160,7 +161,7 @@ kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi); * {.bits = _type}, * {.bits = _mask}, * {.bits = _value}, - * {.handler = _handler}, + * {.action = _handler}, * * Initialising a specified member of the union means that the compiler * will produce a warning if the argument is of an incorrect type. @@ -173,19 +174,23 @@ kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi); * Instruction decoding jumps to parsing the new sub-table 'table'. * * DECODE_CUSTOM(mask, value, decoder) - * The custom function 'decoder' is called to the complete decoding - * of an instruction. + * The value of 'decoder' is used as an index into the array of + * action functions, and the retrieved decoder function is invoked + * to complete decoding of the instruction. * * DECODE_SIMULATE(mask, value, handler) - * Set the probes instruction handler to 'handler', this will be used - * to simulate the instruction when the probe is hit. Decoding returns - * with INSN_GOOD_NO_SLOT. + * The probes instruction handler is set to the value found by + * indexing into the action array using the value of 'handler'. This + * will be used to simulate the instruction when the probe is hit. + * Decoding returns with INSN_GOOD_NO_SLOT. * * DECODE_EMULATE(mask, value, handler) - * Set the probes instruction handler to 'handler', this will be used - * to emulate the instruction when the probe is hit. The modified - * instruction (see below) is placed in the probes instruction slot so it - * may be called by the emulation code. Decoding returns with INSN_GOOD. + * The probes instruction handler is set to the value found by + * indexing into the action array using the value of 'handler'. This + * will be used to emulate the instruction when the probe is hit. The + * modified instruction (see below) is placed in the probes instruction + * slot so it may be called by the emulation code. Decoding returns + * with INSN_GOOD. * * DECODE_REJECT(mask, value) * Instruction decoding fails with INSN_REJECTED @@ -238,7 +243,7 @@ kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi); * Here is a real example which matches ARM instructions of the form * "AND ,,, " * - * DECODE_EMULATEX (0x0e000090, 0x00000010, emulate_rd12rn16rm0rs8_rwflags, + * DECODE_EMULATEX (0x0e000090, 0x00000010, PROBES_DATA_PROCESSING_REG, * REGS(ANY, ANY, NOPC, 0, ANY)), * ^ ^ ^ ^ * Rn Rd Rs Rm @@ -249,7 +254,8 @@ kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi); * Decoding the instruction "AND R4, R5, R6, ASL R7" will be accepted and the * instruction will be modified to "AND R0, R2, R3, ASL R1" and then placed into * the kprobes instruction slot. This can then be called later by the handler - * function emulate_rd12rn16rm0rs8_rwflags in order to simulate the instruction. + * function emulate_rd12rn16rm0rs8_rwflags (a pointer to which is retrieved from + * the indicated slot in the action array), in order to simulate the instruction. */ enum decode_type { @@ -298,10 +304,17 @@ enum decode_reg_type { union decode_item { u32 bits; const union decode_item *table; - kprobe_insn_handler_t *handler; - kprobe_decode_insn_t *decoder; + int action; }; +typedef enum kprobe_insn (probes_custom_decode_t)(kprobe_opcode_t, + struct arch_specific_insn *, + const struct decode_header *); + +union decode_action { + kprobe_insn_handler_t *handler; + probes_custom_decode_t *decoder; +}; #define DECODE_END \ {.bits = DECODE_TYPE_END} @@ -336,7 +349,7 @@ struct decode_custom { #define DECODE_CUSTOM(_mask, _value, _decoder) \ DECODE_HEADER(DECODE_TYPE_CUSTOM, _mask, _value, 0), \ - {.decoder = (_decoder)} + {.action = (_decoder)} struct decode_simulate { @@ -346,7 +359,7 @@ struct decode_simulate { #define DECODE_SIMULATEX(_mask, _value, _handler, _regs) \ DECODE_HEADER(DECODE_TYPE_SIMULATE, _mask, _value, _regs), \ - {.handler = (_handler)} + {.action = (_handler)} #define DECODE_SIMULATE(_mask, _value, _handler) \ DECODE_SIMULATEX(_mask, _value, _handler, 0) @@ -359,7 +372,7 @@ struct decode_emulate { #define DECODE_EMULATEX(_mask, _value, _handler, _regs) \ DECODE_HEADER(DECODE_TYPE_EMULATE, _mask, _value, _regs), \ - {.handler = (_handler)} + {.action = (_handler)} #define DECODE_EMULATE(_mask, _value, _handler) \ DECODE_EMULATEX(_mask, _value, _handler, 0) @@ -384,14 +397,18 @@ struct decode_reject { #ifdef CONFIG_THUMB2_KERNEL extern const union decode_item kprobe_decode_thumb16_table[]; extern const union decode_item kprobe_decode_thumb32_table[]; +extern const union decode_action kprobes_t32_actions[]; +extern const union decode_action kprobes_t16_actions[]; #else extern const union decode_item kprobe_decode_arm_table[]; +extern const union decode_action kprobes_arm_actions[]; #endif extern kprobe_check_cc * const kprobe_condition_checks[16]; int kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, - const union decode_item *table, bool thumb16); + const union decode_item *table, bool thumb16, + const union decode_action *actions); #endif From 7579f4b3764337b39087d10496af0e741cbfe570 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Fri, 7 Mar 2014 11:19:32 -0500 Subject: [PATCH 53/89] ARM: Remove use of struct kprobe from generic probes code Change the generic ARM probes code to pass in the opcode and architecture-specific structure separately instead of using struct kprobe, so we do not pollute code being used only for uprobes or other non-kprobes instruction interpretation. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/include/asm/probes.h | 9 +- arch/arm/kernel/kprobes-arm.c | 61 +++++------ arch/arm/kernel/kprobes-common.c | 40 ++++--- arch/arm/kernel/kprobes-thumb.c | 175 +++++++++++++++---------------- arch/arm/kernel/kprobes.c | 2 +- arch/arm/kernel/probes-arm.c | 33 +++--- arch/arm/kernel/probes-arm.h | 15 ++- arch/arm/kernel/probes-thumb.c | 15 +-- arch/arm/kernel/probes.c | 13 ++- arch/arm/kernel/probes.h | 8 +- 10 files changed, 201 insertions(+), 170 deletions(-) diff --git a/arch/arm/include/asm/probes.h b/arch/arm/include/asm/probes.h index 737a9b310efc..4d014c4aa1e7 100644 --- a/arch/arm/include/asm/probes.h +++ b/arch/arm/include/asm/probes.h @@ -21,9 +21,14 @@ struct kprobe; -typedef void (kprobe_insn_handler_t)(struct kprobe *, struct pt_regs *); +struct arch_specific_insn; +typedef void (kprobe_insn_handler_t)(kprobe_opcode_t, + struct arch_specific_insn *, + struct pt_regs *); typedef unsigned long (kprobe_check_cc)(unsigned long); -typedef void (kprobe_insn_singlestep_t)(struct kprobe *, struct pt_regs *); +typedef void (kprobe_insn_singlestep_t)(kprobe_opcode_t, + struct arch_specific_insn *, + struct pt_regs *); typedef void (kprobe_insn_fn_t)(void); /* Architecture specific copy of original instruction. */ diff --git a/arch/arm/kernel/kprobes-arm.c b/arch/arm/kernel/kprobes-arm.c index 8ebd84c48867..4a232e682e5a 100644 --- a/arch/arm/kernel/kprobes-arm.c +++ b/arch/arm/kernel/kprobes-arm.c @@ -72,12 +72,11 @@ "mov pc, "reg" \n\t" #endif - static void __kprobes -emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) +emulate_ldrdstrd(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = (unsigned long)p->addr + 8; + unsigned long pc = regs->ARM_pc + 4; int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -92,7 +91,7 @@ emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) BLX("%[fn]") : "=r" (rtv), "=r" (rt2v), "=r" (rnv) : "0" (rtv), "1" (rt2v), "2" (rnv), "r" (rmv), - [fn] "r" (p->ainsn.insn_fn) + [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -103,10 +102,10 @@ emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_ldr(struct kprobe *p, struct pt_regs *regs) +emulate_ldr(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = (unsigned long)p->addr + 8; + unsigned long pc = regs->ARM_pc + 4; int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -119,7 +118,7 @@ emulate_ldr(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( BLX("%[fn]") : "=r" (rtv), "=r" (rnv) - : "1" (rnv), "r" (rmv), [fn] "r" (p->ainsn.insn_fn) + : "1" (rnv), "r" (rmv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -133,11 +132,11 @@ emulate_ldr(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_str(struct kprobe *p, struct pt_regs *regs) +emulate_str(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long rtpc = (unsigned long)p->addr + str_pc_offset; - unsigned long rnpc = (unsigned long)p->addr + 8; + unsigned long rtpc = regs->ARM_pc - 4 + str_pc_offset; + unsigned long rnpc = regs->ARM_pc + 4; int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -151,7 +150,7 @@ emulate_str(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( BLX("%[fn]") : "=r" (rnv) - : "r" (rtv), "0" (rnv), "r" (rmv), [fn] "r" (p->ainsn.insn_fn) + : "r" (rtv), "0" (rnv), "r" (rmv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -160,10 +159,10 @@ emulate_str(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) +emulate_rd12rn16rm0rs8_rwflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = (unsigned long)p->addr + 8; + unsigned long pc = regs->ARM_pc + 4; int rd = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -183,7 +182,7 @@ emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdv), [cpsr] "=r" (cpsr) : "0" (rdv), "r" (rnv), "r" (rmv), "r" (rsv), - "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -195,9 +194,9 @@ emulate_rd12rn16rm0rs8_rwflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) +emulate_rd12rn16rm0_rwflags_nopc(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -213,7 +212,7 @@ emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdv), [cpsr] "=r" (cpsr) : "0" (rdv), "r" (rnv), "r" (rmv), - "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -222,9 +221,10 @@ emulate_rd12rn16rm0_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) +emulate_rd16rn12rm0rs8_rwflags_nopc(kprobe_opcode_t insn, + struct arch_specific_insn *asi, + struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 16) & 0xf; int rn = (insn >> 12) & 0xf; int rm = insn & 0xf; @@ -242,7 +242,7 @@ emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdv), [cpsr] "=r" (cpsr) : "0" (rdv), "r" (rnv), "r" (rmv), "r" (rsv), - "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -251,9 +251,9 @@ emulate_rd16rn12rm0rs8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) +emulate_rd12rm0_noflags_nopc(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 12) & 0xf; int rm = insn & 0xf; @@ -263,7 +263,7 @@ emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( BLX("%[fn]") : "=r" (rdv) - : "0" (rdv), "r" (rmv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rdv), "r" (rmv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -271,9 +271,10 @@ emulate_rd12rm0_noflags_nopc(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) +emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(kprobe_opcode_t insn, + struct arch_specific_insn *asi, + struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rdlo = (insn >> 12) & 0xf; int rdhi = (insn >> 16) & 0xf; int rn = insn & 0xf; @@ -291,7 +292,7 @@ emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdlov), "=r" (rdhiv), [cpsr] "=r" (cpsr) : "0" (rdlov), "1" (rdhiv), "r" (rnv), "r" (rmv), - "2" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "2" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index 029b79c6face..abe03890f84d 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -17,9 +17,10 @@ #include "kprobes.h" -static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs) +static void __kprobes simulate_ldm1stm1(kprobe_opcode_t insn, + struct arch_specific_insn *asi, + struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rn = (insn >> 16) & 0xf; int lbit = insn & (1 << 20); int wbit = insn & (1 << 21); @@ -58,24 +59,31 @@ static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs) } } -static void __kprobes simulate_stm1_pc(struct kprobe *p, struct pt_regs *regs) +static void __kprobes simulate_stm1_pc(kprobe_opcode_t insn, + struct arch_specific_insn *asi, + struct pt_regs *regs) { - regs->ARM_pc = (long)p->addr + str_pc_offset; - simulate_ldm1stm1(p, regs); - regs->ARM_pc = (long)p->addr + 4; + unsigned long addr = regs->ARM_pc - 4; + + regs->ARM_pc = (long)addr + str_pc_offset; + simulate_ldm1stm1(insn, asi, regs); + regs->ARM_pc = (long)addr + 4; } -static void __kprobes simulate_ldm1_pc(struct kprobe *p, struct pt_regs *regs) +static void __kprobes simulate_ldm1_pc(kprobe_opcode_t insn, + struct arch_specific_insn *asi, + struct pt_regs *regs) { - simulate_ldm1stm1(p, regs); + simulate_ldm1stm1(insn, asi, regs); load_write_pc(regs->ARM_pc, regs); } static void __kprobes -emulate_generic_r0_12_noflags(struct kprobe *p, struct pt_regs *regs) +emulate_generic_r0_12_noflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { register void *rregs asm("r1") = regs; - register void *rfn asm("lr") = p->ainsn.insn_fn; + register void *rfn asm("lr") = asi->insn_fn; __asm__ __volatile__ ( "stmdb sp!, {%[regs], r11} \n\t" @@ -99,15 +107,19 @@ emulate_generic_r0_12_noflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -emulate_generic_r2_14_noflags(struct kprobe *p, struct pt_regs *regs) +emulate_generic_r2_14_noflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - emulate_generic_r0_12_noflags(p, (struct pt_regs *)(regs->uregs+2)); + emulate_generic_r0_12_noflags(insn, asi, + (struct pt_regs *)(regs->uregs+2)); } static void __kprobes -emulate_ldm_r3_15(struct kprobe *p, struct pt_regs *regs) +emulate_ldm_r3_15(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - emulate_generic_r0_12_noflags(p, (struct pt_regs *)(regs->uregs+3)); + emulate_generic_r0_12_noflags(insn, asi, + (struct pt_regs *)(regs->uregs+3)); load_write_pc(regs->ARM_pc, regs); } diff --git a/arch/arm/kernel/kprobes-thumb.c b/arch/arm/kernel/kprobes-thumb.c index d83f6092920a..adc08f8d4a1c 100644 --- a/arch/arm/kernel/kprobes-thumb.c +++ b/arch/arm/kernel/kprobes-thumb.c @@ -20,24 +20,13 @@ #define t32_emulate_rd8rn16rm0ra12_noflags \ t32_emulate_rdlo12rdhi8rn16rm0_noflags -/* - * Return the PC value for a probe in thumb code. - * This is the address of the probed instruction plus 4. - * We subtract one because the address will have bit zero set to indicate - * a pointer to thumb code. - */ -static inline unsigned long __kprobes thumb_probe_pc(struct kprobe *p) -{ - return (unsigned long)p->addr - 1 + 4; -} - /* t32 thumb actions */ static void __kprobes -t32_simulate_table_branch(struct kprobe *p, struct pt_regs *regs) +t32_simulate_table_branch(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -54,19 +43,19 @@ t32_simulate_table_branch(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_simulate_mrs(struct kprobe *p, struct pt_regs *regs) +t32_simulate_mrs(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 8) & 0xf; unsigned long mask = 0xf8ff03df; /* Mask out execution state */ regs->uregs[rd] = regs->ARM_cpsr & mask; } static void __kprobes -t32_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) +t32_simulate_cond_branch(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc; long offset = insn & 0x7ff; /* imm11 */ offset += (insn & 0x003f0000) >> 5; /* imm6 */ @@ -88,10 +77,10 @@ t32_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) +t32_simulate_branch(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc; long offset = insn & 0x7ff; /* imm11 */ offset += (insn & 0x03ff0000) >> 5; /* imm10 */ @@ -104,7 +93,7 @@ t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) if (insn & (1 << 14)) { /* BL or BLX */ - regs->ARM_lr = (unsigned long)p->addr + 4; + regs->ARM_lr = regs->ARM_pc | 1; if (!(insn & (1 << 12))) { /* BLX so switch to ARM mode */ regs->ARM_cpsr &= ~PSR_T_BIT; @@ -116,10 +105,10 @@ t32_simulate_branch(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) +t32_simulate_ldr_literal(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long addr = thumb_probe_pc(p) & ~3; + unsigned long addr = regs->ARM_pc & ~3; int rt = (insn >> 12) & 0xf; unsigned long rtv; @@ -168,10 +157,10 @@ t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) +t32_emulate_ldrdstrd(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p) & ~3; + unsigned long pc = regs->ARM_pc & ~3; int rt1 = (insn >> 12) & 0xf; int rt2 = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; @@ -184,7 +173,7 @@ t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( "blx %[fn]" : "=r" (rt1v), "=r" (rt2v), "=r" (rnv) - : "0" (rt1v), "1" (rt2v), "2" (rnv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rt1v), "1" (rt2v), "2" (rnv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -195,9 +184,9 @@ t32_emulate_ldrdstrd(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) +t32_emulate_ldrstr(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -209,7 +198,7 @@ t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( "blx %[fn]" : "=r" (rtv), "=r" (rnv) - : "0" (rtv), "1" (rnv), "r" (rmv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rtv), "1" (rnv), "r" (rmv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -221,9 +210,9 @@ t32_emulate_ldrstr(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) +t32_emulate_rd8rn16rm0_rwflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; int rm = insn & 0xf; @@ -239,7 +228,7 @@ t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) "mrs %[cpsr], cpsr \n\t" : "=r" (rdv), [cpsr] "=r" (cpsr) : "0" (rdv), "r" (rnv), "r" (rmv), - "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -248,10 +237,10 @@ t32_emulate_rd8rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) +t32_emulate_rd8pc16_noflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc; int rd = (insn >> 8) & 0xf; register unsigned long rdv asm("r1") = regs->uregs[rd]; @@ -260,7 +249,7 @@ t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( "blx %[fn]" : "=r" (rdv) - : "0" (rdv), "r" (rnv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rdv), "r" (rnv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -268,9 +257,9 @@ t32_emulate_rd8pc16_noflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) +t32_emulate_rd8rn16_noflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; @@ -280,7 +269,7 @@ t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) __asm__ __volatile__ ( "blx %[fn]" : "=r" (rdv) - : "0" (rdv), "r" (rnv), [fn] "r" (p->ainsn.insn_fn) + : "0" (rdv), "r" (rnv), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -288,9 +277,10 @@ t32_emulate_rd8rn16_noflags(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) +t32_emulate_rdlo12rdhi8rn16rm0_noflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, + struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rdlo = (insn >> 12) & 0xf; int rdhi = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; @@ -305,7 +295,7 @@ t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) "blx %[fn]" : "=r" (rdlov), "=r" (rdhiv) : "0" (rdlov), "1" (rdhiv), "r" (rnv), "r" (rmv), - [fn] "r" (p->ainsn.insn_fn) + [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -315,33 +305,33 @@ t32_emulate_rdlo12rdhi8rn16rm0_noflags(struct kprobe *p, struct pt_regs *regs) /* t16 thumb actions */ static void __kprobes -t16_simulate_bxblx(struct kprobe *p, struct pt_regs *regs) +t16_simulate_bxblx(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; int rm = (insn >> 3) & 0xf; unsigned long rmv = (rm == 15) ? pc : regs->uregs[rm]; if (insn & (1 << 7)) /* BLX ? */ - regs->ARM_lr = (unsigned long)p->addr + 2; + regs->ARM_lr = regs->ARM_pc | 1; bx_write_pc(rmv, regs); } static void __kprobes -t16_simulate_ldr_literal(struct kprobe *p, struct pt_regs *regs) +t16_simulate_ldr_literal(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long* base = (unsigned long *)(thumb_probe_pc(p) & ~3); + unsigned long *base = (unsigned long *)((regs->ARM_pc + 2) & ~3); long index = insn & 0xff; int rt = (insn >> 8) & 0x7; regs->uregs[rt] = base[index]; } static void __kprobes -t16_simulate_ldrstr_sp_relative(struct kprobe *p, struct pt_regs *regs) +t16_simulate_ldrstr_sp_relative(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; unsigned long* base = (unsigned long *)regs->ARM_sp; long index = insn & 0xff; int rt = (insn >> 8) & 0x7; @@ -352,20 +342,20 @@ t16_simulate_ldrstr_sp_relative(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t16_simulate_reladr(struct kprobe *p, struct pt_regs *regs) +t16_simulate_reladr(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; unsigned long base = (insn & 0x800) ? regs->ARM_sp - : (thumb_probe_pc(p) & ~3); + : ((regs->ARM_pc + 2) & ~3); long offset = insn & 0xff; int rt = (insn >> 8) & 0x7; regs->uregs[rt] = base + offset * 4; } static void __kprobes -t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs) +t16_simulate_add_sp_imm(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; long imm = insn & 0x7f; if (insn & 0x80) /* SUB */ regs->ARM_sp -= imm * 4; @@ -374,21 +364,22 @@ t16_simulate_add_sp_imm(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t16_simulate_cbz(struct kprobe *p, struct pt_regs *regs) +t16_simulate_cbz(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rn = insn & 0x7; kprobe_opcode_t nonzero = regs->uregs[rn] ? insn : ~insn; if (nonzero & 0x800) { long i = insn & 0x200; long imm5 = insn & 0xf8; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; regs->ARM_pc = pc + (i >> 3) + (imm5 >> 2); } } static void __kprobes -t16_simulate_it(struct kprobe *p, struct pt_regs *regs) +t16_simulate_it(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { /* * The 8 IT state bits are split into two parts in CPSR: @@ -396,7 +387,6 @@ t16_simulate_it(struct kprobe *p, struct pt_regs *regs) * ITSTATE<7:2> are in CPSR<15:10> * The new IT state is in the lower byte of insn. */ - kprobe_opcode_t insn = p->opcode; unsigned long cpsr = regs->ARM_cpsr; cpsr &= ~PSR_IT_MASK; cpsr |= (insn & 0xfc) << 8; @@ -405,10 +395,11 @@ t16_simulate_it(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t16_singlestep_it(struct kprobe *p, struct pt_regs *regs) +t16_singlestep_it(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 2; - t16_simulate_it(p, regs); + t16_simulate_it(insn, asi, regs); } static enum kprobe_insn __kprobes @@ -420,10 +411,10 @@ t16_decode_it(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t16_simulate_cond_branch(struct kprobe *p, struct pt_regs *regs) +t16_simulate_cond_branch(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; long offset = insn & 0x7f; offset -= insn & 0x80; /* Apply sign bit */ regs->ARM_pc = pc + (offset * 2); @@ -440,17 +431,18 @@ t16_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t16_simulate_branch(struct kprobe *p, struct pt_regs *regs) +t16_simulate_branch(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; long offset = insn & 0x3ff; offset -= insn & 0x400; /* Apply sign bit */ regs->ARM_pc = pc + (offset * 2); } static unsigned long __kprobes -t16_emulate_loregs(struct kprobe *p, struct pt_regs *regs) +t16_emulate_loregs(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long oldcpsr = regs->ARM_cpsr; unsigned long newcpsr; @@ -463,7 +455,7 @@ t16_emulate_loregs(struct kprobe *p, struct pt_regs *regs) "mrs %[newcpsr], cpsr \n\t" : [newcpsr] "=r" (newcpsr) : [oldcpsr] "r" (oldcpsr), [regs] "r" (regs), - [fn] "r" (p->ainsn.insn_fn) + [fn] "r" (asi->insn_fn) : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "lr", "memory", "cc" ); @@ -472,24 +464,26 @@ t16_emulate_loregs(struct kprobe *p, struct pt_regs *regs) } static void __kprobes -t16_emulate_loregs_rwflags(struct kprobe *p, struct pt_regs *regs) +t16_emulate_loregs_rwflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - regs->ARM_cpsr = t16_emulate_loregs(p, regs); + regs->ARM_cpsr = t16_emulate_loregs(insn, asi, regs); } static void __kprobes -t16_emulate_loregs_noitrwflags(struct kprobe *p, struct pt_regs *regs) +t16_emulate_loregs_noitrwflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - unsigned long cpsr = t16_emulate_loregs(p, regs); + unsigned long cpsr = t16_emulate_loregs(insn, asi, regs); if (!in_it_block(cpsr)) regs->ARM_cpsr = cpsr; } static void __kprobes -t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) +t16_emulate_hiregs(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - unsigned long pc = thumb_probe_pc(p); + unsigned long pc = regs->ARM_pc + 2; int rdn = (insn & 0x7) | ((insn & 0x80) >> 4); int rm = (insn >> 3) & 0xf; @@ -505,7 +499,7 @@ t16_emulate_hiregs(struct kprobe *p, struct pt_regs *regs) "blx %[fn] \n\t" "mrs %[cpsr], cpsr \n\t" : "=r" (rdnv), [cpsr] "=r" (cpsr) - : "0" (rdnv), "r" (rmv), "1" (cpsr), [fn] "r" (p->ainsn.insn_fn) + : "0" (rdnv), "r" (rmv), "1" (cpsr), [fn] "r" (asi->insn_fn) : "lr", "memory", "cc" ); @@ -528,7 +522,8 @@ t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t16_emulate_push(struct kprobe *p, struct pt_regs *regs) +t16_emulate_push(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( "ldr r9, [%[regs], #13*4] \n\t" @@ -537,7 +532,7 @@ t16_emulate_push(struct kprobe *p, struct pt_regs *regs) "blx %[fn] \n\t" "str r9, [%[regs], #13*4] \n\t" : - : [regs] "r" (regs), [fn] "r" (p->ainsn.insn_fn) + : [regs] "r" (regs), [fn] "r" (asi->insn_fn) : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "lr", "memory", "cc" ); @@ -559,7 +554,8 @@ t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs) +t16_emulate_pop_nopc(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( "ldr r9, [%[regs], #13*4] \n\t" @@ -568,14 +564,15 @@ t16_emulate_pop_nopc(struct kprobe *p, struct pt_regs *regs) "stmia %[regs], {r0-r7} \n\t" "str r9, [%[regs], #13*4] \n\t" : - : [regs] "r" (regs), [fn] "r" (p->ainsn.insn_fn) + : [regs] "r" (regs), [fn] "r" (asi->insn_fn) : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r9", "lr", "memory", "cc" ); } static void __kprobes -t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) +t16_emulate_pop_pc(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { register unsigned long pc asm("r8"); @@ -586,7 +583,7 @@ t16_emulate_pop_pc(struct kprobe *p, struct pt_regs *regs) "stmia %[regs], {r0-r7} \n\t" "str r9, [%[regs], #13*4] \n\t" : "=r" (pc) - : [regs] "r" (regs), [fn] "r" (p->ainsn.insn_fn) + : [regs] "r" (regs), [fn] "r" (asi->insn_fn) : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r9", "lr", "memory", "cc" ); diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c index a757c3c22381..b4a3028edffe 100644 --- a/arch/arm/kernel/kprobes.c +++ b/arch/arm/kernel/kprobes.c @@ -204,7 +204,7 @@ singlestep_skip(struct kprobe *p, struct pt_regs *regs) static inline void __kprobes singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) { - p->ainsn.insn_singlestep(p, regs); + p->ainsn.insn_singlestep(p->opcode, &p->ainsn, regs); } /* diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c index 496e0e913fa6..36002e5d0f0e 100644 --- a/arch/arm/kernel/probes-arm.c +++ b/arch/arm/kernel/probes-arm.c @@ -19,9 +19,8 @@ #include #include #include -#include -#include "kprobes.h" +#include "probes.h" #include "probes-arm.h" #define sign_extend(x, signbit) ((x) | (0 - ((x) & (1 << (signbit))))) @@ -58,10 +57,10 @@ * read and write of flags. */ -void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs) +void __kprobes simulate_bbl(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - long iaddr = (long)p->addr; + long iaddr = (long) regs->ARM_pc - 4; int disp = branch_displacement(insn); if (insn & (1 << 24)) @@ -70,10 +69,10 @@ void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs) regs->ARM_pc = iaddr + 8 + disp; } -void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs) +void __kprobes simulate_blx1(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; - long iaddr = (long)p->addr; + long iaddr = (long) regs->ARM_pc - 4; int disp = branch_displacement(insn); regs->ARM_lr = iaddr + 4; @@ -81,14 +80,14 @@ void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr |= PSR_T_BIT; } -void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs) +void __kprobes simulate_blx2bx(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rm = insn & 0xf; long rmv = regs->uregs[rm]; if (insn & (1 << 5)) - regs->ARM_lr = (long)p->addr + 4; + regs->ARM_lr = (long) regs->ARM_pc; regs->ARM_pc = rmv & ~0x1; regs->ARM_cpsr &= ~PSR_T_BIT; @@ -96,15 +95,16 @@ void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs) regs->ARM_cpsr |= PSR_T_BIT; } -void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs) +void __kprobes simulate_mrs(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { - kprobe_opcode_t insn = p->opcode; int rd = (insn >> 12) & 0xf; unsigned long mask = 0xf8ff03df; /* Mask out execution state */ regs->uregs[rd] = regs->ARM_cpsr & mask; } -void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs) +void __kprobes simulate_mov_ipsp(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { regs->uregs[12] = regs->uregs[13]; } @@ -704,10 +704,11 @@ const union decode_item kprobe_decode_arm_table[] = { EXPORT_SYMBOL_GPL(kprobe_decode_arm_table); #endif -static void __kprobes arm_singlestep(struct kprobe *p, struct pt_regs *regs) +static void __kprobes arm_singlestep(kprobe_opcode_t insn, + struct arch_specific_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 4; - p->ainsn.insn_handler(p, regs); + asi->insn_handler(insn, asi, regs); } /* Return: diff --git a/arch/arm/kernel/probes-arm.h b/arch/arm/kernel/probes-arm.h index ef3089419a0b..d0ac8a42caa0 100644 --- a/arch/arm/kernel/probes-arm.h +++ b/arch/arm/kernel/probes-arm.h @@ -53,10 +53,15 @@ enum probes_arm_action { NUM_PROBES_ARM_ACTIONS }; -void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs); -void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs); -void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs); -void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs); -void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs); +void __kprobes simulate_bbl(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, struct pt_regs *regs); +void __kprobes simulate_blx1(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, struct pt_regs *regs); +void __kprobes simulate_blx2bx(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, struct pt_regs *regs); +void __kprobes simulate_mrs(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, struct pt_regs *regs); +void __kprobes simulate_mov_ipsp(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, struct pt_regs *regs); #endif diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c index 2abe8ceeb670..aa3176da1b29 100644 --- a/arch/arm/kernel/probes-thumb.c +++ b/arch/arm/kernel/probes-thumb.c @@ -10,10 +10,9 @@ #include #include -#include #include -#include "kprobes.h" +#include "probes.h" #include "probes-thumb.h" @@ -844,17 +843,21 @@ static unsigned long __kprobes thumb_check_cc(unsigned long cpsr) return true; } -static void __kprobes thumb16_singlestep(struct kprobe *p, struct pt_regs *regs) +static void __kprobes thumb16_singlestep(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, + struct pt_regs *regs) { regs->ARM_pc += 2; - p->ainsn.insn_handler(p, regs); + asi->insn_handler(opcode, asi, regs); regs->ARM_cpsr = it_advance(regs->ARM_cpsr); } -static void __kprobes thumb32_singlestep(struct kprobe *p, struct pt_regs *regs) +static void __kprobes thumb32_singlestep(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, + struct pt_regs *regs) { regs->ARM_pc += 4; - p->ainsn.insn_handler(p, regs); + asi->insn_handler(opcode, asi, regs); regs->ARM_cpsr = it_advance(regs->ARM_cpsr); } diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c index efd92c5b4a52..f2ab8856ba2b 100644 --- a/arch/arm/kernel/probes.c +++ b/arch/arm/kernel/probes.c @@ -13,12 +13,11 @@ #include #include -#include #include #include #include -#include "kprobes.h" +#include "probes.h" #ifndef find_str_pc_offset @@ -176,13 +175,17 @@ kprobe_check_cc * const kprobe_condition_checks[16] = { }; -void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs) +void __kprobes kprobe_simulate_nop(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, + struct pt_regs *regs) { } -void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs) +void __kprobes kprobe_emulate_none(kprobe_opcode_t opcode, + struct arch_specific_insn *asi, + struct pt_regs *regs) { - p->ainsn.insn_fn(); + asi->insn_fn(); } /* diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h index 5554f161bdac..efea63c02742 100644 --- a/arch/arm/kernel/probes.h +++ b/arch/arm/kernel/probes.h @@ -22,6 +22,7 @@ #include #include #include +#include "kprobes.h" #if __LINUX_ARM_ARCH__ >= 7 @@ -37,6 +38,7 @@ void __init find_str_pc_offset(void); #endif +struct decode_header; /* * Update ITSTATE after normal execution of an IT block instruction. @@ -129,8 +131,10 @@ static inline void __kprobes alu_write_pc(long pcv, struct pt_regs *regs) } -void __kprobes kprobe_simulate_nop(struct kprobe *p, struct pt_regs *regs); -void __kprobes kprobe_emulate_none(struct kprobe *p, struct pt_regs *regs); +void __kprobes kprobe_simulate_nop(kprobe_opcode_t, struct arch_specific_insn *, + struct pt_regs *regs); +void __kprobes kprobe_emulate_none(kprobe_opcode_t, struct arch_specific_insn *, + struct pt_regs *regs); enum kprobe_insn __kprobes kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, From f145d664df502585618b12ed68c681f82153e02a Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Wed, 5 Mar 2014 21:17:23 -0500 Subject: [PATCH 54/89] ARM: Make the kprobes condition_check symbol names more generic In preparation for sharing the ARM kprobes instruction interpreting code with uprobes, make the symbols names less kprobes-specific. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/include/asm/probes.h | 11 ++--- arch/arm/kernel/kprobes-arm.c | 16 +++---- arch/arm/kernel/kprobes-common.c | 14 +++--- arch/arm/kernel/kprobes-thumb.c | 76 ++++++++++++++++---------------- arch/arm/kernel/kprobes.h | 8 ++-- arch/arm/kernel/probes-arm.c | 16 +++---- arch/arm/kernel/probes-arm.h | 10 ++--- arch/arm/kernel/probes-thumb.c | 10 ++--- arch/arm/kernel/probes.c | 22 ++++----- arch/arm/kernel/probes.h | 12 ++--- 10 files changed, 98 insertions(+), 97 deletions(-) diff --git a/arch/arm/include/asm/probes.h b/arch/arm/include/asm/probes.h index 4d014c4aa1e7..c4acf6c8a2d4 100644 --- a/arch/arm/include/asm/probes.h +++ b/arch/arm/include/asm/probes.h @@ -20,22 +20,23 @@ #define _ASM_PROBES_H struct kprobe; +typedef u32 probes_opcode_t; struct arch_specific_insn; -typedef void (kprobe_insn_handler_t)(kprobe_opcode_t, +typedef void (kprobe_insn_handler_t)(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *); -typedef unsigned long (kprobe_check_cc)(unsigned long); -typedef void (kprobe_insn_singlestep_t)(kprobe_opcode_t, +typedef unsigned long (probes_check_cc)(unsigned long); +typedef void (kprobe_insn_singlestep_t)(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *); typedef void (kprobe_insn_fn_t)(void); /* Architecture specific copy of original instruction. */ struct arch_specific_insn { - kprobe_opcode_t *insn; + probes_opcode_t *insn; kprobe_insn_handler_t *insn_handler; - kprobe_check_cc *insn_check_cc; + probes_check_cc *insn_check_cc; kprobe_insn_singlestep_t *insn_singlestep; kprobe_insn_fn_t *insn_fn; }; diff --git a/arch/arm/kernel/kprobes-arm.c b/arch/arm/kernel/kprobes-arm.c index 4a232e682e5a..09a7b1e19c14 100644 --- a/arch/arm/kernel/kprobes-arm.c +++ b/arch/arm/kernel/kprobes-arm.c @@ -73,7 +73,7 @@ #endif static void __kprobes -emulate_ldrdstrd(kprobe_opcode_t insn, +emulate_ldrdstrd(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 4; @@ -102,7 +102,7 @@ emulate_ldrdstrd(kprobe_opcode_t insn, } static void __kprobes -emulate_ldr(kprobe_opcode_t insn, +emulate_ldr(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 4; @@ -132,7 +132,7 @@ emulate_ldr(kprobe_opcode_t insn, } static void __kprobes -emulate_str(kprobe_opcode_t insn, +emulate_str(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long rtpc = regs->ARM_pc - 4 + str_pc_offset; @@ -159,7 +159,7 @@ emulate_str(kprobe_opcode_t insn, } static void __kprobes -emulate_rd12rn16rm0rs8_rwflags(kprobe_opcode_t insn, +emulate_rd12rn16rm0rs8_rwflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 4; @@ -194,7 +194,7 @@ emulate_rd12rn16rm0rs8_rwflags(kprobe_opcode_t insn, } static void __kprobes -emulate_rd12rn16rm0_rwflags_nopc(kprobe_opcode_t insn, +emulate_rd12rn16rm0_rwflags_nopc(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rd = (insn >> 12) & 0xf; @@ -221,7 +221,7 @@ emulate_rd12rn16rm0_rwflags_nopc(kprobe_opcode_t insn, } static void __kprobes -emulate_rd16rn12rm0rs8_rwflags_nopc(kprobe_opcode_t insn, +emulate_rd16rn12rm0rs8_rwflags_nopc(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { @@ -251,7 +251,7 @@ emulate_rd16rn12rm0rs8_rwflags_nopc(kprobe_opcode_t insn, } static void __kprobes -emulate_rd12rm0_noflags_nopc(kprobe_opcode_t insn, +emulate_rd12rm0_noflags_nopc(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rd = (insn >> 12) & 0xf; @@ -271,7 +271,7 @@ emulate_rd12rm0_noflags_nopc(kprobe_opcode_t insn, } static void __kprobes -emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(kprobe_opcode_t insn, +emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index abe03890f84d..08b55605c1ec 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -17,7 +17,7 @@ #include "kprobes.h" -static void __kprobes simulate_ldm1stm1(kprobe_opcode_t insn, +static void __kprobes simulate_ldm1stm1(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { @@ -59,7 +59,7 @@ static void __kprobes simulate_ldm1stm1(kprobe_opcode_t insn, } } -static void __kprobes simulate_stm1_pc(kprobe_opcode_t insn, +static void __kprobes simulate_stm1_pc(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { @@ -70,7 +70,7 @@ static void __kprobes simulate_stm1_pc(kprobe_opcode_t insn, regs->ARM_pc = (long)addr + 4; } -static void __kprobes simulate_ldm1_pc(kprobe_opcode_t insn, +static void __kprobes simulate_ldm1_pc(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { @@ -79,7 +79,7 @@ static void __kprobes simulate_ldm1_pc(kprobe_opcode_t insn, } static void __kprobes -emulate_generic_r0_12_noflags(kprobe_opcode_t insn, +emulate_generic_r0_12_noflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { register void *rregs asm("r1") = regs; @@ -107,7 +107,7 @@ emulate_generic_r0_12_noflags(kprobe_opcode_t insn, } static void __kprobes -emulate_generic_r2_14_noflags(kprobe_opcode_t insn, +emulate_generic_r2_14_noflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { emulate_generic_r0_12_noflags(insn, asi, @@ -115,7 +115,7 @@ emulate_generic_r2_14_noflags(kprobe_opcode_t insn, } static void __kprobes -emulate_ldm_r3_15(kprobe_opcode_t insn, +emulate_ldm_r3_15(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { emulate_generic_r0_12_noflags(insn, asi, @@ -124,7 +124,7 @@ emulate_ldm_r3_15(kprobe_opcode_t insn, } enum kprobe_insn __kprobes -kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, +kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *h) { kprobe_insn_handler_t *handler = 0; diff --git a/arch/arm/kernel/kprobes-thumb.c b/arch/arm/kernel/kprobes-thumb.c index adc08f8d4a1c..610d6932e9bf 100644 --- a/arch/arm/kernel/kprobes-thumb.c +++ b/arch/arm/kernel/kprobes-thumb.c @@ -23,7 +23,7 @@ /* t32 thumb actions */ static void __kprobes -t32_simulate_table_branch(kprobe_opcode_t insn, +t32_simulate_table_branch(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc; @@ -43,7 +43,7 @@ t32_simulate_table_branch(kprobe_opcode_t insn, } static void __kprobes -t32_simulate_mrs(kprobe_opcode_t insn, +t32_simulate_mrs(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rd = (insn >> 8) & 0xf; @@ -52,7 +52,7 @@ t32_simulate_mrs(kprobe_opcode_t insn, } static void __kprobes -t32_simulate_cond_branch(kprobe_opcode_t insn, +t32_simulate_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc; @@ -67,17 +67,17 @@ t32_simulate_cond_branch(kprobe_opcode_t insn, } static enum kprobe_insn __kprobes -t32_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi, +t32_decode_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { int cc = (insn >> 22) & 0xf; - asi->insn_check_cc = kprobe_condition_checks[cc]; + asi->insn_check_cc = probes_condition_checks[cc]; asi->insn_handler = t32_simulate_cond_branch; return INSN_GOOD_NO_SLOT; } static void __kprobes -t32_simulate_branch(kprobe_opcode_t insn, +t32_simulate_branch(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc; @@ -105,7 +105,7 @@ t32_simulate_branch(kprobe_opcode_t insn, } static void __kprobes -t32_simulate_ldr_literal(kprobe_opcode_t insn, +t32_simulate_ldr_literal(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long addr = regs->ARM_pc & ~3; @@ -143,7 +143,7 @@ t32_simulate_ldr_literal(kprobe_opcode_t insn, } static enum kprobe_insn __kprobes -t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, +t32_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { enum kprobe_insn ret = kprobe_decode_ldmstm(insn, asi, d); @@ -157,7 +157,7 @@ t32_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t32_emulate_ldrdstrd(kprobe_opcode_t insn, +t32_emulate_ldrdstrd(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc & ~3; @@ -184,7 +184,7 @@ t32_emulate_ldrdstrd(kprobe_opcode_t insn, } static void __kprobes -t32_emulate_ldrstr(kprobe_opcode_t insn, +t32_emulate_ldrstr(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rt = (insn >> 12) & 0xf; @@ -210,7 +210,7 @@ t32_emulate_ldrstr(kprobe_opcode_t insn, } static void __kprobes -t32_emulate_rd8rn16rm0_rwflags(kprobe_opcode_t insn, +t32_emulate_rd8rn16rm0_rwflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rd = (insn >> 8) & 0xf; @@ -237,7 +237,7 @@ t32_emulate_rd8rn16rm0_rwflags(kprobe_opcode_t insn, } static void __kprobes -t32_emulate_rd8pc16_noflags(kprobe_opcode_t insn, +t32_emulate_rd8pc16_noflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc; @@ -257,7 +257,7 @@ t32_emulate_rd8pc16_noflags(kprobe_opcode_t insn, } static void __kprobes -t32_emulate_rd8rn16_noflags(kprobe_opcode_t insn, +t32_emulate_rd8rn16_noflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rd = (insn >> 8) & 0xf; @@ -277,7 +277,7 @@ t32_emulate_rd8rn16_noflags(kprobe_opcode_t insn, } static void __kprobes -t32_emulate_rdlo12rdhi8rn16rm0_noflags(kprobe_opcode_t insn, +t32_emulate_rdlo12rdhi8rn16rm0_noflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { @@ -305,7 +305,7 @@ t32_emulate_rdlo12rdhi8rn16rm0_noflags(kprobe_opcode_t insn, /* t16 thumb actions */ static void __kprobes -t16_simulate_bxblx(kprobe_opcode_t insn, +t16_simulate_bxblx(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 2; @@ -319,7 +319,7 @@ t16_simulate_bxblx(kprobe_opcode_t insn, } static void __kprobes -t16_simulate_ldr_literal(kprobe_opcode_t insn, +t16_simulate_ldr_literal(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long *base = (unsigned long *)((regs->ARM_pc + 2) & ~3); @@ -329,7 +329,7 @@ t16_simulate_ldr_literal(kprobe_opcode_t insn, } static void __kprobes -t16_simulate_ldrstr_sp_relative(kprobe_opcode_t insn, +t16_simulate_ldrstr_sp_relative(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long* base = (unsigned long *)regs->ARM_sp; @@ -342,7 +342,7 @@ t16_simulate_ldrstr_sp_relative(kprobe_opcode_t insn, } static void __kprobes -t16_simulate_reladr(kprobe_opcode_t insn, +t16_simulate_reladr(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long base = (insn & 0x800) ? regs->ARM_sp @@ -353,7 +353,7 @@ t16_simulate_reladr(kprobe_opcode_t insn, } static void __kprobes -t16_simulate_add_sp_imm(kprobe_opcode_t insn, +t16_simulate_add_sp_imm(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { long imm = insn & 0x7f; @@ -364,11 +364,11 @@ t16_simulate_add_sp_imm(kprobe_opcode_t insn, } static void __kprobes -t16_simulate_cbz(kprobe_opcode_t insn, +t16_simulate_cbz(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rn = insn & 0x7; - kprobe_opcode_t nonzero = regs->uregs[rn] ? insn : ~insn; + probes_opcode_t nonzero = regs->uregs[rn] ? insn : ~insn; if (nonzero & 0x800) { long i = insn & 0x200; long imm5 = insn & 0xf8; @@ -378,7 +378,7 @@ t16_simulate_cbz(kprobe_opcode_t insn, } static void __kprobes -t16_simulate_it(kprobe_opcode_t insn, +t16_simulate_it(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { /* @@ -395,7 +395,7 @@ t16_simulate_it(kprobe_opcode_t insn, } static void __kprobes -t16_singlestep_it(kprobe_opcode_t insn, +t16_singlestep_it(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 2; @@ -403,7 +403,7 @@ t16_singlestep_it(kprobe_opcode_t insn, } static enum kprobe_insn __kprobes -t16_decode_it(kprobe_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_it(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { asi->insn_singlestep = t16_singlestep_it; @@ -411,7 +411,7 @@ t16_decode_it(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t16_simulate_cond_branch(kprobe_opcode_t insn, +t16_simulate_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 2; @@ -421,17 +421,17 @@ t16_simulate_cond_branch(kprobe_opcode_t insn, } static enum kprobe_insn __kprobes -t16_decode_cond_branch(kprobe_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { int cc = (insn >> 8) & 0xf; - asi->insn_check_cc = kprobe_condition_checks[cc]; + asi->insn_check_cc = probes_condition_checks[cc]; asi->insn_handler = t16_simulate_cond_branch; return INSN_GOOD_NO_SLOT; } static void __kprobes -t16_simulate_branch(kprobe_opcode_t insn, +t16_simulate_branch(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 2; @@ -441,7 +441,7 @@ t16_simulate_branch(kprobe_opcode_t insn, } static unsigned long __kprobes -t16_emulate_loregs(kprobe_opcode_t insn, +t16_emulate_loregs(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long oldcpsr = regs->ARM_cpsr; @@ -464,14 +464,14 @@ t16_emulate_loregs(kprobe_opcode_t insn, } static void __kprobes -t16_emulate_loregs_rwflags(kprobe_opcode_t insn, +t16_emulate_loregs_rwflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { regs->ARM_cpsr = t16_emulate_loregs(insn, asi, regs); } static void __kprobes -t16_emulate_loregs_noitrwflags(kprobe_opcode_t insn, +t16_emulate_loregs_noitrwflags(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long cpsr = t16_emulate_loregs(insn, asi, regs); @@ -480,7 +480,7 @@ t16_emulate_loregs_noitrwflags(kprobe_opcode_t insn, } static void __kprobes -t16_emulate_hiregs(kprobe_opcode_t insn, +t16_emulate_hiregs(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 2; @@ -511,7 +511,7 @@ t16_emulate_hiregs(kprobe_opcode_t insn, } static enum kprobe_insn __kprobes -t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_hiregs(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { insn &= ~0x00ff; @@ -522,7 +522,7 @@ t16_decode_hiregs(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t16_emulate_push(kprobe_opcode_t insn, +t16_emulate_push(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( @@ -539,7 +539,7 @@ t16_emulate_push(kprobe_opcode_t insn, } static enum kprobe_insn __kprobes -t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_push(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { /* @@ -554,7 +554,7 @@ t16_decode_push(kprobe_opcode_t insn, struct arch_specific_insn *asi, } static void __kprobes -t16_emulate_pop_nopc(kprobe_opcode_t insn, +t16_emulate_pop_nopc(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( @@ -571,7 +571,7 @@ t16_emulate_pop_nopc(kprobe_opcode_t insn, } static void __kprobes -t16_emulate_pop_pc(kprobe_opcode_t insn, +t16_emulate_pop_pc(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { register unsigned long pc asm("r8"); @@ -592,7 +592,7 @@ t16_emulate_pop_pc(kprobe_opcode_t insn, } static enum kprobe_insn __kprobes -t16_decode_pop(kprobe_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_pop(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { /* diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index 7798035d6003..d0530b15e473 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -36,22 +36,22 @@ enum kprobe_insn { INSN_GOOD_NO_SLOT }; -typedef enum kprobe_insn (kprobe_decode_insn_t)(kprobe_opcode_t, +typedef enum kprobe_insn (kprobe_decode_insn_t)(probes_opcode_t, struct arch_specific_insn *, const union decode_action *); #ifdef CONFIG_THUMB2_KERNEL -enum kprobe_insn thumb16_kprobe_decode_insn(kprobe_opcode_t, +enum kprobe_insn thumb16_kprobe_decode_insn(probes_opcode_t, struct arch_specific_insn *, const union decode_action *); -enum kprobe_insn thumb32_kprobe_decode_insn(kprobe_opcode_t, +enum kprobe_insn thumb32_kprobe_decode_insn(probes_opcode_t, struct arch_specific_insn *, const union decode_action *); #else /* !CONFIG_THUMB2_KERNEL */ -enum kprobe_insn arm_kprobe_decode_insn(kprobe_opcode_t, +enum kprobe_insn arm_kprobe_decode_insn(probes_opcode_t, struct arch_specific_insn *, const union decode_action *); diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c index 36002e5d0f0e..3357a074597d 100644 --- a/arch/arm/kernel/probes-arm.c +++ b/arch/arm/kernel/probes-arm.c @@ -57,7 +57,7 @@ * read and write of flags. */ -void __kprobes simulate_bbl(kprobe_opcode_t insn, +void __kprobes simulate_bbl(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { long iaddr = (long) regs->ARM_pc - 4; @@ -69,7 +69,7 @@ void __kprobes simulate_bbl(kprobe_opcode_t insn, regs->ARM_pc = iaddr + 8 + disp; } -void __kprobes simulate_blx1(kprobe_opcode_t insn, +void __kprobes simulate_blx1(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { long iaddr = (long) regs->ARM_pc - 4; @@ -80,7 +80,7 @@ void __kprobes simulate_blx1(kprobe_opcode_t insn, regs->ARM_cpsr |= PSR_T_BIT; } -void __kprobes simulate_blx2bx(kprobe_opcode_t insn, +void __kprobes simulate_blx2bx(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rm = insn & 0xf; @@ -95,7 +95,7 @@ void __kprobes simulate_blx2bx(kprobe_opcode_t insn, regs->ARM_cpsr |= PSR_T_BIT; } -void __kprobes simulate_mrs(kprobe_opcode_t insn, +void __kprobes simulate_mrs(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { int rd = (insn >> 12) & 0xf; @@ -103,7 +103,7 @@ void __kprobes simulate_mrs(kprobe_opcode_t insn, regs->uregs[rd] = regs->ARM_cpsr & mask; } -void __kprobes simulate_mov_ipsp(kprobe_opcode_t insn, +void __kprobes simulate_mov_ipsp(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { regs->uregs[12] = regs->uregs[13]; @@ -704,7 +704,7 @@ const union decode_item kprobe_decode_arm_table[] = { EXPORT_SYMBOL_GPL(kprobe_decode_arm_table); #endif -static void __kprobes arm_singlestep(kprobe_opcode_t insn, +static void __kprobes arm_singlestep(probes_opcode_t insn, struct arch_specific_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 4; @@ -724,11 +724,11 @@ static void __kprobes arm_singlestep(kprobe_opcode_t insn, * should also be very rare. */ enum kprobe_insn __kprobes -arm_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, +arm_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { asi->insn_singlestep = arm_singlestep; - asi->insn_check_cc = kprobe_condition_checks[insn>>28]; + asi->insn_check_cc = probes_condition_checks[insn>>28]; return kprobe_decode_insn(insn, asi, kprobe_decode_arm_table, false, actions); } diff --git a/arch/arm/kernel/probes-arm.h b/arch/arm/kernel/probes-arm.h index d0ac8a42caa0..9a9d379dbf33 100644 --- a/arch/arm/kernel/probes-arm.h +++ b/arch/arm/kernel/probes-arm.h @@ -53,15 +53,15 @@ enum probes_arm_action { NUM_PROBES_ARM_ACTIONS }; -void __kprobes simulate_bbl(kprobe_opcode_t opcode, +void __kprobes simulate_bbl(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs); -void __kprobes simulate_blx1(kprobe_opcode_t opcode, +void __kprobes simulate_blx1(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs); -void __kprobes simulate_blx2bx(kprobe_opcode_t opcode, +void __kprobes simulate_blx2bx(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs); -void __kprobes simulate_mrs(kprobe_opcode_t opcode, +void __kprobes simulate_mrs(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs); -void __kprobes simulate_mov_ipsp(kprobe_opcode_t opcode, +void __kprobes simulate_mov_ipsp(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs); #endif diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c index aa3176da1b29..a15a79b7c9c5 100644 --- a/arch/arm/kernel/probes-thumb.c +++ b/arch/arm/kernel/probes-thumb.c @@ -839,11 +839,11 @@ EXPORT_SYMBOL_GPL(kprobe_decode_thumb16_table); static unsigned long __kprobes thumb_check_cc(unsigned long cpsr) { if (unlikely(in_it_block(cpsr))) - return kprobe_condition_checks[current_cond(cpsr)](cpsr); + return probes_condition_checks[current_cond(cpsr)](cpsr); return true; } -static void __kprobes thumb16_singlestep(kprobe_opcode_t opcode, +static void __kprobes thumb16_singlestep(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs) { @@ -852,7 +852,7 @@ static void __kprobes thumb16_singlestep(kprobe_opcode_t opcode, regs->ARM_cpsr = it_advance(regs->ARM_cpsr); } -static void __kprobes thumb32_singlestep(kprobe_opcode_t opcode, +static void __kprobes thumb32_singlestep(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs) { @@ -862,7 +862,7 @@ static void __kprobes thumb32_singlestep(kprobe_opcode_t opcode, } enum kprobe_insn __kprobes -thumb16_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, +thumb16_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { asi->insn_singlestep = thumb16_singlestep; @@ -872,7 +872,7 @@ thumb16_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, } enum kprobe_insn __kprobes -thumb32_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, +thumb32_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { asi->insn_singlestep = thumb32_singlestep; diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c index f2ab8856ba2b..c1cdc0d27e05 100644 --- a/arch/arm/kernel/probes.c +++ b/arch/arm/kernel/probes.c @@ -167,7 +167,7 @@ static unsigned long __kprobes __check_al(unsigned long cpsr) return true; } -kprobe_check_cc * const kprobe_condition_checks[16] = { +probes_check_cc * const probes_condition_checks[16] = { &__check_eq, &__check_ne, &__check_cs, &__check_cc, &__check_mi, &__check_pl, &__check_vs, &__check_vc, &__check_hi, &__check_ls, &__check_ge, &__check_lt, @@ -175,13 +175,13 @@ kprobe_check_cc * const kprobe_condition_checks[16] = { }; -void __kprobes kprobe_simulate_nop(kprobe_opcode_t opcode, +void __kprobes kprobe_simulate_nop(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs) { } -void __kprobes kprobe_emulate_none(kprobe_opcode_t opcode, +void __kprobes kprobe_emulate_none(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs) { @@ -195,8 +195,8 @@ void __kprobes kprobe_emulate_none(kprobe_opcode_t opcode, * unconditional as the condition code will already be checked before any * emulation handler is called. */ -static kprobe_opcode_t __kprobes -prepare_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, +static probes_opcode_t __kprobes +prepare_emulated_insn(probes_opcode_t insn, struct arch_specific_insn *asi, bool thumb) { #ifdef CONFIG_THUMB2_KERNEL @@ -221,7 +221,7 @@ prepare_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, * prepare_emulated_insn */ static void __kprobes -set_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, +set_emulated_insn(probes_opcode_t insn, struct arch_specific_insn *asi, bool thumb) { #ifdef CONFIG_THUMB2_KERNEL @@ -257,14 +257,14 @@ set_emulated_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, * non-zero value, the corresponding nibble in pinsn is validated and modified * according to the type. */ -static bool __kprobes decode_regs(kprobe_opcode_t *pinsn, u32 regs) +static bool __kprobes decode_regs(probes_opcode_t *pinsn, u32 regs) { - kprobe_opcode_t insn = *pinsn; - kprobe_opcode_t mask = 0xf; /* Start at least significant nibble */ + probes_opcode_t insn = *pinsn; + probes_opcode_t mask = 0xf; /* Start at least significant nibble */ for (; regs != 0; regs >>= 4, mask <<= 4) { - kprobe_opcode_t new_bits = INSN_NEW_BITS; + probes_opcode_t new_bits = INSN_NEW_BITS; switch (regs & 0xf) { @@ -383,7 +383,7 @@ static const int decode_struct_sizes[NUM_DECODE_TYPES] = { * */ int __kprobes -kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, +kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_item *table, bool thumb, const union decode_action *actions) { diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h index efea63c02742..5a0497f5a8f7 100644 --- a/arch/arm/kernel/probes.h +++ b/arch/arm/kernel/probes.h @@ -131,13 +131,13 @@ static inline void __kprobes alu_write_pc(long pcv, struct pt_regs *regs) } -void __kprobes kprobe_simulate_nop(kprobe_opcode_t, struct arch_specific_insn *, +void __kprobes kprobe_simulate_nop(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *regs); -void __kprobes kprobe_emulate_none(kprobe_opcode_t, struct arch_specific_insn *, +void __kprobes kprobe_emulate_none(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *regs); enum kprobe_insn __kprobes -kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, +kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *h); /* @@ -311,7 +311,7 @@ union decode_item { int action; }; -typedef enum kprobe_insn (probes_custom_decode_t)(kprobe_opcode_t, +typedef enum kprobe_insn (probes_custom_decode_t)(probes_opcode_t, struct arch_specific_insn *, const struct decode_header *); @@ -408,10 +408,10 @@ extern const union decode_item kprobe_decode_arm_table[]; extern const union decode_action kprobes_arm_actions[]; #endif -extern kprobe_check_cc * const kprobe_condition_checks[16]; +extern probes_check_cc * const probes_condition_checks[16]; -int kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi, +int kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_item *table, bool thumb16, const union decode_action *actions); From eb73ea97e63bb06bf98ff052615ce181bc7f69ec Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Wed, 5 Mar 2014 21:20:25 -0500 Subject: [PATCH 55/89] ARM: Change more ARM kprobes symbol names to something more generic Change kprobe_emulate_none, kprobe_simulate_nop, and arm_kprobe_decode_init function names to something more appropriate for code being shared outside of the kprobes subsystem. Also, move the new arm_probes_decode_init declaration out of the kprobes.h include file and into the probes.h include file. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/kernel/kprobes-arm.c | 12 ++++++------ arch/arm/kernel/kprobes-thumb.c | 10 +++++----- arch/arm/kernel/kprobes.c | 2 +- arch/arm/kernel/kprobes.h | 2 -- arch/arm/kernel/probes.c | 6 +++--- arch/arm/kernel/probes.h | 6 ++++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/arch/arm/kernel/kprobes-arm.c b/arch/arm/kernel/kprobes-arm.c index 09a7b1e19c14..d01d9f56f583 100644 --- a/arch/arm/kernel/kprobes-arm.c +++ b/arch/arm/kernel/kprobes-arm.c @@ -302,10 +302,10 @@ emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(probes_opcode_t insn, } const union decode_action kprobes_arm_actions[NUM_PROBES_ARM_ACTIONS] = { - [PROBES_EMULATE_NONE] = {.handler = kprobe_emulate_none}, - [PROBES_SIMULATE_NOP] = {.handler = kprobe_simulate_nop}, - [PROBES_PRELOAD_IMM] = {.handler = kprobe_simulate_nop}, - [PROBES_PRELOAD_REG] = {.handler = kprobe_simulate_nop}, + [PROBES_EMULATE_NONE] = {.handler = probes_emulate_none}, + [PROBES_SIMULATE_NOP] = {.handler = probes_simulate_nop}, + [PROBES_PRELOAD_IMM] = {.handler = probes_simulate_nop}, + [PROBES_PRELOAD_REG] = {.handler = probes_simulate_nop}, [PROBES_BRANCH_IMM] = {.handler = simulate_blx1}, [PROBES_MRS] = {.handler = simulate_mrs}, [PROBES_BRANCH_REG] = {.handler = simulate_blx2bx}, @@ -326,8 +326,8 @@ const union decode_action kprobes_arm_actions[NUM_PROBES_ARM_ACTIONS] = { [PROBES_DATA_PROCESSING_IMM] = { .handler = emulate_rd12rn16rm0rs8_rwflags}, [PROBES_MOV_HALFWORD] = {.handler = emulate_rd12rm0_noflags_nopc}, - [PROBES_SEV] = {.handler = kprobe_emulate_none}, - [PROBES_WFE] = {.handler = kprobe_simulate_nop}, + [PROBES_SEV] = {.handler = probes_emulate_none}, + [PROBES_WFE] = {.handler = probes_simulate_nop}, [PROBES_SATURATE] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, [PROBES_REV] = {.handler = emulate_rd12rm0_noflags_nopc}, [PROBES_MMI] = {.handler = emulate_rd12rn16rm0_rwflags_nopc}, diff --git a/arch/arm/kernel/kprobes-thumb.c b/arch/arm/kernel/kprobes-thumb.c index 610d6932e9bf..2a73330fb19f 100644 --- a/arch/arm/kernel/kprobes-thumb.c +++ b/arch/arm/kernel/kprobes-thumb.c @@ -613,8 +613,8 @@ const union decode_action kprobes_t16_actions[NUM_PROBES_T16_ACTIONS] = { [PROBES_T16_SIGN_EXTEND] = {.handler = t16_emulate_loregs_rwflags}, [PROBES_T16_PUSH] = {.decoder = t16_decode_push}, [PROBES_T16_POP] = {.decoder = t16_decode_pop}, - [PROBES_T16_SEV] = {.handler = kprobe_emulate_none}, - [PROBES_T16_WFE] = {.handler = kprobe_simulate_nop}, + [PROBES_T16_SEV] = {.handler = probes_emulate_none}, + [PROBES_T16_WFE] = {.handler = probes_simulate_nop}, [PROBES_T16_IT] = {.decoder = t16_decode_it}, [PROBES_T16_CMP] = {.handler = t16_emulate_loregs_rwflags}, [PROBES_T16_ADDSUB] = {.handler = t16_emulate_loregs_noitrwflags}, @@ -644,12 +644,12 @@ const union decode_action kprobes_t32_actions[NUM_PROBES_T32_ACTIONS] = { [PROBES_T32_MOVW] = {.handler = t32_emulate_rd8rn16_noflags}, [PROBES_T32_SAT] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, [PROBES_T32_BITFIELD] = {.handler = t32_emulate_rd8rn16_noflags}, - [PROBES_T32_SEV] = {.handler = kprobe_emulate_none}, - [PROBES_T32_WFE] = {.handler = kprobe_simulate_nop}, + [PROBES_T32_SEV] = {.handler = probes_emulate_none}, + [PROBES_T32_WFE] = {.handler = probes_simulate_nop}, [PROBES_T32_MRS] = {.handler = t32_simulate_mrs}, [PROBES_T32_BRANCH_COND] = {.decoder = t32_decode_cond_branch}, [PROBES_T32_BRANCH] = {.handler = t32_simulate_branch}, - [PROBES_T32_PLDI] = {.handler = kprobe_simulate_nop}, + [PROBES_T32_PLDI] = {.handler = probes_simulate_nop}, [PROBES_T32_LDR_LIT] = {.handler = t32_simulate_ldr_literal}, [PROBES_T32_LDRSTR] = {.handler = t32_emulate_ldrstr}, [PROBES_T32_SIGN_EXTEND] = {.handler = t32_emulate_rd8rn16rm0_rwflags}, diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c index b4a3028edffe..bfd7b8161c58 100644 --- a/arch/arm/kernel/kprobes.c +++ b/arch/arm/kernel/kprobes.c @@ -614,7 +614,7 @@ static struct undef_hook kprobes_arm_break_hook = { int __init arch_init_kprobes() { - arm_kprobe_decode_init(); + arm_probes_decode_init(); #ifdef CONFIG_THUMB2_KERNEL register_undef_hook(&kprobes_thumb16_break_hook); register_undef_hook(&kprobes_thumb32_break_hook); diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index d0530b15e473..e2ae4ed168cd 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -57,8 +57,6 @@ enum kprobe_insn arm_kprobe_decode_insn(probes_opcode_t, #endif -void __init arm_kprobe_decode_init(void); - #include "probes.h" #endif /* _ARM_KERNEL_KPROBES_H */ diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c index c1cdc0d27e05..92d359a22843 100644 --- a/arch/arm/kernel/probes.c +++ b/arch/arm/kernel/probes.c @@ -76,7 +76,7 @@ void __init test_alu_write_pc_interworking(void) #endif /* !test_alu_write_pc_interworking */ -void __init arm_kprobe_decode_init(void) +void __init arm_probes_decode_init(void) { find_str_pc_offset(); test_load_write_pc_interworking(); @@ -175,13 +175,13 @@ probes_check_cc * const probes_condition_checks[16] = { }; -void __kprobes kprobe_simulate_nop(probes_opcode_t opcode, +void __kprobes probes_simulate_nop(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs) { } -void __kprobes kprobe_emulate_none(probes_opcode_t opcode, +void __kprobes probes_emulate_none(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs) { diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h index 5a0497f5a8f7..dedff8a5a924 100644 --- a/arch/arm/kernel/probes.h +++ b/arch/arm/kernel/probes.h @@ -24,6 +24,8 @@ #include #include "kprobes.h" +void __init arm_probes_decode_init(void); + #if __LINUX_ARM_ARCH__ >= 7 /* str_pc_offset is architecturally defined from ARMv7 onwards */ @@ -131,9 +133,9 @@ static inline void __kprobes alu_write_pc(long pcv, struct pt_regs *regs) } -void __kprobes kprobe_simulate_nop(probes_opcode_t, struct arch_specific_insn *, +void __kprobes probes_simulate_nop(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *regs); -void __kprobes kprobe_emulate_none(probes_opcode_t, struct arch_specific_insn *, +void __kprobes probes_emulate_none(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *regs); enum kprobe_insn __kprobes From 44a0a59c535004eac9f18210cb2ce10b23861630 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Wed, 5 Mar 2014 21:23:42 -0500 Subject: [PATCH 56/89] ARM: Rename the shared kprobes/uprobe return value enum Change the name of kprobes_insn to probes_insn so it can be shared between kprobes and uprobes without confusion. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/kernel/kprobes-common.c | 2 +- arch/arm/kernel/kprobes-thumb.c | 16 ++++++++-------- arch/arm/kernel/kprobes.h | 14 ++++---------- arch/arm/kernel/probes-arm.c | 2 +- arch/arm/kernel/probes-thumb.c | 4 ++-- arch/arm/kernel/probes.h | 9 +++++++-- 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index 08b55605c1ec..f151e15f566a 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -123,7 +123,7 @@ emulate_ldm_r3_15(probes_opcode_t insn, load_write_pc(regs->ARM_pc, regs); } -enum kprobe_insn __kprobes +enum probes_insn __kprobes kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *h) { diff --git a/arch/arm/kernel/kprobes-thumb.c b/arch/arm/kernel/kprobes-thumb.c index 2a73330fb19f..c271d5d2810c 100644 --- a/arch/arm/kernel/kprobes-thumb.c +++ b/arch/arm/kernel/kprobes-thumb.c @@ -66,7 +66,7 @@ t32_simulate_cond_branch(probes_opcode_t insn, regs->ARM_pc = pc + (offset * 2); } -static enum kprobe_insn __kprobes +static enum probes_insn __kprobes t32_decode_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { @@ -142,11 +142,11 @@ t32_simulate_ldr_literal(probes_opcode_t insn, regs->uregs[rt] = rtv; } -static enum kprobe_insn __kprobes +static enum probes_insn __kprobes t32_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { - enum kprobe_insn ret = kprobe_decode_ldmstm(insn, asi, d); + enum probes_insn ret = kprobe_decode_ldmstm(insn, asi, d); /* Fixup modified instruction to have halfwords in correct order...*/ insn = asi->insn[0]; @@ -402,7 +402,7 @@ t16_singlestep_it(probes_opcode_t insn, t16_simulate_it(insn, asi, regs); } -static enum kprobe_insn __kprobes +static enum probes_insn __kprobes t16_decode_it(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { @@ -420,7 +420,7 @@ t16_simulate_cond_branch(probes_opcode_t insn, regs->ARM_pc = pc + (offset * 2); } -static enum kprobe_insn __kprobes +static enum probes_insn __kprobes t16_decode_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { @@ -510,7 +510,7 @@ t16_emulate_hiregs(probes_opcode_t insn, regs->ARM_cpsr = (regs->ARM_cpsr & ~APSR_MASK) | (cpsr & APSR_MASK); } -static enum kprobe_insn __kprobes +static enum probes_insn __kprobes t16_decode_hiregs(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { @@ -538,7 +538,7 @@ t16_emulate_push(probes_opcode_t insn, ); } -static enum kprobe_insn __kprobes +static enum probes_insn __kprobes t16_decode_push(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { @@ -591,7 +591,7 @@ t16_emulate_pop_pc(probes_opcode_t insn, bx_write_pc(pc, regs); } -static enum kprobe_insn __kprobes +static enum probes_insn __kprobes t16_decode_pop(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *d) { diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index e2ae4ed168cd..3684fc9e27cc 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -30,28 +30,22 @@ struct decode_header; union decode_action; -enum kprobe_insn { - INSN_REJECTED, - INSN_GOOD, - INSN_GOOD_NO_SLOT -}; - -typedef enum kprobe_insn (kprobe_decode_insn_t)(probes_opcode_t, +typedef enum probes_insn (kprobe_decode_insn_t)(probes_opcode_t, struct arch_specific_insn *, const union decode_action *); #ifdef CONFIG_THUMB2_KERNEL -enum kprobe_insn thumb16_kprobe_decode_insn(probes_opcode_t, +enum probes_insn thumb16_kprobe_decode_insn(probes_opcode_t, struct arch_specific_insn *, const union decode_action *); -enum kprobe_insn thumb32_kprobe_decode_insn(probes_opcode_t, +enum probes_insn thumb32_kprobe_decode_insn(probes_opcode_t, struct arch_specific_insn *, const union decode_action *); #else /* !CONFIG_THUMB2_KERNEL */ -enum kprobe_insn arm_kprobe_decode_insn(probes_opcode_t, +enum probes_insn arm_kprobe_decode_insn(probes_opcode_t, struct arch_specific_insn *, const union decode_action *); diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c index 3357a074597d..a9439e607ac0 100644 --- a/arch/arm/kernel/probes-arm.c +++ b/arch/arm/kernel/probes-arm.c @@ -723,7 +723,7 @@ static void __kprobes arm_singlestep(probes_opcode_t insn, * if the work was put into it, but low return considering they * should also be very rare. */ -enum kprobe_insn __kprobes +enum probes_insn __kprobes arm_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c index a15a79b7c9c5..d23ef009fe63 100644 --- a/arch/arm/kernel/probes-thumb.c +++ b/arch/arm/kernel/probes-thumb.c @@ -861,7 +861,7 @@ static void __kprobes thumb32_singlestep(probes_opcode_t opcode, regs->ARM_cpsr = it_advance(regs->ARM_cpsr); } -enum kprobe_insn __kprobes +enum probes_insn __kprobes thumb16_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { @@ -871,7 +871,7 @@ thumb16_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, actions); } -enum kprobe_insn __kprobes +enum probes_insn __kprobes thumb32_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h index dedff8a5a924..870282a39261 100644 --- a/arch/arm/kernel/probes.h +++ b/arch/arm/kernel/probes.h @@ -138,7 +138,7 @@ void __kprobes probes_simulate_nop(probes_opcode_t, struct arch_specific_insn *, void __kprobes probes_emulate_none(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *regs); -enum kprobe_insn __kprobes +enum probes_insn __kprobes kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *h); @@ -313,7 +313,7 @@ union decode_item { int action; }; -typedef enum kprobe_insn (probes_custom_decode_t)(probes_opcode_t, +typedef enum probes_insn (probes_custom_decode_t)(probes_opcode_t, struct arch_specific_insn *, const struct decode_header *); @@ -391,6 +391,11 @@ struct decode_or { #define DECODE_OR(_mask, _value) \ DECODE_HEADER(DECODE_TYPE_OR, _mask, _value, 0) +enum probes_insn { + INSN_REJECTED, + INSN_GOOD, + INSN_GOOD_NO_SLOT +}; struct decode_reject { struct decode_header header; From 47e190fafde49ff8ca732fa137e39cb2b8baba8c Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Thu, 6 Mar 2014 18:12:07 -0500 Subject: [PATCH 57/89] ARM: Change the remaining shared kprobes/uprobes symbols to something generic Any more ARM kprobes/uprobes symbols which have "kprobe" in the name must be changed to the more generic "probes" or other non-kprobes specific symbol. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/include/asm/probes.h | 13 +++++----- arch/arm/kernel/kprobes-common.c | 2 +- arch/arm/kernel/kprobes-test.c | 8 +++--- arch/arm/kernel/kprobes.c | 10 +++++--- arch/arm/kernel/kprobes.h | 21 ++++++---------- arch/arm/kernel/probes-arm.c | 8 +++--- arch/arm/kernel/probes-arm.h | 6 +++++ arch/arm/kernel/probes-thumb.c | 18 +++++++------- arch/arm/kernel/probes-thumb.h | 10 ++++++++ arch/arm/kernel/probes.c | 4 +-- arch/arm/kernel/probes.h | 42 +++++++++----------------------- 11 files changed, 69 insertions(+), 73 deletions(-) diff --git a/arch/arm/include/asm/probes.h b/arch/arm/include/asm/probes.h index c4acf6c8a2d4..c37252c73ee5 100644 --- a/arch/arm/include/asm/probes.h +++ b/arch/arm/include/asm/probes.h @@ -19,26 +19,25 @@ #ifndef _ASM_PROBES_H #define _ASM_PROBES_H -struct kprobe; typedef u32 probes_opcode_t; struct arch_specific_insn; -typedef void (kprobe_insn_handler_t)(probes_opcode_t, +typedef void (probes_insn_handler_t)(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *); typedef unsigned long (probes_check_cc)(unsigned long); -typedef void (kprobe_insn_singlestep_t)(probes_opcode_t, +typedef void (probes_insn_singlestep_t)(probes_opcode_t, struct arch_specific_insn *, struct pt_regs *); -typedef void (kprobe_insn_fn_t)(void); +typedef void (probes_insn_fn_t)(void); /* Architecture specific copy of original instruction. */ struct arch_specific_insn { probes_opcode_t *insn; - kprobe_insn_handler_t *insn_handler; + probes_insn_handler_t *insn_handler; probes_check_cc *insn_check_cc; - kprobe_insn_singlestep_t *insn_singlestep; - kprobe_insn_fn_t *insn_fn; + probes_insn_singlestep_t *insn_singlestep; + probes_insn_fn_t *insn_fn; }; #endif diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index f151e15f566a..6159725597a1 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -127,7 +127,7 @@ enum probes_insn __kprobes kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, const struct decode_header *h) { - kprobe_insn_handler_t *handler = 0; + probes_insn_handler_t *handler = 0; unsigned reglist = insn & 0xffff; int is_ldm = insn & 0x100000; int rn = (insn >> 16) & 0xf; diff --git a/arch/arm/kernel/kprobes-test.c b/arch/arm/kernel/kprobes-test.c index 4a774d40c946..c2fd06b4c389 100644 --- a/arch/arm/kernel/kprobes-test.c +++ b/arch/arm/kernel/kprobes-test.c @@ -207,6 +207,8 @@ #include #include "kprobes.h" +#include "probes-arm.h" +#include "probes-thumb.h" #include "kprobes-test.h" @@ -1610,7 +1612,7 @@ static int __init run_all_tests(void) goto out; pr_info("ARM instruction simulation\n"); - ret = run_test_cases(kprobe_arm_test_cases, kprobe_decode_arm_table); + ret = run_test_cases(kprobe_arm_test_cases, probes_decode_arm_table); if (ret) goto out; @@ -1633,13 +1635,13 @@ static int __init run_all_tests(void) pr_info("16-bit Thumb instruction simulation\n"); ret = run_test_cases(kprobe_thumb16_test_cases, - kprobe_decode_thumb16_table); + probes_decode_thumb16_table); if (ret) goto out; pr_info("32-bit Thumb instruction simulation\n"); ret = run_test_cases(kprobe_thumb32_test_cases, - kprobe_decode_thumb32_table); + probes_decode_thumb32_table); if (ret) goto out; #endif diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c index bfd7b8161c58..468d4a980c6c 100644 --- a/arch/arm/kernel/kprobes.c +++ b/arch/arm/kernel/kprobes.c @@ -31,6 +31,8 @@ #include #include "kprobes.h" +#include "probes-arm.h" +#include "probes-thumb.h" #include "patch.h" #define MIN_STACK_SIZE(addr) \ @@ -69,10 +71,10 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) if (is_wide_instruction(insn)) { insn <<= 16; insn |= ((u16 *)addr)[1]; - decode_insn = thumb32_kprobe_decode_insn; + decode_insn = thumb32_probes_decode_insn; actions = kprobes_t32_actions; } else { - decode_insn = thumb16_kprobe_decode_insn; + decode_insn = thumb16_probes_decode_insn; actions = kprobes_t16_actions; } #else /* !CONFIG_THUMB2_KERNEL */ @@ -80,7 +82,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) if (addr & 0x3) return -EINVAL; insn = *p->addr; - decode_insn = arm_kprobe_decode_insn; + decode_insn = arm_probes_decode_insn; actions = kprobes_arm_actions; #endif @@ -99,7 +101,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) p->ainsn.insn[is] = tmp_insn[is]; flush_insns(p->ainsn.insn, sizeof(p->ainsn.insn[0]) * MAX_INSN_SIZE); - p->ainsn.insn_fn = (kprobe_insn_fn_t *) + p->ainsn.insn_fn = (probes_insn_fn_t *) ((uintptr_t)p->ainsn.insn | thumb); break; diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index 3684fc9e27cc..eee8089b1b93 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -19,6 +19,8 @@ #ifndef _ARM_KERNEL_KPROBES_H #define _ARM_KERNEL_KPROBES_H +#include "probes.h" + /* * These undefined instructions must be unique and * reserved solely for kprobes' use. @@ -27,8 +29,9 @@ #define KPROBE_THUMB16_BREAKPOINT_INSTRUCTION 0xde18 #define KPROBE_THUMB32_BREAKPOINT_INSTRUCTION 0xf7f0a018 -struct decode_header; -union decode_action; +enum probes_insn __kprobes +kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, + const struct decode_header *h); typedef enum probes_insn (kprobe_decode_insn_t)(probes_opcode_t, struct arch_specific_insn *, @@ -36,21 +39,13 @@ typedef enum probes_insn (kprobe_decode_insn_t)(probes_opcode_t, #ifdef CONFIG_THUMB2_KERNEL -enum probes_insn thumb16_kprobe_decode_insn(probes_opcode_t, - struct arch_specific_insn *, - const union decode_action *); -enum probes_insn thumb32_kprobe_decode_insn(probes_opcode_t, - struct arch_specific_insn *, - const union decode_action *); +extern const union decode_action kprobes_t32_actions[]; +extern const union decode_action kprobes_t16_actions[]; #else /* !CONFIG_THUMB2_KERNEL */ -enum probes_insn arm_kprobe_decode_insn(probes_opcode_t, - struct arch_specific_insn *, - const union decode_action *); +extern const union decode_action kprobes_arm_actions[]; #endif -#include "probes.h" - #endif /* _ARM_KERNEL_KPROBES_H */ diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c index a9439e607ac0..738e5fc58928 100644 --- a/arch/arm/kernel/probes-arm.c +++ b/arch/arm/kernel/probes-arm.c @@ -610,7 +610,7 @@ static const union decode_item arm_cccc_100x_table[] = { DECODE_END }; -const union decode_item kprobe_decode_arm_table[] = { +const union decode_item probes_decode_arm_table[] = { /* * Unconditional instructions * 1111 xxxx xxxx xxxx xxxx xxxx xxxx xxxx @@ -701,7 +701,7 @@ const union decode_item kprobe_decode_arm_table[] = { DECODE_END }; #ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_arm_table); +EXPORT_SYMBOL_GPL(probes_decode_arm_table); #endif static void __kprobes arm_singlestep(probes_opcode_t insn, @@ -724,11 +724,11 @@ static void __kprobes arm_singlestep(probes_opcode_t insn, * should also be very rare. */ enum probes_insn __kprobes -arm_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +arm_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { asi->insn_singlestep = arm_singlestep; asi->insn_check_cc = probes_condition_checks[insn>>28]; - return kprobe_decode_insn(insn, asi, kprobe_decode_arm_table, false, + return probes_decode_insn(insn, asi, probes_decode_arm_table, false, actions); } diff --git a/arch/arm/kernel/probes-arm.h b/arch/arm/kernel/probes-arm.h index 9a9d379dbf33..7a5cce497a9b 100644 --- a/arch/arm/kernel/probes-arm.h +++ b/arch/arm/kernel/probes-arm.h @@ -64,4 +64,10 @@ void __kprobes simulate_mrs(probes_opcode_t opcode, void __kprobes simulate_mov_ipsp(probes_opcode_t opcode, struct arch_specific_insn *asi, struct pt_regs *regs); +extern const union decode_item probes_decode_arm_table[]; + +enum probes_insn arm_probes_decode_insn(probes_opcode_t, + struct arch_specific_insn *, + const union decode_action *actions); + #endif diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c index d23ef009fe63..eab440f6b2d4 100644 --- a/arch/arm/kernel/probes-thumb.c +++ b/arch/arm/kernel/probes-thumb.c @@ -1,5 +1,5 @@ /* - * arch/arm/kernel/kprobes-thumb.c + * arch/arm/kernel/probes-thumb.c * * Copyright (C) 2011 Jon Medhurst . * @@ -552,7 +552,7 @@ static const union decode_item t32_table_1111_1011_1[] = { DECODE_END }; -const union decode_item kprobe_decode_thumb32_table[] = { +const union decode_item probes_decode_thumb32_table[] = { /* * Load/store multiple instructions @@ -641,7 +641,7 @@ const union decode_item kprobe_decode_thumb32_table[] = { DECODE_END }; #ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_thumb32_table); +EXPORT_SYMBOL_GPL(probes_decode_thumb32_table); #endif static const union decode_item t16_table_1011[] = { @@ -696,7 +696,7 @@ static const union decode_item t16_table_1011[] = { DECODE_END }; -const union decode_item kprobe_decode_thumb16_table[] = { +const union decode_item probes_decode_thumb16_table[] = { /* * Shift (immediate), add, subtract, move, and compare @@ -833,7 +833,7 @@ const union decode_item kprobe_decode_thumb16_table[] = { DECODE_END }; #ifdef CONFIG_ARM_KPROBES_TEST_MODULE -EXPORT_SYMBOL_GPL(kprobe_decode_thumb16_table); +EXPORT_SYMBOL_GPL(probes_decode_thumb16_table); #endif static unsigned long __kprobes thumb_check_cc(unsigned long cpsr) @@ -862,21 +862,21 @@ static void __kprobes thumb32_singlestep(probes_opcode_t opcode, } enum probes_insn __kprobes -thumb16_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { asi->insn_singlestep = thumb16_singlestep; asi->insn_check_cc = thumb_check_cc; - return kprobe_decode_insn(insn, asi, kprobe_decode_thumb16_table, true, + return probes_decode_insn(insn, asi, probes_decode_thumb16_table, true, actions); } enum probes_insn __kprobes -thumb32_kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_action *actions) { asi->insn_singlestep = thumb32_singlestep; asi->insn_check_cc = thumb_check_cc; - return kprobe_decode_insn(insn, asi, kprobe_decode_thumb32_table, true, + return probes_decode_insn(insn, asi, probes_decode_thumb32_table, true, actions); } diff --git a/arch/arm/kernel/probes-thumb.h b/arch/arm/kernel/probes-thumb.h index 8d6b4eefa706..d6f67c1df7af 100644 --- a/arch/arm/kernel/probes-thumb.h +++ b/arch/arm/kernel/probes-thumb.h @@ -84,4 +84,14 @@ enum probes_t16_action { NUM_PROBES_T16_ACTIONS }; +extern const union decode_item probes_decode_thumb32_table[]; +extern const union decode_item probes_decode_thumb16_table[]; + +enum probes_insn __kprobes +thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, + const union decode_action *actions); +enum probes_insn __kprobes +thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, + const union decode_action *actions); + #endif diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c index 92d359a22843..b6d9b855273c 100644 --- a/arch/arm/kernel/probes.c +++ b/arch/arm/kernel/probes.c @@ -340,7 +340,7 @@ static const int decode_struct_sizes[NUM_DECODE_TYPES] = { }; /* - * kprobe_decode_insn operates on data tables in order to decode an ARM + * probes_decode_insn operates on data tables in order to decode an ARM * architecture instruction onto which a kprobe has been placed. * * These instruction decoding tables are a concatenation of entries each @@ -383,7 +383,7 @@ static const int decode_struct_sizes[NUM_DECODE_TYPES] = { * */ int __kprobes -kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_item *table, bool thumb, const union decode_action *actions) { diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h index 870282a39261..0c72e544175d 100644 --- a/arch/arm/kernel/probes.h +++ b/arch/arm/kernel/probes.h @@ -21,11 +21,12 @@ #include #include -#include -#include "kprobes.h" +#include void __init arm_probes_decode_init(void); +extern probes_check_cc * const probes_condition_checks[16]; + #if __LINUX_ARM_ARCH__ >= 7 /* str_pc_offset is architecturally defined from ARMv7 onwards */ @@ -40,7 +41,6 @@ void __init find_str_pc_offset(void); #endif -struct decode_header; /* * Update ITSTATE after normal execution of an IT block instruction. @@ -133,15 +133,6 @@ static inline void __kprobes alu_write_pc(long pcv, struct pt_regs *regs) } -void __kprobes probes_simulate_nop(probes_opcode_t, struct arch_specific_insn *, - struct pt_regs *regs); -void __kprobes probes_emulate_none(probes_opcode_t, struct arch_specific_insn *, - struct pt_regs *regs); - -enum probes_insn __kprobes -kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, - const struct decode_header *h); - /* * Test if load/store instructions writeback the address register. * if P (bit 24) == 0 or W (bit 21) == 1 @@ -150,7 +141,7 @@ kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, /* * The following definitions and macros are used to build instruction - * decoding tables for use by kprobe_decode_insn. + * decoding tables for use by probes_decode_insn. * * These tables are a concatenation of entries each of which consist of one of * the decode_* structs. All of the fields in every type of decode structure @@ -313,12 +304,13 @@ union decode_item { int action; }; +struct decode_header; typedef enum probes_insn (probes_custom_decode_t)(probes_opcode_t, struct arch_specific_insn *, const struct decode_header *); union decode_action { - kprobe_insn_handler_t *handler; + probes_insn_handler_t *handler; probes_custom_decode_t *decoder; }; @@ -404,22 +396,12 @@ struct decode_reject { #define DECODE_REJECT(_mask, _value) \ DECODE_HEADER(DECODE_TYPE_REJECT, _mask, _value, 0) +probes_insn_handler_t probes_simulate_nop; +probes_insn_handler_t probes_emulate_none; -#ifdef CONFIG_THUMB2_KERNEL -extern const union decode_item kprobe_decode_thumb16_table[]; -extern const union decode_item kprobe_decode_thumb32_table[]; -extern const union decode_action kprobes_t32_actions[]; -extern const union decode_action kprobes_t16_actions[]; -#else -extern const union decode_item kprobe_decode_arm_table[]; -extern const union decode_action kprobes_arm_actions[]; -#endif - -extern probes_check_cc * const probes_condition_checks[16]; - - -int kprobe_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, - const union decode_item *table, bool thumb16, - const union decode_action *actions); +int __kprobes +probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, + const union decode_item *table, bool thumb, + const union decode_action *actions); #endif From 602cd2609eee92d338a83e400774e97c60535ba2 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Wed, 5 Mar 2014 21:40:12 -0500 Subject: [PATCH 58/89] ARM: Add an emulate flag to the kprobes/uprobes instruction decode functions Add an emulate flag into the instruction interpreter, primarily for uprobes support. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/kernel/kprobes.c | 2 +- arch/arm/kernel/kprobes.h | 1 + arch/arm/kernel/probes-arm.c | 4 ++-- arch/arm/kernel/probes-arm.h | 2 +- arch/arm/kernel/probes-thumb.c | 8 ++++---- arch/arm/kernel/probes-thumb.h | 4 ++-- arch/arm/kernel/probes.c | 18 +++++++++++++----- arch/arm/kernel/probes.h | 2 +- 8 files changed, 25 insertions(+), 16 deletions(-) diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c index 468d4a980c6c..8795f9f819d5 100644 --- a/arch/arm/kernel/kprobes.c +++ b/arch/arm/kernel/kprobes.c @@ -89,7 +89,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) p->opcode = insn; p->ainsn.insn = tmp_insn; - switch ((*decode_insn)(insn, &p->ainsn, actions)) { + switch ((*decode_insn)(insn, &p->ainsn, true, actions)) { case INSN_REJECTED: /* not supported */ return -EINVAL; diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index eee8089b1b93..d0a24b73bcfa 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -35,6 +35,7 @@ kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, typedef enum probes_insn (kprobe_decode_insn_t)(probes_opcode_t, struct arch_specific_insn *, + bool, const union decode_action *); #ifdef CONFIG_THUMB2_KERNEL diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c index 738e5fc58928..8e7fde876521 100644 --- a/arch/arm/kernel/probes-arm.c +++ b/arch/arm/kernel/probes-arm.c @@ -725,10 +725,10 @@ static void __kprobes arm_singlestep(probes_opcode_t insn, */ enum probes_insn __kprobes arm_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, - const union decode_action *actions) + bool emulate, const union decode_action *actions) { asi->insn_singlestep = arm_singlestep; asi->insn_check_cc = probes_condition_checks[insn>>28]; return probes_decode_insn(insn, asi, probes_decode_arm_table, false, - actions); + emulate, actions); } diff --git a/arch/arm/kernel/probes-arm.h b/arch/arm/kernel/probes-arm.h index 7a5cce497a9b..ea614dc5aaa3 100644 --- a/arch/arm/kernel/probes-arm.h +++ b/arch/arm/kernel/probes-arm.h @@ -67,7 +67,7 @@ void __kprobes simulate_mov_ipsp(probes_opcode_t opcode, extern const union decode_item probes_decode_arm_table[]; enum probes_insn arm_probes_decode_insn(probes_opcode_t, - struct arch_specific_insn *, + struct arch_specific_insn *, bool emulate, const union decode_action *actions); #endif diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c index eab440f6b2d4..23e2cbdb37cb 100644 --- a/arch/arm/kernel/probes-thumb.c +++ b/arch/arm/kernel/probes-thumb.c @@ -863,20 +863,20 @@ static void __kprobes thumb32_singlestep(probes_opcode_t opcode, enum probes_insn __kprobes thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, - const union decode_action *actions) + bool emulate, const union decode_action *actions) { asi->insn_singlestep = thumb16_singlestep; asi->insn_check_cc = thumb_check_cc; return probes_decode_insn(insn, asi, probes_decode_thumb16_table, true, - actions); + emulate, actions); } enum probes_insn __kprobes thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, - const union decode_action *actions) + bool emulate, const union decode_action *actions) { asi->insn_singlestep = thumb32_singlestep; asi->insn_check_cc = thumb_check_cc; return probes_decode_insn(insn, asi, probes_decode_thumb32_table, true, - actions); + emulate, actions); } diff --git a/arch/arm/kernel/probes-thumb.h b/arch/arm/kernel/probes-thumb.h index d6f67c1df7af..65e4250e9b78 100644 --- a/arch/arm/kernel/probes-thumb.h +++ b/arch/arm/kernel/probes-thumb.h @@ -89,9 +89,9 @@ extern const union decode_item probes_decode_thumb16_table[]; enum probes_insn __kprobes thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, - const union decode_action *actions); + bool emulate, const union decode_action *actions); enum probes_insn __kprobes thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, - const union decode_action *actions); + bool emulate, const union decode_action *actions); #endif diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c index b6d9b855273c..f9dff12cf85c 100644 --- a/arch/arm/kernel/probes.c +++ b/arch/arm/kernel/probes.c @@ -257,7 +257,7 @@ set_emulated_insn(probes_opcode_t insn, struct arch_specific_insn *asi, * non-zero value, the corresponding nibble in pinsn is validated and modified * according to the type. */ -static bool __kprobes decode_regs(probes_opcode_t *pinsn, u32 regs) +static bool __kprobes decode_regs(probes_opcode_t *pinsn, u32 regs, bool modify) { probes_opcode_t insn = *pinsn; probes_opcode_t mask = 0xf; /* Start at least significant nibble */ @@ -323,7 +323,9 @@ static bool __kprobes decode_regs(probes_opcode_t *pinsn, u32 regs) insn |= new_bits & mask; } - *pinsn = insn; + if (modify) + *pinsn = insn; + return true; reject: @@ -385,13 +387,14 @@ static const int decode_struct_sizes[NUM_DECODE_TYPES] = { int __kprobes probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, const union decode_item *table, bool thumb, - const union decode_action *actions) + bool emulate, const union decode_action *actions) { const struct decode_header *h = (struct decode_header *)table; const struct decode_header *next; bool matched = false; - insn = prepare_emulated_insn(insn, asi, thumb); + if (emulate) + insn = prepare_emulated_insn(insn, asi, thumb); for (;; h = next) { enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK; @@ -406,7 +409,7 @@ probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, if (!matched && (insn & h->mask.bits) != h->value.bits) continue; - if (!decode_regs(&insn, regs)) + if (!decode_regs(&insn, regs, emulate)) return INSN_REJECTED; switch (type) { @@ -430,6 +433,11 @@ probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, case DECODE_TYPE_EMULATE: { struct decode_emulate *d = (struct decode_emulate *)h; + + if (!emulate) + return actions[d->handler.action].decoder(insn, + asi, h); + asi->insn_handler = actions[d->handler.action].handler; set_emulated_insn(insn, asi, thumb); return INSN_GOOD; diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h index 0c72e544175d..33cc30c50cf5 100644 --- a/arch/arm/kernel/probes.h +++ b/arch/arm/kernel/probes.h @@ -401,7 +401,7 @@ probes_insn_handler_t probes_emulate_none; int __kprobes probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, - const union decode_item *table, bool thumb, + const union decode_item *table, bool thumb, bool emulate, const union decode_action *actions); #endif From b4cd605ca92d9a8a2f71355cb45dd943ebcb0c97 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Wed, 5 Mar 2014 21:41:29 -0500 Subject: [PATCH 59/89] ARM: Make arch_specific_insn a define for new arch_probes_insn structure Because the common underlying code for ARM kprobes and uprobes needs to share a common architecrure-specific context structure, and because the generic kprobes include file insists on defining this to a dummy structure when kprobes is not configured, a new common structure is required which can exist when uprobes is configured without kprobes. In this case kprobes will define a dummy structure, but without the define aliasing the two structure tags it will not affect uprobes and the shared probes code. Signed-off-by: David A. Long Acked-by: Jon Medhurst --- arch/arm/include/asm/kprobes.h | 2 + arch/arm/include/asm/probes.h | 8 ++-- arch/arm/kernel/kprobes-arm.c | 16 ++++---- arch/arm/kernel/kprobes-common.c | 14 +++---- arch/arm/kernel/kprobes-thumb.c | 70 ++++++++++++++++---------------- arch/arm/kernel/kprobes.h | 4 +- arch/arm/kernel/probes-arm.c | 14 +++---- arch/arm/kernel/probes-arm.h | 12 +++--- arch/arm/kernel/probes-thumb.c | 8 ++-- arch/arm/kernel/probes-thumb.h | 4 +- arch/arm/kernel/probes.c | 10 ++--- arch/arm/kernel/probes.h | 4 +- 12 files changed, 84 insertions(+), 82 deletions(-) diff --git a/arch/arm/include/asm/kprobes.h b/arch/arm/include/asm/kprobes.h index 6e1046661f07..49fa0dfaad33 100644 --- a/arch/arm/include/asm/kprobes.h +++ b/arch/arm/include/asm/kprobes.h @@ -31,6 +31,8 @@ typedef u32 kprobe_opcode_t; struct kprobe; #include +#define arch_specific_insn arch_probes_insn + struct prev_kprobe { struct kprobe *kp; unsigned int status; diff --git a/arch/arm/include/asm/probes.h b/arch/arm/include/asm/probes.h index c37252c73ee5..806cfe622a9e 100644 --- a/arch/arm/include/asm/probes.h +++ b/arch/arm/include/asm/probes.h @@ -21,18 +21,18 @@ typedef u32 probes_opcode_t; -struct arch_specific_insn; +struct arch_probes_insn; typedef void (probes_insn_handler_t)(probes_opcode_t, - struct arch_specific_insn *, + struct arch_probes_insn *, struct pt_regs *); typedef unsigned long (probes_check_cc)(unsigned long); typedef void (probes_insn_singlestep_t)(probes_opcode_t, - struct arch_specific_insn *, + struct arch_probes_insn *, struct pt_regs *); typedef void (probes_insn_fn_t)(void); /* Architecture specific copy of original instruction. */ -struct arch_specific_insn { +struct arch_probes_insn { probes_opcode_t *insn; probes_insn_handler_t *insn_handler; probes_check_cc *insn_check_cc; diff --git a/arch/arm/kernel/kprobes-arm.c b/arch/arm/kernel/kprobes-arm.c index d01d9f56f583..ac300c60d656 100644 --- a/arch/arm/kernel/kprobes-arm.c +++ b/arch/arm/kernel/kprobes-arm.c @@ -74,7 +74,7 @@ static void __kprobes emulate_ldrdstrd(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 4; int rt = (insn >> 12) & 0xf; @@ -103,7 +103,7 @@ emulate_ldrdstrd(probes_opcode_t insn, static void __kprobes emulate_ldr(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 4; int rt = (insn >> 12) & 0xf; @@ -133,7 +133,7 @@ emulate_ldr(probes_opcode_t insn, static void __kprobes emulate_str(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long rtpc = regs->ARM_pc - 4 + str_pc_offset; unsigned long rnpc = regs->ARM_pc + 4; @@ -160,7 +160,7 @@ emulate_str(probes_opcode_t insn, static void __kprobes emulate_rd12rn16rm0rs8_rwflags(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 4; int rd = (insn >> 12) & 0xf; @@ -195,7 +195,7 @@ emulate_rd12rn16rm0rs8_rwflags(probes_opcode_t insn, static void __kprobes emulate_rd12rn16rm0_rwflags_nopc(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rd = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; @@ -222,7 +222,7 @@ emulate_rd12rn16rm0_rwflags_nopc(probes_opcode_t insn, static void __kprobes emulate_rd16rn12rm0rs8_rwflags_nopc(probes_opcode_t insn, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { int rd = (insn >> 16) & 0xf; @@ -252,7 +252,7 @@ emulate_rd16rn12rm0rs8_rwflags_nopc(probes_opcode_t insn, static void __kprobes emulate_rd12rm0_noflags_nopc(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rd = (insn >> 12) & 0xf; int rm = insn & 0xf; @@ -272,7 +272,7 @@ emulate_rd12rm0_noflags_nopc(probes_opcode_t insn, static void __kprobes emulate_rdlo12rdhi16rn0rm8_rwflags_nopc(probes_opcode_t insn, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { int rdlo = (insn >> 12) & 0xf; diff --git a/arch/arm/kernel/kprobes-common.c b/arch/arm/kernel/kprobes-common.c index 6159725597a1..c311ed94ff1c 100644 --- a/arch/arm/kernel/kprobes-common.c +++ b/arch/arm/kernel/kprobes-common.c @@ -18,7 +18,7 @@ static void __kprobes simulate_ldm1stm1(probes_opcode_t insn, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { int rn = (insn >> 16) & 0xf; @@ -60,7 +60,7 @@ static void __kprobes simulate_ldm1stm1(probes_opcode_t insn, } static void __kprobes simulate_stm1_pc(probes_opcode_t insn, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long addr = regs->ARM_pc - 4; @@ -71,7 +71,7 @@ static void __kprobes simulate_stm1_pc(probes_opcode_t insn, } static void __kprobes simulate_ldm1_pc(probes_opcode_t insn, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { simulate_ldm1stm1(insn, asi, regs); @@ -80,7 +80,7 @@ static void __kprobes simulate_ldm1_pc(probes_opcode_t insn, static void __kprobes emulate_generic_r0_12_noflags(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { register void *rregs asm("r1") = regs; register void *rfn asm("lr") = asi->insn_fn; @@ -108,7 +108,7 @@ emulate_generic_r0_12_noflags(probes_opcode_t insn, static void __kprobes emulate_generic_r2_14_noflags(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { emulate_generic_r0_12_noflags(insn, asi, (struct pt_regs *)(regs->uregs+2)); @@ -116,7 +116,7 @@ emulate_generic_r2_14_noflags(probes_opcode_t insn, static void __kprobes emulate_ldm_r3_15(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { emulate_generic_r0_12_noflags(insn, asi, (struct pt_regs *)(regs->uregs+3)); @@ -124,7 +124,7 @@ emulate_ldm_r3_15(probes_opcode_t insn, } enum probes_insn __kprobes -kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, +kprobe_decode_ldmstm(probes_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *h) { probes_insn_handler_t *handler = 0; diff --git a/arch/arm/kernel/kprobes-thumb.c b/arch/arm/kernel/kprobes-thumb.c index c271d5d2810c..6619188619ae 100644 --- a/arch/arm/kernel/kprobes-thumb.c +++ b/arch/arm/kernel/kprobes-thumb.c @@ -24,7 +24,7 @@ static void __kprobes t32_simulate_table_branch(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc; int rn = (insn >> 16) & 0xf; @@ -44,7 +44,7 @@ t32_simulate_table_branch(probes_opcode_t insn, static void __kprobes t32_simulate_mrs(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rd = (insn >> 8) & 0xf; unsigned long mask = 0xf8ff03df; /* Mask out execution state */ @@ -53,7 +53,7 @@ t32_simulate_mrs(probes_opcode_t insn, static void __kprobes t32_simulate_cond_branch(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc; @@ -67,7 +67,7 @@ t32_simulate_cond_branch(probes_opcode_t insn, } static enum probes_insn __kprobes -t32_decode_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, +t32_decode_cond_branch(probes_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *d) { int cc = (insn >> 22) & 0xf; @@ -78,7 +78,7 @@ t32_decode_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, static void __kprobes t32_simulate_branch(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc; @@ -106,7 +106,7 @@ t32_simulate_branch(probes_opcode_t insn, static void __kprobes t32_simulate_ldr_literal(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long addr = regs->ARM_pc & ~3; int rt = (insn >> 12) & 0xf; @@ -143,7 +143,7 @@ t32_simulate_ldr_literal(probes_opcode_t insn, } static enum probes_insn __kprobes -t32_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, +t32_decode_ldmstm(probes_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *d) { enum probes_insn ret = kprobe_decode_ldmstm(insn, asi, d); @@ -158,7 +158,7 @@ t32_decode_ldmstm(probes_opcode_t insn, struct arch_specific_insn *asi, static void __kprobes t32_emulate_ldrdstrd(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc & ~3; int rt1 = (insn >> 12) & 0xf; @@ -185,7 +185,7 @@ t32_emulate_ldrdstrd(probes_opcode_t insn, static void __kprobes t32_emulate_ldrstr(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rt = (insn >> 12) & 0xf; int rn = (insn >> 16) & 0xf; @@ -211,7 +211,7 @@ t32_emulate_ldrstr(probes_opcode_t insn, static void __kprobes t32_emulate_rd8rn16rm0_rwflags(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rd = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; @@ -238,7 +238,7 @@ t32_emulate_rd8rn16rm0_rwflags(probes_opcode_t insn, static void __kprobes t32_emulate_rd8pc16_noflags(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc; int rd = (insn >> 8) & 0xf; @@ -258,7 +258,7 @@ t32_emulate_rd8pc16_noflags(probes_opcode_t insn, static void __kprobes t32_emulate_rd8rn16_noflags(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rd = (insn >> 8) & 0xf; int rn = (insn >> 16) & 0xf; @@ -278,7 +278,7 @@ t32_emulate_rd8rn16_noflags(probes_opcode_t insn, static void __kprobes t32_emulate_rdlo12rdhi8rn16rm0_noflags(probes_opcode_t insn, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { int rdlo = (insn >> 12) & 0xf; @@ -306,7 +306,7 @@ t32_emulate_rdlo12rdhi8rn16rm0_noflags(probes_opcode_t insn, static void __kprobes t16_simulate_bxblx(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 2; int rm = (insn >> 3) & 0xf; @@ -320,7 +320,7 @@ t16_simulate_bxblx(probes_opcode_t insn, static void __kprobes t16_simulate_ldr_literal(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long *base = (unsigned long *)((regs->ARM_pc + 2) & ~3); long index = insn & 0xff; @@ -330,7 +330,7 @@ t16_simulate_ldr_literal(probes_opcode_t insn, static void __kprobes t16_simulate_ldrstr_sp_relative(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long* base = (unsigned long *)regs->ARM_sp; long index = insn & 0xff; @@ -343,7 +343,7 @@ t16_simulate_ldrstr_sp_relative(probes_opcode_t insn, static void __kprobes t16_simulate_reladr(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long base = (insn & 0x800) ? regs->ARM_sp : ((regs->ARM_pc + 2) & ~3); @@ -354,7 +354,7 @@ t16_simulate_reladr(probes_opcode_t insn, static void __kprobes t16_simulate_add_sp_imm(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { long imm = insn & 0x7f; if (insn & 0x80) /* SUB */ @@ -365,7 +365,7 @@ t16_simulate_add_sp_imm(probes_opcode_t insn, static void __kprobes t16_simulate_cbz(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rn = insn & 0x7; probes_opcode_t nonzero = regs->uregs[rn] ? insn : ~insn; @@ -379,7 +379,7 @@ t16_simulate_cbz(probes_opcode_t insn, static void __kprobes t16_simulate_it(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { /* * The 8 IT state bits are split into two parts in CPSR: @@ -396,14 +396,14 @@ t16_simulate_it(probes_opcode_t insn, static void __kprobes t16_singlestep_it(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 2; t16_simulate_it(insn, asi, regs); } static enum probes_insn __kprobes -t16_decode_it(probes_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_it(probes_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *d) { asi->insn_singlestep = t16_singlestep_it; @@ -412,7 +412,7 @@ t16_decode_it(probes_opcode_t insn, struct arch_specific_insn *asi, static void __kprobes t16_simulate_cond_branch(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 2; long offset = insn & 0x7f; @@ -421,7 +421,7 @@ t16_simulate_cond_branch(probes_opcode_t insn, } static enum probes_insn __kprobes -t16_decode_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_cond_branch(probes_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *d) { int cc = (insn >> 8) & 0xf; @@ -432,7 +432,7 @@ t16_decode_cond_branch(probes_opcode_t insn, struct arch_specific_insn *asi, static void __kprobes t16_simulate_branch(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 2; long offset = insn & 0x3ff; @@ -442,7 +442,7 @@ t16_simulate_branch(probes_opcode_t insn, static unsigned long __kprobes t16_emulate_loregs(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long oldcpsr = regs->ARM_cpsr; unsigned long newcpsr; @@ -465,14 +465,14 @@ t16_emulate_loregs(probes_opcode_t insn, static void __kprobes t16_emulate_loregs_rwflags(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { regs->ARM_cpsr = t16_emulate_loregs(insn, asi, regs); } static void __kprobes t16_emulate_loregs_noitrwflags(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long cpsr = t16_emulate_loregs(insn, asi, regs); if (!in_it_block(cpsr)) @@ -481,7 +481,7 @@ t16_emulate_loregs_noitrwflags(probes_opcode_t insn, static void __kprobes t16_emulate_hiregs(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { unsigned long pc = regs->ARM_pc + 2; int rdn = (insn & 0x7) | ((insn & 0x80) >> 4); @@ -511,7 +511,7 @@ t16_emulate_hiregs(probes_opcode_t insn, } static enum probes_insn __kprobes -t16_decode_hiregs(probes_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_hiregs(probes_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *d) { insn &= ~0x00ff; @@ -523,7 +523,7 @@ t16_decode_hiregs(probes_opcode_t insn, struct arch_specific_insn *asi, static void __kprobes t16_emulate_push(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( "ldr r9, [%[regs], #13*4] \n\t" @@ -539,7 +539,7 @@ t16_emulate_push(probes_opcode_t insn, } static enum probes_insn __kprobes -t16_decode_push(probes_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_push(probes_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *d) { /* @@ -555,7 +555,7 @@ t16_decode_push(probes_opcode_t insn, struct arch_specific_insn *asi, static void __kprobes t16_emulate_pop_nopc(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { __asm__ __volatile__ ( "ldr r9, [%[regs], #13*4] \n\t" @@ -572,7 +572,7 @@ t16_emulate_pop_nopc(probes_opcode_t insn, static void __kprobes t16_emulate_pop_pc(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { register unsigned long pc asm("r8"); @@ -592,7 +592,7 @@ t16_emulate_pop_pc(probes_opcode_t insn, } static enum probes_insn __kprobes -t16_decode_pop(probes_opcode_t insn, struct arch_specific_insn *asi, +t16_decode_pop(probes_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *d) { /* diff --git a/arch/arm/kernel/kprobes.h b/arch/arm/kernel/kprobes.h index d0a24b73bcfa..9a2712ecefc3 100644 --- a/arch/arm/kernel/kprobes.h +++ b/arch/arm/kernel/kprobes.h @@ -30,11 +30,11 @@ #define KPROBE_THUMB32_BREAKPOINT_INSTRUCTION 0xf7f0a018 enum probes_insn __kprobes -kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_specific_insn *asi, +kprobe_decode_ldmstm(kprobe_opcode_t insn, struct arch_probes_insn *asi, const struct decode_header *h); typedef enum probes_insn (kprobe_decode_insn_t)(probes_opcode_t, - struct arch_specific_insn *, + struct arch_probes_insn *, bool, const union decode_action *); diff --git a/arch/arm/kernel/probes-arm.c b/arch/arm/kernel/probes-arm.c index 8e7fde876521..51a13a027989 100644 --- a/arch/arm/kernel/probes-arm.c +++ b/arch/arm/kernel/probes-arm.c @@ -58,7 +58,7 @@ */ void __kprobes simulate_bbl(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { long iaddr = (long) regs->ARM_pc - 4; int disp = branch_displacement(insn); @@ -70,7 +70,7 @@ void __kprobes simulate_bbl(probes_opcode_t insn, } void __kprobes simulate_blx1(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { long iaddr = (long) regs->ARM_pc - 4; int disp = branch_displacement(insn); @@ -81,7 +81,7 @@ void __kprobes simulate_blx1(probes_opcode_t insn, } void __kprobes simulate_blx2bx(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rm = insn & 0xf; long rmv = regs->uregs[rm]; @@ -96,7 +96,7 @@ void __kprobes simulate_blx2bx(probes_opcode_t insn, } void __kprobes simulate_mrs(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { int rd = (insn >> 12) & 0xf; unsigned long mask = 0xf8ff03df; /* Mask out execution state */ @@ -104,7 +104,7 @@ void __kprobes simulate_mrs(probes_opcode_t insn, } void __kprobes simulate_mov_ipsp(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { regs->uregs[12] = regs->uregs[13]; } @@ -705,7 +705,7 @@ EXPORT_SYMBOL_GPL(probes_decode_arm_table); #endif static void __kprobes arm_singlestep(probes_opcode_t insn, - struct arch_specific_insn *asi, struct pt_regs *regs) + struct arch_probes_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 4; asi->insn_handler(insn, asi, regs); @@ -724,7 +724,7 @@ static void __kprobes arm_singlestep(probes_opcode_t insn, * should also be very rare. */ enum probes_insn __kprobes -arm_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +arm_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, bool emulate, const union decode_action *actions) { asi->insn_singlestep = arm_singlestep; diff --git a/arch/arm/kernel/probes-arm.h b/arch/arm/kernel/probes-arm.h index ea614dc5aaa3..ace6572f6e26 100644 --- a/arch/arm/kernel/probes-arm.h +++ b/arch/arm/kernel/probes-arm.h @@ -54,20 +54,20 @@ enum probes_arm_action { }; void __kprobes simulate_bbl(probes_opcode_t opcode, - struct arch_specific_insn *asi, struct pt_regs *regs); + struct arch_probes_insn *asi, struct pt_regs *regs); void __kprobes simulate_blx1(probes_opcode_t opcode, - struct arch_specific_insn *asi, struct pt_regs *regs); + struct arch_probes_insn *asi, struct pt_regs *regs); void __kprobes simulate_blx2bx(probes_opcode_t opcode, - struct arch_specific_insn *asi, struct pt_regs *regs); + struct arch_probes_insn *asi, struct pt_regs *regs); void __kprobes simulate_mrs(probes_opcode_t opcode, - struct arch_specific_insn *asi, struct pt_regs *regs); + struct arch_probes_insn *asi, struct pt_regs *regs); void __kprobes simulate_mov_ipsp(probes_opcode_t opcode, - struct arch_specific_insn *asi, struct pt_regs *regs); + struct arch_probes_insn *asi, struct pt_regs *regs); extern const union decode_item probes_decode_arm_table[]; enum probes_insn arm_probes_decode_insn(probes_opcode_t, - struct arch_specific_insn *, bool emulate, + struct arch_probes_insn *, bool emulate, const union decode_action *actions); #endif diff --git a/arch/arm/kernel/probes-thumb.c b/arch/arm/kernel/probes-thumb.c index 23e2cbdb37cb..4131351e812f 100644 --- a/arch/arm/kernel/probes-thumb.c +++ b/arch/arm/kernel/probes-thumb.c @@ -844,7 +844,7 @@ static unsigned long __kprobes thumb_check_cc(unsigned long cpsr) } static void __kprobes thumb16_singlestep(probes_opcode_t opcode, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 2; @@ -853,7 +853,7 @@ static void __kprobes thumb16_singlestep(probes_opcode_t opcode, } static void __kprobes thumb32_singlestep(probes_opcode_t opcode, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { regs->ARM_pc += 4; @@ -862,7 +862,7 @@ static void __kprobes thumb32_singlestep(probes_opcode_t opcode, } enum probes_insn __kprobes -thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, bool emulate, const union decode_action *actions) { asi->insn_singlestep = thumb16_singlestep; @@ -872,7 +872,7 @@ thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, } enum probes_insn __kprobes -thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, bool emulate, const union decode_action *actions) { asi->insn_singlestep = thumb32_singlestep; diff --git a/arch/arm/kernel/probes-thumb.h b/arch/arm/kernel/probes-thumb.h index 65e4250e9b78..7c6f6ebe514f 100644 --- a/arch/arm/kernel/probes-thumb.h +++ b/arch/arm/kernel/probes-thumb.h @@ -88,10 +88,10 @@ extern const union decode_item probes_decode_thumb32_table[]; extern const union decode_item probes_decode_thumb16_table[]; enum probes_insn __kprobes -thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +thumb16_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, bool emulate, const union decode_action *actions); enum probes_insn __kprobes -thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +thumb32_probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, bool emulate, const union decode_action *actions); #endif diff --git a/arch/arm/kernel/probes.c b/arch/arm/kernel/probes.c index f9dff12cf85c..b41873f33e69 100644 --- a/arch/arm/kernel/probes.c +++ b/arch/arm/kernel/probes.c @@ -176,13 +176,13 @@ probes_check_cc * const probes_condition_checks[16] = { void __kprobes probes_simulate_nop(probes_opcode_t opcode, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { } void __kprobes probes_emulate_none(probes_opcode_t opcode, - struct arch_specific_insn *asi, + struct arch_probes_insn *asi, struct pt_regs *regs) { asi->insn_fn(); @@ -196,7 +196,7 @@ void __kprobes probes_emulate_none(probes_opcode_t opcode, * emulation handler is called. */ static probes_opcode_t __kprobes -prepare_emulated_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +prepare_emulated_insn(probes_opcode_t insn, struct arch_probes_insn *asi, bool thumb) { #ifdef CONFIG_THUMB2_KERNEL @@ -221,7 +221,7 @@ prepare_emulated_insn(probes_opcode_t insn, struct arch_specific_insn *asi, * prepare_emulated_insn */ static void __kprobes -set_emulated_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +set_emulated_insn(probes_opcode_t insn, struct arch_probes_insn *asi, bool thumb) { #ifdef CONFIG_THUMB2_KERNEL @@ -385,7 +385,7 @@ static const int decode_struct_sizes[NUM_DECODE_TYPES] = { * */ int __kprobes -probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, const union decode_item *table, bool thumb, bool emulate, const union decode_action *actions) { diff --git a/arch/arm/kernel/probes.h b/arch/arm/kernel/probes.h index 33cc30c50cf5..dba9f2466a93 100644 --- a/arch/arm/kernel/probes.h +++ b/arch/arm/kernel/probes.h @@ -306,7 +306,7 @@ union decode_item { struct decode_header; typedef enum probes_insn (probes_custom_decode_t)(probes_opcode_t, - struct arch_specific_insn *, + struct arch_probes_insn *, const struct decode_header *); union decode_action { @@ -400,7 +400,7 @@ probes_insn_handler_t probes_simulate_nop; probes_insn_handler_t probes_emulate_none; int __kprobes -probes_decode_insn(probes_opcode_t insn, struct arch_specific_insn *asi, +probes_decode_insn(probes_opcode_t insn, struct arch_probes_insn *asi, const union decode_item *table, bool thumb, bool emulate, const union decode_action *actions); From c7edc9e326d53ca5ef9bed82de0740c6b107d55b Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Fri, 7 Mar 2014 11:23:04 -0500 Subject: [PATCH 60/89] ARM: add uprobes support Using Rabin Vincent's ARM uprobes patches as a base, enable uprobes support on ARM. Caveats: - Thumb is not supported Signed-off-by: Rabin Vincent Signed-off-by: David A. Long --- arch/arm/Kconfig | 3 + arch/arm/include/asm/ptrace.h | 6 + arch/arm/include/asm/thread_info.h | 5 +- arch/arm/include/asm/uprobes.h | 45 ++++++ arch/arm/kernel/Makefile | 1 + arch/arm/kernel/signal.c | 4 + arch/arm/kernel/uprobes-arm.c | 234 +++++++++++++++++++++++++++++ arch/arm/kernel/uprobes.c | 210 ++++++++++++++++++++++++++ arch/arm/kernel/uprobes.h | 35 +++++ 9 files changed, 542 insertions(+), 1 deletion(-) create mode 100644 arch/arm/include/asm/uprobes.h create mode 100644 arch/arm/kernel/uprobes-arm.c create mode 100644 arch/arm/kernel/uprobes.c create mode 100644 arch/arm/kernel/uprobes.h diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e25419817791..4d05bb93714a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -207,6 +207,9 @@ config ZONE_DMA config NEED_DMA_MAP_STATE def_bool y +config ARCH_SUPPORTS_UPROBES + def_bool y + config ARCH_HAS_DMA_SET_COHERENT_MASK bool diff --git a/arch/arm/include/asm/ptrace.h b/arch/arm/include/asm/ptrace.h index 04c99f36ff7f..ee688b0a13c3 100644 --- a/arch/arm/include/asm/ptrace.h +++ b/arch/arm/include/asm/ptrace.h @@ -80,6 +80,12 @@ static inline long regs_return_value(struct pt_regs *regs) #define instruction_pointer(regs) (regs)->ARM_pc +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + instruction_pointer(regs) = val; +} + #ifdef CONFIG_SMP extern unsigned long profile_pc(struct pt_regs *regs); #else diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 71a06b293489..f989d7c22dc5 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -153,6 +153,7 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, #define TIF_SIGPENDING 0 #define TIF_NEED_RESCHED 1 #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ +#define TIF_UPROBE 7 #define TIF_SYSCALL_TRACE 8 #define TIF_SYSCALL_AUDIT 9 #define TIF_SYSCALL_TRACEPOINT 10 @@ -165,6 +166,7 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) @@ -178,7 +180,8 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, /* * Change these and you break ASM code in entry-common.S */ -#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | _TIF_NOTIFY_RESUME) +#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ + _TIF_NOTIFY_RESUME | _TIF_UPROBE) #endif /* __KERNEL__ */ #endif /* __ASM_ARM_THREAD_INFO_H */ diff --git a/arch/arm/include/asm/uprobes.h b/arch/arm/include/asm/uprobes.h new file mode 100644 index 000000000000..9472c20b7d49 --- /dev/null +++ b/arch/arm/include/asm/uprobes.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012 Rabin Vincent + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ASM_UPROBES_H +#define _ASM_UPROBES_H + +#include +#include + +typedef u32 uprobe_opcode_t; + +#define MAX_UINSN_BYTES 4 +#define UPROBE_XOL_SLOT_BYTES 64 + +#define UPROBE_SWBP_ARM_INSN 0xe7f001f9 +#define UPROBE_SS_ARM_INSN 0xe7f001fa +#define UPROBE_SWBP_INSN __opcode_to_mem_arm(UPROBE_SWBP_ARM_INSN) +#define UPROBE_SWBP_INSN_SIZE 4 + +struct arch_uprobe_task { + u32 backup; + unsigned long saved_trap_no; +}; + +struct arch_uprobe { + u8 insn[MAX_UINSN_BYTES]; + unsigned long ixol[2]; + uprobe_opcode_t bpinsn; + bool simulate; + u32 pcreg; + void (*prehandler)(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs); + void (*posthandler)(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs); + struct arch_probes_insn asi; +}; + +#endif diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index bb739f28dd80..a766bcbaf8ad 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -50,6 +50,7 @@ obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o insn.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o insn.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o insn.o patch.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_UPROBES) += probes.o probes-arm.o uprobes.o uprobes-arm.o obj-$(CONFIG_KPROBES) += probes.o kprobes.o kprobes-common.o patch.o ifdef CONFIG_THUMB2_KERNEL obj-$(CONFIG_KPROBES) += kprobes-thumb.o probes-thumb.o diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index 04d63880037f..bd1983437205 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -590,6 +591,9 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) return restart; } syscall = 0; + } else if (thread_flags & _TIF_UPROBE) { + clear_thread_flag(TIF_UPROBE); + uprobe_notify_resume(regs); } else { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); diff --git a/arch/arm/kernel/uprobes-arm.c b/arch/arm/kernel/uprobes-arm.c new file mode 100644 index 000000000000..d3b655ff17da --- /dev/null +++ b/arch/arm/kernel/uprobes-arm.c @@ -0,0 +1,234 @@ +/* + * Copyright (C) 2012 Rabin Vincent + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include + +#include "probes.h" +#include "probes-arm.h" +#include "uprobes.h" + +static int uprobes_substitute_pc(unsigned long *pinsn, u32 oregs) +{ + probes_opcode_t insn = __mem_to_opcode_arm(*pinsn); + probes_opcode_t temp; + probes_opcode_t mask; + int freereg; + u32 free = 0xffff; + u32 regs; + + for (regs = oregs; regs; regs >>= 4, insn >>= 4) { + if ((regs & 0xf) == REG_TYPE_NONE) + continue; + + free &= ~(1 << (insn & 0xf)); + } + + /* No PC, no problem */ + if (free & (1 << 15)) + return 15; + + if (!free) + return -1; + + /* + * fls instead of ffs ensures that for "ldrd r0, r1, [pc]" we would + * pick LR instead of R1. + */ + freereg = free = fls(free) - 1; + + temp = __mem_to_opcode_arm(*pinsn); + insn = temp; + regs = oregs; + mask = 0xf; + + for (; regs; regs >>= 4, mask <<= 4, free <<= 4, temp >>= 4) { + if ((regs & 0xf) == REG_TYPE_NONE) + continue; + + if ((temp & 0xf) != 15) + continue; + + insn &= ~mask; + insn |= free & mask; + } + + *pinsn = __opcode_to_mem_arm(insn); + return freereg; +} + +static void uprobe_set_pc(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs) +{ + u32 pcreg = auprobe->pcreg; + + autask->backup = regs->uregs[pcreg]; + regs->uregs[pcreg] = regs->ARM_pc + 8; +} + +static void uprobe_unset_pc(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs) +{ + /* PC will be taken care of by common code */ + regs->uregs[auprobe->pcreg] = autask->backup; +} + +static void uprobe_aluwrite_pc(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs) +{ + u32 pcreg = auprobe->pcreg; + + alu_write_pc(regs->uregs[pcreg], regs); + regs->uregs[pcreg] = autask->backup; +} + +static void uprobe_write_pc(struct arch_uprobe *auprobe, + struct arch_uprobe_task *autask, + struct pt_regs *regs) +{ + u32 pcreg = auprobe->pcreg; + + load_write_pc(regs->uregs[pcreg], regs); + regs->uregs[pcreg] = autask->backup; +} + +enum probes_insn +decode_pc_ro(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) +{ + struct arch_uprobe *auprobe = container_of(asi, struct arch_uprobe, + asi); + struct decode_emulate *decode = (struct decode_emulate *) d; + u32 regs = decode->header.type_regs.bits >> DECODE_TYPE_BITS; + int reg; + + reg = uprobes_substitute_pc(&auprobe->ixol[0], regs); + if (reg == 15) + return INSN_GOOD; + + if (reg == -1) + return INSN_REJECTED; + + auprobe->pcreg = reg; + auprobe->prehandler = uprobe_set_pc; + auprobe->posthandler = uprobe_unset_pc; + + return INSN_GOOD; +} + +enum probes_insn +decode_wb_pc(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d, bool alu) +{ + struct arch_uprobe *auprobe = container_of(asi, struct arch_uprobe, + asi); + enum probes_insn ret = decode_pc_ro(insn, asi, d); + + if (((insn >> 12) & 0xf) == 15) + auprobe->posthandler = alu ? uprobe_aluwrite_pc + : uprobe_write_pc; + + return ret; +} + +enum probes_insn +decode_rd12rn16rm0rs8_rwflags(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d) +{ + return decode_wb_pc(insn, asi, d, true); +} + +enum probes_insn +decode_ldr(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d) +{ + return decode_wb_pc(insn, asi, d, false); +} + +enum probes_insn +uprobe_decode_ldmstm(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d) +{ + struct arch_uprobe *auprobe = container_of(asi, struct arch_uprobe, + asi); + unsigned reglist = insn & 0xffff; + int rn = (insn >> 16) & 0xf; + int lbit = insn & (1 << 20); + unsigned used = reglist | (1 << rn); + + if (rn == 15) + return INSN_REJECTED; + + if (!(used & (1 << 15))) + return INSN_GOOD; + + if (used & (1 << 14)) + return INSN_REJECTED; + + /* Use LR instead of PC */ + insn ^= 0xc000; + + auprobe->pcreg = 14; + auprobe->ixol[0] = __opcode_to_mem_arm(insn); + + auprobe->prehandler = uprobe_set_pc; + if (lbit) + auprobe->posthandler = uprobe_write_pc; + else + auprobe->posthandler = uprobe_unset_pc; + + return INSN_GOOD; +} + +const union decode_action uprobes_probes_actions[] = { + [PROBES_EMULATE_NONE] = {.handler = probes_simulate_nop}, + [PROBES_SIMULATE_NOP] = {.handler = probes_simulate_nop}, + [PROBES_PRELOAD_IMM] = {.handler = probes_simulate_nop}, + [PROBES_PRELOAD_REG] = {.handler = probes_simulate_nop}, + [PROBES_BRANCH_IMM] = {.handler = simulate_blx1}, + [PROBES_MRS] = {.handler = simulate_mrs}, + [PROBES_BRANCH_REG] = {.handler = simulate_blx2bx}, + [PROBES_CLZ] = {.handler = probes_simulate_nop}, + [PROBES_SATURATING_ARITHMETIC] = {.handler = probes_simulate_nop}, + [PROBES_MUL1] = {.handler = probes_simulate_nop}, + [PROBES_MUL2] = {.handler = probes_simulate_nop}, + [PROBES_SWP] = {.handler = probes_simulate_nop}, + [PROBES_LDRSTRD] = {.decoder = decode_pc_ro}, + [PROBES_LOAD_EXTRA] = {.decoder = decode_pc_ro}, + [PROBES_LOAD] = {.decoder = decode_ldr}, + [PROBES_STORE_EXTRA] = {.decoder = decode_pc_ro}, + [PROBES_STORE] = {.decoder = decode_pc_ro}, + [PROBES_MOV_IP_SP] = {.handler = simulate_mov_ipsp}, + [PROBES_DATA_PROCESSING_REG] = { + .decoder = decode_rd12rn16rm0rs8_rwflags}, + [PROBES_DATA_PROCESSING_IMM] = { + .decoder = decode_rd12rn16rm0rs8_rwflags}, + [PROBES_MOV_HALFWORD] = {.handler = probes_simulate_nop}, + [PROBES_SEV] = {.handler = probes_simulate_nop}, + [PROBES_WFE] = {.handler = probes_simulate_nop}, + [PROBES_SATURATE] = {.handler = probes_simulate_nop}, + [PROBES_REV] = {.handler = probes_simulate_nop}, + [PROBES_MMI] = {.handler = probes_simulate_nop}, + [PROBES_PACK] = {.handler = probes_simulate_nop}, + [PROBES_EXTEND] = {.handler = probes_simulate_nop}, + [PROBES_EXTEND_ADD] = {.handler = probes_simulate_nop}, + [PROBES_MUL_ADD_LONG] = {.handler = probes_simulate_nop}, + [PROBES_MUL_ADD] = {.handler = probes_simulate_nop}, + [PROBES_BITFIELD] = {.handler = probes_simulate_nop}, + [PROBES_BRANCH] = {.handler = simulate_bbl}, + [PROBES_LDMSTM] = {.decoder = uprobe_decode_ldmstm} +}; diff --git a/arch/arm/kernel/uprobes.c b/arch/arm/kernel/uprobes.c new file mode 100644 index 000000000000..f9bacee973bf --- /dev/null +++ b/arch/arm/kernel/uprobes.c @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2012 Rabin Vincent + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "probes.h" +#include "probes-arm.h" +#include "uprobes.h" + +#define UPROBE_TRAP_NR UINT_MAX + +bool is_swbp_insn(uprobe_opcode_t *insn) +{ + return (__mem_to_opcode_arm(*insn) & 0x0fffffff) == + (UPROBE_SWBP_ARM_INSN & 0x0fffffff); +} + +int set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long vaddr) +{ + return uprobe_write_opcode(mm, vaddr, + __opcode_to_mem_arm(auprobe->bpinsn)); +} + +bool arch_uprobe_ignore(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + if (!auprobe->asi.insn_check_cc(regs->ARM_cpsr)) { + regs->ARM_pc += 4; + return true; + } + + return false; +} + +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + probes_opcode_t opcode; + + if (!auprobe->simulate) + return false; + + opcode = __mem_to_opcode_arm(*(unsigned int *) auprobe->insn); + + auprobe->asi.insn_singlestep(opcode, &auprobe->asi, regs); + + return true; +} + +unsigned long +arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long orig_ret_vaddr; + + orig_ret_vaddr = regs->ARM_lr; + /* Replace the return addr with trampoline addr */ + regs->ARM_lr = trampoline_vaddr; + return orig_ret_vaddr; +} + +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long addr) +{ + unsigned int insn; + unsigned int bpinsn; + enum probes_insn ret; + + /* Thumb not yet support */ + if (addr & 0x3) + return -EINVAL; + + insn = __mem_to_opcode_arm(*(unsigned int *)auprobe->insn); + auprobe->ixol[0] = __opcode_to_mem_arm(insn); + auprobe->ixol[1] = __opcode_to_mem_arm(UPROBE_SS_ARM_INSN); + + ret = arm_probes_decode_insn(insn, &auprobe->asi, false, + uprobes_probes_actions); + switch (ret) { + case INSN_REJECTED: + return -EINVAL; + + case INSN_GOOD_NO_SLOT: + auprobe->simulate = true; + break; + + case INSN_GOOD: + default: + break; + } + + bpinsn = UPROBE_SWBP_ARM_INSN & 0x0fffffff; + if (insn >= 0xe0000000) + bpinsn |= 0xe0000000; /* Unconditional instruction */ + else + bpinsn |= insn & 0xf0000000; /* Copy condition from insn */ + + auprobe->bpinsn = bpinsn; + + return 0; +} + +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + if (auprobe->prehandler) + auprobe->prehandler(auprobe, &utask->autask, regs); + + utask->autask.saved_trap_no = current->thread.trap_no; + current->thread.trap_no = UPROBE_TRAP_NR; + regs->ARM_pc = utask->xol_vaddr; + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + WARN_ON_ONCE(current->thread.trap_no != UPROBE_TRAP_NR); + + current->thread.trap_no = utask->autask.saved_trap_no; + regs->ARM_pc = utask->vaddr + 4; + + if (auprobe->posthandler) + auprobe->posthandler(auprobe, &utask->autask, regs); + + return 0; +} + +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + if (t->thread.trap_no != UPROBE_TRAP_NR) + return true; + + return false; +} + +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + current->thread.trap_no = utask->autask.saved_trap_no; + instruction_pointer_set(regs, utask->vaddr); +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} + +static int uprobe_trap_handler(struct pt_regs *regs, unsigned int instr) +{ + unsigned long flags; + + local_irq_save(flags); + instr &= 0x0fffffff; + if (instr == (UPROBE_SWBP_ARM_INSN & 0x0fffffff)) + uprobe_pre_sstep_notifier(regs); + else if (instr == (UPROBE_SS_ARM_INSN & 0x0fffffff)) + uprobe_post_sstep_notifier(regs); + local_irq_restore(flags); + + return 0; +} + +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +static struct undef_hook uprobes_arm_break_hook = { + .instr_mask = 0x0fffffff, + .instr_val = (UPROBE_SWBP_ARM_INSN & 0x0fffffff), + .cpsr_mask = MODE_MASK, + .cpsr_val = USR_MODE, + .fn = uprobe_trap_handler, +}; + +static struct undef_hook uprobes_arm_ss_hook = { + .instr_mask = 0x0fffffff, + .instr_val = (UPROBE_SS_ARM_INSN & 0x0fffffff), + .cpsr_mask = MODE_MASK, + .cpsr_val = USR_MODE, + .fn = uprobe_trap_handler, +}; + +static int arch_uprobes_init(void) +{ + register_undef_hook(&uprobes_arm_break_hook); + register_undef_hook(&uprobes_arm_ss_hook); + + return 0; +} +device_initcall(arch_uprobes_init); diff --git a/arch/arm/kernel/uprobes.h b/arch/arm/kernel/uprobes.h new file mode 100644 index 000000000000..1d0c12dfbd03 --- /dev/null +++ b/arch/arm/kernel/uprobes.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2012 Rabin Vincent + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __ARM_KERNEL_UPROBES_H +#define __ARM_KERNEL_UPROBES_H + +enum probes_insn uprobe_decode_ldmstm(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d); + +enum probes_insn decode_ldr(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d); + +enum probes_insn +decode_rd12rn16rm0rs8_rwflags(probes_opcode_t insn, + struct arch_probes_insn *asi, + const struct decode_header *d); + +enum probes_insn +decode_wb_pc(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d, bool alu); + +enum probes_insn +decode_pc_ro(probes_opcode_t insn, struct arch_probes_insn *asi, + const struct decode_header *d); + +extern const union decode_action uprobes_probes_actions[]; + +#endif From 1ef2bf8227abe928e149772f9e6ee9d59c1bd97e Mon Sep 17 00:00:00 2001 From: Alexander Shiyan Date: Sun, 16 Mar 2014 05:54:57 +0100 Subject: [PATCH 61/89] ARM: 8006/1: Remove redundant code Statements following return will never be executed. This patch removes this code. Signed-off-by: Alexander Shiyan Signed-off-by: Russell King --- arch/arm/common/scoop.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm/common/scoop.c b/arch/arm/common/scoop.c index a5c3dc38aa18..6ef146edd0cd 100644 --- a/arch/arm/common/scoop.c +++ b/arch/arm/common/scoop.c @@ -232,8 +232,6 @@ static int scoop_probe(struct platform_device *pdev) return 0; - if (devptr->gpio.base != -1) - temp = gpiochip_remove(&devptr->gpio); err_gpio: platform_set_drvdata(pdev, NULL); err_ioremap: From 95c52fe063351192e0f4ffb70ef9bac1aa26f5a4 Mon Sep 17 00:00:00 2001 From: Christopher Covington Date: Wed, 19 Mar 2014 18:12:25 +0100 Subject: [PATCH 62/89] ARM: 8007/1: Remove extraneous kcmp syscall ignore The kcmp system call was ported to ARM in commit 3f7d1fe108dbaefd0c57a41753fc2c90b395f458 "ARM: 7665/1: Wire up kcmp syscall". Fixes: 3f7d1fe108db ("ARM: 7665/1: Wire up kcmp syscall") Signed-off-by: Christopher Covington Signed-off-by: Russell King --- arch/arm/include/asm/unistd.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h index acabef1a75df..43876245fc57 100644 --- a/arch/arm/include/asm/unistd.h +++ b/arch/arm/include/asm/unistd.h @@ -48,6 +48,5 @@ */ #define __IGNORE_fadvise64_64 #define __IGNORE_migrate_pages -#define __IGNORE_kcmp #endif /* __ASM_ARM_UNISTD_H */ From e26a9e00afc482b971afcaef1db8c9034d4d6d7c Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 25 Mar 2014 19:45:31 +0000 Subject: [PATCH 63/89] ARM: Better virt_to_page() handling virt_to_page() is incredibly inefficient when virt-to-phys patching is enabled. This is because we end up with this calculation: page = &mem_map[asm virt_to_phys(addr) >> 12 - __pv_phys_offset >> 12] in assembly. The asm virt_to_phys() is equivalent this this operation: addr - PAGE_OFFSET + __pv_phys_offset and we can see that because this is assembly, the compiler has no chance to optimise some of that away. This should reduce down to: page = &mem_map[(addr - PAGE_OFFSET) >> 12] for the common cases. Permit the compiler to make this optimisation by giving it more of the information it needs - do this by providing a virt_to_pfn() macro. Another issue which makes this more complex is that __pv_phys_offset is a 64-bit type on all platforms. This is needlessly wasteful - if we store the physical offset as a PFN, we can save a lot of work having to deal with 64-bit values, which sometimes ends up producing incredibly horrid code: a4c: e3009000 movw r9, #0 a4c: R_ARM_MOVW_ABS_NC __pv_phys_offset a50: e3409000 movt r9, #0 ; r9 = &__pv_phys_offset a50: R_ARM_MOVT_ABS __pv_phys_offset a54: e3002000 movw r2, #0 a54: R_ARM_MOVW_ABS_NC __pv_phys_offset a58: e3402000 movt r2, #0 ; r2 = &__pv_phys_offset a58: R_ARM_MOVT_ABS __pv_phys_offset a5c: e5999004 ldr r9, [r9, #4] ; r9 = high word of __pv_phys_offset a60: e3001000 movw r1, #0 a60: R_ARM_MOVW_ABS_NC mem_map a64: e592c000 ldr ip, [r2] ; ip = low word of __pv_phys_offset Reviewed-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/include/asm/memory.h | 41 ++++++++++++++++++++--------------- arch/arm/kernel/armksyms.c | 2 +- arch/arm/kernel/head.S | 17 ++++++++------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index 8756e4bcdba0..2438d72cf4e6 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -169,9 +169,17 @@ * Physical vs virtual RAM address space conversion. These are * private definitions which should NOT be used outside memory.h * files. Use virt_to_phys/phys_to_virt/__pa/__va instead. + * + * PFNs are used to describe any physical page; this means + * PFN 0 == physical address 0. */ -#ifndef __virt_to_phys -#ifdef CONFIG_ARM_PATCH_PHYS_VIRT +#if defined(__virt_to_phys) +#define PHYS_OFFSET PLAT_PHYS_OFFSET +#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT)) + +#define virt_to_pfn(kaddr) (__pa(kaddr) >> PAGE_SHIFT) + +#elif defined(CONFIG_ARM_PATCH_PHYS_VIRT) /* * Constants used to force the right instruction encodings and shifts @@ -180,12 +188,17 @@ #define __PV_BITS_31_24 0x81000000 #define __PV_BITS_7_0 0x81 -extern u64 __pv_phys_offset; +extern unsigned long __pv_phys_pfn_offset; extern u64 __pv_offset; extern void fixup_pv_table(const void *, unsigned long); extern const void *__pv_table_begin, *__pv_table_end; -#define PHYS_OFFSET __pv_phys_offset +#define PHYS_OFFSET ((phys_addr_t)__pv_phys_pfn_offset << PAGE_SHIFT) +#define PHYS_PFN_OFFSET (__pv_phys_pfn_offset) + +#define virt_to_pfn(kaddr) \ + ((((unsigned long)(kaddr) - PAGE_OFFSET) >> PAGE_SHIFT) + \ + PHYS_PFN_OFFSET) #define __pv_stub(from,to,instr,type) \ __asm__("@ __pv_stub\n" \ @@ -246,6 +259,7 @@ static inline unsigned long __phys_to_virt(phys_addr_t x) #else #define PHYS_OFFSET PLAT_PHYS_OFFSET +#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT)) static inline phys_addr_t __virt_to_phys(unsigned long x) { @@ -257,18 +271,11 @@ static inline unsigned long __phys_to_virt(phys_addr_t x) return x - PHYS_OFFSET + PAGE_OFFSET; } -#endif -#endif +#define virt_to_pfn(kaddr) \ + ((((unsigned long)(kaddr) - PAGE_OFFSET) >> PAGE_SHIFT) + \ + PHYS_PFN_OFFSET) -/* - * PFNs are used to describe any physical page; this means - * PFN 0 == physical address 0. - * - * This is the PFN of the first RAM page in the kernel - * direct-mapped view. We assume this is the first page - * of RAM in the mem_map as well. - */ -#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT)) +#endif /* * These are *only* valid on the kernel direct mapped RAM memory. @@ -346,9 +353,9 @@ static inline __deprecated void *bus_to_virt(unsigned long x) */ #define ARCH_PFN_OFFSET PHYS_PFN_OFFSET -#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) +#define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr)) #define virt_addr_valid(kaddr) (((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory) \ - && pfn_valid(__pa(kaddr) >> PAGE_SHIFT) ) + && pfn_valid(virt_to_pfn(kaddr))) #endif diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c index 85e664b6a5f1..f7b450f97e68 100644 --- a/arch/arm/kernel/armksyms.c +++ b/arch/arm/kernel/armksyms.c @@ -158,6 +158,6 @@ EXPORT_SYMBOL(__gnu_mcount_nc); #endif #ifdef CONFIG_ARM_PATCH_PHYS_VIRT -EXPORT_SYMBOL(__pv_phys_offset); +EXPORT_SYMBOL(__pv_phys_pfn_offset); EXPORT_SYMBOL(__pv_offset); #endif diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 914616e0bdcd..3aca959fee8d 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -584,9 +584,10 @@ __fixup_pv_table: subs r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET add r4, r4, r3 @ adjust table start address add r5, r5, r3 @ adjust table end address - add r6, r6, r3 @ adjust __pv_phys_offset address + add r6, r6, r3 @ adjust __pv_phys_pfn_offset address add r7, r7, r3 @ adjust __pv_offset address - str r8, [r6, #LOW_OFFSET] @ save computed PHYS_OFFSET to __pv_phys_offset + mov r0, r8, lsr #12 @ convert to PFN + str r0, [r6, #LOW_OFFSET] @ save computed PHYS_OFFSET to __pv_phys_pfn_offset strcc ip, [r7, #HIGH_OFFSET] @ save to __pv_offset high bits mov r6, r3, lsr #24 @ constant for add/sub instructions teq r3, r6, lsl #24 @ must be 16MiB aligned @@ -600,7 +601,7 @@ ENDPROC(__fixup_pv_table) 1: .long . .long __pv_table_begin .long __pv_table_end -2: .long __pv_phys_offset +2: .long __pv_phys_pfn_offset .long __pv_offset .text @@ -688,11 +689,11 @@ ENTRY(fixup_pv_table) ENDPROC(fixup_pv_table) .data - .globl __pv_phys_offset - .type __pv_phys_offset, %object -__pv_phys_offset: - .quad 0 - .size __pv_phys_offset, . -__pv_phys_offset + .globl __pv_phys_pfn_offset + .type __pv_phys_pfn_offset, %object +__pv_phys_pfn_offset: + .word 0 + .size __pv_phys_pfn_offset, . -__pv_phys_pfn_offset .globl __pv_offset .type __pv_offset, %object From 104fce73fdbd174eb08a493eeb2920fd59e6d3f4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 12:58:29 +0000 Subject: [PATCH 64/89] dmaengine: omap-dma: use devm_kzalloc() to allocate omap_dmadev. Use devm_kzalloc() to allocate omap_dmadev() so that we don't need complex error cleanup paths. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 362e7c49f2e1..98034e8c558f 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -594,7 +594,6 @@ static void omap_dma_free(struct omap_dmadev *od) tasklet_kill(&c->vc.task); kfree(c); } - kfree(od); } static int omap_dma_probe(struct platform_device *pdev) @@ -602,7 +601,7 @@ static int omap_dma_probe(struct platform_device *pdev) struct omap_dmadev *od; int rc, i; - od = kzalloc(sizeof(*od), GFP_KERNEL); + od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); if (!od) return -ENOMEM; From 1b416c4b41351c3eb8fc42dbb4cd8eba463c0813 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 13:00:03 +0000 Subject: [PATCH 65/89] dmaengine: omap-dma: provide a hook to get the underlying DMA platform ops Provide and use a hook to obtain the underlying DMA platform operations so that omap-dma.c can access the hardware more directly without involving the legacy DMA driver. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- arch/arm/plat-omap/dma.c | 6 ++++++ drivers/dma/omap-dma.c | 7 +++++++ include/linux/omap-dma.h | 2 ++ 3 files changed, 15 insertions(+) diff --git a/arch/arm/plat-omap/dma.c b/arch/arm/plat-omap/dma.c index 01619c2910e3..d4d9a5e62152 100644 --- a/arch/arm/plat-omap/dma.c +++ b/arch/arm/plat-omap/dma.c @@ -2000,6 +2000,12 @@ void omap_dma_global_context_restore(void) omap_clear_dma(ch); } +struct omap_system_dma_plat_info *omap_get_plat_info(void) +{ + return p; +} +EXPORT_SYMBOL_GPL(omap_get_plat_info); + static int omap_system_dma_probe(struct platform_device *pdev) { int ch, ret = 0; diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 98034e8c558f..4ac26bf0ad30 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -26,11 +26,13 @@ struct omap_dmadev { spinlock_t lock; struct tasklet_struct task; struct list_head pending; + struct omap_system_dma_plat_info *plat; }; struct omap_chan { struct virt_dma_chan vc; struct list_head node; + struct omap_system_dma_plat_info *plat; struct dma_slave_config cfg; unsigned dma_sig; @@ -573,6 +575,7 @@ static int omap_dma_chan_init(struct omap_dmadev *od, int dma_sig) if (!c) return -ENOMEM; + c->plat = od->plat; c->dma_sig = dma_sig; c->vc.desc_free = omap_dma_desc_free; vchan_init(&c->vc, &od->ddev); @@ -605,6 +608,10 @@ static int omap_dma_probe(struct platform_device *pdev) if (!od) return -ENOMEM; + od->plat = omap_get_plat_info(); + if (!od->plat) + return -EPROBE_DEFER; + dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources; diff --git a/include/linux/omap-dma.h b/include/linux/omap-dma.h index 7af25a9c9c51..14742fc2aefe 100644 --- a/include/linux/omap-dma.h +++ b/include/linux/omap-dma.h @@ -292,6 +292,8 @@ struct omap_system_dma_plat_info { #define dma_omap15xx() ((dma_omap1() && (d->dev_caps & ENABLE_1510_MODE))) #define dma_omap16xx() ((dma_omap1() && (d->dev_caps & ENABLE_16XX_MODE))) +extern struct omap_system_dma_plat_info *omap_get_plat_info(void); + extern void omap_set_dma_priority(int lch, int dst_port, int priority); extern int omap_request_dma(int dev_id, const char *dev_name, void (*callback)(int lch, u16 ch_status, void *data), From b9e97822da374f52aaf99cb502f531ff2184b8f5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 13:26:57 +0000 Subject: [PATCH 66/89] dmaengine: omap-dma: program hardware directly Program the transfer parameters directly into the hardware, rather than using the functions in arch/arm/plat-omap/dma.c. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 142 ++++++++++++++++++++++++++++++++++----- include/linux/omap-dma.h | 6 +- 2 files changed, 130 insertions(+), 18 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 4ac26bf0ad30..47a3fa5bc38e 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -99,16 +99,94 @@ static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, unsigned idx) { struct omap_sg *sg = d->sg + idx; + uint32_t val; - if (d->dir == DMA_DEV_TO_MEM) - omap_set_dma_dest_params(c->dma_ch, OMAP_DMA_PORT_EMIFF, - OMAP_DMA_AMODE_POST_INC, sg->addr, 0, 0); - else - omap_set_dma_src_params(c->dma_ch, OMAP_DMA_PORT_EMIFF, - OMAP_DMA_AMODE_POST_INC, sg->addr, 0, 0); + if (d->dir == DMA_DEV_TO_MEM) { + if (dma_omap1()) { + val = c->plat->dma_read(CSDP, c->dma_ch); + val &= ~(0x1f << 9); + val |= OMAP_DMA_PORT_EMIFF << 9; + c->plat->dma_write(val, CSDP, c->dma_ch); + } - omap_set_dma_transfer_params(c->dma_ch, d->es, sg->en, sg->fn, - d->sync_mode, c->dma_sig, d->sync_type); + val = c->plat->dma_read(CCR, c->dma_ch); + val &= ~(0x03 << 14); + val |= OMAP_DMA_AMODE_POST_INC << 14; + c->plat->dma_write(val, CCR, c->dma_ch); + + c->plat->dma_write(sg->addr, CDSA, c->dma_ch); + c->plat->dma_write(0, CDEI, c->dma_ch); + c->plat->dma_write(0, CDFI, c->dma_ch); + } else { + if (dma_omap1()) { + val = c->plat->dma_read(CSDP, c->dma_ch); + val &= ~(0x1f << 2); + val |= OMAP_DMA_PORT_EMIFF << 2; + c->plat->dma_write(val, CSDP, c->dma_ch); + } + + val = c->plat->dma_read(CCR, c->dma_ch); + val &= ~(0x03 << 12); + val |= OMAP_DMA_AMODE_POST_INC << 12; + c->plat->dma_write(val, CCR, c->dma_ch); + + c->plat->dma_write(sg->addr, CSSA, c->dma_ch); + c->plat->dma_write(0, CSEI, c->dma_ch); + c->plat->dma_write(0, CSFI, c->dma_ch); + } + + val = c->plat->dma_read(CSDP, c->dma_ch); + val &= ~0x03; + val |= d->es; + c->plat->dma_write(val, CSDP, c->dma_ch); + + if (dma_omap1()) { + val = c->plat->dma_read(CCR, c->dma_ch); + val &= ~(1 << 5); + if (d->sync_mode == OMAP_DMA_SYNC_FRAME) + val |= 1 << 5; + c->plat->dma_write(val, CCR, c->dma_ch); + + val = c->plat->dma_read(CCR2, c->dma_ch); + val &= ~(1 << 2); + if (d->sync_mode == OMAP_DMA_SYNC_BLOCK) + val |= 1 << 2; + c->plat->dma_write(val, CCR2, c->dma_ch); + } else if (c->dma_sig) { + val = c->plat->dma_read(CCR, c->dma_ch); + + /* DMA_SYNCHRO_CONTROL_UPPER depends on the channel number */ + val &= ~((1 << 23) | (3 << 19) | 0x1f); + val |= (c->dma_sig & ~0x1f) << 14; + val |= c->dma_sig & 0x1f; + + if (d->sync_mode & OMAP_DMA_SYNC_FRAME) + val |= 1 << 5; + else + val &= ~(1 << 5); + + if (d->sync_mode & OMAP_DMA_SYNC_BLOCK) + val |= 1 << 18; + else + val &= ~(1 << 18); + + switch (d->sync_type) { + case OMAP_DMA_DST_SYNC_PREFETCH: + val &= ~(1 << 24); /* dest synch */ + val |= 1 << 23; /* Prefetch */ + break; + case 0: + val &= ~(1 << 24); /* dest synch */ + break; + default: + val |= 1 << 24; /* source synch */ + break; + } + c->plat->dma_write(val, CCR, c->dma_ch); + } + + c->plat->dma_write(sg->en, CEN, c->dma_ch); + c->plat->dma_write(sg->fn, CFN, c->dma_ch); omap_start_dma(c->dma_ch); } @@ -117,6 +195,7 @@ static void omap_dma_start_desc(struct omap_chan *c) { struct virt_dma_desc *vd = vchan_next_desc(&c->vc); struct omap_desc *d; + uint32_t val; if (!vd) { c->desc = NULL; @@ -128,12 +207,39 @@ static void omap_dma_start_desc(struct omap_chan *c) c->desc = d = to_omap_dma_desc(&vd->tx); c->sgidx = 0; - if (d->dir == DMA_DEV_TO_MEM) - omap_set_dma_src_params(c->dma_ch, d->periph_port, - OMAP_DMA_AMODE_CONSTANT, d->dev_addr, 0, d->fi); - else - omap_set_dma_dest_params(c->dma_ch, d->periph_port, - OMAP_DMA_AMODE_CONSTANT, d->dev_addr, 0, d->fi); + if (d->dir == DMA_DEV_TO_MEM) { + if (dma_omap1()) { + val = c->plat->dma_read(CSDP, c->dma_ch); + val &= ~(0x1f << 2); + val |= d->periph_port << 2; + c->plat->dma_write(val, CSDP, c->dma_ch); + } + + val = c->plat->dma_read(CCR, c->dma_ch); + val &= ~(0x03 << 12); + val |= OMAP_DMA_AMODE_CONSTANT << 12; + c->plat->dma_write(val, CCR, c->dma_ch); + + c->plat->dma_write(d->dev_addr, CSSA, c->dma_ch); + c->plat->dma_write(0, CSEI, c->dma_ch); + c->plat->dma_write(d->fi, CSFI, c->dma_ch); + } else { + if (dma_omap1()) { + val = c->plat->dma_read(CSDP, c->dma_ch); + val &= ~(0x1f << 9); + val |= d->periph_port << 9; + c->plat->dma_write(val, CSDP, c->dma_ch); + } + + val = c->plat->dma_read(CCR, c->dma_ch); + val &= ~(0x03 << 14); + val |= OMAP_DMA_AMODE_CONSTANT << 14; + c->plat->dma_write(val, CCR, c->dma_ch); + + c->plat->dma_write(d->dev_addr, CDSA, c->dma_ch); + c->plat->dma_write(0, CDEI, c->dma_ch); + c->plat->dma_write(d->fi, CDFI, c->dma_ch); + } omap_dma_start_sg(c, d, 0); } @@ -452,8 +558,12 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( } if (dma_omap2plus()) { - omap_set_dma_src_burst_mode(c->dma_ch, OMAP_DMA_DATA_BURST_16); - omap_set_dma_dest_burst_mode(c->dma_ch, OMAP_DMA_DATA_BURST_16); + uint32_t val; + + val = c->plat->dma_read(CSDP, c->dma_ch); + val |= 0x03 << 7; /* src burst mode 16 */ + val |= 0x03 << 14; /* dst burst mode 16 */ + c->plat->dma_write(val, CSDP, c->dma_ch); } return vchan_tx_prep(&c->vc, &d->vd, flags); diff --git a/include/linux/omap-dma.h b/include/linux/omap-dma.h index 14742fc2aefe..d631658e2237 100644 --- a/include/linux/omap-dma.h +++ b/include/linux/omap-dma.h @@ -289,8 +289,10 @@ struct omap_system_dma_plat_info { #define dma_omap2plus() 0 #endif #define dma_omap1() (!dma_omap2plus()) -#define dma_omap15xx() ((dma_omap1() && (d->dev_caps & ENABLE_1510_MODE))) -#define dma_omap16xx() ((dma_omap1() && (d->dev_caps & ENABLE_16XX_MODE))) +#define __dma_omap15xx(d) (dma_omap1() && (d)->dev_caps & ENABLE_1510_MODE) +#define __dma_omap16xx(d) (dma_omap1() && (d)->dev_caps & ENABLE_16XX_MODE) +#define dma_omap15xx() __dma_omap15xx(d) +#define dma_omap16xx() __dma_omap16xx(d) extern struct omap_system_dma_plat_info *omap_get_plat_info(void); From 913a2d0c6952283bc9323cb9152af87f792ff4c4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 14:41:42 +0000 Subject: [PATCH 67/89] dmaengine: omap-dma: consolidate writes to DMA registers There's no need to keep writing registers which don't change value in omap_dma_start_sg(). Move this into omap_dma_start_desc() and merge the register updates together. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 131 ++++++++++++++++------------------------- 1 file changed, 52 insertions(+), 79 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 47a3fa5bc38e..8c5c862f01ed 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -99,92 +99,17 @@ static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, unsigned idx) { struct omap_sg *sg = d->sg + idx; - uint32_t val; if (d->dir == DMA_DEV_TO_MEM) { - if (dma_omap1()) { - val = c->plat->dma_read(CSDP, c->dma_ch); - val &= ~(0x1f << 9); - val |= OMAP_DMA_PORT_EMIFF << 9; - c->plat->dma_write(val, CSDP, c->dma_ch); - } - - val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(0x03 << 14); - val |= OMAP_DMA_AMODE_POST_INC << 14; - c->plat->dma_write(val, CCR, c->dma_ch); - c->plat->dma_write(sg->addr, CDSA, c->dma_ch); c->plat->dma_write(0, CDEI, c->dma_ch); c->plat->dma_write(0, CDFI, c->dma_ch); } else { - if (dma_omap1()) { - val = c->plat->dma_read(CSDP, c->dma_ch); - val &= ~(0x1f << 2); - val |= OMAP_DMA_PORT_EMIFF << 2; - c->plat->dma_write(val, CSDP, c->dma_ch); - } - - val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(0x03 << 12); - val |= OMAP_DMA_AMODE_POST_INC << 12; - c->plat->dma_write(val, CCR, c->dma_ch); - c->plat->dma_write(sg->addr, CSSA, c->dma_ch); c->plat->dma_write(0, CSEI, c->dma_ch); c->plat->dma_write(0, CSFI, c->dma_ch); } - val = c->plat->dma_read(CSDP, c->dma_ch); - val &= ~0x03; - val |= d->es; - c->plat->dma_write(val, CSDP, c->dma_ch); - - if (dma_omap1()) { - val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(1 << 5); - if (d->sync_mode == OMAP_DMA_SYNC_FRAME) - val |= 1 << 5; - c->plat->dma_write(val, CCR, c->dma_ch); - - val = c->plat->dma_read(CCR2, c->dma_ch); - val &= ~(1 << 2); - if (d->sync_mode == OMAP_DMA_SYNC_BLOCK) - val |= 1 << 2; - c->plat->dma_write(val, CCR2, c->dma_ch); - } else if (c->dma_sig) { - val = c->plat->dma_read(CCR, c->dma_ch); - - /* DMA_SYNCHRO_CONTROL_UPPER depends on the channel number */ - val &= ~((1 << 23) | (3 << 19) | 0x1f); - val |= (c->dma_sig & ~0x1f) << 14; - val |= c->dma_sig & 0x1f; - - if (d->sync_mode & OMAP_DMA_SYNC_FRAME) - val |= 1 << 5; - else - val &= ~(1 << 5); - - if (d->sync_mode & OMAP_DMA_SYNC_BLOCK) - val |= 1 << 18; - else - val &= ~(1 << 18); - - switch (d->sync_type) { - case OMAP_DMA_DST_SYNC_PREFETCH: - val &= ~(1 << 24); /* dest synch */ - val |= 1 << 23; /* Prefetch */ - break; - case 0: - val &= ~(1 << 24); /* dest synch */ - break; - default: - val |= 1 << 24; /* source synch */ - break; - } - c->plat->dma_write(val, CCR, c->dma_ch); - } - c->plat->dma_write(sg->en, CEN, c->dma_ch); c->plat->dma_write(sg->fn, CFN, c->dma_ch); @@ -210,13 +135,15 @@ static void omap_dma_start_desc(struct omap_chan *c) if (d->dir == DMA_DEV_TO_MEM) { if (dma_omap1()) { val = c->plat->dma_read(CSDP, c->dma_ch); - val &= ~(0x1f << 2); + val &= ~(0x1f << 9 | 0x1f << 2); + val |= OMAP_DMA_PORT_EMIFF << 9; val |= d->periph_port << 2; c->plat->dma_write(val, CSDP, c->dma_ch); } val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(0x03 << 12); + val &= ~(0x03 << 14 | 0x03 << 12); + val |= OMAP_DMA_AMODE_POST_INC << 14; val |= OMAP_DMA_AMODE_CONSTANT << 12; c->plat->dma_write(val, CCR, c->dma_ch); @@ -226,14 +153,16 @@ static void omap_dma_start_desc(struct omap_chan *c) } else { if (dma_omap1()) { val = c->plat->dma_read(CSDP, c->dma_ch); - val &= ~(0x1f << 9); + val &= ~(0x1f << 9 | 0x1f << 2); val |= d->periph_port << 9; + val |= OMAP_DMA_PORT_EMIFF << 2; c->plat->dma_write(val, CSDP, c->dma_ch); } val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(0x03 << 14); + val &= ~(0x03 << 12 | 0x03 << 14); val |= OMAP_DMA_AMODE_CONSTANT << 14; + val |= OMAP_DMA_AMODE_POST_INC << 12; c->plat->dma_write(val, CCR, c->dma_ch); c->plat->dma_write(d->dev_addr, CDSA, c->dma_ch); @@ -241,6 +170,50 @@ static void omap_dma_start_desc(struct omap_chan *c) c->plat->dma_write(d->fi, CDFI, c->dma_ch); } + val = c->plat->dma_read(CSDP, c->dma_ch); + val &= ~0x03; + val |= d->es; + c->plat->dma_write(val, CSDP, c->dma_ch); + + if (dma_omap1()) { + val = c->plat->dma_read(CCR, c->dma_ch); + val &= ~(1 << 5); + if (d->sync_mode == OMAP_DMA_SYNC_FRAME) + val |= 1 << 5; + c->plat->dma_write(val, CCR, c->dma_ch); + + val = c->plat->dma_read(CCR2, c->dma_ch); + val &= ~(1 << 2); + if (d->sync_mode == OMAP_DMA_SYNC_BLOCK) + val |= 1 << 2; + c->plat->dma_write(val, CCR2, c->dma_ch); + } else if (c->dma_sig) { + val = c->plat->dma_read(CCR, c->dma_ch); + + /* DMA_SYNCHRO_CONTROL_UPPER depends on the channel number */ + val &= ~(1 << 24 | 1 << 23 | 3 << 19 | 1 << 18 | 1 << 5 | 0x1f); + val |= (c->dma_sig & ~0x1f) << 14; + val |= c->dma_sig & 0x1f; + + if (d->sync_mode & OMAP_DMA_SYNC_FRAME) + val |= 1 << 5; + + if (d->sync_mode & OMAP_DMA_SYNC_BLOCK) + val |= 1 << 18; + + switch (d->sync_type) { + case OMAP_DMA_DST_SYNC_PREFETCH:/* dest synch */ + val |= 1 << 23; /* Prefetch */ + break; + case 0: + break; + default: + val |= 1 << 24; /* source synch */ + break; + } + c->plat->dma_write(val, CCR, c->dma_ch); + } + omap_dma_start_sg(c, d, 0); } From fa3ad86ae0576b2c721800cc4d46864aa6d31ffd Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 17:07:09 +0000 Subject: [PATCH 68/89] dmaengine: omap-dma: control start/stop directly Program the non-cyclic mode DMA start/stop directly, rather than via arch/arm/plat-omap/dma.c. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 151 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 141 insertions(+), 10 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 8c5c862f01ed..7aa5ff7ab935 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -5,6 +5,7 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#include #include #include #include @@ -60,6 +61,7 @@ struct omap_desc { uint8_t sync_mode; /* OMAP_DMA_SYNC_xxx */ uint8_t sync_type; /* OMAP_DMA_xxx_SYNC* */ uint8_t periph_port; /* Peripheral port */ + uint16_t cicr; /* CICR value */ unsigned sglen; struct omap_sg sg[0]; @@ -95,6 +97,111 @@ static void omap_dma_desc_free(struct virt_dma_desc *vd) kfree(container_of(vd, struct omap_desc, vd)); } +static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + uint32_t val; + + if (__dma_omap15xx(od->plat->dma_attr)) + c->plat->dma_write(0, CPC, c->dma_ch); + else + c->plat->dma_write(0, CDAC, c->dma_ch); + + if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { + val = c->plat->dma_read(CLNK_CTRL, c->dma_ch); + + if (dma_omap1()) + val &= ~(1 << 14); + + val |= c->dma_ch | 1 << 15; + + c->plat->dma_write(val, CLNK_CTRL, c->dma_ch); + } else if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) + c->plat->dma_write(c->dma_ch, CLNK_CTRL, c->dma_ch); + + /* Clear CSR */ + if (dma_omap1()) + c->plat->dma_read(CSR, c->dma_ch); + else + c->plat->dma_write(~0, CSR, c->dma_ch); + + /* Enable interrupts */ + c->plat->dma_write(d->cicr, CICR, c->dma_ch); + + val = c->plat->dma_read(CCR, c->dma_ch); + if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) + val |= OMAP_DMA_CCR_BUFFERING_DISABLE; + val |= OMAP_DMA_CCR_EN; + mb(); + c->plat->dma_write(val, CCR, c->dma_ch); +} + +static void omap_dma_stop(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + uint32_t val; + + /* disable irq */ + c->plat->dma_write(0, CICR, c->dma_ch); + + /* Clear CSR */ + if (dma_omap1()) + c->plat->dma_read(CSR, c->dma_ch); + else + c->plat->dma_write(~0, CSR, c->dma_ch); + + val = c->plat->dma_read(CCR, c->dma_ch); + if (od->plat->errata & DMA_ERRATA_i541 && + val & OMAP_DMA_CCR_SEL_SRC_DST_SYNC) { + uint32_t sysconfig; + unsigned i; + + sysconfig = c->plat->dma_read(OCP_SYSCONFIG, c->dma_ch); + val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK; + val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE); + c->plat->dma_write(val, OCP_SYSCONFIG, c->dma_ch); + + val = c->plat->dma_read(CCR, c->dma_ch); + val &= ~OMAP_DMA_CCR_EN; + c->plat->dma_write(val, CCR, c->dma_ch); + + /* Wait for sDMA FIFO to drain */ + for (i = 0; ; i++) { + val = c->plat->dma_read(CCR, c->dma_ch); + if (!(val & (OMAP_DMA_CCR_RD_ACTIVE | OMAP_DMA_CCR_WR_ACTIVE))) + break; + + if (i > 100) + break; + + udelay(5); + } + + if (val & (OMAP_DMA_CCR_RD_ACTIVE | OMAP_DMA_CCR_WR_ACTIVE)) + dev_err(c->vc.chan.device->dev, + "DMA drain did not complete on lch %d\n", + c->dma_ch); + + c->plat->dma_write(sysconfig, OCP_SYSCONFIG, c->dma_ch); + } else { + val &= ~OMAP_DMA_CCR_EN; + c->plat->dma_write(val, CCR, c->dma_ch); + } + + mb(); + + if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { + val = c->plat->dma_read(CLNK_CTRL, c->dma_ch); + + if (dma_omap1()) + val |= 1 << 14; /* set the STOP_LNK bit */ + else + val &= ~(1 << 15); /* Clear the ENABLE_LNK bit */ + + c->plat->dma_write(val, CLNK_CTRL, c->dma_ch); + } +} + static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, unsigned idx) { @@ -113,7 +220,7 @@ static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, c->plat->dma_write(sg->en, CEN, c->dma_ch); c->plat->dma_write(sg->fn, CFN, c->dma_ch); - omap_start_dma(c->dma_ch); + omap_dma_start(c, d); } static void omap_dma_start_desc(struct omap_chan *c) @@ -434,6 +541,12 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->sync_mode = OMAP_DMA_SYNC_FRAME; d->sync_type = sync_type; d->periph_port = OMAP_DMA_PORT_TIPB; + d->cicr = OMAP_DMA_DROP_IRQ | OMAP_DMA_BLOCK_IRQ; + + if (dma_omap1()) + d->cicr |= OMAP1_DMA_TOUT_IRQ; + else + d->cicr |= OMAP2_DMA_MISALIGNED_ERR_IRQ | OMAP2_DMA_TRANS_ERR_IRQ; /* * Build our scatterlist entries: each contains the address, @@ -463,6 +576,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( size_t period_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); enum dma_slave_buswidth dev_width; struct omap_desc *d; @@ -519,15 +633,25 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->sg[0].en = period_len / es_bytes[es]; d->sg[0].fn = buf_len / period_len; d->sglen = 1; + d->cicr = OMAP_DMA_DROP_IRQ; + if (flags & DMA_PREP_INTERRUPT) + d->cicr |= OMAP_DMA_FRAME_IRQ; + + if (dma_omap1()) + d->cicr |= OMAP1_DMA_TOUT_IRQ; + else + d->cicr |= OMAP2_DMA_MISALIGNED_ERR_IRQ | OMAP2_DMA_TRANS_ERR_IRQ; if (!c->cyclic) { c->cyclic = true; - omap_dma_link_lch(c->dma_ch, c->dma_ch); - if (flags & DMA_PREP_INTERRUPT) - omap_enable_dma_irq(c->dma_ch, OMAP_DMA_FRAME_IRQ); + if (__dma_omap15xx(od->plat->dma_attr)) { + uint32_t val; - omap_disable_dma_irq(c->dma_ch, OMAP_DMA_BLOCK_IRQ); + val = c->plat->dma_read(CCR, c->dma_ch); + val |= 3 << 8; + c->plat->dma_write(val, CCR, c->dma_ch); + } } if (dma_omap2plus()) { @@ -568,20 +692,27 @@ static int omap_dma_terminate_all(struct omap_chan *c) /* * Stop DMA activity: we assume the callback will not be called - * after omap_stop_dma() returns (even if it does, it will see + * after omap_dma_stop() returns (even if it does, it will see * c->desc is NULL and exit.) */ if (c->desc) { c->desc = NULL; /* Avoid stopping the dma twice */ if (!c->paused) - omap_stop_dma(c->dma_ch); + omap_dma_stop(c); } if (c->cyclic) { c->cyclic = false; c->paused = false; - omap_dma_unlink_lch(c->dma_ch, c->dma_ch); + + if (__dma_omap15xx(od->plat->dma_attr)) { + uint32_t val; + + val = c->plat->dma_read(CCR, c->dma_ch); + val &= ~(3 << 8); + c->plat->dma_write(val, CCR, c->dma_ch); + } } vchan_get_all_descriptors(&c->vc, &head); @@ -598,7 +729,7 @@ static int omap_dma_pause(struct omap_chan *c) return -EINVAL; if (!c->paused) { - omap_stop_dma(c->dma_ch); + omap_dma_stop(c); c->paused = true; } @@ -612,7 +743,7 @@ static int omap_dma_resume(struct omap_chan *c) return -EINVAL; if (c->paused) { - omap_start_dma(c->dma_ch); + omap_dma_start(c, c->desc); c->paused = false; } From 3997cab391b38e126f217e36ad7bdc9672c9fb4d Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 18:04:17 +0000 Subject: [PATCH 69/89] dmaengine: omap-dma: move reading of dma position to omap-dma.c Read the current DMA position from the hardware directly rather than via arch/arm/plat-omap/dma.c. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 66 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 7aa5ff7ab935..323eae2c9d08 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -427,6 +427,68 @@ static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr) return size; } +static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + dma_addr_t addr; + + if (__dma_omap15xx(od->plat->dma_attr)) + addr = c->plat->dma_read(CPC, c->dma_ch); + else + addr = c->plat->dma_read(CSAC, c->dma_ch); + + if (od->plat->errata & DMA_ERRATA_3_3 && addr == 0) + addr = c->plat->dma_read(CSAC, c->dma_ch); + + if (!__dma_omap15xx(od->plat->dma_attr)) { + /* + * CDAC == 0 indicates that the DMA transfer on the channel has + * not been started (no data has been transferred so far). + * Return the programmed source start address in this case. + */ + if (c->plat->dma_read(CDAC, c->dma_ch)) + addr = c->plat->dma_read(CSAC, c->dma_ch); + else + addr = c->plat->dma_read(CSSA, c->dma_ch); + } + + if (dma_omap1()) + addr |= c->plat->dma_read(CSSA, c->dma_ch) & 0xffff0000; + + return addr; +} + +static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + dma_addr_t addr; + + if (__dma_omap15xx(od->plat->dma_attr)) + addr = c->plat->dma_read(CPC, c->dma_ch); + else + addr = c->plat->dma_read(CDAC, c->dma_ch); + + /* + * omap 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is + * read before the DMA controller finished disabling the channel. + */ + if (!__dma_omap15xx(od->plat->dma_attr) && addr == 0) { + addr = c->plat->dma_read(CDAC, c->dma_ch); + /* + * CDAC == 0 indicates that the DMA transfer on the channel has + * not been started (no data has been transferred so far). + * Return the programmed destination start address in this case. + */ + if (addr == 0) + addr = c->plat->dma_read(CDSA, c->dma_ch); + } + + if (dma_omap1()) + addr |= c->plat->dma_read(CDSA, c->dma_ch) & 0xffff0000; + + return addr; +} + static enum dma_status omap_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { @@ -448,9 +510,9 @@ static enum dma_status omap_dma_tx_status(struct dma_chan *chan, dma_addr_t pos; if (d->dir == DMA_MEM_TO_DEV) - pos = omap_get_dma_src_pos(c->dma_ch); + pos = omap_dma_get_src_pos(c); else if (d->dir == DMA_DEV_TO_MEM) - pos = omap_get_dma_dst_pos(c->dma_ch); + pos = omap_dma_get_dst_pos(c); else pos = 0; From 2f0d13bdf6440906bb52fe94681ce7927145f4d7 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 18:51:53 +0000 Subject: [PATCH 70/89] dmaengine: omap-dma: consolidate setup of CSDP Consolidate the setup of the channel source destination parameters register. This way, we calculate the required CSDP value when we setup a transfer descriptor, and only write it to the device registers once when we start the descriptor. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 64 ++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 323eae2c9d08..ec7cc10d4594 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -60,8 +60,8 @@ struct omap_desc { uint8_t es; /* OMAP_DMA_DATA_TYPE_xxx */ uint8_t sync_mode; /* OMAP_DMA_SYNC_xxx */ uint8_t sync_type; /* OMAP_DMA_xxx_SYNC* */ - uint8_t periph_port; /* Peripheral port */ uint16_t cicr; /* CICR value */ + uint32_t csdp; /* CSDP value */ unsigned sglen; struct omap_sg sg[0]; @@ -240,14 +240,6 @@ static void omap_dma_start_desc(struct omap_chan *c) c->sgidx = 0; if (d->dir == DMA_DEV_TO_MEM) { - if (dma_omap1()) { - val = c->plat->dma_read(CSDP, c->dma_ch); - val &= ~(0x1f << 9 | 0x1f << 2); - val |= OMAP_DMA_PORT_EMIFF << 9; - val |= d->periph_port << 2; - c->plat->dma_write(val, CSDP, c->dma_ch); - } - val = c->plat->dma_read(CCR, c->dma_ch); val &= ~(0x03 << 14 | 0x03 << 12); val |= OMAP_DMA_AMODE_POST_INC << 14; @@ -258,14 +250,6 @@ static void omap_dma_start_desc(struct omap_chan *c) c->plat->dma_write(0, CSEI, c->dma_ch); c->plat->dma_write(d->fi, CSFI, c->dma_ch); } else { - if (dma_omap1()) { - val = c->plat->dma_read(CSDP, c->dma_ch); - val &= ~(0x1f << 9 | 0x1f << 2); - val |= d->periph_port << 9; - val |= OMAP_DMA_PORT_EMIFF << 2; - c->plat->dma_write(val, CSDP, c->dma_ch); - } - val = c->plat->dma_read(CCR, c->dma_ch); val &= ~(0x03 << 12 | 0x03 << 14); val |= OMAP_DMA_AMODE_CONSTANT << 14; @@ -277,10 +261,7 @@ static void omap_dma_start_desc(struct omap_chan *c) c->plat->dma_write(d->fi, CDFI, c->dma_ch); } - val = c->plat->dma_read(CSDP, c->dma_ch); - val &= ~0x03; - val |= d->es; - c->plat->dma_write(val, CSDP, c->dma_ch); + c->plat->dma_write(d->csdp, CSDP, c->dma_ch); if (dma_omap1()) { val = c->plat->dma_read(CCR, c->dma_ch); @@ -602,13 +583,21 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->es = es; d->sync_mode = OMAP_DMA_SYNC_FRAME; d->sync_type = sync_type; - d->periph_port = OMAP_DMA_PORT_TIPB; d->cicr = OMAP_DMA_DROP_IRQ | OMAP_DMA_BLOCK_IRQ; + d->csdp = es; - if (dma_omap1()) + if (dma_omap1()) { d->cicr |= OMAP1_DMA_TOUT_IRQ; - else + + if (dir == DMA_DEV_TO_MEM) + d->csdp |= OMAP_DMA_PORT_EMIFF << 9 | + OMAP_DMA_PORT_TIPB << 2; + else + d->csdp |= OMAP_DMA_PORT_TIPB << 9 | + OMAP_DMA_PORT_EMIFF << 2; + } else { d->cicr |= OMAP2_DMA_MISALIGNED_ERR_IRQ | OMAP2_DMA_TRANS_ERR_IRQ; + } /* * Build our scatterlist entries: each contains the address, @@ -690,7 +679,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( else d->sync_mode = OMAP_DMA_SYNC_ELEMENT; d->sync_type = sync_type; - d->periph_port = OMAP_DMA_PORT_MPUI; d->sg[0].addr = buf_addr; d->sg[0].en = period_len / es_bytes[es]; d->sg[0].fn = buf_len / period_len; @@ -699,11 +687,24 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( if (flags & DMA_PREP_INTERRUPT) d->cicr |= OMAP_DMA_FRAME_IRQ; - if (dma_omap1()) + d->csdp = es; + + if (dma_omap1()) { d->cicr |= OMAP1_DMA_TOUT_IRQ; - else + + if (dir == DMA_DEV_TO_MEM) + d->csdp |= OMAP_DMA_PORT_EMIFF << 9 | + OMAP_DMA_PORT_MPUI << 2; + else + d->csdp |= OMAP_DMA_PORT_MPUI << 9 | + OMAP_DMA_PORT_EMIFF << 2; + } else { d->cicr |= OMAP2_DMA_MISALIGNED_ERR_IRQ | OMAP2_DMA_TRANS_ERR_IRQ; + /* src and dst burst mode 16 */ + d->csdp |= 3 << 14 | 3 << 7; + } + if (!c->cyclic) { c->cyclic = true; @@ -716,15 +717,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( } } - if (dma_omap2plus()) { - uint32_t val; - - val = c->plat->dma_read(CSDP, c->dma_ch); - val |= 0x03 << 7; /* src burst mode 16 */ - val |= 0x03 << 14; /* dst burst mode 16 */ - c->plat->dma_write(val, CSDP, c->dma_ch); - } - return vchan_tx_prep(&c->vc, &d->vd, flags); } From 3ed4d18f39bcd8cb8d8218c0a5f89a4d81ba8730 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 19:16:09 +0000 Subject: [PATCH 71/89] dmaengine: omap-dma: consolidate setup of CCR Consolidate the setup of the channel control register. Prepare the basic value in the preparation of the DMA descriptor, and write it into the register upon descriptor execution. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 147 +++++++++++++++++------------------------ 1 file changed, 61 insertions(+), 86 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index ec7cc10d4594..9a9e81907475 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -58,8 +58,7 @@ struct omap_desc { int16_t fi; /* for OMAP_DMA_SYNC_PACKET */ uint8_t es; /* OMAP_DMA_DATA_TYPE_xxx */ - uint8_t sync_mode; /* OMAP_DMA_SYNC_xxx */ - uint8_t sync_type; /* OMAP_DMA_xxx_SYNC* */ + uint32_t ccr; /* CCR value */ uint16_t cicr; /* CICR value */ uint32_t csdp; /* CSDP value */ @@ -227,7 +226,6 @@ static void omap_dma_start_desc(struct omap_chan *c) { struct virt_dma_desc *vd = vchan_next_desc(&c->vc); struct omap_desc *d; - uint32_t val; if (!vd) { c->desc = NULL; @@ -239,23 +237,15 @@ static void omap_dma_start_desc(struct omap_chan *c) c->desc = d = to_omap_dma_desc(&vd->tx); c->sgidx = 0; - if (d->dir == DMA_DEV_TO_MEM) { - val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(0x03 << 14 | 0x03 << 12); - val |= OMAP_DMA_AMODE_POST_INC << 14; - val |= OMAP_DMA_AMODE_CONSTANT << 12; - c->plat->dma_write(val, CCR, c->dma_ch); + c->plat->dma_write(d->ccr, CCR, c->dma_ch); + if (dma_omap1()) + c->plat->dma_write(d->ccr >> 16, CCR2, c->dma_ch); + if (d->dir == DMA_DEV_TO_MEM) { c->plat->dma_write(d->dev_addr, CSSA, c->dma_ch); c->plat->dma_write(0, CSEI, c->dma_ch); c->plat->dma_write(d->fi, CSFI, c->dma_ch); } else { - val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(0x03 << 12 | 0x03 << 14); - val |= OMAP_DMA_AMODE_CONSTANT << 14; - val |= OMAP_DMA_AMODE_POST_INC << 12; - c->plat->dma_write(val, CCR, c->dma_ch); - c->plat->dma_write(d->dev_addr, CDSA, c->dma_ch); c->plat->dma_write(0, CDEI, c->dma_ch); c->plat->dma_write(d->fi, CDFI, c->dma_ch); @@ -263,45 +253,6 @@ static void omap_dma_start_desc(struct omap_chan *c) c->plat->dma_write(d->csdp, CSDP, c->dma_ch); - if (dma_omap1()) { - val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(1 << 5); - if (d->sync_mode == OMAP_DMA_SYNC_FRAME) - val |= 1 << 5; - c->plat->dma_write(val, CCR, c->dma_ch); - - val = c->plat->dma_read(CCR2, c->dma_ch); - val &= ~(1 << 2); - if (d->sync_mode == OMAP_DMA_SYNC_BLOCK) - val |= 1 << 2; - c->plat->dma_write(val, CCR2, c->dma_ch); - } else if (c->dma_sig) { - val = c->plat->dma_read(CCR, c->dma_ch); - - /* DMA_SYNCHRO_CONTROL_UPPER depends on the channel number */ - val &= ~(1 << 24 | 1 << 23 | 3 << 19 | 1 << 18 | 1 << 5 | 0x1f); - val |= (c->dma_sig & ~0x1f) << 14; - val |= c->dma_sig & 0x1f; - - if (d->sync_mode & OMAP_DMA_SYNC_FRAME) - val |= 1 << 5; - - if (d->sync_mode & OMAP_DMA_SYNC_BLOCK) - val |= 1 << 18; - - switch (d->sync_type) { - case OMAP_DMA_DST_SYNC_PREFETCH:/* dest synch */ - val |= 1 << 23; /* Prefetch */ - break; - case 0: - break; - default: - val |= 1 << 24; /* source synch */ - break; - } - c->plat->dma_write(val, CCR, c->dma_ch); - } - omap_dma_start_sg(c, d, 0); } @@ -540,19 +491,17 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( struct scatterlist *sgent; struct omap_desc *d; dma_addr_t dev_addr; - unsigned i, j = 0, es, en, frame_bytes, sync_type; + unsigned i, j = 0, es, en, frame_bytes; u32 burst; if (dir == DMA_DEV_TO_MEM) { dev_addr = c->cfg.src_addr; dev_width = c->cfg.src_addr_width; burst = c->cfg.src_maxburst; - sync_type = OMAP_DMA_SRC_SYNC; } else if (dir == DMA_MEM_TO_DEV) { dev_addr = c->cfg.dst_addr; dev_width = c->cfg.dst_addr_width; burst = c->cfg.dst_maxburst; - sync_type = OMAP_DMA_DST_SYNC; } else { dev_err(chan->device->dev, "%s: bad direction?\n", __func__); return NULL; @@ -581,12 +530,28 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->dir = dir; d->dev_addr = dev_addr; d->es = es; - d->sync_mode = OMAP_DMA_SYNC_FRAME; - d->sync_type = sync_type; + + d->ccr = 0; + if (dir == DMA_DEV_TO_MEM) + d->ccr |= OMAP_DMA_AMODE_POST_INC << 14 | + OMAP_DMA_AMODE_CONSTANT << 12; + else + d->ccr |= OMAP_DMA_AMODE_CONSTANT << 14 | + OMAP_DMA_AMODE_POST_INC << 12; + d->cicr = OMAP_DMA_DROP_IRQ | OMAP_DMA_BLOCK_IRQ; d->csdp = es; if (dma_omap1()) { + d->ccr |= 1 << 5; /* frame sync */ + if (__dma_omap16xx(od->plat->dma_attr)) { + d->ccr |= 1 << 10; /* disable 3.0/3.1 compatibility mode */ + /* Duplicate what plat-omap/dma.c does */ + d->ccr |= c->dma_ch + 1; + } else { + d->ccr |= c->dma_sig & 0x1f; + } + d->cicr |= OMAP1_DMA_TOUT_IRQ; if (dir == DMA_DEV_TO_MEM) @@ -596,6 +561,13 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->csdp |= OMAP_DMA_PORT_TIPB << 9 | OMAP_DMA_PORT_EMIFF << 2; } else { + d->ccr |= (c->dma_sig & ~0x1f) << 14; + d->ccr |= c->dma_sig & 0x1f; + d->ccr |= 1 << 5; /* frame sync */ + + if (dir == DMA_DEV_TO_MEM) + d->ccr |= 1 << 24; /* source synch */ + d->cicr |= OMAP2_DMA_MISALIGNED_ERR_IRQ | OMAP2_DMA_TRANS_ERR_IRQ; } @@ -632,19 +604,17 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( enum dma_slave_buswidth dev_width; struct omap_desc *d; dma_addr_t dev_addr; - unsigned es, sync_type; + unsigned es; u32 burst; if (dir == DMA_DEV_TO_MEM) { dev_addr = c->cfg.src_addr; dev_width = c->cfg.src_addr_width; burst = c->cfg.src_maxburst; - sync_type = OMAP_DMA_SRC_SYNC; } else if (dir == DMA_MEM_TO_DEV) { dev_addr = c->cfg.dst_addr; dev_width = c->cfg.dst_addr_width; burst = c->cfg.dst_maxburst; - sync_type = OMAP_DMA_DST_SYNC; } else { dev_err(chan->device->dev, "%s: bad direction?\n", __func__); return NULL; @@ -674,15 +644,21 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->dev_addr = dev_addr; d->fi = burst; d->es = es; - if (burst) - d->sync_mode = OMAP_DMA_SYNC_PACKET; - else - d->sync_mode = OMAP_DMA_SYNC_ELEMENT; - d->sync_type = sync_type; d->sg[0].addr = buf_addr; d->sg[0].en = period_len / es_bytes[es]; d->sg[0].fn = buf_len / period_len; d->sglen = 1; + + d->ccr = 0; + if (__dma_omap15xx(od->plat->dma_attr)) + d->ccr = 3 << 8; + if (dir == DMA_DEV_TO_MEM) + d->ccr |= OMAP_DMA_AMODE_POST_INC << 14 | + OMAP_DMA_AMODE_CONSTANT << 12; + else + d->ccr |= OMAP_DMA_AMODE_CONSTANT << 14 | + OMAP_DMA_AMODE_POST_INC << 12; + d->cicr = OMAP_DMA_DROP_IRQ; if (flags & DMA_PREP_INTERRUPT) d->cicr |= OMAP_DMA_FRAME_IRQ; @@ -690,6 +666,14 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->csdp = es; if (dma_omap1()) { + if (__dma_omap16xx(od->plat->dma_attr)) { + d->ccr |= 1 << 10; /* disable 3.0/3.1 compatibility mode */ + /* Duplicate what plat-omap/dma.c does */ + d->ccr |= c->dma_ch + 1; + } else { + d->ccr |= c->dma_sig & 0x1f; + } + d->cicr |= OMAP1_DMA_TOUT_IRQ; if (dir == DMA_DEV_TO_MEM) @@ -699,23 +683,22 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->csdp |= OMAP_DMA_PORT_MPUI << 9 | OMAP_DMA_PORT_EMIFF << 2; } else { + d->ccr |= (c->dma_sig & ~0x1f) << 14; + d->ccr |= c->dma_sig & 0x1f; + + if (burst) + d->ccr |= 1 << 18 | 1 << 5; /* packet */ + + if (dir == DMA_DEV_TO_MEM) + d->ccr |= 1 << 24; /* source synch */ + d->cicr |= OMAP2_DMA_MISALIGNED_ERR_IRQ | OMAP2_DMA_TRANS_ERR_IRQ; /* src and dst burst mode 16 */ d->csdp |= 3 << 14 | 3 << 7; } - if (!c->cyclic) { - c->cyclic = true; - - if (__dma_omap15xx(od->plat->dma_attr)) { - uint32_t val; - - val = c->plat->dma_read(CCR, c->dma_ch); - val |= 3 << 8; - c->plat->dma_write(val, CCR, c->dma_ch); - } - } + c->cyclic = true; return vchan_tx_prep(&c->vc, &d->vd, flags); } @@ -759,14 +742,6 @@ static int omap_dma_terminate_all(struct omap_chan *c) if (c->cyclic) { c->cyclic = false; c->paused = false; - - if (__dma_omap15xx(od->plat->dma_attr)) { - uint32_t val; - - val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~(3 << 8); - c->plat->dma_write(val, CCR, c->dma_ch); - } } vchan_get_all_descriptors(&c->vc, &head); From 9043826d88467091543c1d3ab06eb4afeed34789 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 19:57:06 +0000 Subject: [PATCH 72/89] dmaengine: omap-dma: provide register definitions Provide our own set of more complete register definitions; this allows us to get rid of the meaningless 1 << n constants scattered throughout this code. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 171 ++++++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 54 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 9a9e81907475..6cf66e608338 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -57,7 +57,7 @@ struct omap_desc { dma_addr_t dev_addr; int16_t fi; /* for OMAP_DMA_SYNC_PACKET */ - uint8_t es; /* OMAP_DMA_DATA_TYPE_xxx */ + uint8_t es; /* CSDP_DATA_TYPE_xxx */ uint32_t ccr; /* CCR value */ uint16_t cicr; /* CICR value */ uint32_t csdp; /* CSDP value */ @@ -66,10 +66,83 @@ struct omap_desc { struct omap_sg sg[0]; }; +enum { + CCR_FS = BIT(5), + CCR_READ_PRIORITY = BIT(6), + CCR_ENABLE = BIT(7), + CCR_AUTO_INIT = BIT(8), /* OMAP1 only */ + CCR_REPEAT = BIT(9), /* OMAP1 only */ + CCR_OMAP31_DISABLE = BIT(10), /* OMAP1 only */ + CCR_SUSPEND_SENSITIVE = BIT(8), /* OMAP2+ only */ + CCR_RD_ACTIVE = BIT(9), /* OMAP2+ only */ + CCR_WR_ACTIVE = BIT(10), /* OMAP2+ only */ + CCR_SRC_AMODE_CONSTANT = 0 << 12, + CCR_SRC_AMODE_POSTINC = 1 << 12, + CCR_SRC_AMODE_SGLIDX = 2 << 12, + CCR_SRC_AMODE_DBLIDX = 3 << 12, + CCR_DST_AMODE_CONSTANT = 0 << 14, + CCR_DST_AMODE_POSTINC = 1 << 14, + CCR_DST_AMODE_SGLIDX = 2 << 14, + CCR_DST_AMODE_DBLIDX = 3 << 14, + CCR_CONSTANT_FILL = BIT(16), + CCR_TRANSPARENT_COPY = BIT(17), + CCR_BS = BIT(18), + CCR_SUPERVISOR = BIT(22), + CCR_PREFETCH = BIT(23), + CCR_TRIGGER_SRC = BIT(24), + CCR_BUFFERING_DISABLE = BIT(25), + CCR_WRITE_PRIORITY = BIT(26), + CCR_SYNC_ELEMENT = 0, + CCR_SYNC_FRAME = CCR_FS, + CCR_SYNC_BLOCK = CCR_BS, + CCR_SYNC_PACKET = CCR_BS | CCR_FS, + + CSDP_DATA_TYPE_8 = 0, + CSDP_DATA_TYPE_16 = 1, + CSDP_DATA_TYPE_32 = 2, + CSDP_SRC_PORT_EMIFF = 0 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_EMIFS = 1 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_OCP_T1 = 2 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_TIPB = 3 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_OCP_T2 = 4 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_MPUI = 5 << 2, /* OMAP1 only */ + CSDP_SRC_PACKED = BIT(6), + CSDP_SRC_BURST_1 = 0 << 7, + CSDP_SRC_BURST_16 = 1 << 7, + CSDP_SRC_BURST_32 = 2 << 7, + CSDP_SRC_BURST_64 = 3 << 7, + CSDP_DST_PORT_EMIFF = 0 << 9, /* OMAP1 only */ + CSDP_DST_PORT_EMIFS = 1 << 9, /* OMAP1 only */ + CSDP_DST_PORT_OCP_T1 = 2 << 9, /* OMAP1 only */ + CSDP_DST_PORT_TIPB = 3 << 9, /* OMAP1 only */ + CSDP_DST_PORT_OCP_T2 = 4 << 9, /* OMAP1 only */ + CSDP_DST_PORT_MPUI = 5 << 9, /* OMAP1 only */ + CSDP_DST_PACKED = BIT(13), + CSDP_DST_BURST_1 = 0 << 14, + CSDP_DST_BURST_16 = 1 << 14, + CSDP_DST_BURST_32 = 2 << 14, + CSDP_DST_BURST_64 = 3 << 14, + + CICR_TOUT_IE = BIT(0), /* OMAP1 only */ + CICR_DROP_IE = BIT(1), + CICR_HALF_IE = BIT(2), + CICR_FRAME_IE = BIT(3), + CICR_LAST_IE = BIT(4), + CICR_BLOCK_IE = BIT(5), + CICR_PKT_IE = BIT(7), /* OMAP2+ only */ + CICR_TRANS_ERR_IE = BIT(8), /* OMAP2+ only */ + CICR_SUPERVISOR_ERR_IE = BIT(10), /* OMAP2+ only */ + CICR_MISALIGNED_ERR_IE = BIT(11), /* OMAP2+ only */ + CICR_DRAIN_IE = BIT(12), /* OMAP2+ only */ + CICR_SUPER_BLOCK_IE = BIT(14), /* OMAP2+ only */ + + CLNK_CTRL_ENABLE_LNK = BIT(15), +}; + static const unsigned es_bytes[] = { - [OMAP_DMA_DATA_TYPE_S8] = 1, - [OMAP_DMA_DATA_TYPE_S16] = 2, - [OMAP_DMA_DATA_TYPE_S32] = 4, + [CSDP_DATA_TYPE_8] = 1, + [CSDP_DATA_TYPE_16] = 2, + [CSDP_DATA_TYPE_32] = 4, }; static struct of_dma_filter_info omap_dma_info = { @@ -112,7 +185,7 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) if (dma_omap1()) val &= ~(1 << 14); - val |= c->dma_ch | 1 << 15; + val |= c->dma_ch | CLNK_CTRL_ENABLE_LNK; c->plat->dma_write(val, CLNK_CTRL, c->dma_ch); } else if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) @@ -129,8 +202,8 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) val = c->plat->dma_read(CCR, c->dma_ch); if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) - val |= OMAP_DMA_CCR_BUFFERING_DISABLE; - val |= OMAP_DMA_CCR_EN; + val |= CCR_BUFFERING_DISABLE; + val |= CCR_ENABLE; mb(); c->plat->dma_write(val, CCR, c->dma_ch); } @@ -150,8 +223,7 @@ static void omap_dma_stop(struct omap_chan *c) c->plat->dma_write(~0, CSR, c->dma_ch); val = c->plat->dma_read(CCR, c->dma_ch); - if (od->plat->errata & DMA_ERRATA_i541 && - val & OMAP_DMA_CCR_SEL_SRC_DST_SYNC) { + if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { uint32_t sysconfig; unsigned i; @@ -161,13 +233,13 @@ static void omap_dma_stop(struct omap_chan *c) c->plat->dma_write(val, OCP_SYSCONFIG, c->dma_ch); val = c->plat->dma_read(CCR, c->dma_ch); - val &= ~OMAP_DMA_CCR_EN; + val &= ~CCR_ENABLE; c->plat->dma_write(val, CCR, c->dma_ch); /* Wait for sDMA FIFO to drain */ for (i = 0; ; i++) { val = c->plat->dma_read(CCR, c->dma_ch); - if (!(val & (OMAP_DMA_CCR_RD_ACTIVE | OMAP_DMA_CCR_WR_ACTIVE))) + if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))) break; if (i > 100) @@ -176,14 +248,14 @@ static void omap_dma_stop(struct omap_chan *c) udelay(5); } - if (val & (OMAP_DMA_CCR_RD_ACTIVE | OMAP_DMA_CCR_WR_ACTIVE)) + if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)) dev_err(c->vc.chan.device->dev, "DMA drain did not complete on lch %d\n", c->dma_ch); c->plat->dma_write(sysconfig, OCP_SYSCONFIG, c->dma_ch); } else { - val &= ~OMAP_DMA_CCR_EN; + val &= ~CCR_ENABLE; c->plat->dma_write(val, CCR, c->dma_ch); } @@ -195,7 +267,7 @@ static void omap_dma_stop(struct omap_chan *c) if (dma_omap1()) val |= 1 << 14; /* set the STOP_LNK bit */ else - val &= ~(1 << 15); /* Clear the ENABLE_LNK bit */ + val &= ~CLNK_CTRL_ENABLE_LNK; c->plat->dma_write(val, CLNK_CTRL, c->dma_ch); } @@ -510,13 +582,13 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( /* Bus width translates to the element size (ES) */ switch (dev_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: - es = OMAP_DMA_DATA_TYPE_S8; + es = CSDP_DATA_TYPE_8; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: - es = OMAP_DMA_DATA_TYPE_S16; + es = CSDP_DATA_TYPE_16; break; case DMA_SLAVE_BUSWIDTH_4_BYTES: - es = OMAP_DMA_DATA_TYPE_S32; + es = CSDP_DATA_TYPE_32; break; default: /* not reached */ return NULL; @@ -531,44 +603,38 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->dev_addr = dev_addr; d->es = es; - d->ccr = 0; + d->ccr = CCR_SYNC_FRAME; if (dir == DMA_DEV_TO_MEM) - d->ccr |= OMAP_DMA_AMODE_POST_INC << 14 | - OMAP_DMA_AMODE_CONSTANT << 12; + d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; else - d->ccr |= OMAP_DMA_AMODE_CONSTANT << 14 | - OMAP_DMA_AMODE_POST_INC << 12; + d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; - d->cicr = OMAP_DMA_DROP_IRQ | OMAP_DMA_BLOCK_IRQ; + d->cicr = CICR_DROP_IE | CICR_BLOCK_IE; d->csdp = es; if (dma_omap1()) { - d->ccr |= 1 << 5; /* frame sync */ if (__dma_omap16xx(od->plat->dma_attr)) { - d->ccr |= 1 << 10; /* disable 3.0/3.1 compatibility mode */ + d->ccr |= CCR_OMAP31_DISABLE; /* Duplicate what plat-omap/dma.c does */ d->ccr |= c->dma_ch + 1; } else { d->ccr |= c->dma_sig & 0x1f; } - d->cicr |= OMAP1_DMA_TOUT_IRQ; + d->cicr |= CICR_TOUT_IE; if (dir == DMA_DEV_TO_MEM) - d->csdp |= OMAP_DMA_PORT_EMIFF << 9 | - OMAP_DMA_PORT_TIPB << 2; + d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB; else - d->csdp |= OMAP_DMA_PORT_TIPB << 9 | - OMAP_DMA_PORT_EMIFF << 2; + d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF; } else { d->ccr |= (c->dma_sig & ~0x1f) << 14; d->ccr |= c->dma_sig & 0x1f; - d->ccr |= 1 << 5; /* frame sync */ if (dir == DMA_DEV_TO_MEM) - d->ccr |= 1 << 24; /* source synch */ + d->ccr |= CCR_TRIGGER_SRC; - d->cicr |= OMAP2_DMA_MISALIGNED_ERR_IRQ | OMAP2_DMA_TRANS_ERR_IRQ; + d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; } /* @@ -623,13 +689,13 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( /* Bus width translates to the element size (ES) */ switch (dev_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: - es = OMAP_DMA_DATA_TYPE_S8; + es = CSDP_DATA_TYPE_8; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: - es = OMAP_DMA_DATA_TYPE_S16; + es = CSDP_DATA_TYPE_16; break; case DMA_SLAVE_BUSWIDTH_4_BYTES: - es = OMAP_DMA_DATA_TYPE_S32; + es = CSDP_DATA_TYPE_32; break; default: /* not reached */ return NULL; @@ -651,51 +717,48 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->ccr = 0; if (__dma_omap15xx(od->plat->dma_attr)) - d->ccr = 3 << 8; + d->ccr = CCR_AUTO_INIT | CCR_REPEAT; if (dir == DMA_DEV_TO_MEM) - d->ccr |= OMAP_DMA_AMODE_POST_INC << 14 | - OMAP_DMA_AMODE_CONSTANT << 12; + d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; else - d->ccr |= OMAP_DMA_AMODE_CONSTANT << 14 | - OMAP_DMA_AMODE_POST_INC << 12; + d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; - d->cicr = OMAP_DMA_DROP_IRQ; + d->cicr = CICR_DROP_IE; if (flags & DMA_PREP_INTERRUPT) - d->cicr |= OMAP_DMA_FRAME_IRQ; + d->cicr |= CICR_FRAME_IE; d->csdp = es; if (dma_omap1()) { if (__dma_omap16xx(od->plat->dma_attr)) { - d->ccr |= 1 << 10; /* disable 3.0/3.1 compatibility mode */ + d->ccr |= CCR_OMAP31_DISABLE; /* Duplicate what plat-omap/dma.c does */ d->ccr |= c->dma_ch + 1; } else { d->ccr |= c->dma_sig & 0x1f; } - d->cicr |= OMAP1_DMA_TOUT_IRQ; + d->cicr |= CICR_TOUT_IE; if (dir == DMA_DEV_TO_MEM) - d->csdp |= OMAP_DMA_PORT_EMIFF << 9 | - OMAP_DMA_PORT_MPUI << 2; + d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI; else - d->csdp |= OMAP_DMA_PORT_MPUI << 9 | - OMAP_DMA_PORT_EMIFF << 2; + d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF; } else { d->ccr |= (c->dma_sig & ~0x1f) << 14; d->ccr |= c->dma_sig & 0x1f; if (burst) - d->ccr |= 1 << 18 | 1 << 5; /* packet */ + d->ccr |= CCR_SYNC_PACKET; + else + d->ccr |= CCR_SYNC_ELEMENT; if (dir == DMA_DEV_TO_MEM) - d->ccr |= 1 << 24; /* source synch */ + d->ccr |= CCR_TRIGGER_SRC; - d->cicr |= OMAP2_DMA_MISALIGNED_ERR_IRQ | OMAP2_DMA_TRANS_ERR_IRQ; + d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; - /* src and dst burst mode 16 */ - d->csdp |= 3 << 14 | 3 << 7; + d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; } c->cyclic = true; From 49ae0b29439446cff81c32bf01fb7b7cce195373 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 21:09:18 +0000 Subject: [PATCH 73/89] dmaengine: omap-dma: move CCR buffering disable errata out of the fast path Since we record the CCR register in the dma transaction, we can move the processing of the iframe buffering errata out of the omap_dma_start(). Move it to the preparation functions. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 6cf66e608338..324f4c7c5dd0 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -201,8 +201,6 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) c->plat->dma_write(d->cicr, CICR, c->dma_ch); val = c->plat->dma_read(CCR, c->dma_ch); - if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) - val |= CCR_BUFFERING_DISABLE; val |= CCR_ENABLE; mb(); c->plat->dma_write(val, CCR, c->dma_ch); @@ -558,6 +556,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen, enum dma_transfer_direction dir, unsigned long tx_flags, void *context) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); enum dma_slave_buswidth dev_width; struct scatterlist *sgent; @@ -636,6 +635,8 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; } + if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) + d->ccr |= CCR_BUFFERING_DISABLE; /* * Build our scatterlist entries: each contains the address, @@ -760,6 +761,8 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; } + if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) + d->ccr |= CCR_BUFFERING_DISABLE; c->cyclic = true; From 470b23f7308dd2af25bd075d14a724f8ccd93985 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 2 Nov 2013 21:23:06 +0000 Subject: [PATCH 74/89] dmaengine: omap-dma: consolidate clearing channel status register Consolidate clearing of the channel status register, rather than open coding the same functionality in two places. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 324f4c7c5dd0..d1641aa9d113 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -169,6 +169,14 @@ static void omap_dma_desc_free(struct virt_dma_desc *vd) kfree(container_of(vd, struct omap_desc, vd)); } +static void omap_dma_clear_csr(struct omap_chan *c) +{ + if (dma_omap1()) + c->plat->dma_read(CSR, c->dma_ch); + else + c->plat->dma_write(~0, CSR, c->dma_ch); +} + static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) { struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); @@ -191,11 +199,7 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) } else if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) c->plat->dma_write(c->dma_ch, CLNK_CTRL, c->dma_ch); - /* Clear CSR */ - if (dma_omap1()) - c->plat->dma_read(CSR, c->dma_ch); - else - c->plat->dma_write(~0, CSR, c->dma_ch); + omap_dma_clear_csr(c); /* Enable interrupts */ c->plat->dma_write(d->cicr, CICR, c->dma_ch); @@ -214,11 +218,7 @@ static void omap_dma_stop(struct omap_chan *c) /* disable irq */ c->plat->dma_write(0, CICR, c->dma_ch); - /* Clear CSR */ - if (dma_omap1()) - c->plat->dma_read(CSR, c->dma_ch); - else - c->plat->dma_write(~0, CSR, c->dma_ch); + omap_dma_clear_csr(c); val = c->plat->dma_read(CCR, c->dma_ch); if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { From 893e63e301e37cd3be7afb55c95eb8ef6ead304b Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 3 Nov 2013 11:17:11 +0000 Subject: [PATCH 75/89] dmaengine: omap-dma: improve efficiency loading C.SA/C.EI/C.FI registers The only thing which changes is which registers are written, so put this in local variables instead. This results in smaller code. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index d1641aa9d113..06727a78e883 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -275,17 +275,21 @@ static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, unsigned idx) { struct omap_sg *sg = d->sg + idx; + unsigned cxsa, cxei, cxfi; if (d->dir == DMA_DEV_TO_MEM) { - c->plat->dma_write(sg->addr, CDSA, c->dma_ch); - c->plat->dma_write(0, CDEI, c->dma_ch); - c->plat->dma_write(0, CDFI, c->dma_ch); + cxsa = CDSA; + cxei = CDEI; + cxfi = CDFI; } else { - c->plat->dma_write(sg->addr, CSSA, c->dma_ch); - c->plat->dma_write(0, CSEI, c->dma_ch); - c->plat->dma_write(0, CSFI, c->dma_ch); + cxsa = CSSA; + cxei = CSEI; + cxfi = CSFI; } + c->plat->dma_write(sg->addr, cxsa, c->dma_ch); + c->plat->dma_write(0, cxei, c->dma_ch); + c->plat->dma_write(0, cxfi, c->dma_ch); c->plat->dma_write(sg->en, CEN, c->dma_ch); c->plat->dma_write(sg->fn, CFN, c->dma_ch); @@ -296,6 +300,7 @@ static void omap_dma_start_desc(struct omap_chan *c) { struct virt_dma_desc *vd = vchan_next_desc(&c->vc); struct omap_desc *d; + unsigned cxsa, cxei, cxfi; if (!vd) { c->desc = NULL; @@ -312,15 +317,18 @@ static void omap_dma_start_desc(struct omap_chan *c) c->plat->dma_write(d->ccr >> 16, CCR2, c->dma_ch); if (d->dir == DMA_DEV_TO_MEM) { - c->plat->dma_write(d->dev_addr, CSSA, c->dma_ch); - c->plat->dma_write(0, CSEI, c->dma_ch); - c->plat->dma_write(d->fi, CSFI, c->dma_ch); + cxsa = CSSA; + cxei = CSEI; + cxfi = CSFI; } else { - c->plat->dma_write(d->dev_addr, CDSA, c->dma_ch); - c->plat->dma_write(0, CDEI, c->dma_ch); - c->plat->dma_write(d->fi, CDFI, c->dma_ch); + cxsa = CDSA; + cxei = CDEI; + cxfi = CDFI; } + c->plat->dma_write(d->dev_addr, cxsa, c->dma_ch); + c->plat->dma_write(0, cxei, c->dma_ch); + c->plat->dma_write(d->fi, cxfi, c->dma_ch); c->plat->dma_write(d->csdp, CSDP, c->dma_ch); omap_dma_start_sg(c, d, 0); From 965aeb4df1f2142f5a6407c6d40b7196be719582 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 6 Nov 2013 17:12:30 +0000 Subject: [PATCH 76/89] dmaengine: omap-dma: move clnk_ctrl setting to preparation functions Move the clnk_ctrl setup to the preparation functions, saving its value in the omap_desc. This only needs to be set once per descriptor, not for each segment, so set it in omap_dma_start_desc() rather than omap_dma_start(). Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 06727a78e883..49609275b2e7 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -59,6 +59,7 @@ struct omap_desc { int16_t fi; /* for OMAP_DMA_SYNC_PACKET */ uint8_t es; /* CSDP_DATA_TYPE_xxx */ uint32_t ccr; /* CCR value */ + uint16_t clnk_ctrl; /* CLNK_CTRL value */ uint16_t cicr; /* CICR value */ uint32_t csdp; /* CSDP value */ @@ -187,18 +188,6 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) else c->plat->dma_write(0, CDAC, c->dma_ch); - if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { - val = c->plat->dma_read(CLNK_CTRL, c->dma_ch); - - if (dma_omap1()) - val &= ~(1 << 14); - - val |= c->dma_ch | CLNK_CTRL_ENABLE_LNK; - - c->plat->dma_write(val, CLNK_CTRL, c->dma_ch); - } else if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) - c->plat->dma_write(c->dma_ch, CLNK_CTRL, c->dma_ch); - omap_dma_clear_csr(c); /* Enable interrupts */ @@ -330,6 +319,7 @@ static void omap_dma_start_desc(struct omap_chan *c) c->plat->dma_write(0, cxei, c->dma_ch); c->plat->dma_write(d->fi, cxfi, c->dma_ch); c->plat->dma_write(d->csdp, CSDP, c->dma_ch); + c->plat->dma_write(d->clnk_ctrl, CLNK_CTRL, c->dma_ch); omap_dma_start_sg(c, d, 0); } @@ -645,6 +635,8 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( } if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) d->ccr |= CCR_BUFFERING_DISABLE; + if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) + d->clnk_ctrl = c->dma_ch; /* * Build our scatterlist entries: each contains the address, @@ -725,8 +717,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->sglen = 1; d->ccr = 0; - if (__dma_omap15xx(od->plat->dma_attr)) - d->ccr = CCR_AUTO_INIT | CCR_REPEAT; if (dir == DMA_DEV_TO_MEM) d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; else @@ -772,6 +762,11 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) d->ccr |= CCR_BUFFERING_DISABLE; + if (__dma_omap15xx(od->plat->dma_attr)) + d->ccr |= CCR_AUTO_INIT | CCR_REPEAT; + else + d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK; + c->cyclic = true; return vchan_tx_prep(&c->vc, &d->vd, flags); From 59871902703c47acc730555be41bd9cb36d3700c Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 6 Nov 2013 17:15:16 +0000 Subject: [PATCH 77/89] dmaengine: omap-dma: move barrier to omap_dma_start_desc() We don't need to issue a barrier for every segment of a DMA transfer; doing this just once per descriptor will do. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 49609275b2e7..49b303296d75 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -195,7 +195,6 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) val = c->plat->dma_read(CCR, c->dma_ch); val |= CCR_ENABLE; - mb(); c->plat->dma_write(val, CCR, c->dma_ch); } @@ -301,6 +300,13 @@ static void omap_dma_start_desc(struct omap_chan *c) c->desc = d = to_omap_dma_desc(&vd->tx); c->sgidx = 0; + /* + * This provides the necessary barrier to ensure data held in + * DMA coherent memory is visible to the DMA engine prior to + * the transfer starting. + */ + mb(); + c->plat->dma_write(d->ccr, CCR, c->dma_ch); if (dma_omap1()) c->plat->dma_write(d->ccr >> 16, CCR2, c->dma_ch); From 45da7b0451b1fe15e882b08c79be58458cbe7a2f Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 6 Nov 2013 17:18:42 +0000 Subject: [PATCH 78/89] dmaengine: omap-dma: use cached CCR value when enabling DMA We don't need to read-modify-write the CCR register; we already know what value it should contain at this point. Use the cached CCR value when setting the enable bit. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 49b303296d75..b270aedf1d15 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -181,7 +181,6 @@ static void omap_dma_clear_csr(struct omap_chan *c) static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) { struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); - uint32_t val; if (__dma_omap15xx(od->plat->dma_attr)) c->plat->dma_write(0, CPC, c->dma_ch); @@ -193,9 +192,8 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) /* Enable interrupts */ c->plat->dma_write(d->cicr, CICR, c->dma_ch); - val = c->plat->dma_read(CCR, c->dma_ch); - val |= CCR_ENABLE; - c->plat->dma_write(val, CCR, c->dma_ch); + /* Enable channel */ + c->plat->dma_write(d->ccr | CCR_ENABLE, CCR, c->dma_ch); } static void omap_dma_stop(struct omap_chan *c) From c5ed98b6ae79545284b7855a07ded32934865a6d Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 6 Nov 2013 17:33:09 +0000 Subject: [PATCH 79/89] dmaengine: omap-dma: provide register read/write functions Provide a pair of channel register accessors, and a pair of global accessors for non-channel specific registers. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 102 ++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 41 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index b270aedf1d15..a1baada7dcd4 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -170,12 +170,32 @@ static void omap_dma_desc_free(struct virt_dma_desc *vd) kfree(container_of(vd, struct omap_desc, vd)); } +static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val) +{ + od->plat->dma_write(val, reg, 0); +} + +static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg) +{ + return od->plat->dma_read(reg, 0); +} + +static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val) +{ + c->plat->dma_write(val, reg, c->dma_ch); +} + +static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg) +{ + return c->plat->dma_read(reg, c->dma_ch); +} + static void omap_dma_clear_csr(struct omap_chan *c) { if (dma_omap1()) - c->plat->dma_read(CSR, c->dma_ch); + omap_dma_chan_read(c, CSR); else - c->plat->dma_write(~0, CSR, c->dma_ch); + omap_dma_chan_write(c, CSR, ~0); } static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) @@ -183,17 +203,17 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); if (__dma_omap15xx(od->plat->dma_attr)) - c->plat->dma_write(0, CPC, c->dma_ch); + omap_dma_chan_write(c, CPC, 0); else - c->plat->dma_write(0, CDAC, c->dma_ch); + omap_dma_chan_write(c, CDAC, 0); omap_dma_clear_csr(c); /* Enable interrupts */ - c->plat->dma_write(d->cicr, CICR, c->dma_ch); + omap_dma_chan_write(c, CICR, d->cicr); /* Enable channel */ - c->plat->dma_write(d->ccr | CCR_ENABLE, CCR, c->dma_ch); + omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE); } static void omap_dma_stop(struct omap_chan *c) @@ -202,27 +222,27 @@ static void omap_dma_stop(struct omap_chan *c) uint32_t val; /* disable irq */ - c->plat->dma_write(0, CICR, c->dma_ch); + omap_dma_chan_write(c, CICR, 0); omap_dma_clear_csr(c); - val = c->plat->dma_read(CCR, c->dma_ch); + val = omap_dma_chan_read(c, CCR); if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { uint32_t sysconfig; unsigned i; - sysconfig = c->plat->dma_read(OCP_SYSCONFIG, c->dma_ch); + sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK; val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE); - c->plat->dma_write(val, OCP_SYSCONFIG, c->dma_ch); + omap_dma_glbl_write(od, OCP_SYSCONFIG, val); - val = c->plat->dma_read(CCR, c->dma_ch); + val = omap_dma_chan_read(c, CCR); val &= ~CCR_ENABLE; - c->plat->dma_write(val, CCR, c->dma_ch); + omap_dma_chan_write(c, CCR, val); /* Wait for sDMA FIFO to drain */ for (i = 0; ; i++) { - val = c->plat->dma_read(CCR, c->dma_ch); + val = omap_dma_chan_read(c, CCR); if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))) break; @@ -237,23 +257,23 @@ static void omap_dma_stop(struct omap_chan *c) "DMA drain did not complete on lch %d\n", c->dma_ch); - c->plat->dma_write(sysconfig, OCP_SYSCONFIG, c->dma_ch); + omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig); } else { val &= ~CCR_ENABLE; - c->plat->dma_write(val, CCR, c->dma_ch); + omap_dma_chan_write(c, CCR, val); } mb(); if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { - val = c->plat->dma_read(CLNK_CTRL, c->dma_ch); + val = omap_dma_chan_read(c, CLNK_CTRL); if (dma_omap1()) val |= 1 << 14; /* set the STOP_LNK bit */ else val &= ~CLNK_CTRL_ENABLE_LNK; - c->plat->dma_write(val, CLNK_CTRL, c->dma_ch); + omap_dma_chan_write(c, CLNK_CTRL, val); } } @@ -273,11 +293,11 @@ static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, cxfi = CSFI; } - c->plat->dma_write(sg->addr, cxsa, c->dma_ch); - c->plat->dma_write(0, cxei, c->dma_ch); - c->plat->dma_write(0, cxfi, c->dma_ch); - c->plat->dma_write(sg->en, CEN, c->dma_ch); - c->plat->dma_write(sg->fn, CFN, c->dma_ch); + omap_dma_chan_write(c, cxsa, sg->addr); + omap_dma_chan_write(c, cxei, 0); + omap_dma_chan_write(c, cxfi, 0); + omap_dma_chan_write(c, CEN, sg->en); + omap_dma_chan_write(c, CFN, sg->fn); omap_dma_start(c, d); } @@ -305,9 +325,9 @@ static void omap_dma_start_desc(struct omap_chan *c) */ mb(); - c->plat->dma_write(d->ccr, CCR, c->dma_ch); + omap_dma_chan_write(c, CCR, d->ccr); if (dma_omap1()) - c->plat->dma_write(d->ccr >> 16, CCR2, c->dma_ch); + omap_dma_chan_write(c, CCR2, d->ccr >> 16); if (d->dir == DMA_DEV_TO_MEM) { cxsa = CSSA; @@ -319,11 +339,11 @@ static void omap_dma_start_desc(struct omap_chan *c) cxfi = CDFI; } - c->plat->dma_write(d->dev_addr, cxsa, c->dma_ch); - c->plat->dma_write(0, cxei, c->dma_ch); - c->plat->dma_write(d->fi, cxfi, c->dma_ch); - c->plat->dma_write(d->csdp, CSDP, c->dma_ch); - c->plat->dma_write(d->clnk_ctrl, CLNK_CTRL, c->dma_ch); + omap_dma_chan_write(c, cxsa, d->dev_addr); + omap_dma_chan_write(c, cxei, 0); + omap_dma_chan_write(c, cxfi, d->fi); + omap_dma_chan_write(c, CSDP, d->csdp); + omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl); omap_dma_start_sg(c, d, 0); } @@ -437,12 +457,12 @@ static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c) dma_addr_t addr; if (__dma_omap15xx(od->plat->dma_attr)) - addr = c->plat->dma_read(CPC, c->dma_ch); + addr = omap_dma_chan_read(c, CPC); else - addr = c->plat->dma_read(CSAC, c->dma_ch); + addr = omap_dma_chan_read(c, CSAC); if (od->plat->errata & DMA_ERRATA_3_3 && addr == 0) - addr = c->plat->dma_read(CSAC, c->dma_ch); + addr = omap_dma_chan_read(c, CSAC); if (!__dma_omap15xx(od->plat->dma_attr)) { /* @@ -450,14 +470,14 @@ static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c) * not been started (no data has been transferred so far). * Return the programmed source start address in this case. */ - if (c->plat->dma_read(CDAC, c->dma_ch)) - addr = c->plat->dma_read(CSAC, c->dma_ch); + if (omap_dma_chan_read(c, CDAC)) + addr = omap_dma_chan_read(c, CSAC); else - addr = c->plat->dma_read(CSSA, c->dma_ch); + addr = omap_dma_chan_read(c, CSSA); } if (dma_omap1()) - addr |= c->plat->dma_read(CSSA, c->dma_ch) & 0xffff0000; + addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000; return addr; } @@ -468,27 +488,27 @@ static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c) dma_addr_t addr; if (__dma_omap15xx(od->plat->dma_attr)) - addr = c->plat->dma_read(CPC, c->dma_ch); + addr = omap_dma_chan_read(c, CPC); else - addr = c->plat->dma_read(CDAC, c->dma_ch); + addr = omap_dma_chan_read(c, CDAC); /* * omap 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is * read before the DMA controller finished disabling the channel. */ if (!__dma_omap15xx(od->plat->dma_attr) && addr == 0) { - addr = c->plat->dma_read(CDAC, c->dma_ch); + addr = omap_dma_chan_read(c, CDAC); /* * CDAC == 0 indicates that the DMA transfer on the channel has * not been started (no data has been transferred so far). * Return the programmed destination start address in this case. */ if (addr == 0) - addr = c->plat->dma_read(CDSA, c->dma_ch); + addr = omap_dma_chan_read(c, CDSA); } if (dma_omap1()) - addr |= c->plat->dma_read(CDSA, c->dma_ch) & 0xffff0000; + addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000; return addr; } From b07fd625ac9df7412bd996edbdc298eb343dd501 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 6 Nov 2013 19:26:45 +0000 Subject: [PATCH 80/89] dmaengine: omap-dma: cleanup errata 3.3 handling Provide a function to read the CSAC/CDAC register, working around the OMAP 3.2/3.3 erratum (which requires two reads of the register if the first returned zero. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 52 +++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index a1baada7dcd4..1e0018f36384 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -451,28 +451,39 @@ static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr) return size; } +/* + * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is + * read before the DMA controller finished disabling the channel. + */ +static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + uint32_t val; + + val = omap_dma_chan_read(c, reg); + if (val == 0 && od->plat->errata & DMA_ERRATA_3_3) + val = omap_dma_chan_read(c, reg); + + return val; +} + static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c) { struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); - dma_addr_t addr; + dma_addr_t addr, cdac; - if (__dma_omap15xx(od->plat->dma_attr)) + if (__dma_omap15xx(od->plat->dma_attr)) { addr = omap_dma_chan_read(c, CPC); - else - addr = omap_dma_chan_read(c, CSAC); + } else { + addr = omap_dma_chan_read_3_3(c, CSAC); + cdac = omap_dma_chan_read_3_3(c, CDAC); - if (od->plat->errata & DMA_ERRATA_3_3 && addr == 0) - addr = omap_dma_chan_read(c, CSAC); - - if (!__dma_omap15xx(od->plat->dma_attr)) { /* * CDAC == 0 indicates that the DMA transfer on the channel has * not been started (no data has been transferred so far). * Return the programmed source start address in this case. */ - if (omap_dma_chan_read(c, CDAC)) - addr = omap_dma_chan_read(c, CSAC); - else + if (cdac == 0) addr = omap_dma_chan_read(c, CSSA); } @@ -487,21 +498,16 @@ static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c) struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); dma_addr_t addr; - if (__dma_omap15xx(od->plat->dma_attr)) + if (__dma_omap15xx(od->plat->dma_attr)) { addr = omap_dma_chan_read(c, CPC); - else - addr = omap_dma_chan_read(c, CDAC); + } else { + addr = omap_dma_chan_read_3_3(c, CDAC); - /* - * omap 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is - * read before the DMA controller finished disabling the channel. - */ - if (!__dma_omap15xx(od->plat->dma_attr) && addr == 0) { - addr = omap_dma_chan_read(c, CDAC); /* - * CDAC == 0 indicates that the DMA transfer on the channel has - * not been started (no data has been transferred so far). - * Return the programmed destination start address in this case. + * CDAC == 0 indicates that the DMA transfer on the channel + * has not been started (no data has been transferred so + * far). Return the programmed destination start address in + * this case. */ if (addr == 0) addr = omap_dma_chan_read(c, CDSA); From e38b1485fde832f72ab478f947f6e78a8e28c58b Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 8 Nov 2013 14:48:55 +0000 Subject: [PATCH 81/89] ARM: omap: remove references to disable_irq_lch The disable_irq_lch method is never actually used, so there's not much point it existing; remove it. Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mach-omap1/dma.c | 1 - arch/arm/mach-omap2/dma.c | 10 ---------- include/linux/omap-dma.h | 1 - 3 files changed, 12 deletions(-) diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c index 5bb8ce86d54b..d292055d3117 100644 --- a/arch/arm/mach-omap1/dma.c +++ b/arch/arm/mach-omap1/dma.c @@ -363,7 +363,6 @@ static int __init omap1_system_dma_init(void) p->clear_dma = omap1_clear_dma; p->dma_write = dma_write; p->dma_read = dma_read; - p->disable_irq_lch = NULL; p->errata = configure_dma_errata(); diff --git a/arch/arm/mach-omap2/dma.c b/arch/arm/mach-omap2/dma.c index 49fd0d501c9b..81c2d3383bc5 100644 --- a/arch/arm/mach-omap2/dma.c +++ b/arch/arm/mach-omap2/dma.c @@ -112,15 +112,6 @@ static inline u32 dma_read(int reg, int lch) return val; } -static inline void omap2_disable_irq_lch(int lch) -{ - u32 val; - - val = dma_read(IRQENABLE_L0, lch); - val &= ~(1 << lch); - dma_write(val, IRQENABLE_L0, lch); -} - static void omap2_clear_dma(int lch) { int i = dma_common_ch_start; @@ -239,7 +230,6 @@ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) } p->dma_attr = (struct omap_dma_dev_attr *)oh->dev_attr; - p->disable_irq_lch = omap2_disable_irq_lch; p->show_dma_caps = omap2_show_dma_caps; p->clear_dma = omap2_clear_dma; p->dma_write = dma_write; diff --git a/include/linux/omap-dma.h b/include/linux/omap-dma.h index d631658e2237..0bb7de77d478 100644 --- a/include/linux/omap-dma.h +++ b/include/linux/omap-dma.h @@ -275,7 +275,6 @@ struct omap_dma_dev_attr { struct omap_system_dma_plat_info { struct omap_dma_dev_attr *dma_attr; u32 errata; - void (*disable_irq_lch)(int lch); void (*show_dma_caps)(void); void (*clear_lch_regs)(int lch); void (*clear_dma)(int lch); From ad0c381a8b3a15b8edfca0996729ea45692470ca Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 8 Nov 2013 14:53:35 +0000 Subject: [PATCH 82/89] ARM: omap: remove almost-const variables dma_stride and dma_common_ch_start are only ever initialised to one known value at initialisation, and are private to each of these files. There's no point these being variables at all. Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mach-omap1/dma.c | 14 ++++---------- arch/arm/mach-omap2/dma.c | 14 +++++--------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c index d292055d3117..11f0b0ee67a3 100644 --- a/arch/arm/mach-omap1/dma.c +++ b/arch/arm/mach-omap1/dma.c @@ -36,8 +36,6 @@ static u32 errata; static u32 enable_1510_mode; -static u8 dma_stride; -static enum omap_reg_offsets dma_common_ch_start, dma_common_ch_end; static u16 reg_map[] = { [GCR] = 0x400, @@ -184,7 +182,7 @@ static inline void dma_write(u32 val, int reg, int lch) u8 stride; u32 offset; - stride = (reg >= dma_common_ch_start) ? dma_stride : 0; + stride = (reg >= CPC) ? OMAP1_DMA_STRIDE : 0; offset = reg_map[reg] + (stride * lch); __raw_writew(val, dma_base + offset); @@ -200,7 +198,7 @@ static inline u32 dma_read(int reg, int lch) u8 stride; u32 offset, val; - stride = (reg >= dma_common_ch_start) ? dma_stride : 0; + stride = (reg >= CPC) ? OMAP1_DMA_STRIDE : 0; offset = reg_map[reg] + (stride * lch); val = __raw_readw(dma_base + offset); @@ -216,9 +214,9 @@ static inline u32 dma_read(int reg, int lch) static void omap1_clear_lch_regs(int lch) { - int i = dma_common_ch_start; + int i; - for (; i <= dma_common_ch_end; i += 1) + for (i = CPC; i <= COLOR; i += 1) dma_write(0, i, lch); } @@ -380,10 +378,6 @@ static int __init omap1_system_dma_init(void) goto exit_release_chan; } - dma_stride = OMAP1_DMA_STRIDE; - dma_common_ch_start = CPC; - dma_common_ch_end = COLOR; - dma_pdev = platform_device_register_full(&omap_dma_dev_info); if (IS_ERR(dma_pdev)) { ret = PTR_ERR(dma_pdev); diff --git a/arch/arm/mach-omap2/dma.c b/arch/arm/mach-omap2/dma.c index 81c2d3383bc5..e4ac7ac9a228 100644 --- a/arch/arm/mach-omap2/dma.c +++ b/arch/arm/mach-omap2/dma.c @@ -38,11 +38,10 @@ #define OMAP2_DMA_STRIDE 0x60 static u32 errata; -static u8 dma_stride; static struct omap_dma_dev_attr *d; -static enum omap_reg_offsets dma_common_ch_start, dma_common_ch_end; +static enum omap_reg_offsets dma_common_ch_end; static u16 reg_map[] = { [REVISION] = 0x00, @@ -96,7 +95,7 @@ static inline void dma_write(u32 val, int reg, int lch) u8 stride; u32 offset; - stride = (reg >= dma_common_ch_start) ? dma_stride : 0; + stride = (reg >= CSDP) ? OMAP2_DMA_STRIDE : 0; offset = reg_map[reg] + (stride * lch); __raw_writel(val, dma_base + offset); } @@ -106,7 +105,7 @@ static inline u32 dma_read(int reg, int lch) u8 stride; u32 offset, val; - stride = (reg >= dma_common_ch_start) ? dma_stride : 0; + stride = (reg >= CSDP) ? OMAP2_DMA_STRIDE : 0; offset = reg_map[reg] + (stride * lch); val = __raw_readl(dma_base + offset); return val; @@ -114,9 +113,9 @@ static inline u32 dma_read(int reg, int lch) static void omap2_clear_dma(int lch) { - int i = dma_common_ch_start; + int i; - for (; i <= dma_common_ch_end; i += 1) + for (i = CSDP; i <= dma_common_ch_end; i += 1) dma_write(0, i, lch); } @@ -219,9 +218,6 @@ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) struct resource *mem; char *name = "omap_dma_system"; - dma_stride = OMAP2_DMA_STRIDE; - dma_common_ch_start = CSDP; - p = kzalloc(sizeof(struct omap_system_dma_plat_info), GFP_KERNEL); if (!p) { pr_err("%s: Unable to allocate pdata for %s:%s\n", From 64a2dc3d3de4235eb73921d870a674a23d9888f0 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 8 Nov 2013 18:04:06 +0000 Subject: [PATCH 83/89] ARM: omap: clean up DMA register accesses We can do much better with this by using a structure to describe each register, rather than code. Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mach-omap1/dma.c | 113 ++++++++++++++++++-------------------- arch/arm/mach-omap2/dma.c | 99 ++++++++++++++++----------------- include/linux/omap-dma.h | 13 +++++ 3 files changed, 113 insertions(+), 112 deletions(-) diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c index 11f0b0ee67a3..b680db7a35d4 100644 --- a/arch/arm/mach-omap1/dma.c +++ b/arch/arm/mach-omap1/dma.c @@ -32,53 +32,52 @@ #define OMAP1_DMA_BASE (0xfffed800) #define OMAP1_LOGICAL_DMA_CH_COUNT 17 -#define OMAP1_DMA_STRIDE 0x40 static u32 errata; static u32 enable_1510_mode; -static u16 reg_map[] = { - [GCR] = 0x400, - [GSCR] = 0x404, - [GRST1] = 0x408, - [HW_ID] = 0x442, - [PCH2_ID] = 0x444, - [PCH0_ID] = 0x446, - [PCH1_ID] = 0x448, - [PCHG_ID] = 0x44a, - [PCHD_ID] = 0x44c, - [CAPS_0] = 0x44e, - [CAPS_1] = 0x452, - [CAPS_2] = 0x456, - [CAPS_3] = 0x458, - [CAPS_4] = 0x45a, - [PCH2_SR] = 0x460, - [PCH0_SR] = 0x480, - [PCH1_SR] = 0x482, - [PCHD_SR] = 0x4c0, +static const struct omap_dma_reg reg_map[] = { + [GCR] = { 0x0400, 0x00, OMAP_DMA_REG_16BIT }, + [GSCR] = { 0x0404, 0x00, OMAP_DMA_REG_16BIT }, + [GRST1] = { 0x0408, 0x00, OMAP_DMA_REG_16BIT }, + [HW_ID] = { 0x0442, 0x00, OMAP_DMA_REG_16BIT }, + [PCH2_ID] = { 0x0444, 0x00, OMAP_DMA_REG_16BIT }, + [PCH0_ID] = { 0x0446, 0x00, OMAP_DMA_REG_16BIT }, + [PCH1_ID] = { 0x0448, 0x00, OMAP_DMA_REG_16BIT }, + [PCHG_ID] = { 0x044a, 0x00, OMAP_DMA_REG_16BIT }, + [PCHD_ID] = { 0x044c, 0x00, OMAP_DMA_REG_16BIT }, + [CAPS_0] = { 0x044e, 0x00, OMAP_DMA_REG_2X16BIT }, + [CAPS_1] = { 0x0452, 0x00, OMAP_DMA_REG_2X16BIT }, + [CAPS_2] = { 0x0456, 0x00, OMAP_DMA_REG_16BIT }, + [CAPS_3] = { 0x0458, 0x00, OMAP_DMA_REG_16BIT }, + [CAPS_4] = { 0x045a, 0x00, OMAP_DMA_REG_16BIT }, + [PCH2_SR] = { 0x0460, 0x00, OMAP_DMA_REG_16BIT }, + [PCH0_SR] = { 0x0480, 0x00, OMAP_DMA_REG_16BIT }, + [PCH1_SR] = { 0x0482, 0x00, OMAP_DMA_REG_16BIT }, + [PCHD_SR] = { 0x04c0, 0x00, OMAP_DMA_REG_16BIT }, /* Common Registers */ - [CSDP] = 0x00, - [CCR] = 0x02, - [CICR] = 0x04, - [CSR] = 0x06, - [CEN] = 0x10, - [CFN] = 0x12, - [CSFI] = 0x14, - [CSEI] = 0x16, - [CPC] = 0x18, /* 15xx only */ - [CSAC] = 0x18, - [CDAC] = 0x1a, - [CDEI] = 0x1c, - [CDFI] = 0x1e, - [CLNK_CTRL] = 0x28, + [CSDP] = { 0x0000, 0x40, OMAP_DMA_REG_16BIT }, + [CCR] = { 0x0002, 0x40, OMAP_DMA_REG_16BIT }, + [CICR] = { 0x0004, 0x40, OMAP_DMA_REG_16BIT }, + [CSR] = { 0x0006, 0x40, OMAP_DMA_REG_16BIT }, + [CEN] = { 0x0010, 0x40, OMAP_DMA_REG_16BIT }, + [CFN] = { 0x0012, 0x40, OMAP_DMA_REG_16BIT }, + [CSFI] = { 0x0014, 0x40, OMAP_DMA_REG_16BIT }, + [CSEI] = { 0x0016, 0x40, OMAP_DMA_REG_16BIT }, + [CPC] = { 0x0018, 0x40, OMAP_DMA_REG_16BIT }, /* 15xx only */ + [CSAC] = { 0x0018, 0x40, OMAP_DMA_REG_16BIT }, + [CDAC] = { 0x001a, 0x40, OMAP_DMA_REG_16BIT }, + [CDEI] = { 0x001c, 0x40, OMAP_DMA_REG_16BIT }, + [CDFI] = { 0x001e, 0x40, OMAP_DMA_REG_16BIT }, + [CLNK_CTRL] = { 0x0028, 0x40, OMAP_DMA_REG_16BIT }, /* Channel specific register offsets */ - [CSSA] = 0x08, - [CDSA] = 0x0c, - [COLOR] = 0x20, - [CCR2] = 0x24, - [LCH_CTRL] = 0x2a, + [CSSA] = { 0x0008, 0x40, OMAP_DMA_REG_2X16BIT }, + [CDSA] = { 0x000c, 0x40, OMAP_DMA_REG_2X16BIT }, + [COLOR] = { 0x0020, 0x40, OMAP_DMA_REG_2X16BIT }, + [CCR2] = { 0x0024, 0x40, OMAP_DMA_REG_16BIT }, + [LCH_CTRL] = { 0x002a, 0x40, OMAP_DMA_REG_16BIT }, }; static struct resource res[] __initdata = { @@ -179,36 +178,28 @@ static struct resource res[] __initdata = { static void __iomem *dma_base; static inline void dma_write(u32 val, int reg, int lch) { - u8 stride; - u32 offset; + void __iomem *addr = dma_base; - stride = (reg >= CPC) ? OMAP1_DMA_STRIDE : 0; - offset = reg_map[reg] + (stride * lch); + addr += reg_map[reg].offset; + addr += reg_map[reg].stride * lch; - __raw_writew(val, dma_base + offset); - if ((reg > CLNK_CTRL && reg < CCEN) || - (reg > PCHD_ID && reg < CAPS_2)) { - u32 offset2 = reg_map[reg] + 2 + (stride * lch); - __raw_writew(val >> 16, dma_base + offset2); - } + __raw_writew(val, addr); + if (reg_map[reg].type == OMAP_DMA_REG_2X16BIT) + __raw_writew(val >> 16, addr + 2); } static inline u32 dma_read(int reg, int lch) { - u8 stride; - u32 offset, val; + void __iomem *addr = dma_base; + uint32_t val; - stride = (reg >= CPC) ? OMAP1_DMA_STRIDE : 0; - offset = reg_map[reg] + (stride * lch); + addr += reg_map[reg].offset; + addr += reg_map[reg].stride * lch; + + val = __raw_readw(addr); + if (reg_map[reg].type == OMAP_DMA_REG_2X16BIT) + val |= __raw_readw(addr + 2) << 16; - val = __raw_readw(dma_base + offset); - if ((reg > CLNK_CTRL && reg < CCEN) || - (reg > PCHD_ID && reg < CAPS_2)) { - u16 upper; - u32 offset2 = reg_map[reg] + 2 + (stride * lch); - upper = __raw_readw(dma_base + offset2); - val |= (upper << 16); - } return val; } diff --git a/arch/arm/mach-omap2/dma.c b/arch/arm/mach-omap2/dma.c index e4ac7ac9a228..e633b48a3fcb 100644 --- a/arch/arm/mach-omap2/dma.c +++ b/arch/arm/mach-omap2/dma.c @@ -35,80 +35,77 @@ #include "omap_hwmod.h" #include "omap_device.h" -#define OMAP2_DMA_STRIDE 0x60 - static u32 errata; static struct omap_dma_dev_attr *d; static enum omap_reg_offsets dma_common_ch_end; -static u16 reg_map[] = { - [REVISION] = 0x00, - [GCR] = 0x78, - [IRQSTATUS_L0] = 0x08, - [IRQSTATUS_L1] = 0x0c, - [IRQSTATUS_L2] = 0x10, - [IRQSTATUS_L3] = 0x14, - [IRQENABLE_L0] = 0x18, - [IRQENABLE_L1] = 0x1c, - [IRQENABLE_L2] = 0x20, - [IRQENABLE_L3] = 0x24, - [SYSSTATUS] = 0x28, - [OCP_SYSCONFIG] = 0x2c, - [CAPS_0] = 0x64, - [CAPS_2] = 0x6c, - [CAPS_3] = 0x70, - [CAPS_4] = 0x74, +static const struct omap_dma_reg reg_map[] = { + [REVISION] = { 0x0000, 0x00, OMAP_DMA_REG_32BIT }, + [GCR] = { 0x0078, 0x00, OMAP_DMA_REG_32BIT }, + [IRQSTATUS_L0] = { 0x0008, 0x00, OMAP_DMA_REG_32BIT }, + [IRQSTATUS_L1] = { 0x000c, 0x00, OMAP_DMA_REG_32BIT }, + [IRQSTATUS_L2] = { 0x0010, 0x00, OMAP_DMA_REG_32BIT }, + [IRQSTATUS_L3] = { 0x0014, 0x00, OMAP_DMA_REG_32BIT }, + [IRQENABLE_L0] = { 0x0018, 0x00, OMAP_DMA_REG_32BIT }, + [IRQENABLE_L1] = { 0x001c, 0x00, OMAP_DMA_REG_32BIT }, + [IRQENABLE_L2] = { 0x0020, 0x00, OMAP_DMA_REG_32BIT }, + [IRQENABLE_L3] = { 0x0024, 0x00, OMAP_DMA_REG_32BIT }, + [SYSSTATUS] = { 0x0028, 0x00, OMAP_DMA_REG_32BIT }, + [OCP_SYSCONFIG] = { 0x002c, 0x00, OMAP_DMA_REG_32BIT }, + [CAPS_0] = { 0x0064, 0x00, OMAP_DMA_REG_32BIT }, + [CAPS_2] = { 0x006c, 0x00, OMAP_DMA_REG_32BIT }, + [CAPS_3] = { 0x0070, 0x00, OMAP_DMA_REG_32BIT }, + [CAPS_4] = { 0x0074, 0x00, OMAP_DMA_REG_32BIT }, /* Common register offsets */ - [CCR] = 0x80, - [CLNK_CTRL] = 0x84, - [CICR] = 0x88, - [CSR] = 0x8c, - [CSDP] = 0x90, - [CEN] = 0x94, - [CFN] = 0x98, - [CSEI] = 0xa4, - [CSFI] = 0xa8, - [CDEI] = 0xac, - [CDFI] = 0xb0, - [CSAC] = 0xb4, - [CDAC] = 0xb8, + [CCR] = { 0x0080, 0x60, OMAP_DMA_REG_32BIT }, + [CLNK_CTRL] = { 0x0084, 0x60, OMAP_DMA_REG_32BIT }, + [CICR] = { 0x0088, 0x60, OMAP_DMA_REG_32BIT }, + [CSR] = { 0x008c, 0x60, OMAP_DMA_REG_32BIT }, + [CSDP] = { 0x0090, 0x60, OMAP_DMA_REG_32BIT }, + [CEN] = { 0x0094, 0x60, OMAP_DMA_REG_32BIT }, + [CFN] = { 0x0098, 0x60, OMAP_DMA_REG_32BIT }, + [CSEI] = { 0x00a4, 0x60, OMAP_DMA_REG_32BIT }, + [CSFI] = { 0x00a8, 0x60, OMAP_DMA_REG_32BIT }, + [CDEI] = { 0x00ac, 0x60, OMAP_DMA_REG_32BIT }, + [CDFI] = { 0x00b0, 0x60, OMAP_DMA_REG_32BIT }, + [CSAC] = { 0x00b4, 0x60, OMAP_DMA_REG_32BIT }, + [CDAC] = { 0x00b8, 0x60, OMAP_DMA_REG_32BIT }, /* Channel specific register offsets */ - [CSSA] = 0x9c, - [CDSA] = 0xa0, - [CCEN] = 0xbc, - [CCFN] = 0xc0, - [COLOR] = 0xc4, + [CSSA] = { 0x009c, 0x60, OMAP_DMA_REG_32BIT }, + [CDSA] = { 0x00a0, 0x60, OMAP_DMA_REG_32BIT }, + [CCEN] = { 0x00bc, 0x60, OMAP_DMA_REG_32BIT }, + [CCFN] = { 0x00c0, 0x60, OMAP_DMA_REG_32BIT }, + [COLOR] = { 0x00c4, 0x60, OMAP_DMA_REG_32BIT }, /* OMAP4 specific registers */ - [CDP] = 0xd0, - [CNDP] = 0xd4, - [CCDN] = 0xd8, + [CDP] = { 0x00d0, 0x60, OMAP_DMA_REG_32BIT }, + [CNDP] = { 0x00d4, 0x60, OMAP_DMA_REG_32BIT }, + [CCDN] = { 0x00d8, 0x60, OMAP_DMA_REG_32BIT }, }; static void __iomem *dma_base; static inline void dma_write(u32 val, int reg, int lch) { - u8 stride; - u32 offset; + void __iomem *addr = dma_base; - stride = (reg >= CSDP) ? OMAP2_DMA_STRIDE : 0; - offset = reg_map[reg] + (stride * lch); - __raw_writel(val, dma_base + offset); + addr += reg_map[reg].offset; + addr += reg_map[reg].stride * lch; + + __raw_writel(val, addr); } static inline u32 dma_read(int reg, int lch) { - u8 stride; - u32 offset, val; + void __iomem *addr = dma_base; - stride = (reg >= CSDP) ? OMAP2_DMA_STRIDE : 0; - offset = reg_map[reg] + (stride * lch); - val = __raw_readl(dma_base + offset); - return val; + addr += reg_map[reg].offset; + addr += reg_map[reg].stride * lch; + + return __raw_readl(addr); } static void omap2_clear_dma(int lch) diff --git a/include/linux/omap-dma.h b/include/linux/omap-dma.h index 0bb7de77d478..41328725721a 100644 --- a/include/linux/omap-dma.h +++ b/include/linux/omap-dma.h @@ -271,6 +271,19 @@ struct omap_dma_dev_attr { struct omap_dma_lch *chan; }; +enum { + OMAP_DMA_REG_NONE, + OMAP_DMA_REG_16BIT, + OMAP_DMA_REG_2X16BIT, + OMAP_DMA_REG_32BIT, +}; + +struct omap_dma_reg { + u16 offset; + u8 stride; + u8 type; +}; + /* System DMA platform data structure */ struct omap_system_dma_plat_info { struct omap_dma_dev_attr *dma_attr; From 0ef64986d69602ba8df933c62d0b6d6edfaf0557 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 8 Nov 2013 18:06:37 +0000 Subject: [PATCH 84/89] ARM: omap: dma: get rid of errata global There's no need for this to be a global variable; move it into the errata configuration function instead. Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mach-omap1/dma.c | 4 ++-- arch/arm/mach-omap2/dma.c | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c index b680db7a35d4..c979e3f88bbb 100644 --- a/arch/arm/mach-omap1/dma.c +++ b/arch/arm/mach-omap1/dma.c @@ -33,7 +33,6 @@ #define OMAP1_DMA_BASE (0xfffed800) #define OMAP1_LOGICAL_DMA_CH_COUNT 17 -static u32 errata; static u32 enable_1510_mode; static const struct omap_dma_reg reg_map[] = { @@ -244,8 +243,9 @@ static void omap1_show_dma_caps(void) return; } -static u32 configure_dma_errata(void) +static unsigned configure_dma_errata(void) { + unsigned errata = 0; /* * Erratum 3.2/3.3: sometimes 0 is returned if CSAC/CDAC is diff --git a/arch/arm/mach-omap2/dma.c b/arch/arm/mach-omap2/dma.c index e633b48a3fcb..244ff5012aed 100644 --- a/arch/arm/mach-omap2/dma.c +++ b/arch/arm/mach-omap2/dma.c @@ -35,8 +35,6 @@ #include "omap_hwmod.h" #include "omap_device.h" -static u32 errata; - static struct omap_dma_dev_attr *d; static enum omap_reg_offsets dma_common_ch_end; @@ -124,8 +122,9 @@ static void omap2_show_dma_caps(void) return; } -static u32 configure_dma_errata(void) +static unsigned configure_dma_errata(void) { + unsigned errata = 0; /* * Errata applicable for OMAP2430ES1.0 and all omap2420 From 9834f81314b2a5b1bb3cd0d8f46e61528b60d580 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 8 Nov 2013 18:10:42 +0000 Subject: [PATCH 85/89] ARM: omap: move dma channel allocation into plat-omap code This really needs to be there, because otherwise the plat-omap code can kfree() this data structure, and then re-use the pointer later. Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mach-omap1/dma.c | 17 ++--------------- arch/arm/mach-omap2/dma.c | 7 ------- arch/arm/plat-omap/dma.c | 11 ++++++++--- include/linux/omap-dma.h | 1 - 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c index c979e3f88bbb..d170f7d0a6ac 100644 --- a/arch/arm/mach-omap1/dma.c +++ b/arch/arm/mach-omap1/dma.c @@ -325,17 +325,6 @@ static int __init omap1_system_dma_init(void) d->dev_caps |= CLEAR_CSR_ON_READ; d->dev_caps |= IS_WORD_16; - - d->chan = kzalloc(sizeof(struct omap_dma_lch) * - (d->lch_count), GFP_KERNEL); - if (!d->chan) { - dev_err(&pdev->dev, - "%s: Memory allocation failed for d->chan!\n", - __func__); - ret = -ENOMEM; - goto exit_release_d; - } - if (cpu_is_omap15xx()) d->chan_count = 9; else if (cpu_is_omap16xx() || cpu_is_omap7xx()) { @@ -359,14 +348,14 @@ static int __init omap1_system_dma_init(void) if (ret) { dev_err(&pdev->dev, "%s: Unable to add resources for %s%d\n", __func__, pdev->name, pdev->id); - goto exit_release_chan; + goto exit_release_d; } ret = platform_device_add(pdev); if (ret) { dev_err(&pdev->dev, "%s: Unable to add resources for %s%d\n", __func__, pdev->name, pdev->id); - goto exit_release_chan; + goto exit_release_d; } dma_pdev = platform_device_register_full(&omap_dma_dev_info); @@ -379,8 +368,6 @@ static int __init omap1_system_dma_init(void) exit_release_pdev: platform_device_del(pdev); -exit_release_chan: - kfree(d->chan); exit_release_d: kfree(d); exit_release_p: diff --git a/arch/arm/mach-omap2/dma.c b/arch/arm/mach-omap2/dma.c index 244ff5012aed..9f210d637354 100644 --- a/arch/arm/mach-omap2/dma.c +++ b/arch/arm/mach-omap2/dma.c @@ -251,13 +251,6 @@ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) } d = oh->dev_attr; - d->chan = kzalloc(sizeof(struct omap_dma_lch) * - (d->lch_count), GFP_KERNEL); - - if (!d->chan) { - dev_err(&pdev->dev, "%s: kzalloc fail\n", __func__); - return -ENOMEM; - } if (cpu_is_omap34xx() && (omap_type() != OMAP2_DEVICE_TYPE_GP)) d->dev_caps |= HS_CHANNELS_RESERVED; diff --git a/arch/arm/plat-omap/dma.c b/arch/arm/plat-omap/dma.c index d4d9a5e62152..5f5b975887fc 100644 --- a/arch/arm/plat-omap/dma.c +++ b/arch/arm/plat-omap/dma.c @@ -2030,9 +2030,16 @@ static int omap_system_dma_probe(struct platform_device *pdev) dma_lch_count = d->lch_count; dma_chan_count = dma_lch_count; - dma_chan = d->chan; enable_1510_mode = d->dev_caps & ENABLE_1510_MODE; + dma_chan = devm_kcalloc(&pdev->dev, dma_lch_count, + sizeof(struct omap_dma_lch), GFP_KERNEL); + if (!dma_chan) { + dev_err(&pdev->dev, "%s: kzalloc fail\n", __func__); + return -ENOMEM; + } + + if (dma_omap2plus()) { dma_linked_lch = kzalloc(sizeof(struct dma_link_info) * dma_lch_count, GFP_KERNEL); @@ -2117,7 +2124,6 @@ exit_dma_irq_fail: } exit_dma_lch_fail: - kfree(dma_chan); return ret; } @@ -2137,7 +2143,6 @@ static int omap_system_dma_remove(struct platform_device *pdev) free_irq(dma_irq, (void *)(irq_rel + 1)); } } - kfree(dma_chan); return 0; } diff --git a/include/linux/omap-dma.h b/include/linux/omap-dma.h index 41328725721a..7813636a193d 100644 --- a/include/linux/omap-dma.h +++ b/include/linux/omap-dma.h @@ -268,7 +268,6 @@ struct omap_dma_dev_attr { u32 dev_caps; u16 lch_count; u16 chan_count; - struct omap_dma_lch *chan; }; enum { From 34a378fcb9273d73dbd6b209952d2f6bf88a3dd0 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 8 Nov 2013 18:21:35 +0000 Subject: [PATCH 86/89] ARM: omap: dma: get rid of 'p' allocation and clean up The omap_system_dma_plat_info structure is only seven words, it's not worth the expense of kmalloc()'ing backing store for this only to release it later. Note that platform_device_add_data() copies the data anyway. Clean up the initialisation of this structure - we don't even need code to initialise most of this structure. Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mach-omap1/dma.c | 36 ++++++++++++++---------------------- arch/arm/mach-omap2/dma.c | 34 +++++++++++++--------------------- 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c index d170f7d0a6ac..a8c83ccc36fb 100644 --- a/arch/arm/mach-omap1/dma.c +++ b/arch/arm/mach-omap1/dma.c @@ -263,9 +263,17 @@ static const struct platform_device_info omap_dma_dev_info = { .dma_mask = DMA_BIT_MASK(32), }; +static struct omap_system_dma_plat_info dma_plat_info __initdata = { + .show_dma_caps = omap1_show_dma_caps, + .clear_lch_regs = omap1_clear_lch_regs, + .clear_dma = omap1_clear_dma, + .dma_write = dma_write, + .dma_read = dma_read, +}; + static int __init omap1_system_dma_init(void) { - struct omap_system_dma_plat_info *p; + struct omap_system_dma_plat_info p; struct omap_dma_dev_attr *d; struct platform_device *pdev, *dma_pdev; int ret; @@ -291,20 +299,12 @@ static int __init omap1_system_dma_init(void) goto exit_iounmap; } - p = kzalloc(sizeof(struct omap_system_dma_plat_info), GFP_KERNEL); - if (!p) { - dev_err(&pdev->dev, "%s: Unable to allocate 'p' for %s\n", - __func__, pdev->name); - ret = -ENOMEM; - goto exit_iounmap; - } - d = kzalloc(sizeof(struct omap_dma_dev_attr), GFP_KERNEL); if (!d) { dev_err(&pdev->dev, "%s: Unable to allocate 'd' for %s\n", __func__, pdev->name); ret = -ENOMEM; - goto exit_release_p; + goto exit_iounmap; } d->lch_count = OMAP1_LOGICAL_DMA_CH_COUNT; @@ -334,17 +334,11 @@ static int __init omap1_system_dma_init(void) d->chan_count = 9; } - p->dma_attr = d; + p = dma_plat_info; + p.dma_attr = d; + p.errata = configure_dma_errata(); - p->show_dma_caps = omap1_show_dma_caps; - p->clear_lch_regs = omap1_clear_lch_regs; - p->clear_dma = omap1_clear_dma; - p->dma_write = dma_write; - p->dma_read = dma_read; - - p->errata = configure_dma_errata(); - - ret = platform_device_add_data(pdev, p, sizeof(*p)); + ret = platform_device_add_data(pdev, &p, sizeof(p)); if (ret) { dev_err(&pdev->dev, "%s: Unable to add resources for %s%d\n", __func__, pdev->name, pdev->id); @@ -370,8 +364,6 @@ exit_release_pdev: platform_device_del(pdev); exit_release_d: kfree(d); -exit_release_p: - kfree(p); exit_iounmap: iounmap(dma_base); exit_device_put: diff --git a/arch/arm/mach-omap2/dma.c b/arch/arm/mach-omap2/dma.c index 9f210d637354..6331fc4b4054 100644 --- a/arch/arm/mach-omap2/dma.c +++ b/arch/arm/mach-omap2/dma.c @@ -35,8 +35,6 @@ #include "omap_hwmod.h" #include "omap_device.h" -static struct omap_dma_dev_attr *d; - static enum omap_reg_offsets dma_common_ch_end; static const struct omap_dma_reg reg_map[] = { @@ -206,33 +204,27 @@ static unsigned configure_dma_errata(void) return errata; } +static struct omap_system_dma_plat_info dma_plat_info __initdata = { + .show_dma_caps = omap2_show_dma_caps, + .clear_dma = omap2_clear_dma, + .dma_write = dma_write, + .dma_read = dma_read, +}; + /* One time initializations */ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) { struct platform_device *pdev; - struct omap_system_dma_plat_info *p; + struct omap_system_dma_plat_info p; + struct omap_dma_dev_attr *d; struct resource *mem; char *name = "omap_dma_system"; - p = kzalloc(sizeof(struct omap_system_dma_plat_info), GFP_KERNEL); - if (!p) { - pr_err("%s: Unable to allocate pdata for %s:%s\n", - __func__, name, oh->name); - return -ENOMEM; - } + p = dma_plat_info; + p.dma_attr = (struct omap_dma_dev_attr *)oh->dev_attr; + p.errata = configure_dma_errata(); - p->dma_attr = (struct omap_dma_dev_attr *)oh->dev_attr; - p->show_dma_caps = omap2_show_dma_caps; - p->clear_dma = omap2_clear_dma; - p->dma_write = dma_write; - p->dma_read = dma_read; - - p->clear_lch_regs = NULL; - - p->errata = configure_dma_errata(); - - pdev = omap_device_build(name, 0, oh, p, sizeof(*p)); - kfree(p); + pdev = omap_device_build(name, 0, oh, &p, sizeof(p)); if (IS_ERR(pdev)) { pr_err("%s: Can't build omap_device for %s:%s.\n", __func__, name, oh->name); From 596c471b69249764d8e241b004736878204daa0f Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 10 Dec 2013 11:08:01 +0000 Subject: [PATCH 87/89] dmaengine: omap-dma: move register read/writes into omap-dma.c Export the DMA register information from the SoC specific data, such that we can access the registers directly in omap-dma.c, mapping the register region ourselves as well. Rather than calculating the DMA channel register in its entirety for each access, we pre-calculate an offset base address for the allocated DMA channel and then just use the appropriate register offset. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- arch/arm/mach-omap1/dma.c | 4 ++ arch/arm/mach-omap2/dma.c | 18 +++++--- drivers/dma/omap-dma.c | 96 +++++++++++++++++++++++++++++++++++---- include/linux/omap-dma.h | 2 + 4 files changed, 105 insertions(+), 15 deletions(-) diff --git a/arch/arm/mach-omap1/dma.c b/arch/arm/mach-omap1/dma.c index a8c83ccc36fb..4be601b638d7 100644 --- a/arch/arm/mach-omap1/dma.c +++ b/arch/arm/mach-omap1/dma.c @@ -261,9 +261,13 @@ static const struct platform_device_info omap_dma_dev_info = { .name = "omap-dma-engine", .id = -1, .dma_mask = DMA_BIT_MASK(32), + .res = res, + .num_res = 1, }; static struct omap_system_dma_plat_info dma_plat_info __initdata = { + .reg_map = reg_map, + .channel_stride = 0x40, .show_dma_caps = omap1_show_dma_caps, .clear_lch_regs = omap1_clear_lch_regs, .clear_dma = omap1_clear_dma, diff --git a/arch/arm/mach-omap2/dma.c b/arch/arm/mach-omap2/dma.c index 6331fc4b4054..5689c88d986d 100644 --- a/arch/arm/mach-omap2/dma.c +++ b/arch/arm/mach-omap2/dma.c @@ -205,12 +205,20 @@ static unsigned configure_dma_errata(void) } static struct omap_system_dma_plat_info dma_plat_info __initdata = { + .reg_map = reg_map, + .channel_stride = 0x60, .show_dma_caps = omap2_show_dma_caps, .clear_dma = omap2_clear_dma, .dma_write = dma_write, .dma_read = dma_read, }; +static struct platform_device_info omap_dma_dev_info = { + .name = "omap-dma-engine", + .id = -1, + .dma_mask = DMA_BIT_MASK(32), +}; + /* One time initializations */ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) { @@ -231,11 +239,15 @@ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) return PTR_ERR(pdev); } + omap_dma_dev_info.res = pdev->resource; + omap_dma_dev_info.num_res = pdev->num_resources; + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!mem) { dev_err(&pdev->dev, "%s: no mem resource\n", __func__); return -EINVAL; } + dma_base = ioremap(mem->start, resource_size(mem)); if (!dma_base) { dev_err(&pdev->dev, "%s: ioremap fail\n", __func__); @@ -256,12 +268,6 @@ static int __init omap2_system_dma_init_dev(struct omap_hwmod *oh, void *unused) return 0; } -static const struct platform_device_info omap_dma_dev_info = { - .name = "omap-dma-engine", - .id = -1, - .dma_mask = DMA_BIT_MASK(32), -}; - static int __init omap2_system_dma_init(void) { struct platform_device *pdev; diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 1e0018f36384..00f8e566cf12 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -27,13 +27,16 @@ struct omap_dmadev { spinlock_t lock; struct tasklet_struct task; struct list_head pending; + void __iomem *base; + const struct omap_dma_reg *reg_map; struct omap_system_dma_plat_info *plat; }; struct omap_chan { struct virt_dma_chan vc; struct list_head node; - struct omap_system_dma_plat_info *plat; + void __iomem *channel_base; + const struct omap_dma_reg *reg_map; struct dma_slave_config cfg; unsigned dma_sig; @@ -170,24 +173,77 @@ static void omap_dma_desc_free(struct virt_dma_desc *vd) kfree(container_of(vd, struct omap_desc, vd)); } +static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr) +{ + switch (type) { + case OMAP_DMA_REG_16BIT: + writew_relaxed(val, addr); + break; + case OMAP_DMA_REG_2X16BIT: + writew_relaxed(val, addr); + writew_relaxed(val >> 16, addr + 2); + break; + case OMAP_DMA_REG_32BIT: + writel_relaxed(val, addr); + break; + default: + WARN_ON(1); + } +} + +static unsigned omap_dma_read(unsigned type, void __iomem *addr) +{ + unsigned val; + + switch (type) { + case OMAP_DMA_REG_16BIT: + val = readw_relaxed(addr); + break; + case OMAP_DMA_REG_2X16BIT: + val = readw_relaxed(addr); + val |= readw_relaxed(addr + 2) << 16; + break; + case OMAP_DMA_REG_32BIT: + val = readl_relaxed(addr); + break; + default: + WARN_ON(1); + val = 0; + } + + return val; +} + static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val) { - od->plat->dma_write(val, reg, 0); + const struct omap_dma_reg *r = od->reg_map + reg; + + WARN_ON(r->stride); + + omap_dma_write(val, r->type, od->base + r->offset); } static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg) { - return od->plat->dma_read(reg, 0); + const struct omap_dma_reg *r = od->reg_map + reg; + + WARN_ON(r->stride); + + return omap_dma_read(r->type, od->base + r->offset); } static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val) { - c->plat->dma_write(val, reg, c->dma_ch); + const struct omap_dma_reg *r = c->reg_map + reg; + + omap_dma_write(val, r->type, c->channel_base + r->offset); } static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg) { - return c->plat->dma_read(reg, c->dma_ch); + const struct omap_dma_reg *r = c->reg_map + reg; + + return omap_dma_read(r->type, c->channel_base + r->offset); } static void omap_dma_clear_csr(struct omap_chan *c) @@ -198,6 +254,12 @@ static void omap_dma_clear_csr(struct omap_chan *c) omap_dma_chan_write(c, CSR, ~0); } +static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c, + unsigned lch) +{ + c->channel_base = od->base + od->plat->channel_stride * lch; +} + static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) { struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); @@ -400,18 +462,26 @@ static void omap_dma_sched(unsigned long data) static int omap_dma_alloc_chan_resources(struct dma_chan *chan) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); + int ret; - dev_dbg(c->vc.chan.device->dev, "allocating channel for %u\n", c->dma_sig); + dev_dbg(od->ddev.dev, "allocating channel for %u\n", c->dma_sig); - return omap_request_dma(c->dma_sig, "DMA engine", - omap_dma_callback, c, &c->dma_ch); + ret = omap_request_dma(c->dma_sig, "DMA engine", omap_dma_callback, + c, &c->dma_ch); + + if (ret >= 0) + omap_dma_assign(od, c, c->dma_ch); + + return ret; } static void omap_dma_free_chan_resources(struct dma_chan *chan) { struct omap_chan *c = to_omap_dma_chan(chan); + c->channel_base = NULL; vchan_free_chan_resources(&c->vc); omap_free_dma(c->dma_ch); @@ -917,7 +987,7 @@ static int omap_dma_chan_init(struct omap_dmadev *od, int dma_sig) if (!c) return -ENOMEM; - c->plat = od->plat; + c->reg_map = od->reg_map; c->dma_sig = dma_sig; c->vc.desc_free = omap_dma_desc_free; vchan_init(&c->vc, &od->ddev); @@ -944,16 +1014,24 @@ static void omap_dma_free(struct omap_dmadev *od) static int omap_dma_probe(struct platform_device *pdev) { struct omap_dmadev *od; + struct resource *res; int rc, i; od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); if (!od) return -ENOMEM; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + od->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(od->base)) + return PTR_ERR(od->base); + od->plat = omap_get_plat_info(); if (!od->plat) return -EPROBE_DEFER; + od->reg_map = od->plat->reg_map; + dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources; diff --git a/include/linux/omap-dma.h b/include/linux/omap-dma.h index 7813636a193d..41a13e70f41f 100644 --- a/include/linux/omap-dma.h +++ b/include/linux/omap-dma.h @@ -285,6 +285,8 @@ struct omap_dma_reg { /* System DMA platform data structure */ struct omap_system_dma_plat_info { + const struct omap_dma_reg *reg_map; + unsigned channel_stride; struct omap_dma_dev_attr *dma_attr; u32 errata; void (*show_dma_caps)(void); From 6ddeb6d844596cac13c4a3665c0bd61f074a81a7 Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 10 Dec 2013 19:05:50 +0000 Subject: [PATCH 88/89] dmaengine: omap-dma: move IRQ handling to omap-dma Move the interrupt handling for OMAP2+ into omap-dma, rather than using the legacy support in the platform code. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 121 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 6 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 00f8e566cf12..ec98e718de70 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -30,6 +30,10 @@ struct omap_dmadev { void __iomem *base; const struct omap_dma_reg *reg_map; struct omap_system_dma_plat_info *plat; + bool legacy; + spinlock_t irq_lock; + uint32_t irq_enable_mask; + struct omap_chan *lch_map[32]; }; struct omap_chan { @@ -254,10 +258,22 @@ static void omap_dma_clear_csr(struct omap_chan *c) omap_dma_chan_write(c, CSR, ~0); } +static unsigned omap_dma_get_csr(struct omap_chan *c) +{ + unsigned val = omap_dma_chan_read(c, CSR); + + if (!dma_omap1()) + omap_dma_chan_write(c, CSR, val); + + return val; +} + static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c, unsigned lch) { c->channel_base = od->base + od->plat->channel_stride * lch; + + od->lch_map[lch] = c; } static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) @@ -460,32 +476,103 @@ static void omap_dma_sched(unsigned long data) } } +static irqreturn_t omap_dma_irq(int irq, void *devid) +{ + struct omap_dmadev *od = devid; + unsigned status, channel; + + spin_lock(&od->irq_lock); + + status = omap_dma_glbl_read(od, IRQSTATUS_L1); + status &= od->irq_enable_mask; + if (status == 0) { + spin_unlock(&od->irq_lock); + return IRQ_NONE; + } + + while ((channel = ffs(status)) != 0) { + unsigned mask, csr; + struct omap_chan *c; + + channel -= 1; + mask = BIT(channel); + status &= ~mask; + + c = od->lch_map[channel]; + if (c == NULL) { + /* This should never happen */ + dev_err(od->ddev.dev, "invalid channel %u\n", channel); + continue; + } + + csr = omap_dma_get_csr(c); + omap_dma_glbl_write(od, IRQSTATUS_L1, mask); + + omap_dma_callback(channel, csr, c); + } + + spin_unlock(&od->irq_lock); + + return IRQ_HANDLED; +} + static int omap_dma_alloc_chan_resources(struct dma_chan *chan) { struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); int ret; - dev_dbg(od->ddev.dev, "allocating channel for %u\n", c->dma_sig); + if (od->legacy) { + ret = omap_request_dma(c->dma_sig, "DMA engine", + omap_dma_callback, c, &c->dma_ch); + } else { + ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL, + &c->dma_ch); + } - ret = omap_request_dma(c->dma_sig, "DMA engine", omap_dma_callback, - c, &c->dma_ch); + dev_dbg(od->ddev.dev, "allocating channel %u for %u\n", + c->dma_ch, c->dma_sig); - if (ret >= 0) + if (ret >= 0) { omap_dma_assign(od, c, c->dma_ch); + if (!od->legacy) { + unsigned val; + + spin_lock_irq(&od->irq_lock); + val = BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQSTATUS_L1, val); + od->irq_enable_mask |= val; + omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); + + val = omap_dma_glbl_read(od, IRQENABLE_L0); + val &= ~BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQENABLE_L0, val); + spin_unlock_irq(&od->irq_lock); + } + } + return ret; } static void omap_dma_free_chan_resources(struct dma_chan *chan) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); + if (!od->legacy) { + spin_lock_irq(&od->irq_lock); + od->irq_enable_mask &= ~BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); + spin_unlock_irq(&od->irq_lock); + } + c->channel_base = NULL; + od->lch_map[c->dma_ch] = NULL; vchan_free_chan_resources(&c->vc); omap_free_dma(c->dma_ch); - dev_dbg(c->vc.chan.device->dev, "freeing channel for %u\n", c->dma_sig); + dev_dbg(od->ddev.dev, "freeing channel for %u\n", c->dma_sig); } static size_t omap_dma_sg_size(struct omap_sg *sg) @@ -1015,7 +1102,7 @@ static int omap_dma_probe(struct platform_device *pdev) { struct omap_dmadev *od; struct resource *res; - int rc, i; + int rc, i, irq; od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); if (!od) @@ -1045,6 +1132,7 @@ static int omap_dma_probe(struct platform_device *pdev) INIT_LIST_HEAD(&od->ddev.channels); INIT_LIST_HEAD(&od->pending); spin_lock_init(&od->lock); + spin_lock_init(&od->irq_lock); tasklet_init(&od->task, omap_dma_sched, (unsigned long)od); @@ -1056,6 +1144,21 @@ static int omap_dma_probe(struct platform_device *pdev) } } + irq = platform_get_irq(pdev, 1); + if (irq <= 0) { + dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq); + od->legacy = true; + } else { + /* Disable all interrupts */ + od->irq_enable_mask = 0; + omap_dma_glbl_write(od, IRQENABLE_L1, 0); + + rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq, + IRQF_SHARED, "omap-dma-engine", od); + if (rc) + return rc; + } + rc = dma_async_device_register(&od->ddev); if (rc) { pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n", @@ -1092,6 +1195,12 @@ static int omap_dma_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&od->ddev); + + if (!od->legacy) { + /* Disable all interrupts */ + omap_dma_glbl_write(od, IRQENABLE_L0, 0); + } + omap_dma_free(od); return 0; From aa4c5b962a7a03eb6b43b3d2677c3677022c1223 Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 14 Jan 2014 23:58:10 +0000 Subject: [PATCH 89/89] dmaengine: omap-dma: more consolidation of CCR register setup We can move the handling of the DMA synchronisation control out of the prepare functions; this can be pre-calculated when the DMA channel has been allocated, so we don't need to duplicate this in both prepare functions. Acked-by: Tony Lindgren Acked-by: Vinod Koul Signed-off-by: Russell King --- drivers/dma/omap-dma.c | 46 +++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index ec98e718de70..64ceca2920b8 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -41,6 +41,7 @@ struct omap_chan { struct list_head node; void __iomem *channel_base; const struct omap_dma_reg *reg_map; + uint32_t ccr; struct dma_slave_config cfg; unsigned dma_sig; @@ -552,6 +553,21 @@ static int omap_dma_alloc_chan_resources(struct dma_chan *chan) } } + if (dma_omap1()) { + if (__dma_omap16xx(od->plat->dma_attr)) { + c->ccr = CCR_OMAP31_DISABLE; + /* Duplicate what plat-omap/dma.c does */ + c->ccr |= c->dma_ch + 1; + } else { + c->ccr = c->dma_sig & 0x1f; + } + } else { + c->ccr = c->dma_sig & 0x1f; + c->ccr |= (c->dma_sig & ~0x1f) << 14; + } + if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) + c->ccr |= CCR_BUFFERING_DISABLE; + return ret; } @@ -787,7 +803,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->dev_addr = dev_addr; d->es = es; - d->ccr = CCR_SYNC_FRAME; + d->ccr = c->ccr | CCR_SYNC_FRAME; if (dir == DMA_DEV_TO_MEM) d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; else @@ -797,14 +813,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->csdp = es; if (dma_omap1()) { - if (__dma_omap16xx(od->plat->dma_attr)) { - d->ccr |= CCR_OMAP31_DISABLE; - /* Duplicate what plat-omap/dma.c does */ - d->ccr |= c->dma_ch + 1; - } else { - d->ccr |= c->dma_sig & 0x1f; - } - d->cicr |= CICR_TOUT_IE; if (dir == DMA_DEV_TO_MEM) @@ -812,16 +820,11 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( else d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF; } else { - d->ccr |= (c->dma_sig & ~0x1f) << 14; - d->ccr |= c->dma_sig & 0x1f; - if (dir == DMA_DEV_TO_MEM) d->ccr |= CCR_TRIGGER_SRC; d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; } - if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) - d->ccr |= CCR_BUFFERING_DISABLE; if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) d->clnk_ctrl = c->dma_ch; @@ -903,7 +906,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->sg[0].fn = buf_len / period_len; d->sglen = 1; - d->ccr = 0; + d->ccr = c->ccr; if (dir == DMA_DEV_TO_MEM) d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; else @@ -916,14 +919,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->csdp = es; if (dma_omap1()) { - if (__dma_omap16xx(od->plat->dma_attr)) { - d->ccr |= CCR_OMAP31_DISABLE; - /* Duplicate what plat-omap/dma.c does */ - d->ccr |= c->dma_ch + 1; - } else { - d->ccr |= c->dma_sig & 0x1f; - } - d->cicr |= CICR_TOUT_IE; if (dir == DMA_DEV_TO_MEM) @@ -931,9 +926,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( else d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF; } else { - d->ccr |= (c->dma_sig & ~0x1f) << 14; - d->ccr |= c->dma_sig & 0x1f; - if (burst) d->ccr |= CCR_SYNC_PACKET; else @@ -946,8 +938,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; } - if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) - d->ccr |= CCR_BUFFERING_DISABLE; if (__dma_omap15xx(od->plat->dma_attr)) d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;