Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-10 14:11:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

ARM: 8930/1: Add support for generic vDSO

Browse Source 
The arm vDSO library requires some adaptations to take advantage of
the newly introduced generic vDSO library.

Introduce the following changes:
 - Modification vdso.c to be compliant with the common vdso datapage
 - Use of lib/vdso for gettimeofday
 - Implementation of elf note

Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
This commit is contained in:
Vincenzo Frascino 2019-11-04 11:59:59 +01:00 committed by Russell King
parent 9f1984c6ae
commit 20e2fc4231
8 changed files with 195 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
arch/arm/include/asm/vdso/gettimeofday.h Normal file
View File

@ -0,0 +1,74 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2018 ARM Limited
*/
#ifndef __ASM_VDSO_GETTIMEOFDAY_H
#define __ASM_VDSO_GETTIMEOFDAY_H
#ifndef __ASSEMBLY__
#include <asm/barrier.h>
#include <asm/cp15.h>
#include <asm/unistd.h>
#include <uapi/linux/time.h>
extern struct vdso_data *__get_datapage(void);
static __always_inline int gettimeofday_fallback(
struct __kernel_old_timeval *_tv,
struct timezone *_tz)
{
register struct timezone *tz asm("r1") = _tz;
register struct __kernel_old_timeval *tv asm("r0") = _tv;
register long ret asm ("r0");
register long nr asm("r7") = __NR_gettimeofday;
asm volatile(
" swi #0\n"
: "=r" (ret)
: "r" (tv), "r" (tz), "r" (nr)
: "memory");
return ret;
}
static __always_inline long clock_gettime_fallback(
clockid_t _clkid,
struct __kernel_timespec *_ts)
{
register struct __kernel_timespec *ts asm("r1") = _ts;
register clockid_t clkid asm("r0") = _clkid;
register long ret asm ("r0");
register long nr asm("r7") = __NR_clock_gettime64;
asm volatile(
" swi #0\n"
: "=r" (ret)
: "r" (clkid), "r" (ts), "r" (nr)
: "memory");
return ret;
}
static __always_inline u64 __arch_get_hw_counter(int clock_mode)
{
#ifdef CONFIG_ARM_ARCH_TIMER
u64 cycle_now;
isb();
cycle_now = read_sysreg(CNTVCT);
return cycle_now;
#else
return -EINVAL; /* use fallback */
#endif
}
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
{
return __get_datapage();
}
#endif /* !__ASSEMBLY__ */
#endif /* __ASM_VDSO_GETTIMEOFDAY_H */

71
arch/arm/include/asm/vdso/vsyscall.h Normal file
View File

@ -0,0 +1,71 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_VDSO_VSYSCALL_H
#define __ASM_VDSO_VSYSCALL_H
#ifndef __ASSEMBLY__
#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
#include <asm/cacheflush.h>
extern struct vdso_data *vdso_data;
extern bool cntvct_ok;
static __always_inline
bool tk_is_cntvct(const struct timekeeper *tk)
{
if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER))
return false;
if (!tk->tkr_mono.clock->archdata.vdso_direct)
return false;
return true;
}
/*
* Update the vDSO data page to keep in sync with kernel timekeeping.
*/
static __always_inline
struct vdso_data *__arm_get_k_vdso_data(void)
{
return vdso_data;
}
#define __arch_get_k_vdso_data __arm_get_k_vdso_data
static __always_inline
int __arm_update_vdso_data(void)
{
return !cntvct_ok;
}
#define __arch_update_vdso_data __arm_update_vdso_data
static __always_inline
int __arm_get_clock_mode(struct timekeeper *tk)
{
u32 __tk_is_cntvct = tk_is_cntvct(tk);
return __tk_is_cntvct;
}
#define __arch_get_clock_mode __arm_get_clock_mode
static __always_inline
int __arm_use_vsyscall(struct vdso_data *vdata)
{
return vdata[CS_HRES_COARSE].clock_mode;
}
#define __arch_use_vsyscall __arm_use_vsyscall
static __always_inline
void __arm_sync_vdso_data(struct vdso_data *vdata)
{
flush_dcache_page(virt_to_page(vdata));
}
#define __arch_sync_vdso_data __arm_sync_vdso_data
/* The asm-generic header needs to be included after the definitions above */
#include <asm-generic/vdso/vsyscall.h>
#endif /* !__ASSEMBLY__ */
#endif /* __ASM_VDSO_VSYSCALL_H */

29
arch/arm/include/asm/vdso_datapage.h
View File

@ -11,35 +11,12 @@
#ifndef __ASSEMBLY__
#include <vdso/datapage.h>
#include <asm/page.h>
/* Try to be cache-friendly on systems that don't implement the
* generic timer: fit the unconditionally updated fields in the first
* 32 bytes.
*/
struct vdso_data {
u32 seq_count; /* sequence count - odd during updates */
u16 tk_is_cntvct; /* fall back to syscall if false */
u16 cs_shift; /* clocksource shift */
u32 xtime_coarse_sec; /* coarse time */
u32 xtime_coarse_nsec;
u32 wtm_clock_sec; /* wall to monotonic offset */
u32 wtm_clock_nsec;
u32 xtime_clock_sec; /* CLOCK_REALTIME - seconds */
u32 cs_mult; /* clocksource multiplier */
u64 cs_cycle_last; /* last cycle value */
u64 cs_mask; /* clocksource mask */
u64 xtime_clock_snsec; /* CLOCK_REALTIME sub-ns base */
u32 tz_minuteswest; /* timezone info for gettimeofday(2) */
u32 tz_dsttime;
};
union vdso_data_store {
struct vdso_data data;
u8 page[PAGE_SIZE];
struct vdso_data data[CS_BASES];
u8 page[PAGE_SIZE];
};
#endif /* !__ASSEMBLY__ */

87
arch/arm/kernel/vdso.c
View File

@ -23,6 +23,8 @@
#include <asm/vdso.h>
#include <asm/vdso_datapage.h>
#include <clocksource/arm_arch_timer.h>
#include <vdso/helpers.h>
#include <vdso/vsyscall.h>
#define MAX_SYMNAME 64
@ -37,7 +39,7 @@ unsigned int vdso_total_pages __ro_after_init;
* The VDSO data page.
*/
static union vdso_data_store vdso_data_store __page_aligned_data;
static struct vdso_data *vdso_data = &vdso_data_store.data;
struct vdso_data *vdso_data = vdso_data_store.data;
static struct page *vdso_data_page __ro_after_init;
static const struct vm_special_mapping vdso_data_mapping = {
@ -77,7 +79,7 @@ struct elfinfo {
/* Cached result of boot-time check for whether the arch timer exists,
* and if so, whether the virtual counter is useable.
*/
static bool cntvct_ok __ro_after_init;
bool cntvct_ok __ro_after_init;
static bool __init cntvct_functional(void)
{
@ -262,84 +264,3 @@ void arm_install_vdso(struct mm_struct *mm, unsigned long addr)
mm->context.vdso = addr;
}
static void vdso_write_begin(struct vdso_data *vdata)
{
++vdso_data->seq_count;
smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */
}
static void vdso_write_end(struct vdso_data *vdata)
{
smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */
++vdso_data->seq_count;
}
static bool tk_is_cntvct(const struct timekeeper *tk)
{
if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER))
return false;
if (!tk->tkr_mono.clock->archdata.vdso_direct)
return false;
return true;
}
/**
* update_vsyscall - update the vdso data page
*
* Increment the sequence counter, making it odd, indicating to
* userspace that an update is in progress. Update the fields used
* for coarse clocks and, if the architected system timer is in use,
* the fields used for high precision clocks. Increment the sequence
* counter again, making it even, indicating to userspace that the
* update is finished.
*
* Userspace is expected to sample seq_count before reading any other
* fields from the data page. If seq_count is odd, userspace is
* expected to wait until it becomes even. After copying data from
* the page, userspace must sample seq_count again; if it has changed
* from its previous value, userspace must retry the whole sequence.
*
* Calls to update_vsyscall are serialized by the timekeeping core.
*/
void update_vsyscall(struct timekeeper *tk)
{
struct timespec64 *wtm = &tk->wall_to_monotonic;
if (!cntvct_ok) {
/* The entry points have been zeroed, so there is no
* point in updating the data page.
*/
return;
}
vdso_write_begin(vdso_data);
vdso_data->tk_is_cntvct = tk_is_cntvct(tk);
vdso_data->xtime_coarse_sec = tk->xtime_sec;
vdso_data->xtime_coarse_nsec = (u32)(tk->tkr_mono.xtime_nsec >>
tk->tkr_mono.shift);
vdso_data->wtm_clock_sec = wtm->tv_sec;
vdso_data->wtm_clock_nsec = wtm->tv_nsec;
if (vdso_data->tk_is_cntvct) {
vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
vdso_data->xtime_clock_sec = tk->xtime_sec;
vdso_data->xtime_clock_snsec = tk->tkr_mono.xtime_nsec;
vdso_data->cs_mult = tk->tkr_mono.mult;
vdso_data->cs_shift = tk->tkr_mono.shift;
vdso_data->cs_mask = tk->tkr_mono.mask;
}
vdso_write_end(vdso_data);
flush_dcache_page(virt_to_page(vdso_data));
}
void update_vsyscall_tz(void)
{
vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;
vdso_data->tz_dsttime = sys_tz.tz_dsttime;
flush_dcache_page(virt_to_page(vdso_data));
}

3
arch/arm/mm/Kconfig
View File

@ -896,7 +896,10 @@ config VDSO
bool "Enable VDSO for acceleration of some system calls"
depends on AEABI && MMU && CPU_V7
default y if ARM_ARCH_TIMER
select HAVE_GENERIC_VDSO
select GENERIC_TIME_VSYSCALL
select GENERIC_VDSO_32
select GENERIC_GETTIMEOFDAY
help
Place in the process address space an ELF shared object
providing fast implementations of gettimeofday and

18
arch/arm/vdso/Makefile
View File

@ -1,7 +1,13 @@
# SPDX-License-Identifier: GPL-2.0
# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before
# the inclusion of generic Makefile.
ARCH_REL_TYPE_ABS := R_ARM_JUMP_SLOT|R_ARM_GLOB_DAT|R_ARM_ABS32
include $(srctree)/lib/vdso/Makefile
hostprogs-y := vdsomunge
obj-vdso := vgettimeofday.o datapage.o
obj-vdso := vgettimeofday.o datapage.o note.o
# Build rules
targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.so.raw vdso.lds
@ -24,7 +30,11 @@ CFLAGS_REMOVE_vdso.o = -pg
# Force -O2 to avoid libgcc dependencies
CFLAGS_REMOVE_vgettimeofday.o = -pg -Os
ifeq ($(c-gettimeofday-y),)
CFLAGS_vgettimeofday.o = -O2
else
CFLAGS_vgettimeofday.o = -O2 -include $(c-gettimeofday-y)
endif
# Disable gcov profiling for VDSO code
GCOV_PROFILE := n
@ -37,7 +47,7 @@ $(obj)/vdso.o : $(obj)/vdso.so
# Link rule for the .so file
$(obj)/vdso.so.raw: $(obj)/vdso.lds $(obj-vdso) FORCE
$(call if_changed,ld)
$(call if_changed,vdsold_and_vdso_check)
$(obj)/vdso.so.dbg: $(obj)/vdso.so.raw $(obj)/vdsomunge FORCE
$(call if_changed,vdsomunge)
@ -47,6 +57,10 @@ $(obj)/%.so: OBJCOPYFLAGS := -S
$(obj)/%.so: $(obj)/%.so.dbg FORCE
$(call if_changed,objcopy)
# Actual build commands
quiet_cmd_vdsold_and_vdso_check = LD $@
cmd_vdsold_and_vdso_check = $(cmd_ld); $(cmd_vdso_check)
quiet_cmd_vdsomunge = MUNGE $@
cmd_vdsomunge = $(objtree)/$(obj)/vdsomunge $< $@

15
arch/arm/vdso/note.c Normal file
View File

@ -0,0 +1,15 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2012-2018 ARM Limited
*
* This supplies .note.* sections to go into the PT_NOTE inside the vDSO text.
* Here we can supply some information useful to userland.
*/
#include <linux/uts.h>
#include <linux/version.h>
#include <linux/elfnote.h>
#include <linux/build-salt.h>
ELFNOTE32("Linux", 0, LINUX_VERSION_CODE);
BUILD_SALT;

255
arch/arm/vdso/vgettimeofday.c
View File

@ -1,259 +1,22 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* ARM userspace implementations of gettimeofday() and similar.
*
* Copyright 2015 Mentor Graphics Corporation.
*/
#include <linux/compiler.h>
#include <linux/hrtimer.h>
#include <linux/time.h>
#include <asm/barrier.h>
#include <asm/bug.h>
#include <asm/cp15.h>
#include <asm/page.h>
#include <asm/unistd.h>
#include <asm/vdso_datapage.h>
#include <linux/types.h>
#ifndef CONFIG_AEABI
#error This code depends on AEABI system call conventions
#endif
extern struct vdso_data *__get_datapage(void);
static notrace u32 __vdso_read_begin(const struct vdso_data *vdata)
int __vdso_clock_gettime(clockid_t clock,
struct old_timespec32 *ts)
{
u32 seq;
repeat:
seq = READ_ONCE(vdata->seq_count);
if (seq & 1) {
cpu_relax();
goto repeat;
}
return seq;
return __cvdso_clock_gettime32(clock, ts);
}
static notrace u32 vdso_read_begin(const struct vdso_data *vdata)
int __vdso_gettimeofday(struct __kernel_old_timeval *tv,
struct timezone *tz)
{
u32 seq;
seq = __vdso_read_begin(vdata);
smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */
return seq;
}
static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)
{
smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */
return vdata->seq_count != start;
}
static notrace long clock_gettime_fallback(clockid_t _clkid,
struct timespec *_ts)
{
register struct timespec *ts asm("r1") = _ts;
register clockid_t clkid asm("r0") = _clkid;
register long ret asm ("r0");
register long nr asm("r7") = __NR_clock_gettime;
asm volatile(
" swi #0\n"
: "=r" (ret)
: "r" (clkid), "r" (ts), "r" (nr)
: "memory");
return ret;
}
static notrace int do_realtime_coarse(struct timespec *ts,
struct vdso_data *vdata)
{
u32 seq;
do {
seq = vdso_read_begin(vdata);
ts->tv_sec = vdata->xtime_coarse_sec;
ts->tv_nsec = vdata->xtime_coarse_nsec;
} while (vdso_read_retry(vdata, seq));
return 0;
}
static notrace int do_monotonic_coarse(struct timespec *ts,
struct vdso_data *vdata)
{
struct timespec tomono;
u32 seq;
do {
seq = vdso_read_begin(vdata);
ts->tv_sec = vdata->xtime_coarse_sec;
ts->tv_nsec = vdata->xtime_coarse_nsec;
tomono.tv_sec = vdata->wtm_clock_sec;
tomono.tv_nsec = vdata->wtm_clock_nsec;
} while (vdso_read_retry(vdata, seq));
ts->tv_sec += tomono.tv_sec;
timespec_add_ns(ts, tomono.tv_nsec);
return 0;
}
#ifdef CONFIG_ARM_ARCH_TIMER
static notrace u64 get_ns(struct vdso_data *vdata)
{
u64 cycle_delta;
u64 cycle_now;
u64 nsec;
isb();
cycle_now = read_sysreg(CNTVCT);
cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;
nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec;
nsec >>= vdata->cs_shift;
return nsec;
}
static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
{
u64 nsecs;
u32 seq;
do {
seq = vdso_read_begin(vdata);
if (!vdata->tk_is_cntvct)
return -1;
ts->tv_sec = vdata->xtime_clock_sec;
nsecs = get_ns(vdata);
} while (vdso_read_retry(vdata, seq));
ts->tv_nsec = 0;
timespec_add_ns(ts, nsecs);
return 0;
}
static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
{
struct timespec tomono;
u64 nsecs;
u32 seq;
do {
seq = vdso_read_begin(vdata);
if (!vdata->tk_is_cntvct)
return -1;
ts->tv_sec = vdata->xtime_clock_sec;
nsecs = get_ns(vdata);
tomono.tv_sec = vdata->wtm_clock_sec;
tomono.tv_nsec = vdata->wtm_clock_nsec;
} while (vdso_read_retry(vdata, seq));
ts->tv_sec += tomono.tv_sec;
ts->tv_nsec = 0;
timespec_add_ns(ts, nsecs + tomono.tv_nsec);
return 0;
}
#else /* CONFIG_ARM_ARCH_TIMER */
static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
{
return -1;
}
static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
{
return -1;
}
#endif /* CONFIG_ARM_ARCH_TIMER */
notrace int __vdso_clock_gettime(clockid_t clkid, struct timespec *ts)
{
struct vdso_data *vdata;
int ret = -1;
vdata = __get_datapage();
switch (clkid) {
case CLOCK_REALTIME_COARSE:
ret = do_realtime_coarse(ts, vdata);
break;
case CLOCK_MONOTONIC_COARSE:
ret = do_monotonic_coarse(ts, vdata);
break;
case CLOCK_REALTIME:
ret = do_realtime(ts, vdata);
break;
case CLOCK_MONOTONIC:
ret = do_monotonic(ts, vdata);
break;
default:
break;
}
if (ret)
ret = clock_gettime_fallback(clkid, ts);
return ret;
}
static notrace long gettimeofday_fallback(struct timeval *_tv,
struct timezone *_tz)
{
register struct timezone *tz asm("r1") = _tz;
register struct timeval *tv asm("r0") = _tv;
register long ret asm ("r0");
register long nr asm("r7") = __NR_gettimeofday;
asm volatile(
" swi #0\n"
: "=r" (ret)
: "r" (tv), "r" (tz), "r" (nr)
: "memory");
return ret;
}
notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
{
struct timespec ts;
struct vdso_data *vdata;
int ret;
vdata = __get_datapage();
ret = do_realtime(&ts, vdata);
if (ret)
return gettimeofday_fallback(tv, tz);
if (tv) {
tv->tv_sec = ts.tv_sec;
tv->tv_usec = ts.tv_nsec / 1000;
}
if (tz) {
tz->tz_minuteswest = vdata->tz_minuteswest;
tz->tz_dsttime = vdata->tz_dsttime;
}
return ret;
return __cvdso_gettimeofday(tv, tz);
}
/* Avoid unresolved references emitted by GCC */