leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rcu: Check cond_resched_rcu_qs() state less often to reduce GP overhead

Browse Source 
Commit 4a81e8328d ("rcu: Reduce overhead of cond_resched() checks
for RCU") moved quiescent-state generation out of cond_resched()
and commit bde6c3aa99 ("rcu: Provide cond_resched_rcu_qs() to force
quiescent states in long loops") introduced cond_resched_rcu_qs(), and
commit 5cd37193ce ("rcu: Make cond_resched_rcu_qs() apply to normal RCU
flavors") introduced the per-CPU rcu_qs_ctr variable, which is frequently
polled by the RCU core state machine.

This frequent polling can increase grace-period rate, which in turn
increases grace-period overhead, which is visible in some benchmarks
(for example, the "open1" benchmark in Anton Blanchard's "will it scale"
suite).  This commit therefore reduces the rate at which rcu_qs_ctr
is polled by moving that polling into the force-quiescent-state (FQS)
machinery, and by further polling it only after the grace period has
been in effect for at least jiffies_till_sched_qs jiffies.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
This commit is contained in:
Paul E. McKenney 2016-11-30 11:21:21 -08:00
parent 02a5c550b2
commit 3a19b46a5c
2 changed files with 39 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/trace/events/rcu.h
View File

@ -385,11 +385,11 @@ TRACE_EVENT(rcu_quiescent_state_report,
/* /*
* Tracepoint for quiescent states detected by force_quiescent_state(). * Tracepoint for quiescent states detected by force_quiescent_state().
* These trace events include the type of RCU, the grace-period number * These trace events include the type of RCU, the grace-period number that
* that was blocked by the CPU, the CPU itself, and the type of quiescent * was blocked by the CPU, the CPU itself, and the type of quiescent state,
* state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline, * which can be "dti" for dyntick-idle mode, "ofl" for CPU offline, "kick"
* or "kick" when kicking a CPU that has been in dyntick-idle mode for * when kicking a CPU that has been in dyntick-idle mode for too long, or
* too long. * "rqc" if the CPU got a quiescent state via its rcu_qs_ctr.
*/ */
TRACE_EVENT(rcu_fqs, TRACE_EVENT(rcu_fqs,

46
kernel/rcu/tree.c
View File

@ -1232,7 +1232,10 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
static int rcu_implicit_dynticks_qs(struct rcu_data *rdp, static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
bool *isidle, unsigned long *maxj) bool *isidle, unsigned long *maxj)
{ {
unsigned long jtsq;
int *rcrmp; int *rcrmp;
unsigned long rjtsc;
struct rcu_node *rnp;
/* /*
* If the CPU passed through or entered a dynticks idle phase with * If the CPU passed through or entered a dynticks idle phase with
@ -1248,6 +1251,31 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
return 1; return 1;
} }
/* Compute and saturate jiffies_till_sched_qs. */
jtsq = jiffies_till_sched_qs;
rjtsc = rcu_jiffies_till_stall_check();
if (jtsq > rjtsc / 2) {
WRITE_ONCE(jiffies_till_sched_qs, rjtsc);
jtsq = rjtsc / 2;
} else if (jtsq < 1) {
WRITE_ONCE(jiffies_till_sched_qs, 1);
jtsq = 1;
}
/*
* Has this CPU encountered a cond_resched_rcu_qs() since the
* beginning of the grace period? For this to be the case,
* the CPU has to have noticed the current grace period. This
* might not be the case for nohz_full CPUs looping in the kernel.
*/
rnp = rdp->mynode;
if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_qs_ctr, rdp->cpu) &&
READ_ONCE(rdp->gpnum) == rnp->gpnum && !rdp->gpwrap) {
trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("rqc"));
return 1;
}
/* /*
* Check for the CPU being offline, but only if the grace period * Check for the CPU being offline, but only if the grace period
* is old enough. We don't need to worry about the CPU changing * is old enough. We don't need to worry about the CPU changing
@ -1290,9 +1318,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
* warning delay. * warning delay.
*/ */
rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu); rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu);
if (ULONG_CMP_GE(jiffies, if (time_after(jiffies, rdp->rsp->gp_start + jtsq) ||
rdp->rsp->gp_start + jiffies_till_sched_qs) || time_after(jiffies, rdp->rsp->jiffies_resched)) {
ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) { if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) {
WRITE_ONCE(rdp->cond_resched_completed, WRITE_ONCE(rdp->cond_resched_completed,
READ_ONCE(rdp->mynode->completed)); READ_ONCE(rdp->mynode->completed));
@ -2550,10 +2577,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
rnp = rdp->mynode; rnp = rdp->mynode;
raw_spin_lock_irqsave_rcu_node(rnp, flags); raw_spin_lock_irqsave_rcu_node(rnp, flags);
if ((rdp->cpu_no_qs.b.norm && if (rdp->cpu_no_qs.b.norm || rdp->gpnum != rnp->gpnum ||
rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) || rnp->completed == rnp->gpnum || rdp->gpwrap) {
rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
rdp->gpwrap) {
/* /*
* The grace period in which this quiescent state was * The grace period in which this quiescent state was
@ -2608,8 +2633,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
* Was there a quiescent state since the beginning of the grace * Was there a quiescent state since the beginning of the grace
* period? If no, then exit and wait for the next call. * period? If no, then exit and wait for the next call.
*/ */
if (rdp->cpu_no_qs.b.norm && if (rdp->cpu_no_qs.b.norm)
rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
return; return;
/* /*
@ -3563,9 +3587,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
rdp->core_needs_qs && rdp->cpu_no_qs.b.norm && rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) { rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
rdp->n_rp_core_needs_qs++; rdp->n_rp_core_needs_qs++;
} else if (rdp->core_needs_qs && } else if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm) {
(!rdp->cpu_no_qs.b.norm ||
rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
rdp->n_rp_report_qs++; rdp->n_rp_report_qs++;
return 1; return 1;
} }