leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

epoll: simplify and optimize busy loop logic

Browse Source 
ep_events_available() is called multiple times around the busy loop logic,
even though the logic is generally not used.  ep_reset_busy_poll_napi_id()
is similarly always called, even when busy loop is not used.

Eliminate ep_reset_busy_poll_napi_id() and inline it inside
ep_busy_loop().  Make ep_busy_loop() return whether there are any events
available after the busy loop.  This will eliminate unnecessary loads and
branches, and simplifies the loop.

Link: https://lkml.kernel.org/r/20201106231635.3528496-6-soheil.kdev@gmail.com
Signed-off-by: Soheil Hassas Yeganeh <soheil@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Khazhismel Kumykov <khazhy@google.com>
Cc: Guantao Liu <guantaol@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Soheil Hassas Yeganeh
2020-12-18 14:01:57 -08:00
committed by Linus Torvalds
parent e411596d48
commit 1493c47fb1
1 changed files with 17 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
fs/eventpoll.c
View File

@@ -389,19 +389,24 @@ static bool ep_busy_loop_end(void *p, unsigned long start_time)
* *
* we must do our busy polling with irqs enabled * we must do our busy polling with irqs enabled
*/ */
static void ep_busy_loop(struct eventpoll *ep, int nonblock) static bool ep_busy_loop(struct eventpoll *ep, int nonblock)
{ {
unsigned int napi_id = READ_ONCE(ep->napi_id); unsigned int napi_id = READ_ONCE(ep->napi_id);
if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on()) if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on()) {
napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep, false, napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep, false,
BUSY_POLL_BUDGET); BUSY_POLL_BUDGET);
} if (ep_events_available(ep))
return true;
static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep) /*
{ * Busy poll timed out. Drop NAPI ID for now, we can add
if (ep->napi_id) * it back in when we have moved a socket with a valid NAPI
* ID onto the ready list.
*/
ep->napi_id = 0; ep->napi_id = 0;
return false;
}
return false;
} }
/* /*
@@ -441,12 +446,9 @@ static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
#else #else
static inline void ep_busy_loop(struct eventpoll *ep, int nonblock) static inline bool ep_busy_loop(struct eventpoll *ep, int nonblock)
{
}
static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
{ {
return false;
} }
static inline void ep_set_busy_poll_napi_id(struct epitem *epi) static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
@@ -1772,21 +1774,13 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
} }
fetch_events: fetch_events:
if (!ep_events_available(ep))
ep_busy_loop(ep, timed_out);
eavail = ep_events_available(ep); eavail = ep_events_available(ep);
if (!eavail)
eavail = ep_busy_loop(ep, timed_out);
if (eavail) if (eavail)
goto send_events; goto send_events;
/*
* Busy poll timed out. Drop NAPI ID for now, we can add
* it back in when we have moved a socket with a valid NAPI
* ID onto the ready list.
*/
ep_reset_busy_poll_napi_id(ep);
do { do {
if (signal_pending(current)) if (signal_pending(current))
return -EINTR; return -EINTR;