mirror of
https://github.com/torvalds/linux.git
synced 2024-11-15 00:21:59 +00:00
7dc836187f
pid_list_fill_irq() runs via irq_work.
When CONFIG_PREEMPT_RT is disabled, it would run in irq_context.
so it shouldn't sleep while memory allocation.
Change gfp flags from GFP_KERNEL to GFP_NOWAIT to prevent sleep in
irq_work.
This change wouldn't impact functionality in practice because the worst-size
is 2K.
Cc: stable@goodmis.org
Fixes: 8d6e90983a
("tracing: Create a sparse bitmask for pid filtering")
Link: https://lore.kernel.org/20240704150226.1359936-1-yeoreum.yun@arm.com
Acked-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: levi.yun <yeoreum.yun@arm.com>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
497 lines
12 KiB
C
497 lines
12 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org>
|
|
*/
|
|
#include <linux/spinlock.h>
|
|
#include <linux/irq_work.h>
|
|
#include <linux/slab.h>
|
|
#include "trace.h"
|
|
|
|
/* See pid_list.h for details */
|
|
|
|
static inline union lower_chunk *get_lower_chunk(struct trace_pid_list *pid_list)
|
|
{
|
|
union lower_chunk *chunk;
|
|
|
|
lockdep_assert_held(&pid_list->lock);
|
|
|
|
if (!pid_list->lower_list)
|
|
return NULL;
|
|
|
|
chunk = pid_list->lower_list;
|
|
pid_list->lower_list = chunk->next;
|
|
pid_list->free_lower_chunks--;
|
|
WARN_ON_ONCE(pid_list->free_lower_chunks < 0);
|
|
chunk->next = NULL;
|
|
/*
|
|
* If a refill needs to happen, it can not happen here
|
|
* as the scheduler run queue locks are held.
|
|
*/
|
|
if (pid_list->free_lower_chunks <= CHUNK_REALLOC)
|
|
irq_work_queue(&pid_list->refill_irqwork);
|
|
|
|
return chunk;
|
|
}
|
|
|
|
static inline union upper_chunk *get_upper_chunk(struct trace_pid_list *pid_list)
|
|
{
|
|
union upper_chunk *chunk;
|
|
|
|
lockdep_assert_held(&pid_list->lock);
|
|
|
|
if (!pid_list->upper_list)
|
|
return NULL;
|
|
|
|
chunk = pid_list->upper_list;
|
|
pid_list->upper_list = chunk->next;
|
|
pid_list->free_upper_chunks--;
|
|
WARN_ON_ONCE(pid_list->free_upper_chunks < 0);
|
|
chunk->next = NULL;
|
|
/*
|
|
* If a refill needs to happen, it can not happen here
|
|
* as the scheduler run queue locks are held.
|
|
*/
|
|
if (pid_list->free_upper_chunks <= CHUNK_REALLOC)
|
|
irq_work_queue(&pid_list->refill_irqwork);
|
|
|
|
return chunk;
|
|
}
|
|
|
|
static inline void put_lower_chunk(struct trace_pid_list *pid_list,
|
|
union lower_chunk *chunk)
|
|
{
|
|
lockdep_assert_held(&pid_list->lock);
|
|
|
|
chunk->next = pid_list->lower_list;
|
|
pid_list->lower_list = chunk;
|
|
pid_list->free_lower_chunks++;
|
|
}
|
|
|
|
static inline void put_upper_chunk(struct trace_pid_list *pid_list,
|
|
union upper_chunk *chunk)
|
|
{
|
|
lockdep_assert_held(&pid_list->lock);
|
|
|
|
chunk->next = pid_list->upper_list;
|
|
pid_list->upper_list = chunk;
|
|
pid_list->free_upper_chunks++;
|
|
}
|
|
|
|
static inline bool upper_empty(union upper_chunk *chunk)
|
|
{
|
|
/*
|
|
* If chunk->data has no lower chunks, it will be the same
|
|
* as a zeroed bitmask. Use find_first_bit() to test it
|
|
* and if it doesn't find any bits set, then the array
|
|
* is empty.
|
|
*/
|
|
int bit = find_first_bit((unsigned long *)chunk->data,
|
|
sizeof(chunk->data) * 8);
|
|
return bit >= sizeof(chunk->data) * 8;
|
|
}
|
|
|
|
static inline int pid_split(unsigned int pid, unsigned int *upper1,
|
|
unsigned int *upper2, unsigned int *lower)
|
|
{
|
|
/* MAX_PID should cover all pids */
|
|
BUILD_BUG_ON(MAX_PID < PID_MAX_LIMIT);
|
|
|
|
/* In case a bad pid is passed in, then fail */
|
|
if (unlikely(pid >= MAX_PID))
|
|
return -1;
|
|
|
|
*upper1 = (pid >> UPPER1_SHIFT) & UPPER_MASK;
|
|
*upper2 = (pid >> UPPER2_SHIFT) & UPPER_MASK;
|
|
*lower = pid & LOWER_MASK;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static inline unsigned int pid_join(unsigned int upper1,
|
|
unsigned int upper2, unsigned int lower)
|
|
{
|
|
return ((upper1 & UPPER_MASK) << UPPER1_SHIFT) |
|
|
((upper2 & UPPER_MASK) << UPPER2_SHIFT) |
|
|
(lower & LOWER_MASK);
|
|
}
|
|
|
|
/**
|
|
* trace_pid_list_is_set - test if the pid is set in the list
|
|
* @pid_list: The pid list to test
|
|
* @pid: The pid to see if set in the list.
|
|
*
|
|
* Tests if @pid is set in the @pid_list. This is usually called
|
|
* from the scheduler when a task is scheduled. Its pid is checked
|
|
* if it should be traced or not.
|
|
*
|
|
* Return true if the pid is in the list, false otherwise.
|
|
*/
|
|
bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)
|
|
{
|
|
union upper_chunk *upper_chunk;
|
|
union lower_chunk *lower_chunk;
|
|
unsigned long flags;
|
|
unsigned int upper1;
|
|
unsigned int upper2;
|
|
unsigned int lower;
|
|
bool ret = false;
|
|
|
|
if (!pid_list)
|
|
return false;
|
|
|
|
if (pid_split(pid, &upper1, &upper2, &lower) < 0)
|
|
return false;
|
|
|
|
raw_spin_lock_irqsave(&pid_list->lock, flags);
|
|
upper_chunk = pid_list->upper[upper1];
|
|
if (upper_chunk) {
|
|
lower_chunk = upper_chunk->data[upper2];
|
|
if (lower_chunk)
|
|
ret = test_bit(lower, lower_chunk->data);
|
|
}
|
|
raw_spin_unlock_irqrestore(&pid_list->lock, flags);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* trace_pid_list_set - add a pid to the list
|
|
* @pid_list: The pid list to add the @pid to.
|
|
* @pid: The pid to add.
|
|
*
|
|
* Adds @pid to @pid_list. This is usually done explicitly by a user
|
|
* adding a task to be traced, or indirectly by the fork function
|
|
* when children should be traced and a task's pid is in the list.
|
|
*
|
|
* Return 0 on success, negative otherwise.
|
|
*/
|
|
int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)
|
|
{
|
|
union upper_chunk *upper_chunk;
|
|
union lower_chunk *lower_chunk;
|
|
unsigned long flags;
|
|
unsigned int upper1;
|
|
unsigned int upper2;
|
|
unsigned int lower;
|
|
int ret;
|
|
|
|
if (!pid_list)
|
|
return -ENODEV;
|
|
|
|
if (pid_split(pid, &upper1, &upper2, &lower) < 0)
|
|
return -EINVAL;
|
|
|
|
raw_spin_lock_irqsave(&pid_list->lock, flags);
|
|
upper_chunk = pid_list->upper[upper1];
|
|
if (!upper_chunk) {
|
|
upper_chunk = get_upper_chunk(pid_list);
|
|
if (!upper_chunk) {
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
pid_list->upper[upper1] = upper_chunk;
|
|
}
|
|
lower_chunk = upper_chunk->data[upper2];
|
|
if (!lower_chunk) {
|
|
lower_chunk = get_lower_chunk(pid_list);
|
|
if (!lower_chunk) {
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
upper_chunk->data[upper2] = lower_chunk;
|
|
}
|
|
set_bit(lower, lower_chunk->data);
|
|
ret = 0;
|
|
out:
|
|
raw_spin_unlock_irqrestore(&pid_list->lock, flags);
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* trace_pid_list_clear - remove a pid from the list
|
|
* @pid_list: The pid list to remove the @pid from.
|
|
* @pid: The pid to remove.
|
|
*
|
|
* Removes @pid from @pid_list. This is usually done explicitly by a user
|
|
* removing tasks from tracing, or indirectly by the exit function
|
|
* when a task that is set to be traced exits.
|
|
*
|
|
* Return 0 on success, negative otherwise.
|
|
*/
|
|
int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)
|
|
{
|
|
union upper_chunk *upper_chunk;
|
|
union lower_chunk *lower_chunk;
|
|
unsigned long flags;
|
|
unsigned int upper1;
|
|
unsigned int upper2;
|
|
unsigned int lower;
|
|
|
|
if (!pid_list)
|
|
return -ENODEV;
|
|
|
|
if (pid_split(pid, &upper1, &upper2, &lower) < 0)
|
|
return -EINVAL;
|
|
|
|
raw_spin_lock_irqsave(&pid_list->lock, flags);
|
|
upper_chunk = pid_list->upper[upper1];
|
|
if (!upper_chunk)
|
|
goto out;
|
|
|
|
lower_chunk = upper_chunk->data[upper2];
|
|
if (!lower_chunk)
|
|
goto out;
|
|
|
|
clear_bit(lower, lower_chunk->data);
|
|
|
|
/* if there's no more bits set, add it to the free list */
|
|
if (find_first_bit(lower_chunk->data, LOWER_MAX) >= LOWER_MAX) {
|
|
put_lower_chunk(pid_list, lower_chunk);
|
|
upper_chunk->data[upper2] = NULL;
|
|
if (upper_empty(upper_chunk)) {
|
|
put_upper_chunk(pid_list, upper_chunk);
|
|
pid_list->upper[upper1] = NULL;
|
|
}
|
|
}
|
|
out:
|
|
raw_spin_unlock_irqrestore(&pid_list->lock, flags);
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* trace_pid_list_next - return the next pid in the list
|
|
* @pid_list: The pid list to examine.
|
|
* @pid: The pid to start from
|
|
* @next: The pointer to place the pid that is set starting from @pid.
|
|
*
|
|
* Looks for the next consecutive pid that is in @pid_list starting
|
|
* at the pid specified by @pid. If one is set (including @pid), then
|
|
* that pid is placed into @next.
|
|
*
|
|
* Return 0 when a pid is found, -1 if there are no more pids included.
|
|
*/
|
|
int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
|
|
unsigned int *next)
|
|
{
|
|
union upper_chunk *upper_chunk;
|
|
union lower_chunk *lower_chunk;
|
|
unsigned long flags;
|
|
unsigned int upper1;
|
|
unsigned int upper2;
|
|
unsigned int lower;
|
|
|
|
if (!pid_list)
|
|
return -ENODEV;
|
|
|
|
if (pid_split(pid, &upper1, &upper2, &lower) < 0)
|
|
return -EINVAL;
|
|
|
|
raw_spin_lock_irqsave(&pid_list->lock, flags);
|
|
for (; upper1 <= UPPER_MASK; upper1++, upper2 = 0) {
|
|
upper_chunk = pid_list->upper[upper1];
|
|
|
|
if (!upper_chunk)
|
|
continue;
|
|
|
|
for (; upper2 <= UPPER_MASK; upper2++, lower = 0) {
|
|
lower_chunk = upper_chunk->data[upper2];
|
|
if (!lower_chunk)
|
|
continue;
|
|
|
|
lower = find_next_bit(lower_chunk->data, LOWER_MAX,
|
|
lower);
|
|
if (lower < LOWER_MAX)
|
|
goto found;
|
|
}
|
|
}
|
|
|
|
found:
|
|
raw_spin_unlock_irqrestore(&pid_list->lock, flags);
|
|
if (upper1 > UPPER_MASK)
|
|
return -1;
|
|
|
|
*next = pid_join(upper1, upper2, lower);
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* trace_pid_list_first - return the first pid in the list
|
|
* @pid_list: The pid list to examine.
|
|
* @pid: The pointer to place the pid first found pid that is set.
|
|
*
|
|
* Looks for the first pid that is set in @pid_list, and places it
|
|
* into @pid if found.
|
|
*
|
|
* Return 0 when a pid is found, -1 if there are no pids set.
|
|
*/
|
|
int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)
|
|
{
|
|
return trace_pid_list_next(pid_list, 0, pid);
|
|
}
|
|
|
|
static void pid_list_refill_irq(struct irq_work *iwork)
|
|
{
|
|
struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,
|
|
refill_irqwork);
|
|
union upper_chunk *upper = NULL;
|
|
union lower_chunk *lower = NULL;
|
|
union upper_chunk **upper_next = &upper;
|
|
union lower_chunk **lower_next = &lower;
|
|
int upper_count;
|
|
int lower_count;
|
|
int ucnt = 0;
|
|
int lcnt = 0;
|
|
|
|
again:
|
|
raw_spin_lock(&pid_list->lock);
|
|
upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;
|
|
lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;
|
|
raw_spin_unlock(&pid_list->lock);
|
|
|
|
if (upper_count <= 0 && lower_count <= 0)
|
|
return;
|
|
|
|
while (upper_count-- > 0) {
|
|
union upper_chunk *chunk;
|
|
|
|
chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
|
|
if (!chunk)
|
|
break;
|
|
*upper_next = chunk;
|
|
upper_next = &chunk->next;
|
|
ucnt++;
|
|
}
|
|
|
|
while (lower_count-- > 0) {
|
|
union lower_chunk *chunk;
|
|
|
|
chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
|
|
if (!chunk)
|
|
break;
|
|
*lower_next = chunk;
|
|
lower_next = &chunk->next;
|
|
lcnt++;
|
|
}
|
|
|
|
raw_spin_lock(&pid_list->lock);
|
|
if (upper) {
|
|
*upper_next = pid_list->upper_list;
|
|
pid_list->upper_list = upper;
|
|
pid_list->free_upper_chunks += ucnt;
|
|
}
|
|
if (lower) {
|
|
*lower_next = pid_list->lower_list;
|
|
pid_list->lower_list = lower;
|
|
pid_list->free_lower_chunks += lcnt;
|
|
}
|
|
raw_spin_unlock(&pid_list->lock);
|
|
|
|
/*
|
|
* On success of allocating all the chunks, both counters
|
|
* will be less than zero. If they are not, then an allocation
|
|
* failed, and we should not try again.
|
|
*/
|
|
if (upper_count >= 0 || lower_count >= 0)
|
|
return;
|
|
/*
|
|
* When the locks were released, free chunks could have
|
|
* been used and allocation needs to be done again. Might as
|
|
* well allocate it now.
|
|
*/
|
|
goto again;
|
|
}
|
|
|
|
/**
|
|
* trace_pid_list_alloc - create a new pid_list
|
|
*
|
|
* Allocates a new pid_list to store pids into.
|
|
*
|
|
* Returns the pid_list on success, NULL otherwise.
|
|
*/
|
|
struct trace_pid_list *trace_pid_list_alloc(void)
|
|
{
|
|
struct trace_pid_list *pid_list;
|
|
int i;
|
|
|
|
/* According to linux/thread.h, pids can be no bigger that 30 bits */
|
|
WARN_ON_ONCE(pid_max > (1 << 30));
|
|
|
|
pid_list = kzalloc(sizeof(*pid_list), GFP_KERNEL);
|
|
if (!pid_list)
|
|
return NULL;
|
|
|
|
init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);
|
|
|
|
raw_spin_lock_init(&pid_list->lock);
|
|
|
|
for (i = 0; i < CHUNK_ALLOC; i++) {
|
|
union upper_chunk *chunk;
|
|
|
|
chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
|
|
if (!chunk)
|
|
break;
|
|
chunk->next = pid_list->upper_list;
|
|
pid_list->upper_list = chunk;
|
|
pid_list->free_upper_chunks++;
|
|
}
|
|
|
|
for (i = 0; i < CHUNK_ALLOC; i++) {
|
|
union lower_chunk *chunk;
|
|
|
|
chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
|
|
if (!chunk)
|
|
break;
|
|
chunk->next = pid_list->lower_list;
|
|
pid_list->lower_list = chunk;
|
|
pid_list->free_lower_chunks++;
|
|
}
|
|
|
|
return pid_list;
|
|
}
|
|
|
|
/**
|
|
* trace_pid_list_free - Frees an allocated pid_list.
|
|
* @pid_list: The pid list to free.
|
|
*
|
|
* Frees the memory for a pid_list that was allocated.
|
|
*/
|
|
void trace_pid_list_free(struct trace_pid_list *pid_list)
|
|
{
|
|
union upper_chunk *upper;
|
|
union lower_chunk *lower;
|
|
int i, j;
|
|
|
|
if (!pid_list)
|
|
return;
|
|
|
|
irq_work_sync(&pid_list->refill_irqwork);
|
|
|
|
while (pid_list->lower_list) {
|
|
union lower_chunk *chunk;
|
|
|
|
chunk = pid_list->lower_list;
|
|
pid_list->lower_list = pid_list->lower_list->next;
|
|
kfree(chunk);
|
|
}
|
|
|
|
while (pid_list->upper_list) {
|
|
union upper_chunk *chunk;
|
|
|
|
chunk = pid_list->upper_list;
|
|
pid_list->upper_list = pid_list->upper_list->next;
|
|
kfree(chunk);
|
|
}
|
|
|
|
for (i = 0; i < UPPER1_SIZE; i++) {
|
|
upper = pid_list->upper[i];
|
|
if (upper) {
|
|
for (j = 0; j < UPPER2_SIZE; j++) {
|
|
lower = upper->data[j];
|
|
kfree(lower);
|
|
}
|
|
kfree(upper);
|
|
}
|
|
}
|
|
kfree(pid_list);
|
|
}
|