leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

percpu: use block scan_hint to only scan forward

Browse Source 
Blocks now remember the latest scan_hint. This can be used on the
allocation path as when a contig_hint is broken, we can promote the
scan_hint to the contig_hint and scan forward from there. This works
because pcpu_block_refresh_hint() is only called on the allocation path
while block free regions are updated manually in
pcpu_block_update_hint_free().

Signed-off-by: Dennis Zhou <dennis@kernel.org>
This commit is contained in:
Dennis Zhou
2019-02-25 14:10:15 -08:00
parent b89462a9c5
commit da3afdd5bb
1 changed files with 17 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
mm/percpu.c
View File

@@ -761,14 +761,23 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index)
{ {
struct pcpu_block_md *block = chunk->md_blocks + index; struct pcpu_block_md *block = chunk->md_blocks + index;
unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index); unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index);
int rs, re; /* region start, region end */ int rs, re, start; /* region start, region end */
/* clear hints */ /* promote scan_hint to contig_hint */
block->contig_hint = block->scan_hint = 0; if (block->scan_hint) {
block->left_free = block->right_free = 0; start = block->scan_hint_start + block->scan_hint;
block->contig_hint_start = block->scan_hint_start;
block->contig_hint = block->scan_hint;
block->scan_hint = 0;
} else {
start = block->first_free;
block->contig_hint = 0;
}
block->right_free = 0;
/* iterate over free areas and update the contig hints */ /* iterate over free areas and update the contig hints */
pcpu_for_each_unpop_region(alloc_map, rs, re, block->first_free, pcpu_for_each_unpop_region(alloc_map, rs, re, start,
PCPU_BITMAP_BLOCK_BITS) { PCPU_BITMAP_BLOCK_BITS) {
pcpu_block_update(block, rs, re); pcpu_block_update(block, rs, re);
} }
@@ -833,6 +842,8 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
s_off, s_off,
s_off + bits)) { s_off + bits)) {
/* block contig hint is broken - scan to fix it */ /* block contig hint is broken - scan to fix it */
if (!s_off)
s_block->left_free = 0;
pcpu_block_refresh_hint(chunk, s_index); pcpu_block_refresh_hint(chunk, s_index);
} else { } else {
/* update left and right contig manually */ /* update left and right contig manually */
@@ -866,11 +877,11 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
if (e_off > e_block->scan_hint_start) if (e_off > e_block->scan_hint_start)
e_block->scan_hint = 0; e_block->scan_hint = 0;
e_block->left_free = 0;
if (e_off > e_block->contig_hint_start) { if (e_off > e_block->contig_hint_start) {
/* contig hint is broken - scan to fix it */ /* contig hint is broken - scan to fix it */
pcpu_block_refresh_hint(chunk, e_index); pcpu_block_refresh_hint(chunk, e_index);
} else { } else {
e_block->left_free = 0;
e_block->right_free = e_block->right_free =
min_t(int, e_block->right_free, min_t(int, e_block->right_free,
PCPU_BITMAP_BLOCK_BITS - e_off); PCPU_BITMAP_BLOCK_BITS - e_off);