leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mm: memcontrol: eliminate raw access to stat and event counters

Browse Source 
Replace all raw 'this_cpu_' modifications of the stat and event per-cpu
counters with API functions such as mod_memcg_state().

This makes the code easier to read, but is also in preparation for the
next patch, which changes the per-cpu implementation of those counters.

Link: http://lkml.kernel.org/r/20171103153336.24044-1-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: Michal Hocko <mhocko@suse.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Johannes Weiner 2018-01-31 16:16:37 -08:00 committed by Linus Torvalds
parent 2b9fceb3b4
commit c9019e9bf4
2 changed files with 45 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
include/linux/memcontrol.h
View File

@ -272,13 +272,6 @@ static inline bool mem_cgroup_disabled(void)
return !cgroup_subsys_enabled(memory_cgrp_subsys); return !cgroup_subsys_enabled(memory_cgrp_subsys);
} }
static inline void mem_cgroup_event(struct mem_cgroup *memcg,
enum memcg_event_item event)
{
this_cpu_inc(memcg->stat->events[event]);
cgroup_file_notify(&memcg->events_file);
}
bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg); bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
@ -627,15 +620,23 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
gfp_t gfp_mask, gfp_t gfp_mask,
unsigned long *total_scanned); unsigned long *total_scanned);
/* idx can be of type enum memcg_event_item or vm_event_item */
static inline void __count_memcg_events(struct mem_cgroup *memcg,
int idx, unsigned long count)
{
if (!mem_cgroup_disabled())
__this_cpu_add(memcg->stat->events[idx], count);
}
/* idx can be of type enum memcg_event_item or vm_event_item */
static inline void count_memcg_events(struct mem_cgroup *memcg, static inline void count_memcg_events(struct mem_cgroup *memcg,
enum vm_event_item idx, int idx, unsigned long count)
unsigned long count)
{ {
if (!mem_cgroup_disabled()) if (!mem_cgroup_disabled())
this_cpu_add(memcg->stat->events[idx], count); this_cpu_add(memcg->stat->events[idx], count);
} }
/* idx can be of type enum memcg_stat_item or node_stat_item */ /* idx can be of type enum memcg_event_item or vm_event_item */
static inline void count_memcg_page_event(struct page *page, static inline void count_memcg_page_event(struct page *page,
int idx) int idx)
{ {
@ -654,12 +655,20 @@ static inline void count_memcg_event_mm(struct mm_struct *mm,
rcu_read_lock(); rcu_read_lock();
memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
if (likely(memcg)) { if (likely(memcg)) {
this_cpu_inc(memcg->stat->events[idx]); count_memcg_events(memcg, idx, 1);
if (idx == OOM_KILL) if (idx == OOM_KILL)
cgroup_file_notify(&memcg->events_file); cgroup_file_notify(&memcg->events_file);
} }
rcu_read_unlock(); rcu_read_unlock();
} }
static inline void mem_cgroup_event(struct mem_cgroup *memcg,
enum memcg_event_item event)
{
count_memcg_events(memcg, event, 1);
cgroup_file_notify(&memcg->events_file);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
void mem_cgroup_split_huge_fixup(struct page *head); void mem_cgroup_split_huge_fixup(struct page *head);
#endif #endif

59
mm/memcontrol.c
View File

@ -586,23 +586,23 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
* counted as CACHE even if it's on ANON LRU. * counted as CACHE even if it's on ANON LRU.
*/ */
if (PageAnon(page)) if (PageAnon(page))
__this_cpu_add(memcg->stat->count[MEMCG_RSS], nr_pages); __mod_memcg_state(memcg, MEMCG_RSS, nr_pages);
else { else {
__this_cpu_add(memcg->stat->count[MEMCG_CACHE], nr_pages); __mod_memcg_state(memcg, MEMCG_CACHE, nr_pages);
if (PageSwapBacked(page)) if (PageSwapBacked(page))
__this_cpu_add(memcg->stat->count[NR_SHMEM], nr_pages); __mod_memcg_state(memcg, NR_SHMEM, nr_pages);
} }
if (compound) { if (compound) {
VM_BUG_ON_PAGE(!PageTransHuge(page), page); VM_BUG_ON_PAGE(!PageTransHuge(page), page);
__this_cpu_add(memcg->stat->count[MEMCG_RSS_HUGE], nr_pages); __mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages);
} }
/* pagein of a big page is an event. So, ignore page size */ /* pagein of a big page is an event. So, ignore page size */
if (nr_pages > 0) if (nr_pages > 0)
__this_cpu_inc(memcg->stat->events[PGPGIN]); __count_memcg_events(memcg, PGPGIN, 1);
else { else {
__this_cpu_inc(memcg->stat->events[PGPGOUT]); __count_memcg_events(memcg, PGPGOUT, 1);
nr_pages = -nr_pages; /* for event */ nr_pages = -nr_pages; /* for event */
} }
@ -2415,18 +2415,11 @@ void mem_cgroup_split_huge_fixup(struct page *head)
for (i = 1; i < HPAGE_PMD_NR; i++) for (i = 1; i < HPAGE_PMD_NR; i++)
head[i].mem_cgroup = head->mem_cgroup; head[i].mem_cgroup = head->mem_cgroup;
__this_cpu_sub(head->mem_cgroup->stat->count[MEMCG_RSS_HUGE], __mod_memcg_state(head->mem_cgroup, MEMCG_RSS_HUGE, -HPAGE_PMD_NR);
HPAGE_PMD_NR);
} }
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifdef CONFIG_MEMCG_SWAP #ifdef CONFIG_MEMCG_SWAP
static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
int nr_entries)
{
this_cpu_add(memcg->stat->count[MEMCG_SWAP], nr_entries);
}
/** /**
* mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
* @entry: swap entry to be moved * @entry: swap entry to be moved
@ -2450,8 +2443,8 @@ static int mem_cgroup_move_swap_account(swp_entry_t entry,
new_id = mem_cgroup_id(to); new_id = mem_cgroup_id(to);
if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) { if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
mem_cgroup_swap_statistics(from, -1); mod_memcg_state(from, MEMCG_SWAP, -1);
mem_cgroup_swap_statistics(to, 1); mod_memcg_state(to, MEMCG_SWAP, 1);
return 0; return 0;
} }
return -EINVAL; return -EINVAL;
@ -4584,8 +4577,8 @@ static int mem_cgroup_move_account(struct page *page,
spin_lock_irqsave(&from->move_lock, flags); spin_lock_irqsave(&from->move_lock, flags);
if (!anon && page_mapped(page)) { if (!anon && page_mapped(page)) {
__this_cpu_sub(from->stat->count[NR_FILE_MAPPED], nr_pages); __mod_memcg_state(from, NR_FILE_MAPPED, -nr_pages);
__this_cpu_add(to->stat->count[NR_FILE_MAPPED], nr_pages); __mod_memcg_state(to, NR_FILE_MAPPED, nr_pages);
} }
/* /*
@ -4597,16 +4590,14 @@ static int mem_cgroup_move_account(struct page *page,
struct address_space *mapping = page_mapping(page); struct address_space *mapping = page_mapping(page);
if (mapping_cap_account_dirty(mapping)) { if (mapping_cap_account_dirty(mapping)) {
__this_cpu_sub(from->stat->count[NR_FILE_DIRTY], __mod_memcg_state(from, NR_FILE_DIRTY, -nr_pages);
nr_pages); __mod_memcg_state(to, NR_FILE_DIRTY, nr_pages);
__this_cpu_add(to->stat->count[NR_FILE_DIRTY],
nr_pages);
} }
} }
if (PageWriteback(page)) { if (PageWriteback(page)) {
__this_cpu_sub(from->stat->count[NR_WRITEBACK], nr_pages); __mod_memcg_state(from, NR_WRITEBACK, -nr_pages);
__this_cpu_add(to->stat->count[NR_WRITEBACK], nr_pages); __mod_memcg_state(to, NR_WRITEBACK, nr_pages);
} }
/* /*
@ -5642,11 +5633,11 @@ static void uncharge_batch(const struct uncharge_gather *ug)
} }
local_irq_save(flags); local_irq_save(flags);
__this_cpu_sub(ug->memcg->stat->count[MEMCG_RSS], ug->nr_anon); __mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon);
__this_cpu_sub(ug->memcg->stat->count[MEMCG_CACHE], ug->nr_file); __mod_memcg_state(ug->memcg, MEMCG_CACHE, -ug->nr_file);
__this_cpu_sub(ug->memcg->stat->count[MEMCG_RSS_HUGE], ug->nr_huge); __mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);
__this_cpu_sub(ug->memcg->stat->count[NR_SHMEM], ug->nr_shmem); __mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem);
__this_cpu_add(ug->memcg->stat->events[PGPGOUT], ug->pgpgout); __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
__this_cpu_add(ug->memcg->stat->nr_page_events, nr_pages); __this_cpu_add(ug->memcg->stat->nr_page_events, nr_pages);
memcg_check_events(ug->memcg, ug->dummy_page); memcg_check_events(ug->memcg, ug->dummy_page);
local_irq_restore(flags); local_irq_restore(flags);
@ -5874,7 +5865,7 @@ bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
if (in_softirq()) if (in_softirq())
gfp_mask = GFP_NOWAIT; gfp_mask = GFP_NOWAIT;
this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages); mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
if (try_charge(memcg, gfp_mask, nr_pages) == 0) if (try_charge(memcg, gfp_mask, nr_pages) == 0)
return true; return true;
@ -5895,7 +5886,7 @@ void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
return; return;
} }
this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages); mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
refill_stock(memcg, nr_pages); refill_stock(memcg, nr_pages);
} }
@ -6019,7 +6010,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg), oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
nr_entries); nr_entries);
VM_BUG_ON_PAGE(oldid, page); VM_BUG_ON_PAGE(oldid, page);
mem_cgroup_swap_statistics(swap_memcg, nr_entries); mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
page->mem_cgroup = NULL; page->mem_cgroup = NULL;
@ -6085,7 +6076,7 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
mem_cgroup_id_get_many(memcg, nr_pages - 1); mem_cgroup_id_get_many(memcg, nr_pages - 1);
oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages); oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
VM_BUG_ON_PAGE(oldid, page); VM_BUG_ON_PAGE(oldid, page);
mem_cgroup_swap_statistics(memcg, nr_pages); mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
return 0; return 0;
} }
@ -6113,7 +6104,7 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
else else
page_counter_uncharge(&memcg->memsw, nr_pages); page_counter_uncharge(&memcg->memsw, nr_pages);
} }
mem_cgroup_swap_statistics(memcg, -nr_pages); mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
mem_cgroup_id_put_many(memcg, nr_pages); mem_cgroup_id_put_many(memcg, nr_pages);
} }
rcu_read_unlock(); rcu_read_unlock();