2007-05-06 21:49:50 +00:00
|
|
|
/*
|
|
|
|
|
* Quicklist support.
|
|
|
|
|
*
|
|
|
|
|
* Quicklists are light weight lists of pages that have a defined state
|
|
|
|
|
* on alloc and free. Pages must be in the quicklist specific defined state
|
|
|
|
|
* (zero by default) when the page is freed. It seems that the initial idea
|
|
|
|
|
* for such lists first came from Dave Miller and then various other people
|
|
|
|
|
* improved on it.
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2007 SGI,
|
|
|
|
|
* Christoph Lameter <clameter@sgi.com>
|
|
|
|
|
* Generalized, added support for multiple lists and
|
|
|
|
|
* constructors / destructors.
|
|
|
|
|
*/
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
|
|
|
|
|
|
#include <linux/mm.h>
|
|
|
|
|
#include <linux/mmzone.h>
|
|
|
|
|
#include <linux/module.h>
|
|
|
|
|
#include <linux/quicklist.h>
|
|
|
|
|
|
2009-06-24 06:13:45 +00:00
|
|
|
DEFINE_PER_CPU(struct quicklist [CONFIG_NR_QUICK], quicklist);
|
2007-05-06 21:49:50 +00:00
|
|
|
|
|
|
|
|
#define FRACTION_OF_NODE_MEM 16
|
|
|
|
|
|
|
|
|
|
static unsigned long max_pages(unsigned long min_pages)
|
|
|
|
|
{
|
|
|
|
|
unsigned long node_free_pages, max;
|
mm: size of quicklists shouldn't be proportional to the number of CPUs
Quicklists store pages for each CPU as caches. (Each CPU can cache
node_free_pages/16 pages)
It is used for page table cache. exit() will increase the cache size,
while fork() consumes it.
So for example if an apache-style application runs (one parent and many
child model), one CPU process will fork() while another CPU will process
the middleware work and exit().
At that time, the CPU on which the parent runs doesn't have page table
cache at all. Others (on which children runs) have maximum caches.
QList_max = (#ofCPUs - 1) x Free / 16
=> QList_max / (Free + QList_max) = (#ofCPUs - 1) / (16 + #ofCPUs - 1)
So, How much quicklist memory is used in the maximum case?
This is proposional to # of CPUs because the limit of per cpu quicklist
cache doesn't see the number of cpus.
Above calculation mean
Number of CPUs per node 2 4 8 16
============================== ====================
QList_max / (Free + QList_max) 5.8% 16% 30% 48%
Wow! Quicklist can spend about 50% memory at worst case.
My demonstration program is here
--------------------------------------------------------------------------------
#define _GNU_SOURCE
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sched.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/wait.h>
#define BUFFSIZE 512
int max_cpu(void) /* get max number of logical cpus from /proc/cpuinfo */
{
FILE *fd;
char *ret, buffer[BUFFSIZE];
int cpu = 1;
fd = fopen("/proc/cpuinfo", "r");
if (fd == NULL) {
perror("fopen(/proc/cpuinfo)");
exit(EXIT_FAILURE);
}
while (1) {
ret = fgets(buffer, BUFFSIZE, fd);
if (ret == NULL)
break;
if (!strncmp(buffer, "processor", 9))
cpu = atoi(strchr(buffer, ':') + 2);
}
fclose(fd);
return cpu;
}
void cpu_bind(int cpu) /* bind current process to one cpu */
{
cpu_set_t mask;
int ret;
CPU_ZERO(&mask);
CPU_SET(cpu, &mask);
ret = sched_setaffinity(0, sizeof(mask), &mask);
if (ret == -1) {
perror("sched_setaffinity()");
exit(EXIT_FAILURE);
}
sched_yield(); /* not necessary */
}
#define MMAP_SIZE (10 * 1024 * 1024) /* 10 MB */
#define FORK_INTERVAL 1 /* 1 second */
main(int argc, char *argv[])
{
int cpu_max, nextcpu;
long pagesize;
pid_t pid;
/* set max number of logical cpu */
if (argc > 1)
cpu_max = atoi(argv[1]) - 1;
else
cpu_max = max_cpu();
/* get the page size */
pagesize = sysconf(_SC_PAGESIZE);
if (pagesize == -1) {
perror("sysconf(_SC_PAGESIZE)");
exit(EXIT_FAILURE);
}
/* prepare parent process */
cpu_bind(0);
nextcpu = cpu_max;
loop:
/* select destination cpu for child process by round-robin rule */
if (++nextcpu > cpu_max)
nextcpu = 1;
pid = fork();
if (pid == 0) { /* child action */
char *p;
int i;
/* consume page tables */
p = mmap(0, MMAP_SIZE, PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
i = MMAP_SIZE / pagesize;
while (i-- > 0) {
*p = 1;
p += pagesize;
}
/* move to other cpu */
cpu_bind(nextcpu);
/*
printf("a child moved to cpu%d after mmap().\n", nextcpu);
fflush(stdout);
*/
/* back page tables to pgtable_quicklist */
exit(0);
} else if (pid > 0) { /* parent action */
sleep(FORK_INTERVAL);
waitpid(pid, NULL, WNOHANG);
}
goto loop;
}
----------------------------------------
When above program which does task migration runs, my 8GB box spends
800MB of memory for quicklist. This is not memory leak but doesn't seem
good.
% cat /proc/meminfo
MemTotal: 7701568 kB
MemFree: 4724672 kB
(snip)
Quicklists: 844800 kB
because
- My machine spec is
number of numa node: 2
number of cpus: 8 (4CPU x2 node)
total mem: 8GB (4GB x2 node)
free mem: about 5GB
- Then, 4.7GB x 16% ~= 880MB.
So, Quicklist can use 800MB.
So, if following spec machine run that program
CPUs: 64 (8cpu x 8node)
Mem: 1TB (128GB x8node)
Then, quicklist can waste 300GB (= 1TB x 30%). It is too large.
So, I don't like cache policies which is proportional to # of cpus.
My patch changes the number of caches
from:
per-cpu-cache-amount = memory_on_node / 16
to
per-cpu-cache-amount = memory_on_node / 16 / number_of_cpus_on_node.
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Keiichiro Tokunaga <tokunaga.keiich@jp.fujitsu.com>
Acked-by: Christoph Lameter <cl@linux-foundation.org>
Tested-by: David Miller <davem@davemloft.net>
Acked-by: Mike Travis <travis@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-09-02 21:35:58 +00:00
|
|
|
int node = numa_node_id();
|
|
|
|
|
struct zone *zones = NODE_DATA(node)->node_zones;
|
|
|
|
|
int num_cpus_on_node;
|
2009-03-13 04:19:46 +00:00
|
|
|
const struct cpumask *cpumask_on_node = cpumask_of_node(node);
|
2008-01-14 08:55:14 +00:00
|
|
|
|
|
|
|
|
node_free_pages =
|
|
|
|
|
#ifdef CONFIG_ZONE_DMA
|
|
|
|
|
zone_page_state(&zones[ZONE_DMA], NR_FREE_PAGES) +
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef CONFIG_ZONE_DMA32
|
|
|
|
|
zone_page_state(&zones[ZONE_DMA32], NR_FREE_PAGES) +
|
|
|
|
|
#endif
|
|
|
|
|
zone_page_state(&zones[ZONE_NORMAL], NR_FREE_PAGES);
|
2007-05-06 21:49:50 +00:00
|
|
|
|
|
|
|
|
max = node_free_pages / FRACTION_OF_NODE_MEM;
|
mm: size of quicklists shouldn't be proportional to the number of CPUs
Quicklists store pages for each CPU as caches. (Each CPU can cache
node_free_pages/16 pages)
It is used for page table cache. exit() will increase the cache size,
while fork() consumes it.
So for example if an apache-style application runs (one parent and many
child model), one CPU process will fork() while another CPU will process
the middleware work and exit().
At that time, the CPU on which the parent runs doesn't have page table
cache at all. Others (on which children runs) have maximum caches.
QList_max = (#ofCPUs - 1) x Free / 16
=> QList_max / (Free + QList_max) = (#ofCPUs - 1) / (16 + #ofCPUs - 1)
So, How much quicklist memory is used in the maximum case?
This is proposional to # of CPUs because the limit of per cpu quicklist
cache doesn't see the number of cpus.
Above calculation mean
Number of CPUs per node 2 4 8 16
============================== ====================
QList_max / (Free + QList_max) 5.8% 16% 30% 48%
Wow! Quicklist can spend about 50% memory at worst case.
My demonstration program is here
--------------------------------------------------------------------------------
#define _GNU_SOURCE
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sched.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/wait.h>
#define BUFFSIZE 512
int max_cpu(void) /* get max number of logical cpus from /proc/cpuinfo */
{
FILE *fd;
char *ret, buffer[BUFFSIZE];
int cpu = 1;
fd = fopen("/proc/cpuinfo", "r");
if (fd == NULL) {
perror("fopen(/proc/cpuinfo)");
exit(EXIT_FAILURE);
}
while (1) {
ret = fgets(buffer, BUFFSIZE, fd);
if (ret == NULL)
break;
if (!strncmp(buffer, "processor", 9))
cpu = atoi(strchr(buffer, ':') + 2);
}
fclose(fd);
return cpu;
}
void cpu_bind(int cpu) /* bind current process to one cpu */
{
cpu_set_t mask;
int ret;
CPU_ZERO(&mask);
CPU_SET(cpu, &mask);
ret = sched_setaffinity(0, sizeof(mask), &mask);
if (ret == -1) {
perror("sched_setaffinity()");
exit(EXIT_FAILURE);
}
sched_yield(); /* not necessary */
}
#define MMAP_SIZE (10 * 1024 * 1024) /* 10 MB */
#define FORK_INTERVAL 1 /* 1 second */
main(int argc, char *argv[])
{
int cpu_max, nextcpu;
long pagesize;
pid_t pid;
/* set max number of logical cpu */
if (argc > 1)
cpu_max = atoi(argv[1]) - 1;
else
cpu_max = max_cpu();
/* get the page size */
pagesize = sysconf(_SC_PAGESIZE);
if (pagesize == -1) {
perror("sysconf(_SC_PAGESIZE)");
exit(EXIT_FAILURE);
}
/* prepare parent process */
cpu_bind(0);
nextcpu = cpu_max;
loop:
/* select destination cpu for child process by round-robin rule */
if (++nextcpu > cpu_max)
nextcpu = 1;
pid = fork();
if (pid == 0) { /* child action */
char *p;
int i;
/* consume page tables */
p = mmap(0, MMAP_SIZE, PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
i = MMAP_SIZE / pagesize;
while (i-- > 0) {
*p = 1;
p += pagesize;
}
/* move to other cpu */
cpu_bind(nextcpu);
/*
printf("a child moved to cpu%d after mmap().\n", nextcpu);
fflush(stdout);
*/
/* back page tables to pgtable_quicklist */
exit(0);
} else if (pid > 0) { /* parent action */
sleep(FORK_INTERVAL);
waitpid(pid, NULL, WNOHANG);
}
goto loop;
}
----------------------------------------
When above program which does task migration runs, my 8GB box spends
800MB of memory for quicklist. This is not memory leak but doesn't seem
good.
% cat /proc/meminfo
MemTotal: 7701568 kB
MemFree: 4724672 kB
(snip)
Quicklists: 844800 kB
because
- My machine spec is
number of numa node: 2
number of cpus: 8 (4CPU x2 node)
total mem: 8GB (4GB x2 node)
free mem: about 5GB
- Then, 4.7GB x 16% ~= 880MB.
So, Quicklist can use 800MB.
So, if following spec machine run that program
CPUs: 64 (8cpu x 8node)
Mem: 1TB (128GB x8node)
Then, quicklist can waste 300GB (= 1TB x 30%). It is too large.
So, I don't like cache policies which is proportional to # of cpus.
My patch changes the number of caches
from:
per-cpu-cache-amount = memory_on_node / 16
to
per-cpu-cache-amount = memory_on_node / 16 / number_of_cpus_on_node.
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Keiichiro Tokunaga <tokunaga.keiich@jp.fujitsu.com>
Acked-by: Christoph Lameter <cl@linux-foundation.org>
Tested-by: David Miller <davem@davemloft.net>
Acked-by: Mike Travis <travis@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-09-02 21:35:58 +00:00
|
|
|
|
|
|
|
|
num_cpus_on_node = cpus_weight_nr(*cpumask_on_node);
|
|
|
|
|
max /= num_cpus_on_node;
|
|
|
|
|
|
2007-05-06 21:49:50 +00:00
|
|
|
return max(max, min_pages);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static long min_pages_to_free(struct quicklist *q,
|
|
|
|
|
unsigned long min_pages, long max_free)
|
|
|
|
|
{
|
|
|
|
|
long pages_to_free;
|
|
|
|
|
|
|
|
|
|
pages_to_free = q->nr_pages - max_pages(min_pages);
|
|
|
|
|
|
|
|
|
|
return min(pages_to_free, max_free);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Trim down the number of pages in the quicklist
|
|
|
|
|
*/
|
|
|
|
|
void quicklist_trim(int nr, void (*dtor)(void *),
|
|
|
|
|
unsigned long min_pages, unsigned long max_free)
|
|
|
|
|
{
|
|
|
|
|
long pages_to_free;
|
|
|
|
|
struct quicklist *q;
|
|
|
|
|
|
|
|
|
|
q = &get_cpu_var(quicklist)[nr];
|
|
|
|
|
if (q->nr_pages > min_pages) {
|
|
|
|
|
pages_to_free = min_pages_to_free(q, min_pages, max_free);
|
|
|
|
|
|
|
|
|
|
while (pages_to_free > 0) {
|
|
|
|
|
/*
|
|
|
|
|
* We pass a gfp_t of 0 to quicklist_alloc here
|
|
|
|
|
* because we will never call into the page allocator.
|
|
|
|
|
*/
|
|
|
|
|
void *p = quicklist_alloc(nr, 0, NULL);
|
|
|
|
|
|
|
|
|
|
if (dtor)
|
|
|
|
|
dtor(p);
|
|
|
|
|
free_page((unsigned long)p);
|
|
|
|
|
pages_to_free--;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
put_cpu_var(quicklist);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unsigned long quicklist_total_size(void)
|
|
|
|
|
{
|
|
|
|
|
unsigned long count = 0;
|
|
|
|
|
int cpu;
|
|
|
|
|
struct quicklist *ql, *q;
|
|
|
|
|
|
|
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
|
ql = per_cpu(quicklist, cpu);
|
|
|
|
|
for (q = ql; q < ql + CONFIG_NR_QUICK; q++)
|
|
|
|
|
count += q->nr_pages;
|
|
|
|
|
}
|
|
|
|
|
return count;
|
|
|
|
|
}
|
|
|
|
|
|
