leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

inode: convert inode_sb_list_lock to per-sb

Browse Source 
The process of reducing contention on per-superblock inode lists
starts with moving the locking to match the per-superblock inode
list. This takes the global lock out of the picture and reduces the
contention problems to within a single filesystem. This doesn't get
rid of contention as the locks still have global CPU scope, but it
does isolate operations on different superblocks form each other.

Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Josef Bacik <jbacik@fb.com>
Reviewed-by: Jan Kara <jack@suse.cz>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Tested-by: Dave Chinner <dchinner@redhat.com>
This commit is contained in:
Dave Chinner 2015-03-04 12:37:22 -05:00 committed by Josef Bacik
parent cbedaac634
commit 74278da9f7
10 changed files with 57 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
fs/block_dev.c
View File

@ -1769,7 +1769,7 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
{ {
struct inode *inode, *old_inode = NULL; struct inode *inode, *old_inode = NULL;
spin_lock(&inode_sb_list_lock); spin_lock(&blockdev_superblock->s_inode_list_lock);
list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
struct address_space *mapping = inode->i_mapping; struct address_space *mapping = inode->i_mapping;
@ -1781,13 +1781,13 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
} }
__iget(inode); __iget(inode);
spin_unlock(&inode->i_lock); spin_unlock(&inode->i_lock);
spin_unlock(&inode_sb_list_lock); spin_unlock(&blockdev_superblock->s_inode_list_lock);
/* /*
* We hold a reference to 'inode' so it couldn't have been * We hold a reference to 'inode' so it couldn't have been
* removed from s_inodes list while we dropped the * removed from s_inodes list while we dropped the
* inode_sb_list_lock. We cannot iput the inode now as we can * s_inode_list_lock We cannot iput the inode now as we can
* be holding the last reference and we cannot iput it under * be holding the last reference and we cannot iput it under
* inode_sb_list_lock. So we keep the reference and iput it * s_inode_list_lock. So we keep the reference and iput it
* later. * later.
*/ */
iput(old_inode); iput(old_inode);
@ -1795,8 +1795,8 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
func(I_BDEV(inode), arg); func(I_BDEV(inode), arg);
spin_lock(&inode_sb_list_lock); spin_lock(&blockdev_superblock->s_inode_list_lock);
} }
spin_unlock(&inode_sb_list_lock); spin_unlock(&blockdev_superblock->s_inode_list_lock);
iput(old_inode); iput(old_inode);
} }

10
fs/drop_caches.c
View File

@ -17,7 +17,7 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused)
{ {
struct inode *inode, *toput_inode = NULL; struct inode *inode, *toput_inode = NULL;
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
spin_lock(&inode->i_lock); spin_lock(&inode->i_lock);
if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) || if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
@ -27,13 +27,15 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused)
} }
__iget(inode); __iget(inode);
spin_unlock(&inode->i_lock); spin_unlock(&inode->i_lock);
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
invalidate_mapping_pages(inode->i_mapping, 0, -1); invalidate_mapping_pages(inode->i_mapping, 0, -1);
iput(toput_inode); iput(toput_inode);
toput_inode = inode; toput_inode = inode;
spin_lock(&inode_sb_list_lock);
spin_lock(&sb->s_inode_list_lock);
} }
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
iput(toput_inode); iput(toput_inode);
} }

12
fs/fs-writeback.c
View File

@ -2124,7 +2124,7 @@ static void wait_sb_inodes(struct super_block *sb)
*/ */
WARN_ON(!rwsem_is_locked(&sb->s_umount)); WARN_ON(!rwsem_is_locked(&sb->s_umount));
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
/* /*
* Data integrity sync. Must wait for all pages under writeback, * Data integrity sync. Must wait for all pages under writeback,
@ -2144,14 +2144,14 @@ static void wait_sb_inodes(struct super_block *sb)
} }
__iget(inode); __iget(inode);
spin_unlock(&inode->i_lock); spin_unlock(&inode->i_lock);
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
/* /*
* We hold a reference to 'inode' so it couldn't have been * We hold a reference to 'inode' so it couldn't have been
* removed from s_inodes list while we dropped the * removed from s_inodes list while we dropped the
* inode_sb_list_lock. We cannot iput the inode now as we can * s_inode_list_lock. We cannot iput the inode now as we can
* be holding the last reference and we cannot iput it under * be holding the last reference and we cannot iput it under
* inode_sb_list_lock. So we keep the reference and iput it * s_inode_list_lock. So we keep the reference and iput it
* later. * later.
*/ */
iput(old_inode); iput(old_inode);
@ -2161,9 +2161,9 @@ static void wait_sb_inodes(struct super_block *sb)
cond_resched(); cond_resched();
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
} }
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
iput(old_inode); iput(old_inode);
} }

28
fs/inode.c
View File

@ -28,8 +28,8 @@
* inode->i_state, inode->i_hash, __iget() * inode->i_state, inode->i_hash, __iget()
* Inode LRU list locks protect: * Inode LRU list locks protect:
* inode->i_sb->s_inode_lru, inode->i_lru * inode->i_sb->s_inode_lru, inode->i_lru
* inode_sb_list_lock protects: * inode->i_sb->s_inode_list_lock protects:
* sb->s_inodes, inode->i_sb_list * inode->i_sb->s_inodes, inode->i_sb_list
* bdi->wb.list_lock protects: * bdi->wb.list_lock protects:
* bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_wb_list * bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_wb_list
* inode_hash_lock protects: * inode_hash_lock protects:
@ -37,7 +37,7 @@
* *
* Lock ordering: * Lock ordering:
* *
* inode_sb_list_lock * inode->i_sb->s_inode_list_lock
* inode->i_lock * inode->i_lock
* Inode LRU list locks * Inode LRU list locks
* *
@ -45,7 +45,7 @@
* inode->i_lock * inode->i_lock
* *
* inode_hash_lock * inode_hash_lock
* inode_sb_list_lock * inode->i_sb->s_inode_list_lock
* inode->i_lock * inode->i_lock
* *
* iunique_lock * iunique_lock
@ -57,8 +57,6 @@ static unsigned int i_hash_shift __read_mostly;
static struct hlist_head *inode_hashtable __read_mostly; static struct hlist_head *inode_hashtable __read_mostly;
static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock); static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
__cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_sb_list_lock);
/* /*
* Empty aops. Can be used for the cases where the user does not * Empty aops. Can be used for the cases where the user does not
* define any of the address_space operations. * define any of the address_space operations.
@ -426,18 +424,18 @@ static void inode_lru_list_del(struct inode *inode)
*/ */
void inode_sb_list_add(struct inode *inode) void inode_sb_list_add(struct inode *inode)
{ {
spin_lock(&inode_sb_list_lock); spin_lock(&inode->i_sb->s_inode_list_lock);
list_add(&inode->i_sb_list, &inode->i_sb->s_inodes); list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
spin_unlock(&inode_sb_list_lock); spin_unlock(&inode->i_sb->s_inode_list_lock);
} }
EXPORT_SYMBOL_GPL(inode_sb_list_add); EXPORT_SYMBOL_GPL(inode_sb_list_add);
static inline void inode_sb_list_del(struct inode *inode) static inline void inode_sb_list_del(struct inode *inode)
{ {
if (!list_empty(&inode->i_sb_list)) { if (!list_empty(&inode->i_sb_list)) {
spin_lock(&inode_sb_list_lock); spin_lock(&inode->i_sb->s_inode_list_lock);
list_del_init(&inode->i_sb_list); list_del_init(&inode->i_sb_list);
spin_unlock(&inode_sb_list_lock); spin_unlock(&inode->i_sb->s_inode_list_lock);
} }
} }
@ -594,7 +592,7 @@ void evict_inodes(struct super_block *sb)
struct inode *inode, *next; struct inode *inode, *next;
LIST_HEAD(dispose); LIST_HEAD(dispose);
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) { list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
if (atomic_read(&inode->i_count)) if (atomic_read(&inode->i_count))
continue; continue;
@ -610,7 +608,7 @@ void evict_inodes(struct super_block *sb)
spin_unlock(&inode->i_lock); spin_unlock(&inode->i_lock);
list_add(&inode->i_lru, &dispose); list_add(&inode->i_lru, &dispose);
} }
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
dispose_list(&dispose); dispose_list(&dispose);
} }
@ -631,7 +629,7 @@ int invalidate_inodes(struct super_block *sb, bool kill_dirty)
struct inode *inode, *next; struct inode *inode, *next;
LIST_HEAD(dispose); LIST_HEAD(dispose);
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) { list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
spin_lock(&inode->i_lock); spin_lock(&inode->i_lock);
if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) { if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
@ -654,7 +652,7 @@ int invalidate_inodes(struct super_block *sb, bool kill_dirty)
spin_unlock(&inode->i_lock); spin_unlock(&inode->i_lock);
list_add(&inode->i_lru, &dispose); list_add(&inode->i_lru, &dispose);
} }
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
dispose_list(&dispose); dispose_list(&dispose);
@ -890,7 +888,7 @@ struct inode *new_inode(struct super_block *sb)
{ {
struct inode *inode; struct inode *inode;
spin_lock_prefetch(&inode_sb_list_lock); spin_lock_prefetch(&sb->s_inode_list_lock);
inode = new_inode_pseudo(sb); inode = new_inode_pseudo(sb);
if (inode) if (inode)

1
fs/internal.h
View File

@ -112,7 +112,6 @@ extern int vfs_open(const struct path *, struct file *, const struct cred *);
/* /*
* inode.c * inode.c
*/ */
extern spinlock_t inode_sb_list_lock;
extern long prune_icache_sb(struct super_block *sb, struct shrink_control *sc); extern long prune_icache_sb(struct super_block *sb, struct shrink_control *sc);
extern void inode_add_lru(struct inode *inode); extern void inode_add_lru(struct inode *inode);

20
fs/notify/inode_mark.c
View File

@ -163,17 +163,17 @@ int fsnotify_add_inode_mark(struct fsnotify_mark *mark,
/** /**
* fsnotify_unmount_inodes - an sb is unmounting. handle any watched inodes. * fsnotify_unmount_inodes - an sb is unmounting. handle any watched inodes.
* @list: list of inodes being unmounted (sb->s_inodes) * @sb: superblock being unmounted.
* *
* Called during unmount with no locks held, so needs to be safe against * Called during unmount with no locks held, so needs to be safe against
* concurrent modifiers. We temporarily drop inode_sb_list_lock and CAN block. * concurrent modifiers. We temporarily drop sb->s_inode_list_lock and CAN block.
*/ */
void fsnotify_unmount_inodes(struct list_head *list) void fsnotify_unmount_inodes(struct super_block *sb)
{ {
struct inode *inode, *next_i, *need_iput = NULL; struct inode *inode, *next_i, *need_iput = NULL;
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
list_for_each_entry_safe(inode, next_i, list, i_sb_list) { list_for_each_entry_safe(inode, next_i, &sb->s_inodes, i_sb_list) {
struct inode *need_iput_tmp; struct inode *need_iput_tmp;
/* /*
@ -209,7 +209,7 @@ void fsnotify_unmount_inodes(struct list_head *list)
spin_unlock(&inode->i_lock); spin_unlock(&inode->i_lock);
/* In case the dropping of a reference would nuke next_i. */ /* In case the dropping of a reference would nuke next_i. */
while (&next_i->i_sb_list != list) { while (&next_i->i_sb_list != &sb->s_inodes) {
spin_lock(&next_i->i_lock); spin_lock(&next_i->i_lock);
if (!(next_i->i_state & (I_FREEING | I_WILL_FREE)) && if (!(next_i->i_state & (I_FREEING | I_WILL_FREE)) &&
atomic_read(&next_i->i_count)) { atomic_read(&next_i->i_count)) {
@ -224,12 +224,12 @@ void fsnotify_unmount_inodes(struct list_head *list)
} }
/* /*
* We can safely drop inode_sb_list_lock here because either * We can safely drop s_inode_list_lock here because either
* we actually hold references on both inode and next_i or * we actually hold references on both inode and next_i or
* end of list. Also no new inodes will be added since the * end of list. Also no new inodes will be added since the
* umount has begun. * umount has begun.
*/ */
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
if (need_iput_tmp) if (need_iput_tmp)
iput(need_iput_tmp); iput(need_iput_tmp);
@ -241,7 +241,7 @@ void fsnotify_unmount_inodes(struct list_head *list)
iput(inode); iput(inode);
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
} }
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
} }

16
fs/quota/dquot.c
View File

@ -923,7 +923,7 @@ static void add_dquot_ref(struct super_block *sb, int type)
int reserved = 0; int reserved = 0;
#endif #endif
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
spin_lock(&inode->i_lock); spin_lock(&inode->i_lock);
if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) || if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
@ -934,7 +934,7 @@ static void add_dquot_ref(struct super_block *sb, int type)
} }
__iget(inode); __iget(inode);
spin_unlock(&inode->i_lock); spin_unlock(&inode->i_lock);
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
#ifdef CONFIG_QUOTA_DEBUG #ifdef CONFIG_QUOTA_DEBUG
if (unlikely(inode_get_rsv_space(inode) > 0)) if (unlikely(inode_get_rsv_space(inode) > 0))
@ -946,15 +946,15 @@ static void add_dquot_ref(struct super_block *sb, int type)
/* /*
* We hold a reference to 'inode' so it couldn't have been * We hold a reference to 'inode' so it couldn't have been
* removed from s_inodes list while we dropped the * removed from s_inodes list while we dropped the
* inode_sb_list_lock We cannot iput the inode now as we can be * s_inode_list_lock. We cannot iput the inode now as we can be
* holding the last reference and we cannot iput it under * holding the last reference and we cannot iput it under
* inode_sb_list_lock. So we keep the reference and iput it * s_inode_list_lock. So we keep the reference and iput it
* later. * later.
*/ */
old_inode = inode; old_inode = inode;
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
} }
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
iput(old_inode); iput(old_inode);
#ifdef CONFIG_QUOTA_DEBUG #ifdef CONFIG_QUOTA_DEBUG
@ -1023,7 +1023,7 @@ static void remove_dquot_ref(struct super_block *sb, int type,
struct inode *inode; struct inode *inode;
int reserved = 0; int reserved = 0;
spin_lock(&inode_sb_list_lock); spin_lock(&sb->s_inode_list_lock);
list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
/* /*
* We have to scan also I_NEW inodes because they can already * We have to scan also I_NEW inodes because they can already
@ -1039,7 +1039,7 @@ static void remove_dquot_ref(struct super_block *sb, int type,
} }
spin_unlock(&dq_data_lock); spin_unlock(&dq_data_lock);
} }
spin_unlock(&inode_sb_list_lock); spin_unlock(&sb->s_inode_list_lock);
#ifdef CONFIG_QUOTA_DEBUG #ifdef CONFIG_QUOTA_DEBUG
if (reserved) { if (reserved) {
printk(KERN_WARNING "VFS (%s): Writes happened after quota" printk(KERN_WARNING "VFS (%s): Writes happened after quota"

3
fs/super.c
View File

@ -191,6 +191,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags)
INIT_HLIST_NODE(&s->s_instances); INIT_HLIST_NODE(&s->s_instances);
INIT_HLIST_BL_HEAD(&s->s_anon); INIT_HLIST_BL_HEAD(&s->s_anon);
INIT_LIST_HEAD(&s->s_inodes); INIT_LIST_HEAD(&s->s_inodes);
spin_lock_init(&s->s_inode_list_lock);
if (list_lru_init_memcg(&s->s_dentry_lru)) if (list_lru_init_memcg(&s->s_dentry_lru))
goto fail; goto fail;
@ -399,7 +400,7 @@ void generic_shutdown_super(struct super_block *sb)
sync_filesystem(sb); sync_filesystem(sb);
sb->s_flags &= ~MS_ACTIVE; sb->s_flags &= ~MS_ACTIVE;
fsnotify_unmount_inodes(&sb->s_inodes); fsnotify_unmount_inodes(sb);
evict_inodes(sb); evict_inodes(sb);

5
include/linux/fs.h
View File

@ -1309,7 +1309,6 @@ struct super_block {
#endif #endif
const struct xattr_handler **s_xattr; const struct xattr_handler **s_xattr;
struct list_head s_inodes; /* all inodes */
struct hlist_bl_head s_anon; /* anonymous dentries for (nfs) exporting */ struct hlist_bl_head s_anon; /* anonymous dentries for (nfs) exporting */
struct list_head s_mounts; /* list of mounts; _not_ for fs use */ struct list_head s_mounts; /* list of mounts; _not_ for fs use */
struct block_device *s_bdev; struct block_device *s_bdev;
@ -1380,6 +1379,10 @@ struct super_block {
* Indicates how deep in a filesystem stack this SB is * Indicates how deep in a filesystem stack this SB is
*/ */
int s_stack_depth; int s_stack_depth;
/* s_inode_list_lock protects s_inodes */
spinlock_t s_inode_list_lock ____cacheline_aligned_in_smp;
struct list_head s_inodes; /* all inodes */
}; };
extern struct timespec current_fs_time(struct super_block *sb); extern struct timespec current_fs_time(struct super_block *sb);

4
include/linux/fsnotify_backend.h
View File

@ -357,7 +357,7 @@ extern void fsnotify_clear_marks_by_group_flags(struct fsnotify_group *group, un
extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group); extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group);
extern void fsnotify_get_mark(struct fsnotify_mark *mark); extern void fsnotify_get_mark(struct fsnotify_mark *mark);
extern void fsnotify_put_mark(struct fsnotify_mark *mark); extern void fsnotify_put_mark(struct fsnotify_mark *mark);
extern void fsnotify_unmount_inodes(struct list_head *list); extern void fsnotify_unmount_inodes(struct super_block *sb);
/* put here because inotify does some weird stuff when destroying watches */ /* put here because inotify does some weird stuff when destroying watches */
extern void fsnotify_init_event(struct fsnotify_event *event, extern void fsnotify_init_event(struct fsnotify_event *event,
@ -393,7 +393,7 @@ static inline u32 fsnotify_get_cookie(void)
return 0; return 0;
} }
static inline void fsnotify_unmount_inodes(struct list_head *list) static inline void fsnotify_unmount_inodes(struct super_block *sb)
{} {}
#endif /* CONFIG_FSNOTIFY */ #endif /* CONFIG_FSNOTIFY */