[PATCH 6/7] writeback: add lazy bdi->task creation

From: Jens Axboe
Date: Thu Mar 12 2009 - 10:35:41 EST

Instead of creating the bdi flusher threads when the bdi is registered,
defer that to the point where we have dirty IO pending and someone
attempts to start the flushing.

A bdi is put on the normal bdi_list when it is registered. When someone
attempts to schedule writeback on this bdi, we move it to a pending list
and wake up the default bdi forker thread to take care of setting up a
task and putting the bdi back on the normal bdi_list. If task creation
should fail, the forker thread will writeout some data on behalf of the
pending bdi. This should ensure progress always.

Signed-off-by: Jens Axboe <jens.axboe@xxxxxxxxxx>
---
fs/fs-writeback.c | 42 +++++-----
include/linux/backing-dev.h | 8 ++-
mm/backing-dev.c | 196 +++++++++++++++++++++++++++++++++++++++----
3 files changed, 206 insertions(+), 40 deletions(-)

diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 37b042f..c25c261 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -74,14 +74,17 @@ static void writeback_release(struct backing_dev_info *bdi)
clear_bit(BDI_pdflush, &bdi->state);
}

-void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
+int bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
long nr_pages)
{
/*
- * Should not happen, complain?
+ * This only happens the first time someone kicks this bdi, so put
+ * it out-of-line.
*/
- if (unlikely(!bdi->task))
- return;
+ if (unlikely(!bdi->task)) {
+ bdi_add_flusher_task(bdi);
+ return 1;
+ }

if (writeback_acquire(bdi)) {
bdi->wb_arg.nr_pages = nr_pages;
@@ -92,6 +95,8 @@ void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
smp_mb();
wake_up(&bdi->wait);
}
+
+ return 0;
}

/*
@@ -185,24 +190,13 @@ static void bdi_pdflush(struct backing_dev_info *bdi)
* Handle writeback of dirty data for the device backed by this bdi. Also
* wakes up periodically and does kupdated style flushing.
*/
-int bdi_writeback_task(void *ptr)
+int bdi_writeback_task(struct backing_dev_info *bdi)
{
- struct backing_dev_info *bdi = ptr;
- struct task_struct *tsk = current;
-
- tsk->flags |= PF_FLUSHER | PF_SWAPWRITE;
- set_freezable();
-
- /*
- * Our parent may run at a different priority, just set us to normal
- */
- set_user_nice(tsk, 0);
-
while (!kthread_should_stop()) {
- DECLARE_WAITQUEUE(wait, tsk);
+ DECLARE_WAITQUEUE(wait, current);

add_wait_queue(&bdi->wait, &wait);
- set_task_state(tsk, TASK_INTERRUPTIBLE);
+ set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(dirty_writeback_interval);
try_to_freeze();

@@ -226,7 +220,7 @@ int bdi_writeback_task(void *ptr)
bdi_pdflush(bdi);

writeback_release(bdi);
- set_task_state(tsk, TASK_RUNNING);
+ set_current_state(TASK_RUNNING);
finish_wait(&bdi->wait, &wait);
}

@@ -239,9 +233,13 @@ void bdi_writeback_all(struct super_block *sb, long nr_pages)

rcu_read_lock();

- list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
- if (bdi_has_dirty_io(bdi))
- bdi_start_writeback(bdi, sb, nr_pages);
+restart:
+ list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
+ if (!bdi_has_dirty_io(bdi))
+ continue;
+ if (bdi_start_writeback(bdi, sb, nr_pages))
+ goto restart;
+ }

rcu_read_unlock();
}
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index 3c94fbd..b9e2085 100644
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -24,6 +24,7 @@ struct dentry;
*/
enum bdi_state {
BDI_pdflush, /* A pdflush thread is working this device */
+ BDI_pending, /* On its way to being activated */
BDI_write_congested, /* The write queue is getting full */
BDI_read_congested, /* The read queue is getting full */
BDI_unused, /* Available bits start here */
@@ -46,6 +47,7 @@ struct bdi_writeback_arg {

struct backing_dev_info {
struct list_head bdi_list;
+ struct rcu_head rcu_head;

unsigned long ra_pages; /* max readahead in PAGE_CACHE_SIZE units */
unsigned long state; /* Always use atomic bitops on this */
@@ -85,10 +87,11 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
const char *fmt, ...);
int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev);
void bdi_unregister(struct backing_dev_info *bdi);
-void bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
+int bdi_start_writeback(struct backing_dev_info *bdi, struct super_block *sb,
long nr_pages);
-int bdi_writeback_task(void *);
+int bdi_writeback_task(struct backing_dev_info *bdi);
void bdi_writeback_all(struct super_block *sb, long nr_pages);
+void bdi_add_flusher_task(struct backing_dev_info *bdi);

extern spinlock_t bdi_lock;
extern struct list_head bdi_list;
@@ -215,6 +218,7 @@ int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);
#define BDI_CAP_EXEC_MAP 0x00000040
#define BDI_CAP_NO_ACCT_WB 0x00000080
#define BDI_CAP_SWAP_BACKED 0x00000100
+#define BDI_CAP_FLUSH_FORKER 0x00000200

#define BDI_CAP_VMFLAGS \
(BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP | BDI_CAP_EXEC_MAP)
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index 0096b96..500d1fc 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -2,6 +2,7 @@
#include <linux/wait.h>
#include <linux/backing-dev.h>
#include <linux/kthread.h>
+#include <linux/freezer.h>
#include <linux/fs.h>
#include <linux/sched.h>
#include <linux/mm.h>
@@ -18,7 +19,7 @@ EXPORT_SYMBOL(default_unplug_io_fn);
struct backing_dev_info default_backing_dev_info = {
.ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE,
.state = 0,
- .capabilities = BDI_CAP_MAP_COPY,
+ .capabilities = BDI_CAP_MAP_COPY | BDI_CAP_FLUSH_FORKER,
.unplug_io_fn = default_unplug_io_fn,
};
EXPORT_SYMBOL_GPL(default_backing_dev_info);
@@ -26,6 +27,7 @@ EXPORT_SYMBOL_GPL(default_backing_dev_info);
static struct class *bdi_class;
DEFINE_SPINLOCK(bdi_lock);
LIST_HEAD(bdi_list);
+LIST_HEAD(bdi_pending_list);

#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
@@ -197,6 +199,147 @@ static int __init default_bdi_init(void)
}
subsys_initcall(default_bdi_init);

+static int bdi_start_fn(void *ptr)
+{
+ struct backing_dev_info *bdi = ptr;
+ struct task_struct *tsk = current;
+
+ /*
+ * Add us to the active bdi_list
+ */
+ spin_lock_bh(&bdi_lock);
+ list_add_rcu(&bdi->bdi_list, &bdi_list);
+ spin_unlock_bh(&bdi_lock);
+
+ tsk->flags |= PF_FLUSHER | PF_SWAPWRITE;
+ set_freezable();
+
+ /*
+ * Our parent may run at a different priority, just set us to normal
+ */
+ set_user_nice(tsk, 0);
+
+ /*
+ * Clear pending bit and wakeup anybody waiting to tear us down
+ */
+ clear_bit(BDI_pending, &bdi->state);
+ wake_up_bit(&bdi->state, BDI_pending);
+
+ return bdi_writeback_task(bdi);
+}
+
+static int bdi_forker_task(void *ptr)
+{
+ struct backing_dev_info *bdi = ptr;
+ struct task_struct *tsk = current;
+
+ for (;;) {
+ DECLARE_WAITQUEUE(wait, tsk);
+
+ /*
+ * Should never trigger on the default bdi
+ */
+ WARN_ON(bdi_has_dirty_io(bdi));
+
+ add_wait_queue(&bdi->wait, &wait);
+ set_task_state(tsk, TASK_INTERRUPTIBLE);
+ smp_mb();
+ if (list_empty(&bdi_pending_list))
+ schedule();
+ else {
+ struct backing_dev_info *bdi = NULL;
+
+ spin_lock_bh(&bdi_lock);
+ if (!list_empty(&bdi_pending_list)) {
+ bdi = list_entry(bdi_pending_list.next,
+ struct backing_dev_info,
+ bdi_list);
+ list_del_init(&bdi->bdi_list);
+ }
+ spin_unlock_bh(&bdi_lock);
+
+ /*
+ * If no bdi or bdi already got setup, continue
+ */
+ if (!bdi || bdi->task)
+ continue;
+
+ bdi->task = kthread_run(bdi_start_fn, bdi, "bdi-%s",
+ dev_name(bdi->dev));
+ /*
+ * If task creation fails, then readd the bdi to
+ * the pending list and force writeout of the bdi
+ * from this forker thread. That will free some memory
+ * and we can try again.
+ */
+ if (!bdi->task) {
+ struct writeback_control wbc = {
+ .bdi = bdi,
+ .sync_mode = WB_SYNC_NONE,
+ .older_than_this = NULL,
+ .range_cyclic = 1,
+ };
+
+ /*
+ * Add this 'bdi' to the back, so we get
+ * a chance to flush other bdi's to free
+ * memory.
+ */
+ spin_lock_bh(&bdi_lock);
+ list_add_tail(&bdi->bdi_list,
+ &bdi_pending_list);
+ spin_unlock_bh(&bdi_lock);
+
+ wbc.nr_to_write = 1024;
+ generic_sync_bdi_inodes(NULL, &wbc);
+ }
+ }
+
+ set_task_state(tsk, TASK_RUNNING);
+ finish_wait(&bdi->wait, &wait);
+ }
+
+ return 0;
+}
+
+/*
+ * Grace period has now ended, init bdi->bdi_list and add us to the
+ * list of bdi's that are pending for task creation. Wake up
+ * bdi_forker_task() to finish the job and add us back to the
+ * active bdi_list.
+ */
+static void bdi_add_to_pending(struct rcu_head *head)
+{
+ struct backing_dev_info *bdi;
+
+ bdi = container_of(head, struct backing_dev_info, rcu_head);
+ INIT_LIST_HEAD(&bdi->bdi_list);
+
+ spin_lock(&bdi_lock);
+ list_add_tail(&bdi->bdi_list, &bdi_pending_list);
+ spin_unlock(&bdi_lock);
+
+ wake_up(&default_backing_dev_info.wait);
+}
+
+void bdi_add_flusher_task(struct backing_dev_info *bdi)
+{
+ if (test_and_set_bit(BDI_pending, &bdi->state))
+ return;
+
+ spin_lock_bh(&bdi_lock);
+ list_del_rcu(&bdi->bdi_list);
+ spin_unlock_bh(&bdi_lock);
+
+ /*
+ * We need to wait for the current grace period to end,
+ * in case others were browsing the bdi_list as well.
+ * So defer the adding and wakeup to after the RCU
+ * grace period has ended.
+ */
+ call_rcu(&bdi->rcu_head, bdi_add_to_pending);
+}
+
int bdi_register(struct backing_dev_info *bdi, struct device *parent,
const char *fmt, ...)
{
@@ -215,17 +358,24 @@ int bdi_register(struct backing_dev_info *bdi, struct device *parent,
goto exit;
}

- bdi->task = kthread_run(bdi_writeback_task, bdi, "bdi-%s",
- dev_name(dev));
- if (!bdi->task) {
- ret = -ENOMEM;
- goto exit;
+ /*
+ * Just start the forker thread for our default backing_dev_info,
+ * and add other bdi's to the list. They will get a thread created
+ * on-demand when they need it.
+ */
+ if (bdi->capabilities & BDI_CAP_FLUSH_FORKER) {
+ bdi->task = kthread_run(bdi_forker_task, bdi, "bdi-%s",
+ dev_name(dev));
+ if (!bdi->task) {
+ ret = -ENOMEM;
+ goto exit;
+ }
+ } else {
+ spin_lock_bh(&bdi_lock);
+ list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
+ spin_unlock_bh(&bdi_lock);
}

- spin_lock(&bdi_lock);
- list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
- spin_unlock(&bdi_lock);
-
bdi->dev = dev;
bdi_debug_register(bdi, dev_name(dev));

@@ -240,11 +390,22 @@ int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
}
EXPORT_SYMBOL(bdi_register_dev);

+static int sched_wait(void *word)
+{
+ schedule();
+ return 0;
+}
+
static void bdi_remove_from_list(struct backing_dev_info *bdi)
{
- spin_lock(&bdi_lock);
+ /*
+ * If setup is pending, wait for that to complete first
+ */
+ wait_on_bit(&bdi->state, BDI_pending, sched_wait, TASK_UNINTERRUPTIBLE);
+
+ spin_lock_bh(&bdi_lock);
list_del_rcu(&bdi->bdi_list);
- spin_unlock(&bdi_lock);
+ spin_unlock_bh(&bdi_lock);

/*
* In case the bdi is freed right after unregister, we need to
@@ -256,10 +417,12 @@ static void bdi_remove_from_list(struct backing_dev_info *bdi)
void bdi_unregister(struct backing_dev_info *bdi)
{
if (bdi->dev) {
- bdi_remove_from_list(bdi);
- if (bdi->task) {
- kthread_stop(bdi->task);
- bdi->task = NULL;
+ if (!(bdi->capabilities & BDI_CAP_FLUSH_FORKER)) {
+ bdi_remove_from_list(bdi);
+ if (bdi->task) {
+ kthread_stop(bdi->task);
+ bdi->task = NULL;
+ }
}
bdi_debug_unregister(bdi);
device_unregister(bdi->dev);
@@ -272,6 +435,7 @@ int bdi_init(struct backing_dev_info *bdi)
{
int i, err;

+ INIT_RCU_HEAD(&bdi->rcu_head);
bdi->dev = NULL;

bdi->min_ratio = 0;
--
1.6.2

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/