[PATCH] move put_task_struct() reaping into a thread [Re: 2.6.18-rt1]
From: hui
Date: Thu Sep 21 2006 - 02:57:00 EST
On Wed, Sep 20, 2006 at 04:19:07PM +0200, Ingo Molnar wrote:
> I'm pleased to announce the 2.6.18-rt1 tree, which can be downloaded
> from the usual place:
...
> as usual, bugreports, fixes and suggestions are welcome,
Speaking of which...
This patch moves put_task_struct() reaping into a thread instead of an
RCU callback function as discussed with Esben publically and Ingo privately:
bill
============================================================
--- include/linux/init_task.h 7998a1b8676588d10f9ca05c681b99fc3ee869d1
+++ include/linux/init_task.h ca1562e79859bb022e6e9b140cc1edad1116fca9
@@ -77,6 +77,12 @@
extern struct group_info init_groups;
+#ifdef CONFIG_PREEMPT_RT
+#define INIT_DELAYED_DROP(a) .delayed_drop = LIST_HEAD_INIT(a.delayed_drop),
+#else
+#define INIT_DELAYED_DROP()
+#endif
+
/*
* INIT_TASK is used to set up the first task table, touch at
* your own risk!. Base=0, limit=0x1fffff (=2MB)
@@ -128,6 +134,7 @@
.fs_excl = ATOMIC_INIT(0), \
.posix_timer_list = NULL, \
.pi_lock = RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock), \
+ INIT_DELAYED_DROP(tsk) \
INIT_TRACE_IRQFLAGS \
INIT_LOCKDEP \
}
============================================================
--- include/linux/sched.h afe9eb8f7f85de2d61a92d35807386aa9d79a52e
+++ include/linux/sched.h e8907a986e9567267f290b504177faf6a96d4dbd
@@ -1176,6 +1176,9 @@
#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info *delays;
#endif
+#ifdef CONFIG_PREEMPT_RT
+ struct list_head delayed_drop;
+#endif
};
static inline pid_t process_group(struct task_struct *tsk)
@@ -1199,15 +1202,6 @@
extern void free_task(struct task_struct *tsk);
#define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0)
-#ifdef CONFIG_PREEMPT_RT
-extern void __put_task_struct_cb(struct rcu_head *rhp);
-
-static inline void put_task_struct(struct task_struct *t)
-{
- if (atomic_dec_and_test(&t->usage))
- call_rcu(&t->rcu, __put_task_struct_cb);
-}
-#else
extern void __put_task_struct(struct task_struct *t);
static inline void put_task_struct(struct task_struct *t)
@@ -1215,7 +1209,6 @@
if (atomic_dec_and_test(&t->usage))
__put_task_struct(t);
}
-#endif
/*
* Per process flags
============================================================
--- kernel/exit.c 98f9cbf2db74c4cf03c792c75b63991856793263
+++ kernel/exit.c 5a6655dad5c3e72723c9b42adcecab12daf6b933
@@ -131,11 +131,6 @@
}
}
-static void delayed_put_task_struct(struct rcu_head *rhp)
-{
- put_task_struct(container_of(rhp, struct task_struct, rcu));
-}
-
void release_task(struct task_struct * p)
{
struct task_struct *leader;
@@ -172,7 +167,7 @@
write_unlock_irq(&tasklist_lock);
proc_flush_task(p);
release_thread(p);
- call_rcu(&p->rcu, delayed_put_task_struct);
+ put_task_struct(p);
p = leader;
if (unlikely(zap_leader))
============================================================
--- kernel/fork.c a26a13e186fd7a595fb24745cd6060c155cb4e10
+++ kernel/fork.c 5ea1f9118ab66e1668ab7f5e6549e184d1b05d74
@@ -75,7 +75,10 @@
*/
static DEFINE_PER_CPU(struct task_struct *, desched_task);
-static DEFINE_PER_CPU(struct list_head, delayed_drop_list);
+static DEFINE_PER_CPU(struct list_head, delayed_mmdrop_list);
+#ifdef CONFIG_PREEMPT_RT
+static DEFINE_PER_CPU(struct list_head, delayed_put_task_struct_list);
+#endif
int nr_processes(void)
{
@@ -120,28 +123,33 @@
}
EXPORT_SYMBOL(free_task);
+
#ifdef CONFIG_PREEMPT_RT
-void __put_task_struct_cb(struct rcu_head *rhp)
+/*
+ * We dont want to do complex work from the scheduler with preemption
+ * disabled, therefore we delay the work to a per-CPU worker thread.
+ */
+static void _wake_cpu_desched_task(void);
+
+void fastcall __put_task_struct(struct task_struct *task)
{
- struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
+ struct list_head *head;
- BUG_ON(atomic_read(&tsk->usage));
- WARN_ON(!(tsk->flags & PF_DEAD));
- WARN_ON(!(tsk->exit_state & (EXIT_DEAD | EXIT_ZOMBIE)));
- WARN_ON(tsk == current);
+ head = &get_cpu_var(delayed_put_task_struct_list);
+ list_add_tail(&task->delayed_drop, head);
- security_task_free(tsk);
- free_uid(tsk->user);
- put_group_info(tsk->group_info);
- delayacct_tsk_free(tsk);
+ _wake_cpu_desched_task();
- if (!profile_handoff_task(tsk))
- free_task(tsk);
+ put_cpu_var(delayed_put_task_struct_list);
}
+/*
+ * Delay if this is in an atomic critical section otherwise inline the deallocation
+ */
+void __put_task_struct_inline(struct task_struct *tsk)
#else
-
void __put_task_struct(struct task_struct *tsk)
+#endif
{
WARN_ON(!(tsk->exit_state & (EXIT_DEAD | EXIT_ZOMBIE)));
BUG_ON(atomic_read(&tsk->usage));
@@ -155,7 +163,6 @@
if (!profile_handoff_task(tsk))
free_task(tsk);
}
-#endif
void __init fork_init(unsigned long mempages)
{
@@ -189,8 +196,12 @@
init_task.signal->rlim[RLIMIT_SIGPENDING] =
init_task.signal->rlim[RLIMIT_NPROC];
- for (i = 0; i < NR_CPUS; i++)
- INIT_LIST_HEAD(&per_cpu(delayed_drop_list, i));
+ for (i = 0; i < NR_CPUS; i++) {
+ INIT_LIST_HEAD(&per_cpu(delayed_mmdrop_list, i));
+#ifdef CONFIG_PREEMPT_RT
+ INIT_LIST_HEAD(&per_cpu(delayed_put_task_struct_list, i));
+#endif
+ }
}
static struct task_struct *dup_task_struct(struct task_struct *orig)
@@ -1123,6 +1134,9 @@
#endif
rt_mutex_init_task(p);
+#ifdef CONFIG_PREEMPT_RT
+ INIT_LIST_HEAD(&p->delayed_drop);
+#endif
#ifdef CONFIG_DEBUG_MUTEXES
p->blocked_on = NULL; /* not blocked yet */
@@ -1740,24 +1754,58 @@
return err;
}
+static void _wake_cpu_desched_task(void)
+{
+ struct task_struct *desched_task;
+
+ desched_task = __get_cpu_var(desched_task);
+ if (desched_task)
+ wake_up_process(desched_task);
+}
+
+#ifdef CONFIG_PREEMPT_RT
+static int put_task_struct_complete(void)
+{
+ struct list_head *head;
+ int ret = 0;
+
+ head = &get_cpu_var(delayed_put_task_struct_list);
+ while (!list_empty(head)) {
+ struct task_struct *task = list_entry(head->next,
+ struct task_struct, delayed_drop);
+ list_del(&task->delayed_drop);
+ put_cpu_var(delayed_put_task_struct_list);
+
+ __put_task_struct_inline(task); /* call the original function to perform the operation */
+ ret = 1;
+
+ head = &get_cpu_var(delayed_put_task_struct_list);
+ }
+ put_cpu_var(delayed_put_task_struct_list);
+
+ return ret;
+}
+
+#endif
+
static int mmdrop_complete(void)
{
struct list_head *head;
int ret = 0;
- head = &get_cpu_var(delayed_drop_list);
+ head = &get_cpu_var(delayed_mmdrop_list);
while (!list_empty(head)) {
struct mm_struct *mm = list_entry(head->next,
struct mm_struct, delayed_drop);
list_del(&mm->delayed_drop);
- put_cpu_var(delayed_drop_list);
+ put_cpu_var(delayed_mmdrop_list);
__mmdrop(mm);
ret = 1;
- head = &get_cpu_var(delayed_drop_list);
+ head = &get_cpu_var(delayed_mmdrop_list);
}
- put_cpu_var(delayed_drop_list);
+ put_cpu_var(delayed_mmdrop_list);
return ret;
}
@@ -1768,15 +1816,14 @@
*/
void fastcall __mmdrop_delayed(struct mm_struct *mm)
{
- struct task_struct *desched_task;
struct list_head *head;
- head = &get_cpu_var(delayed_drop_list);
+ head = &get_cpu_var(delayed_mmdrop_list);
list_add_tail(&mm->delayed_drop, head);
- desched_task = __get_cpu_var(desched_task);
- if (desched_task)
- wake_up_process(desched_task);
- put_cpu_var(delayed_drop_list);
+
+ _wake_cpu_desched_task();
+
+ put_cpu_var(delayed_mmdrop_list);
}
static int desched_thread(void * __bind_cpu)
@@ -1790,6 +1837,9 @@
if (mmdrop_complete())
continue;
+ if (put_task_struct_complete())
+ continue;
+
schedule();
/* This must be called from time to time on ia64, and is a no-op on other archs.
@@ -1814,7 +1864,10 @@
case CPU_UP_PREPARE:
BUG_ON(per_cpu(desched_task, hotcpu));
- INIT_LIST_HEAD(&per_cpu(delayed_drop_list, hotcpu));
+ INIT_LIST_HEAD(&per_cpu(delayed_mmdrop_list, hotcpu));
+#ifdef CONFIG_PREEMPT_RT
+ INIT_LIST_HEAD(&per_cpu(delayed_put_task_struct_list, hotcpu));
+#endif
p = kthread_create(desched_thread, hcpu, "desched/%d", hotcpu);
if (IS_ERR(p)) {
printk("desched_thread for %i failed\n", hotcpu);