[PATCH] thread-aware oom-killer

From Ingo

- performance optimization: do not kill threads in the same thread group
  as the OOM-ing thread. (it's still necessery to scan over every thread
  though, as it's possible to have CLONE_VM threads in a different thread
  group - we do not want those to escape the OOM-kill.)

- to not let newly created child threads slip out of the group-kill. Note
  that the 2.4 kernel's OOM handler has the same problem, and it could be
  the reason why forkbombs occasionally slip out of the OOM kill.

kernel/fork.c

@@ -863,6 +863,14 @@ static struct task_struct *copy_process(unsigned long clone_flags,
 
 	/* Need tasklist lock for parent etc handling! */
 	write_lock_irq(&tasklist_lock);
+	/*
+	 * Check for pending SIGKILL! The new thread should not be allowed
+	 * to slip out of an OOM kill. (or normal SIGKILL.)
+	 */
+	if (sigismember(&current->pending.signal, SIGKILL)) {
+		write_unlock_irq(&tasklist_lock);
+		goto bad_fork_cleanup_namespace;
+	}
 
 	/* CLONE_PARENT re-uses the old parent */
 	if (clone_flags & CLONE_PARENT)

mm/oom_kill.c

@@ -175,9 +175,13 @@ static void oom_kill(void)
 	if (p == NULL)
 		panic("Out of memory and no killable processes...\n");
 
-	/* kill all processes that share the ->mm (i.e. all threads) */
+	oom_kill_task(p);
+	/*
+	 * kill all processes that share the ->mm (i.e. all threads),
+	 * but are in a different thread group
+	 */
 	do_each_thread(g, q)
-		if (q->mm == p->mm)
+		if (q->mm == p->mm && q->tgid != p->tgid)
 			oom_kill_task(q);
 	while_each_thread(g, q);