Commit 38432d05 authored by Ingo Molnar's avatar Ingo Molnar

- new, less intrusive and faster migration method:

/*
 * This is how migration works:
 *
 * 1) we queue a migration_req_t structure in the source CPU's
 *    runqueue and wake up that CPU's migration thread.
 * 2) we down() the locked semaphore => thread blocks.
 * 3) migration thread wakes up (implicitly it forces the migrated
 *    thread off the CPU)
 * 4) it gets the migration request and checks whether the migrated
 *    task is still in the wrong runqueue.
 * 5) if it's in the wrong runqueue then the migration thread removes
 *    it and puts it into the right queue.
 * 6) migration thread up()s the semaphore.
 * 7) we wake up and the migration is done.
 */
parent eac94688
...@@ -52,8 +52,6 @@ ...@@ -52,8 +52,6 @@
mm: NULL, \ mm: NULL, \
active_mm: &init_mm, \ active_mm: &init_mm, \
run_list: LIST_HEAD_INIT(tsk.run_list), \ run_list: LIST_HEAD_INIT(tsk.run_list), \
migration_list: LIST_HEAD_INIT(tsk.migration_list), \
migration_sem: __MUTEX_INITIALIZER(tsk.migration_sem), \
time_slice: HZ, \ time_slice: HZ, \
next_task: &tsk, \ next_task: &tsk, \
prev_task: &tsk, \ prev_task: &tsk, \
......
...@@ -286,9 +286,6 @@ struct task_struct { ...@@ -286,9 +286,6 @@ struct task_struct {
wait_queue_head_t wait_chldexit; /* for wait4() */ wait_queue_head_t wait_chldexit; /* for wait4() */
struct completion *vfork_done; /* for vfork() */ struct completion *vfork_done; /* for vfork() */
list_t migration_list;
struct semaphore migration_sem;
unsigned long rt_priority; unsigned long rt_priority;
unsigned long it_real_value, it_prof_value, it_virt_value; unsigned long it_real_value, it_prof_value, it_virt_value;
unsigned long it_real_incr, it_prof_incr, it_virt_incr; unsigned long it_real_incr, it_prof_incr, it_virt_incr;
......
...@@ -365,12 +365,10 @@ void wake_up_forked_process(task_t * p) ...@@ -365,12 +365,10 @@ void wake_up_forked_process(task_t * p)
p->sleep_avg = p->sleep_avg * CHILD_PENALTY / 100; p->sleep_avg = p->sleep_avg * CHILD_PENALTY / 100;
p->prio = effective_prio(p); p->prio = effective_prio(p);
} }
INIT_LIST_HEAD(&p->migration_list);
p->thread_info->cpu = smp_processor_id(); p->thread_info->cpu = smp_processor_id();
activate_task(p, rq); activate_task(p, rq);
spin_unlock_irq(&rq->lock); spin_unlock_irq(&rq->lock);
init_MUTEX(&p->migration_sem);
preempt_enable(); preempt_enable();
} }
...@@ -974,65 +972,6 @@ long sleep_on_timeout(wait_queue_head_t *q, long timeout) ...@@ -974,65 +972,6 @@ long sleep_on_timeout(wait_queue_head_t *q, long timeout)
void scheduling_functions_end_here(void) { } void scheduling_functions_end_here(void) { }
#if CONFIG_SMP
/*
* Change a given task's CPU affinity. Migrate the process to a
* proper CPU and schedule it away if the current CPU is removed
* from the allowed bitmask.
*/
void set_cpus_allowed(task_t *p, unsigned long new_mask)
{
unsigned long flags;
runqueue_t *rq;
int dest_cpu;
down(&p->migration_sem);
if (!list_empty(&p->migration_list))
BUG();
new_mask &= cpu_online_map;
if (!new_mask)
BUG();
rq = task_rq_lock(p, &flags);
p->cpus_allowed = new_mask;
/*
* Can the task run on the task's current CPU? If not then
* migrate the process off to a proper CPU.
*/
if (new_mask & (1UL << p->thread_info->cpu)) {
task_rq_unlock(rq, &flags);
goto out;
}
/*
* We mark the process as nonrunnable, and kick it to
* schedule away from its current CPU. We also add
* the task to the migration queue and wake up the
* target CPU's migration thread, so that it can pick
* up this task and insert it into the local runqueue.
*/
p->state = TASK_UNINTERRUPTIBLE;
kick_if_running(p);
task_rq_unlock(rq, &flags);
dest_cpu = __ffs(new_mask);
rq = cpu_rq(dest_cpu);
spin_lock_irq(&rq->lock);
list_add(&p->migration_list, &rq->migration_queue);
spin_unlock_irq(&rq->lock);
wake_up_process(rq->migration_thread);
while (!((1UL << p->thread_info->cpu) & p->cpus_allowed) &&
(p->state != TASK_ZOMBIE))
yield();
out:
up(&p->migration_sem);
}
#endif
void set_user_nice(task_t *p, long nice) void set_user_nice(task_t *p, long nice)
{ {
unsigned long flags; unsigned long flags;
...@@ -1198,7 +1137,7 @@ static int setscheduler(pid_t pid, int policy, struct sched_param *param) ...@@ -1198,7 +1137,7 @@ static int setscheduler(pid_t pid, int policy, struct sched_param *param)
retval = 0; retval = 0;
p->policy = policy; p->policy = policy;
p->rt_priority = lp.sched_priority; p->rt_priority = lp.sched_priority;
if (rt_task(p)) if (policy != SCHED_OTHER)
p->prio = 99 - p->rt_priority; p->prio = 99 - p->rt_priority;
else else
p->prio = p->static_prio; p->prio = p->static_prio;
...@@ -1556,15 +1495,78 @@ void __init sched_init(void) ...@@ -1556,15 +1495,78 @@ void __init sched_init(void)
#if CONFIG_SMP #if CONFIG_SMP
/*
* This is how migration works:
*
* 1) we queue a migration_req_t structure in the source CPU's
* runqueue and wake up that CPU's migration thread.
* 2) we down() the locked semaphore => thread blocks.
* 3) migration thread wakes up (implicitly it forces the migrated
* thread off the CPU)
* 4) it gets the migration request and checks whether the migrated
* task is still in the wrong runqueue.
* 5) if it's in the wrong runqueue then the migration thread removes
* it and puts it into the right queue.
* 6) migration thread up()s the semaphore.
* 7) we wake up and the migration is done.
*/
typedef struct {
list_t list;
task_t *task;
struct semaphore sem;
} migration_req_t;
/*
* Change a given task's CPU affinity. Migrate the process to a
* proper CPU and schedule it away if the CPU it's executing on
* is removed from the allowed bitmask.
*
* NOTE: the caller must have a valid reference to the task, the
* task must not exit() & deallocate itself prematurely.
*/
void set_cpus_allowed(task_t *p, unsigned long new_mask)
{
unsigned long flags;
migration_req_t req;
runqueue_t *rq;
new_mask &= cpu_online_map;
if (!new_mask)
BUG();
rq = task_rq_lock(p, &flags);
p->cpus_allowed = new_mask;
/*
* Can the task run on the task's current CPU? If not then
* migrate the process off to a proper CPU.
*/
if (new_mask & (1UL << p->thread_info->cpu)) {
task_rq_unlock(rq, &flags);
return;
}
init_MUTEX_LOCKED(&req.sem);
req.task = p;
list_add(&req.list, &rq->migration_queue);
task_rq_unlock(rq, &flags);
wake_up_process(rq->migration_thread);
down(&req.sem);
}
static volatile unsigned long migration_mask; static volatile unsigned long migration_mask;
static int migration_thread(void * unused) static int migration_thread(void * unused)
{ {
struct sched_param param = { sched_priority: 99 };
runqueue_t *rq; runqueue_t *rq;
int ret;
daemonize(); daemonize();
sigfillset(&current->blocked); sigfillset(&current->blocked);
set_user_nice(current, -20); set_fs(KERNEL_DS);
ret = setscheduler(0, SCHED_FIFO, &param);
/* /*
* We have to migrate manually - there is no migration thread * We have to migrate manually - there is no migration thread
...@@ -1598,41 +1600,47 @@ static int migration_thread(void * unused) ...@@ -1598,41 +1600,47 @@ static int migration_thread(void * unused)
sprintf(current->comm, "migration_CPU%d", smp_processor_id()); sprintf(current->comm, "migration_CPU%d", smp_processor_id());
for (;;) { for (;;) {
runqueue_t *rq_src, *rq_dest;
struct list_head *head; struct list_head *head;
int cpu_src, cpu_dest;
migration_req_t *req;
unsigned long flags; unsigned long flags;
task_t *p = NULL; task_t *p;
spin_lock_irqsave(&rq->lock, flags); spin_lock_irqsave(&rq->lock, flags);
head = &rq->migration_queue; head = &rq->migration_queue;
if (list_empty(head)) {
current->state = TASK_UNINTERRUPTIBLE; current->state = TASK_UNINTERRUPTIBLE;
if (list_empty(head)) {
spin_unlock_irqrestore(&rq->lock, flags); spin_unlock_irqrestore(&rq->lock, flags);
schedule(); schedule();
continue; continue;
} }
p = list_entry(head->next, task_t, migration_list); req = list_entry(head->next, migration_req_t, list);
list_del_init(head->next); list_del_init(head->next);
spin_unlock_irqrestore(&rq->lock, flags); spin_unlock_irqrestore(&rq->lock, flags);
for (;;) { p = req->task;
runqueue_t *rq2 = task_rq_lock(p, &flags); cpu_dest = __ffs(p->cpus_allowed);
rq_dest = cpu_rq(cpu_dest);
repeat:
cpu_src = p->thread_info->cpu;
rq_src = cpu_rq(cpu_src);
if (!p->array) { double_rq_lock(rq_src, rq_dest);
p->thread_info->cpu = smp_processor_id(); if (p->thread_info->cpu != cpu_src) {
task_rq_unlock(rq2, &flags); double_rq_unlock(rq_src, rq_dest);
wake_up_process(p); goto repeat;
break;
}
if (p->state != TASK_UNINTERRUPTIBLE) {
p->state = TASK_UNINTERRUPTIBLE;
kick_if_running(p);
} }
task_rq_unlock(rq2, &flags); if (rq_src == rq) {
while ((p->state == TASK_UNINTERRUPTIBLE) && p->array) { p->thread_info->cpu = cpu_dest;
cpu_relax(); if (p->array) {
barrier(); deactivate_task(p, rq_src);
activate_task(p, rq_dest);
} }
} }
double_rq_unlock(rq_src, rq_dest);
up(&req->sem);
} }
} }
...@@ -1645,11 +1653,11 @@ void __init migration_init(void) ...@@ -1645,11 +1653,11 @@ void __init migration_init(void)
CLONE_FS | CLONE_FILES | CLONE_SIGNAL) < 0) CLONE_FS | CLONE_FILES | CLONE_SIGNAL) < 0)
BUG(); BUG();
migration_mask = (1 << smp_num_cpus) -1; migration_mask = (1 << smp_num_cpus) - 1;
for (cpu = 0; cpu < smp_num_cpus; cpu++) for (cpu = 0; cpu < smp_num_cpus; cpu++)
while (!cpu_rq(cpu)->migration_thread) while (!cpu_rq(cpu)->migration_thread)
yield(); schedule_timeout(2);
if (migration_mask) if (migration_mask)
BUG(); BUG();
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment