Commit b4fb015e authored by Konstantin Khlebnikov's avatar Konstantin Khlebnikov Committed by Ingo Molnar

sched/rt: Optimize checking group RT scheduler constraints

Group RT scheduler contains protection against setting zero runtime for
cgroup with RT tasks. Right now function tg_set_rt_bandwidth() iterates
over all CPU cgroups and calls tg_has_rt_tasks() for any cgroup which
runtime is zero (not only for changed one). Default RT runtime is zero,
thus tg_has_rt_tasks() will is called for almost at CPU cgroups.

This protection already is slightly racy: runtime limit could be changed
between cpu_cgroup_can_attach() and cpu_cgroup_attach() because changing
cgroup attribute does not lock cgroup_mutex while attach does not lock
rt_constraints_mutex. Changing task scheduler class also races with
changing rt runtime: check in __sched_setscheduler() isn't protected.

Function tg_has_rt_tasks() iterates over all threads in the system.
This gives NR_CGROUPS * NR_TASKS operations under single tasklist_lock
locked for read tg_set_rt_bandwidth(). Any concurrent attempt of locking
tasklist_lock for write (for example fork) will stuck with disabled irqs.

This patch makes two optimizations:
1) Remove locking tasklist_lock and iterate only tasks in cgroup
2) Call tg_has_rt_tasks() iff rt runtime changes from non-zero to zero

All changed code is under CONFIG_RT_GROUP_SCHED.

Testcase:

 # mkdir /sys/fs/cgroup/cpu/test{1..10000}
 # echo 0 | tee /sys/fs/cgroup/cpu/test*/cpu.rt_runtime_us

At the same time without patch fork time will be >100ms:

 # perf trace -e clone --duration 100 stress-ng --fork 1

Also remote ping will show timings >100ms caused by irq latency.
Signed-off-by: default avatarKonstantin Khlebnikov <khlebnikov@yandex-team.ru>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: https://lkml.kernel.org/r/157996383820.4651.11292439232549211693.stgit@buzz
parent bec2860a
...@@ -2449,10 +2449,11 @@ const struct sched_class rt_sched_class = { ...@@ -2449,10 +2449,11 @@ const struct sched_class rt_sched_class = {
*/ */
static DEFINE_MUTEX(rt_constraints_mutex); static DEFINE_MUTEX(rt_constraints_mutex);
/* Must be called with tasklist_lock held */
static inline int tg_has_rt_tasks(struct task_group *tg) static inline int tg_has_rt_tasks(struct task_group *tg)
{ {
struct task_struct *g, *p; struct task_struct *task;
struct css_task_iter it;
int ret = 0;
/* /*
* Autogroups do not have RT tasks; see autogroup_create(). * Autogroups do not have RT tasks; see autogroup_create().
...@@ -2460,12 +2461,12 @@ static inline int tg_has_rt_tasks(struct task_group *tg) ...@@ -2460,12 +2461,12 @@ static inline int tg_has_rt_tasks(struct task_group *tg)
if (task_group_is_autogroup(tg)) if (task_group_is_autogroup(tg))
return 0; return 0;
for_each_process_thread(g, p) { css_task_iter_start(&tg->css, 0, &it);
if (rt_task(p) && task_group(p) == tg) while (!ret && (task = css_task_iter_next(&it)))
return 1; ret |= rt_task(task);
} css_task_iter_end(&it);
return 0; return ret;
} }
struct rt_schedulable_data { struct rt_schedulable_data {
...@@ -2496,9 +2497,10 @@ static int tg_rt_schedulable(struct task_group *tg, void *data) ...@@ -2496,9 +2497,10 @@ static int tg_rt_schedulable(struct task_group *tg, void *data)
return -EINVAL; return -EINVAL;
/* /*
* Ensure we don't starve existing RT tasks. * Ensure we don't starve existing RT tasks if runtime turns zero.
*/ */
if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg)) if (rt_bandwidth_enabled() && !runtime &&
tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg))
return -EBUSY; return -EBUSY;
total = to_ratio(period, runtime); total = to_ratio(period, runtime);
...@@ -2564,7 +2566,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg, ...@@ -2564,7 +2566,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg,
return -EINVAL; return -EINVAL;
mutex_lock(&rt_constraints_mutex); mutex_lock(&rt_constraints_mutex);
read_lock(&tasklist_lock);
err = __rt_schedulable(tg, rt_period, rt_runtime); err = __rt_schedulable(tg, rt_period, rt_runtime);
if (err) if (err)
goto unlock; goto unlock;
...@@ -2582,7 +2583,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg, ...@@ -2582,7 +2583,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg,
} }
raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
unlock: unlock:
read_unlock(&tasklist_lock);
mutex_unlock(&rt_constraints_mutex); mutex_unlock(&rt_constraints_mutex);
return err; return err;
...@@ -2641,9 +2641,7 @@ static int sched_rt_global_constraints(void) ...@@ -2641,9 +2641,7 @@ static int sched_rt_global_constraints(void)
int ret = 0; int ret = 0;
mutex_lock(&rt_constraints_mutex); mutex_lock(&rt_constraints_mutex);
read_lock(&tasklist_lock);
ret = __rt_schedulable(NULL, 0, 0); ret = __rt_schedulable(NULL, 0, 0);
read_unlock(&tasklist_lock);
mutex_unlock(&rt_constraints_mutex); mutex_unlock(&rt_constraints_mutex);
return ret; return ret;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment