Commit aa464ba9 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Andrew Morton

lazy tlb: introduce lazy tlb mm refcount helper functions

Add explicit _lazy_tlb annotated functions for lazy tlb mm refcounting. 
This makes the lazy tlb mm references more obvious, and allows the
refcounting scheme to be modified in later changes.  There is no
functional change with this patch.

Link: https://lkml.kernel.org/r/20230203071837.1136453-3-npiggin@gmail.comSigned-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Acked-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 6cad87b0
...@@ -253,7 +253,7 @@ static int ecard_init_mm(void) ...@@ -253,7 +253,7 @@ static int ecard_init_mm(void)
current->mm = mm; current->mm = mm;
current->active_mm = mm; current->active_mm = mm;
activate_mm(active_mm, mm); activate_mm(active_mm, mm);
mmdrop(active_mm); mmdrop_lazy_tlb(active_mm);
ecard_init_pgtables(mm); ecard_init_pgtables(mm);
return 0; return 0;
} }
......
...@@ -1611,7 +1611,7 @@ void start_secondary(void *unused) ...@@ -1611,7 +1611,7 @@ void start_secondary(void *unused)
if (IS_ENABLED(CONFIG_PPC32)) if (IS_ENABLED(CONFIG_PPC32))
setup_kup(); setup_kup();
mmgrab(&init_mm); mmgrab_lazy_tlb(&init_mm);
current->active_mm = &init_mm; current->active_mm = &init_mm;
smp_store_cpu_info(cpu); smp_store_cpu_info(cpu);
......
...@@ -797,10 +797,10 @@ void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush) ...@@ -797,10 +797,10 @@ void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush)
if (current->active_mm == mm) { if (current->active_mm == mm) {
WARN_ON_ONCE(current->mm != NULL); WARN_ON_ONCE(current->mm != NULL);
/* Is a kernel thread and is using mm as the lazy tlb */ /* Is a kernel thread and is using mm as the lazy tlb */
mmgrab(&init_mm); mmgrab_lazy_tlb(&init_mm);
current->active_mm = &init_mm; current->active_mm = &init_mm;
switch_mm_irqs_off(mm, &init_mm, current); switch_mm_irqs_off(mm, &init_mm, current);
mmdrop(mm); mmdrop_lazy_tlb(mm);
} }
/* /*
......
...@@ -1034,7 +1034,7 @@ static int exec_mmap(struct mm_struct *mm) ...@@ -1034,7 +1034,7 @@ static int exec_mmap(struct mm_struct *mm)
mmput(old_mm); mmput(old_mm);
return 0; return 0;
} }
mmdrop(active_mm); mmdrop_lazy_tlb(active_mm);
return 0; return 0;
} }
......
...@@ -79,6 +79,22 @@ static inline void mmdrop_sched(struct mm_struct *mm) ...@@ -79,6 +79,22 @@ static inline void mmdrop_sched(struct mm_struct *mm)
} }
#endif #endif
/* Helpers for lazy TLB mm refcounting */
static inline void mmgrab_lazy_tlb(struct mm_struct *mm)
{
mmgrab(mm);
}
static inline void mmdrop_lazy_tlb(struct mm_struct *mm)
{
mmdrop(mm);
}
static inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm)
{
mmdrop_sched(mm);
}
/** /**
* mmget() - Pin the address space associated with a &struct mm_struct. * mmget() - Pin the address space associated with a &struct mm_struct.
* @mm: The address space to pin. * @mm: The address space to pin.
......
...@@ -623,7 +623,7 @@ static int finish_cpu(unsigned int cpu) ...@@ -623,7 +623,7 @@ static int finish_cpu(unsigned int cpu)
*/ */
if (mm != &init_mm) if (mm != &init_mm)
idle->active_mm = &init_mm; idle->active_mm = &init_mm;
mmdrop(mm); mmdrop_lazy_tlb(mm);
return 0; return 0;
} }
......
...@@ -537,7 +537,7 @@ static void exit_mm(void) ...@@ -537,7 +537,7 @@ static void exit_mm(void)
return; return;
sync_mm_rss(mm); sync_mm_rss(mm);
mmap_read_lock(mm); mmap_read_lock(mm);
mmgrab(mm); mmgrab_lazy_tlb(mm);
BUG_ON(mm != current->active_mm); BUG_ON(mm != current->active_mm);
/* more a memory barrier than a real lock */ /* more a memory barrier than a real lock */
task_lock(current); task_lock(current);
......
...@@ -1415,6 +1415,11 @@ void kthread_use_mm(struct mm_struct *mm) ...@@ -1415,6 +1415,11 @@ void kthread_use_mm(struct mm_struct *mm)
WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
WARN_ON_ONCE(tsk->mm); WARN_ON_ONCE(tsk->mm);
/*
* It is possible for mm to be the same as tsk->active_mm, but
* we must still mmgrab(mm) and mmdrop_lazy_tlb(active_mm),
* because these references are not equivalent.
*/
mmgrab(mm); mmgrab(mm);
task_lock(tsk); task_lock(tsk);
...@@ -1438,9 +1443,9 @@ void kthread_use_mm(struct mm_struct *mm) ...@@ -1438,9 +1443,9 @@ void kthread_use_mm(struct mm_struct *mm)
* memory barrier after storing to tsk->mm, before accessing * memory barrier after storing to tsk->mm, before accessing
* user-space memory. A full memory barrier for membarrier * user-space memory. A full memory barrier for membarrier
* {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by * {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by
* mmdrop(). * mmdrop_lazy_tlb().
*/ */
mmdrop(active_mm); mmdrop_lazy_tlb(active_mm);
} }
EXPORT_SYMBOL_GPL(kthread_use_mm); EXPORT_SYMBOL_GPL(kthread_use_mm);
...@@ -1468,10 +1473,13 @@ void kthread_unuse_mm(struct mm_struct *mm) ...@@ -1468,10 +1473,13 @@ void kthread_unuse_mm(struct mm_struct *mm)
local_irq_disable(); local_irq_disable();
tsk->mm = NULL; tsk->mm = NULL;
membarrier_update_current_mm(NULL); membarrier_update_current_mm(NULL);
mmgrab_lazy_tlb(mm);
/* active_mm is still 'mm' */ /* active_mm is still 'mm' */
enter_lazy_tlb(mm, tsk); enter_lazy_tlb(mm, tsk);
local_irq_enable(); local_irq_enable();
task_unlock(tsk); task_unlock(tsk);
mmdrop(mm);
} }
EXPORT_SYMBOL_GPL(kthread_unuse_mm); EXPORT_SYMBOL_GPL(kthread_unuse_mm);
......
...@@ -5203,13 +5203,14 @@ static struct rq *finish_task_switch(struct task_struct *prev) ...@@ -5203,13 +5203,14 @@ static struct rq *finish_task_switch(struct task_struct *prev)
* rq->curr, before returning to userspace, so provide them here: * rq->curr, before returning to userspace, so provide them here:
* *
* - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
* provided by mmdrop(), * provided by mmdrop_lazy_tlb(),
* - a sync_core for SYNC_CORE. * - a sync_core for SYNC_CORE.
*/ */
if (mm) { if (mm) {
membarrier_mm_sync_core_before_usermode(mm); membarrier_mm_sync_core_before_usermode(mm);
mmdrop_sched(mm); mmdrop_lazy_tlb_sched(mm);
} }
if (unlikely(prev_state == TASK_DEAD)) { if (unlikely(prev_state == TASK_DEAD)) {
if (prev->sched_class->task_dead) if (prev->sched_class->task_dead)
prev->sched_class->task_dead(prev); prev->sched_class->task_dead(prev);
...@@ -5266,9 +5267,9 @@ context_switch(struct rq *rq, struct task_struct *prev, ...@@ -5266,9 +5267,9 @@ context_switch(struct rq *rq, struct task_struct *prev,
/* /*
* kernel -> kernel lazy + transfer active * kernel -> kernel lazy + transfer active
* user -> kernel lazy + mmgrab() active * user -> kernel lazy + mmgrab_lazy_tlb() active
* *
* kernel -> user switch + mmdrop() active * kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch * user -> user switch
*/ */
if (!next->mm) { // to kernel if (!next->mm) { // to kernel
...@@ -5276,7 +5277,7 @@ context_switch(struct rq *rq, struct task_struct *prev, ...@@ -5276,7 +5277,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
next->active_mm = prev->active_mm; next->active_mm = prev->active_mm;
if (prev->mm) // from user if (prev->mm) // from user
mmgrab(prev->active_mm); mmgrab_lazy_tlb(prev->active_mm);
else else
prev->active_mm = NULL; prev->active_mm = NULL;
} else { // to user } else { // to user
...@@ -5293,7 +5294,7 @@ context_switch(struct rq *rq, struct task_struct *prev, ...@@ -5293,7 +5294,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
lru_gen_use_mm(next->mm); lru_gen_use_mm(next->mm);
if (!prev->mm) { // from kernel if (!prev->mm) { // from kernel
/* will mmdrop() in finish_task_switch(). */ /* will mmdrop_lazy_tlb() in finish_task_switch(). */
rq->prev_mm = prev->active_mm; rq->prev_mm = prev->active_mm;
prev->active_mm = NULL; prev->active_mm = NULL;
} }
...@@ -9935,7 +9936,7 @@ void __init sched_init(void) ...@@ -9935,7 +9936,7 @@ void __init sched_init(void)
/* /*
* The boot idle thread does lazy MMU switching as well: * The boot idle thread does lazy MMU switching as well:
*/ */
mmgrab(&init_mm); mmgrab_lazy_tlb(&init_mm);
enter_lazy_tlb(&init_mm, current); enter_lazy_tlb(&init_mm, current);
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment