Commit 138e4ad6 authored by Oleg Nesterov's avatar Oleg Nesterov Committed by Linus Torvalds

epoll: fix race between ep_poll_callback(POLLFREE) and ep_free()/ep_remove()

The race was introduced by me in commit 971316f0 ("epoll:
ep_unregister_pollwait() can use the freed pwq->whead").  I did not
realize that nothing can protect eventpoll after ep_poll_callback() sets
->whead = NULL, only whead->lock can save us from the race with
ep_free() or ep_remove().

Move ->whead = NULL to the end of ep_poll_callback() and add the
necessary barriers.

TODO: cleanup the ewake/EPOLLEXCLUSIVE logic, it was confusing even
before this patch.

Hopefully this explains use-after-free reported by syzcaller:

	BUG: KASAN: use-after-free in debug_spin_lock_before
	...
	 _raw_spin_lock_irqsave+0x4a/0x60 kernel/locking/spinlock.c:159
	 ep_poll_callback+0x29f/0xff0 fs/eventpoll.c:1148

this is spin_lock(eventpoll->lock),

	...
	Freed by task 17774:
	...
	 kfree+0xe8/0x2c0 mm/slub.c:3883
	 ep_free+0x22c/0x2a0 fs/eventpoll.c:865

Fixes: 971316f0 ("epoll: ep_unregister_pollwait() can use the freed pwq->whead")
Reported-by: default avatar范龙飞 <long7573@126.com>
Cc: stable@vger.kernel.org
Signed-off-by: default avatarOleg Nesterov <oleg@redhat.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 8cf9f2a2
...@@ -600,8 +600,13 @@ static void ep_remove_wait_queue(struct eppoll_entry *pwq) ...@@ -600,8 +600,13 @@ static void ep_remove_wait_queue(struct eppoll_entry *pwq)
wait_queue_head_t *whead; wait_queue_head_t *whead;
rcu_read_lock(); rcu_read_lock();
/* If it is cleared by POLLFREE, it should be rcu-safe */ /*
whead = rcu_dereference(pwq->whead); * If it is cleared by POLLFREE, it should be rcu-safe.
* If we read NULL we need a barrier paired with
* smp_store_release() in ep_poll_callback(), otherwise
* we rely on whead->lock.
*/
whead = smp_load_acquire(&pwq->whead);
if (whead) if (whead)
remove_wait_queue(whead, &pwq->wait); remove_wait_queue(whead, &pwq->wait);
rcu_read_unlock(); rcu_read_unlock();
...@@ -1134,17 +1139,6 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v ...@@ -1134,17 +1139,6 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v
struct eventpoll *ep = epi->ep; struct eventpoll *ep = epi->ep;
int ewake = 0; int ewake = 0;
if ((unsigned long)key & POLLFREE) {
ep_pwq_from_wait(wait)->whead = NULL;
/*
* whead = NULL above can race with ep_remove_wait_queue()
* which can do another remove_wait_queue() after us, so we
* can't use __remove_wait_queue(). whead->lock is held by
* the caller.
*/
list_del_init(&wait->entry);
}
spin_lock_irqsave(&ep->lock, flags); spin_lock_irqsave(&ep->lock, flags);
ep_set_busy_poll_napi_id(epi); ep_set_busy_poll_napi_id(epi);
...@@ -1228,10 +1222,26 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v ...@@ -1228,10 +1222,26 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v
if (pwake) if (pwake)
ep_poll_safewake(&ep->poll_wait); ep_poll_safewake(&ep->poll_wait);
if (epi->event.events & EPOLLEXCLUSIVE) if (!(epi->event.events & EPOLLEXCLUSIVE))
return ewake; ewake = 1;
if ((unsigned long)key & POLLFREE) {
/*
* If we race with ep_remove_wait_queue() it can miss
* ->whead = NULL and do another remove_wait_queue() after
* us, so we can't use __remove_wait_queue().
*/
list_del_init(&wait->entry);
/*
* ->whead != NULL protects us from the race with ep_free()
* or ep_remove(), ep_remove_wait_queue() takes whead->lock
* held by the caller. Once we nullify it, nothing protects
* ep/epi or even wait.
*/
smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL);
}
return 1; return ewake;
} }
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment