[PATCH 4.14 121/173] sched/core: Use smp_mb() in wake_woken_function()
4.14-stable review patch. If anyone has any objections, please let me know.
--
From: Andrea Parri
[ Upstream commit 76e079fefc8f62bd9b2cd2950814d1ee806e31a5 ]
wake_woken_function() synchronizes with wait_woken() as follows:
[wait_woken] [wake_woken_function]
entry->flags &= ~wq_flag_woken;condition = true;
smp_mb(); smp_wmb();
if (condition) wq_entry->flags |= wq_flag_woken;
break;
This commit replaces the above smp_wmb() with an smp_mb() in order to
guarantee that either wait_woken() sees the wait condition being true
or the store to wq_entry->flags in woken_wake_function() follows the
store in wait_woken() in the coherence order (so that the former can
eventually be observed by wait_woken()).
The commit also fixes a comment associated to set_current_state() in
wait_woken(): the comment pairs the barrier in set_current_state() to
the above smp_wmb(), while the actual pairing involves the barrier in
set_current_state() and the barrier executed by the try_to_wake_up()
in wake_woken_function().
Signed-off-by: Andrea Parri
Signed-off-by: Paul E. McKenney
Acked-by: Peter Zijlstra (Intel)
Cc: Linus Torvalds
Cc: Peter Zijlstra
Cc: Thomas Gleixner
Cc: aki...@gmail.com
Cc: boqun.f...@gmail.com
Cc: dhowe...@redhat.com
Cc: j.algl...@ucl.ac.uk
Cc: linux-a...@vger.kernel.org
Cc: luc.maran...@inria.fr
Cc: npig...@gmail.com
Cc: parri.and...@gmail.com
Cc: st...@rowland.harvard.edu
Cc: will.dea...@arm.com
Link:
http://lkml.kernel.org/r/20180716180605.16115-10-paul...@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar
Signed-off-by: Sasha Levin
Signed-off-by: Greg Kroah-Hartman
---
kernel/sched/wait.c | 47 +--
1 file changed, 21 insertions(+), 26 deletions(-)
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -395,35 +395,36 @@ static inline bool is_kthread_should_sto
* if (condition)
* break;
*
- * p->state = mode;condition = true;
- * smp_mb(); // A smp_wmb(); // C
- * if (!wq_entry->flags & WQ_FLAG_WOKEN) wq_entry->flags |=
WQ_FLAG_WOKEN;
- * schedule() try_to_wake_up();
- * p->state = TASK_RUNNING;~~
- * wq_entry->flags &= ~WQ_FLAG_WOKEN; condition = true;
- * smp_mb() // B smp_wmb(); // C
- * wq_entry->flags |=
WQ_FLAG_WOKEN;
- * }
- * remove_wait_queue(_head, );
+ * // in wait_woken() // in woken_wake_function()
*
+ * p->state = mode;wq_entry->flags |=
WQ_FLAG_WOKEN;
+ * smp_mb(); // A try_to_wake_up():
+ * if (!(wq_entry->flags & WQ_FLAG_WOKEN))
+ * schedule() if (p->state & mode)
+ * p->state = TASK_RUNNING; p->state =
TASK_RUNNING;
+ * wq_entry->flags &= ~WQ_FLAG_WOKEN; ~~
+ * smp_mb(); // B condition = true;
+ * } smp_mb(); // C
+ * remove_wait_queue(_head, ); wq_entry->flags |=
WQ_FLAG_WOKEN;
*/
long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout)
{
- set_current_state(mode); /* A */
/*
-* The above implies an smp_mb(), which matches with the smp_wmb() from
-* woken_wake_function() such that if we observe WQ_FLAG_WOKEN we must
-* also observe all state before the wakeup.
+* The below executes an smp_mb(), which matches with the full barrier
+* executed by the try_to_wake_up() in woken_wake_function() such that
+* either we see the store to wq_entry->flags in woken_wake_function()
+* or woken_wake_function() sees our store to current->state.
*/
+ set_current_state(mode); /* A */
if (!(wq_entry->flags & WQ_FLAG_WOKEN) && !is_kthread_should_stop())
timeout = schedule_timeout(timeout);
__set_current_state(TASK_RUNNING);
/*
-* The below implies an smp_mb(), it too pairs with the smp_wmb() from
-* woken_wake_function() such that we must either observe the wait
-* condition being true _OR_ WQ_FLAG_WOKEN such that we will not miss
-* an event.
+* The below executes an smp_mb(), which matches with the smp_mb() (C)
+* in woken_wake_function() such that either we see the wait condition
+* being true or the store to wq_entry->flags in woken_wake_function()
+* follows ours in the coherence order.
*/
smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN); /* B */
@@ -433,14 +434,8 @@ EXPORT_SYMBOL(wait_woken);
int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int
[PATCH 4.14 121/173] sched/core: Use smp_mb() in wake_woken_function()
4.14-stable review patch. If anyone has any objections, please let me know.
--
From: Andrea Parri
[ Upstream commit 76e079fefc8f62bd9b2cd2950814d1ee806e31a5 ]
wake_woken_function() synchronizes with wait_woken() as follows:
[wait_woken] [wake_woken_function]
entry->flags &= ~wq_flag_woken;condition = true;
smp_mb(); smp_wmb();
if (condition) wq_entry->flags |= wq_flag_woken;
break;
This commit replaces the above smp_wmb() with an smp_mb() in order to
guarantee that either wait_woken() sees the wait condition being true
or the store to wq_entry->flags in woken_wake_function() follows the
store in wait_woken() in the coherence order (so that the former can
eventually be observed by wait_woken()).
The commit also fixes a comment associated to set_current_state() in
wait_woken(): the comment pairs the barrier in set_current_state() to
the above smp_wmb(), while the actual pairing involves the barrier in
set_current_state() and the barrier executed by the try_to_wake_up()
in wake_woken_function().
Signed-off-by: Andrea Parri
Signed-off-by: Paul E. McKenney
Acked-by: Peter Zijlstra (Intel)
Cc: Linus Torvalds
Cc: Peter Zijlstra
Cc: Thomas Gleixner
Cc: aki...@gmail.com
Cc: boqun.f...@gmail.com
Cc: dhowe...@redhat.com
Cc: j.algl...@ucl.ac.uk
Cc: linux-a...@vger.kernel.org
Cc: luc.maran...@inria.fr
Cc: npig...@gmail.com
Cc: parri.and...@gmail.com
Cc: st...@rowland.harvard.edu
Cc: will.dea...@arm.com
Link:
http://lkml.kernel.org/r/20180716180605.16115-10-paul...@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar
Signed-off-by: Sasha Levin
Signed-off-by: Greg Kroah-Hartman
---
kernel/sched/wait.c | 47 +--
1 file changed, 21 insertions(+), 26 deletions(-)
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -395,35 +395,36 @@ static inline bool is_kthread_should_sto
* if (condition)
* break;
*
- * p->state = mode;condition = true;
- * smp_mb(); // A smp_wmb(); // C
- * if (!wq_entry->flags & WQ_FLAG_WOKEN) wq_entry->flags |=
WQ_FLAG_WOKEN;
- * schedule() try_to_wake_up();
- * p->state = TASK_RUNNING;~~
- * wq_entry->flags &= ~WQ_FLAG_WOKEN; condition = true;
- * smp_mb() // B smp_wmb(); // C
- * wq_entry->flags |=
WQ_FLAG_WOKEN;
- * }
- * remove_wait_queue(_head, );
+ * // in wait_woken() // in woken_wake_function()
*
+ * p->state = mode;wq_entry->flags |=
WQ_FLAG_WOKEN;
+ * smp_mb(); // A try_to_wake_up():
+ * if (!(wq_entry->flags & WQ_FLAG_WOKEN))
+ * schedule() if (p->state & mode)
+ * p->state = TASK_RUNNING; p->state =
TASK_RUNNING;
+ * wq_entry->flags &= ~WQ_FLAG_WOKEN; ~~
+ * smp_mb(); // B condition = true;
+ * } smp_mb(); // C
+ * remove_wait_queue(_head, ); wq_entry->flags |=
WQ_FLAG_WOKEN;
*/
long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout)
{
- set_current_state(mode); /* A */
/*
-* The above implies an smp_mb(), which matches with the smp_wmb() from
-* woken_wake_function() such that if we observe WQ_FLAG_WOKEN we must
-* also observe all state before the wakeup.
+* The below executes an smp_mb(), which matches with the full barrier
+* executed by the try_to_wake_up() in woken_wake_function() such that
+* either we see the store to wq_entry->flags in woken_wake_function()
+* or woken_wake_function() sees our store to current->state.
*/
+ set_current_state(mode); /* A */
if (!(wq_entry->flags & WQ_FLAG_WOKEN) && !is_kthread_should_stop())
timeout = schedule_timeout(timeout);
__set_current_state(TASK_RUNNING);
/*
-* The below implies an smp_mb(), it too pairs with the smp_wmb() from
-* woken_wake_function() such that we must either observe the wait
-* condition being true _OR_ WQ_FLAG_WOKEN such that we will not miss
-* an event.
+* The below executes an smp_mb(), which matches with the smp_mb() (C)
+* in woken_wake_function() such that either we see the wait condition
+* being true or the store to wq_entry->flags in woken_wake_function()
+* follows ours in the coherence order.
*/
smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN); /* B */
@@ -433,14 +434,8 @@ EXPORT_SYMBOL(wait_woken);
int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int
