Re: [PATCH 01/19] configure: add --enable-tsan flag + fiber annotations for coroutine-ucontext
Hi,
On Wed, 17 Jun 2020 at 10:24, Stefan Hajnoczi wrote:
>
> On Fri, May 22, 2020 at 12:07:37PM -0400, Robert Foley wrote:
> > +#define UC_DEBUG 0
> > +#if UC_DEBUG && defined(CONFIG_TSAN)
> > +#define UC_TRACE(fmt, ...) fprintf(stderr, "%s:%d:%p " fmt "\n", \
> > +__func__, __LINE__, __tsan_get_current_fiber(), ##__VA_ARGS__);
> > +#else
> > +#define UC_TRACE(fmt, ...)
> > +#endif
>
> QEMU has tracing support, see docs/devel/tracing.txt. These fprintfs
> should be trace events defined in the util/trace-events file.
Thanks for the details. We removed this tracing in a later patch.
https://lists.gnu.org/archive/html/qemu-devel/2020-06/msg02506.html
>
> > +
> > /**
> > * Per-thread coroutine bookkeeping
> > */
> > @@ -65,7 +80,20 @@ union cc_arg {
> > int i[2];
> > };
> >
> > -static void finish_switch_fiber(void *fake_stack_save)
> > +/* QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. */
> > +static inline __attribute__((always_inline))
>
> Please document why always_inline is necessary here and in other
> functions. Is it for performance or because the __tsan_*() functions
> need to be called from a the parent function?
We will look into this and add documentation here or (if it is no
longer needed),
remove the inline.
> > -static void start_switch_fiber(void **fake_stack_save,
> > - const void *bottom, size_t size)
> > +static inline __attribute__((always_inline)) void start_switch_fiber(
> > +CoroutineAction action, void **fake_stack_save,
> > +const void *bottom, size_t size, void *new_fiber)
> > {
> > #ifdef CONFIG_ASAN
> > -__sanitizer_start_switch_fiber(fake_stack_save, bottom, size);
> > +if (action == COROUTINE_TERMINATE) {
> > +__sanitizer_start_switch_fiber(
> > +action == COROUTINE_TERMINATE ? NULL : fake_stack_save,
>
> The if statement already checks action == COROUTINE_TERMINATE, why is it
> being checked again?
>
> I think the old behavior can be retained by dropping the if statement
> like this:
>
> __sanitizer_start_switch_fiber(action == COROUTINE_TERMINATE ?
> NULL : fake_stack_save,
> bottom, size);
>
> > +bottom, size);
Good point. We did change this by dropping the if in a later patch.
https://lists.gnu.org/archive/html/qemu-devel/2020-06/msg02506.html
> > +}
> > +#endif
> > +#ifdef CONFIG_TSAN
> > +void *curr_fiber =
> > +__tsan_get_current_fiber();
> > +__tsan_acquire(curr_fiber);
> > +
> > +UC_TRACE("Current fiber: %p.", curr_fiber);
> > +*fake_stack_save = curr_fiber;
> > +UC_TRACE("Switch to fiber %p", new_fiber);
> > +__tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */
> > #endif
> > }
>
> Please split start_switch_fiber() into two functions:
> start_switch_fiber_asan() and start_switch_fiber_tsan(). That way the
> asan- and tsan-specific arguments can be kept separate and the
> co->tsan_* fields only need to be compiled in when CONFIG_TSAN is
> defined.
>
> For example:
>
> static inline __attribute__((always_inline))
> void start_switch_fiber_tsan(void **fake_stack_save,
>CoroutineUContext *co,
> bool caller)
> {
> #ifdef CONFIG_TSAN
> void *new_fiber = caller ?
> co->tsan_caller_fiber :
> co->tsan_co_fiber;
> void *curr_fiber = __tsan_get_current_fiber();
> __tsan_acquire(curr_fiber);
>
> UC_TRACE("Current fiber: %p.", curr_fiber);
> *fake_stack_save = curr_fiber;
> UC_TRACE("Switch to fiber %p", new_fiber);
> __tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */
> #endif
> }
>
> This does two things:
> 1. Unrelated ASAN and TSAN code is separate and each function only
>has arguments that are actually needed.
> 2. The co->tsan_caller_fiber and co->tsan_co_fiber fields are only
>access from within #ifdef CONFIG_TSAN.
Makes sense, we will make these changes and submit a cleanup patch.
Thanks & Regards,
-Rob
Re: [PATCH 01/19] configure: add --enable-tsan flag + fiber annotations for coroutine-ucontext
Stefan Hajnoczi writes: > On Fri, May 22, 2020 at 12:07:37PM -0400, Robert Foley wrote: >> +#define UC_DEBUG 0 >> +#if UC_DEBUG && defined(CONFIG_TSAN) >> +#define UC_TRACE(fmt, ...) fprintf(stderr, "%s:%d:%p " fmt "\n", \ >> +__func__, __LINE__, __tsan_get_current_fiber(), ##__VA_ARGS__); >> +#else >> +#define UC_TRACE(fmt, ...) >> +#endif > > QEMU has tracing support, see docs/devel/tracing.txt. These fprintfs > should be trace events defined in the util/trace-events file. These got dropped on the final version. You might want to check against the version I pulled: Subject: [PATCH v3 00/13] Add Thread Sanitizer support to QEMU Date: Tue, 9 Jun 2020 16:07:25 -0400 Message-Id: <20200609200738.445-1-robert.fo...@linaro.org> -- Alex Bennée
Re: [PATCH 01/19] configure: add --enable-tsan flag + fiber annotations for coroutine-ucontext
On Fri, May 22, 2020 at 12:07:37PM -0400, Robert Foley wrote:
> +#define UC_DEBUG 0
> +#if UC_DEBUG && defined(CONFIG_TSAN)
> +#define UC_TRACE(fmt, ...) fprintf(stderr, "%s:%d:%p " fmt "\n", \
> +__func__, __LINE__, __tsan_get_current_fiber(), ##__VA_ARGS__);
> +#else
> +#define UC_TRACE(fmt, ...)
> +#endif
QEMU has tracing support, see docs/devel/tracing.txt. These fprintfs
should be trace events defined in the util/trace-events file.
> +
> /**
> * Per-thread coroutine bookkeeping
> */
> @@ -65,7 +80,20 @@ union cc_arg {
> int i[2];
> };
>
> -static void finish_switch_fiber(void *fake_stack_save)
> +/* QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. */
> +static inline __attribute__((always_inline))
Please document why always_inline is necessary here and in other
functions. Is it for performance or because the __tsan_*() functions
need to be called from a the parent function?
> +void on_new_fiber(CoroutineUContext *co)
> +{
> +#ifdef CONFIG_TSAN
> +co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */
> +co->tsan_caller_fiber = __tsan_get_current_fiber();
> +UC_TRACE("Create new TSAN co fiber. co: %p co fiber: %p caller fiber: %p
> ",
> + co, co->tsan_co_fiber, co->tsan_caller_fiber);
> +#endif
> +}
> +
> +static inline __attribute__((always_inline))
> +void finish_switch_fiber(void *fake_stack_save)
> {
> #ifdef CONFIG_ASAN
> const void *bottom_old;
> @@ -78,18 +106,40 @@ static void finish_switch_fiber(void *fake_stack_save)
> leader.stack_size = size_old;
> }
> #endif
> +#ifdef CONFIG_TSAN
> +if (fake_stack_save) {
> +__tsan_release(fake_stack_save);
> +__tsan_switch_to_fiber(fake_stack_save, 0); /* 0=synchronize */
> +}
> +#endif
> }
>
> -static void start_switch_fiber(void **fake_stack_save,
> - const void *bottom, size_t size)
> +static inline __attribute__((always_inline)) void start_switch_fiber(
> +CoroutineAction action, void **fake_stack_save,
> +const void *bottom, size_t size, void *new_fiber)
> {
> #ifdef CONFIG_ASAN
> -__sanitizer_start_switch_fiber(fake_stack_save, bottom, size);
> +if (action == COROUTINE_TERMINATE) {
> +__sanitizer_start_switch_fiber(
> +action == COROUTINE_TERMINATE ? NULL : fake_stack_save,
The if statement already checks action == COROUTINE_TERMINATE, why is it
being checked again?
I think the old behavior can be retained by dropping the if statement
like this:
__sanitizer_start_switch_fiber(action == COROUTINE_TERMINATE ?
NULL : fake_stack_save,
bottom, size);
> +bottom, size);
> +}
> +#endif
> +#ifdef CONFIG_TSAN
> +void *curr_fiber =
> +__tsan_get_current_fiber();
> +__tsan_acquire(curr_fiber);
> +
> +UC_TRACE("Current fiber: %p.", curr_fiber);
> +*fake_stack_save = curr_fiber;
> +UC_TRACE("Switch to fiber %p", new_fiber);
> +__tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */
> #endif
> }
Please split start_switch_fiber() into two functions:
start_switch_fiber_asan() and start_switch_fiber_tsan(). That way the
asan- and tsan-specific arguments can be kept separate and the
co->tsan_* fields only need to be compiled in when CONFIG_TSAN is
defined.
For example:
static inline __attribute__((always_inline))
void start_switch_fiber_tsan(void **fake_stack_save,
CoroutineUContext *co,
bool caller)
{
#ifdef CONFIG_TSAN
void *new_fiber = caller ?
co->tsan_caller_fiber :
co->tsan_co_fiber;
void *curr_fiber = __tsan_get_current_fiber();
__tsan_acquire(curr_fiber);
UC_TRACE("Current fiber: %p.", curr_fiber);
*fake_stack_save = curr_fiber;
UC_TRACE("Switch to fiber %p", new_fiber);
__tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */
#endif
}
This does two things:
1. Unrelated ASAN and TSAN code is separate and each function only
has arguments that are actually needed.
2. The co->tsan_caller_fiber and co->tsan_co_fiber fields are only
access from within #ifdef CONFIG_TSAN.
signature.asc
Description: PGP signature
Re: [PATCH 01/19] configure: add --enable-tsan flag + fiber annotations for coroutine-ucontext
On Sat, 23 May 2020 at 12:55, Philippe Mathieu-Daudé wrote: > > Hi Robert, > > On 5/22/20 6:07 PM, Robert Foley wrote: > > From: Lingfeng Yang > > > > We tried running QEMU under tsan in 2016, but tsan's lack of support for > > longjmp-based fibers was a blocker: > > @@ -6277,6 +6304,14 @@ if test "$have_asan" = "yes"; then > > "Without code annotation, the report may be inferior." > >fi > > fi > > +if test "$have_tsan" = "yes" ; then > > + if test "$have_tsan_iface_fiber" = "yes" ; then > > +QEMU_CFLAGS="-fsanitize=thread $QEMU_CFLAGS" > > +QEMU_LDFLAGS="-fsanitize=thread $QEMU_LDFLAGS" > > + else > > +echo "Cannot enable TSAN due to missing fiber annotation interface." > > I tried your series and there were no changes anywhere, then I looked at > how TSan work, started to debug, to finally realize my build was not > using TSan (clang8). Please use to something such: > > if test "$tsan" = "yes" ; then > error_exit "Cannot enable TSAN due to missing fiber" \ >"annotation interface." > fi This is a good point. Will make these changes. Thanks & Regards, -Rob
Re: [PATCH 01/19] configure: add --enable-tsan flag + fiber annotations for coroutine-ucontext
Hi Robert,
On 5/22/20 6:07 PM, Robert Foley wrote:
> From: Lingfeng Yang
>
> We tried running QEMU under tsan in 2016, but tsan's lack of support for
> longjmp-based fibers was a blocker:
> https://groups.google.com/forum/#!topic/thread-sanitizer/se0YuzfWazw
>
> Fortunately, thread sanitizer gained fiber support in early 2019:
> https://reviews.llvm.org/D54889
>
> This patch brings tsan support upstream by importing the patch that annotated
> QEMU's coroutines as tsan fibers in Android's QEMU fork:
> https://android-review.googlesource.com/c/platform/external/qemu/+/844675
>
> Tested with '--enable-tsan --cc=clang-9 --cxx=clang++-9 --disable-werror'
> configure flags.
>
> Signed-off-by: Lingfeng Yang
> Signed-off-by: Emilio G. Cota
> [cota: minor modifications + configure changes]
> Signed-off-by: Robert Foley
> [RF: minor changes to clean up checkpatch warnings/errors]
> ---
> configure | 39
> util/coroutine-ucontext.c | 97 +++
> 2 files changed, 127 insertions(+), 9 deletions(-)
>
> diff --git a/configure b/configure
> index 26084fc53a..c95c54fb48 100755
> --- a/configure
> +++ b/configure
> @@ -395,6 +395,7 @@ gprof="no"
> debug_tcg="no"
> debug="no"
> sanitizers="no"
> +tsan="no"
> fortify_source=""
> strip_opt="yes"
> tcg_interpreter="no"
> @@ -1150,6 +1151,10 @@ for opt do
>;;
>--disable-sanitizers) sanitizers="no"
>;;
> + --enable-tsan) tsan="yes"
> + ;;
> + --disable-tsan) tsan="no"
> + ;;
>--enable-sparse) sparse="yes"
>;;
>--disable-sparse) sparse="no"
> @@ -1750,6 +1755,7 @@ Advanced options (experts only):
>--with-pkgversion=VERS use specified string as sub-version of the package
>--enable-debug enable common debug build options
>--enable-sanitizers enable default sanitizers
> + --enable-tsanenable thread sanitizer
>--disable-strip disable stripping binaries
>--disable-werror disable compilation abort on warning
>--disable-stack-protector disable compiler-provided stack protection
> @@ -6176,6 +6182,27 @@ if test "$fuzzing" = "yes" ; then
>fi
> fi
>
> +# Thread sanitizer is, for now, much noisier than the other sanitizers;
> +# keep it separate until that is not the case.
> +have_tsan=no
> +have_tsan_iface_fiber=no
> +if test "$tsan" = "yes" ; then
> + write_c_skeleton
> + if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then
> + have_tsan=yes
> + fi
> + cat > $TMPC << EOF
> +#include
> +int main(void) {
> + __tsan_create_fiber(0);
> + return 0;
> +}
> +EOF
> + if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then
> + have_tsan_iface_fiber=yes
> + fi
> +fi
> +
> ##
> # check for libpmem
>
> @@ -6277,6 +6304,14 @@ if test "$have_asan" = "yes"; then
> "Without code annotation, the report may be inferior."
>fi
> fi
> +if test "$have_tsan" = "yes" ; then
> + if test "$have_tsan_iface_fiber" = "yes" ; then
> +QEMU_CFLAGS="-fsanitize=thread $QEMU_CFLAGS"
> +QEMU_LDFLAGS="-fsanitize=thread $QEMU_LDFLAGS"
> + else
> +echo "Cannot enable TSAN due to missing fiber annotation interface."
I tried your series and there were no changes anywhere, then I looked at
how TSan work, started to debug, to finally realize my build was not
using TSan (clang8). Please use to something such:
if test "$tsan" = "yes" ; then
error_exit "Cannot enable TSAN due to missing fiber" \
"annotation interface."
fi
> + fi
> +fi
> if test "$have_ubsan" = "yes"; then
>QEMU_CFLAGS="-fsanitize=undefined $QEMU_CFLAGS"
>QEMU_LDFLAGS="-fsanitize=undefined $QEMU_LDFLAGS"
> @@ -7365,6 +7400,10 @@ if test "$have_asan_iface_fiber" = "yes" ; then
> echo "CONFIG_ASAN_IFACE_FIBER=y" >> $config_host_mak
> fi
>
> +if test "$have_tsan" = "yes" && test "$have_tsan_iface_fiber" = "yes" ; then
> +echo "CONFIG_TSAN=y" >> $config_host_mak
> +fi
> +
> if test "$has_environ" = "yes" ; then
>echo "CONFIG_HAS_ENVIRON=y" >> $config_host_mak
> fi
[...]
[PATCH 01/19] configure: add --enable-tsan flag + fiber annotations for coroutine-ucontext
From: Lingfeng Yang
We tried running QEMU under tsan in 2016, but tsan's lack of support for
longjmp-based fibers was a blocker:
https://groups.google.com/forum/#!topic/thread-sanitizer/se0YuzfWazw
Fortunately, thread sanitizer gained fiber support in early 2019:
https://reviews.llvm.org/D54889
This patch brings tsan support upstream by importing the patch that annotated
QEMU's coroutines as tsan fibers in Android's QEMU fork:
https://android-review.googlesource.com/c/platform/external/qemu/+/844675
Tested with '--enable-tsan --cc=clang-9 --cxx=clang++-9 --disable-werror'
configure flags.
Signed-off-by: Lingfeng Yang
Signed-off-by: Emilio G. Cota
[cota: minor modifications + configure changes]
Signed-off-by: Robert Foley
[RF: minor changes to clean up checkpatch warnings/errors]
---
configure | 39
util/coroutine-ucontext.c | 97 +++
2 files changed, 127 insertions(+), 9 deletions(-)
diff --git a/configure b/configure
index 26084fc53a..c95c54fb48 100755
--- a/configure
+++ b/configure
@@ -395,6 +395,7 @@ gprof="no"
debug_tcg="no"
debug="no"
sanitizers="no"
+tsan="no"
fortify_source=""
strip_opt="yes"
tcg_interpreter="no"
@@ -1150,6 +1151,10 @@ for opt do
;;
--disable-sanitizers) sanitizers="no"
;;
+ --enable-tsan) tsan="yes"
+ ;;
+ --disable-tsan) tsan="no"
+ ;;
--enable-sparse) sparse="yes"
;;
--disable-sparse) sparse="no"
@@ -1750,6 +1755,7 @@ Advanced options (experts only):
--with-pkgversion=VERS use specified string as sub-version of the package
--enable-debug enable common debug build options
--enable-sanitizers enable default sanitizers
+ --enable-tsanenable thread sanitizer
--disable-strip disable stripping binaries
--disable-werror disable compilation abort on warning
--disable-stack-protector disable compiler-provided stack protection
@@ -6176,6 +6182,27 @@ if test "$fuzzing" = "yes" ; then
fi
fi
+# Thread sanitizer is, for now, much noisier than the other sanitizers;
+# keep it separate until that is not the case.
+have_tsan=no
+have_tsan_iface_fiber=no
+if test "$tsan" = "yes" ; then
+ write_c_skeleton
+ if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then
+ have_tsan=yes
+ fi
+ cat > $TMPC << EOF
+#include
+int main(void) {
+ __tsan_create_fiber(0);
+ return 0;
+}
+EOF
+ if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then
+ have_tsan_iface_fiber=yes
+ fi
+fi
+
##
# check for libpmem
@@ -6277,6 +6304,14 @@ if test "$have_asan" = "yes"; then
"Without code annotation, the report may be inferior."
fi
fi
+if test "$have_tsan" = "yes" ; then
+ if test "$have_tsan_iface_fiber" = "yes" ; then
+QEMU_CFLAGS="-fsanitize=thread $QEMU_CFLAGS"
+QEMU_LDFLAGS="-fsanitize=thread $QEMU_LDFLAGS"
+ else
+echo "Cannot enable TSAN due to missing fiber annotation interface."
+ fi
+fi
if test "$have_ubsan" = "yes"; then
QEMU_CFLAGS="-fsanitize=undefined $QEMU_CFLAGS"
QEMU_LDFLAGS="-fsanitize=undefined $QEMU_LDFLAGS"
@@ -7365,6 +7400,10 @@ if test "$have_asan_iface_fiber" = "yes" ; then
echo "CONFIG_ASAN_IFACE_FIBER=y" >> $config_host_mak
fi
+if test "$have_tsan" = "yes" && test "$have_tsan_iface_fiber" = "yes" ; then
+echo "CONFIG_TSAN=y" >> $config_host_mak
+fi
+
if test "$has_environ" = "yes" ; then
echo "CONFIG_HAS_ENVIRON=y" >> $config_host_mak
fi
diff --git a/util/coroutine-ucontext.c b/util/coroutine-ucontext.c
index bd593e61bc..a3dc78e67a 100644
--- a/util/coroutine-ucontext.c
+++ b/util/coroutine-ucontext.c
@@ -37,18 +37,33 @@
#endif
#endif
+#ifdef CONFIG_TSAN
+#include
+#endif
+
typedef struct {
Coroutine base;
void *stack;
size_t stack_size;
sigjmp_buf env;
+void *tsan_co_fiber;
+void *tsan_caller_fiber;
+
#ifdef CONFIG_VALGRIND_H
unsigned int valgrind_stack_id;
#endif
} CoroutineUContext;
+#define UC_DEBUG 0
+#if UC_DEBUG && defined(CONFIG_TSAN)
+#define UC_TRACE(fmt, ...) fprintf(stderr, "%s:%d:%p " fmt "\n", \
+__func__, __LINE__, __tsan_get_current_fiber(), ##__VA_ARGS__);
+#else
+#define UC_TRACE(fmt, ...)
+#endif
+
/**
* Per-thread coroutine bookkeeping
*/
@@ -65,7 +80,20 @@ union cc_arg {
int i[2];
};
-static void finish_switch_fiber(void *fake_stack_save)
+/* QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. */
+static inline __attribute__((always_inline))
+void on_new_fiber(CoroutineUContext *co)
+{
+#ifdef CONFIG_TSAN
+co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */
+co->tsan_caller_fiber = __tsan_get_current_fiber();
+UC_TRACE("Create new TSAN co fiber. co: %p co fiber: %p caller fiber: %p ",
+ co, co->tsan_co_fiber, co->tsan_caller_fiber);
+#endif
+}
+
+static inline __attribute__((always_inline))
+void finish_switch_fiber(void
