Re: [PATCH RFC 3/7] util: Introduce ThreadContext user-creatable object
On 7/21/22 14:07, David Hildenbrand wrote:
> Setting the CPU affinity of QEMU threads is a bit problematic, because
> QEMU doesn't always have permissions to set the CPU affinity itself,
> for example, with seccomp after initialized by QEMU:
> -sandbox enable=on,resourcecontrol=deny
>
> While upper layers are already aware how to handl;e CPU affinities for
> long-lived threads like iothreads or vcpu threads, especially short-lived
> threads, as used for memory-backend preallocation, are more involved to
> handle. These threads are created on demand and upper layers are not even
> able to identify and configure them.
>
> Introduce the concept of a ThreadContext, that is essentially a thread
> used for creating new threads. All threads created via that context
> thread inherit the configured CPU affinity. Consequently, it's
> sufficient to create a ThreadContext and configure it once, and have all
> threads created via that ThreadContext inherit the same CPU affinity.
>
> The CPU affinity of a ThreadContext can be configured two ways:
>
> (1) Obtaining the thread id via the "thread-id" property and setting the
> CPU affinity manually.
>
> (2) Setting the "cpu-affinity" property and letting QEMU try set the
> CPU affinity itself. This will fail if QEMU doesn't have permissions
> to do so anymore after seccomp was initialized.
>
> A ThreadContext can be reused, simply be reconfiguring the CPU affinity.
>
> Signed-off-by: David Hildenbrand
> ---
> include/qemu/thread-context.h | 58 +++
> qapi/qom.json | 16 ++
> util/meson.build | 1 +
> util/oslib-posix.c| 1 +
> util/thread-context.c | 279 ++
> 5 files changed, 355 insertions(+)
> create mode 100644 include/qemu/thread-context.h
> create mode 100644 util/thread-context.c
>
> diff --git a/include/qemu/thread-context.h b/include/qemu/thread-context.h
> new file mode 100644
> index 00..c799cbe7a1
> --- /dev/null
> +++ b/include/qemu/thread-context.h
> @@ -0,0 +1,58 @@
> +/*
> + * QEMU Thread Context
> + *
> + * Copyright Red Hat Inc., 2022
> + *
> + * Authors:
> + * David Hildenbrand
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + *
> + */
> +
> +#ifndef SYSEMU_THREAD_CONTEXT_H
> +#define SYSEMU_THREAD_CONTEXT_H
> +
> +#include "qapi/qapi-types-machine.h"
> +#include "qemu/thread.h"
> +#include "qom/object.h"
> +
> +#define TYPE_THREAD_CONTEXT "thread-context"
> +OBJECT_DECLARE_TYPE(ThreadContext, ThreadContextClass,
> +THREAD_CONTEXT)
> +
> +struct ThreadContextClass {
> +ObjectClass parent_class;
> +};
> +
> +struct ThreadContext {
> +/* private */
> +Object parent;
> +
> +/* private */
> +unsigned int thread_id;
> +QemuThread thread;
> +
> +/* Semaphore to wait for context thread action. */
> +QemuSemaphore sem;
> +/* Semaphore to wait for action in context thread. */
> +QemuSemaphore sem_thread;
> +/* Mutex to synchronize requests. */
> +QemuMutex mutex;
> +
> +/* Commands for the thread to execute. */
> +int thread_cmd;
> +void *thread_cmd_data;
> +
> +/* CPU affinity bitmap used for initialization. */
> +unsigned long *init_cpu_bitmap;
> +int init_cpu_nbits;
> +};
> +
> +void thread_context_create_thread(ThreadContext *tc, QemuThread *thread,
> + const char *name,
> + void *(*start_routine)(void *), void *arg,
> + int mode);
> +
> +#endif /* SYSEMU_THREAD_CONTEXT_H */
> diff --git a/qapi/qom.json b/qapi/qom.json
> index 80dd419b39..4775a333ed 100644
> --- a/qapi/qom.json
> +++ b/qapi/qom.json
> @@ -830,6 +830,20 @@
> 'reduced-phys-bits': 'uint32',
> '*kernel-hashes': 'bool' } }
>
> +##
> +# @ThreadContextProperties:
> +#
> +# Properties for thread context objects.
> +#
> +# @cpu-affinity: the CPU affinity for all threads created in the thread
> +#context (default: QEMU main thread affinity)
> +#
> +# Since: 7.2
> +##
> +{ 'struct': 'ThreadContextProperties',
> + 'data': { '*cpu-affinity': ['uint16'] } }
> +
> +
> ##
> # @ObjectType:
> #
> @@ -882,6 +896,7 @@
> { 'name': 'secret_keyring',
>'if': 'CONFIG_SECRET_KEYRING' },
> 'sev-guest',
> +'thread-context',
> 's390-pv-guest',
> 'throttle-group',
> 'tls-creds-anon',
> @@ -948,6 +963,7 @@
>'secret_keyring': { 'type': 'SecretKeyringProperties',
>'if': 'CONFIG_SECRET_KEYRING' },
>'sev-guest': 'SevGuestProperties',
> + 'thread-context': 'ThreadContextProperties',
>'throttle-group': 'ThrottleGroupProperties',
>'tls-creds-anon': 'TlsCredsAnonProperties',
>'tls-creds-psk':
[PATCH RFC 3/7] util: Introduce ThreadContext user-creatable object
Setting the CPU affinity of QEMU threads is a bit problematic, because
QEMU doesn't always have permissions to set the CPU affinity itself,
for example, with seccomp after initialized by QEMU:
-sandbox enable=on,resourcecontrol=deny
While upper layers are already aware how to handl;e CPU affinities for
long-lived threads like iothreads or vcpu threads, especially short-lived
threads, as used for memory-backend preallocation, are more involved to
handle. These threads are created on demand and upper layers are not even
able to identify and configure them.
Introduce the concept of a ThreadContext, that is essentially a thread
used for creating new threads. All threads created via that context
thread inherit the configured CPU affinity. Consequently, it's
sufficient to create a ThreadContext and configure it once, and have all
threads created via that ThreadContext inherit the same CPU affinity.
The CPU affinity of a ThreadContext can be configured two ways:
(1) Obtaining the thread id via the "thread-id" property and setting the
CPU affinity manually.
(2) Setting the "cpu-affinity" property and letting QEMU try set the
CPU affinity itself. This will fail if QEMU doesn't have permissions
to do so anymore after seccomp was initialized.
A ThreadContext can be reused, simply be reconfiguring the CPU affinity.
Signed-off-by: David Hildenbrand
---
include/qemu/thread-context.h | 58 +++
qapi/qom.json | 16 ++
util/meson.build | 1 +
util/oslib-posix.c| 1 +
util/thread-context.c | 279 ++
5 files changed, 355 insertions(+)
create mode 100644 include/qemu/thread-context.h
create mode 100644 util/thread-context.c
diff --git a/include/qemu/thread-context.h b/include/qemu/thread-context.h
new file mode 100644
index 00..c799cbe7a1
--- /dev/null
+++ b/include/qemu/thread-context.h
@@ -0,0 +1,58 @@
+/*
+ * QEMU Thread Context
+ *
+ * Copyright Red Hat Inc., 2022
+ *
+ * Authors:
+ * David Hildenbrand
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef SYSEMU_THREAD_CONTEXT_H
+#define SYSEMU_THREAD_CONTEXT_H
+
+#include "qapi/qapi-types-machine.h"
+#include "qemu/thread.h"
+#include "qom/object.h"
+
+#define TYPE_THREAD_CONTEXT "thread-context"
+OBJECT_DECLARE_TYPE(ThreadContext, ThreadContextClass,
+THREAD_CONTEXT)
+
+struct ThreadContextClass {
+ObjectClass parent_class;
+};
+
+struct ThreadContext {
+/* private */
+Object parent;
+
+/* private */
+unsigned int thread_id;
+QemuThread thread;
+
+/* Semaphore to wait for context thread action. */
+QemuSemaphore sem;
+/* Semaphore to wait for action in context thread. */
+QemuSemaphore sem_thread;
+/* Mutex to synchronize requests. */
+QemuMutex mutex;
+
+/* Commands for the thread to execute. */
+int thread_cmd;
+void *thread_cmd_data;
+
+/* CPU affinity bitmap used for initialization. */
+unsigned long *init_cpu_bitmap;
+int init_cpu_nbits;
+};
+
+void thread_context_create_thread(ThreadContext *tc, QemuThread *thread,
+ const char *name,
+ void *(*start_routine)(void *), void *arg,
+ int mode);
+
+#endif /* SYSEMU_THREAD_CONTEXT_H */
diff --git a/qapi/qom.json b/qapi/qom.json
index 80dd419b39..4775a333ed 100644
--- a/qapi/qom.json
+++ b/qapi/qom.json
@@ -830,6 +830,20 @@
'reduced-phys-bits': 'uint32',
'*kernel-hashes': 'bool' } }
+##
+# @ThreadContextProperties:
+#
+# Properties for thread context objects.
+#
+# @cpu-affinity: the CPU affinity for all threads created in the thread
+#context (default: QEMU main thread affinity)
+#
+# Since: 7.2
+##
+{ 'struct': 'ThreadContextProperties',
+ 'data': { '*cpu-affinity': ['uint16'] } }
+
+
##
# @ObjectType:
#
@@ -882,6 +896,7 @@
{ 'name': 'secret_keyring',
'if': 'CONFIG_SECRET_KEYRING' },
'sev-guest',
+'thread-context',
's390-pv-guest',
'throttle-group',
'tls-creds-anon',
@@ -948,6 +963,7 @@
'secret_keyring': { 'type': 'SecretKeyringProperties',
'if': 'CONFIG_SECRET_KEYRING' },
'sev-guest': 'SevGuestProperties',
+ 'thread-context': 'ThreadContextProperties',
'throttle-group': 'ThrottleGroupProperties',
'tls-creds-anon': 'TlsCredsAnonProperties',
'tls-creds-psk': 'TlsCredsPskProperties',
diff --git a/util/meson.build b/util/meson.build
index 5e282130df..e97cd2d779 100644
--- a/util/meson.build
+++ b/util/meson.build
@@ -1,4 +1,5 @@
util_ss.add(files('osdep.c', 'cutils.c', 'unicode.c', 'qemu-timer-common.c'))
+util_ss.add(files('thread-context.c'))
if not config_host_data.ge
