[PATCH 0/1] i386/tcg fix for IRET as used in dotnet runtime

2024-06-11 Thread Robert R. Henry
This patch fixes the i386/tcg implementation of the IRET instruction
so that IRET can return from user space to user space, as used by the
dotnet runtime to switch threads.

This fixes https://gitlab.com/qemu-project/qemu/-/issues/249

I debugged this issue 4+ years ago, and wrote this patch then.

At the time, I did not fully understand the nuances of the priority
levels in the TCG emulation of the x86, nor of the x86 itself.
I understand less now!

I do not recall exactly how I was led to the conclusion that an
unhandled page fault in kernel space was due to a bug in the code
executed in the tcg emulator for IRET. Eventually, my approach to
debugging was to modify the source for the dotnet runtime so that
immediately prior to the IRET I executed an x87 fpatan2 instruction,
knowing that no modern program used that instruction, and that there
was a single point in QEMU source code that emulated that, making it a
convenient place to put gdb breakpoints to enable further breakpoints in
the IRET emulation code.

With this change the page faults go away, and that the dotnet program
completes as expected. For the curious,
https://github.com/dotnet/runtime/blob/main/src/coreclr/pal/src/arch/amd64/context2.S#L241
shows how the dotnet runtime uses iret.

I have booted BSD, solaris and macosX with this change, and await
results for booting Windows from the Windows kernel team.

I have not tested this with other modern JITers, such as Java,
v8, or HHVM.

Robert R. Henry (1):
  i386/tcg: Allow IRET from user mode to user mode for dotnet runtime

 target/i386/tcg/seg_helper.c | 78 ++--
 1 file changed, 47 insertions(+), 31 deletions(-)

-- 
2.34.1




[PATCH 1/1] i386/tcg: Allow IRET from user mode to user mode for dotnet runtime

2024-06-11 Thread Robert R. Henry
This fixes a bug wherein i386/tcg assumed an interrupt return using
the IRET instruction was always returning from kernel mode to either
kernel mode or user mode. This assumption is violated when IRET is used
as a clever way to restore thread state, as for example in the dotnet
runtime. There, IRET returns from user mode to user mode.

This bug manifested itself as a page fault in the guest Linux kernel.

This bug appears to have been in QEMU since the beginning.

Resolves: https://gitlab.com/qemu-project/qemu/-/issues/249
Signed-off-by: Robert R. Henry 
---
 target/i386/tcg/seg_helper.c | 78 ++--
 1 file changed, 47 insertions(+), 31 deletions(-)

diff --git a/target/i386/tcg/seg_helper.c b/target/i386/tcg/seg_helper.c
index 715db1f232..815d26e61d 100644
--- a/target/i386/tcg/seg_helper.c
+++ b/target/i386/tcg/seg_helper.c
@@ -843,20 +843,35 @@ static void do_interrupt_protected(CPUX86State *env, int 
intno, int is_int,
 
 #ifdef TARGET_X86_64
 
-#define PUSHQ_RA(sp, val, ra)   \
-{   \
-sp -= 8;\
-cpu_stq_kernel_ra(env, sp, (val), ra);  \
-}
-
-#define POPQ_RA(sp, val, ra)\
-{   \
-val = cpu_ldq_kernel_ra(env, sp, ra);   \
-sp += 8;\
-}
+#define PUSHQ_RA(sp, val, ra, cpl, dpl) \
+  FUNC_PUSHQ_RA(env, , val, ra, cpl, dpl)
+
+static inline void FUNC_PUSHQ_RA(
+CPUX86State *env, target_ulong *sp,
+target_ulong val, target_ulong ra, int cpl, int dpl) {
+  *sp -= 8;
+  if (dpl == 0) {
+cpu_stq_kernel_ra(env, *sp, val, ra);
+  } else {
+cpu_stq_data_ra(env, *sp, val, ra);
+  } 
+}
 
-#define PUSHQ(sp, val) PUSHQ_RA(sp, val, 0)
-#define POPQ(sp, val) POPQ_RA(sp, val, 0)
+#define POPQ_RA(sp, val, ra, cpl, dpl) \
+  val = FUNC_POPQ_RA(env, , ra, cpl, dpl)
+
+static inline target_ulong FUNC_POPQ_RA(
+CPUX86State *env, target_ulong *sp,
+target_ulong ra, int cpl, int dpl) {
+  target_ulong val;
+  if (cpl == 0) {  /* TODO perhaps both arms reduce to cpu_ldq_data_ra? */
+val = cpu_ldq_kernel_ra(env, *sp, ra);
+  } else {
+val = cpu_ldq_data_ra(env, *sp, ra);
+  }
+  *sp += 8;
+  return val;
+}
 
 static inline target_ulong get_rsp_from_tss(CPUX86State *env, int level)
 {
@@ -901,6 +916,7 @@ static void do_interrupt64(CPUX86State *env, int intno, int 
is_int,
 uint32_t e1, e2, e3, ss, eflags;
 target_ulong old_eip, esp, offset;
 bool set_rf;
+const target_ulong retaddr = 0;
 
 has_error_code = 0;
 if (!is_int && !is_hw) {
@@ -989,13 +1005,13 @@ static void do_interrupt64(CPUX86State *env, int intno, 
int is_int,
 eflags |= RF_MASK;
 }
 
-PUSHQ(esp, env->segs[R_SS].selector);
-PUSHQ(esp, env->regs[R_ESP]);
-PUSHQ(esp, eflags);
-PUSHQ(esp, env->segs[R_CS].selector);
-PUSHQ(esp, old_eip);
+PUSHQ_RA(esp, env->segs[R_SS].selector, retaddr, cpl, dpl);
+PUSHQ_RA(esp, env->regs[R_ESP], retaddr, cpl, dpl);
+PUSHQ_RA(esp, eflags,   retaddr, cpl, dpl);
+PUSHQ_RA(esp, env->segs[R_CS].selector, retaddr, cpl, dpl);
+PUSHQ_RA(esp, old_eip,  retaddr, cpl, dpl);
 if (has_error_code) {
-PUSHQ(esp, error_code);
+PUSHQ_RA(esp, error_code, retaddr, cpl, dpl);
 }
 
 /* interrupt gate clear IF mask */
@@ -1621,8 +1637,8 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, 
target_ulong new_eip,
 
 /* 64 bit case */
 rsp = env->regs[R_ESP];
-PUSHQ_RA(rsp, env->segs[R_CS].selector, GETPC());
-PUSHQ_RA(rsp, next_eip, GETPC());
+PUSHQ_RA(rsp, env->segs[R_CS].selector, GETPC(), cpl, dpl);
+PUSHQ_RA(rsp, next_eip, GETPC(), cpl, dpl);
 /* from this point, not restartable */
 env->regs[R_ESP] = rsp;
 cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) | cpl,
@@ -1792,8 +1808,8 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, 
target_ulong new_eip,
 #ifdef TARGET_X86_64
 if (shift == 2) {
 /* XXX: verify if new stack address is canonical */
-PUSHQ_RA(sp, env->segs[R_SS].selector, GETPC());
-PUSHQ_RA(sp, env->regs[R_ESP], GETPC());
+PUSHQ_RA(sp, env->segs[R_SS].selector, GETPC(), cpl, dpl);
+PUSHQ_RA(sp, env->regs[R_ESP], GETPC(), cpl, dpl);
 /* parameters aren't supported for 64-bit call gates */
 } else
 #endif
@@ -1828,8 +1844,8 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, 
target_ulong new_eip,
 
 #ifdef TARGET_X86_64
 if (shift == 2) {
-PUSHQ_RA(sp, env->segs[R_CS].selector, GETPC());
-PUSHQ_RA(sp, next_eip, GETPC());
+PUSHQ_RA(sp, env->segs[R_CS].sele

Help with QEMU DBUS display

2023-08-28 Thread Elijah R

Hi!

I'm currently attempting to use qemu's DBUS display from within a go 
application, using the godbus 
<https://pkg.go.dev/github.com/godbus/dbus/v5> library. However, I'm 
hitting some roadblocks, and this is probably because I'm a bit confused 
about how qemu's peer-to-peer dbus connection works, and it's not 
explained in the documentation.



I invoke QEMU with the following argument

-display dbus,p2p=yes

and then connect to QMP through a separate UNIX socket. From my 
understanding, I need to listen on the socket separately in the program, 
and then pass qemu the file descriptor over QMP and run add_client.


In the same program I listen to the socket, and start waiting for a 
connection. Then, on another thread, get a file descriptor for the 
socket by first dialing it. Here's my code for that


sock, err := net.Dial("unix", "/tmp/qemudbus.sock")
if err != nil {
return err
}
uc, ok := sock.(*net.UnixConn)
if !ok {
	return fmt.Errorf("Could not cast 
Conn to UnixConn")

}
file, err := uc.File()

I then pass this file descriptor to QEMU using QMP (I'm using 
DigitalOcean's qmp library)


res, err := mon.RunWithFile([]byte(`{"execute": "getfd", "arguments": {"fdname": 
"dbusmon"}}`), file)

And add the dbus client:

res, err = mon.Run([]byte(`{"execute": "add_client", "arguments": {"protocol": "@dbus-display", 
"fdname": "dbusmon"}}`))

This seems to work fine, as I'm then (apparently) able to make a DBUS 
connection with that socket without any error.


However, when I then try to do anything with that connection (In this 
case I'm attempting to introspect /org/qemu/Display1/VM), the call hangs 
and never returns.


node, err := introspect.Call(session.Object("org.qemu.Display1.VM", 
"/org/qemu/Display1/VM"))

I'm not sure what's going wrong here, but I suspect I'm obtaining the 
file descriptor wrong (am I supposed to dial the socket?) or I'm doing 
things in the wrong order.


Thanks in advance for any help.



Regards,

Elijah R


publickey - elijah@elijahr.dev - a0a86704.asc
Description: application/pgp-keys


signature.asc
Description: OpenPGP digital signature


MacOS cocoa/OpenGL

2022-03-04 Thread R Kim
Hello Guys,

Looks like we haven't yet official support for OpenGL on MacOS? I need to apply 
third-party patches to make it work. It would be great if these parts are 
imported to qemu and be official. Nowaways, it's really needed Acceleration on 
VMs for who uses Graphical interfaces browsing, eg, chrome.

I have found it:
https://mail.gnu.org/archive/html/qemu-devel/2021-02/msg04235.html

It uses Angle from Google and the developer seems to be creating you own 
branch, with a lot of patches in https://github.com/akihikodaki/qemu/tree/macos

Another it this one, did small changes, but uses SDL instead of cocoa:
Based on initial work "Virgil 3D on macOS" by 小田喜陽彦 akihikodaki. 
[https://mail.gnu.org/archive/html/qem...](https://www.youtube.com/redirect?event=video_description_token=QUFFLUhqa3FxTDVuaWhiSFZkWWRkbHZkeTRTckc0bTJIQXxBQ3Jtc0trY3kza3JQT1dPS09vWmhCZktYSTRuWUdJMzB5MGdmV01DYUN4UGpOVTFWS3RwLUNZVGw5MzE3Zzg3WXZLNlJqNVNFZkM5QUZwRUN3UnJfOU5kVWxQaUJLdGo4N0IxVXd3X3pIb1lQUThMcmpJVktfWQ=https%3A%2F%2Fmail.gnu.org%2Farchive%2Fhtml%2Fqemu-devel%2F2021-02%2Fmsg04235.html)
 With improvements: - Get rid of ANGLE as EGL libraries. Virgil 3D solely uses 
OpenGL Core backend through SDL2 which wraps to NSOpenGL. - No patch to "cocoa" 
display of QEMU. Minimal modification to SDL2 display to support Virgil 3D on 
macOS. QEMU cannot have both "cocoa" and "sdl2" display in the same build. - 
Minimal change to libepoxy and virglrenderer, basically only have to make sure 
they built without libdrm and libEGL. - Fix known issues with OpenGL Core - 
Performance comparable to ANGLE OpenGL ES backend. WebGL Aquarium at 60 FPS. 
Chromium web rendering is accelerated. The accelerated ArchLinux ARM is very 
snappy & usable with Apple HVF virtualization. It matches or even exceeds the 
performance of typical ARM Linux flavors running on RK3399 or RPi4 bare-metal. 
It probably makes M1 MacBooks the world fastest (... most expensive) Linux ARM 
laptops. Nested VM is possible, and there is a demo of qemu-3dfx Glide 
passthrough from nested QEMU i386 instance running Blood 1.2 3Dfx DOS game.

https://www.youtube.com/watch?v=FVv8UjGhYPU

What should we do to have this changes in?

Sent with [ProtonMail](https://protonmail.com/) Secure Email.

[PATCH v3 1/2] modules: introduces module_kconfig directive

2021-09-28 Thread Jose R. Ziviani
module_kconfig is a new directive that should be used with module_obj
whenever that module depends on the Kconfig to be enabled.

When the module is enabled in Kconfig we are sure that its dependencies
will be enabled as well, thus the module will be loaded without any
problem.

The correct way to use module_kconfig is by passing the Kconfig option
to module_kconfig (or the *config-devices.mak without CONFIG_).

Signed-off-by: Jose R. Ziviani 
---
 hw/display/qxl.c|  1 +
 hw/display/vhost-user-gpu-pci.c |  1 +
 hw/display/vhost-user-gpu.c |  1 +
 hw/display/vhost-user-vga.c |  1 +
 hw/display/virtio-gpu-base.c|  1 +
 hw/display/virtio-gpu-gl.c  |  1 +
 hw/display/virtio-gpu-pci-gl.c  |  1 +
 hw/display/virtio-gpu-pci.c |  1 +
 hw/display/virtio-gpu.c |  1 +
 hw/display/virtio-vga-gl.c  |  1 +
 hw/display/virtio-vga.c |  1 +
 hw/s390x/virtio-ccw-gpu.c   |  1 +
 hw/usb/ccid-card-emulated.c |  1 +
 hw/usb/ccid-card-passthru.c |  1 +
 hw/usb/host-libusb.c|  1 +
 hw/usb/redirect.c   |  1 +
 include/qemu/module.h   | 10 ++
 scripts/modinfo-generate.py |  2 ++
 18 files changed, 28 insertions(+)

diff --git a/hw/display/qxl.c b/hw/display/qxl.c
index 29c80b4289..d02d50014b 100644
--- a/hw/display/qxl.c
+++ b/hw/display/qxl.c
@@ -2494,6 +2494,7 @@ static const TypeInfo qxl_primary_info = {
 .class_init= qxl_primary_class_init,
 };
 module_obj("qxl-vga");
+module_kconfig(QXL);
 
 static void qxl_secondary_class_init(ObjectClass *klass, void *data)
 {
diff --git a/hw/display/vhost-user-gpu-pci.c b/hw/display/vhost-user-gpu-pci.c
index daefcf7101..d119bcae45 100644
--- a/hw/display/vhost-user-gpu-pci.c
+++ b/hw/display/vhost-user-gpu-pci.c
@@ -44,6 +44,7 @@ static const VirtioPCIDeviceTypeInfo vhost_user_gpu_pci_info 
= {
 .instance_init = vhost_user_gpu_pci_initfn,
 };
 module_obj(TYPE_VHOST_USER_GPU_PCI);
+module_kconfig(VHOST_USER_GPU);
 
 static void vhost_user_gpu_pci_register_types(void)
 {
diff --git a/hw/display/vhost-user-gpu.c b/hw/display/vhost-user-gpu.c
index 49df56cd14..c961d2a2d8 100644
--- a/hw/display/vhost-user-gpu.c
+++ b/hw/display/vhost-user-gpu.c
@@ -599,6 +599,7 @@ static const TypeInfo vhost_user_gpu_info = {
 .class_init = vhost_user_gpu_class_init,
 };
 module_obj(TYPE_VHOST_USER_GPU);
+module_kconfig(VHOST_USER_GPU);
 
 static void vhost_user_gpu_register_types(void)
 {
diff --git a/hw/display/vhost-user-vga.c b/hw/display/vhost-user-vga.c
index 072c9c65bc..0c146080fd 100644
--- a/hw/display/vhost-user-vga.c
+++ b/hw/display/vhost-user-vga.c
@@ -45,6 +45,7 @@ static const VirtioPCIDeviceTypeInfo vhost_user_vga_info = {
 .instance_init = vhost_user_vga_inst_initfn,
 };
 module_obj(TYPE_VHOST_USER_VGA);
+module_kconfig(VHOST_USER_VGA);
 
 static void vhost_user_vga_register_types(void)
 {
diff --git a/hw/display/virtio-gpu-base.c b/hw/display/virtio-gpu-base.c
index c8da4806e0..cde96325ca 100644
--- a/hw/display/virtio-gpu-base.c
+++ b/hw/display/virtio-gpu-base.c
@@ -257,6 +257,7 @@ static const TypeInfo virtio_gpu_base_info = {
 .abstract = true
 };
 module_obj(TYPE_VIRTIO_GPU_BASE);
+module_kconfig(VIRTIO_GPU);
 
 static void
 virtio_register_types(void)
diff --git a/hw/display/virtio-gpu-gl.c b/hw/display/virtio-gpu-gl.c
index 6cc4313b1a..f7837cc44d 100644
--- a/hw/display/virtio-gpu-gl.c
+++ b/hw/display/virtio-gpu-gl.c
@@ -160,6 +160,7 @@ static const TypeInfo virtio_gpu_gl_info = {
 .class_init = virtio_gpu_gl_class_init,
 };
 module_obj(TYPE_VIRTIO_GPU_GL);
+module_kconfig(VIRTIO_GPU);
 
 static void virtio_register_types(void)
 {
diff --git a/hw/display/virtio-gpu-pci-gl.c b/hw/display/virtio-gpu-pci-gl.c
index 99b14a0718..a2819e1ca9 100644
--- a/hw/display/virtio-gpu-pci-gl.c
+++ b/hw/display/virtio-gpu-pci-gl.c
@@ -47,6 +47,7 @@ static const VirtioPCIDeviceTypeInfo virtio_gpu_gl_pci_info = 
{
 .instance_init = virtio_gpu_gl_initfn,
 };
 module_obj(TYPE_VIRTIO_GPU_GL_PCI);
+module_kconfig(VIRTIO_PCI);
 
 static void virtio_gpu_gl_pci_register_types(void)
 {
diff --git a/hw/display/virtio-gpu-pci.c b/hw/display/virtio-gpu-pci.c
index e36eee0c40..93f214ff58 100644
--- a/hw/display/virtio-gpu-pci.c
+++ b/hw/display/virtio-gpu-pci.c
@@ -65,6 +65,7 @@ static const TypeInfo virtio_gpu_pci_base_info = {
 .abstract = true
 };
 module_obj(TYPE_VIRTIO_GPU_PCI_BASE);
+module_kconfig(VIRTIO_PCI);
 
 #define TYPE_VIRTIO_GPU_PCI "virtio-gpu-pci"
 typedef struct VirtIOGPUPCI VirtIOGPUPCI;
diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c
index 182e0868b0..6cc6bb1f1f 100644
--- a/hw/display/virtio-gpu.c
+++ b/hw/display/virtio-gpu.c
@@ -1450,6 +1450,7 @@ static const TypeInfo virtio_gpu_info = {
 .class_init = virtio_gpu_class_init,
 };
 module_obj(TYPE_VIRTIO_GPU);
+module_kconfig(VIRTIO_GPU);
 
 static void virtio_register_types(void)
 {
diff --git a/hw/display/virtio-vga-gl.c b/hw/display/virtio-vga-gl.c
index f2

[PATCH v3 2/2] modules: generates per-target modinfo

2021-09-28 Thread Jose R. Ziviani
This patch changes the way modinfo is generated and built. Instead of
one modinfo.c it generates one modinfo--softmmu.c per target. It
aims a fine-tune control of modules by configuring Kconfig.

Signed-off-by: Jose R. Ziviani 
---
 meson.build | 25 +++---
 scripts/modinfo-generate.py | 42 ++---
 2 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/meson.build b/meson.build
index 15ef4d3c41..a617be73de 100644
--- a/meson.build
+++ b/meson.build
@@ -2403,14 +2403,23 @@ foreach d, list : target_modules
 endforeach
 
 if enable_modules
-  modinfo_src = custom_target('modinfo.c',
-  output: 'modinfo.c',
-  input: modinfo_files,
-  command: [modinfo_generate, '@INPUT@'],
-  capture: true)
-  modinfo_lib = static_library('modinfo', modinfo_src)
-  modinfo_dep = declare_dependency(link_whole: modinfo_lib)
-  softmmu_ss.add(modinfo_dep)
+  foreach target : target_dirs
+if target.endswith('-softmmu')
+  config_target = config_target_mak[target]
+  config_devices_mak = target + '-config-devices.mak'
+  modinfo_src = custom_target('modinfo-' + target + '.c',
+  output: 'modinfo-' + target + '.c',
+  input: modinfo_files,
+  command: [modinfo_generate, '--devices', 
config_devices_mak, '@INPUT@'],
+  capture: true)
+
+  modinfo_lib = static_library('modinfo-' + target + '.c', modinfo_src)
+  modinfo_dep = declare_dependency(link_with: modinfo_lib)
+
+  arch = config_target['TARGET_NAME'] == 'sparc64' ? 'sparc64' : 
config_target['TARGET_BASE_ARCH']
+  hw_arch[arch].add(modinfo_dep)
+endif
+  endforeach
 endif
 
 nm = find_program('nm')
diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py
index 689f33c0f2..a0c09edae1 100755
--- a/scripts/modinfo-generate.py
+++ b/scripts/modinfo-generate.py
@@ -32,7 +32,7 @@ def parse_line(line):
 continue
 return (kind, data)
 
-def generate(name, lines):
+def generate(name, lines, core_modules):
 arch = ""
 objs = []
 deps = []
@@ -49,7 +49,13 @@ def generate(name, lines):
 elif kind == 'arch':
 arch = data;
 elif kind == 'kconfig':
-pass # ignore
+# don't add a module which dependency is not enabled
+# in kconfig
+if data.strip() not in core_modules:
+print("/* module {} isn't enabled in Kconfig. */"
+  .format(data.strip()))
+print("/* },{ */")
+return []
 else:
 print("unknown:", kind)
 exit(1)
@@ -60,7 +66,7 @@ def generate(name, lines):
 print_array("objs", objs)
 print_array("deps", deps)
 print_array("opts", opts)
-print("},{");
+print("},{")
 return deps
 
 def print_pre():
@@ -74,26 +80,28 @@ def print_post():
 print("}};")
 
 def main(args):
+if len(args) < 3 or args[0] != '--devices':
+print('Expected: modinfo-generate.py --devices '
+  'config-device.mak [modinfo files]', file=sys.stderr)
+exit(1)
+
+# get all devices enabled in kconfig, from *-config-device.mak
+enabled_core_modules = set()
+with open(args[1]) as file:
+for line in file.readlines():
+config = line.split('=')
+if config[1].rstrip() == 'y':
+enabled_core_modules.add(config[0][7:]) # remove CONFIG_
+
 deps = {}
 print_pre()
-for modinfo in args:
+for modinfo in args[2:]:
 with open(modinfo) as f:
 lines = f.readlines()
 print("/* %s */" % modinfo)
-(basename, ext) = os.path.splitext(modinfo)
-deps[basename] = generate(basename, lines)
+(basename, _) = os.path.splitext(modinfo)
+deps[basename] = generate(basename, lines, enabled_core_modules)
 print_post()
 
-flattened_deps = {flat.strip('" ') for dep in deps.values() for flat in 
dep}
-error = False
-for dep in flattened_deps:
-if dep not in deps.keys():
-print("Dependency {} cannot be satisfied".format(dep),
-  file=sys.stderr)
-error = True
-
-if error:
-exit(1)
-
 if __name__ == "__main__":
 main(sys.argv[1:])
-- 
2.33.0




[PATCH v3 0/2] modules: Improve modinfo.c support

2021-09-28 Thread Jose R. Ziviani
This patchset introduces the modinfo_kconfig aiming for a fine-tune
control of module loading by simply checking Kconfig options during the
compile time, then generates one modinfo--softmmu.c per target.

The main reason of this change is to fix problems like:
$ ./qemu-system-s390x -nodefaults -display none -accel tcg -M none -device help 
| head
Failed to open module: /.../hw-display-qxl.so: undefined symbol: vga_ioport_read
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga-gl.so: undefined symbol: 
have_vga
Failed to open module: /.../hw-usb-smartcard.so: undefined symbol: 
ccid_card_ccid_attach
Failed to open module: /.../hw-usb-redirect.so: undefined symbol: 
vmstate_usb_device
Failed to open module: /.../hw-usb-host.so: undefined symbol: vmstate_usb_device

With this patch, I run this small script successfuly:
#!/bin/bash
pushd ~/suse/virtualization/qemu/build
for qemu in qemu-system-*
do
[[ -f "$qemu" ]] || continue
res=$(./$qemu -nodefaults -display none -accel tcg -M none -device help 
2>&1 | grep "Failed to" > /dev/null; echo $?)
[[ $res -eq 0 ]] && echo "Error: $qemu"
done
popd

Also run 'make check' and 'check-acceptance' without any failures.

v2 -> v3:
 - Renamed module_needs to module_kconfig [Gerd]
 - Reworded the commit message a bit to improve a better understanding [myself]

v1 -> v2:
 - Changed the approach to this problem after suggestions made by Paolo and 
Gerd.

Thank you!

Jose R. Ziviani (2):
  modules: introduces module_kconfig directive
  modules: generates per-target modinfo

 hw/display/qxl.c|  1 +
 hw/display/vhost-user-gpu-pci.c |  1 +
 hw/display/vhost-user-gpu.c |  1 +
 hw/display/vhost-user-vga.c |  1 +
 hw/display/virtio-gpu-base.c|  1 +
 hw/display/virtio-gpu-gl.c  |  1 +
 hw/display/virtio-gpu-pci-gl.c  |  1 +
 hw/display/virtio-gpu-pci.c |  1 +
 hw/display/virtio-gpu.c |  1 +
 hw/display/virtio-vga-gl.c  |  1 +
 hw/display/virtio-vga.c |  1 +
 hw/s390x/virtio-ccw-gpu.c   |  1 +
 hw/usb/ccid-card-emulated.c |  1 +
 hw/usb/ccid-card-passthru.c |  1 +
 hw/usb/host-libusb.c|  1 +
 hw/usb/redirect.c   |  1 +
 include/qemu/module.h   | 10 
 meson.build | 25 +---
 scripts/modinfo-generate.py | 42 -
 19 files changed, 69 insertions(+), 24 deletions(-)

-- 
2.33.0




Re: [PATCH v2 0/2] modules: Improve modinfo.c support

2021-09-28 Thread Jose R. Ziviani
Hello, Gerd!

On Tue, Sep 28, 2021 at 07:06:28AM +0200, Gerd Hoffmann wrote:
> On Mon, Sep 27, 2021 at 11:11:58AM -0300, Jose R. Ziviani wrote:
> > This patchset introduces the modinfo_need and changes
> > modinfo-generate.py/meson.build to generate/link one modinfo per target.
> > 
> > modinfo-generate.py will know, thanks to modinfo_need, which modules are
> > currently enabled for a given target before adding it in the array of
> > modules. It will give a hint about why some modules failed, so
> > developers can have a clue about it:
> 
> The approach looks good to me.

Awesome, I'll apply your review and send a new version.

Thank you!

> 
> > /* hw-display-qxl.modinfo */
> > /* module  QXL is missing. */
> 
> You are using kconfig symbols here, so the comment should say so ;)
> 
> Renaming modinfo_need to modinfo_kconfig will probably also help
> to make that clear.
> 
> > /* hw-display-virtio-gpu.modinfo */
> > .name = "hw-display-virtio-gpu",
> > .objs = ((const char*[]){  "virtio-gpu-base",  "virtio-gpu-device",  
> > "vhost-user-gpu", NULL }),
> 
> Hmm?  Leftover from an older version of the series?
> 
> >  - accelerators can be filtered as well (this only covers the device
> >part), then the field QemuModinfo.arch can be removed.
> 
> It's target-specific modules.  Although accelerators are the only
> in-tree users right this is not limited to accelerators.
> 
> take care,
>   Gerd
> 


signature.asc
Description: Digital signature


[PATCH v2 0/2] modules: Improve modinfo.c support

2021-09-27 Thread Jose R. Ziviani
This patchset introduces the modinfo_need and changes
modinfo-generate.py/meson.build to generate/link one modinfo per target.

modinfo-generate.py will know, thanks to modinfo_need, which modules are
currently enabled for a given target before adding it in the array of
modules. It will give a hint about why some modules failed, so
developers can have a clue about it:

},{
/* hw-display-qxl.modinfo */
/* module  QXL is missing. */

/* hw-display-virtio-gpu.modinfo */
.name = "hw-display-virtio-gpu",
.objs = ((const char*[]){  "virtio-gpu-base",  "virtio-gpu-device",  
"vhost-user-gpu", NULL }),
},{

The main reason of this change is to fix problems like:
$ ./qemu-system-s390x -nodefaults -display none -accel tcg -M none -device help 
| head
Failed to open module: /.../hw-display-qxl.so: undefined symbol: vga_ioport_read
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga-gl.so: undefined symbol: 
have_vga
Failed to open module: /.../hw-usb-smartcard.so: undefined symbol: 
ccid_card_ccid_attach
Failed to open module: /.../hw-usb-redirect.so: undefined symbol: 
vmstate_usb_device
Failed to open module: /.../hw-usb-host.so: undefined symbol: vmstate_usb_device

With this patch, I run this small script successfuly:
#!/bin/bash
pushd ~/suse/virtualization/qemu/build
for qemu in qemu-system-*
do
[[ -f "$qemu" ]] || continue
res=$(./$qemu -nodefaults -display none -accel tcg -M none -device help 
2>&1 | grep "Failed to" > /dev/null; echo $?)
[[ $res -eq 0 ]] && echo "Error: $qemu"
done
popd

Also run make check and check-acceptance without any failures.

Todo:
 - accelerators can be filtered as well (this only covers the device
   part), then the field QemuModinfo.arch can be removed.

v1 -> v2:
 - Changed the approach to this problem after suggestions made by Paolo and 
Gerd.

Thank you!

Jose R. Ziviani (2):
  modules: introduces module_needs directive
  modules: generates per-target modinfo

 hw/display/qxl.c|  1 +
 hw/display/vhost-user-gpu-pci.c |  1 +
 hw/display/vhost-user-gpu.c |  1 +
 hw/display/vhost-user-vga.c |  1 +
 hw/display/virtio-gpu-base.c|  1 +
 hw/display/virtio-gpu-gl.c  |  1 +
 hw/display/virtio-gpu-pci-gl.c  |  1 +
 hw/display/virtio-gpu-pci.c |  1 +
 hw/display/virtio-gpu.c |  1 +
 hw/display/virtio-vga-gl.c  |  1 +
 hw/display/virtio-vga.c |  1 +
 hw/s390x/virtio-ccw-gpu.c   |  1 +
 hw/usb/ccid-card-emulated.c |  1 +
 hw/usb/ccid-card-passthru.c |  1 +
 hw/usb/host-libusb.c|  1 +
 hw/usb/redirect.c   |  1 +
 include/qemu/module.h   | 10 +
 meson.build | 25 ++---
 scripts/modinfo-generate.py | 40 -
 19 files changed, 67 insertions(+), 24 deletions(-)

-- 
2.33.0




[PATCH v2 1/2] modules: introduces module_needs directive

2021-09-27 Thread Jose R. Ziviani
module_needs is a new directive that shoule be used with module_obj
whenever that module depends on the Kconfig to be enabled.

When the module is enabled in Kconfig, we are sure that all of its
dependencies will be enabled as well, and that the program will be
able to load that module without any problem.

The correct way to use module_needs is by passing the Kconfig option
(or the *config-devices.mak without CONFIG_).

Signed-off-by: Jose R. Ziviani 
---
 hw/display/qxl.c|  1 +
 hw/display/vhost-user-gpu-pci.c |  1 +
 hw/display/vhost-user-gpu.c |  1 +
 hw/display/vhost-user-vga.c |  1 +
 hw/display/virtio-gpu-base.c|  1 +
 hw/display/virtio-gpu-gl.c  |  1 +
 hw/display/virtio-gpu-pci-gl.c  |  1 +
 hw/display/virtio-gpu-pci.c |  1 +
 hw/display/virtio-gpu.c |  1 +
 hw/display/virtio-vga-gl.c  |  1 +
 hw/display/virtio-vga.c |  1 +
 hw/s390x/virtio-ccw-gpu.c   |  1 +
 hw/usb/ccid-card-emulated.c |  1 +
 hw/usb/ccid-card-passthru.c |  1 +
 hw/usb/host-libusb.c|  1 +
 hw/usb/redirect.c   |  1 +
 include/qemu/module.h   | 10 ++
 scripts/modinfo-generate.py |  2 ++
 18 files changed, 28 insertions(+)

diff --git a/hw/display/qxl.c b/hw/display/qxl.c
index 29c80b4289..33647d59ad 100644
--- a/hw/display/qxl.c
+++ b/hw/display/qxl.c
@@ -2494,6 +2494,7 @@ static const TypeInfo qxl_primary_info = {
 .class_init= qxl_primary_class_init,
 };
 module_obj("qxl-vga");
+module_needs(QXL);
 
 static void qxl_secondary_class_init(ObjectClass *klass, void *data)
 {
diff --git a/hw/display/vhost-user-gpu-pci.c b/hw/display/vhost-user-gpu-pci.c
index daefcf7101..d47219f294 100644
--- a/hw/display/vhost-user-gpu-pci.c
+++ b/hw/display/vhost-user-gpu-pci.c
@@ -44,6 +44,7 @@ static const VirtioPCIDeviceTypeInfo vhost_user_gpu_pci_info 
= {
 .instance_init = vhost_user_gpu_pci_initfn,
 };
 module_obj(TYPE_VHOST_USER_GPU_PCI);
+module_needs(VHOST_USER_GPU);
 
 static void vhost_user_gpu_pci_register_types(void)
 {
diff --git a/hw/display/vhost-user-gpu.c b/hw/display/vhost-user-gpu.c
index 49df56cd14..6a229f2b34 100644
--- a/hw/display/vhost-user-gpu.c
+++ b/hw/display/vhost-user-gpu.c
@@ -599,6 +599,7 @@ static const TypeInfo vhost_user_gpu_info = {
 .class_init = vhost_user_gpu_class_init,
 };
 module_obj(TYPE_VHOST_USER_GPU);
+module_needs(VHOST_USER_GPU);
 
 static void vhost_user_gpu_register_types(void)
 {
diff --git a/hw/display/vhost-user-vga.c b/hw/display/vhost-user-vga.c
index 072c9c65bc..a7b9f3580d 100644
--- a/hw/display/vhost-user-vga.c
+++ b/hw/display/vhost-user-vga.c
@@ -45,6 +45,7 @@ static const VirtioPCIDeviceTypeInfo vhost_user_vga_info = {
 .instance_init = vhost_user_vga_inst_initfn,
 };
 module_obj(TYPE_VHOST_USER_VGA);
+module_needs(VHOST_USER_VGA);
 
 static void vhost_user_vga_register_types(void)
 {
diff --git a/hw/display/virtio-gpu-base.c b/hw/display/virtio-gpu-base.c
index c8da4806e0..9c485151fc 100644
--- a/hw/display/virtio-gpu-base.c
+++ b/hw/display/virtio-gpu-base.c
@@ -257,6 +257,7 @@ static const TypeInfo virtio_gpu_base_info = {
 .abstract = true
 };
 module_obj(TYPE_VIRTIO_GPU_BASE);
+module_needs(VIRTIO_GPU);
 
 static void
 virtio_register_types(void)
diff --git a/hw/display/virtio-gpu-gl.c b/hw/display/virtio-gpu-gl.c
index 6cc4313b1a..c57707f6f1 100644
--- a/hw/display/virtio-gpu-gl.c
+++ b/hw/display/virtio-gpu-gl.c
@@ -160,6 +160,7 @@ static const TypeInfo virtio_gpu_gl_info = {
 .class_init = virtio_gpu_gl_class_init,
 };
 module_obj(TYPE_VIRTIO_GPU_GL);
+module_needs(VIRTIO_GPU);
 
 static void virtio_register_types(void)
 {
diff --git a/hw/display/virtio-gpu-pci-gl.c b/hw/display/virtio-gpu-pci-gl.c
index 99b14a0718..3817f0dd9d 100644
--- a/hw/display/virtio-gpu-pci-gl.c
+++ b/hw/display/virtio-gpu-pci-gl.c
@@ -47,6 +47,7 @@ static const VirtioPCIDeviceTypeInfo virtio_gpu_gl_pci_info = 
{
 .instance_init = virtio_gpu_gl_initfn,
 };
 module_obj(TYPE_VIRTIO_GPU_GL_PCI);
+module_needs(VIRTIO_PCI);
 
 static void virtio_gpu_gl_pci_register_types(void)
 {
diff --git a/hw/display/virtio-gpu-pci.c b/hw/display/virtio-gpu-pci.c
index e36eee0c40..3219adcf2d 100644
--- a/hw/display/virtio-gpu-pci.c
+++ b/hw/display/virtio-gpu-pci.c
@@ -65,6 +65,7 @@ static const TypeInfo virtio_gpu_pci_base_info = {
 .abstract = true
 };
 module_obj(TYPE_VIRTIO_GPU_PCI_BASE);
+module_needs(VIRTIO_PCI);
 
 #define TYPE_VIRTIO_GPU_PCI "virtio-gpu-pci"
 typedef struct VirtIOGPUPCI VirtIOGPUPCI;
diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c
index 182e0868b0..874808326f 100644
--- a/hw/display/virtio-gpu.c
+++ b/hw/display/virtio-gpu.c
@@ -1450,6 +1450,7 @@ static const TypeInfo virtio_gpu_info = {
 .class_init = virtio_gpu_class_init,
 };
 module_obj(TYPE_VIRTIO_GPU);
+module_needs(VIRTIO_GPU);
 
 static void virtio_register_types(void)
 {
diff --git a/hw/display/virtio-vga-gl.c b/hw/display/virtio-vga-gl.c
index f22549097

[PATCH v2 2/2] modules: generates per-target modinfo

2021-09-27 Thread Jose R. Ziviani
This patch changes the way modinfo is generated and built. Today we have
only modinfo.c being genereated and linked to all targets, now it
generates (and link) one modinfo per target.

It also makes use of the module_need to add modules that makes sense for
the selected target.

Signed-off-by: Jose R. Ziviani 
---
 meson.build | 25 +++
 scripts/modinfo-generate.py | 40 +
 2 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/meson.build b/meson.build
index 2711cbb789..9d25ebb2f9 100644
--- a/meson.build
+++ b/meson.build
@@ -2395,14 +2395,23 @@ foreach d, list : target_modules
 endforeach
 
 if enable_modules
-  modinfo_src = custom_target('modinfo.c',
-  output: 'modinfo.c',
-  input: modinfo_files,
-  command: [modinfo_generate, '@INPUT@'],
-  capture: true)
-  modinfo_lib = static_library('modinfo', modinfo_src)
-  modinfo_dep = declare_dependency(link_whole: modinfo_lib)
-  softmmu_ss.add(modinfo_dep)
+  foreach target : target_dirs
+if target.endswith('-softmmu')
+  config_target = config_target_mak[target]
+  config_devices_mak = target + '-config-devices.mak'
+  modinfo_src = custom_target('modinfo-' + target + '.c',
+  output: 'modinfo-' + target + '.c',
+  input: modinfo_files,
+  command: [modinfo_generate, '--devices', 
config_devices_mak, '@INPUT@'],
+  capture: true)
+
+  modinfo_lib = static_library('modinfo-' + target + '.c', modinfo_src)
+  modinfo_dep = declare_dependency(link_with: modinfo_lib)
+
+  arch = config_target['TARGET_NAME'] == 'sparc64' ? 'sparc64' : 
config_target['TARGET_BASE_ARCH']
+  hw_arch[arch].add(modinfo_dep)
+endif
+  endforeach
 endif
 
 nm = find_program('nm')
diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py
index 9d3e037b15..25fb241b2d 100755
--- a/scripts/modinfo-generate.py
+++ b/scripts/modinfo-generate.py
@@ -32,7 +32,7 @@ def parse_line(line):
 continue
 return (kind, data)
 
-def generate(name, lines):
+def generate(name, lines, core_modules):
 arch = ""
 objs = []
 deps = []
@@ -49,7 +49,11 @@ def generate(name, lines):
 elif kind == 'arch':
 arch = data;
 elif kind == 'need':
-pass # ignore
+# don't add a module which dependency is not enabled
+# in kconfig
+if data.strip() not in core_modules:
+print("/* module {} is missing. */\n".format(data))
+return []
 else:
 print("unknown:", kind)
 exit(1)
@@ -60,7 +64,7 @@ def generate(name, lines):
 print_array("objs", objs)
 print_array("deps", deps)
 print_array("opts", opts)
-print("},{");
+print("},{")
 return deps
 
 def print_pre():
@@ -74,26 +78,28 @@ def print_post():
 print("}};")
 
 def main(args):
+if len(args) < 3 or args[0] != '--devices':
+print('Expected: modinfo-generate.py --devices '
+  'config-device.mak [modinfo files]', file=sys.stderr)
+exit(1)
+
+# get all devices enabled in kconfig, from *-config-device.mak
+enabled_core_modules = set()
+with open(args[1]) as file:
+for line in file.readlines():
+config = line.split('=')
+if config[1].rstrip() == 'y':
+enabled_core_modules.add(config[0][7:]) # remove CONFIG_
+
 deps = {}
 print_pre()
-for modinfo in args:
+for modinfo in args[2:]:
 with open(modinfo) as f:
 lines = f.readlines()
 print("/* %s */" % modinfo)
-(basename, ext) = os.path.splitext(modinfo)
-deps[basename] = generate(basename, lines)
+(basename, _) = os.path.splitext(modinfo)
+deps[basename] = generate(basename, lines, enabled_core_modules)
 print_post()
 
-flattened_deps = {flat.strip('" ') for dep in deps.values() for flat in 
dep}
-error = False
-for dep in flattened_deps:
-if dep not in deps.keys():
-print("Dependency {} cannot be satisfied".format(dep),
-  file=sys.stderr)
-error = True
-
-if error:
-exit(1)
-
 if __name__ == "__main__":
 main(sys.argv[1:])
-- 
2.33.0




[PATCH v2 0/2] modules: Improve modinfo.c support

2021-09-27 Thread Jose R. Ziviani
This patchset introduces the modinfo_need and changes
modinfo-generate.py/meson.build to generate/link one modinfo per target.

modinfo-generate.py will know, thanks to modinfo_need, which modules are
currently enabled for a given target before adding it in the array of
modules. It will give a hint about why some modules failed, so
developers can have a clue about it:

},{
/* hw-display-qxl.modinfo */
/* module  QXL is missing. */

/* hw-display-virtio-gpu.modinfo */
.name = "hw-display-virtio-gpu",
.objs = ((const char*[]){  "virtio-gpu-base",  "virtio-gpu-device",  
"vhost-user-gpu", NULL }),
},{

The main reason of this change is to fix problems like:
$ ./qemu-system-s390x -nodefaults -display none -accel tcg -M none -device help 
| head
Failed to open module: /.../hw-display-qxl.so: undefined symbol: vga_ioport_read
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga-gl.so: undefined symbol: 
have_vga
Failed to open module: /.../hw-usb-smartcard.so: undefined symbol: 
ccid_card_ccid_attach
Failed to open module: /.../hw-usb-redirect.so: undefined symbol: 
vmstate_usb_device
Failed to open module: /.../hw-usb-host.so: undefined symbol: vmstate_usb_device

With this patch, I run this small script successfuly:
#!/bin/bash
pushd ~/suse/virtualization/qemu/build
for qemu in qemu-system-*
do
[[ -f "$qemu" ]] || continue
res=$(./$qemu -nodefaults -display none -accel tcg -M none -device help 
2>&1 | grep "Failed to" > /dev/null; echo $?)
[[ $res -eq 0 ]] && echo "Error: $qemu"
done
popd

Also run make check and check-acceptance without any failures.

Todo:
 - accelerators can be filtered as well (this only covers the device
   part), then the field QemuModinfo.arch can be removed.

v1 -> v2:
 - Changed the approach to this problem after suggestions made by Paolo and 
Gerd.

Thank you!

Jose R. Ziviani (2):
  modules: introduces module_needs directive
  modules: generates per-target modinfo

 hw/display/qxl.c|  1 +
 hw/display/vhost-user-gpu-pci.c |  1 +
 hw/display/vhost-user-gpu.c |  1 +
 hw/display/vhost-user-vga.c |  1 +
 hw/display/virtio-gpu-base.c|  1 +
 hw/display/virtio-gpu-gl.c  |  1 +
 hw/display/virtio-gpu-pci-gl.c  |  1 +
 hw/display/virtio-gpu-pci.c |  1 +
 hw/display/virtio-gpu.c |  1 +
 hw/display/virtio-vga-gl.c  |  1 +
 hw/display/virtio-vga.c |  1 +
 hw/s390x/virtio-ccw-gpu.c   |  1 +
 hw/usb/ccid-card-emulated.c |  1 +
 hw/usb/ccid-card-passthru.c |  1 +
 hw/usb/host-libusb.c|  1 +
 hw/usb/redirect.c   |  1 +
 include/qemu/module.h   | 10 +
 meson.build | 25 ++---
 scripts/modinfo-generate.py | 40 -
 19 files changed, 67 insertions(+), 24 deletions(-)

-- 
2.33.0




Re: [PATCH 1/2] meson: introduce modules_arch

2021-09-21 Thread Jose R. Ziviani
Hello!!

On Mon, Sep 20, 2021 at 09:03:28PM +0200, Paolo Bonzini wrote:
> On 20/09/21 15:02, Jose R. Ziviani wrote:
> > But, in anyway, I'll still need to store the target architecture that
> > can use such core module, like I did here in this patch. Otherwise,
> > if I compile different targets at the same time, I'll end up with the
> > same problem of targets trying to load wrong modules.
> > 
> > I thought of using qom, but I think it will pollute module.c.
> 
> Alternatively, you could C-ify the contents of config-devices.mak, and embed
> them in the per-arch modinfo-*.c; and record CONFIG_* symbols for each
> module (e.g. '{ "CONFIG_QXL", "hw-display-qxl" }' from a
> 'module_config("CONFIG_QXL")' line in the global modinfo.c file.  Then
> before loading a module you do a binary search on the per-arch
> config-devices array.

With a per-arch modinfo-*.c we don't even need a modinfo.c global, do
we?

Each target could be linked to its own modinfo-target.c only.

> 
> I hope the above is readable. :)

Absolutely, thank you for your suggestion!!

> 
> Paolo


signature.asc
Description: Digital signature


Re: [PATCH 1/2] meson: introduce modules_arch

2021-09-21 Thread Jose R. Ziviani
Hello!!

On Tue, Sep 21, 2021 at 07:25:42AM +0200, Gerd Hoffmann wrote:
>   Hi,
> 
> > But, in anyway, I'll still need to store the target architecture that
> > can use such core module, like I did here in this patch. Otherwise,
> > if I compile different targets at the same time, I'll end up with the
> > same problem of targets trying to load wrong modules.
> 
> That all works just fine today.  If you have target-specific modules
> (i.e. source files added to specific_ss instead of softmmu_ss when
> compiling into core qemu) you only need to add those to the
> target_modules[] (instead of modules[]) and you are set.
> 
> In-tree example: qtest accelerator.

Exactly, but what it doesn't seem to work (or I'm not understanding it
well) is: how the target will know whether it can or cannot load a
module.

For example, suppose I build target-list=s390x-softmmu,x86_64-softmmu.
Both targets will be linked to the same modinfo.c object, which will
have a 'hw-display-virtio-gpu-pci' entry (it wouldn't if I build only
s390x-softmmu). When I execute ./qemu-system-s390x, it will try to
load hw-display-virtio-gpu-pci and will throw the errors I mentioned
earlier.

If, for example, I add that module_need(PCI_BUS), we will continue
not knowing whether the target in execution has the required bus
(without injecting dependencies in module.c). But it would be easier if
we have such information in modinfo.c directly (of an modinfo-.c).
I understood that it's not an arch issue, it's the target that doesn't
current implement such core module. But, in practice, we will end up
need to query that information.

Please, correct me if I'm not getting the point correctly.

Thank you!!


> 
> take care,
>   Gerd
> 


signature.asc
Description: Digital signature


Re: [PATCH 1/2] meson: introduce modules_arch

2021-09-20 Thread Jose R. Ziviani
On Mon, Sep 20, 2021 at 07:15:32AM +0200, Gerd Hoffmann wrote:
>   Hi,
> 
> > Yes, I really like your approach, makes more sense indeed. But, how do I
> > get the core modules that other modules depend on?
> > 
> > I see that Kconfig already has something in this line:
> > 
> > config VGA  (from hw/display)
> > bool
> > 
> > config PCI  (from hw/pci)
> > bool
> > 
> > config QXL  (from hw/display)
> > bool
> > depends on SPICE && PCI
> > select VGA
> > 
> > I assume that independent entries (like VGA and PCI) are core and that I
> > can rely on it to add
> >   module_need(PCI)
> >   module_need(VGA)
> > for hw-display-qxl. Am I right?
> 
> Yes, looking at kconfig for core dependencies makes sense.

But, in anyway, I'll still need to store the target architecture that
can use such core module, like I did here in this patch. Otherwise,
if I compile different targets at the same time, I'll end up with the
same problem of targets trying to load wrong modules.

I thought of using qom, but I think it will pollute module.c.

What do you think if I simply create one modinfo.c per target, like
modinfo-s390x.c, modinfo-avr.c, etc? Each will only have the data
structure filled with the right modules and linked only to its own
qemu-system-arch.

Best regards,

Jose R Ziviani

> 
> take care,
>   Gerd
> 


signature.asc
Description: Digital signature


Re: [PATCH 1/2] meson: introduce modules_arch

2021-09-17 Thread Jose R. Ziviani
Hello!

On Fri, Sep 17, 2021 at 09:14:04AM +0200, Gerd Hoffmann wrote:
>   Hi,
> 
> > This variable keeps track of all modules enabled for a target
> > architecture. This will be used in modinfo to refine the
> > architectures that can really load the .so to avoid errors.
> 
> I think this is the wrong approach.  The reason why modules are
> not loading is typically *not* the architecture, but a feature
> or subsystem the device needs not being compiled in.  Often the
> subsystem is a bus (pci, usb, ccw), but there are also other
> cases (virtio, vga).
> 
> We can stick that into modinfo, simliar to module_dep() but for bits
> provided by core qemu instead of other modules.  i.e. add something
> along the lines of ...

Yes, I really like your approach, makes more sense indeed. But, how do I
get the core modules that other modules depend on?

I see that Kconfig already has something in this line:

config VGA  (from hw/display)
bool

config PCI  (from hw/pci)
bool

config QXL  (from hw/display)
bool
depends on SPICE && PCI
select VGA

I assume that independent entries (like VGA and PCI) are core and that I
can rely on it to add
  module_need(PCI)
  module_need(VGA)
for hw-display-qxl. Am I right?

Thanks for reviewing it!!

> 
>   module_need(BUS_PCI);
> 
> ... to the modules, store that in modinfo and check it before trying
> to load.
> 
> That would also allow to remove hacks like commit 2dd9d8cfb4f3 ("s390x:
> add have_virtio_ccw")
> 
> take care,
>   Gerd
> 


signature.asc
Description: Digital signature


[PATCH 2/2] modules: use a list of supported arch for each module

2021-09-16 Thread Jose R. Ziviani
When compiling QEMU with more than one target, for instance,
--target-list=s390x-softmmu,x86_64-softmmu, modinfo.c will be
filled with modules available for both, with no specification
of what modules can/cannot be loaded for a particular target.

This will cause message errors when executing the target that
shouldn't be loading that module, such as:

$ qemu-system-s390x -nodefaults -display none -accel qtest -M none -device help
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common

This patch changes the module infrastructure to use a list of
architectures, obtained during the build time, to specify what
targets can load each module.

Signed-off-by: Jose R. Ziviani 
---
 include/qemu/module.h   |  2 +-
 meson.build | 18 +-
 scripts/modinfo-collect.py  | 10 ++
 scripts/modinfo-generate.py |  7 +++
 util/module.c   | 18 +-
 5 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/include/qemu/module.h b/include/qemu/module.h
index 3deac0078b..3b487c646c 100644
--- a/include/qemu/module.h
+++ b/include/qemu/module.h
@@ -144,7 +144,7 @@ void module_allow_arch(const char *arch);
 typedef struct QemuModinfo QemuModinfo;
 struct QemuModinfo {
 const char *name;
-const char *arch;
+const char **archs;
 const char **objs;
 const char **deps;
 const char **opts;
diff --git a/meson.build b/meson.build
index d1d3fd84ec..efba275092 100644
--- a/meson.build
+++ b/meson.build
@@ -2343,11 +2343,19 @@ foreach d, list : modules
 # unique when it comes to lookup in compile_commands.json.
 # Depnds on a mesion version with
 # https://github.com/mesonbuild/meson/pull/8900
-modinfo_files += custom_target(d + '-' + m + '.modinfo',
-   output: d + '-' + m + '.modinfo',
-   input: module_ss.sources() + genh,
-   capture: true,
-   command: [modinfo_collect, 
module_ss.sources()])
+if modules_arch.has_key(m)
+  modinfo_files += custom_target(d + '-' + m + '.modinfo',
+output: d + '-' + m + '.modinfo',
+input: module_ss.sources() + genh,
+capture: true,
+command: [modinfo_collect, 
module_ss.sources(), '--archs', modules_arch[m]])
+else
+  modinfo_files += custom_target(d + '-' + m + '.modinfo',
+output: d + '-' + m + '.modinfo',
+input: module_ss.sources() + genh,
+capture: true,
+command: [modinfo_collect, 
module_ss.sources()])
+endif
   endif
 else
   if d == 'block'
diff --git a/scripts/modinfo-collect.py b/scripts/modinfo-collect.py
index 4acb188c3e..739cd23e2f 100755
--- a/scripts/modinfo-collect.py
+++ b/scripts/modinfo-collect.py
@@ -50,6 +50,16 @@ def main(args):
 print("MODINFO_START arch \"%s\" MODINFO_END" % arch)
 with open('compile_commands.json') as f:
 compile_commands = json.load(f)
+
+try:
+arch_idx = args.index('--archs')
+archs = args[arch_idx + 1:]
+args = args[:arch_idx]
+for arch in archs:
+print("MODINFO_START arch \"%s\" MODINFO_END" % arch)
+except ValueError:
+pass
+
 for src in args:
 print("MODINFO_DEBUG src %s" % src)
 command = find_command(src, target, compile_commands)
diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py
index f559eed007..e1d13acd92 100755
--- a/scripts/modinfo-generate.py
+++ b/scripts/modinfo-generate.py
@@ -33,7 +33,7 @@ def parse_line(line):
 return (kind, data)
 
 def generate(name, lines):
-arch = ""
+archs = []
 objs = []
 deps = []
 opts = []
@@ -47,14 +47,13 @@ def generate(name, lines):
 elif kind == 'opts':
 opts.append(data)
 elif kind == 'arch':
-arch = data;
+archs.append(data);
 else:
 print("unknown:", kind)
 exit(1)
 
 print(".name = \"%s\"," % name)
-if arch != "":
-print(".arch = %s," % arch)
+print_array("archs", archs)
 print_array("objs", objs)
 print_array("deps", deps)
 print_array("opts", opts)
diff --git a/util/module.c b/util/module.c
index 6bb4ad915a..7009143bfc 100644
--- a/util/module.c
+++ b/util/module.c
@@ -131,16 +131,24 @@ void module_allow_arch(const char *arch)
 
 static bool module_check_arch(const QemuModinfo *modinfo)
 {

[PATCH 1/2] meson: introduce modules_arch

2021-09-16 Thread Jose R. Ziviani
This variable keeps track of all modules enabled for a target
architecture. This will be used in modinfo to refine the
architectures that can really load the .so to avoid errors.

Signed-off-by: Jose R. Ziviani 
---
 hw/display/meson.build | 48 ++
 hw/usb/meson.build | 36 +++
 meson.build|  1 +
 3 files changed, 85 insertions(+)

diff --git a/hw/display/meson.build b/hw/display/meson.build
index 861c43ff98..ba06f58ff1 100644
--- a/hw/display/meson.build
+++ b/hw/display/meson.build
@@ -43,6 +43,18 @@ if config_all_devices.has_key('CONFIG_QXL')
   qxl_ss.add(when: 'CONFIG_QXL', if_true: [files('qxl.c', 'qxl-logger.c', 
'qxl-render.c'),
pixman, spice])
   hw_display_modules += {'qxl': qxl_ss}
+
+  archs = []
+  foreach target: target_dirs
+if target.endswith('-softmmu')
+  cfg_target = config_target_mak[target]
+  if cfg_target.has_key('CONFIG_QXL') and cfg_target['CONFIG_QXL'] == 'y'
+archs += [cfg_target['TARGET_NAME']]
+  endif
+endif
+  endforeach
+
+  modules_arch += {'qxl': archs}
 endif
 
 softmmu_ss.add(when: 'CONFIG_DPCD', if_true: files('dpcd.c'))
@@ -65,6 +77,18 @@ if config_all_devices.has_key('CONFIG_VIRTIO_GPU')
   virtio_gpu_gl_ss.add(when: ['CONFIG_VIRTIO_GPU', virgl, opengl],
if_true: [files('virtio-gpu-gl.c', 
'virtio-gpu-virgl.c'), pixman, virgl])
   hw_display_modules += {'virtio-gpu-gl': virtio_gpu_gl_ss}
+
+  archs = []
+  foreach target: target_dirs
+if target.endswith('-softmmu')
+  cfg_target = config_target_mak[target]
+  if cfg_target.has_key('CONFIG_VIRTIO_GPU') and 
cfg_target['CONFIG_VIRTIO_GPU'] == 'y'
+archs += [cfg_target['TARGET_NAME']]
+  endif
+endif
+  endforeach
+
+  modules_arch += {'virtio-gpu': archs, 'virtio-gpu-gl': archs}
 endif
 
 if config_all_devices.has_key('CONFIG_VIRTIO_PCI')
@@ -79,6 +103,18 @@ if config_all_devices.has_key('CONFIG_VIRTIO_PCI')
   virtio_gpu_pci_gl_ss.add(when: ['CONFIG_VIRTIO_GPU', 'CONFIG_VIRTIO_PCI', 
virgl, opengl],
if_true: [files('virtio-gpu-pci-gl.c'), pixman])
   hw_display_modules += {'virtio-gpu-pci-gl': virtio_gpu_pci_gl_ss}
+
+  archs = []
+  foreach target: target_dirs
+if target.endswith('-softmmu')
+  cfg_target = config_target_mak[target]
+  if cfg_target.has_key('CONFIG_VIRTIO_PCI') and 
cfg_target['CONFIG_VIRTIO_PCI'] == 'y'
+archs += [cfg_target['TARGET_NAME']]
+  endif
+endif
+  endforeach
+
+  modules_arch += {'virtio-gpu-pci': archs, 'virtio-gpu-pci-gl': archs}
 endif
 
 if config_all_devices.has_key('CONFIG_VIRTIO_VGA')
@@ -93,6 +129,18 @@ if config_all_devices.has_key('CONFIG_VIRTIO_VGA')
   virtio_vga_gl_ss.add(when: ['CONFIG_VIRTIO_VGA', virgl, opengl],
if_true: [files('virtio-vga-gl.c'), pixman])
   hw_display_modules += {'virtio-vga-gl': virtio_vga_gl_ss}
+
+  archs = []
+  foreach target: target_dirs
+if target.endswith('-softmmu')
+  cfg_target = config_target_mak[target]
+  if cfg_target.has_key('CONFIG_VIRTIO_VGA') and 
cfg_target['CONFIG_VIRTIO_VGA'] == 'y'
+archs += [cfg_target['TARGET_NAME']]
+  endif
+endif
+  endforeach
+
+  modules_arch += {'virtio-vga': archs, 'virtio-vga-gl': archs}
 endif
 
 specific_ss.add(when: 'CONFIG_OMAP', if_true: files('omap_lcdc.c'))
diff --git a/hw/usb/meson.build b/hw/usb/meson.build
index de853d780d..6b889d2ee2 100644
--- a/hw/usb/meson.build
+++ b/hw/usb/meson.build
@@ -54,6 +54,18 @@ if cacard.found()
   usbsmartcard_ss.add(when: 'CONFIG_USB_SMARTCARD',
   if_true: [cacard, files('ccid-card-emulated.c', 
'ccid-card-passthru.c')])
   hw_usb_modules += {'smartcard': usbsmartcard_ss}
+
+  archs = []
+  foreach target: target_dirs
+if target.endswith('-softmmu')
+  cfg_target = config_target_mak[target]
+  if cfg_target.has_key('CONFIG_USB_SMARTCARD') and 
cfg_target['CONFIG_USB_SMARTCARD'] == 'y'
+archs += [cfg_target['TARGET_NAME']]
+  endif
+endif
+  endforeach
+
+  modules_arch += {'smartcard': archs}
 endif
 
 # U2F
@@ -69,6 +81,18 @@ if usbredir.found()
   usbredir_ss.add(when: 'CONFIG_USB',
   if_true: [usbredir, files('redirect.c', 'quirks.c')])
   hw_usb_modules += {'redirect': usbredir_ss}
+
+  archs = []
+  foreach target: target_dirs
+if target.endswith('-softmmu')
+  cfg_target = config_target_mak[target]
+  if cfg_target.has_key('CONFIG_USB') and cfg_target['CONFIG_USB'] == 'y'
+archs += [cfg_target['TARGET_NAME']]
+  endif
+endif
+  endforeach
+
+  modules_arch += {'redirect': archs}
 endif
 
 # usb pass-through
@@ -77,6 +101,18 @@ if libusb.found()
   usbhost_ss.add(when: ['CONFIG_USB', libusb],
  if_true: files('host-libusb.c'))
   hw_usb_modules += {'host': usbhost_ss}
+
+  archs = []
+  foreach target: target_dirs
+if target.endswith

[PATCH 0/2] modules: Improve modinfo.c architecture support

2021-09-16 Thread Jose R. Ziviani
When building a single target, the build system detects the architecture
and generates a modinfo.c with modules related to that arch only.
However, when more than one target is built, modinfo.c is generated with
modules available for each architecture - without any way to know what
arch supports what.

The problem is when executing the target, it will try to load modules
that is not supported by it and will throw errors to users, for
instance:

$ ./configure --enable-modules # all targets
$ ./qemu-system-avr -nodefaults -display none -accel tcg -M none -device help | 
head
Failed to open module: /.../hw-display-qxl.so: undefined symbol: vga_ioport_read
Failed to open module: /.../hw-display-virtio-gpu.so: undefined symbol: 
virtio_vmstate_info
Failed to open module: /.../hw-display-virtio-gpu.so: undefined symbol: 
virtio_vmstate_info
Failed to open module: /.../hw-display-virtio-gpu-gl.so: undefined symbol: 
virtio_gpu_ctrl_response
Failed to open module: /.../hw-display-virtio-gpu-pci.so: undefined symbol: 
virtio_instance_init_common
Failed to open module: /.../hw-display-virtio-gpu-pci.so: undefined symbol: 
virtio_instance_init_common
Failed to open module: /.../hw-display-virtio-gpu-pci-gl.so: undefined symbol: 
virtio_instance_init_common
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga-gl.so: undefined symbol: 
have_vga
Failed to open module: /.../hw-usb-smartcard.so: undefined symbol: 
ccid_card_ccid_attach
Failed to open module: /.../hw-usb-redirect.so: undefined symbol: 
vmstate_usb_device
Failed to open module: /.../hw-usb-host.so: undefined symbol: vmstate_usb_device
...

$ ./qemu-system-s390x -nodefaults -display none -accel tcg -M none -device help 
| head 
Failed to open module: /.../hw-display-qxl.so: undefined symbol: vga_ioport_read
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga.so: undefined symbol: 
vmstate_vga_common
Failed to open module: /.../hw-display-virtio-vga-gl.so: undefined symbol: 
have_vga
Failed to open module: /.../hw-usb-smartcard.so: undefined symbol: 
ccid_card_ccid_attach
Failed to open module: /.../hw-usb-redirect.so: undefined symbol: 
vmstate_usb_device
Failed to open module: /.../hw-usb-host.so: undefined symbol: vmstate_usb_device

This patchset tries to improve by collecting the modules are currently
enabled for each target, obtaining information in meson from kconfig
and passing that to modinfo.c, which now uses a list to store supported
architectures for each module.

$ ./qemu-system-s390x -nodefaults -display none -accel tcg -M none -device help 
| head
Controller/Bridge/Hub devices:
name "pci-bridge", bus PCI, desc "Standard PCI Bridge"
name "pci-bridge-seat", bus PCI, desc "Standard PCI Bridge (multiseat)"
...

$ ./qemu-system-avr -nodefaults -display none -accel tcg -M none -device help | 
head
...
Misc devices:
name "guest-loader", desc "Guest Loader"
name "loader", desc "Generic Loader"

Jose R. Ziviani (2):
  meson: introduce modules_arch
  modules: use a list of supported arch for each module

 hw/display/meson.build  | 48 +
 hw/usb/meson.build  | 36 
 include/qemu/module.h   |  2 +-
 meson.build | 19 +++
 scripts/modinfo-collect.py  | 10 
 scripts/modinfo-generate.py |  7 +++---
 util/module.c   | 18 ++
 7 files changed, 125 insertions(+), 15 deletions(-)

-- 
2.33.0




[PATCH] tcg/arm: Fix tcg_out_vec_op function signature

2021-09-08 Thread Jose R. Ziviani
Commit 5e8892db93 fixed several function signatures but tcg_out_vec_op
for arm is missing. It causes a build error on armv6 and armv7:

tcg-target.c.inc:2718:42: error: argument 5 of type 'const TCGArg *'
{aka 'const unsigned int *'} declared as a pointer [-Werror=array-parameter=]
   const TCGArg *args, const int *const_args)
  ~~^~~~
../tcg/tcg.c:120:41: note: previously declared as an array 'const TCGArg[16]'
{aka 'const unsigned int[16]'}
   const TCGArg args[TCG_MAX_OP_ARGS],
  ~~^~~~

Signed-off-by: Jose R. Ziviani 
---
 tcg/arm/tcg-target.c.inc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index 007ceee68e..e5b4f86841 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -2715,7 +2715,8 @@ static const ARMInsn vec_cmp0_insn[16] = {
 
 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
unsigned vecl, unsigned vece,
-   const TCGArg *args, const int *const_args)
+   const TCGArg args[TCG_MAX_OP_ARGS],
+   const int const_args[TCG_MAX_OP_ARGS])
 {
 TCGType type = vecl + TCG_TYPE_V64;
 unsigned q = vecl;
-- 
2.33.0




Re: [PATCH] qemu-img: Allow target be aligned to sector size

2021-08-19 Thread Jose R. Ziviani
On Thu, Aug 19, 2021 at 05:14:30PM +0200, Hanna Reitz wrote:
> On 19.08.21 16:31, Jose R. Ziviani wrote:
> > Hello Hanna,
> > 
> > On Thu, Aug 19, 2021 at 12:12:00PM +0200, Hanna Reitz wrote:
> > > We cannot write to images opened with O_DIRECT unless we allow them to
> > > be resized so they are aligned to the sector size: Since 9c60a5d1978,
> > > bdrv_node_refresh_perm() ensures that for nodes whose length is not
> > > aligned to the request alignment and where someone has taken a WRITE
> > > permission, the RESIZE permission is taken, too).
> > > 
> > > Let qemu-img convert pass the BDRV_O_RESIZE flag (which causes
> > > blk_new_open() to take the RESIZE permission) when using cache=none for
> > > the target, so that when writing to it, it can be aligned to the target
> > > sector size.
> > > 
> > > Without this patch, an error is returned:
> > > 
> > > $ qemu-img convert -f raw -O raw -t none foo.img /mnt/tmp/foo.img
> > > qemu-img: Could not open '/mnt/tmp/foo.img': Cannot get 'write'
> > > permission without 'resize': Image size is not a multiple of request
> > > alignment
> > > 
> > > Buglink: https://bugzilla.redhat.com/show_bug.cgi?id=1994266
> > > Signed-off-by: Hanna Reitz 
> > > ---
> > > As I have written in the BZ linked above, I am not sure what behavior we
> > > want.  It can be argued that the current behavior is perfectly OK
> > > because we want the target to have the same size as the source, so if
> > > this cannot be done, we should just print the above error (which I think
> > > explains the problem well enough that users can figure out they need to
> > > resize the source image).
> > > 
> > > OTOH, it is difficult for me to imagine a case where the user would
> > > prefer the above error to just having qemu-img align the target image's
> > > length.
> > This is timely convenient because I'm currently investigating an issue 
> > detected
> > by a libvirt-tck test:
> > 
> > https://gitlab.com/libvirt/libvirt-tck/-/blob/master/scripts/qemu/200-qcow2-single-backing-file.t
> > 
> > It fails with the same message:
> > qemu-system-x86_64: -device 
> > virtio-blk-pci,bus=pci.0,addr=0x3,drive=libvirt-1-format,id=virtio-disk1,write-cache=on:
> >  Cannot get 'write' permission without 'resize': Image size is not a 
> > multiple of request alignment
> > 
> > Libvirt-tck basically wants to make sure that libvirt won't pass a 'backing'
> > argument if we force a QCOW2 image to be interpreted as a RAW image. But, 
> > the
> > actual size of a (not preallocated) QCOW2 is usually unaligned so we ended 
> > up
> > failing at that requirement.
> > 
> > I crafted a reproducer (tck-main is a QCOW2 image):
> >   $ ./qemu-system-x86_64 \
> >-machine pc-i440fx-6.0,accel=kvm -m 1024 -display none -no-user-config 
> > -nodefaults \
> >-kernel vmlinuz -initrd initrd \
> >-blockdev 
> > driver=file,filename=/var/cache/libvirt-tck/storage-fs/tck/tck-main,node-name=a,cache.direct=on
> >  \
> >-blockdev node-name=name,driver=raw,file=a \
> >-device virtio-blk-pci,drive=name
> > 
> > And if I remove 'cache.direct=on' OR if I remove the device virtio-blk-pci I
> > don't hit the failure.
> > 
> > In order to fix the libvirt-tck test case, I think that creating a 
> > preallocated
> > QCOW2 image is the best approach, considering their test case goal. But, if 
> > you
> > don't mind, could you explain why cache.direct=on doesn't set resize 
> > permission
> > with virtio-blk-pci?
> 
> Well, users only take the permissions they need.  Because the device doesn’t
> actively want to resize the disk, it doesn’t take the permission (because
> other simultaneous users might not share that permission, and so taking more
> permissions than you need may cause problems).
> 
> So the decision whether to take the permission or not is a tradeoff.  I
> mean, there’s always a work-around: The error message tells you that the
> image is not aligned to the request alignment, so you can align the image
> size manually.  The question is whether taking the permissions may be
> problematic, and whether the user can be expected to align the image size.
> 
> (And we also need to keep in mind that this case is extremely rare. I don’t
> think that users in practice will ever have images that are not 4k-aligned. 
> What the test is doing is of course something that would never happen in a
> practical set-up, in fact, I believe attaching a qcow

Re: [PATCH] qemu-img: Allow target be aligned to sector size

2021-08-19 Thread Jose R. Ziviani
Hello Hanna,

On Thu, Aug 19, 2021 at 12:12:00PM +0200, Hanna Reitz wrote:
> We cannot write to images opened with O_DIRECT unless we allow them to
> be resized so they are aligned to the sector size: Since 9c60a5d1978,
> bdrv_node_refresh_perm() ensures that for nodes whose length is not
> aligned to the request alignment and where someone has taken a WRITE
> permission, the RESIZE permission is taken, too).
> 
> Let qemu-img convert pass the BDRV_O_RESIZE flag (which causes
> blk_new_open() to take the RESIZE permission) when using cache=none for
> the target, so that when writing to it, it can be aligned to the target
> sector size.
> 
> Without this patch, an error is returned:
> 
> $ qemu-img convert -f raw -O raw -t none foo.img /mnt/tmp/foo.img
> qemu-img: Could not open '/mnt/tmp/foo.img': Cannot get 'write'
> permission without 'resize': Image size is not a multiple of request
> alignment
> 
> Buglink: https://bugzilla.redhat.com/show_bug.cgi?id=1994266
> Signed-off-by: Hanna Reitz 
> ---
> As I have written in the BZ linked above, I am not sure what behavior we
> want.  It can be argued that the current behavior is perfectly OK
> because we want the target to have the same size as the source, so if
> this cannot be done, we should just print the above error (which I think
> explains the problem well enough that users can figure out they need to
> resize the source image).
> 
> OTOH, it is difficult for me to imagine a case where the user would
> prefer the above error to just having qemu-img align the target image's
> length.

This is timely convenient because I'm currently investigating an issue detected
by a libvirt-tck test:

https://gitlab.com/libvirt/libvirt-tck/-/blob/master/scripts/qemu/200-qcow2-single-backing-file.t

It fails with the same message:
qemu-system-x86_64: -device 
virtio-blk-pci,bus=pci.0,addr=0x3,drive=libvirt-1-format,id=virtio-disk1,write-cache=on:
 Cannot get 'write' permission without 'resize': Image size is not a multiple 
of request alignment

Libvirt-tck basically wants to make sure that libvirt won't pass a 'backing'
argument if we force a QCOW2 image to be interpreted as a RAW image. But, the
actual size of a (not preallocated) QCOW2 is usually unaligned so we ended up
failing at that requirement.

I crafted a reproducer (tck-main is a QCOW2 image):
 $ ./qemu-system-x86_64 \
  -machine pc-i440fx-6.0,accel=kvm -m 1024 -display none -no-user-config 
-nodefaults \
  -kernel vmlinuz -initrd initrd \
  -blockdev 
driver=file,filename=/var/cache/libvirt-tck/storage-fs/tck/tck-main,node-name=a,cache.direct=on
 \
  -blockdev node-name=name,driver=raw,file=a \
  -device virtio-blk-pci,drive=name

And if I remove 'cache.direct=on' OR if I remove the device virtio-blk-pci I
don't hit the failure.

In order to fix the libvirt-tck test case, I think that creating a preallocated
QCOW2 image is the best approach, considering their test case goal. But, if you
don't mind, could you explain why cache.direct=on doesn't set resize permission
with virtio-blk-pci?

Thank you!

> ---
>  qemu-img.c | 8 
>  1 file changed, 8 insertions(+)
> 
> diff --git a/qemu-img.c b/qemu-img.c
> index 908fd0cce5..d4b29bf73e 100644
> --- a/qemu-img.c
> +++ b/qemu-img.c
> @@ -2628,6 +2628,14 @@ static int img_convert(int argc, char **argv)
>  goto out;
>  }
>  
> +if (flags & BDRV_O_NOCACHE) {
> +/*
> + * If we open the target with O_DIRECT, it may be necessary to
> + * extend its size to align to the physical sector size.
> + */
> +flags |= BDRV_O_RESIZE;
> +}
> +
>  if (skip_create) {
>  s.target = img_open(tgt_image_opts, out_filename, out_fmt,
>  flags, writethrough, s.quiet, false);
> -- 
> 2.31.1
> 
> 


signature.asc
Description: Digital signature


[PATCH v4] vga: don't abort when adding a duplicate isa-vga device

2021-08-17 Thread Jose R. Ziviani
If users try to add an isa-vga device that was already registered,
still in command line, qemu will crash:

$ qemu-system-mips64el -M pica61 -device isa-vga
RAMBlock "vga.vram" already registered, abort!
Aborted (core dumped)

That particular board registers the device automaticaly, so it's
not obvious that a VGA device already exists. This patch changes
this behavior by displaying a message and exiting without crashing.

Resolves: https://gitlab.com/qemu-project/qemu/-/issues/44
Signed-off-by: Jose R. Ziviani 
---
v3 to v4: Used object_resolve_path_type instead of qemu_ram_block_by_name
  and copied the message from virgl, to give the same user exp.
v2 to v3: Improved error message
v1 to v2: Use error_setg instead of error_report

 hw/display/vga-isa.c | 10 ++
 1 file changed, 10 insertions(+)

diff --git a/hw/display/vga-isa.c b/hw/display/vga-isa.c
index 90851e730b..8cea84f2be 100644
--- a/hw/display/vga-isa.c
+++ b/hw/display/vga-isa.c
@@ -33,6 +33,7 @@
 #include "hw/loader.h"
 #include "hw/qdev-properties.h"
 #include "qom/object.h"
+#include "qapi/error.h"
 
 #define TYPE_ISA_VGA "isa-vga"
 OBJECT_DECLARE_SIMPLE_TYPE(ISAVGAState, ISA_VGA)
@@ -61,6 +62,15 @@ static void vga_isa_realizefn(DeviceState *dev, Error **errp)
 MemoryRegion *vga_io_memory;
 const MemoryRegionPortio *vga_ports, *vbe_ports;
 
+/*
+ * make sure this device is not being added twice, if so
+ * exit without crashing qemu
+ */
+if (object_resolve_path_type("", TYPE_ISA_VGA, NULL)) {
+error_setg(errp, "at most one %s device is permitted", TYPE_ISA_VGA);
+return;
+}
+
 s->global_vmstate = true;
 vga_common_init(s, OBJECT(dev));
 s->legacy_address_space = isa_address_space(isadev);
-- 
2.32.0




Re: [PATCH v3] vga: don't abort when adding a duplicate isa-vga device

2021-08-17 Thread Jose R. Ziviani
On Tue, Aug 17, 2021 at 10:07:55AM +0200, Philippe Mathieu-Daudé wrote:
> On 8/17/21 9:36 AM, Mark Cave-Ayland wrote:
> > On 17/08/2021 08:25, Thomas Huth wrote:
> > 
> >> On 16/08/2021 15.55, Jose R. Ziviani wrote:
> >>> If users try to add an isa-vga device that was already registered,
> >>> still in command line, qemu will crash:
> >>>
> >>> $ qemu-system-mips64el -M pica61 -device isa-vga
> >>> RAMBlock "vga.vram" already registered, abort!
> >>> Aborted (core dumped)
> >>>
> >>> That particular board registers the device automaticaly, so it's
> >>> not obvious that a VGA device already exists. This patch changes
> >>> this behavior by displaying a message and exiting without crashing.
> >>>
> >>> Resolves: https://gitlab.com/qemu-project/qemu/-/issues/44
> >>> Reviewed-by: Philippe Mathieu-Daudé 
> >>> Signed-off-by: Jose R. Ziviani 
> >>> ---
> >>> v2 to v3: Improved error message
> >>> v1 to v2: Use error_setg instead of error_report
> >>>
> >>>   hw/display/vga-isa.c | 10 ++
> >>>   1 file changed, 10 insertions(+)
> >>>
> >>> diff --git a/hw/display/vga-isa.c b/hw/display/vga-isa.c
> >>> index 90851e730b..30d55b41c3 100644
> >>> --- a/hw/display/vga-isa.c
> >>> +++ b/hw/display/vga-isa.c
> >>> @@ -33,6 +33,7 @@
> >>>   #include "hw/loader.h"
> >>>   #include "hw/qdev-properties.h"
> >>>   #include "qom/object.h"
> >>> +#include "qapi/error.h"
> >>>   #define TYPE_ISA_VGA "isa-vga"
> >>>   OBJECT_DECLARE_SIMPLE_TYPE(ISAVGAState, ISA_VGA)
> >>> @@ -61,6 +62,15 @@ static void vga_isa_realizefn(DeviceState *dev,
> >>> Error **errp)
> >>>   MemoryRegion *vga_io_memory;
> >>>   const MemoryRegionPortio *vga_ports, *vbe_ports;
> >>> +    /*
> >>> + * make sure this device is not being added twice, if so
> >>> + * exit without crashing qemu
> >>> + */
> >>> +    if (qemu_ram_block_by_name("vga.vram")) {
> >>> +    error_setg(errp, "'isa-vga' device already registered");
> >>> +    return;
> >>> +    }
> >>> +
> >>>   s->global_vmstate = true;
> >>>   vga_common_init(s, OBJECT(dev));
> >>>   s->legacy_address_space = isa_address_space(isadev);
> >>>
> >>
> >> Reviewed-by: Thomas Huth 
> > 
> > Instead of checking for the presence of the vga.vram block, would it be
> > better to use the standard object_resolve_path_type() method to check
> > for the presence of the existing isa-vga device instead? See
> > https://lists.gnu.org/archive/html/qemu-devel/2021-07/msg00717.html for
> > how this was done for virgl.
> 
> I remembered there was a nicer way but couldn't find it.
> If this patch were for 6.1, it was good enough. Now it
> will be merged in 6.2, I prefer Mark's suggestion.
> Jose, do you mind a v4?
> 

Hello people! Thanks for reviewing it.

Sure, I'll send a v4. It's not for 6.1 anyway.

Thank you very much


signature.asc
Description: Digital signature


[PATCH v3] vga: don't abort when adding a duplicate isa-vga device

2021-08-16 Thread Jose R. Ziviani
If users try to add an isa-vga device that was already registered,
still in command line, qemu will crash:

$ qemu-system-mips64el -M pica61 -device isa-vga
RAMBlock "vga.vram" already registered, abort!
Aborted (core dumped)

That particular board registers the device automaticaly, so it's
not obvious that a VGA device already exists. This patch changes
this behavior by displaying a message and exiting without crashing.

Resolves: https://gitlab.com/qemu-project/qemu/-/issues/44
Reviewed-by: Philippe Mathieu-Daudé 
Signed-off-by: Jose R. Ziviani 
---
v2 to v3: Improved error message
v1 to v2: Use error_setg instead of error_report

 hw/display/vga-isa.c | 10 ++
 1 file changed, 10 insertions(+)

diff --git a/hw/display/vga-isa.c b/hw/display/vga-isa.c
index 90851e730b..30d55b41c3 100644
--- a/hw/display/vga-isa.c
+++ b/hw/display/vga-isa.c
@@ -33,6 +33,7 @@
 #include "hw/loader.h"
 #include "hw/qdev-properties.h"
 #include "qom/object.h"
+#include "qapi/error.h"
 
 #define TYPE_ISA_VGA "isa-vga"
 OBJECT_DECLARE_SIMPLE_TYPE(ISAVGAState, ISA_VGA)
@@ -61,6 +62,15 @@ static void vga_isa_realizefn(DeviceState *dev, Error **errp)
 MemoryRegion *vga_io_memory;
 const MemoryRegionPortio *vga_ports, *vbe_ports;
 
+/*
+ * make sure this device is not being added twice, if so
+ * exit without crashing qemu
+ */
+if (qemu_ram_block_by_name("vga.vram")) {
+error_setg(errp, "'isa-vga' device already registered");
+return;
+}
+
 s->global_vmstate = true;
 vga_common_init(s, OBJECT(dev));
 s->legacy_address_space = isa_address_space(isadev);
-- 
2.32.0




[PATCH v2] vga: don't abort when adding a duplicate isa-vga device

2021-08-16 Thread Jose R. Ziviani
If users try to add an isa-vga device that was already registered,
still in command line, qemu will crash:

$ qemu-system-mips64el -M pica61 -device isa-vga
RAMBlock "vga.vram" already registered, abort!
Aborted (core dumped)

That particular board registers the device automaticaly, so it's
not obvious that a VGA device already exists. This patch changes
this behavior by displaying a message and exiting without crashing.

Resolves: https://gitlab.com/qemu-project/qemu/-/issues/44
Signed-off-by: Jose R. Ziviani 
---
 hw/display/vga-isa.c | 10 ++
 1 file changed, 10 insertions(+)

diff --git a/hw/display/vga-isa.c b/hw/display/vga-isa.c
index 90851e730b..1fba33b22b 100644
--- a/hw/display/vga-isa.c
+++ b/hw/display/vga-isa.c
@@ -33,6 +33,7 @@
 #include "hw/loader.h"
 #include "hw/qdev-properties.h"
 #include "qom/object.h"
+#include "qapi/error.h"
 
 #define TYPE_ISA_VGA "isa-vga"
 OBJECT_DECLARE_SIMPLE_TYPE(ISAVGAState, ISA_VGA)
@@ -61,6 +62,15 @@ static void vga_isa_realizefn(DeviceState *dev, Error **errp)
 MemoryRegion *vga_io_memory;
 const MemoryRegionPortio *vga_ports, *vbe_ports;
 
+/*
+ * make sure this device is not being added twice, if so
+ * exit without crashing qemu
+ */
+if (qemu_ram_block_by_name("vga.vram")) {
+error_setg(errp, "vga.vram is already registered");
+return;
+}
+
 s->global_vmstate = true;
 vga_common_init(s, OBJECT(dev));
 s->legacy_address_space = isa_address_space(isadev);
-- 
2.32.0




[PATCH] vga: don't abort when adding a duplicate isa-vga device

2021-08-13 Thread Jose R. Ziviani
If users try to add an isa-vga device that was already registered,
still in command line, qemu will crash:

$ qemu-system-mips64el -M pica61 -device isa-vga
RAMBlock "vga.vram" already registered, abort!
Aborted (core dumped)

That particular board registers such device automaticaly, so it's
not obvious that a VGA device already exists. This patch changes
this behavior by displaying a message and ignoring that device,
starting qemu normally.

Resolves: https://gitlab.com/qemu-project/qemu/-/issues/44
Signed-off-by: Jose R. Ziviani 
---
 hw/display/vga-isa.c | 9 +
 1 file changed, 9 insertions(+)

diff --git a/hw/display/vga-isa.c b/hw/display/vga-isa.c
index 90851e730b..69db502dde 100644
--- a/hw/display/vga-isa.c
+++ b/hw/display/vga-isa.c
@@ -61,6 +61,15 @@ static void vga_isa_realizefn(DeviceState *dev, Error **errp)
 MemoryRegion *vga_io_memory;
 const MemoryRegionPortio *vga_ports, *vbe_ports;
 
+/*
+ * some machines register VGA by default, so instead of aborting
+ * it, show a message and ignore this device.
+ */
+if (qemu_ram_block_by_name("vga.vram")) {
+error_report("vga.vram is already registered, ignoring this device");
+return;
+}
+
 s->global_vmstate = true;
 vga_common_init(s, OBJECT(dev));
 s->legacy_address_space = isa_address_space(isadev);
-- 
2.32.0




Re: [PATCH v2 1/1] modules: Improve error message when module is not found

2021-07-23 Thread Jose R. Ziviani
On Fri, Jul 23, 2021 at 05:27:25PM +0200, Claudio Fontana wrote:
> On 7/23/21 4:36 PM, Jose R. Ziviani wrote:
> > On Fri, Jul 23, 2021 at 04:02:26PM +0200, Claudio Fontana wrote:
> >> On 7/23/21 3:50 PM, Jose R. Ziviani wrote:
> >>> On Fri, Jul 23, 2021 at 11:41:19AM +0200, Claudio Fontana wrote:
> >>>> On 7/23/21 12:09 AM, Jose R. Ziviani wrote:
> >>>>> When a module is not found, specially accelerators, QEMU displays
> >>>>> a error message that not easy to understand[1]. This patch improves
> >>>>> the readability by offering a user-friendly message[2].
> >>>>>
> >>>>> This patch also moves the accelerator ops check to runtine (instead
> >>>>> of the original g_assert) because it works better with dynamic
> >>>>> modules.
> >>>>>
> >>>>> [1] qemu-system-x86_64 -accel tcg
> >>>>> ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion 
> >>>>> failed:
> >>>>> (ops != NULL)
> >>>>> Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces:
> >>>>> assertion failed: (ops != NULL)
> >>>>> 31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...
> >>>>>
> >>>>> [2] qemu-system-x86_64 -accel tcg
> >>>>> accel-tcg-x86_64 module is missing, install the package or config the 
> >>>>> library path correctly.
> >>>>>
> >>>>> Signed-off-by: Jose R. Ziviani 
> >>>>> ---
> >>>>>  accel/accel-softmmu.c |  5 -
> >>>>>  util/module.c | 14 --
> >>>>>  2 files changed, 12 insertions(+), 7 deletions(-)
> >>>>>
> >>>>> diff --git a/accel/accel-softmmu.c b/accel/accel-softmmu.c
> >>>>> index 67276e4f52..52449ac2d0 100644
> >>>>> --- a/accel/accel-softmmu.c
> >>>>> +++ b/accel/accel-softmmu.c
> >>>>> @@ -79,7 +79,10 @@ void accel_init_ops_interfaces(AccelClass *ac)
> >>>>>   * all accelerators need to define ops, providing at least a 
> >>>>> mandatory
> >>>>>   * non-NULL create_vcpu_thread operation.
> >>>>>   */
> >>>>> -g_assert(ops != NULL);
> >>>>> +if (ops == NULL) {
> >>>>> +exit(1);
> >>>>> +}
> >>>>> +
> >>>>
> >>>>
> >>>> Ah, again, why?
> >>>> This change looks wrong to me, 
> >>>>
> >>>> the ops code should be present when ops interfaces are initialized:
> >>>> it should be a code level assertion, as it has to do with the proper 
> >>>> order of initializations in QEMU,
> >>>>
> >>>> why would we want to do anything else but to assert here?
> >>>>
> >>>> Am I blind to something obvious?
> >>>
> >>> Hello!
> >>>
> >>> Thank you for reviewing it!
> >>>
> >>> The problem is that if your TCG module is not installed and you start
> >>> QEMU like:
> >>>
> >>> ./qemu-system-x86_64 -accel tcg
> >>>
> >>> You'll get the error message + a crash with a core dump:
> >>>
> >>> accel-tcg-x86_64 module is missing, install the package or config the 
> >>> library path correctly.
> >>> **
> >>> ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion 
> >>> failed: (ops != NULL)
> >>> Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: 
> >>> assertion failed: (ops != NULL)
> >>> [1]5740 IOT instruction (core dumped)  ./qemu-system-x86_64 -accel tcg
> >>>
> >>> I was digging a little bit more in order to move this responsibility to
> >>> module.c but there isn't enough information there to safely exit() in
> >>> all situations that a module may be loaded. As Gerd mentioned, more work
> >>> is needed in order to achieve that.
> >>>
> >>> However, it's not nice to have a crash due to an optional module missing.
> >>> It's specially confusing because TCG has always been native. Considering
> >>> also that we're already in hard freeze for 6.1, I thought to have this
> >>> simpler check instead.
> >>>
>

Re: [PATCH v2 1/1] modules: Improve error message when module is not found

2021-07-23 Thread Jose R. Ziviani
On Fri, Jul 23, 2021 at 04:02:26PM +0200, Claudio Fontana wrote:
> On 7/23/21 3:50 PM, Jose R. Ziviani wrote:
> > On Fri, Jul 23, 2021 at 11:41:19AM +0200, Claudio Fontana wrote:
> >> On 7/23/21 12:09 AM, Jose R. Ziviani wrote:
> >>> When a module is not found, specially accelerators, QEMU displays
> >>> a error message that not easy to understand[1]. This patch improves
> >>> the readability by offering a user-friendly message[2].
> >>>
> >>> This patch also moves the accelerator ops check to runtine (instead
> >>> of the original g_assert) because it works better with dynamic
> >>> modules.
> >>>
> >>> [1] qemu-system-x86_64 -accel tcg
> >>> ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion 
> >>> failed:
> >>> (ops != NULL)
> >>> Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces:
> >>> assertion failed: (ops != NULL)
> >>> 31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...
> >>>
> >>> [2] qemu-system-x86_64 -accel tcg
> >>> accel-tcg-x86_64 module is missing, install the package or config the 
> >>> library path correctly.
> >>>
> >>> Signed-off-by: Jose R. Ziviani 
> >>> ---
> >>>  accel/accel-softmmu.c |  5 -
> >>>  util/module.c | 14 --
> >>>  2 files changed, 12 insertions(+), 7 deletions(-)
> >>>
> >>> diff --git a/accel/accel-softmmu.c b/accel/accel-softmmu.c
> >>> index 67276e4f52..52449ac2d0 100644
> >>> --- a/accel/accel-softmmu.c
> >>> +++ b/accel/accel-softmmu.c
> >>> @@ -79,7 +79,10 @@ void accel_init_ops_interfaces(AccelClass *ac)
> >>>   * all accelerators need to define ops, providing at least a 
> >>> mandatory
> >>>   * non-NULL create_vcpu_thread operation.
> >>>   */
> >>> -g_assert(ops != NULL);
> >>> +if (ops == NULL) {
> >>> +exit(1);
> >>> +}
> >>> +
> >>
> >>
> >> Ah, again, why?
> >> This change looks wrong to me, 
> >>
> >> the ops code should be present when ops interfaces are initialized:
> >> it should be a code level assertion, as it has to do with the proper order 
> >> of initializations in QEMU,
> >>
> >> why would we want to do anything else but to assert here?
> >>
> >> Am I blind to something obvious?
> > 
> > Hello!
> > 
> > Thank you for reviewing it!
> > 
> > The problem is that if your TCG module is not installed and you start
> > QEMU like:
> > 
> > ./qemu-system-x86_64 -accel tcg
> > 
> > You'll get the error message + a crash with a core dump:
> > 
> > accel-tcg-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > **
> > ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion 
> > failed: (ops != NULL)
> > Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: 
> > assertion failed: (ops != NULL)
> > [1]5740 IOT instruction (core dumped)  ./qemu-system-x86_64 -accel tcg
> > 
> > I was digging a little bit more in order to move this responsibility to
> > module.c but there isn't enough information there to safely exit() in
> > all situations that a module may be loaded. As Gerd mentioned, more work
> > is needed in order to achieve that.
> > 
> > However, it's not nice to have a crash due to an optional module missing.
> > It's specially confusing because TCG has always been native. Considering
> > also that we're already in hard freeze for 6.1, I thought to have this
> > simpler check instead.
> > 
> > What do you think if we have something like:
> > 
> > /* FIXME: this isn't the right place to handle a missing module and
> >must be reverted when the module refactoring is completely done */
> > #ifdef CONFIG_MODULES
> > if (ops == NULL) {
> > exit(1);
> > }
> > #else
> > g_assert(ops != NULL);
> > #endif
> > 
> > Regards!
> 
> 
> For the normal builds (without modular tcg), this issue does not appear right?

Yes, but OpenSUSE already builds with --enable-modules, we've already been 
shipping
several modules as optional RPMs, like qemu-hw-display-virtio-gpu for example. 
I sent
a patch some weeks ago to add "--enable-tcg-builtin" in the build system but 
there're
more work required in that area 

Re: [PATCH v2 1/1] modules: Improve error message when module is not found

2021-07-23 Thread Jose R. Ziviani
On Fri, Jul 23, 2021 at 11:41:19AM +0200, Claudio Fontana wrote:
> On 7/23/21 12:09 AM, Jose R. Ziviani wrote:
> > When a module is not found, specially accelerators, QEMU displays
> > a error message that not easy to understand[1]. This patch improves
> > the readability by offering a user-friendly message[2].
> > 
> > This patch also moves the accelerator ops check to runtine (instead
> > of the original g_assert) because it works better with dynamic
> > modules.
> > 
> > [1] qemu-system-x86_64 -accel tcg
> > ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion 
> > failed:
> > (ops != NULL)
> > Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces:
> > assertion failed: (ops != NULL)
> > 31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...
> > 
> > [2] qemu-system-x86_64 -accel tcg
> > accel-tcg-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > 
> > Signed-off-by: Jose R. Ziviani 
> > ---
> >  accel/accel-softmmu.c |  5 -
> >  util/module.c | 14 --
> >  2 files changed, 12 insertions(+), 7 deletions(-)
> > 
> > diff --git a/accel/accel-softmmu.c b/accel/accel-softmmu.c
> > index 67276e4f52..52449ac2d0 100644
> > --- a/accel/accel-softmmu.c
> > +++ b/accel/accel-softmmu.c
> > @@ -79,7 +79,10 @@ void accel_init_ops_interfaces(AccelClass *ac)
> >   * all accelerators need to define ops, providing at least a mandatory
> >   * non-NULL create_vcpu_thread operation.
> >   */
> > -g_assert(ops != NULL);
> > +if (ops == NULL) {
> > +exit(1);
> > +}
> > +
> 
> 
> Ah, again, why?
> This change looks wrong to me, 
> 
> the ops code should be present when ops interfaces are initialized:
> it should be a code level assertion, as it has to do with the proper order of 
> initializations in QEMU,
> 
> why would we want to do anything else but to assert here?
> 
> Am I blind to something obvious?

Hello!

Thank you for reviewing it!

The problem is that if your TCG module is not installed and you start
QEMU like:

./qemu-system-x86_64 -accel tcg

You'll get the error message + a crash with a core dump:

accel-tcg-x86_64 module is missing, install the package or config the library 
path correctly.
**
ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion failed: 
(ops != NULL)
Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: 
assertion failed: (ops != NULL)
[1]5740 IOT instruction (core dumped)  ./qemu-system-x86_64 -accel tcg

I was digging a little bit more in order to move this responsibility to
module.c but there isn't enough information there to safely exit() in
all situations that a module may be loaded. As Gerd mentioned, more work
is needed in order to achieve that.

However, it's not nice to have a crash due to an optional module missing.
It's specially confusing because TCG has always been native. Considering
also that we're already in hard freeze for 6.1, I thought to have this
simpler check instead.

What do you think if we have something like:

/* FIXME: this isn't the right place to handle a missing module and
   must be reverted when the module refactoring is completely done */
#ifdef CONFIG_MODULES
if (ops == NULL) {
exit(1);
}
#else
g_assert(ops != NULL);
#endif

Regards!

> 
> >  if (ops->ops_init) {
> >  ops->ops_init(ops);
> >  }
> > diff --git a/util/module.c b/util/module.c
> > index 6bb4ad915a..268a8563fd 100644
> > --- a/util/module.c
> > +++ b/util/module.c
> > @@ -206,13 +206,10 @@ static int module_load_file(const char *fname, bool 
> > mayfail, bool export_symbols
> >  out:
> >  return ret;
> >  }
> > -#endif
> >  
> >  bool module_load_one(const char *prefix, const char *lib_name, bool 
> > mayfail)
> >  {
> >  bool success = false;
> > -
> > -#ifdef CONFIG_MODULES
> >  char *fname = NULL;
> >  #ifdef CONFIG_MODULE_UPGRADES
> >  char *version_dir;
> > @@ -300,6 +297,9 @@ bool module_load_one(const char *prefix, const char 
> > *lib_name, bool mayfail)
> >  
> >  if (!success) {
> >  g_hash_table_remove(loaded_modules, module_name);
> > +fprintf(stderr, "%s module is missing, install the "
> > +"package or config the library path "
> > +"correctly.\n", module_name);
> >  g_free(module_name);
> >  }
> >  
> > @@ -307,12 +307,9 @@ bool module_load_one(const char *prefix, c

[PATCH v2 1/1] modules: Improve error message when module is not found

2021-07-22 Thread Jose R. Ziviani
When a module is not found, specially accelerators, QEMU displays
a error message that not easy to understand[1]. This patch improves
the readability by offering a user-friendly message[2].

This patch also moves the accelerator ops check to runtine (instead
of the original g_assert) because it works better with dynamic
modules.

[1] qemu-system-x86_64 -accel tcg
ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion failed:
(ops != NULL)
Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces:
assertion failed: (ops != NULL)
31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...

[2] qemu-system-x86_64 -accel tcg
accel-tcg-x86_64 module is missing, install the package or config the library 
path correctly.

Signed-off-by: Jose R. Ziviani 
---
 accel/accel-softmmu.c |  5 -
 util/module.c | 14 --
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/accel/accel-softmmu.c b/accel/accel-softmmu.c
index 67276e4f52..52449ac2d0 100644
--- a/accel/accel-softmmu.c
+++ b/accel/accel-softmmu.c
@@ -79,7 +79,10 @@ void accel_init_ops_interfaces(AccelClass *ac)
  * all accelerators need to define ops, providing at least a mandatory
  * non-NULL create_vcpu_thread operation.
  */
-g_assert(ops != NULL);
+if (ops == NULL) {
+exit(1);
+}
+
 if (ops->ops_init) {
 ops->ops_init(ops);
 }
diff --git a/util/module.c b/util/module.c
index 6bb4ad915a..268a8563fd 100644
--- a/util/module.c
+++ b/util/module.c
@@ -206,13 +206,10 @@ static int module_load_file(const char *fname, bool 
mayfail, bool export_symbols
 out:
 return ret;
 }
-#endif
 
 bool module_load_one(const char *prefix, const char *lib_name, bool mayfail)
 {
 bool success = false;
-
-#ifdef CONFIG_MODULES
 char *fname = NULL;
 #ifdef CONFIG_MODULE_UPGRADES
 char *version_dir;
@@ -300,6 +297,9 @@ bool module_load_one(const char *prefix, const char 
*lib_name, bool mayfail)
 
 if (!success) {
 g_hash_table_remove(loaded_modules, module_name);
+fprintf(stderr, "%s module is missing, install the "
+"package or config the library path "
+"correctly.\n", module_name);
 g_free(module_name);
 }
 
@@ -307,12 +307,9 @@ bool module_load_one(const char *prefix, const char 
*lib_name, bool mayfail)
 g_free(dirs[i]);
 }
 
-#endif
 return success;
 }
 
-#ifdef CONFIG_MODULES
-
 static bool module_loaded_qom_all;
 
 void module_load_qom_one(const char *type)
@@ -384,4 +381,9 @@ void qemu_load_module_for_opts(const char *group) {}
 void module_load_qom_one(const char *type) {}
 void module_load_qom_all(void) {}
 
+bool module_load_one(const char *prefix, const char *lib_name, bool mayfail)
+{
+return false;
+}
+
 #endif
-- 
2.32.0




[PATCH v2 0/1] Improve module accelerator error message

2021-07-22 Thread Jose R. Ziviani
v1 -> v2:
* Moved the code to module.c
* Simplified a lot by using current module DB to get info

The main objective is to improve the error message when trying to
load a not found/not installed module TCG.

For example:

$ qemu-system-x86_64 -accel tcg
ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion failed: 
(ops != NULL)
Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: 
assertion failed: (ops != NULL)
[1]31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...

To:
$ qemu-system-x86_64 -accel tcg
accel-tcg-x86_64 module is missing, install the package or config the library 
path correctly.

Jose R. Ziviani (1):
  modules: Improve error message when module is not found

 accel/accel-softmmu.c |  5 -
 util/module.c | 14 --
 2 files changed, 12 insertions(+), 7 deletions(-)

-- 
2.32.0




Re: [RFC 3/3] qom: Improve error message in module_object_class_by_name()

2021-07-21 Thread Jose R. Ziviani
On Wed, Jul 21, 2021 at 10:57:37AM +0100, Daniel P. Berrangé wrote:
> On Wed, Jul 21, 2021 at 11:54:45AM +0200, Gerd Hoffmann wrote:
> > >  ObjectClass *module_object_class_by_name(const char *typename)
> > >  {
> > >  ObjectClass *oc;
> > > @@ -1031,8 +1049,20 @@ ObjectClass *module_object_class_by_name(const 
> > > char *typename)
> > >  oc = object_class_by_name(typename);
> > >  #ifdef CONFIG_MODULES
> > >  if (!oc) {
> > > +char *module_name;
> > >  module_load_qom_one(typename);
> > >  oc = object_class_by_name(typename);
> > > +module_name = get_accel_module_name(typename);
> > > +if (module_name) {
> > > +if (!module_is_loaded(module_name)) {
> > > +fprintf(stderr, "%s module is missing, install the "
> > > +"package or config the library path "
> > > +"correctly.\n", module_name);
> > > +g_free(module_name);
> > > +exit(1);
> > > +}
> > > +g_free(module_name);
> > > +}
> > 
> > This error logging should IMHO be moved to util/module.c.  Either have a
> > helper function to print the error message, or have
> > module_load_qom_one() print it.
> > 
> > There is also no need to hard-code the module names.  We have the module
> > database and module_load_qom_one() uses it to figure which module must
> > be loaded for a specific qom object.  We can likewise use the database
> > for printing the error message.
> 
> IIUC, module loading can be triggered from hotplug of backends/devices,
> If so, we really shouldn't be printing to stderr, but using Error *errp
> throughout, so that QMP can give back useful error messages

Thank you Gerd and Daniel,

I'll improve it and send a v2.

Thank you very much,

Jose

> 
> 
> Regards,
> Daniel
> -- 
> |: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o-https://fstop138.berrange.com :|
> |: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|
> 


signature.asc
Description: Digital signature


Re: [PATCH 0/1]

2021-07-21 Thread Jose R. Ziviani
On Wed, Jul 21, 2021 at 07:24:02AM +0200, Thomas Huth wrote:
> On 21/07/2021 00.13, Jose R. Ziviani wrote:
> > Hello!
> > 
> > This patch gives the ability to build TCG builtin even if
> > --enable-modules is selected. This is useful to have a base
> > QEMU with TCG native product but still using the benefits of
> > modules.
> 
> Could you please elaborate why this is required? Did you see a performance
> improvement? Or is there another problem?

Hello Thomas,

Please, disconsider this patch. There's a more general discussion about
modules happening here:

https://lists.nongnu.org/archive/html/qemu-devel/2021-07/msg00632.html

A more general solution may be required to actually give us a
fine-grained control on modules.

The case is to allow us to generate customized QEMU packages attending
different user needs.

Thank you very much!!

Jose

> 
>  Thomas
> 


signature.asc
Description: Digital signature


[PATCH 1/2] modules: Implement new helper functions

2021-07-20 Thread Jose R. Ziviani
The function module_load_one() fills a hash table with modules that
were successfuly loaded. However, that table is a static variable of
module_load_one(). This patch changes it and creates a function that
informs whether a given module was loaded or not.

It also creates a function that returns the module name given the
typename.

Signed-off-by: Jose R. Ziviani 
---
 include/qemu/module.h |  4 +++
 util/module.c | 57 +--
 2 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/include/qemu/module.h b/include/qemu/module.h
index 3deac0078b..f45773718d 100644
--- a/include/qemu/module.h
+++ b/include/qemu/module.h
@@ -14,6 +14,7 @@
 #ifndef QEMU_MODULE_H
 #define QEMU_MODULE_H
 
+#include 
 
 #define DSO_STAMP_FUN glue(qemu_stamp, CONFIG_STAMP)
 #define DSO_STAMP_FUN_STR stringify(DSO_STAMP_FUN)
@@ -74,6 +75,9 @@ void module_load_qom_one(const char *type);
 void module_load_qom_all(void);
 void module_allow_arch(const char *arch);
 
+bool module_is_loaded(const char *name);
+const char *module_get_name_from_obj(const char *obj);
+
 /**
  * DOC: module info annotation macros
  *
diff --git a/util/module.c b/util/module.c
index 6bb4ad915a..f23adc65bf 100644
--- a/util/module.c
+++ b/util/module.c
@@ -119,6 +119,8 @@ static const QemuModinfo module_info_stub[] = { {
 static const QemuModinfo *module_info = module_info_stub;
 static const char *module_arch;
 
+static GHashTable *loaded_modules;
+
 void module_init_info(const QemuModinfo *info)
 {
 module_info = info;
@@ -206,13 +208,10 @@ static int module_load_file(const char *fname, bool 
mayfail, bool export_symbols
 out:
 return ret;
 }
-#endif
 
 bool module_load_one(const char *prefix, const char *lib_name, bool mayfail)
 {
 bool success = false;
-
-#ifdef CONFIG_MODULES
 char *fname = NULL;
 #ifdef CONFIG_MODULE_UPGRADES
 char *version_dir;
@@ -223,7 +222,6 @@ bool module_load_one(const char *prefix, const char 
*lib_name, bool mayfail)
 int i = 0, n_dirs = 0;
 int ret;
 bool export_symbols = false;
-static GHashTable *loaded_modules;
 const QemuModinfo *modinfo;
 const char **sl;
 
@@ -307,12 +305,9 @@ bool module_load_one(const char *prefix, const char 
*lib_name, bool mayfail)
 g_free(dirs[i]);
 }
 
-#endif
 return success;
 }
 
-#ifdef CONFIG_MODULES
-
 static bool module_loaded_qom_all;
 
 void module_load_qom_one(const char *type)
@@ -377,6 +372,39 @@ void qemu_load_module_for_opts(const char *group)
 }
 }
 
+bool module_is_loaded(const char *name)
+{
+if (!loaded_modules || !g_hash_table_contains(loaded_modules, name)) {
+return false;
+}
+
+return true;
+}
+
+const char *module_get_name_from_obj(const char *obj)
+{
+const QemuModinfo *modinfo;
+const char **sl;
+
+if (!obj) {
+return NULL;
+}
+
+for (modinfo = module_info; modinfo->name != NULL; modinfo++) {
+if (!modinfo->objs) {
+continue;
+}
+
+for (sl = modinfo->objs; *sl != NULL; sl++) {
+if (strcmp(obj, *sl) == 0) {
+return modinfo->name;
+}
+}
+}
+
+return NULL;
+}
+
 #else
 
 void module_allow_arch(const char *arch) {}
@@ -384,4 +412,19 @@ void qemu_load_module_for_opts(const char *group) {}
 void module_load_qom_one(const char *type) {}
 void module_load_qom_all(void) {}
 
+bool module_load_one(const char *prefix, const char *lib_name, bool mayfail)
+{
+return false;
+}
+
+bool module_is_loaded(const char *name)
+{
+return false;
+}
+
+char *module_get_name_from_obj(const char *obj)
+{
+return NULL;
+}
+
 #endif
-- 
2.32.0




[PATCH 2/2] qom: Improve error message in module_object_class_by_name()

2021-07-20 Thread Jose R. Ziviani
module_object_class_by_name() calls module_load_qom_one if the object
is provided by a dynamically linked library. Such library might not be
available at this moment - for instance, it can be a package not yet
installed. Thus, instead of assert error messages, this patch outputs
more friendly messages.

Current error messages:
$ ./qemu-system-x86_64 -machine q35 -accel tcg -kernel /boot/vmlinuz
...
ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion failed: 
(ops != NULL)
Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: 
assertion failed: (ops != NULL)
[1]31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...

New error message:
$ ./qemu-system-x86_64 -machine q35 -accel tcg -kernel /boot/vmlinuz
accel-tcg-x86_64 module is missing, install the package or config the library 
path correctly.

Or with other modules, when possible:
$ ./qemu-system-x86_64 -machine q35 -accel kvm -kernel /boot/vmlinuz -vga qxl   
  ✹
hw-display-qxl module is missing, install the package or config the library 
path correctly.
qemu-system-x86_64: QXL VGA not available

$ make check
...
Running test qtest-x86_64/test-filter-mirror
Running test qtest-x86_64/endianness-test
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-tcg-x86_64 module is missing, install the package or config the library 
path correctly.
...

Signed-off-by: Jose R. Ziviani 
---
 accel/accel-softmmu.c | 5 -
 qom/object.c  | 9 +
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/accel/accel-softmmu.c b/accel/accel-softmmu.c
index 67276e4f52..52449ac2d0 100644
--- a/accel/accel-softmmu.c
+++ b/accel/accel-softmmu.c
@@ -79,7 +79,10 @@ void accel_init_ops_interfaces(AccelClass *ac)
  * all accelerators need to define ops, providing at least a mandatory
  * non-NULL create_vcpu_thread operation.
  */
-g_assert(ops != NULL);
+if (ops == NULL) {
+exit(1);
+}
+
 if (ops->ops_init) {
 ops->ops_init(ops);
 }
diff --git a/qom/object.c b/qom/object.c
index 6a01d56546..3a170ea9df 100644
--- a/qom/object.c
+++ b/qom/object.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/module.h"
 #include "qemu/osdep.h"
 #include "hw/qdev-core.h"
 #include "qapi/error.h"
@@ -1031,8 +1032,16 @@ ObjectClass *module_object_class_by_name(const char 
*typename)
 oc = object_class_by_name(typename);
 #ifdef CONFIG_MODULES
 if (!oc) {
+const char *module_name = module_get_name_from_obj(typename);
 module_load_qom_one(typename);
 oc = object_class_by_name(typename);
+if (!oc && module_name) {
+if (!module_is_loaded(module_name)) {
+fprintf(stderr, "%s module is missing, install the "
+"package or config the library path "
+"correctly.\n", module_name);
+}
+}
 }
 #endif
 return oc;
-- 
2.32.0




[PATCH 0/2] Improve module accelerator error message

2021-07-20 Thread Jose R. Ziviani
The main objective here is to fix an user issue when trying to load TCG
that was built as module, but it's not installed or found in the library
path.

For example:

$ ./qemu-system-x86_64 -machine q35 -accel tcg -kernel /boot/vmlinuz
...
ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion failed: 
(ops != NULL)
Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: 
assertion failed: (ops != NULL)
[1]31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...

The new error message becomes:

$ ./qemu-system-x86_64 -machine q35 -accel tcg -kernel /boot/vmlinuz
accel-tcg-x86_64 module is missing, install the package or config the library 
path correctly.

Jose R. Ziviani (2):
  modules: Implement new helper functions
  qom: Improve error message in module_object_class_by_name()

 accel/accel-softmmu.c |  5 +++-
 include/qemu/module.h |  4 +++
 qom/object.c  |  9 +++
 util/module.c | 57 +--
 4 files changed, 67 insertions(+), 8 deletions(-)

-- 
2.32.0




[PATCH 1/1] modules: Option to build native TCG with --enable-modules

2021-07-20 Thread Jose R. Ziviani
Adds an option (--enable-tcg-builtin) to build TCG natively when
--enable-modules argument is passed to the build system. It gives
the opportunity to have this important accelerator built-in and
still take advantage of the new modular system.

Signed-off-by: Jose R. Ziviani 
---
 configure | 12 ++--
 meson.build   | 11 ++-
 meson_options.txt |  2 ++
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 232c54dcc1..64d7a909ce 100755
--- a/configure
+++ b/configure
@@ -345,6 +345,7 @@ tsan="no"
 fortify_source="$default_feature"
 strip_opt="yes"
 tcg_interpreter="false"
+tcg_builtin="false"
 bigendian="no"
 mingw32="no"
 gcov="no"
@@ -1120,6 +1121,8 @@ for opt do
   ;;
   --enable-tcg) tcg="enabled"
   ;;
+  --enable-tcg-builtin) tcg_builtin="true"
+  ;;
   --disable-malloc-trim) malloc_trim="disabled"
   ;;
   --enable-malloc-trim) malloc_trim="enabled"
@@ -1817,6 +1820,7 @@ Advanced options (experts only):
Default:trace-
   --disable-slirp  disable SLIRP userspace network connectivity
   --enable-tcg-interpreter enable TCI (TCG with bytecode interpreter, 
experimental and slow)
+  --enable-tcg-builtin force TCG builtin even with --enable-modules
   --enable-malloc-trim enable libc malloc_trim() for memory optimization
   --oss-libpath to OSS library
   --cpu=CPUBuild for host CPU [$cpu]
@@ -2318,7 +2322,11 @@ if test "$solaris" = "yes" ; then
   fi
 fi
 
-if test "$tcg" = "enabled"; then
+if test "$tcg" = "disabled"; then
+debug_tcg="no"
+tcg_interpreter="false"
+tcg_builtin="false"
+else
 git_submodules="$git_submodules tests/fp/berkeley-testfloat-3"
 git_submodules="$git_submodules tests/fp/berkeley-softfloat-3"
 fi
@@ -5229,7 +5237,7 @@ if test "$skip_meson" = no; then
 -Dvhost_user_blk_server=$vhost_user_blk_server 
-Dmultiprocess=$multiprocess \
 -Dfuse=$fuse -Dfuse_lseek=$fuse_lseek 
-Dguest_agent_msi=$guest_agent_msi -Dbpf=$bpf\
 $(if test "$default_features" = no; then echo 
"-Dauto_features=disabled"; fi) \
-   -Dtcg_interpreter=$tcg_interpreter \
+-Dtcg_interpreter=$tcg_interpreter -Dtcg_builtin=$tcg_builtin \
 $cross_arg \
 "$PWD" "$source_path"
 
diff --git a/meson.build b/meson.build
index 2f377098d7..2909043aab 100644
--- a/meson.build
+++ b/meson.build
@@ -93,9 +93,13 @@ if cpu in ['x86', 'x86_64']
 endif
 
 modular_tcg = []
+is_tcg_modular = false
 # Darwin does not support references to thread-local variables in modules
 if targetos != 'darwin'
   modular_tcg = ['i386-softmmu', 'x86_64-softmmu']
+  is_tcg_modular = config_host.has_key('CONFIG_MODULES') \
+   and get_option('tcg').enabled() \
+   and not get_option('tcg_builtin')
 endif
 
 edk2_targets = [ 'arm-softmmu', 'aarch64-softmmu', 'i386-softmmu', 
'x86_64-softmmu' ]
@@ -279,6 +283,9 @@ if not get_option('tcg').disabled()
 
   accelerators += 'CONFIG_TCG'
   config_host += { 'CONFIG_TCG': 'y' }
+  if is_tcg_modular
+config_host += { 'CONFIG_TCG_MODULAR': 'y' }
+  endif
 endif
 
 if 'CONFIG_KVM' not in accelerators and get_option('kvm').enabled()
@@ -1567,7 +1574,7 @@ foreach target : target_dirs
   elif sym == 'CONFIG_XEN' and have_xen_pci_passthrough
 config_target += { 'CONFIG_XEN_PCI_PASSTHROUGH': 'y' }
   endif
-  if target in modular_tcg
+  if target in modular_tcg and is_tcg_modular
 config_target += { 'CONFIG_TCG_MODULAR': 'y' }
   else
 config_target += { 'CONFIG_TCG_BUILTIN': 'y' }
@@ -2976,6 +2983,8 @@ summary_info += {'TCG support':   
config_all.has_key('CONFIG_TCG')}
 if config_all.has_key('CONFIG_TCG')
   if get_option('tcg_interpreter')
 summary_info += {'TCG backend':   'TCI (TCG with bytecode interpreter, 
experimental and slow)'}
+  elif is_tcg_modular
+summary_info += {'TCG backend':   'module (@0@)'.format(cpu)}
   else
 summary_info += {'TCG backend':   'native (@0@)'.format(cpu)}
   endif
diff --git a/meson_options.txt b/meson_options.txt
index a9a9b8f4c6..c27749b864 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -43,6 +43,8 @@ option('tcg', type: 'feature', value: 'auto',
description: 'TCG support')
 option('tcg_interpreter', type: 'boolean', value: false,
description: 'TCG with bytecode interpreter (experimental and slow)')
+option('tcg_builtin', type: 'boolean', value: 'false',
+   description: 'Force TCG builtin')
 option('cfi', type: 'boolean', value: 'false',
description: 'Control-Flow Integrity (CFI)')
 option('cfi_debug', type: 'boolean', value: 'false',
-- 
2.32.0




[PATCH 0/1]

2021-07-20 Thread Jose R. Ziviani
Hello!

This patch gives the ability to build TCG builtin even if
--enable-modules is selected. This is useful to have a base
QEMU with TCG native product but still using the benefits of
modules.

Thank you!

Jose R. Ziviani (1):
  modules: Option to build native TCG with --enable-modules

 configure | 12 ++--
 meson.build   | 11 ++-
 meson_options.txt |  2 ++
 3 files changed, 22 insertions(+), 3 deletions(-)

-- 
2.32.0




Re: [RFC 3/3] qom: Improve error message in module_object_class_by_name()

2021-07-19 Thread Jose R. Ziviani
On Mon, Jul 19, 2021 at 05:29:49PM +0200, Claudio Fontana wrote:
> On 7/1/21 1:27 AM, Jose R. Ziviani wrote:
> > module_object_class_by_name() calls module_load_qom_one if the object
> > is provided by a dynamically linked library. Such library might not be
> > available at this moment - for instance, it can be a package not yet
> > installed. Thus, instead of assert error messages, this patch outputs
> > more friendly messages.
> > 
> > Current error messages:
> > $ ./qemu-system-x86_64 -machine q35 -accel tcg -kernel /boot/vmlinuz
> > ...
> > ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion 
> > failed: (ops != NULL)
> > Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: 
> > assertion failed: (ops != NULL)
> > [1]31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...
> > 
> > New error message:
> > $ ./qemu-system-x86_64 -machine q35 -accel tcg -kernel /boot/vmlinuz
> > accel-tcg-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > 
> > $ make check
> > ...
> > Running test qtest-x86_64/test-filter-mirror
> > Running test qtest-x86_64/endianness-test
> > accel-qtest-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > accel-qtest-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > accel-qtest-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > accel-qtest-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > accel-qtest-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > accel-tcg-x86_64 module is missing, install the package or config the 
> > library path correctly.
> > ...
> > 
> > Signed-off-by: Jose R. Ziviani 
> > ---
> >  qom/object.c | 30 ++
> >  1 file changed, 30 insertions(+)
> > 
> > diff --git a/qom/object.c b/qom/object.c
> > index 6a01d56546..2d40245af9 100644
> > --- a/qom/object.c
> > +++ b/qom/object.c
> > @@ -1024,6 +1024,24 @@ ObjectClass *object_class_by_name(const char 
> > *typename)
> >  return type->class;
> >  }
> >  
> > +char *get_accel_module_name(const char *ac_name);
> > +
> > +char *get_accel_module_name(const char *ac_name)
> > +{
> > +size_t len = strlen(ac_name);
> > +char *module_name = NULL;
> > +
> > +if (strncmp(ac_name, "tcg-accel-ops", len) == 0) {
> > +#ifdef CONFIG_TCG_MODULAR
> > +module_name = g_strdup_printf("%s%s", "accel-tcg-", "x86_64");
> > +#endif
> > +} else if (strncmp(ac_name, "qtest-accel-ops", len) == 0) {
> > +module_name = g_strdup_printf("%s%s", "accel-qtest-", "x86_64");
> > +}
> > +
> > +return module_name;
> > +}
> > +
> >  ObjectClass *module_object_class_by_name(const char *typename)
> >  {
> >  ObjectClass *oc;
> > @@ -1031,8 +1049,20 @@ ObjectClass *module_object_class_by_name(const char 
> > *typename)
> >  oc = object_class_by_name(typename);
> >  #ifdef CONFIG_MODULES
> >  if (!oc) {
> > +char *module_name;
> >  module_load_qom_one(typename);
> >  oc = object_class_by_name(typename);
> > +module_name = get_accel_module_name(typename);
> > +if (module_name) {
> > +if (!module_is_loaded(module_name)) {
> > +fprintf(stderr, "%s module is missing, install the "
> > +"package or config the library path "
> > +"correctly.\n", module_name);
> > +g_free(module_name);
> > +exit(1);
> > +}
> > +g_free(module_name);
> > +}
> >  }
> >  #endif
> >  return oc;
> > 
> 
> Hi Jose,
> 
> this inserts accel logic into module_object_class_by_name,
> maybe some other place would be better to insert this check,
> like when trying to select an accelerator?


Hello Claudio,

Yes, I think that 'get_accel_module_name()' may be a more generic
'get_module_name()' to handle any module failure (not only
accelerators).

I'll improve it and send a v2. Thank you for reviewing it.

> 
> Thanks,
> 
> CLaudio


signature.asc
Description: Digital signature


[RFC 1/3] modules: Add CONFIG_TCG_MODULAR in config_host

2021-06-30 Thread Jose R. Ziviani
CONFIG_TCG_MODULAR is a complement to CONFIG_MODULES, in order to
know if TCG will be a module, even if --enable-modules option was
set.

Signed-off-by: Jose R. Ziviani 
---
 meson.build | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/meson.build b/meson.build
index 2d72b8cc06..c37a2358d4 100644
--- a/meson.build
+++ b/meson.build
@@ -277,6 +277,9 @@ if not get_option('tcg').disabled()
 
   accelerators += 'CONFIG_TCG'
   config_host += { 'CONFIG_TCG': 'y' }
+  if is_tcg_modular
+config_host += { 'CONFIG_TCG_MODULAR': 'y' }
+  endif
 endif
 
 if 'CONFIG_KVM' not in accelerators and get_option('kvm').enabled()
-- 
2.32.0




[RFC 0/3] Improve module accelerator error message

2021-06-30 Thread Jose R. Ziviani
Hello!

I'm sending this as RFC because it's based on a patch still on
review[1], so I'd like to see if it makes sense.

Tt will improve the error message when an accelerator module could
not be loaded. Instead of the current assert error, a formated
message will be displayed.

[1] https://patchwork.kernel.org/project/qemu-devel/list/?series=506379

Jose R. Ziviani (3):
  modules: Add CONFIG_TCG_MODULAR in config_host
  modules: Implement module_is_loaded function
  qom: Improve error message in module_object_class_by_name()

 include/qemu/module.h |  3 +++
 meson.build   |  3 +++
 qom/object.c  | 30 ++
 util/module.c | 28 +---
 4 files changed, 57 insertions(+), 7 deletions(-)

-- 
2.32.0




[RFC 3/3] qom: Improve error message in module_object_class_by_name()

2021-06-30 Thread Jose R. Ziviani
module_object_class_by_name() calls module_load_qom_one if the object
is provided by a dynamically linked library. Such library might not be
available at this moment - for instance, it can be a package not yet
installed. Thus, instead of assert error messages, this patch outputs
more friendly messages.

Current error messages:
$ ./qemu-system-x86_64 -machine q35 -accel tcg -kernel /boot/vmlinuz
...
ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: assertion failed: 
(ops != NULL)
Bail out! ERROR:../accel/accel-softmmu.c:82:accel_init_ops_interfaces: 
assertion failed: (ops != NULL)
[1]31964 IOT instruction (core dumped)  ./qemu-system-x86_64 ...

New error message:
$ ./qemu-system-x86_64 -machine q35 -accel tcg -kernel /boot/vmlinuz
accel-tcg-x86_64 module is missing, install the package or config the library 
path correctly.

$ make check
...
Running test qtest-x86_64/test-filter-mirror
Running test qtest-x86_64/endianness-test
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-qtest-x86_64 module is missing, install the package or config the library 
path correctly.
accel-tcg-x86_64 module is missing, install the package or config the library 
path correctly.
...

Signed-off-by: Jose R. Ziviani 
---
 qom/object.c | 30 ++
 1 file changed, 30 insertions(+)

diff --git a/qom/object.c b/qom/object.c
index 6a01d56546..2d40245af9 100644
--- a/qom/object.c
+++ b/qom/object.c
@@ -1024,6 +1024,24 @@ ObjectClass *object_class_by_name(const char *typename)
 return type->class;
 }
 
+char *get_accel_module_name(const char *ac_name);
+
+char *get_accel_module_name(const char *ac_name)
+{
+size_t len = strlen(ac_name);
+char *module_name = NULL;
+
+if (strncmp(ac_name, "tcg-accel-ops", len) == 0) {
+#ifdef CONFIG_TCG_MODULAR
+module_name = g_strdup_printf("%s%s", "accel-tcg-", "x86_64");
+#endif
+} else if (strncmp(ac_name, "qtest-accel-ops", len) == 0) {
+module_name = g_strdup_printf("%s%s", "accel-qtest-", "x86_64");
+}
+
+return module_name;
+}
+
 ObjectClass *module_object_class_by_name(const char *typename)
 {
 ObjectClass *oc;
@@ -1031,8 +1049,20 @@ ObjectClass *module_object_class_by_name(const char 
*typename)
 oc = object_class_by_name(typename);
 #ifdef CONFIG_MODULES
 if (!oc) {
+char *module_name;
 module_load_qom_one(typename);
 oc = object_class_by_name(typename);
+module_name = get_accel_module_name(typename);
+if (module_name) {
+if (!module_is_loaded(module_name)) {
+fprintf(stderr, "%s module is missing, install the "
+"package or config the library path "
+"correctly.\n", module_name);
+g_free(module_name);
+exit(1);
+}
+g_free(module_name);
+}
 }
 #endif
 return oc;
-- 
2.32.0




[RFC 2/3] modules: Implement module_is_loaded function

2021-06-30 Thread Jose R. Ziviani
The function module_load_one() fills a hash table will all modules that
were successfuly loaded. However, that table is a static variable of
module_load_one(). This patch changes it and creates a function that
informs whether a given module was loaded or not.

Signed-off-by: Jose R. Ziviani 
---
 include/qemu/module.h |  3 +++
 util/module.c | 28 +---
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/include/qemu/module.h b/include/qemu/module.h
index 456e190a55..01779cc7fb 100644
--- a/include/qemu/module.h
+++ b/include/qemu/module.h
@@ -14,6 +14,7 @@
 #ifndef QEMU_MODULE_H
 #define QEMU_MODULE_H
 
+#include 
 
 #define DSO_STAMP_FUN glue(qemu_stamp, CONFIG_STAMP)
 #define DSO_STAMP_FUN_STR stringify(DSO_STAMP_FUN)
@@ -74,6 +75,8 @@ void module_load_qom_one(const char *type);
 void module_load_qom_all(void);
 void module_allow_arch(const char *arch);
 
+bool module_is_loaded(const char *name);
+
 /**
  * DOC: module info annotation macros
  *
diff --git a/util/module.c b/util/module.c
index 6bb4ad915a..64307b7a25 100644
--- a/util/module.c
+++ b/util/module.c
@@ -119,6 +119,8 @@ static const QemuModinfo module_info_stub[] = { {
 static const QemuModinfo *module_info = module_info_stub;
 static const char *module_arch;
 
+static GHashTable *loaded_modules;
+
 void module_init_info(const QemuModinfo *info)
 {
 module_info = info;
@@ -206,13 +208,10 @@ static int module_load_file(const char *fname, bool 
mayfail, bool export_symbols
 out:
 return ret;
 }
-#endif
 
 bool module_load_one(const char *prefix, const char *lib_name, bool mayfail)
 {
 bool success = false;
-
-#ifdef CONFIG_MODULES
 char *fname = NULL;
 #ifdef CONFIG_MODULE_UPGRADES
 char *version_dir;
@@ -223,7 +222,6 @@ bool module_load_one(const char *prefix, const char 
*lib_name, bool mayfail)
 int i = 0, n_dirs = 0;
 int ret;
 bool export_symbols = false;
-static GHashTable *loaded_modules;
 const QemuModinfo *modinfo;
 const char **sl;
 
@@ -307,12 +305,9 @@ bool module_load_one(const char *prefix, const char 
*lib_name, bool mayfail)
 g_free(dirs[i]);
 }
 
-#endif
 return success;
 }
 
-#ifdef CONFIG_MODULES
-
 static bool module_loaded_qom_all;
 
 void module_load_qom_one(const char *type)
@@ -377,6 +372,15 @@ void qemu_load_module_for_opts(const char *group)
 }
 }
 
+bool module_is_loaded(const char *name)
+{
+if (!loaded_modules || !g_hash_table_contains(loaded_modules, name)) {
+return false;
+}
+
+return true;
+}
+
 #else
 
 void module_allow_arch(const char *arch) {}
@@ -384,4 +388,14 @@ void qemu_load_module_for_opts(const char *group) {}
 void module_load_qom_one(const char *type) {}
 void module_load_qom_all(void) {}
 
+bool module_load_one(const char *prefix, const char *lib_name, bool mayfail)
+{
+return false;
+}
+
+bool module_is_loaded(const char *name)
+{
+return false;
+}
+
 #endif
-- 
2.32.0




[RFC 2/2] modules: Fix warning in module_arch documentation

2021-06-29 Thread Jose R. Ziviani
Fixes a small issue in the module_arch documentation that
caused the build system to complain:
module.h:127: warning: Function parameter or member
  'name' not described in 'module_arch'
module.h:127: warning: Excess function parameter
  'arch' description in 'module_arch'

Signed-off-by: Jose R. Ziviani 
---
 include/qemu/module.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/qemu/module.h b/include/qemu/module.h
index 8bc80535a4..456e190a55 100644
--- a/include/qemu/module.h
+++ b/include/qemu/module.h
@@ -115,7 +115,7 @@ void module_allow_arch(const char *arch);
 /**
  * module_arch
  *
- * @arch: target architecture
+ * @name: module name
  *
  * This module is for target architecture @arch.
  *
-- 
2.32.0




[RFC 1/2] modules: Option to build native TCG with --enable-modules

2021-06-29 Thread Jose R. Ziviani
Adds an option (--enable-tcg-builtin) to build TCG natively when
--enable-modules argument is passed to the build system. It gives
the opportunity to have the accelerator built-in and still take
advantage of the new modular system.

Signed-off-by: Jose R. Ziviani 
---
 configure | 12 ++--
 meson.build   |  7 ++-
 meson_options.txt |  2 ++
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 38704b4e11..add8cf4970 100755
--- a/configure
+++ b/configure
@@ -344,6 +344,7 @@ tsan="no"
 fortify_source="$default_feature"
 strip_opt="yes"
 tcg_interpreter="false"
+tcg_builtin="false"
 bigendian="no"
 mingw32="no"
 gcov="no"
@@ -1107,6 +1108,8 @@ for opt do
   ;;
   --enable-tcg) tcg="enabled"
   ;;
+  --enable-tcg-builtin) tcg_builtin="true"
+  ;;
   --disable-malloc-trim) malloc_trim="disabled"
   ;;
   --enable-malloc-trim) malloc_trim="enabled"
@@ -1792,6 +1795,7 @@ Advanced options (experts only):
Default:trace-
   --disable-slirp  disable SLIRP userspace network connectivity
   --enable-tcg-interpreter enable TCI (TCG with bytecode interpreter, 
experimental and slow)
+  --enable-tcg-builtin force TCG builtin even with --enable-modules
   --enable-malloc-trim enable libc malloc_trim() for memory optimization
   --oss-libpath to OSS library
   --cpu=CPUBuild for host CPU [$cpu]
@@ -2288,7 +2292,11 @@ if test "$solaris" = "yes" ; then
   fi
 fi
 
-if test "$tcg" = "enabled"; then
+if test "$tcg" = "disabled"; then
+debug_tcg="no"
+tcg_interpreter="false"
+tcg_builtin="false"
+else
 git_submodules="$git_submodules tests/fp/berkeley-testfloat-3"
 git_submodules="$git_submodules tests/fp/berkeley-softfloat-3"
 fi
@@ -6449,7 +6457,7 @@ if test "$skip_meson" = no; then
 -Dvhost_user_blk_server=$vhost_user_blk_server 
-Dmultiprocess=$multiprocess \
 -Dfuse=$fuse -Dfuse_lseek=$fuse_lseek 
-Dguest_agent_msi=$guest_agent_msi -Dbpf=$bpf\
 $(if test "$default_features" = no; then echo 
"-Dauto_features=disabled"; fi) \
-   -Dtcg_interpreter=$tcg_interpreter \
+-Dtcg_interpreter=$tcg_interpreter -Dtcg_builtin=$tcg_builtin \
 $cross_arg \
 "$PWD" "$source_path"
 
diff --git a/meson.build b/meson.build
index fe58793549..2d72b8cc06 100644
--- a/meson.build
+++ b/meson.build
@@ -93,6 +93,9 @@ if cpu in ['x86', 'x86_64']
 endif
 
 modular_tcg = ['i386-softmmu', 'x86_64-softmmu']
+is_tcg_modular = config_host.has_key('CONFIG_MODULES') \
+ and get_option('tcg').enabled() \
+ and not get_option('tcg_builtin')
 
 edk2_targets = [ 'arm-softmmu', 'aarch64-softmmu', 'i386-softmmu', 
'x86_64-softmmu' ]
 install_edk2_blobs = false
@@ -1313,7 +1316,7 @@ foreach target : target_dirs
   elif sym == 'CONFIG_XEN' and have_xen_pci_passthrough
 config_target += { 'CONFIG_XEN_PCI_PASSTHROUGH': 'y' }
   endif
-  if target in modular_tcg
+  if target in modular_tcg and is_tcg_modular
 config_target += { 'CONFIG_TCG_MODULAR': 'y' }
   else
 config_target += { 'CONFIG_TCG_BUILTIN': 'y' }
@@ -2698,6 +2701,8 @@ summary_info += {'TCG support':   
config_all.has_key('CONFIG_TCG')}
 if config_all.has_key('CONFIG_TCG')
   if get_option('tcg_interpreter')
 summary_info += {'TCG backend':   'TCI (TCG with bytecode interpreter, 
experimental and slow)'}
+  elif is_tcg_modular
+summary_info += {'TCG backend':   'module (@0@)'.format(cpu)}
   else
 summary_info += {'TCG backend':   'native (@0@)'.format(cpu)}
   endif
diff --git a/meson_options.txt b/meson_options.txt
index 3d304cac96..fd9f92b333 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -43,6 +43,8 @@ option('tcg', type: 'feature', value: 'auto',
description: 'TCG support')
 option('tcg_interpreter', type: 'boolean', value: false,
description: 'TCG with bytecode interpreter (experimental and slow)')
+option('tcg_builtin', type: 'boolean', value: 'false',
+   description: 'Force TCG builtin')
 option('cfi', type: 'boolean', value: 'false',
description: 'Control-Flow Integrity (CFI)')
 option('cfi_debug', type: 'boolean', value: 'false',
-- 
2.32.0




[RFC 0/2] Option to build native TCG with --enable-modules

2021-06-29 Thread Jose R. Ziviani
Hello!

I'm sending this simple patchset based on a patch still on review[1]
just to understand if it's something that makes sense to the community.

If so, I think it could be included in Gerd's patchset.

Thank you!

[1] https://patchwork.kernel.org/project/qemu-devel/list/?series=506379

Jose R. Ziviani (2):
  modules: Option to build native TCG with --enable-modules
  modules: Fix warning in module_arch documentation

 configure | 12 ++--
 include/qemu/module.h |  2 +-
 meson.build   |  7 ++-
 meson_options.txt |  2 ++
 4 files changed, 19 insertions(+), 4 deletions(-)

-- 
2.32.0




Re: [PATCH v4 00/34] modules: add meta-data database

2021-06-24 Thread Jose R. Ziviani
Hello Gerd,

Reviewed and tested successfully here. Thank you!

Reviewed-by: Jose R. Ziviani 


On Thu, Jun 24, 2021 at 12:38:02PM +0200, Gerd Hoffmann wrote:
> This patch series adds support for module meta-data.  Today this is
> either hard-coded in qemu (see qemu_load_module_for_opts) or handled
> with manually maintained lists in util/module (see module_deps[] and
> qom_modules[]).  This series replaced that scheme with annotation
> macros, so the meta-data can go into the module source code and -- for
> example -- the module_obj() annotations can go next to the TypeInfo
> struct for the object class.
> 
> Patches 1-3 put the infrastructure in place:  Add the annotation macros,
> add a script to collect the meta-data, add a script to compile the
> meta-data into C source code which we can then add to qemu.
> 
> Patch 4 - check module dependencies (Jose, new in v4).
> 
> Patches 5-13 add annotations macros to the modules we have.
> 
> Patches 14-16 put the modinfo database into use and remove the
> module_deps[] and qom_modules[] lists.
> 
> Patch 16 adds two tracepoints for easier trouble-shooting.
> 
> Patches 18-20 add support for target-specific modules.
> 
> Patches 21-24 add documentation for all of the above (new in v4, was
> separate series).
> 
> Patches 25-29 start building accelerators modular.  So far it is
> only qtest (all archs) and a small fraction of tcg (x86 only).
> 
> Patches 30-34 add support for registering hmp commands so they can
> be implemented as module (new in v4, was separate series).
> 
> take care,
>   Gerd
> 
> Gerd Hoffmann (33):
>   modules: add modinfo macros
>   modules: collect module meta-data
>   modules: generate modinfo.c
>   modules: add qxl module annotations
>   modules: add virtio-gpu module annotations
>   modules: add chardev module annotations
>   modules: add audio module annotations
>   modules: add usb-redir module annotations
>   modules: add ccid module annotations
>   modules: add ui module annotations
>   modules: add s390x module annotations
>   modules: add block module annotations
>   modules: use modinfo for dependencies
>   modules: use modinfo for qom load
>   modules: use modinfo for qemu opts load
>   modules: add tracepoints
>   modules: check arch and block load on mismatch
>   modules: check arch on qom lookup
>   modules: target-specific module build infrastructure
>   modules: add documentation for module sourcesets
>   modules: add module_obj() note to QOM docs
>   modules: module.h kerneldoc annotations
>   modules: hook up modules.h to docs build
>   accel: autoload modules
>   accel: add qtest module annotations
>   accel: build qtest modular
>   accel: add tcg module annotations
>   accel: build tcg modular
>   monitor: allow register hmp commands
>   usb: drop usb_host_dev_is_scsi_storage hook
>   monitor/usb: register 'info usbhost' dynamically
>   usb: build usb-host as module
>   monitor/tcg: move tcg hmp commands to accel/tcg, register them
> dynamically
> 
> Jose R. Ziviani (1):
>   modules: check if all dependencies can be satisfied
> 
>  scripts/modinfo-collect.py  |  67 +++
>  scripts/modinfo-generate.py |  97 
>  include/hw/usb.h|   7 +-
>  include/monitor/monitor.h   |   3 +
>  include/qemu/module.h   |  74 
>  accel/accel-common.c|   2 +-
>  accel/accel-softmmu.c   |   2 +-
>  accel/qtest/qtest.c |   2 +
>  accel/tcg/hmp.c |  29 +
>  accel/tcg/tcg-accel-ops.c   |   1 +
>  accel/tcg/tcg-all.c |   1 +
>  audio/spiceaudio.c  |   2 +
>  block/iscsi-opts.c  |   1 +
>  chardev/baum.c  |   1 +
>  chardev/spice.c |   4 +
>  hw/display/qxl.c|   4 +
>  hw/display/vhost-user-gpu-pci.c |   1 +
>  hw/display/vhost-user-gpu.c |   1 +
>  hw/display/vhost-user-vga.c |   1 +
>  hw/display/virtio-gpu-base.c|   1 +
>  hw/display/virtio-gpu-gl.c  |   3 +
>  hw/display/virtio-gpu-pci-gl.c  |   3 +
>  hw/display/virtio-gpu-pci.c |   2 +
>  hw/display/virtio-gpu.c |   1 +
>  hw/display/virtio-vga-gl.c  |   3 +
>  hw/display/virtio-vga.c |   2 +
>  hw/ppc/spapr.c  |   2 +-
>  hw/s390x/virtio-ccw-gpu.c   |   3 +
>  hw/usb/ccid-card-emulated.c |   1 +
>  hw/usb/ccid-card-passthru.c |   1 +
>  hw/usb/dev-storage-bot.c|   1 +
>  hw/usb/dev-storage-classic.c|   1 +
>  hw/usb/dev-uas.c|   1 +
>  hw/usb/host-libusb.c|  38 ++
>  hw/usb/host-stub.c  |  45 ---
>  hw/usb/redirect.c

Re: [PATCH v3 03/24] modules: generate modinfo.c

2021-06-22 Thread Jose R. Ziviani
);
>  /* module registers QemuOpts  */
>  #define module_opts(name) modinfo(opts, name)
>  
> +/*
> + * module info database
> + *
> + * scripts/modinfo-generate.c will build this using the data collected
> + * by scripts/modinfo-collect.py
> + */
> +typedef struct QemuModinfo QemuModinfo;
> +struct QemuModinfo {
> +const char *name;
> +const char *arch;
> +const char **objs;
> +const char **deps;
> +const char **opts;
> +};
> +extern const QemuModinfo qemu_modinfo[];
> +void module_init_info(const QemuModinfo *info);
> +
>  #endif
> diff --git a/softmmu/vl.c b/softmmu/vl.c
> index 326c1e908008..a4857ec43ff3 100644
> --- a/softmmu/vl.c
> +++ b/softmmu/vl.c
> @@ -2740,6 +2740,10 @@ void qemu_init(int argc, char **argv, char **envp)
>  error_init(argv[0]);
>  qemu_init_exec_dir(argv[0]);
>  
> +#ifdef CONFIG_MODULES
> +module_init_info(qemu_modinfo);
> +#endif
> +
>  qemu_init_subsystems();
>  
>  /* first pass of option parsing */
> diff --git a/util/module.c b/util/module.c
> index eee8ff2de136..8d3e8275b9f7 100644
> --- a/util/module.c
> +++ b/util/module.c
> @@ -110,6 +110,17 @@ void module_call_init(module_init_type type)
>  }
>  
>  #ifdef CONFIG_MODULES
> +
> +static const QemuModinfo module_info_stub[] = { {
> +/* end of list */
> +} };
> +static const QemuModinfo *module_info = module_info_stub;
> +
> +void module_init_info(const QemuModinfo *info)
> +{
> +module_info = info;
> +}
> +
>  static int module_load_file(const char *fname, bool mayfail, bool 
> export_symbols)
>  {
>  GModule *g_module;
> diff --git a/meson.build b/meson.build
> index bb99619257d5..9cf50a50d39a 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -2022,6 +2022,7 @@ subdir('tests/qtest/fuzz')
>  
>  
>  modinfo_collect = find_program('scripts/modinfo-collect.py')
> +modinfo_generate = find_program('scripts/modinfo-generate.py')
>  modinfo_files = []
>  
>  block_mods = []
> @@ -2042,7 +2043,6 @@ foreach d, list : modules
> output: d + '-' + m + '.modinfo',
> input: module_ss.sources(),
> capture: true,
> -   build_by_default: true, # to be 
> removed when added to a target
> command: [modinfo_collect, '@INPUT@'])
>endif
>  else
> @@ -2055,6 +2055,17 @@ foreach d, list : modules
>endforeach
>  endforeach
>  
> +if enable_modules
> +  modinfo_src = custom_target('modinfo.c',
> +  output: 'modinfo.c',
> +  input: modinfo_files,
> +  command: [modinfo_generate, '@INPUT@'],
> +  capture: true)
> +  modinfo_lib = static_library('modinfo', modinfo_src)
> +  modinfo_dep = declare_dependency(link_whole: modinfo_lib)
> +  softmmu_ss.add(modinfo_dep)
> +endif
> +
>  nm = find_program('nm')
>  undefsym = find_program('scripts/undefsym.py')
>  block_syms = custom_target('block.syms', output: 'block.syms',
> -- 
> 2.31.1
> 
> 
From 0e6866bcae2d6576e7dbc08d8bbd2837f655d5a3 Mon Sep 17 00:00:00 2001
From: "Jose R. Ziviani" 
Date: Tue, 22 Jun 2021 17:37:42 -0300
Subject: [PATCH] modules: check if all dependencies can be satisfied

Verifies if all dependencies are correctly listed in the modinfo.c too
and stop the builds if they're not.

Signed-off-by: Jose R. Ziviani 
---
 scripts/modinfo-generate.py | 15 ++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py
index 2b92543265..a36ddb77dd 100755
--- a/scripts/modinfo-generate.py
+++ b/scripts/modinfo-generate.py
@@ -59,6 +59,7 @@ def generate(name, lines):
 print_array("deps", deps)
 print_array("opts", opts)
 print("},{");
+return deps
 
 def print_pre():
 print("/* generated by scripts/modinfo.py */")
@@ -71,14 +72,26 @@ def print_post():
 print("}};")
 
 def main(args):
+deps = {}
 print_pre()
 for modinfo in args:
 with open(modinfo) as f:
 lines = f.readlines()
 print("/* %s */" % modinfo)
 (basename, ext) = os.path.splitext(modinfo)
-generate(basename, lines)
+deps[basename] = generate(basename, lines)
 print_post()
 
+flattened_deps = {flat.strip('" ') for dep in deps.values() for flat in dep}
+error = False
+for dep in flattened_deps:
+if dep not in deps.keys():
+print("Dependency {} cannot be satisfied".format(dep),
+  file=sys.stderr)
+error = True
+
+if error:
+exit(1)
+
 if __name__ == "__main__":
 main(sys.argv[1:])
-- 
2.32.0



signature.asc
Description: Digital signature


Re: [PATCH v3 03/24] modules: generate modinfo.c

2021-06-22 Thread Jose R. Ziviani
Hello,

Just a small change.

On Fri, Jun 18, 2021 at 06:53:32AM +0200, Gerd Hoffmann wrote:
> Add script to generate C source with a small
> database containing the module meta-data.
> 
> Signed-off-by: Gerd Hoffmann 
> ---
>  scripts/modinfo-generate.py | 84 +
>  include/qemu/module.h   | 17 
>  softmmu/vl.c|  4 ++
>  util/module.c   | 11 +
>  meson.build | 13 +-
>  5 files changed, 128 insertions(+), 1 deletion(-)
>  create mode 100755 scripts/modinfo-generate.py
> 
> diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py
> new file mode 100755
> index ..2b925432655a
> --- /dev/null
> +++ b/scripts/modinfo-generate.py
> @@ -0,0 +1,84 @@
> +#!/usr/bin/env python3
> +# -*- coding: utf-8 -*-
> +
> +import os
> +import sys
> +
> +def print_array(name, values):
> +if len(values) == 0:
> +return
> +list = ", ".join(values)
> +print(".%s = ((const char*[]){ %s, NULL })," % (name, list))
> +
> +def parse_line(line):
> +kind = ""
> +data = ""
> +get_kind = False
> +get_data = False
> +for item in line.split():
> +if item == "MODINFO_START":
> +get_kind = True
> +continue
> +if item.startswith("MODINFO_END"):
> +get_data = False
> +continue
> +if get_kind:
> +kind = item
> +get_kind = False
> +get_data = True
> +continue
> +if get_data:
> +data += " " + item
> +continue
> +return (kind, data)
> +
> +def generate(name, lines):
> +arch = ""
> +objs = []
> +deps = []
> +opts = []
> +for line in lines:
> +if line.find("MODINFO_START") != -1:
> +(kind, data) = parse_line(line)
> +if kind == 'obj':
> +objs.append(data)
> +elif kind == 'dep':
> +deps.append(data)
> +elif kind == 'opts':
> +opts.append(data)
> +elif kind == 'arch':
> +arch = data;
> +else:
> +print("unknown:", kind)
> +exit(1)
> +
> +print(".name = \"%s\"," % name)
> +if arch != "":
> +print(".arch = %s," % arch)
> +print_array("objs", objs)
> +print_array("deps", deps)
> +print_array("opts", opts)
> +print("},{");
> +
> +def print_pre():
> +print("/* generated by scripts/modinfo.py */")

generated by scripts/modinfo-generate.py

> +print("#include \"qemu/osdep.h\"")
> +print("#include \"qemu/module.h\"")
> +print("const QemuModinfo qemu_modinfo[] = {{")
> +
> +def print_post():
> +print("/* end of list */")
> +print("}};")
> +
> +def main(args):
> +print_pre()
> +for modinfo in args:
> +with open(modinfo) as f:
> +lines = f.readlines()
> +print("/* %s */" % modinfo)
> +(basename, ext) = os.path.splitext(modinfo)
> +generate(basename, lines)
> +print_post()
> +
> +if __name__ == "__main__":
> +main(sys.argv[1:])
> diff --git a/include/qemu/module.h b/include/qemu/module.h
> index 81ef086da023..a98748d501d3 100644
> --- a/include/qemu/module.h
> +++ b/include/qemu/module.h
> @@ -98,4 +98,21 @@ void module_load_qom_all(void);
>  /* module registers QemuOpts  */
>  #define module_opts(name) modinfo(opts, name)
>  
> +/*
> + * module info database
> + *
> + * scripts/modinfo-generate.c will build this using the data collected
> + * by scripts/modinfo-collect.py
> + */
> +typedef struct QemuModinfo QemuModinfo;
> +struct QemuModinfo {
> +const char *name;
> +const char *arch;
> +const char **objs;
> +const char **deps;
> +const char **opts;
> +};
> +extern const QemuModinfo qemu_modinfo[];
> +void module_init_info(const QemuModinfo *info);
> +
>  #endif
> diff --git a/softmmu/vl.c b/softmmu/vl.c
> index 326c1e908008..a4857ec43ff3 100644
> --- a/softmmu/vl.c
> +++ b/softmmu/vl.c
> @@ -2740,6 +2740,10 @@ void qemu_init(int argc, char **argv, char **envp)
>  error_init(argv[0]);
>  qemu_init_exec_dir(argv[0]);
>  
> +#ifdef CONFIG_MODULES
> +module_init_info(qemu_modinfo);
> +#endif
> +
>  qemu_init_subsystems();
>  
>  /* first pass of option parsing */
> diff --git a/util/module.c b/util/module.c
> index eee8ff2de136..8d3e8275b9f7 100644
> --- a/util/module.c
> +++ b/util/module.c
> @@ -110,6 +110,17 @@ void module_call_init(module_init_type type)
>  }
>  
>  #ifdef CONFIG_MODULES
> +
> +static const QemuModinfo module_info_stub[] = { {
> +/* end of list */
> +} };
> +static const QemuModinfo *module_info = module_info_stub;
> +
> +void module_init_info(const QemuModinfo *info)
> +{
> +module_info = info;
> +}
> +
>  static int module_load_file(const char *fname, bool mayfail, bool 
> export_symbols)
>  {
>  GModule *g_module;
> diff --git a/meson.build 

Re: [PATCH 3/4] modules: module.h kerneldoc annotations

2021-06-22 Thread Jose R. Ziviani
Hello Gerd,

On Tue, Jun 22, 2021 at 02:51:09PM +0200, Gerd Hoffmann wrote:
> ---
>  include/qemu/module.h | 59 +--
>  1 file changed, 45 insertions(+), 14 deletions(-)

This header has a copyright date from 2009. Not sure if it requires an
update.

> 
> diff --git a/include/qemu/module.h b/include/qemu/module.h
> index 7f4b1af8198c..8bc80535a4d4 100644
> --- a/include/qemu/module.h
> +++ b/include/qemu/module.h
> @@ -74,11 +74,18 @@ void module_load_qom_one(const char *type);
>  void module_load_qom_all(void);
>  void module_allow_arch(const char *arch);
>  
> -/*
> - * module info annotation macros
> +/**
> + * DOC: module info annotation macros
>   *
> - * scripts/modinfo-collect.py will collect module info,
> - * using the preprocessor and -DQEMU_MODINFO
> + * `scripts/modinfo-collect.py` will collect module info,
> + * using the preprocessor and -DQEMU_MODINFO.
> + *
> + * `scripts/modinfo-generate.py` will create a module meta-data database
> + * from the collected information so qemu knows about module
> + * dependencies and QOM objects implemented by modules.
> + *
> + * See `*.modinfo` and `modinfo.c` in the build directory to check the
> + * script results.
>   */
>  #ifdef QEMU_MODINFO
>  # define modinfo(kind, value) \
> @@ -87,24 +94,48 @@ void module_allow_arch(const char *arch);
>  # define modinfo(kind, value)
>  #endif
>  
> -/* module implements QOM type  */
> +/**
> + * module_obj
> + *
> + * @name: QOM type.
> + *
> + * This module implements QOM type @name.
> + */
>  #define module_obj(name) modinfo(obj, name)
>  
> -/* module has a dependency on  */
> +/**
> + * module_dep
> + *
> + * @name: module name
> + *
> + * This module depends on module @name.
> + */
>  #define module_dep(name) modinfo(dep, name)
>  
> -/* module is for target architecture  */
> +/**
> + * module_arch
> + *
> + * @arch: target architecture
> + *
> + * This module is for target architecture @arch.
> + *
> + * Note that target-dependent modules are tagged automatically, so
> + * this is only needed in case target-independent modules should be
> + * restricted.  Use case example: the ccw bus is implemented by s390x
> + * only.
> + */
>  #define module_arch(name) modinfo(arch, name)
>  
> -/* module registers QemuOpts  */
> +/**
> + * module_opts
> + *
> + * @name: QemuOpts name
> + *
> + * This module registers QemuOpts @name.
> + */
>  #define module_opts(name) modinfo(opts, name)
>  
> -/*
> - * module info database
> - *
> - * scripts/modinfo-generate.c will build this using the data collected
> - * by scripts/modinfo-collect.py
> - */
> +/* module info database (created by scripts/modinfo-generate.py) */
>  typedef struct QemuModinfo QemuModinfo;
>  struct QemuModinfo {
>  const char *name;
> -- 
> 2.31.1
> 
> 


signature.asc
Description: Digital signature


[PATCH] tcg/arm: Fix tcg_out_op function signature

2021-06-10 Thread Jose R. Ziviani
Commit 5e8892db93 fixed several function signatures but tcg_out_op for
arm is missing. This patch fixes it as well.

Signed-off-by: Jose R. Ziviani 
---
 tcg/arm/tcg-target.c.inc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index f4c9cb8f9f..5157143246 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -1984,7 +1984,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg 
*args, bool is64)
 static void tcg_out_epilogue(TCGContext *s);
 
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-const TCGArg *args, const int *const_args)
+const TCGArg args[TCG_MAX_OP_ARGS],
+const int const_args[TCG_MAX_OP_ARGS])
 {
 TCGArg a0, a1, a2, a3, a4, a5;
 int c;
-- 
2.31.1




Performance issue with qcow2/raid

2021-05-27 Thread Jose R. Ziviani
Hello team,

I'm currently investigating a performance regression detected by iozone 
filesystem benchmark (https://www.iozone.org/).

Basically, if I format a QCOW2 image with XFS filesystem in my guest and run 
iozone I'll get the following result:

$ mkfs.xfs -f /dev/xvdb1 && \
  mount -t xfs /dev/xvdb1 /mnt && \
  /opt/iozone/bin/iozone -a -e -s 16777216 -y 4 -q 8 -i 0 -i 1 -f 
/mnt/iozone.dat

kBblock len  read reread
16777216  4K 354790   348796
16777216  8K 362356   364818

However, if I revert the commit 46cd1e8a47 (qcow2: Skip copy-on-write when 
allocating a zero cluster) and run the same, I see a huge improvement:

$ mkfs.xfs -f /dev/xvdb1 && \
  mount -t xfs /dev/xvdb1 /mnt && \
  /opt/iozone/bin/iozone -a -e -s 16777216 -y 4 -q 8 -i 0 -i 1 -f 
/mnt/iozone.dat

kBblock len  read reread
16777216  4K 524067   560057
16777216  8K 538661   537004

Note that if I run iozone without re-formating the disk, I'll get results 
similar to last formatting. In other words, if I my current QEMU executable 
doesn't have commit 46cd1e8a47 and I format the disk, iozone will continue 
showing good results even if I reboot to use QEMU with that commit patched.

My system has a RAID controller[1] and runs QEMU/Xen. I'm not able to reproduce 
such behavior in other systems. 

Do you have any suggestion to help debugging this? What more info could help to 
understand it better?
My next approach is using perf, but I appreciate if you have any hints measure 
qcow efficiently.

[1]
# lspci -vv | grep -i raid
1a:00.0 RAID bus controller: Broadcom / LSI MegaRAID SAS-3 3108 [Invader] (rev 
02)
Kernel driver in use: megaraid_sas
Kernel modules: megaraid_sas

Thank you very much!

Jose R. Ziviani


signature.asc
Description: Digital signature


Re: Incorrect NVMe DLFEAT?

2021-04-29 Thread Harris, James R


On 4/29/21, 10:22 AM, "Klaus Jensen"  wrote:

On Apr 29 16:51, Harris, James R wrote:
>Hi,
>

Hi Jim,

>I’m seeing SPDK test failures with QEMU NVMe controllers that I’ve 
>bisected to QEMU commit 2605257a26 (“hw/block/nvme: add the dataset 
>management command”).
>
>The failing tests are related to write zeroes handling.  If an NVMe 
>controller supports DSM, and DLFEAT indicates that deallocated blocks 
>will read back as zeroes, then SPDK uses DEALLOCATE to implement the 
>write zeroes operation.  (Note: SPDK prefers this method to using NVMe 
>WRITE_ZEROES, since the latter is limited to a 16-bit block count.)
>

The Dataset Management command is advisory, the controller gives no 
guarantee that it will actually deallocate anything. You cannot use DSM 
as a realiable alternative to Write Zeroes. The QEMU emulated nvme 
device exploits this and in many cases wont deallocate anything if it 
does not fit nicely with the underlying block device setup.

[JH] Hmmm, looks like we may need some SPDK changes for this case.

>QEMU sets DLFEAT = 0x9 – and actually set it to 0x9 even before this 
>commit.  Since the lower 3 bits are 0b001, it is reporting that 
>deallocated blocks will read back later as 0.  This does not actually 
>seem to be the case however – reading previously deallocated blocks do 
>not actually return 0s.
>

Can you share the configuration you use for QEMU? DSM works best with 
4096 byte LBAs (logical_block_size=4096) and the raw format driver.

Also, please test with the Error Recovery feature (and set DULBE) to 
test if the device reports that the block is actually deallocated.

[JH] I've tested both 512 and 4096 LBAs and hacked something up to enable
DULBE.  I am not seeing any errors reading blocks that were part of a
previous DSM command.  So at least for the cases in the SPDK test, these
blocks are not actually being deallocated.

>It seems DLFEAT is being set incorrectly here – should probably be 0x8 
>instead?
>
>Thanks,
>
>Jim
>
>
>

-- 
k



Incorrect NVMe DLFEAT?

2021-04-29 Thread Harris, James R
Hi,

I’m seeing SPDK test failures with QEMU NVMe controllers that I’ve bisected to 
QEMU commit 2605257a26 (“hw/block/nvme: add the dataset management command”).

The failing tests are related to write zeroes handling.  If an NVMe controller 
supports DSM, and DLFEAT indicates that deallocated blocks will read back as 
zeroes, then SPDK uses DEALLOCATE to implement the write zeroes operation.  
(Note: SPDK prefers this method to using NVMe WRITE_ZEROES, since the latter is 
limited to a 16-bit block count.)

QEMU sets DLFEAT = 0x9 – and actually set it to 0x9 even before this commit.  
Since the lower 3 bits are 0b001, it is reporting that deallocated blocks will 
read back later as 0.  This does not actually seem to be the case however – 
reading previously deallocated blocks do not actually return 0s.

It seems DLFEAT is being set incorrectly here – should probably be 0x8 instead?

Thanks,

Jim





[Bug 1883784] Re: [ppc64le] qemu behavior differs from ppc64le hardware

2020-06-17 Thread Michael R. Crusoe
I just ran the provided binaries on a qemu-system-ppc64 version 5.0-5
from Debian Bullseye and they also aborted there

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1883784

Title:
  [ppc64le] qemu behavior differs from ppc64le hardware

Status in QEMU:
  New

Bug description:
  I have some code which passes my test suite on PPC64LE hardware when
  compiled with GCC 10, but the saem binary fails with both qemu-ppc64le
  4.2 (on Fedora 32) and qemu-ppc64le-static 5.0.0 (Debian testing).

  I'm not getting any errors about illegal instructions or anything,
  like that; the results are just silently different on qemu.

  I've generated a reduced test case, which is attached along with the
  binaries (both are the same code, one is just statically linked).
  They should execute successufully on PPC64LE hardware, but on qemu
  they hit a __builtin_abort (because the computed value doesn't match
  the expected value).

  Without being familiar with PPC assembly I'm not sure what else I can
  do, but if there is anything please let me know.

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1883784/+subscriptions



Re: [RHEL-8.1 virt 2/2] target/i386: sev: Do not pin the ram device memory region

2019-11-06 Thread Gary R Hook



- Original Message -
> From: "Gary R Hook" 
> To: rhvirt-patc...@redhat.com
> Cc: qemu-devel@nongnu.org, "Paolo Bonzini" , 
> gh...@redhat.com, "Eduardo Habkost"
> , "Richard Henderson" 
> Sent: Tuesday, April 9, 2019 7:08:03 PM
> Subject: [RHEL-8.1 virt 2/2] target/i386: sev: Do not pin the ram device  
> memory region
> 
> BZ: 1667249
> Branch: rhel-8.1.0
> Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=1667249
> Upstream Status: 4.0.0-rc1
> Build Info:
> https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=20980582
> Conflicts: None
> 
> commit cedc0ad539afbbb669dba9e73dfad2915bc1c25b
> Author: Singh, Brijesh 
> Date:   Mon Feb 4 22:23:40 2019 +
> 
> target/i386: sev: Do not pin the ram device memory region
> 
> The RAM device presents a memory region that should be handled
> as an IO region and should not be pinned.
> 
> In the case of the vfio-pci, RAM device represents a MMIO BAR
> and the memory region is not backed by pages hence
> KVM_MEMORY_ENCRYPT_REG_REGION fails to lock the memory range.
> 
> Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1667249
> Cc: Alex Williamson 
> Cc: Paolo Bonzini 
> Signed-off-by: Brijesh Singh 
> Message-Id: <20190204222322.26766-3-brijesh.si...@amd.com>
> Signed-off-by: Paolo Bonzini 
> 
> Cc: Paolo Bonzini 
> Cc: Richard Henderson 
> Cc: Eduardo Habkost 
> Cc: qemu-devel@nongnu.org

NACKed-by: Gary R Hook 

> ---
>  target/i386/sev.c | 11 +++
>  1 file changed, 11 insertions(+)
> 
> diff --git a/target/i386/sev.c b/target/i386/sev.c
> index 2395171acf..b8009b001a 100644
> --- a/target/i386/sev.c
> +++ b/target/i386/sev.c
> @@ -130,6 +130,17 @@ sev_ram_block_added(RAMBlockNotifier *n, void *host,
> size_t size)
>  {
>  int r;
>  struct kvm_enc_region range;
> +ram_addr_t offset;
> +MemoryRegion *mr;
> +
> +/*
> + * The RAM device presents a memory region that should be treated
> + * as IO region and should not be pinned.
> + */
> +mr = memory_region_from_host(host, );
> +if (mr && memory_region_is_ram_device(mr)) {
> + return;
> +}
>  
>  range.addr = (__u64)(unsigned long)host;
>  range.size = size;
> --
> 2.18.1
> 
> 




[Qemu-devel] [Bug 1776920] Re: qemu-img convert on Mac OSX creates corrupt images

2019-07-23 Thread Sven R
Hi,

I recently ran into problems and after a long time trying to find out the cause 
landed here, I got in trouble using a CentOs Cloud image:
https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1905.qcow2.xz
which extracts to a .qcow2 image with sha256 of:
b376afdc0150601f15e53516327d9832216e01a300fecfc302066e36e2ac2b39

image: CentOS-7-x86_64-GenericCloud-1905.qcow2
file format: qcow2
virtual size: 8.0G (8589934592 bytes)
disk size: 898M
cluster_size: 65536
Format specific information:
compat: 0.10
refcount bits: 16

I use this command on a Mac, OS X 10.13.6 (17G7024), qemu installed via brew:
qemu-img convert -f qcow2 -O vmdk -o 
adapter_type=lsilogic,subformat=streamOptimized,compat6=on -p 
CentOS-7-x86_64-GenericCloud-1905.qcow2 -S 0 result.vmdk


941359104 21 Jul 17:11 CentOS-7-x86_64-GenericCloud-1905.qcow2 - original image

Converting this gives these results:
214551040 23 Jul 20:45 conv_mac_v3_1_mit_s_0.vmdk - doesnt work, made on Mac 
(APFS) with -S 0 
402303488 23 Jul 20:50 conv_mac_v3_1_auf_exfat.vmdk - works and is bootable, 
made on same Mac, on external drive, exFAT formatted.
149743104 23 Jul 21:21 conv_mac_v4_0.vmdk - doesnt work, made on Mac (APFS) 
without -S 0 
214551040 23 Jul 21:20 conv_mac_v4_0mit_S0.vmdk - doesnt work, made on Mac 
(APFS) with -S 0 

converting on non-Mac also works fine:
402303488 23 Jul 18:48 conv_centos7_v1-5-3.vmdk - works and is bootable, made 
on Centos, qemu-img version 1.5.3

So it seems that '-S 0' didn't fix it for me, or is that only in the
development branch?

Best Regards

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1776920

Title:
  qemu-img convert on Mac OSX creates corrupt images

Status in QEMU:
  New

Bug description:
  An image created by qemu-img create, then modified by another program
  is converted to bad/corrupt image when using convert sub command on
  Mac OSX. The same convert works on Linux. The version of qemu-img is
  2.12.

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1776920/+subscriptions



[Qemu-devel] [RHEL-8.1 virt 1/2] memory: Fix the memory region type assignment order

2019-04-09 Thread Gary R Hook
BZ: 1667249
Branch: rhel-8.1.0
Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=1667249
Upstream Status: 4.0.0-rc1
Build Info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=20980582
Conflicts: None

commit 2ddb89b00f947f785c9ca6742f28f954e3b75e62
Author: Singh, Brijesh 
Date:   Mon Feb 4 22:23:39 2019 +

memory: Fix the memory region type assignment order

Currently, a callback registered through the RAMBlock notifier
is not able to get the memory region type (i.e callback is not
able to use memory_region_is_ram_device function). This is
because mr->ram assignment happens _after_ the memory is allocated
whereas the callback is executed during allocation.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1667249
Suggested-by: Alex Williamson 
Cc: Paolo Bonzini 
Reviewed-by: Alex Williamson 
Signed-off-by: Brijesh Singh 
Message-Id: <20190204222322.26766-2-brijesh.si...@amd.com>
Signed-off-by: Paolo Bonzini 

Cc: Paolo Bonzini  
Cc: qemu-devel@nongnu.org 
---
 memory.c | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/memory.c b/memory.c
index 4974f972d5..04ff5e9108 100644
--- a/memory.c
+++ b/memory.c
@@ -1631,10 +1631,17 @@ void memory_region_init_ram_device_ptr(MemoryRegion *mr,
uint64_t size,
void *ptr)
 {
-memory_region_init_ram_ptr(mr, owner, name, size, ptr);
+memory_region_init(mr, owner, name, size);
+mr->ram = true;
+mr->terminates = true;
 mr->ram_device = true;
 mr->ops = _device_mem_ops;
 mr->opaque = mr;
+mr->destructor = memory_region_destructor_ram;
+mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
+/* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
+assert(ptr != NULL);
+mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, _fatal);
 }
 
 void memory_region_init_alias(MemoryRegion *mr,
-- 
2.18.1




[Qemu-devel] [RHEL-8.1 virt 2/2] target/i386: sev: Do not pin the ram device memory region

2019-04-09 Thread Gary R Hook
BZ: 1667249
Branch: rhel-8.1.0
Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=1667249
Upstream Status: 4.0.0-rc1
Build Info: https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=20980582
Conflicts: None

commit cedc0ad539afbbb669dba9e73dfad2915bc1c25b
Author: Singh, Brijesh 
Date:   Mon Feb 4 22:23:40 2019 +

target/i386: sev: Do not pin the ram device memory region

The RAM device presents a memory region that should be handled
as an IO region and should not be pinned.

In the case of the vfio-pci, RAM device represents a MMIO BAR
and the memory region is not backed by pages hence
KVM_MEMORY_ENCRYPT_REG_REGION fails to lock the memory range.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1667249
Cc: Alex Williamson 
Cc: Paolo Bonzini 
Signed-off-by: Brijesh Singh 
Message-Id: <20190204222322.26766-3-brijesh.si...@amd.com>
Signed-off-by: Paolo Bonzini 

Cc: Paolo Bonzini  
Cc: Richard Henderson  
Cc: Eduardo Habkost  
Cc: qemu-devel@nongnu.org 
---
 target/i386/sev.c | 11 +++
 1 file changed, 11 insertions(+)

diff --git a/target/i386/sev.c b/target/i386/sev.c
index 2395171acf..b8009b001a 100644
--- a/target/i386/sev.c
+++ b/target/i386/sev.c
@@ -130,6 +130,17 @@ sev_ram_block_added(RAMBlockNotifier *n, void *host, 
size_t size)
 {
     int r;
 struct kvm_enc_region range;
+ram_addr_t offset;
+MemoryRegion *mr;
+
+/*
+ * The RAM device presents a memory region that should be treated
+ * as IO region and should not be pinned.
+ */
+mr = memory_region_from_host(host, );
+if (mr && memory_region_is_ram_device(mr)) {
+   return;
+}
 
 range.addr = (__u64)(unsigned long)host;
 range.size = size;
-- 
2.18.1




[Qemu-devel] [RHEL-8.1 virt 0/2] Enable SEV VM to boot with assigned PCI device

2019-04-09 Thread Gary R Hook
Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=1667249

On an AMD SEV enabled host with an SEV enabled guest, attaching an
assigned device to the VM results in a failure to start the VM:

qemu-kvm: -device vfio-pci,host=01:00.0,id=hostdev0,bus=pci.2,addr=0x0: 
sev_ram_block_added: failed to register region (0x7fd96e6bb000+0x2) error 
'Cannot allocate memory'

In this example the assigned device is a simple Intel 82574L NIC:

01:00.0 Ethernet controller: Intel Corporation 82574L Gigabit Network Connection
Subsystem: Intel Corporation Gigabit CT Desktop Adapter
Flags: bus master, fast devsel, latency 0, IRQ 89, NUMA node 0
Memory at fb9c (32-bit, non-prefetchable) [size=128K]
Memory at fb90 (32-bit, non-prefetchable) [size=512K]

Note that the error indicates the region as (base+size) where a size
of 0x2 is 128K, which matches that of BAR0 for the device.
dmesg on the host also reports:

SVM: SEV: Failure locking 32 pages.

SEV guests make use of the RAMBlock notifier in QEMU to add page
pinnings for SEV; the kernel side of the call only knows how to pin
pages with get_user_pages(), and this currently faults on non-page
backed mappings (e.g. the mmap of an MMIO BAR).

To resolve this failure, change the order of the memory region type
assignment and avoid pinning device memory regions.

Cc: "Danilo C. L. de Paula" 
Cc: Eduardo Habkost 
Cc: Paolo Bonzini 
Cc: qemu-devel@nongnu.org
Cc: Richard Henderson 

Danilo C. L. de Paula (2):
  redhat: branching qemu-kvm to rhel-8.1.0
  redhat: renaming branch to rhel-8.1.0

Gary R Hook (2):
  Subject: memory: Fix the memory region type assignment order
  Subject: target/i386: sev: Do not pin the ram device memory region

 .gitpublish   |  6 +++---
 memory.c  |  9 -
 target/i386/sev.c | 11 +++
 3 files changed, 22 insertions(+), 4 deletions(-)

-- 
2.18.1




[Qemu-devel] [Bug 1818880] Re: Deadlock when detaching network interface

2019-03-07 Thread Heitor R. Alves de Siqueira
** Patch removed: "Debdiff for xenial v2"
   
https://bugs.launchpad.net/cloud-archive/+bug/1818880/+attachment/5244567/+files/xenial_v2.debdiff

** Patch added: "Correct debdiff for xenial v2"
   
https://bugs.launchpad.net/cloud-archive/+bug/1818880/+attachment/5244568/+files/lp1818880-v2.debdiff

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1818880

Title:
  Deadlock when detaching network interface

Status in Ubuntu Cloud Archive:
  Confirmed
Status in QEMU:
  Fix Released
Status in qemu package in Ubuntu:
  Fix Released
Status in qemu source package in Xenial:
  Confirmed
Status in qemu source package in Bionic:
  Fix Released
Status in qemu source package in Cosmic:
  Fix Released
Status in qemu source package in Disco:
  Fix Released

Bug description:
  [Impact]
  Qemu guests hang indefinitely

  [Description]
  When running a Qemu guest with VirtIO network interfaces, detaching an 
interface that's currently being used can result in a deadlock. The guest 
instance will hang and become unresponsive to commands, and the only option is 
to kill -9 the instance.
  The reason for this is a dealock between a monitor and an RCU thread, which 
will fight over the BQL (qemu_global_mutex) and the critical RCU section locks. 
The monitor thread will acquire the BQL for detaching the network interface, 
and fire up a helper thread to deal with detaching the network adapter. That 
new thread needs to wait on the RCU thread to complete the deletion, but the 
RCU thread wants the BQL to commit its transactions.
  This bug is already fixed upstream (73c6e4013b4c rcu: completely disable 
pthread_atfork callbacks as soon as possible) and included for other series 
(see below), so we don't need to backport it to Bionic onwards.

  Upstream commit:
  https://git.qemu.org/?p=qemu.git;a=commit;h=73c6e4013b4c

  $ git describe --contains 73c6e4013b4c
  v2.10.0-rc2~1^2~8

  $ rmadison qemu
  ===> qemu | 1:2.5+dfsg-5ubuntu10.34 | xenial-updates/universe   | amd64, ...
   qemu | 1:2.11+dfsg-1ubuntu7| bionic/universe   | amd64, ...
   qemu | 1:2.12+dfsg-3ubuntu8| cosmic/universe   | amd64, ...
   qemu | 1:3.1+dfsg-2ubuntu2 | disco/universe| amd64, ...

  [Test Case]
  Being a racing condition, this is a tricky bug to reproduce consistently. 
We've had reports of users running into this with OpenStack deployments and 
Windows Server guests, and the scenario is usually like this:
  1) Deploy a 16vCPU Windows Server 2012 R2 guest with a virtio network 
interface
  2) Stress the network interface with e.g. Windows HLK test suite or similar
  3) Repeatedly attach/detach the network adapter that's in use
  It usually takes more than ~4000 attach/detach cycles to trigger the bug.

  [Regression Potential]
  Regressions for this might arise from the fact that the fix changes RCU lock 
code. Since this patch has been upstream and in other series for a while, it's 
unlikely that it would regressions in RCU code specifically. Other code that 
makes use of the RCU locks (MMIO and some monitor events) will be thoroughly 
tested for any regressions with use-case scenarios and scripted runs.

To manage notifications about this bug go to:
https://bugs.launchpad.net/cloud-archive/+bug/1818880/+subscriptions



[Qemu-devel] [Bug 1818880] Re: Deadlock when detaching network interface

2019-03-07 Thread Heitor R. Alves de Siqueira
Patch v2:
Added missing DEP3 info and corrected pkg version

** Patch added: "Debdiff for xenial v2"
   
https://bugs.launchpad.net/cloud-archive/+bug/1818880/+attachment/5244567/+files/xenial_v2.debdiff

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1818880

Title:
  Deadlock when detaching network interface

Status in Ubuntu Cloud Archive:
  Confirmed
Status in QEMU:
  Fix Released
Status in qemu package in Ubuntu:
  Fix Released
Status in qemu source package in Xenial:
  Confirmed
Status in qemu source package in Bionic:
  Fix Released
Status in qemu source package in Cosmic:
  Fix Released
Status in qemu source package in Disco:
  Fix Released

Bug description:
  [Impact]
  Qemu guests hang indefinitely

  [Description]
  When running a Qemu guest with VirtIO network interfaces, detaching an 
interface that's currently being used can result in a deadlock. The guest 
instance will hang and become unresponsive to commands, and the only option is 
to kill -9 the instance.
  The reason for this is a dealock between a monitor and an RCU thread, which 
will fight over the BQL (qemu_global_mutex) and the critical RCU section locks. 
The monitor thread will acquire the BQL for detaching the network interface, 
and fire up a helper thread to deal with detaching the network adapter. That 
new thread needs to wait on the RCU thread to complete the deletion, but the 
RCU thread wants the BQL to commit its transactions.
  This bug is already fixed upstream (73c6e4013b4c rcu: completely disable 
pthread_atfork callbacks as soon as possible) and included for other series 
(see below), so we don't need to backport it to Bionic onwards.

  Upstream commit:
  https://git.qemu.org/?p=qemu.git;a=commit;h=73c6e4013b4c

  $ git describe --contains 73c6e4013b4c
  v2.10.0-rc2~1^2~8

  $ rmadison qemu
  ===> qemu | 1:2.5+dfsg-5ubuntu10.34 | xenial-updates/universe   | amd64, ...
   qemu | 1:2.11+dfsg-1ubuntu7| bionic/universe   | amd64, ...
   qemu | 1:2.12+dfsg-3ubuntu8| cosmic/universe   | amd64, ...
   qemu | 1:3.1+dfsg-2ubuntu2 | disco/universe| amd64, ...

  [Test Case]
  Being a racing condition, this is a tricky bug to reproduce consistently. 
We've had reports of users running into this with OpenStack deployments and 
Windows Server guests, and the scenario is usually like this:
  1) Deploy a 16vCPU Windows Server 2012 R2 guest with a virtio network 
interface
  2) Stress the network interface with e.g. Windows HLK test suite or similar
  3) Repeatedly attach/detach the network adapter that's in use
  It usually takes more than ~4000 attach/detach cycles to trigger the bug.

  [Regression Potential]
  Regressions for this might arise from the fact that the fix changes RCU lock 
code. Since this patch has been upstream and in other series for a while, it's 
unlikely that it would regressions in RCU code specifically. Other code that 
makes use of the RCU locks (MMIO and some monitor events) will be thoroughly 
tested for any regressions with use-case scenarios and scripted runs.

To manage notifications about this bug go to:
https://bugs.launchpad.net/cloud-archive/+bug/1818880/+subscriptions



[Qemu-devel] [Bug 1818880] Re: Deadlock when detaching network interface

2019-03-07 Thread Heitor R. Alves de Siqueira
** Changed in: qemu (Ubuntu Disco)
 Assignee: Heitor R. Alves de Siqueira (halves) => (unassigned)

** Changed in: qemu (Ubuntu Cosmic)
 Assignee: Heitor R. Alves de Siqueira (halves) => (unassigned)

** Changed in: qemu (Ubuntu Bionic)
 Assignee: Heitor R. Alves de Siqueira (halves) => (unassigned)

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1818880

Title:
  Deadlock when detaching network interface

Status in Ubuntu Cloud Archive:
  Confirmed
Status in QEMU:
  Fix Released
Status in qemu package in Ubuntu:
  Fix Released
Status in qemu source package in Xenial:
  Confirmed
Status in qemu source package in Bionic:
  Fix Released
Status in qemu source package in Cosmic:
  Fix Released
Status in qemu source package in Disco:
  Fix Released

Bug description:
  [Impact]
  Qemu guests hang indefinitely

  [Description]
  When running a Qemu guest with VirtIO network interfaces, detaching an 
interface that's currently being used can result in a deadlock. The guest 
instance will hang and become unresponsive to commands, and the only option is 
to kill -9 the instance.
  The reason for this is a dealock between a monitor and an RCU thread, which 
will fight over the BQL (qemu_global_mutex) and the critical RCU section locks. 
The monitor thread will acquire the BQL for detaching the network interface, 
and fire up a helper thread to deal with detaching the network adapter. That 
new thread needs to wait on the RCU thread to complete the deletion, but the 
RCU thread wants the BQL to commit its transactions.
  This bug is already fixed upstream (73c6e4013b4c rcu: completely disable 
pthread_atfork callbacks as soon as possible) and included for other series 
(see below), so we don't need to backport it to Bionic onwards.

  Upstream commit:
  https://git.qemu.org/?p=qemu.git;a=commit;h=73c6e4013b4c

  $ git describe --contains 73c6e4013b4c
  v2.10.0-rc2~1^2~8

  $ rmadison qemu
  ===> qemu | 1:2.5+dfsg-5ubuntu10.34 | xenial-updates/universe   | amd64, ...
   qemu | 1:2.11+dfsg-1ubuntu7| bionic/universe   | amd64, ...
   qemu | 1:2.12+dfsg-3ubuntu8| cosmic/universe   | amd64, ...
   qemu | 1:3.1+dfsg-2ubuntu2 | disco/universe| amd64, ...

  [Test Case]
  Being a racing condition, this is a tricky bug to reproduce consistently. 
We've had reports of users running into this with OpenStack deployments and 
Windows Server guests, and the scenario is usually like this:
  1) Deploy a 16vCPU Windows Server 2012 R2 guest with a virtio network 
interface
  2) Stress the network interface with e.g. Windows HLK test suite or similar
  3) Repeatedly attach/detach the network adapter that's in use
  It usually takes more than ~4000 attach/detach cycles to trigger the bug.

  [Regression Potential]
  Regressions for this might arise from the fact that the fix changes RCU lock 
code. Since this patch has been upstream and in other series for a while, it's 
unlikely that it would regressions in RCU code specifically. Other code that 
makes use of the RCU locks (MMIO and some monitor events) will be thoroughly 
tested for any regressions with use-case scenarios and scripted runs.

To manage notifications about this bug go to:
https://bugs.launchpad.net/cloud-archive/+bug/1818880/+subscriptions



[Qemu-devel] [Bug 1818880] Re: Deadlock when detaching network interface

2019-03-07 Thread Heitor R. Alves de Siqueira
** Patch added: "Debdiff for xenial"
   
https://bugs.launchpad.net/cloud-archive/+bug/1818880/+attachment/5244384/+files/xenial.debdiff

** Description changed:

  [Impact]
  Qemu guests hang indefinitely
  
  [Description]
  When running a Qemu guest with VirtIO network interfaces, detaching an 
interface that's currently being used can result in a deadlock. The guest 
instance will hang and become unresponsive to commands, and the only option is 
to kill -9 the instance.
  The reason for this is a dealock between a monitor and an RCU thread, which 
will fight over the BQL (qemu_global_mutex) and the critical RCU section locks. 
The monitor thread will acquire the BQL for detaching the network interface, 
and fire up a helper thread to deal with detaching the network adapter. That 
new thread needs to wait on the RCU thread to complete the deletion, but the 
RCU thread wants the BQL to commit its transactions.
  This bug is already fixed upstream (73c6e4013b4c rcu: completely disable 
pthread_atfork callbacks as soon as possible) and included for other series 
(see below), so we don't need to backport it to Bionic onwards.
  
  Upstream commit:
  https://git.qemu.org/?p=qemu.git;a=commit;h=73c6e4013b4c
  
  $ git describe --contains 73c6e4013b4c
  v2.10.0-rc2~1^2~8
  
  $ rmadison qemu
  ===> qemu | 1:2.5+dfsg-5ubuntu10.34 | xenial-updates/universe   | amd64, ...
-  qemu | 1:2.11+dfsg-1ubuntu7| bionic/universe   | amd64, ...
-  qemu | 1:2.12+dfsg-3ubuntu8| cosmic/universe   | amd64, ...
-  qemu | 1:3.1+dfsg-2ubuntu2 | disco/universe| amd64, ...
+  qemu | 1:2.11+dfsg-1ubuntu7| bionic/universe   | amd64, ...
+  qemu | 1:2.12+dfsg-3ubuntu8| cosmic/universe   | amd64, ...
+  qemu | 1:3.1+dfsg-2ubuntu2 | disco/universe| amd64, ...
  
  [Test Case]
  Being a racing condition, this is a tricky bug to reproduce consistently. 
We've had reports of users running into this with OpenStack deployments and 
Windows Server guests, and the scenario is usually like this:
  1) Deploy a 16vCPU Windows Server 2012 R2 guest with a virtio network 
interface
  2) Stress the network interface with e.g. Windows HLK test suite or similar
  3) Repeatedly attach/detach the network adapter that's in use
  It usually takes more than ~4000 attach/detach cycles to trigger the bug.
  
  [Regression Potential]
- Regressions for this might arise from the fact that the fix changes RCU lock 
code. Since this patch has been upstream and in other series for a while, it's 
unlikely that it would regressions in RCU code specifically. Other code that 
makes use of the RCU locks (MMIO and some monitor events) will be thoroughly 
tested for any regressions with autokpkgtest and scripted Qemu runs.
+ Regressions for this might arise from the fact that the fix changes RCU lock 
code. Since this patch has been upstream and in other series for a while, it's 
unlikely that it would regressions in RCU code specifically. Other code that 
makes use of the RCU locks (MMIO and some monitor events) will be thoroughly 
tested for any regressions with use-case scenarios and scripted runs.

** Tags added: sts-sponsor

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1818880

Title:
  Deadlock when detaching network interface

Status in Ubuntu Cloud Archive:
  Confirmed
Status in QEMU:
  Fix Released
Status in qemu package in Ubuntu:
  Fix Released
Status in qemu source package in Xenial:
  Confirmed
Status in qemu source package in Bionic:
  Fix Released
Status in qemu source package in Cosmic:
  Fix Released
Status in qemu source package in Disco:
  Fix Released

Bug description:
  [Impact]
  Qemu guests hang indefinitely

  [Description]
  When running a Qemu guest with VirtIO network interfaces, detaching an 
interface that's currently being used can result in a deadlock. The guest 
instance will hang and become unresponsive to commands, and the only option is 
to kill -9 the instance.
  The reason for this is a dealock between a monitor and an RCU thread, which 
will fight over the BQL (qemu_global_mutex) and the critical RCU section locks. 
The monitor thread will acquire the BQL for detaching the network interface, 
and fire up a helper thread to deal with detaching the network adapter. That 
new thread needs to wait on the RCU thread to complete the deletion, but the 
RCU thread wants the BQL to commit its transactions.
  This bug is already fixed upstream (73c6e4013b4c rcu: completely disable 
pthread_atfork callbacks as soon as possible) and included for other series 
(see below), so we don't need to backport it to Bionic onwards.

  Upstream commit:
  https://git.qemu.org/?p=qemu.git;a=commit;h=73c6e4013b4c

  $ git describe --contains 73c6e4013b4c
  v2.10.0-rc2~1^2~8

  $ rmadison qemu
  ===> qemu | 1:2.5+dfsg-5ubuntu10.34 | xenial-updates/universe   | 

[Qemu-devel] [Bug 1818880] Re: Deadlock when detaching network interface

2019-03-06 Thread Heitor R. Alves de Siqueira
** Also affects: cloud-archive
   Importance: Undecided
   Status: New

** Changed in: cloud-archive
   Status: New => Confirmed

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1818880

Title:
  Deadlock when detaching network interface

Status in Ubuntu Cloud Archive:
  Confirmed
Status in QEMU:
  Fix Released
Status in qemu package in Ubuntu:
  Fix Released
Status in qemu source package in Xenial:
  Confirmed
Status in qemu source package in Bionic:
  Fix Released
Status in qemu source package in Cosmic:
  Fix Released
Status in qemu source package in Disco:
  Fix Released

Bug description:
  [Impact]
  Qemu guests hang indefinitely

  [Description]
  When running a Qemu guest with VirtIO network interfaces, detaching an 
interface that's currently being used can result in a deadlock. The guest 
instance will hang and become unresponsive to commands, and the only option is 
to kill -9 the instance.
  The reason for this is a dealock between a monitor and an RCU thread, which 
will fight over the BQL (qemu_global_mutex) and the critical RCU section locks. 
The monitor thread will acquire the BQL for detaching the network interface, 
and fire up a helper thread to deal with detaching the network adapter. That 
new thread needs to wait on the RCU thread to complete the deletion, but the 
RCU thread wants the BQL to commit its transactions.
  This bug is already fixed upstream (73c6e4013b4c rcu: completely disable 
pthread_atfork callbacks as soon as possible) and included for other series 
(see below), so we don't need to backport it to Bionic onwards.

  Upstream commit:
  https://git.qemu.org/?p=qemu.git;a=commit;h=73c6e4013b4c

  $ git describe --contains 73c6e4013b4c
  v2.10.0-rc2~1^2~8

  $ rmadison qemu
  ===> qemu | 1:2.5+dfsg-5ubuntu10.34 | xenial-updates/universe   | amd64, ...
   qemu | 1:2.11+dfsg-1ubuntu7| bionic/universe   | amd64, ...
   qemu | 1:2.12+dfsg-3ubuntu8| cosmic/universe   | amd64, ...
   qemu | 1:3.1+dfsg-2ubuntu2 | disco/universe| amd64, ...

  [Test Case]
  Being a racing condition, this is a tricky bug to reproduce consistently. 
We've had reports of users running into this with OpenStack deployments and 
Windows Server guests, and the scenario is usually like this:
  1) Deploy a 16vCPU Windows Server 2012 R2 guest with a virtio network 
interface
  2) Stress the network interface with e.g. Windows HLK test suite or similar
  3) Repeatedly attach/detach the network adapter that's in use
  It usually takes more than ~4000 attach/detach cycles to trigger the bug.

  [Regression Potential]
  Regressions for this might arise from the fact that the fix changes RCU lock 
code. Since this patch has been upstream and in other series for a while, it's 
unlikely that it would regressions in RCU code specifically. Other code that 
makes use of the RCU locks (MMIO and some monitor events) will be thoroughly 
tested for any regressions with autokpkgtest and scripted Qemu runs.

To manage notifications about this bug go to:
https://bugs.launchpad.net/cloud-archive/+bug/1818880/+subscriptions



[Qemu-devel] [Bug 1818880] [NEW] Deadlock when detaching network interface

2019-03-06 Thread Heitor R. Alves de Siqueira
Public bug reported:

[Impact]
Qemu guests hang indefinitely

[Description]
When running a Qemu guest with VirtIO network interfaces, detaching an 
interface that's currently being used can result in a deadlock. The guest 
instance will hang and become unresponsive to commands, and the only option is 
to kill -9 the instance.
The reason for this is a dealock between a monitor and an RCU thread, which 
will fight over the BQL (qemu_global_mutex) and the critical RCU section locks. 
The monitor thread will acquire the BQL for detaching the network interface, 
and fire up a helper thread to deal with detaching the network adapter. That 
new thread needs to wait on the RCU thread to complete the deletion, but the 
RCU thread wants the BQL to commit its transactions.
This bug is already fixed upstream (73c6e4013b4c rcu: completely disable 
pthread_atfork callbacks as soon as possible) and included for other series 
(see below), so we don't need to backport it to Bionic onwards.

Upstream commit:
https://git.qemu.org/?p=qemu.git;a=commit;h=73c6e4013b4c

$ git describe --contains 73c6e4013b4c
v2.10.0-rc2~1^2~8

$ rmadison qemu
===> qemu | 1:2.5+dfsg-5ubuntu10.34 | xenial-updates/universe   | amd64, ...
 qemu | 1:2.11+dfsg-1ubuntu7| bionic/universe   | amd64, ...
 qemu | 1:2.12+dfsg-3ubuntu8| cosmic/universe   | amd64, ...
 qemu | 1:3.1+dfsg-2ubuntu2 | disco/universe| amd64, ...

[Test Case]
Being a racing condition, this is a tricky bug to reproduce consistently. We've 
had reports of users running into this with OpenStack deployments and Windows 
Server guests, and the scenario is usually like this:
1) Deploy a 16vCPU Windows Server 2012 R2 guest with a virtio network interface
2) Stress the network interface with e.g. Windows HLK test suite or similar
3) Repeatedly attach/detach the network adapter that's in use
It usually takes more than ~4000 attach/detach cycles to trigger the bug.

[Regression Potential]
Regressions for this might arise from the fact that the fix changes RCU lock 
code. Since this patch has been upstream and in other series for a while, it's 
unlikely that it would regressions in RCU code specifically. Other code that 
makes use of the RCU locks (MMIO and some monitor events) will be thoroughly 
tested for any regressions with autokpkgtest and scripted Qemu runs.

** Affects: qemu
 Importance: Undecided
 Status: Fix Released

** Affects: qemu (Ubuntu)
 Importance: Undecided
 Assignee: Heitor R. Alves de Siqueira (halves)
 Status: Fix Released

** Affects: qemu (Ubuntu Xenial)
 Importance: Undecided
 Assignee: Heitor R. Alves de Siqueira (halves)
 Status: Confirmed

** Affects: qemu (Ubuntu Bionic)
 Importance: Undecided
 Assignee: Heitor R. Alves de Siqueira (halves)
 Status: Fix Released

** Affects: qemu (Ubuntu Cosmic)
 Importance: Undecided
 Assignee: Heitor R. Alves de Siqueira (halves)
 Status: Fix Released

** Affects: qemu (Ubuntu Disco)
 Importance: Undecided
 Assignee: Heitor R. Alves de Siqueira (halves)
 Status: Fix Released


** Tags: sts

** Changed in: qemu
   Status: New => Confirmed

** Changed in: qemu
   Status: Confirmed => Fix Released

** Also affects: qemu (Ubuntu)
   Importance: Undecided
   Status: New

** Changed in: qemu (Ubuntu)
 Assignee: (unassigned) => Heitor R. Alves de Siqueira (halves)

** Changed in: qemu (Ubuntu)
   Status: New => Confirmed

** Also affects: qemu (Ubuntu Cosmic)
   Importance: Undecided
   Status: New

** Also affects: qemu (Ubuntu Xenial)
   Importance: Undecided
   Status: New

** Also affects: qemu (Ubuntu Disco)
   Importance: Undecided
 Assignee: Heitor R. Alves de Siqueira (halves)
   Status: Confirmed

** Also affects: qemu (Ubuntu Bionic)
   Importance: Undecided
   Status: New

** Changed in: qemu (Ubuntu Cosmic)
 Assignee: (unassigned) => Heitor R. Alves de Siqueira (halves)

** Changed in: qemu (Ubuntu Bionic)
 Assignee: (unassigned) => Heitor R. Alves de Siqueira (halves)

** Changed in: qemu (Ubuntu Xenial)
 Assignee: (unassigned) => Heitor R. Alves de Siqueira (halves)

** Changed in: qemu (Ubuntu Disco)
   Status: Confirmed => Fix Released

** Changed in: qemu (Ubuntu Cosmic)
   Status: New => Fix Released

** Changed in: qemu (Ubuntu Bionic)
   Status: New => Fix Released

** Changed in: qemu (Ubuntu Xenial)
   Status: New => Confirmed

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1818880

Title:
  Deadlock when detaching network interface

Status in QEMU:
  Fix Released
Status in qemu package in Ubuntu:
  Fix Released
Status in qemu source package in Xenial:
  Confirmed
Status in qemu source package in Bionic:
  Fix Released
Status in qemu source package in Cosmic:
  Fix Re

[Qemu-devel] [Bug 1787754] [NEW] qemu sparc -cpu help does not generate correct display

2018-08-18 Thread Donald R Laster Jr
Public bug reported:

The output for the "-cpu help" on the Sparc executables is not
generating accurate information.

Running

./qemu-sparc64 -cpu help

produces:

Sparc  Fujitsu Sparc64 IU 00040002 FPU  MMU  NWINS 4
Sparc Fujitsu Sparc64 III IU 00040003 FPU  MMU  NWINS 5
Sparc Fujitsu Sparc64 IV IU 00040004 FPU  MMU  NWINS 8
Sparc Fujitsu Sparc64 V IU 000400055100 FPU  MMU  NWINS 8
Sparc  TI UltraSparc I IU 001700104000 FPU  MMU  NWINS 8
Sparc TI UltraSparc II IU 001700112000 FPU  MMU  NWINS 8
Sparc TI UltraSparc IIi IU 001700129100 FPU  MMU  NWINS 8
Sparc TI UltraSparc IIe IU 001700131400 FPU  MMU  NWINS 8
Sparc Sun UltraSparc III IU 003e00143400 FPU  MMU  NWINS 8
Sparc Sun UltraSparc III Cu IU 003e00154100 FPU  MMU 0001 NWINS 
8
Sparc Sun UltraSparc IIIi IU 003e00163400 FPU  MMU  NWINS 8
Sparc Sun UltraSparc IV IU 003e00183100 FPU  MMU 0002 NWINS 8
Sparc Sun UltraSparc IV+ IU 003e00192200 FPU  MMU  NWINS 8 
+cmt
Sparc Sun UltraSparc IIIi+ IU 003e0022 FPU  MMU 0001 NWINS 8
Sparc Sun UltraSparc T1 IU 003e00230200 FPU  MMU 0003 NWINS 8 
+hypv +cmt +gl
Sparc Sun UltraSparc T2 IU 003e00240200 FPU  MMU 0003 NWINS 8 
+hypv +cmt +gl
Sparc NEC UltraSparc I IU 002200104000 FPU  MMU  NWINS 8
Default CPU feature flags (use '-' to remove): float swap mul div flush fsqrt 
fmul vis1 vis2 fsmuld
Available CPU feature flags (use '+' to add): float128 hypv cmt gl
Numerical features (use '=' to set): iu_version fpu_version mmu_version nwindows

The entries appear to supposed to be (partial list from source code):

TI-SuperSparc-II
TI-SuperSparc-II
TI-SuperSparc-II
TI-MicroSparc-I
TI-MicroSparc-I
TI-MicroSparc-I
Sun-UltraSparc-T1
TI-UltraSparc-IIi
Sun-UltraSparc-T1

The output is from qemu 2.12.0.

** Affects: qemu
 Importance: Undecided
 Status: New


** Tags: minor

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1787754

Title:
  qemu sparc -cpu help does not generate correct display

Status in QEMU:
  New

Bug description:
  The output for the "-cpu help" on the Sparc executables is not
  generating accurate information.

  Running

  ./qemu-sparc64 -cpu help

  produces:

  Sparc  Fujitsu Sparc64 IU 00040002 FPU  MMU  NWINS 4
  Sparc Fujitsu Sparc64 III IU 00040003 FPU  MMU  NWINS 
5
  Sparc Fujitsu Sparc64 IV IU 00040004 FPU  MMU  NWINS 8
  Sparc Fujitsu Sparc64 V IU 000400055100 FPU  MMU  NWINS 8
  Sparc  TI UltraSparc I IU 001700104000 FPU  MMU  NWINS 8
  Sparc TI UltraSparc II IU 001700112000 FPU  MMU  NWINS 8
  Sparc TI UltraSparc IIi IU 001700129100 FPU  MMU  NWINS 8
  Sparc TI UltraSparc IIe IU 001700131400 FPU  MMU  NWINS 8
  Sparc Sun UltraSparc III IU 003e00143400 FPU  MMU  NWINS 8
  Sparc Sun UltraSparc III Cu IU 003e00154100 FPU  MMU 0001 
NWINS 8
  Sparc Sun UltraSparc IIIi IU 003e00163400 FPU  MMU  NWINS 
8
  Sparc Sun UltraSparc IV IU 003e00183100 FPU  MMU 0002 NWINS 8
  Sparc Sun UltraSparc IV+ IU 003e00192200 FPU  MMU  NWINS 
8 +cmt
  Sparc Sun UltraSparc IIIi+ IU 003e0022 FPU  MMU 0001 
NWINS 8
  Sparc Sun UltraSparc T1 IU 003e00230200 FPU  MMU 0003 NWINS 8 
+hypv +cmt +gl
  Sparc Sun UltraSparc T2 IU 003e00240200 FPU  MMU 0003 NWINS 8 
+hypv +cmt +gl
  Sparc NEC UltraSparc I IU 002200104000 FPU  MMU  NWINS 8
  Default CPU feature flags (use '-' to remove): float swap mul div flush fsqrt 
fmul vis1 vis2 fsmuld
  Available CPU feature flags (use '+' to add): float128 hypv cmt gl
  Numerical features (use '=' to set): iu_version fpu_version mmu_version 
nwindows

  The entries appear to supposed to be (partial list from source code):

  TI-SuperSparc-II
  TI-SuperSparc-II
  TI-SuperSparc-II
  TI-MicroSparc-I
  TI-MicroSparc-I
  TI-MicroSparc-I
  Sun-UltraSparc-T1
  TI-UltraSparc-IIi
  Sun-UltraSparc-T1

  The output is from qemu 2.12.0.

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1787754/+subscriptions



[Qemu-devel] unknown keycodes

2018-04-21 Thread Mike R
(qemu) unknown keycodes `empty+aliases(qwerty)’, please report to 
qemu-devel@nongnu.org

reported.





signature.asc
Description: Message signed with OpenPGP using GPGMail


Re: [Qemu-devel] [PATCH] virtio_net: flush uncompleted TX on reset

2018-03-07 Thread Nageswara R Sastry
Greg Kurz <gr...@kaod.org> wrote on 08/03/2018 12:27:37 AM:

> From: Greg Kurz <gr...@kaod.org>
> To: qemu-devel@nongnu.org
> Cc: "Michael S. Tsirkin" <m...@redhat.com>, Jason Wang 
> <jasow...@redhat.com>, R Nageswara Sastry <nasas...@in.ibm.com>
> Date: 08/03/2018 12:27 AM
> Subject: [PATCH] virtio_net: flush uncompleted TX on reset
> 
> If the backend could not transmit a packet right away for some reason,
> the packet is queued for asynchronous sending. The corresponding vq
> element is tracked in the async_tx.elem field of the VirtIONetQueue,
> for later freeing when the transmission is complete.
> 
> If a reset happens before completion, virtio_net_tx_complete() will push
> async_tx.elem back to the guest anyway, and we end up with the inuse 
flag
> of the vq being equal to -1. The next call to virtqueue_pop() is then
> likely to fail with "Virtqueue size exceeded".
> 
> This can be reproduced easily by starting a guest without a net backend,
> doing a system reset when it is booted, and finally snapshotting it.
> 
> The appropriate fix is to ensure that such an asynchronous transmission
> cannot survive a device reset. So for all queues, we first try to send
> the packet again, and eventually we purge it if the backend still could
> not deliver it.
> 
> Reported-by: R. Nageswara Sastry <nasas...@in.ibm.com>

Tested-by: R. Nageswara Sastry <nasas...@in.ibm.com>

> Buglink: https://urldefense.proofpoint.com/v2/url?
> 
u=https-3A__github.com_open-2Dpower-2Dhost-2Dos_qemu_issues_37=DwICaQ=jf_iaSHvJObTbx-
> siA1ZOg=mxAxqGE8eb0FlPPFjDZkTNaoci-GdQbkJayE4r-
> 
wzYY=hPoY6b601IXJbUV2uh22jBrnYuByNQpwi1d7gvN4yZs=Ic3NN3mM_Nv1gAJ7dY22-
> ebnJsG7c0yNkbThX8Tu6xg=
> Signed-off-by: Greg Kurz <gr...@kaod.org>
> ---
>  hw/net/virtio-net.c |   11 +++
>  1 file changed, 11 insertions(+)
> 
> diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
> index 188744e17d57..eea3cdb2c700 100644
> --- a/hw/net/virtio-net.c
> +++ b/hw/net/virtio-net.c
> @@ -422,6 +422,7 @@ static RxFilterInfo 
> *virtio_net_query_rxfilter(NetClientState *nc)
>  static void virtio_net_reset(VirtIODevice *vdev)
>  {
>  VirtIONet *n = VIRTIO_NET(vdev);
> +int i;
> 
>  /* Reset back to compatibility mode */
>  n->promisc = 1;
> @@ -445,6 +446,16 @@ static void virtio_net_reset(VirtIODevice *vdev)
>  memcpy(>mac[0], >nic->conf->macaddr, sizeof(n->mac));
>  qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
>  memset(n->vlans, 0, MAX_VLAN >> 3);
> +
> +/* Flush any async TX */
> +for (i = 0;  i < n->max_queues; i++) {
> +NetClientState *nc = qemu_get_subqueue(n->nic, i);
> +
> +if (!qemu_net_queue_flush(nc->peer->incoming_queue)) {
> +qemu_net_queue_purge(nc->peer->incoming_queue, nc);
> +}
> +assert(!virtio_net_get_subqueue(nc)->async_tx.elem);
> +}
>  }
> 
>  static void peer_test_vnet_hdr(VirtIONet *n)
> 


With out patch:
(qemu) system_reset
(qemu) savevm 1
Virtqueue size exceeded
(qemu) loadvm 1
VQ 1 size 0x100 < last_avail_idx 0x0 - used_idx 0x1
Failed to load virtio-net:virtio
error while loading state for instance 0x0 of device
'pci@8002000:00.0/virtio-net'
Error -1 while loading VM state

With patch:
(qemu) system_reset
(qemu) savevm 1
(qemu) loadvm 1






Re: [Qemu-devel] [patch] linux-user/syscall.c: Fix missing break for host_to_target_cmsg

2018-02-16 Thread Nageswara R Sastry

On 2018-02-15 20:17, Laurent Vivier wrote:

Le 08/02/2018 à 10:56, Nageswara R Sastry a écrit :

On 2018-02-07 19:27, Laurent Vivier wrote:

Le 07/02/2018 à 10:49, no-re...@patchew.org a écrit :

Hi,

This series failed build test on s390x host. Please find the details
below.

...

  CC  aarch64_be-linux-user/linux-user/syscall.o
In file included from
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/qemu.h:16:0,
 from
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/syscall.c:118:
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/syscall.c: In
function ‘do_sendrecvmsg_locked’:
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/syscall_defs.h:308:61:
error: ‘tgt_len’ may be used uninitialized in this function
[-Werror=maybe-uninitialized]
 #define TARGET_CMSG_LEN(len) (sizeof(struct target_cmsghdr) + 
(len))
 
^
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/syscall.c:1797:13: 
note:

‘tgt_len’ was declared here
 int tgt_len, tgt_space;
 ^~~


it seems gcc disagrees with Coverity...

I think this should fixed like:

 diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index 74378947f0..d7fbe334eb 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -1824,8 +1824,10 @@ static inline abi_long 
host_to_target_cmsg(struct

target_msghdr *target_msgh,
 tgt_len = sizeof(struct 
target_timeval);

 break;
 default:
+    tgt_len = len;
 break;


In my view this will result in assigning a wrong value to 
‘tgt_len’ at

this ‘switch-case’ condition.
Instead looking at the option of initializing ‘tgt_len' to ‘0’.


According to the comment above the switch():

/* Payload types which need a different size of payload on
 * the target must adjust tgt_len here.
 */

So "tgt_len" must be "len" by default, except if it needs to be 
adjusted

(currently only for SO_TIMESTAMP), so I don't understand why it should
be set to "0".

Thanks,
Laurent


 1814 switch (cmsg->cmsg_level) {
 1815 case SOL_SOCKET:
 1816 switch (cmsg->cmsg_type) {
 1817 case SO_TIMESTAMP:
 1818 tgt_len = sizeof(struct target_timeval);
 1819 break;
 1820 default:
 1821 break;
 1822 }
 1823 default:
 1824 tgt_len = len;
 1825 break;
 1826 }


If setting tgt_len = len at 1820 then it get set to 'case SOL_SOCKET{ 
switch (cmsg_type's default) }' This place am not sure assingn tgt_len 
with len. To eliminate the gcc uninitialized error thought of 
initializing with '0'.



--
Regards,
R.Nageswara Sastry



Re: [Qemu-devel] [patch] linux-user/syscall.c: Fix missing break for host_to_target_cmsg

2018-02-08 Thread Nageswara R Sastry

On 2018-02-07 19:27, Laurent Vivier wrote:

Le 07/02/2018 à 10:49, no-re...@patchew.org a écrit :

Hi,

This series failed build test on s390x host. Please find the details 
below.

...

  CC  aarch64_be-linux-user/linux-user/syscall.o
In file included from 
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/qemu.h:16:0,
 from 
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/syscall.c:118:
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/syscall.c: In 
function ‘do_sendrecvmsg_locked’:
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/syscall_defs.h:308:61: 
error: ‘tgt_len’ may be used uninitialized in this function 
[-Werror=maybe-uninitialized]

 #define TARGET_CMSG_LEN(len) (sizeof(struct target_cmsghdr) + (len))
 ^
/var/tmp/patchew-tester-tmp-ewjgn083/src/linux-user/syscall.c:1797:13: 
note: ‘tgt_len’ was declared here

 int tgt_len, tgt_space;
 ^~~


it seems gcc disagrees with Coverity...

I think this should fixed like:

 diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index 74378947f0..d7fbe334eb 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -1824,8 +1824,10 @@ static inline abi_long 
host_to_target_cmsg(struct

target_msghdr *target_msgh,
 tgt_len = sizeof(struct target_timeval);
 break;
 default:
+tgt_len = len;
 break;


In my view this will result in assigning a wrong value to ‘tgt_len’ 
at this ‘switch-case’ condition.

Instead looking at the option of initializing ‘tgt_len' to ‘0’.

@@ -1789,7 +1789,7 @@
void *target_data = TARGET_CMSG_DATA(target_cmsg);

int len = cmsg->cmsg_len - sizeof(struct cmsghdr);
-int tgt_len, tgt_space;
+int tgt_len = 0, tgt_space;

/* We never copy a half-header but may copy half-data;
 * this is Linux's behaviour in put_cmsg(). Note that
@@ -1821,6 +1821,7 @@
default:
break;
}
+break;
default:
tgt_len = len;
break;

Re-sending this mail because earlier one not reached the mailing list. 
Please accept my apologies if it is a duplicate.



 }
+break;
 default:
 tgt_len = len;
 break;

Peter?

Thanks,
Laurent


--
Regards,
R.Nageswara Sastry




[Qemu-devel] [patch] linux-user/syscall.c: Fix missing break for host_to_target_cmsg

2018-02-07 Thread Nageswara R Sastry
Detected by by Coverity (CID 1385425)
with out this break statement the assigned value of 'tgt_len' at
line 1824 will be replaced by value of 'tgt_len' at line 1830.

Signed-off-by: Nageswara R Sastry <rnsas...@linux.vnet.ibm.com>
---
 linux-user/syscall.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index 74378947f0..3ecd533880 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -1826,6 +1826,7 @@ static inline abi_long host_to_target_cmsg(struct 
target_msghdr *target_msgh,
 default:
 break;
 }
+break;
 default:
 tgt_len = len;
 break;
-- 
2.14.3 (Apple Git-98)




[Qemu-devel] [patch] linux-user/syscall.c: Fix missing break for host_to_target_cmsg

2018-02-07 Thread Nageswara R Sastry
Detected by by Coverity (CID 1385425)
with out this break statement the assigned value of 'tgt_len' at
line 1824 will be replaced by value of 'tgt_len' at line 1830.

Signed-off-by: Nageswara R Sastry <rnsas...@linux.vnet.ibm.com>
---
 linux-user/syscall.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index 74378947f0..3ecd533880 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -1826,6 +1826,7 @@ static inline abi_long host_to_target_cmsg(struct 
target_msghdr *target_msgh,
 default:
 break;
 }
+break;
 default:
 tgt_len = len;
 break;
-- 
2.14.3 (Apple Git-98)




Re: [Qemu-devel] [RFC v1] block/NVMe: introduce a new vhost NVMe host device to QEMU

2018-01-29 Thread Harris, James R


On 1/29/18, 8:29 AM, "Stefan Hajnoczi"  wrote:



Each new feature has a cost in terms of maintainance, testing,
documentation, and support.  Users need to be educated about the role of
each available storage controller and how to choose between them.

I'm not sure why QEMU should go in this direction since it makes the
landscape more complex and harder to support.  You've said the
performance is comparable to vhost-user-blk.  So what does NVMe offer
that makes this worthwhile?

A cool NVMe feature would be the ability to pass through invididual
queues to different guests without SR-IOV.  In other words, bind a queue
to namespace subset so that multiple guests can be isolated from each
other.  That way the data path would not require vmexits.  The control
path and device initialization would still be emulated by QEMU so the
hardware does not need to provide the full resources and state needed
for SR-IOV.  I looked into this but came to the conclusion that it would
require changes to the NVMe specification because the namespace is a
per-command field.

Correct – any command from any queue can access any namespace on the controller.

Another reason this is not possible is that most (if not all?) controllers have 
CAP.DSTRD (Doorbell Stride) = 0, meaning doorbell registers for all queues fall 
within the same page.





Re: [Qemu-devel] [PATCH 2/2] virtio: fix IO request length in virtio SCSI/block

2017-12-18 Thread Harris, James R

> On Dec 18, 2017, at 6:38 AM, Stefan Hajnoczi <stefa...@redhat.com> wrote:
> 
> On Fri, Dec 15, 2017 at 06:02:50PM +0300, Denis V. Lunev wrote:
>> Linux guests submit IO requests no longer than PAGE_SIZE * max_seg
>> field reported by SCSI controler. Thus typical sequential read with
>> 1 MB size results in the following pattern of the IO from the guest:
>>  8,16   115754 2.766095122  2071  D   R 2095104 + 1008 [dd]
>>  8,16   115755 2.766108785  2071  D   R 2096112 + 1008 [dd]
>>  8,16   115756 2.766113486  2071  D   R 2097120 + 32 [dd]
>>  8,16   115757 2.767668961 0  C   R 2095104 + 1008 [0]
>>  8,16   115758 2.768534315 0  C   R 2096112 + 1008 [0]
>>  8,16   115759 2.768539782 0  C   R 2097120 + 32 [0]
>> The IO was generated by
>>  dd if=/dev/sda of=/dev/null bs=1024 iflag=direct
>> 
>> This effectively means that on rotational disks we will observe 3 IOPS
>> for each 2 MBs processed. This definitely negatively affects both
>> guest and host IO performance.
>> 
>> The cure is relatively simple - we should report lengthy scatter-gather
>> ability of the SCSI controller. Fortunately the situation here is very
>> good. VirtIO transport layer can accomodate 1024 items in one request
>> while we are using only 128. This situation is present since almost
>> very beginning. 2 items are dedicated for request metadata thus we
>> should publish VIRTQUEUE_MAX_SIZE - 2 as max_seg.
>> 
>> The following pattern is observed after the patch:
>>  8,16   1 9921 2.662721340  2063  D   R 2095104 + 1024 [dd]
>>  8,16   1 9922 2.662737585  2063  D   R 2096128 + 1024 [dd]
>>  8,16   1 9923 2.665188167 0  C   R 2095104 + 1024 [0]
>>  8,16   1 9924 2.665198777 0  C   R 2096128 + 1024 [0]
>> which is much better.
>> 
>> The dark side of this patch is that we are tweaking guest visible
>> parameter, though this should be relatively safe as above transport
>> layer support is present in QEMU/host Linux for a very long time.
>> The patch adds configurable property for VirtIO SCSI with a new default
>> and hardcode option for VirtBlock which does not provide good
>> configurable framework.
>> 
>> Signed-off-by: Denis V. Lunev <d...@openvz.org>
>> CC: "Michael S. Tsirkin" <m...@redhat.com>
>> CC: Stefan Hajnoczi <stefa...@redhat.com>
>> CC: Kevin Wolf <kw...@redhat.com>
>> CC: Max Reitz <mre...@redhat.com>
>> CC: Paolo Bonzini <pbonz...@redhat.com>
>> CC: Richard Henderson <r...@twiddle.net>
>> CC: Eduardo Habkost <ehabk...@redhat.com>
>> ---
>> include/hw/compat.h | 17 +
>> include/hw/virtio/virtio-blk.h  |  1 +
>> include/hw/virtio/virtio-scsi.h |  1 +
>> hw/block/virtio-blk.c   |  4 +++-
>> hw/scsi/vhost-scsi.c|  2 ++
>> hw/scsi/vhost-user-scsi.c   |  2 ++
>> hw/scsi/virtio-scsi.c   |  4 +++-
>> 7 files changed, 29 insertions(+), 2 deletions(-)
>> 
>> diff --git a/include/hw/compat.h b/include/hw/compat.h
>> index 026fee9..b9be5d7 100644
>> --- a/include/hw/compat.h
>> +++ b/include/hw/compat.h
>> @@ -2,6 +2,23 @@
>> #define HW_COMPAT_H
>> 
>> #define HW_COMPAT_2_11 \
>> +{\
>> +.driver   = "virtio-blk-device",\
>> +.property = "max_segments",\
>> +.value= "126",\
>> +},{\
>> +.driver   = "vhost-scsi",\
>> +.property = "max_segments",\
>> +.value= "126",\
>> +},{\
>> +.driver   = "vhost-user-scsi",\
>> +.property = "max_segments",\
>> +.value= "126",\
> 
> Existing vhost-user-scsi slave programs might not expect up to 1022
> segments.  Hopefully we can get away with this change since there are
> relatively few vhost-user-scsi slave programs.
> 
> CCed Felipe (Nutanix) and Jim (SPDK) in case they have comments.

SPDK vhost-user targets only expect max 128 segments.  They also pre-allocate 
I/O task structures when QEMU connects to the vhost-user device.

Supporting up to 1022 segments would result in significantly higher memory 
usage, reduction in I/O queue depth processed by the vhost-user target, or 
having to dynamically allocate I/O task structures - none of which are ideal.

What if this was just bumped from 126 to 128?  I guess I’m trying to understand 
the level of guest and host I/O performance that is gained with this patch.  
One I/O per 512KB vs. one I/O per 4MB - we are still only talking about a few 
hundred IO/s difference.

-Jim




[Qemu-devel] [Bug 1719870] Re: Converting 100G VHDX fixed image to QCOW2 fails

2017-10-11 Thread R Naveen Kumar
Bug is reproducible with setting VHDX Fixed Logicalsectorsize to 4096bytes
>10G image created reflects as 1.2G virtual size in Qemu-img
PS F:\> new-vhd -path test.vhdx -BlockSizeBytes 134217728 -SizeBytes 
10737418240 -Fixed -LogicalSectorSizeBytes 4096
ComputerName: 
Path: F:\test.vhdx
VhdFormat   : VHDX
VhdType : Fixed
FileSize: 10741612544
Size: 10737418240
MinimumSize :
LogicalSectorSize   : 4096
PhysicalSectorSize  : 4096
BlockSize   : 0
ParentPath  :
DiskIdentifier  : dfa84293-86f2-4ddf-aaff-14c04dae5df9
FragmentationPercentage : 0
Alignment   : 1
Attached: False
DiskNumber  :
Key :
IsDeleted   : False
Number  :
PS F:\> C:\temp\qemu-img\qemu-img.exe info .\test.vhdx
image: .\test.vhdx
file format: vhdx
virtual size: 1.2G (1342177280 bytes)
disk size: 10G
cluster_size: 134217728

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1719870

Title:
  Converting 100G VHDX fixed image to QCOW2 fails

Status in QEMU:
  New

Bug description:
  Virtual Size recognized incorrectly for VHDX fixed disk and conversion
  fails with error Expression: !qiov || bytes == qiov->size


  PS > & 'C:\Program Files\qemu\qemu-img.exe' --version
  qemu-img version 2.10.0 (v2.10.0-11669-g579e69bd5b-dirty)
  Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1719870/+subscriptions



[Qemu-devel] [Bug 1719870] Re: Converting 100G VHDX fixed image to QCOW2 fails

2017-09-27 Thread R Naveen Kumar
PS > Get-VHD .\VM.vhdx
ComputerName: Server1
Path: \VM.vhdx
VhdFormat   : VHDX
VhdType : Fixed
FileSize: 107378376704
Size: 107374182400
MinimumSize : 107374182400
LogicalSectorSize   : 4096
PhysicalSectorSize  : 4096
BlockSize   : 0
ParentPath  :
DiskIdentifier  : 53fd4aa7-562e-4bed-bc1c-2db71222e07e
FragmentationPercentage : 0
Alignment   : 1
Attached: False
DiskNumber  :
Key :
IsDeleted   : False
Number  :
PS > & 'C:\Program Files\qemu\qemu-img.exe' convert -O qcow2 .\VM.vhdx 
.\VM.qcow2
Assertion failed!
Program: C:\Program Files\qemu\qemu-img.exe
File: /home/stefan/src/qemu/repo.or.cz/qemu/ar7/block/io.c, Line 1034
Expression: !qiov || bytes == qiov->size
PS > & 'C:\Program Files\qemu\qemu-img.exe' info .\VM.qcow2
image: .\VM.qcow2
file format: qcow2
virtual size: 13G (13421772800 bytes)
disk size: 1.4G
cluster_size: 65536
Format specific information:
compat: 1.1
lazy refcounts: false
refcount bits: 16
corrupt: false

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1719870

Title:
  Converting 100G VHDX fixed image to QCOW2 fails

Status in QEMU:
  New

Bug description:
  Virtual Size recognized incorrectly for VHDX fixed disk and conversion
  fails with error Expression: !qiov || bytes == qiov->size


  PS > & 'C:\Program Files\qemu\qemu-img.exe' --version
  qemu-img version 2.10.0 (v2.10.0-11669-g579e69bd5b-dirty)
  Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1719870/+subscriptions



[Qemu-devel] [Bug 1719870] Re: Converting 100G VHDX fixed image to QCOW2 fails

2017-09-27 Thread R Naveen Kumar
Command capture,

PS > & 'C:\Program Files\qemu\qemu-img.exe' --version
qemu-img version 2.10.0 (v2.10.0-11669-g579e69bd5b-dirty)
Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1719870

Title:
  Converting 100G VHDX fixed image to QCOW2 fails

Status in QEMU:
  New

Bug description:
  Virtual Size recognized incorrectly for VHDX fixed disk and conversion
  fails with error Expression: !qiov || bytes == qiov->size


  PS > & 'C:\Program Files\qemu\qemu-img.exe' --version
  qemu-img version 2.10.0 (v2.10.0-11669-g579e69bd5b-dirty)
  Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1719870/+subscriptions



[Qemu-devel] [Bug 1719870] [NEW] Converting 100G VHDX fixed image to QCOW2 fails

2017-09-27 Thread R Naveen Kumar
Public bug reported:

Virtual Size recognized incorrectly for VHDX fixed disk and conversion
fails with error Expression: !qiov || bytes == qiov->size


PS > & 'C:\Program Files\qemu\qemu-img.exe' --version
qemu-img version 2.10.0 (v2.10.0-11669-g579e69bd5b-dirty)
Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers

** Affects: qemu
 Importance: Undecided
 Status: New

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1719870

Title:
  Converting 100G VHDX fixed image to QCOW2 fails

Status in QEMU:
  New

Bug description:
  Virtual Size recognized incorrectly for VHDX fixed disk and conversion
  fails with error Expression: !qiov || bytes == qiov->size


  PS > & 'C:\Program Files\qemu\qemu-img.exe' --version
  qemu-img version 2.10.0 (v2.10.0-11669-g579e69bd5b-dirty)
  Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1719870/+subscriptions



Re: [Qemu-devel] [PATCH v5 0/4] Introduce vhost-user-scsi and sample application

2017-07-06 Thread Harris, James R

> On Jul 6, 2017, at 10:11 AM, Marc-André Lureau <marcandre.lur...@redhat.com> 
> wrote:
> 
> Hi
> 
> - Original Message -
>> 
>>> On Jul 6, 2017, at 9:56 AM, Paolo Bonzini <pbonz...@redhat.com> wrote:
>>> 
>>> 
>>> 
>>> On 06/07/2017 18:54, Harris, James R wrote:
>>>> Hi Michael,
>>>> 
>>>> Yes - we (SPDK team at Intel) have this on our TODO list, in addition
>>>> to a polled mode virtio-scsi driver (akin to the DPDK virtio-net driver
>>>> that
>>>> can be used in the guest).
>>> 
>>> Can you also look at virtio-blk poll-mode drivers and vhost-user-blk
>>> (vhost-user-blk would be really easy to write, the sample server being
>>> much more work than the QEMU code)?
>>> 
>>> Paolo
>> 
>> Hi Paolo,
>> 
>> vhost-user-blk:
>> 
>> We have an initial implementation in our SPDK QEMU repo.  Changpeng Liu
>> (added)
>> will be sending this to the QEMU mailing list shortly.  If you are interested
>> in viewing it
>> before it hits the mailing list, you can take a look at the patch here:
>> 
>> https://github.com/spdk/qemu/commit/db2fcb0f5002c3d195e756eab07bd976ccf6e7c8
> 
> Great! I also just started looking into this, I'll take a look at your 
> implementation. Do you also have a "server"/slave side, or plan to share one?

We will add a sample server that can be upstreamed into QEMU, and will also 
provide a morefully-functional vhost-user-blk
target with SPDK.

A lot of the SPDK vhost-user-blk work has been pushed out to our master branch 
already, but there’s still some WIP.

https://github.com/spdk/spdk/tree/master/lib/vhost

-Jim



Re: [Qemu-devel] [PATCH v5 0/4] Introduce vhost-user-scsi and sample application

2017-07-06 Thread Harris, James R

> On Jul 6, 2017, at 10:06 AM, Paolo Bonzini <pbonz...@redhat.com> wrote:
> 
> 
> 
> On 06/07/2017 19:03, Harris, James R wrote:
>> 
>> vhost-user-blk:
>> 
>> We have an initial implementation in our SPDK QEMU repo.  Changpeng Liu 
>> (added)
>> will be sending this to the QEMU mailing list shortly.  If you are 
>> interested in viewing it
>> before it hits the mailing list, you can take a look at the patch here:
>> 
>> https://github.com/spdk/qemu/commit/db2fcb0f5002c3d195e756eab07bd976ccf6e7c8
> 
> Nice.  But please provide some sample server too, it's a prerequisite
> for committing it into QEMU.  Even a simple file backend is enough.
> 
> You can use the contrib/libvhost-user infrastructure already used by the
> vhost-user-scsi sample.

Thanks Paolo.  We will add the sample server before posting to the mailing list.

-Jim




Re: [Qemu-devel] [PATCH v5 0/4] Introduce vhost-user-scsi and sample application

2017-07-06 Thread Harris, James R

> On Jul 6, 2017, at 9:56 AM, Paolo Bonzini <pbonz...@redhat.com> wrote:
> 
> 
> 
> On 06/07/2017 18:54, Harris, James R wrote:
>> Hi Michael,
>> 
>> Yes - we (SPDK team at Intel) have this on our TODO list, in addition
>> to a polled mode virtio-scsi driver (akin to the DPDK virtio-net driver that
>> can be used in the guest).
> 
> Can you also look at virtio-blk poll-mode drivers and vhost-user-blk
> (vhost-user-blk would be really easy to write, the sample server being
> much more work than the QEMU code)?
> 
> Paolo

Hi Paolo,

vhost-user-blk:

We have an initial implementation in our SPDK QEMU repo.  Changpeng Liu (added)
will be sending this to the QEMU mailing list shortly.  If you are interested 
in viewing it
before it hits the mailing list, you can take a look at the patch here:

https://github.com/spdk/qemu/commit/db2fcb0f5002c3d195e756eab07bd976ccf6e7c8

virtio-blk poll-mode driver:

Yes - this is part of our plan as well - once we have virtio-scsi in place, 
adding virtio-blk
will be relatively straightforward.

Thanks,

-Jim




Re: [Qemu-devel] [PATCH v5 0/4] Introduce vhost-user-scsi and sample application

2017-07-06 Thread Harris, James R

> On Jul 6, 2017, at 6:41 AM, Michael S. Tsirkin  wrote:
> 
> On Fri, Apr 21, 2017 at 02:53:33PM +, Felipe Franciosi wrote:
>> 
>>> On 2 Mar 2017, at 21:47, Michael S. Tsirkin  wrote:
>>> 
>>> On Thu, Mar 02, 2017 at 10:25:49AM -0800, Felipe Franciosi wrote:
 Based on various discussions on the 2016 KVM Forum, I'm sending over a
 vhost-user-scsi implementation for your consideration.
 
 This patchset introduces a new vhost-user SCSI device. While heavily
 based on vhost-scsi, it is implemented using vhost's userspace
 counterpart. The device has been coded and tested to work with live
 migration.
 
 As part of this work, a new vhost-scsi-common device was created and
 the existing vhost-scsi device moved underneath it. The new
 vhost-user-scsi device is also placed underneath it.
 
 A sample application based on the newly introduced libvhost-user is
 also included. It makes use of libiscsi for simplicity.
 
 For convenience, I'm maintaining an up-to-date version of these
 patches (including some necessary fixes for libvhost-user) on:
 
 https://github.com/franciozzy/qemu/tree/vus-upstream-v5
 
 See the individual patches for build and use instructions.
 
 Signed-off-by: Felipe Franciosi 
>>> 
>>> BTW, pls remember to ping after QEMU release.
>> 
>> Ping! :-)
>> 
>> F.
> 
> Now that code is upstream, I think the next step would be
> to add vIOMMU support based on what we are doing for virtio-net,
> so people can use SPDK within guest with security. Do you think
> you will have the time to look at it?

Hi Michael,

Yes - we (SPDK team at Intel) have this on our TODO list, in addition
to a polled mode virtio-scsi driver (akin to the DPDK virtio-net driver that
can be used in the guest).

-Jim

> 
> 
>>> 
 v4 -> v5:
 - Add a patch to fix a build issue with libvhost-user.
 
 v3 -> v4:
 - Drop configure switches and build vus by default.
 - Add sample application to .gitignore.
 - Removed a whitespace error.
 - Rebase on master.
 
 v2 -> v3:
 - Rebase after vhost notifier fixes by Paolo.
 - Exposed F_HOTPLUG and F_CHANGE on vhost-user-scsi.
 
 v1 -> v2:
 - Introduce new vhost-scsi-common device type.
 - Move vhost-scsi device underneath vhost-scsi-common.
 - Move sample application from tests/ to contrib/.
 - Make sample application use the glib event loop.
 - Minor fixes.
 
 Felipe Franciosi (4):
 libvhost-user: replace vasprintf() to fix build
 vhost-scsi: create a vhost-scsi-common abstraction
 vus: Introduce vhost-user-scsi host device
 vus: Introduce a vhost-user-scsi sample application
 
 .gitignore|   1 +
 Makefile  |   3 +
 Makefile.objs |   4 +
 configure |   1 +
 contrib/libvhost-user/libvhost-user.c |   2 +-
 contrib/vhost-user-scsi/Makefile.objs |   1 +
 contrib/vhost-user-scsi/vhost-user-scsi.c | 886 
 ++
 default-configs/pci.mak   |   1 +
 hw/scsi/Makefile.objs |   3 +-
 hw/scsi/vhost-scsi-common.c   | 143 +
 hw/scsi/vhost-scsi.c  | 194 ++-
 hw/scsi/vhost-user-scsi.c | 215 
 hw/virtio/virtio-pci.c|  54 ++
 hw/virtio/virtio-pci.h|  11 +
 include/hw/virtio/vhost-scsi-common.h |  48 ++
 include/hw/virtio/vhost-scsi.h|  11 +-
 include/hw/virtio/vhost-user-scsi.h   |  35 ++
 include/hw/virtio/virtio-scsi.h   |   5 +
 18 files changed, 1469 insertions(+), 149 deletions(-)
 create mode 100644 contrib/vhost-user-scsi/Makefile.objs
 create mode 100644 contrib/vhost-user-scsi/vhost-user-scsi.c
 create mode 100644 hw/scsi/vhost-scsi-common.c
 create mode 100644 hw/scsi/vhost-user-scsi.c
 create mode 100644 include/hw/virtio/vhost-scsi-common.h
 create mode 100644 include/hw/virtio/vhost-user-scsi.h
 
 -- 
 1.9.4




Re: [Qemu-devel] What is the best commit for record-replay?

2017-05-02 Thread Igor R
 I'm trying to use the deterministic record/replay feature, and I would
 like to know which commit I should take to get it work.
 In RC0 it seems to be broken. I tried pre-MTTCG commit 2421f381dc, as
>>
>>> Can you retry with the latest rc? There were some fixes regarding rr since 
>>> rc0.
>>
>>
>> I've taken 2.9 release, and RR does not seem to work there.
>> I recorded the boot process of x86 Fedora-21 linux and the replay got
>> stuck almost immediately.
>
> What's your command line?
>
> Does it get stuck at the same place each time?
>
> Can you boot fine with icount but without record/replay?

Here is the exact scenario:
- Get 2.9 from git, configure it as follows: "./configure
--target-list=i386-softmmu --enable-sdl" and  make.
- Download 
https://people.debian.org/~aurel32/qemu/i386/debian_squeeze_i386_standard.qcow2
- Run qemu with the following command line, until login prompt:
-icount shift=7,rr=record,rrfile=replay.bin -drive
file=debian_squeeze_i386_standard.qcow2,if=none,id=img-direct -drive
driver=blkreplay,if=none,image=img-direct,id=img-blkreplay -device
ide-hd,drive=img-blkreplay -monitor stdio
- Replay: -icount shift=7,rr=replay,rrfile=replay.bin -drive
file=debian_squeeze_i386_standard.qcow2,if=none,id=img-direct -drive
driver=blkreplay,if=none,image=img-direct,id=img-blkreplay -device
ide-hd,drive=img-blkreplay -monitor stdio

Every time I attempt to replay, QEMU gets stuck at the same EIP, at a
very early stage.


> Can you boot fine with icount but without record/replay?

Yes. I can also enable icount and recording - it also boots fine. The
problem with the replay.



Re: [Qemu-devel] What is the best commit for record-replay?

2017-04-25 Thread Igor R
>> Hi,
>>
>> I'm trying to use the deterministic record/replay feature, and I would
>> like to know which commit I should take to get it work.
>> In RC0 it seems to be broken. I tried pre-MTTCG commit 2421f381dc, as

> Can you retry with the latest rc? There were some fixes regarding rr since 
> rc0.


I've taken 2.9 release, and RR does not seem to work there.
I recorded the boot process of x86 Fedora-21 linux and the replay got
stuck almost immediately.



[Qemu-devel] What is the best commit for record-replay?

2017-03-23 Thread Igor R
Hi,

I'm trying to use the deterministic record/replay feature, and I would
like to know which commit I should take to get it work.
In RC0 it seems to be broken. I tried pre-MTTCG commit 2421f381dc, as
mentioned here:
http://lists.nongnu.org/archive/html/qemu-devel/2017-03/msg02657.html
with this patch:
http://lists.nongnu.org/archive/html/qemu-devel/2017-02/msg01316.html
My command line is:
./qemu-system-i386 -net none  -icount
shift=7,rr=replay,rrfile=replay.bin -drive
file=MyFedora386.qcow,if=none,id=img-direct -drive
driver=blkreplay,if=none,image=img-direct,id=img-blkreplay -device
ide-hd,drive=img-blkreplay -monitor stdio

The replay advances for a while, but gets stuck in about 10-15 sec,
and it looks like it encounters deadlock trying to acquire rcu lock.

Is there a working commit of RR?


Thanks.



[Qemu-devel] [Bug 1655708] Re: target/ppc/int_helper.c:2806: strange expression ?

2017-01-11 Thread Jose R. Ziviani
** Changed in: qemu
 Assignee: (unassigned) => Jose R. Ziviani (jrziviani)

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1655708

Title:
  target/ppc/int_helper.c:2806: strange expression ?

Status in QEMU:
  New

Bug description:
  target/ppc/int_helper.c:2806:25: warning: ‘*’ in boolean context,
  suggest ‘&&’ instead [-Wint-in-bool-context]

  Source code is

 zone_digit = (i * 2) ? b->u8[BCD_DIG_BYTE(i * 2)] >> 4 :
  zone_lead;

  Which I read as

 zone_digit = (i * 2) ? (b->u8[BCD_DIG_BYTE(i * 2)] >> 4) :
  zone_lead;

  so I think the compiler warning is for the i * 2 lhs of the ?.

  I am not sure what to suggest as a bugfix.

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1655708/+subscriptions



[Qemu-devel] [PATCH v2] target-mips: Implement Loongson 3A processor.

2016-11-30 Thread r
From: Heiher <wa...@lemote.com>

Signed-off-by: Heiher <wa...@lemote.com>
---
 target-mips/helper.c |   4 +-
 target-mips/mips-defs.h  |   5 +
 target-mips/translate.c  | 463 +--
 target-mips/translate_init.c |  24 +++
 4 files changed, 481 insertions(+), 15 deletions(-)

diff --git a/target-mips/helper.c b/target-mips/helper.c
index c864b15..a70b20f 100644
--- a/target-mips/helper.c
+++ b/target-mips/helper.c
@@ -715,7 +715,7 @@ void mips_cpu_do_interrupt(CPUState *cs)
 int KX = (env->CP0_Status & (1 << CP0St_KX)) != 0;
 
 if (((R == 0 && UX) || (R == 1 && SX) || (R == 3 && KX)) &&
-(!(env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F
+(!(env->insn_flags & INSN_LOONGSON)))
 offset = 0x080;
 else
 #endif
@@ -734,7 +734,7 @@ void mips_cpu_do_interrupt(CPUState *cs)
 int KX = (env->CP0_Status & (1 << CP0St_KX)) != 0;
 
 if (((R == 0 && UX) || (R == 1 && SX) || (R == 3 && KX)) &&
-(!(env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F
+(!(env->insn_flags & INSN_LOONGSON)))
 offset = 0x080;
 else
 #endif
diff --git a/target-mips/mips-defs.h b/target-mips/mips-defs.h
index 047554e..02dfb22 100644
--- a/target-mips/mips-defs.h
+++ b/target-mips/mips-defs.h
@@ -50,7 +50,11 @@
 /* Chip specific instructions. */
 #defineINSN_LOONGSON2E  0x2000
 #defineINSN_LOONGSON2F  0x4000
+#define INSN_LOONGSON3A  0x1000
 #defineINSN_VR54XX 0x8000
+#define INSN_LOONGSON2   (INSN_LOONGSON2E | INSN_LOONGSON2F)
+#define INSN_LOONGSON3   (INSN_LOONGSON3A)
+#define INSN_LOONGSON(INSN_LOONGSON2 | INSN_LOONGSON3)
 
 /* MIPS CPU defines. */
 #defineCPU_MIPS1   (ISA_MIPS1)
@@ -70,6 +74,7 @@
 /* MIPS Technologies "Release 2" */
 #defineCPU_MIPS32R2(CPU_MIPS32 | ISA_MIPS32R2)
 #defineCPU_MIPS64R2(CPU_MIPS64 | CPU_MIPS32R2 | 
ISA_MIPS64R2)
+#define CPU_LOONGSON3A  (CPU_MIPS64R2 | INSN_LOONGSON3A)
 
 /* MIPS Technologies "Release 3" */
 #define CPU_MIPS32R3 (CPU_MIPS32R2 | ISA_MIPS32R3)
diff --git a/target-mips/translate.c b/target-mips/translate.c
index 57b824f..67378b4 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -2123,7 +2123,7 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
 {
 TCGv t0, t1, t2;
 
-if (rt == 0 && ctx->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
+if (rt == 0 && ctx->insn_flags & INSN_LOONGSON) {
 /* Loongson CPU uses a load to zero register for prefetch.
We emulate it as a NOP. On other CPU we must perform the
actual memory access. */
@@ -2438,6 +2438,402 @@ static void gen_cop1_ldst(DisasContext *ctx, uint32_t 
op, int rt,
 }
 }
 
+static inline void gen_cop2_gslwlrc1(DisasContext *ctx, TCGv base,
+  int rt, int rs, int offset, int left)
+{
+TCGv_i32 t0, t1, t2;
+
+gen_base_offset_addr(ctx, base, rs, offset);
+t1 = tcg_temp_new_i32();
+/* Do a byte access to possibly trigger a page
+   fault with the unaligned address.  */
+tcg_gen_qemu_ld_i32(t1, base, ctx->mem_idx, MO_UB);
+tcg_gen_trunc_tl_i32(t1, base);
+tcg_gen_andi_i32(t1, t1, 3);
+if (left) {
+tcg_gen_xori_i32(t1, t1, 3);
+}
+tcg_gen_shli_i32(t1, t1, 3);
+tcg_gen_andi_tl(base, base, ~3);
+t0 = tcg_temp_new_i32();
+tcg_gen_qemu_ld_i32(t0, base, ctx->mem_idx, MO_TEUL);
+if (left) {
+tcg_gen_shl_i32(t0, t0, t1);
+t2 = tcg_const_i32(-1);
+} else {
+tcg_gen_shr_i32(t0, t0, t1);
+tcg_gen_xori_i32(t1, t1, 31);
+t2 = tcg_const_i32(0xfffeu);
+}
+tcg_gen_shl_i32(t2, t2, t1);
+gen_load_fpr32(ctx, t1, rt);
+if (left) {
+tcg_gen_andc_i32(t1, t1, t2);
+} else {
+tcg_gen_and_i32(t1, t1, t2);
+}
+tcg_temp_free_i32(t2);
+tcg_gen_or_i32(t0, t0, t1);
+tcg_temp_free_i32(t1);
+gen_store_fpr32(ctx, t0, rt);
+tcg_temp_free_i32(t0);
+}
+
+static inline void gen_cop2_gsldlrc1(DisasContext *ctx, TCGv base,
+  int rt, int rs, int offset, int left)
+{
+TCGv_i64 t0, t1, t2;
+
+gen_base_offset_addr(ctx, base, rs, offset);
+t1 = tcg_temp_new_i64();
+/* Do a byte access to possibly trigger a page
+   fault with the unaligned address.  */
+tcg_gen_qemu_ld_i64(t1, base, ctx->mem_idx, MO_UB);
+tcg_gen_ext_tl_i64(t1, base);
+tcg_gen_andi_i64(t1, t1, 7);
+if (left) {
+tcg_gen_xori_i64(t1, t1, 7);
+}
+tcg_gen_shli_i64(t1, t1, 3);
+tcg_gen_andi_tl(b

[Qemu-devel] [PATCH] target-mips: Implement Loongson 3A processor.

2016-11-22 Thread r
From: Heiher 

Signed-off-by: Heiher 
---
 target-mips/mips-defs.h  |   2 +
 target-mips/translate.c  | 449 ++-
 target-mips/translate_init.c |  24 +++
 3 files changed, 469 insertions(+), 6 deletions(-)

diff --git a/target-mips/mips-defs.h b/target-mips/mips-defs.h
index 047554e..30e3197 100644
--- a/target-mips/mips-defs.h
+++ b/target-mips/mips-defs.h
@@ -50,6 +50,7 @@
 /* Chip specific instructions. */
 #defineINSN_LOONGSON2E  0x2000
 #defineINSN_LOONGSON2F  0x4000
+#defineINSN_LOONGSON3A  0x1000
 #defineINSN_VR54XX 0x8000
 
 /* MIPS CPU defines. */
@@ -70,6 +71,7 @@
 /* MIPS Technologies "Release 2" */
 #defineCPU_MIPS32R2(CPU_MIPS32 | ISA_MIPS32R2)
 #defineCPU_MIPS64R2(CPU_MIPS64 | CPU_MIPS32R2 | 
ISA_MIPS64R2)
+#defineCPU_LOONGSON3A  (CPU_MIPS64R2 | INSN_LOONGSON3A)
 
 /* MIPS Technologies "Release 3" */
 #define CPU_MIPS32R3 (CPU_MIPS32R2 | ISA_MIPS32R3)
diff --git a/target-mips/translate.c b/target-mips/translate.c
index 0fdcb8e..de169db 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -2438,6 +2438,401 @@ static void gen_cop1_ldst(DisasContext *ctx, uint32_t 
op, int rt,
 }
 }
 
+static inline void gen_cop2_gslwlrc1(DisasContext *ctx, TCGv base,
+  int rt, int rs, int offset, int left)
+{
+TCGv_i32 t0, t1, t2;
+
+gen_base_offset_addr(ctx, base, rs, offset);
+t1 = tcg_temp_new_i32();
+/* Do a byte access to possibly trigger a page
+   fault with the unaligned address.  */
+tcg_gen_qemu_ld_i32(t1, base, ctx->mem_idx, MO_UB);
+tcg_gen_trunc_tl_i32(t1, base);
+tcg_gen_andi_i32(t1, t1, 3);
+if (left)
+tcg_gen_xori_i32(t1, t1, 3);
+tcg_gen_shli_i32(t1, t1, 3);
+tcg_gen_andi_tl(base, base, ~3);
+t0 = tcg_temp_new_i32();
+tcg_gen_qemu_ld_i32(t0, base, ctx->mem_idx, MO_TEUL);
+if (left) {
+tcg_gen_shl_i32(t0, t0, t1);
+t2 = tcg_const_i32(-1);
+} else {
+tcg_gen_shr_i32(t0, t0, t1);
+tcg_gen_xori_i32(t1, t1, 31);
+t2 = tcg_const_i32(0xfffeu);
+}
+tcg_gen_shl_i32(t2, t2, t1);
+gen_load_fpr32(ctx, t1, rt);
+if (left)
+tcg_gen_andc_i32(t1, t1, t2);
+else
+tcg_gen_and_i32(t1, t1, t2);
+tcg_temp_free_i32(t2);
+tcg_gen_or_i32(t0, t0, t1);
+tcg_temp_free_i32(t1);
+gen_store_fpr32(ctx, t0, rt);
+tcg_temp_free_i32(t0);
+}
+
+static inline void gen_cop2_gsldlrc1(DisasContext *ctx, TCGv base,
+  int rt, int rs, int offset, int left)
+{
+TCGv_i64 t0, t1, t2;
+
+gen_base_offset_addr(ctx, base, rs, offset);
+t1 = tcg_temp_new_i64();
+/* Do a byte access to possibly trigger a page
+   fault with the unaligned address.  */
+tcg_gen_qemu_ld_i64(t1, base, ctx->mem_idx, MO_UB);
+tcg_gen_ext_tl_i64(t1, base);
+tcg_gen_andi_i64(t1, t1, 7);
+if (left)
+tcg_gen_xori_i64(t1, t1, 7);
+tcg_gen_shli_i64(t1, t1, 3);
+tcg_gen_andi_tl(base, base, ~7);
+t0 = tcg_temp_new_i64();
+tcg_gen_qemu_ld_i64(t0, base, ctx->mem_idx, MO_TEQ);
+if (left) {
+tcg_gen_shl_i64(t0, t0, t1);
+t2 = tcg_const_i64(-1);
+} else {
+tcg_gen_shr_i64(t0, t0, t1);
+tcg_gen_xori_i64(t1, t1, 63);
+t2 = tcg_const_i64(0xfffeull);
+}
+tcg_gen_shl_i64(t2, t2, t1);
+gen_load_fpr64(ctx, t1, rt);
+if (left) {
+tcg_gen_andc_i64(t1, t1, t2);
+} else {
+tcg_gen_and_i64(t1, t1, t2);
+}
+tcg_temp_free_i64(t2);
+tcg_gen_or_i64(t0, t0, t1);
+tcg_temp_free_i64(t1);
+gen_store_fpr64(ctx, t0, rt);
+tcg_temp_free_i64(t0);
+}
+
+static void gen_cop2_ldst(DisasContext *ctx, uint32_t op, int rt,
+  int rs, int rd)
+{
+if (ctx->insn_flags & INSN_LOONGSON3A) {
+TCGv t0 = tcg_temp_new(), t1;
+TCGv_i32 fp32;
+TCGv_i64 fp64;
+int offset, gen_excp_cop2 = 0;
+
+switch (op) {
+case OPC_LWC2:
+offset = (int8_t)(ctx->opcode >> 6);
+switch (ctx->opcode & 0xc03f) {
+case 4:  /* gslwlc1 */
+check_cp1_enabled(ctx);
+gen_cop2_gslwlrc1(ctx, t0, rt, rs, offset, 1);
+break;
+case 5:  /* gslwrc1 */
+check_cp1_enabled(ctx);
+gen_cop2_gslwlrc1(ctx, t0, rt, rs, offset, 0);
+break;
+case 6:  /* gsldlc1 */
+check_cp1_enabled(ctx);
+gen_cop2_gsldlrc1(ctx, t0, rt, rs, offset, 1);
+break;
+case 7:  /* gsldrc1 */
+check_cp1_enabled(ctx);
+gen_cop2_gsldlrc1(ctx, t0, rt, rs, offset, 0);
+  

Re: [Qemu-devel] [PATCH 00/15] optimize Qemu RSS usage

2016-11-01 Thread Michael R. Hines

On 10/31/2016 05:00 PM, Michael R. Hines wrote:

On 10/18/2016 05:47 AM, Peter Lieven wrote:

Am 12.10.2016 um 23:18 schrieb Michael R. Hines:

Peter,

Greetings from DigitalOcean. We're experiencing the same symptoms 
without this patch.
We have, collectively, many gigabytes of un-planned-for RSS being 
used per-hypervisor

that we would like to get rid of =).

Without explicitly trying this patch (will do that ASAP), we 
immediately noticed that the
192MB mentioned immediately melts away (Yay) when we disabled the 
coroutine thread pool explicitly,
with another ~100MB in additional stack usage that would likely also 
go away if we

applied the entirety of your patch.

Is there any chance you have revisited this or have a timeline for it?


Hi Michael,

the current master already includes some of the patches of this 
original series. There are still some changes left, but

what works for me is the current master +

diff --git a/util/qemu-coroutine.c b/util/qemu-coroutine.c
index 5816702..3eaef68 100644
--- a/util/qemu-coroutine.c
+++ b/util/qemu-coroutine.c
@@ -25,8 +25,6 @@ enum {
 };

 /** Free list to speed up creation */
-static QSLIST_HEAD(, Coroutine) release_pool = 
QSLIST_HEAD_INITIALIZER(pool);

-static unsigned int release_pool_size;
 static __thread QSLIST_HEAD(, Coroutine) alloc_pool = 
QSLIST_HEAD_INITIALIZER(pool);

 static __thread unsigned int alloc_pool_size;
 static __thread Notifier coroutine_pool_cleanup_notifier;
@@ -49,20 +47,10 @@ Coroutine *qemu_coroutine_create(CoroutineEntry 
*entry)

 if (CONFIG_COROUTINE_POOL) {
 co = QSLIST_FIRST(_pool);
 if (!co) {
-if (release_pool_size > POOL_BATCH_SIZE) {
-/* Slow path; a good place to register the 
destructor, too.  */

-if (!coroutine_pool_cleanup_notifier.notify) {
-coroutine_pool_cleanup_notifier.notify = 
coroutine_pool_cleanup;

- qemu_thread_atexit_add(_pool_cleanup_notifier);
-}
-
-/* This is not exact; there could be a little skew 
between
- * release_pool_size and the actual size of 
release_pool.  But
- * it is just a heuristic, it does not need to be 
perfect.

- */
-alloc_pool_size = atomic_xchg(_pool_size, 0);
-QSLIST_MOVE_ATOMIC(_pool, _pool);
-co = QSLIST_FIRST(_pool);
+/* Slow path; a good place to register the destructor, 
too.  */

+if (!coroutine_pool_cleanup_notifier.notify) {
+coroutine_pool_cleanup_notifier.notify = 
coroutine_pool_cleanup;

+ qemu_thread_atexit_add(_pool_cleanup_notifier);
 }
 }
 if (co) {
@@ -85,11 +73,6 @@ static void coroutine_delete(Coroutine *co)
 co->caller = NULL;

 if (CONFIG_COROUTINE_POOL) {
-if (release_pool_size < POOL_BATCH_SIZE * 2) {
-QSLIST_INSERT_HEAD_ATOMIC(_pool, co, pool_next);
-atomic_inc(_pool_size);
-return;
-}
 if (alloc_pool_size < POOL_BATCH_SIZE) {
 QSLIST_INSERT_HEAD(_pool, co, pool_next);
 alloc_pool_size++;

+ invoking qemu with the following environemnet variable set:

MALLOC_MMAP_THRESHOLD_=32768 qemu-system-x86_64 

The last one makes glibc automatically using mmap when the malloced 
memory exceeds 32kByte.




Peter,

I tested the above patch (and the environment variable --- it doesn't 
quite come close to as lean of
an RSS tally as the original patchset  there's still about 
70-80 MB of remaining RSS.


Any chance you could trim the remaining fat before merging this? =)




False alarm! I didn't set the MMAP threshold low enough. Now the results 
are on-par with the other patchset.


Thank you!



Re: [Qemu-devel] [PATCH 00/15] optimize Qemu RSS usage

2016-10-31 Thread Michael R. Hines

On 10/18/2016 05:47 AM, Peter Lieven wrote:

Am 12.10.2016 um 23:18 schrieb Michael R. Hines:

Peter,

Greetings from DigitalOcean. We're experiencing the same symptoms 
without this patch.
We have, collectively, many gigabytes of un-planned-for RSS being 
used per-hypervisor

that we would like to get rid of =).

Without explicitly trying this patch (will do that ASAP), we 
immediately noticed that the
192MB mentioned immediately melts away (Yay) when we disabled the 
coroutine thread pool explicitly,
with another ~100MB in additional stack usage that would likely also 
go away if we

applied the entirety of your patch.

Is there any chance you have revisited this or have a timeline for it?


Hi Michael,

the current master already includes some of the patches of this 
original series. There are still some changes left, but

what works for me is the current master +

diff --git a/util/qemu-coroutine.c b/util/qemu-coroutine.c
index 5816702..3eaef68 100644
--- a/util/qemu-coroutine.c
+++ b/util/qemu-coroutine.c
@@ -25,8 +25,6 @@ enum {
 };

 /** Free list to speed up creation */
-static QSLIST_HEAD(, Coroutine) release_pool = 
QSLIST_HEAD_INITIALIZER(pool);

-static unsigned int release_pool_size;
 static __thread QSLIST_HEAD(, Coroutine) alloc_pool = 
QSLIST_HEAD_INITIALIZER(pool);

 static __thread unsigned int alloc_pool_size;
 static __thread Notifier coroutine_pool_cleanup_notifier;
@@ -49,20 +47,10 @@ Coroutine *qemu_coroutine_create(CoroutineEntry 
*entry)

 if (CONFIG_COROUTINE_POOL) {
 co = QSLIST_FIRST(_pool);
 if (!co) {
-if (release_pool_size > POOL_BATCH_SIZE) {
-/* Slow path; a good place to register the 
destructor, too.  */

-if (!coroutine_pool_cleanup_notifier.notify) {
-coroutine_pool_cleanup_notifier.notify = 
coroutine_pool_cleanup;

- qemu_thread_atexit_add(_pool_cleanup_notifier);
-}
-
-/* This is not exact; there could be a little skew 
between
- * release_pool_size and the actual size of 
release_pool.  But
- * it is just a heuristic, it does not need to be 
perfect.

- */
-alloc_pool_size = atomic_xchg(_pool_size, 0);
-QSLIST_MOVE_ATOMIC(_pool, _pool);
-co = QSLIST_FIRST(_pool);
+/* Slow path; a good place to register the destructor, 
too.  */

+if (!coroutine_pool_cleanup_notifier.notify) {
+coroutine_pool_cleanup_notifier.notify = 
coroutine_pool_cleanup;

+ qemu_thread_atexit_add(_pool_cleanup_notifier);
 }
 }
 if (co) {
@@ -85,11 +73,6 @@ static void coroutine_delete(Coroutine *co)
 co->caller = NULL;

 if (CONFIG_COROUTINE_POOL) {
-if (release_pool_size < POOL_BATCH_SIZE * 2) {
-QSLIST_INSERT_HEAD_ATOMIC(_pool, co, pool_next);
-atomic_inc(_pool_size);
-return;
-}
 if (alloc_pool_size < POOL_BATCH_SIZE) {
 QSLIST_INSERT_HEAD(_pool, co, pool_next);
 alloc_pool_size++;

+ invoking qemu with the following environemnet variable set:

MALLOC_MMAP_THRESHOLD_=32768 qemu-system-x86_64 

The last one makes glibc automatically using mmap when the malloced 
memory exceeds 32kByte.




Peter,

I tested the above patch (and the environment variable --- it doesn't 
quite come close to as lean of
an RSS tally as the original patchset  there's still about 70-80 
MB of remaining RSS.


Any chance you could trim the remaining fat before merging this? =)


/*
 * Michael R. Hines
 * Senior Engineer, DigitalOcean.
 */





Re: [Qemu-devel] [PATCH 00/15] optimize Qemu RSS usage

2016-10-19 Thread Michael R. Hines

Thank you for the response! I'll run off and test that. =)

/*
 * Michael R. Hines
 * Senior Engineer, DigitalOcean.
 */

On 10/18/2016 05:47 AM, Peter Lieven wrote:

Am 12.10.2016 um 23:18 schrieb Michael R. Hines:

Peter,

Greetings from DigitalOcean. We're experiencing the same symptoms 
without this patch.
We have, collectively, many gigabytes of un-planned-for RSS being 
used per-hypervisor

that we would like to get rid of =).

Without explicitly trying this patch (will do that ASAP), we 
immediately noticed that the
192MB mentioned immediately melts away (Yay) when we disabled the 
coroutine thread pool explicitly,
with another ~100MB in additional stack usage that would likely also 
go away if we

applied the entirety of your patch.

Is there any chance you have revisited this or have a timeline for it?


Hi Michael,

the current master already includes some of the patches of this 
original series. There are still some changes left, but

what works for me is the current master +

diff --git a/util/qemu-coroutine.c b/util/qemu-coroutine.c
index 5816702..3eaef68 100644
--- a/util/qemu-coroutine.c
+++ b/util/qemu-coroutine.c
@@ -25,8 +25,6 @@ enum {
 };

 /** Free list to speed up creation */
-static QSLIST_HEAD(, Coroutine) release_pool = 
QSLIST_HEAD_INITIALIZER(pool);

-static unsigned int release_pool_size;
 static __thread QSLIST_HEAD(, Coroutine) alloc_pool = 
QSLIST_HEAD_INITIALIZER(pool);

 static __thread unsigned int alloc_pool_size;
 static __thread Notifier coroutine_pool_cleanup_notifier;
@@ -49,20 +47,10 @@ Coroutine *qemu_coroutine_create(CoroutineEntry 
*entry)

 if (CONFIG_COROUTINE_POOL) {
 co = QSLIST_FIRST(_pool);
 if (!co) {
-if (release_pool_size > POOL_BATCH_SIZE) {
-/* Slow path; a good place to register the 
destructor, too.  */

-if (!coroutine_pool_cleanup_notifier.notify) {
-coroutine_pool_cleanup_notifier.notify = 
coroutine_pool_cleanup;

- qemu_thread_atexit_add(_pool_cleanup_notifier);
-}
-
-/* This is not exact; there could be a little skew 
between
- * release_pool_size and the actual size of 
release_pool.  But
- * it is just a heuristic, it does not need to be 
perfect.

- */
-alloc_pool_size = atomic_xchg(_pool_size, 0);
-QSLIST_MOVE_ATOMIC(_pool, _pool);
-co = QSLIST_FIRST(_pool);
+/* Slow path; a good place to register the destructor, 
too.  */

+if (!coroutine_pool_cleanup_notifier.notify) {
+coroutine_pool_cleanup_notifier.notify = 
coroutine_pool_cleanup;

+ qemu_thread_atexit_add(_pool_cleanup_notifier);
 }
 }
 if (co) {
@@ -85,11 +73,6 @@ static void coroutine_delete(Coroutine *co)
 co->caller = NULL;

 if (CONFIG_COROUTINE_POOL) {
-if (release_pool_size < POOL_BATCH_SIZE * 2) {
-QSLIST_INSERT_HEAD_ATOMIC(_pool, co, pool_next);
-atomic_inc(_pool_size);
-return;
-}
 if (alloc_pool_size < POOL_BATCH_SIZE) {
 QSLIST_INSERT_HEAD(_pool, co, pool_next);
 alloc_pool_size++;

+ invoking qemu with the following environemnet variable set:

MALLOC_MMAP_THRESHOLD_=32768 qemu-system-x86_64 

The last one makes glibc automatically using mmap when the malloced 
memory exceeds 32kByte.


Hope this helps,
Peter






Re: [Qemu-devel] [PATCH 00/15] optimize Qemu RSS usage

2016-10-12 Thread Michael R. Hines

Peter,

Greetings from DigitalOcean. We're experiencing the same symptoms 
without this patch.
We have, collectively, many gigabytes of un-planned-for RSS being used 
per-hypervisor

that we would like to get rid of =).

Without explicitly trying this patch (will do that ASAP), we immediately 
noticed that the
192MB mentioned immediately melts away (Yay) when we disabled the 
coroutine thread pool explicitly,
with another ~100MB in additional stack usage that would likely also go 
away if we

applied the entirety of your patch.

Is there any chance you have revisited this or have a timeline for it?

- Michael

/*
 * Michael R. Hines
 * Senior Engineer, DigitalOcean.
 */

On 06/28/2016 04:01 AM, Peter Lieven wrote:

I recently found that Qemu is using several hundred megabytes of RSS memory
more than older versions such as Qemu 2.2.0. So I started tracing
memory allocation and found 2 major reasons for this.

1) We changed the qemu coroutine pool to have a per thread and a global release
pool. The choosen poolsize and the changed algorithm could lead to up to
192 free coroutines with just a single iothread. Each of the coroutines
in the pool each having 1MB of stack memory.

2) Between Qemu 2.2.0 and 2.3.0 RCU was introduced which lead to delayed freeing
of memory. This lead to higher heap allocations which could not effectively
be returned to kernel (most likely due to fragmentation).

The following series is what I came up with. Beside the coroutine patches I 
changed
some allocations to forcibly use mmap. All these allocations are not repeatly 
made
during runtime so the impact of using mmap should be neglectible.

There are still some big malloced allocations left which cannot be easily 
changed
(e.g. the pixman buffers in VNC). So it might an idea to set a lower mmap 
threshold for
malloc since this threshold seems to be in the order of several Megabytes on 
modern systems.

Peter Lieven (15):
   coroutine-ucontext: mmap stack memory
   coroutine-ucontext: add a switch to monitor maximum stack size
   coroutine-ucontext: reduce stack size to 64kB
   coroutine: add a knob to disable the shared release pool
   util: add a helper to mmap private anonymous memory
   exec: use mmap for subpages
   qapi: use mmap for QmpInputVisitor
   virtio: use mmap for VirtQueue
   loader: use mmap for ROMs
   vmware_svga: use mmap for scratch pad
   qom: use mmap for bigger Objects
   util: add a function to realloc mmapped memory
   exec: use mmap for PhysPageMap->nodes
   vnc-tight: make the encoding palette static
   vnc: use mmap for VncState

  configure | 33 ++--
  exec.c| 11 ---
  hw/core/loader.c  | 16 +-
  hw/display/vmware_vga.c   |  3 +-
  hw/virtio/virtio.c|  5 +--
  include/qemu/mmap-alloc.h |  7 +
  include/qom/object.h  |  1 +
  qapi/qmp-input-visitor.c  |  5 +--
  qom/object.c  | 20 ++--
  ui/vnc-enc-tight.c| 21 ++---
  ui/vnc.c  |  5 +--
  ui/vnc.h  |  1 +
  util/coroutine-ucontext.c | 66 +--
  util/mmap-alloc.c | 27 
  util/qemu-coroutine.c | 79 ++-
  15 files changed, 225 insertions(+), 75 deletions(-)





Re: [Qemu-devel] [PATCH 0/3] RDMA error handling

2016-09-23 Thread Michael R. Hines

Reviewed-by: Michael R. Hines <mich...@hinespot.com>

(By the way, I no longer work for IBM and no longer have direct access to RDMA 
hardware. If someone is willing to let me login to something that does in the 
future, I don't mind debugging things. I just don't have any hardware of my own 
anymore to debug, and the last time I tried to use software RDMA it was an 
unpleasurable experience.)

/*
 * Michael R. Hines
 * Senior Engineer, DigitalOcean.
 */

On 09/23/2016 02:14 PM, Dr. David Alan Gilbert (git) wrote:

From: "Dr. David Alan Gilbert" <dgilb...@redhat.com>

lp: https://bugs.launchpad.net/qemu/+bug/1545052

The RDMA code tends to hang if the destination dies
in the wrong place;  this series doesn't completely fix
that, but in cases where the destination knows there's
been an error, it makes sure it tells the source and
that cleans up quickly.
If the destination just dies, then the source still hangs
and I still need to look at better ways to fix that.

Dave

Dr. David Alan Gilbert (3):
   migration/rdma: Pass qemu_file errors across link
   migration: Make failed migration load set file error
   migration/rdma: Don't flag an error when we've been told about one

  migration/rdma.c   |  9 -
  migration/savevm.c | 19 ---
  2 files changed, 20 insertions(+), 8 deletions(-)






Re: [Qemu-devel] [PATCH v5 0/7] Deterministic replay extensions

2016-03-15 Thread Igor R
> This set of patches is related to the reverse execution and deterministic
> replay of qemu execution. It includes recording and replaying of serial 
> devices
> and block devices operations.
>
> With these patches one can record and deterministically replay behavior
> of the system with connected disk drives and serial communication ports
> (e.g., telnet terminal).
>
> Patches for deterministic replay of the block devices intercept calls of
> bdrv coroutine functions at the top of block drivers stack.
> To record and replay block operations the drive must be configured
> as following:
>  -drive file=disk.qcow,if=none,id=img-direct
>  -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay
>  -device ide-hd,drive=img-blkreplay


Thanks for the great feature.

A couple of questions:
* Is it possible to enable/disable recording when QEMU is already
running? I.e. is it ok to call replay_configure() or replay_enable()
from the monitor? Would it be possible to replay such a log?
* Currently, when running QEMU with the above drive configuration
(even w/o icount), savevm command doesn't seem to work correctly, and
subsequent loavm crashes QEMU. Is it just a bug or this feature
doesn't support snapshots?
* The replaying is much slower than the original scenario. Is it
affected by icount shift? What shift values would allow "real time"
speed?


Thanks!



[Qemu-devel] cpu_memory_rw_debug doesn't work on MIPS?

2016-02-25 Thread Igor R
If I understand correctly, the most advanced MMU that QEMU emulates
for MIPS is "R4000-style" MMU - i.e. a "software-managed" TLB, where
on TLB miss QEMU just emulates exception that should be handled by the
guest OS. So, QEMU doesn't walk through the page directory, like it
does when emulating e.g. x86.
While this approach works for the guest code, it results in inability
to read guest virtual memory "externally" - from within a monitor
command, for debugging purposes etc. That is, cpu_memory_rw_debug()
doesn't work reliably for mapped segments - it fails because of TLB
miss, but no one tries to fill the TLB.
If all the above correct, is there any work-around that can be used to
read the guest memory in qemu-system-mips?

Thanks.



[Qemu-devel] Developing custom device for ARM

2016-02-23 Thread Igor R
Hello,

I implemented a simple sys_bus device that only communicates through
port io. The purpose of this device is to be accessible via /dev/port
only, with no need for any additional kernel modules.
I Defined the memory region using memory_region_init_io and registered
it using sysbus_add_io.
I registered the memory region in address 0x9000, and defined custom
read and write functions for it.
In the guest Linux I used dd if=/dev/port skip=$((0x9000)) count=1
bs=1 to access the port io registers.

This worked great for x86 and mips (malta) machines, however, on an
arm (versatilepb) machine, I couldn't get this method to work.
The reads and writes from 0x900X addresses did not get through to my device.

Further inspection revealed that in arm, the kernel tries to reference
the *virtual* address 0x9000, while in mips it references the virtual
address 0x8b009000.
(In x86 a different set of opcodes are used to access the port io area).

Why isn't this method working on arm?

Thanks.



Re: [Qemu-devel] Memory mapping on MIPS

2016-02-22 Thread Igor R
> > Here is an excerpt from r4k_map_address(), related to addresses >=
0x8000.
> > Actually, it maps 0x8010 and 0xA010 to the same physical
> > address. What's the idea behind that?
>
> 0x8010 is kseg0 whereas 0xA010 is kseg1, both segments are
> unmapped thus both refer to the same 0x10 physical address. The
> difference between them is that the latter is uncached, i.e. all
> accesses to this segment bypass the caches, but in QEMU we don't model
> caches so they do the same thing.
>
> > What should happen if I map KSEG2 directly as a continuation of KSEG1,
> > i.e. substitute TLB lookup with "address - (int32_t)KSEG1_BASE"? Guest
> > Linux seems to work correctly (but maybe it's just a matter of luck?).
>
> With such a change it's very likely that guest through kseg2 will be
> accessing different physical addresses than it expects...

I see, thanks.
As far as I can see now, my problem is that cpu_memory_rw_debug doesn't
handle tlb misses at all - that's why it fails for many addresses in kseg2.
Is it the desired behavior or a bug? Is there any other simple way to read
from the guest virt.addrs?

Thanks!


Re: [Qemu-devel] Memory mapping on MIPS

2016-02-22 Thread Igor R
> > Here is an excerpt from r4k_map_address(), related to addresses >=
0x8000.
> > Actually, it maps 0x8010 and 0xA010 to the same physical
> > address. What's the idea behind that?
>
> 0x8010 is kseg0 whereas 0xA010 is kseg1, both segments are
> unmapped thus both refer to the same 0x10 physical address. The
> difference between them is that the latter is uncached, i.e. all
> accesses to this segment bypass the caches, but in QEMU we don't model
> caches so they do the same thing.
>
> > What should happen if I map KSEG2 directly as a continuation of KSEG1,
> > i.e. substitute TLB lookup with "address - (int32_t)KSEG1_BASE"? Guest
> > Linux seems to work correctly (but maybe it's just a matter of luck?).
>
> With such a change it's very likely that guest through kseg2 will be
> accessing different physical addresses than it expects...

I see, thanks.
As far as I can see now, my problem is that cpu_memory_rw_debug doesn't
handle tlb misses at all - that's why it fails for many addresses in kseg2.
Is it the desired behavior or a bug? Is there any other simple way to read
from the guest virt.addrs?

Thanks!


  1   2   3   4   5   6   7   8   >