On Wed, Jun 26, 2019 at 03:36:35PM +0200, Juergen Gross wrote:
>On 26.06.19 14:21, Chao Gao wrote:
>>On Wed, Jun 26, 2019 at 08:17:50AM +0200, Juergen Gross wrote:
>>>On 24.06.19 20:47, Stefano Stabellini wrote:
>>>>+ xen-devel
>>>>
>>>>On Mon, 24 Jun 2019, Stefano Stabellini wrote:
>>>>>Hi all,
>>>>>
>>>>>I might have found a bug with PCI passthrough to a Linux HVM guest on
>>>>>x86 with Xen 4.12. It is not easy for me to get access, and especially
>>>>>change components, on this particular system, and I don't have access to
>>>>>other x86 boxes at the moment, so apologies for the partial information
>>>>>report. The setup is as follow:
>>>>>
>>>>>- two PCI devices have been assigned to a HVM guest, everything is fine
>>>>>- reboot the guest from inside, i.e. `reboot' in Linux
>>>>>- after the reboot completes, only one device is assigned
>>>>>
>>>>>Before the reboot, I see all the appropriate xenstore entries for both
>>>>>devices. Everything is fine. After the reboot, I can only see the
>>>>>xenstore entries of one device. It is as if the other device
>>>>>"disappeared" without throwing any errors.
>>>>>
>>>>>Have you seen this before? Do you know if it has been fixed in staging?
>>>>>I noticed this fix which seems to be very relevant:
>>>>>
>>>>>https://lists.xenproject.org/archives/html/xen-devel/2018-11/msg01616.html
>>>>>
>>>>>but it is already included in 4.12.
>>>
>>>Stefano, could you please try the attached patch? It is only compile
>>>tested for now.
>>>
>>>
>>>Juergen
>>
>>>From ea95dcdfc60a895cc43baf34c8e0fb088e10008d Mon Sep 17 00:00:00 2001
>>>From: Juergen Gross <jgr...@suse.com>
>>>To: xen-devel@lists.xenproject.org
>>>Cc: Ian Jackson <ian.jack...@eu.citrix.com>
>>>Cc: Wei Liu <w...@xen.org>
>>>Date: Wed, 26 Jun 2019 08:15:28 +0200
>>>Subject: [PATCH] libxl: fix pci device re-assigning after domain reboot
>>>
>>>After a reboot of a guest only the first pci device configuration will
>>>be retrieved from Xenstore resulting in loss of any further assigned
>>>passed through pci devices.
>>>
>>>The main reason is that all passed through pci devices reside under a
>>>common root device "0" in Xenstore. So when the device list is rebuilt
>>>from Xenstore after a reboot the sub-devices below that root device
>>>need to be selected instead of using the root device number as a
>>>selector.
>>>
>>>Fix that by adding a new member to struct libxl_device_type which when
>>>set is used to get the number of devices. Add such a member for pci to
>>>get the correct number of pci devices instead of implying it from the
>>>number of pci root devices (which will always be 1).
>>>
>>>While at it fix the type of libxl__device_pci_from_xs_be() to match
>>>the one of the .from_xenstore member of struct libxl_device_type. This
>>>fixes a latent bug checking the return value of a function returning
>>>void.
>>>
>>>Signed-off-by: Juergen Gross <jgr...@suse.com>
>>
>>Tested-by: Chao Gao <chao....@intel.com>
>
>Thanks!
>
>>
>>Note that I just tested it on RELEASE-4.12.0, not staging.
>>
>>I also found USB device would miss across reboot. Is there someone
>>willing to fix it too?
>
>I'll have a look.
>
In guest configuration file, it has following two lines:
usbctrl=['type=devicemodel,version=1']
usbdev=['type=hostdev,hostbus=1,hostaddr=3,controller=0,port=1']
Attachments are output of 'xenstore-ls' before and after reboot
Thanks
Chao
tool = ""
xenstored = ""
local = ""
domain = ""
0 = ""
control = ""
feature-poweroff = "1"
feature-reboot = "1"
feature-suspend = "1"
domid = "0"
name = "Domain-0"
device-model = ""
0 = ""
backends = ""
console = ""
vkbd = ""
qdisk = ""
9pfs = ""
qusb = ""
vfb = ""
qnic = ""
state = "running"
37 = ""
backends = ""
console = ""
vkbd = ""
9pfs = ""
qusb = ""
state = "running"
device = ""
vbd = ""
51712 = ""
backend = "/local/domain/0/backend/qdisk/0/51712"
backend-id = "0"
state = "3"
virtual-device = "51712"
device-type = "disk"
protocol = "x86_64-abi"
ring-ref = "8"
event-channel = "76"
feature-persistent = "1"
backend = ""
qdisk = ""
0 = ""
51712 = ""
frontend = "/local/domain/0/device/vbd/51712"
params = "qcow2:centos.qcow2"
frontend-id = "0"
online = "1"
removable = "0"
bootable = "1"
state = "2"
dev = "xvda"
type = "qdisk"
mode = "w"
device-type = "disk"
discard-enable = "1"
feature-flush-cache = "1"
info = "0"
max-ring-page-order = "4"
feature-discard = "1"
hotplug-status = "connected"
37 = ""
51712 = ""
frontend = "/local/domain/37/device/vbd/51712"
params = "qcow2:/home/nervalusr/chao/centos.qcow2"
frontend-id = "37"
online = "1"
removable = "0"
bootable = "1"
state = "4"
dev = "xvda"
type = "qdisk"
mode = "w"
device-type = "disk"
discard-enable = "1"
feature-discard = "1"
discard-granularity = "512"
feature-flush-cache = "1"
max-ring-page-order = "4"
info = "0"
sector-size = "512"
sectors = "62914560"
hotplug-status = "connected"
console = ""
37 = ""
0 = ""
frontend = "/local/domain/37/console"
frontend-id = "37"
online = "1"
state = "1"
protocol = "vt100"
vkbd = ""
37 = ""
0 = ""
frontend = "/local/domain/37/device/vkbd/0"
frontend-id = "37"
online = "1"
state = "4"
backend-type = "qemu"
feature-abs-pointer = "1"
feature-raw-pointer = "1"
hotplug-status = "connected"
vif = ""
37 = ""
0 = ""
frontend = "/local/domain/37/device/vif/0"
frontend-id = "37"
online = "1"
state = "4"
script = "/etc/xen/scripts/vif-bridge"
mac = "22:16:3e:00:9f:a7"
bridge = "xenbr0"
handle = "0"
type = "vif_ioemu"
feature-sg = "1"
feature-gso-tcpv4 = "1"
feature-gso-tcpv6 = "1"
feature-ipv6-csum-offload = "1"
feature-rx-copy = "1"
feature-rx-flip = "0"
feature-multicast-control = "1"
feature-dynamic-multicast-control = "1"
feature-split-event-channels = "1"
multi-queue-max-queues = "4"
feature-ctrl-ring = "1"
hotplug-status = "connected"
37 = ""
vm = "/vm/9a6857f8-05a0-4c8d-92c3-1ea04bc28c32"
name = "guest_centos7"
cpu = ""
0 = ""
availability = "online"
1 = ""
availability = "online"
2 = ""
availability = "online"
3 = ""
availability = "online"
memory = ""
static-max = "16777216"
target = "16760832"
videoram = "16384"
device = ""
suspend = ""
event-channel = ""
vbd = ""
51712 = ""
backend = "/local/domain/0/backend/qdisk/37/51712"
backend-id = "0"
state = "4"
virtual-device = "51712"
device-type = "disk"
ring-ref = "2312"
event-channel = "46"
protocol = "x86_64-abi"
feature-persistent = "1"
vkbd = ""
0 = ""
backend = "/local/domain/0/backend/vkbd/37/0"
backend-id = "0"
state = "4"
request-abs-pointer = "1"
page-ref = "4156386"
page-gref = "270"
event-channel = "47"
vif = ""
0 = ""
backend = "/local/domain/0/backend/vif/37/0"
backend-id = "0"
state = "4"
handle = "0"
mac = "22:16:3e:00:9f:a7"
multi-queue-num-queues = "4"
queue-0 = ""
tx-ring-ref = "2304"
rx-ring-ref = "2305"
event-channel-tx = "38"
event-channel-rx = "39"
queue-1 = ""
tx-ring-ref = "2306"
rx-ring-ref = "2307"
event-channel-tx = "40"
event-channel-rx = "41"
queue-2 = ""
tx-ring-ref = "2308"
rx-ring-ref = "2309"
event-channel-tx = "42"
event-channel-rx = "43"
queue-3 = ""
tx-ring-ref = "2310"
rx-ring-ref = "2311"
event-channel-tx = "44"
event-channel-rx = "45"
request-rx-copy = "1"
feature-rx-notify = "1"
feature-sg = "1"
feature-gso-tcpv4 = "1"
feature-gso-tcpv6 = "1"
feature-ipv6-csum-offload = "1"
control = ""
shutdown = ""
feature-poweroff = ""
feature-reboot = ""
feature-suspend = ""
feature-s3 = ""
feature-s4 = ""
sysrq = ""
platform-feature-multiprocessor-suspend = "1"
platform-feature-xs_reset_watches = "1"
hvmloader = ""
bios = "seabios"
allow-memory-relocate = "0"
data = ""
drivers = ""
feature = ""
attr = ""
domid = "37"
store = ""
port = "1"
ring-ref = "1044476"
platform = ""
acpi = "1"
acpi_s3 = "1"
acpi_s4 = "1"
acpi_laptop_slate = "0"
device-model = "qemu_xen"
console = ""
backend = "/local/domain/0/backend/console/37/0"
backend-id = "0"
limit = "1048576"
type = "xenconsoled"
output = "pty"
tty = "/dev/pts/4"
port = "2"
ring-ref = "1044479"
vnc-listen = "127.0.0.1"
vnc-port = "5900"
image = ""
device-model-pid = "8409"
serial = ""
0 = ""
tty = "/dev/pts/6"
vm = ""
9a6857f8-05a0-4c8d-92c3-1ea04bc28c32 = ""
name = "guest_centos7"
uuid = "9a6857f8-05a0-4c8d-92c3-1ea04bc28c32"
rtc = ""
timeoffset = ""
image = ""
ostype = "hvm"
start_time = "1561550075.84"
libxl = ""
pciback = ""
0000-88-00-0 = ""
driver_path = "/sys/bus/pci/drivers/intel_nnp"
0 = ""
device = ""
vbd = ""
51712 = ""
frontend = "/local/domain/0/device/vbd/51712"
backend = "/local/domain/0/backend/qdisk/0/51712"
params = "qcow2:centos.qcow2"
frontend-id = "0"
online = "1"
removable = "0"
bootable = "1"
state = "1"
dev = "xvda"
type = "qdisk"
mode = "w"
device-type = "disk"
discard-enable = "1"
37 = ""
device = ""
vbd = ""
51712 = ""
frontend = "/local/domain/37/device/vbd/51712"
backend = "/local/domain/0/backend/qdisk/37/51712"
params = "qcow2:/home/nervalusr/chao/centos.qcow2"
frontend-id = "37"
online = "1"
removable = "0"
bootable = "1"
state = "1"
dev = "xvda"
type = "qdisk"
mode = "w"
device-type = "disk"
discard-enable = "1"
console = ""
0 = ""
frontend = "/local/domain/37/console"
backend = "/local/domain/0/backend/console/37/0"
frontend-id = "37"
online = "1"
state = "1"
protocol = "vt100"
vkbd = ""
0 = ""
frontend = "/local/domain/37/device/vkbd/0"
backend = "/local/domain/0/backend/vkbd/37/0"
frontend-id = "37"
online = "1"
state = "1"
backend-type = "qemu"
vif = ""
0 = ""
frontend = "/local/domain/37/device/vif/0"
backend = "/local/domain/0/backend/vif/37/0"
frontend-id = "37"
online = "1"
state = "1"
script = "/etc/xen/scripts/vif-bridge"
mac = "22:16:3e:00:9f:a7"
bridge = "xenbr0"
handle = "0"
type = "vif_ioemu"
vusb = ""
0 = ""
type = "devicemodel"
usb-ver = "1"
num-ports = "2"
port = ""
1 = "1-4"
2 = ""
type = "hvm"
dm-version = "qemu_xen"
tool = ""
xenstored = ""
local = ""
domain = ""
0 = ""
control = ""
feature-poweroff = "1"
feature-reboot = "1"
feature-suspend = "1"
domid = "0"
name = "Domain-0"
device-model = ""
0 = ""
backends = ""
console = ""
vkbd = ""
qdisk = ""
9pfs = ""
qusb = ""
vfb = ""
qnic = ""
state = "running"
38 = ""
backends = ""
console = ""
vkbd = ""
9pfs = ""
qusb = ""
state = "running"
device = ""
vbd = ""
51712 = ""
backend = "/local/domain/0/backend/qdisk/0/51712"
backend-id = "0"
state = "3"
virtual-device = "51712"
device-type = "disk"
protocol = "x86_64-abi"
ring-ref = "8"
event-channel = "76"
feature-persistent = "1"
backend = ""
qdisk = ""
0 = ""
51712 = ""
frontend = "/local/domain/0/device/vbd/51712"
params = "qcow2:centos.qcow2"
frontend-id = "0"
online = "1"
removable = "0"
bootable = "1"
state = "2"
dev = "xvda"
type = "qdisk"
mode = "w"
device-type = "disk"
discard-enable = "1"
feature-flush-cache = "1"
info = "0"
max-ring-page-order = "4"
feature-discard = "1"
hotplug-status = "connected"
38 = ""
51712 = ""
frontend = "/local/domain/38/device/vbd/51712"
params = "qcow2:/home/nervalusr/chao/centos.qcow2"
frontend-id = "38"
online = "1"
removable = "0"
bootable = "1"
state = "4"
dev = "xvda"
type = "qdisk"
mode = "w"
device-type = "disk"
discard-enable = "1"
feature-discard = "1"
discard-granularity = "512"
feature-flush-cache = "1"
max-ring-page-order = "4"
info = "0"
sector-size = "512"
sectors = "62914560"
hotplug-status = "connected"
console = ""
38 = ""
0 = ""
frontend = "/local/domain/38/console"
frontend-id = "38"
online = "1"
state = "1"
protocol = "vt100"
vkbd = ""
38 = ""
0 = ""
frontend = "/local/domain/38/device/vkbd/0"
frontend-id = "38"
online = "1"
state = "4"
backend-type = "qemu"
feature-abs-pointer = "1"
feature-raw-pointer = "1"
hotplug-status = "connected"
vif = ""
38 = ""
0 = ""
frontend = "/local/domain/38/device/vif/0"
frontend-id = "38"
online = "1"
state = "4"
script = "/etc/xen/scripts/vif-bridge"
mac = "22:16:3e:00:9f:a7"
bridge = "xenbr0"
handle = "0"
type = "vif_ioemu"
feature-sg = "1"
feature-gso-tcpv4 = "1"
feature-gso-tcpv6 = "1"
feature-ipv6-csum-offload = "1"
feature-rx-copy = "1"
feature-rx-flip = "0"
feature-multicast-control = "1"
feature-dynamic-multicast-control = "1"
feature-split-event-channels = "1"
multi-queue-max-queues = "4"
feature-ctrl-ring = "1"
hotplug-status = "connected"
38 = ""
vm = "/vm/9a6857f8-05a0-4c8d-92c3-1ea04bc28c32"
name = "guest_centos7"
cpu = ""
0 = ""
availability = "online"
1 = ""
availability = "online"
2 = ""
availability = "online"
3 = ""
availability = "online"
memory = ""
static-max = "16777216"
target = "16760832"
videoram = "16384"
device = ""
suspend = ""
event-channel = ""
vbd = ""
51712 = ""
backend = "/local/domain/0/backend/qdisk/38/51712"
backend-id = "0"
state = "4"
virtual-device = "51712"
device-type = "disk"
ring-ref = "2312"
event-channel = "45"
protocol = "x86_64-abi"
feature-persistent = "1"
vkbd = ""
0 = ""
backend = "/local/domain/0/backend/vkbd/38/0"
backend-id = "0"
state = "4"
request-abs-pointer = "1"
page-ref = "222612"
page-gref = "343"
event-channel = "46"
vif = ""
0 = ""
backend = "/local/domain/0/backend/vif/38/0"
backend-id = "0"
state = "4"
handle = "0"
mac = "22:16:3e:00:9f:a7"
multi-queue-num-queues = "4"
queue-0 = ""
tx-ring-ref = "2304"
rx-ring-ref = "2305"
event-channel-tx = "37"
event-channel-rx = "38"
queue-1 = ""
tx-ring-ref = "2306"
rx-ring-ref = "2307"
event-channel-tx = "39"
event-channel-rx = "40"
queue-2 = ""
tx-ring-ref = "2308"
rx-ring-ref = "2309"
event-channel-tx = "41"
event-channel-rx = "42"
queue-3 = ""
tx-ring-ref = "2310"
rx-ring-ref = "2311"
event-channel-tx = "43"
event-channel-rx = "44"
request-rx-copy = "1"
feature-rx-notify = "1"
feature-sg = "1"
feature-gso-tcpv4 = "1"
feature-gso-tcpv6 = "1"
feature-ipv6-csum-offload = "1"
control = ""
shutdown = ""
feature-poweroff = ""
feature-reboot = ""
feature-suspend = ""
feature-s3 = ""
feature-s4 = ""
sysrq = ""
platform-feature-multiprocessor-suspend = "1"
platform-feature-xs_reset_watches = "1"
hvmloader = ""
bios = "seabios"
allow-memory-relocate = "0"
data = ""
drivers = ""
feature = ""
attr = ""
domid = "38"
store = ""
port = "1"
ring-ref = "1044476"
platform = ""
acpi = "1"
acpi_s3 = "1"
acpi_s4 = "1"
acpi_laptop_slate = "0"
device-model = "qemu_xen"
console = ""
backend = "/local/domain/0/backend/console/38/0"
backend-id = "0"
limit = "1048576"
type = "xenconsoled"
output = "pty"
tty = "/dev/pts/3"
port = "2"
ring-ref = "1044479"
vnc-listen = "127.0.0.1"
vnc-port = "5900"
image = ""
device-model-pid = "8783"
serial = ""
0 = ""
tty = "/dev/pts/4"
vm = ""
9a6857f8-05a0-4c8d-92c3-1ea04bc28c32 = ""
name = "guest_centos7"
uuid = "9a6857f8-05a0-4c8d-92c3-1ea04bc28c32"
rtc = ""
timeoffset = ""
image = ""
ostype = "hvm"
start_time = "1561550310.23"
libxl = ""
pciback = ""
0000-88-00-0 = ""
driver_path = "/sys/bus/pci/drivers/intel_nnp"
0 = ""
device = ""
vbd = ""
51712 = ""
frontend = "/local/domain/0/device/vbd/51712"
backend = "/local/domain/0/backend/qdisk/0/51712"
params = "qcow2:centos.qcow2"
frontend-id = "0"
online = "1"
removable = "0"
bootable = "1"
state = "1"
dev = "xvda"
type = "qdisk"
mode = "w"
device-type = "disk"
discard-enable = "1"
38 = ""
device = ""
vbd = ""
51712 = ""
frontend = "/local/domain/38/device/vbd/51712"
backend = "/local/domain/0/backend/qdisk/38/51712"
params = "qcow2:/home/nervalusr/chao/centos.qcow2"
frontend-id = "38"
online = "1"
removable = "0"
bootable = "1"
state = "1"
dev = "xvda"
type = "qdisk"
mode = "w"
device-type = "disk"
discard-enable = "1"
console = ""
0 = ""
frontend = "/local/domain/38/console"
backend = "/local/domain/0/backend/console/38/0"
frontend-id = "38"
online = "1"
state = "1"
protocol = "vt100"
vkbd = ""
0 = ""
frontend = "/local/domain/38/device/vkbd/0"
backend = "/local/domain/0/backend/vkbd/38/0"
frontend-id = "38"
online = "1"
state = "1"
backend-type = "qemu"
vif = ""
0 = ""
frontend = "/local/domain/38/device/vif/0"
backend = "/local/domain/0/backend/vif/38/0"
frontend-id = "38"
online = "1"
state = "1"
script = "/etc/xen/scripts/vif-bridge"
mac = "22:16:3e:00:9f:a7"
bridge = "xenbr0"
handle = "0"
type = "vif_ioemu"
type = "hvm"
dm-version = "qemu_xen"
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel
