In order to support the theoretical maximum of devices (16 disks and 8 NICs) we introduce kvm_pci_reservations hvparam that denotes the number of PCI slots that QEMU will implicitly use, e.g. if kvm_pci_reservations is set to 5, QEMU will manage PCI slots 0, 1, 2, 3, 4. By default this will be 12 and Ganeti will start adding disks and NICs from the 13rd slot (or whatever the next slot is) onwards.
Signed-off-by: Dimitris Aragiorgis <[email protected]> --- lib/hypervisor/hv_kvm/__init__.py | 24 +++++++++++++++++++++--- lib/hypervisor/hv_kvm/monitor.py | 4 ++-- man/gnt-instance.rst | 8 ++++++++ src/Ganeti/Constants.hs | 11 +++++++++++ test/py/ganeti.hypervisor.hv_kvm_unittest.py | 5 ++++- 5 files changed, 46 insertions(+), 6 deletions(-) diff --git a/lib/hypervisor/hv_kvm/__init__.py b/lib/hypervisor/hv_kvm/__init__.py index bac72c0..d50a9a0 100644 --- a/lib/hypervisor/hv_kvm/__init__.py +++ b/lib/hypervisor/hv_kvm/__init__.py @@ -515,6 +515,11 @@ class KVMHypervisor(hv_base.BaseHypervisor): constants.HV_KVM_EXTRA: hv_base.NO_CHECK, constants.HV_KVM_MACHINE_VERSION: hv_base.NO_CHECK, constants.HV_KVM_MIGRATION_CAPS: hv_base.NO_CHECK, + constants.HV_KVM_PCI_RESERVATIONS: + (False, lambda x: (x >= 0 and x <= constants.QEMU_PCI_SLOTS), + "The number of PCI slots managed by QEMU (max: %s)" % + constants.QEMU_PCI_SLOTS, + None, None), constants.HV_VNET_HDR: hv_base.NO_CHECK, } @@ -570,6 +575,11 @@ class KVMHypervisor(hv_base.BaseHypervisor): # NOTE: This maps to the default PCI bus created by pc machine type # by default (pci.0). The q35 creates a PCIe bus that is not hotpluggable # and should be handled differently (pcie.0). + # NOTE: This bitarray here is defined for more fine-grained control. + # Currently the number of slots is QEMU_PCI_SLOTS and the reserved + # ones are the first QEMU_DEFAULT_PCI_RESERVATIONS. + # If the above constants change without updating _DEFAULT_PCI_RESERVATIONS + # properly, TestGenerateDeviceHVInfo() will probably break. _DEFAULT_PCI_RESERVATIONS = "11111111111100000000000000000000" # The SCSI bus is created on demand or automatically and is empty. # For simplicity we decide to use a different target (scsi-id) @@ -1225,7 +1235,7 @@ class KVMHypervisor(hv_base.BaseHypervisor): kvm_cmd.extend(["-pidfile", pidfile]) - bus_slots = self._GetBusSlots() + bus_slots = self._GetBusSlots(hvp) # As requested by music lovers if hvp[constants.HV_SOUNDHW]: @@ -2034,7 +2044,7 @@ class KVMHypervisor(hv_base.BaseHypervisor): if (int(v_major), int(v_min)) < (1, 7): raise errors.HotplugError("Hotplug not supported for qemu versions < 1.7") - def _GetBusSlots(self, runtime=None): + def _GetBusSlots(self, hvp=None, runtime=None): """Helper function to get the slots of PCI and SCSI QEMU buses. This will return the status of the first PCI and SCSI buses. By default @@ -2059,6 +2069,14 @@ class KVMHypervisor(hv_base.BaseHypervisor): _SCSI_BUS: bitarray(self._DEFAULT_SCSI_RESERVATIONS), } + # Adjust the empty slots depending of the corresponding hvparam + if hvp and constants.HV_KVM_PCI_RESERVATIONS in hvp: + res = hvp[constants.HV_KVM_PCI_RESERVATIONS] + pci = bitarray(constants.QEMU_PCI_SLOTS) + pci.setall(False) # pylint: disable=E1101 + pci[0:res:1] = True + bus_slots[_PCI_BUS] = pci + # This is during hot-add if runtime: _, nics, _, disks = runtime @@ -2118,7 +2136,7 @@ class KVMHypervisor(hv_base.BaseHypervisor): runtime = self._LoadKVMRuntime(instance) up_hvp = runtime[2] device_type = _DEVICE_TYPE[dev_type](up_hvp) - bus_state = self._GetBusSlots(runtime) + bus_state = self._GetBusSlots(up_hvp, runtime) # in case of hot-mod this is given if not device.hvinfo: device.hvinfo = _GenerateDeviceHVInfo(dev_type, kvm_devid, diff --git a/lib/hypervisor/hv_kvm/monitor.py b/lib/hypervisor/hv_kvm/monitor.py index 74317f6..658f20c 100644 --- a/lib/hypervisor/hv_kvm/monitor.py +++ b/lib/hypervisor/hv_kvm/monitor.py @@ -48,6 +48,7 @@ from bitarray import bitarray from ganeti import errors from ganeti import utils +from ganeti import constants from ganeti import serializer @@ -255,7 +256,6 @@ class QmpConnection(MonitorSocket): _CAPABILITIES_COMMAND = "qmp_capabilities" _QUERY_COMMANDS = "query-commands" _MESSAGE_END_TOKEN = "\r\n" - _QEMU_PCI_SLOTS = 32 # The number of PCI slots QEMU exposes by default # List of valid attributes for the device_add QMP command. # Extra attributes found in device's hvinfo will be ignored. _DEVICE_ATTRIBUTES = [ @@ -669,7 +669,7 @@ class QmpConnection(MonitorSocket): """Get the first available PCI slot of a running instance. """ - slots = bitarray(self._QEMU_PCI_SLOTS) + slots = bitarray(constants.QEMU_PCI_SLOTS) slots.setall(False) # pylint: disable=E1101 for d in self._GetPCIDevices(): slot = d["slot"] diff --git a/man/gnt-instance.rst b/man/gnt-instance.rst index 03d8fc9..caad1d0 100644 --- a/man/gnt-instance.rst +++ b/man/gnt-instance.rst @@ -343,6 +343,14 @@ scsi\_controller\_type - megasas - virtio-scsi-pci +kvm\_pci\_reservations + Valid for the KVM hypervisor. + + The nubmer of PCI slots that QEMU will manage implicitly. By default Ganeti + will let QEMU use the first 12 slots (i.e. PCI slots 0-11) on its own and + will start adding disks and NICs from the 13rd slot (i.e. PCI slot 12) + onwards. So by default one can add 20 PCI devices (32 - 12). To support more + than that, this hypervisor parameter should be set accordingly (e.g. to 8). disk\_type Valid for the Xen HVM and KVM hypervisors. diff --git a/src/Ganeti/Constants.hs b/src/Ganeti/Constants.hs index 98d6805..28efc16 100644 --- a/src/Ganeti/Constants.hs +++ b/src/Ganeti/Constants.hs @@ -1700,6 +1700,9 @@ hvKvmDiskAio = "disk_aio" hvKvmScsiControllerType :: String hvKvmScsiControllerType = "scsi_controller_type" +hvKvmPciReservations :: String +hvKvmPciReservations = "kvm_pci_reservations" + hvKvmSpiceAudioCompr :: String hvKvmSpiceAudioCompr = "spice_playback_compression" @@ -1907,6 +1910,7 @@ hvsParameterTypes = Map.fromList , (hvKvmPath, VTypeString) , (hvKvmDiskAio, VTypeString) , (hvKvmScsiControllerType, VTypeString) + , (hvKvmPciReservations, VTypeInt) , (hvKvmSpiceAudioCompr, VTypeBool) , (hvKvmSpiceBind, VTypeString) , (hvKvmSpiceIpVersion, VTypeInt) @@ -2647,6 +2651,12 @@ vncBasePort = 5900 vncDefaultBindAddress :: String vncDefaultBindAddress = ip4AddressAny +qemuPciSlots :: Int +qemuPciSlots = 32 + +qemuDefaultPciReservations :: Int +qemuDefaultPciReservations = 12 + -- * NIC types htNicE1000 :: String @@ -4058,6 +4068,7 @@ hvcDefaults = , (hvVncX509Verify, PyValueEx False) , (hvVncPasswordFile, PyValueEx "") , (hvKvmScsiControllerType, PyValueEx htScsiControllerLsi) + , (hvKvmPciReservations, PyValueEx qemuDefaultPciReservations) , (hvKvmSpiceBind, PyValueEx "") , (hvKvmSpiceIpVersion, PyValueEx ifaceNoIpVersionSpecified) , (hvKvmSpicePasswordFile, PyValueEx "") diff --git a/test/py/ganeti.hypervisor.hv_kvm_unittest.py b/test/py/ganeti.hypervisor.hv_kvm_unittest.py index 5ea7ebb..16b1e0a 100755 --- a/test/py/ganeti.hypervisor.hv_kvm_unittest.py +++ b/test/py/ganeti.hypervisor.hv_kvm_unittest.py @@ -468,6 +468,7 @@ class TestGenerateDeviceKVMId(unittest.TestCase): class TestGenerateDeviceHVInfo(testutils.GanetiTestCase): def testPCI(self): + """Test the placement of the first PCI device during startup.""" self.MockOut(mock.patch('ganeti.utils.EnsureDirs')) hypervisor = hv_kvm.KVMHypervisor() dev_type = constants.HOTPLUG_TARGET_NIC @@ -478,16 +479,18 @@ class TestGenerateDeviceHVInfo(testutils.GanetiTestCase): kvm_devid, hv_dev_type, bus_slots) + # NOTE: The PCI slot is zero-based, i.e. 13th slot has addr hex(12) expected_hvinfo = { "driver": "virtio-net-pci", "id": kvm_devid, "bus": "pci.0", - "addr": hex(16), + "addr": hex(constants.QEMU_DEFAULT_PCI_RESERVATIONS), } self.assertTrue(hvinfo == expected_hvinfo) def testSCSI(self): + """Test the placement of the first SCSI device during startup.""" self.MockOut(mock.patch('ganeti.utils.EnsureDirs')) hypervisor = hv_kvm.KVMHypervisor() dev_type = constants.HOTPLUG_TARGET_DISK -- 1.7.10.4
