The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/8126
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) ===
From b9ebdef70cac8153b509a06032e0ff98acc794c7 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 6 Nov 2020 09:43:23 +0100 Subject: [PATCH 1/6] shared: Allow volatile uuid config keys Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- shared/instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/instance.go b/shared/instance.go index 06541b4539..5234f5c751 100644 --- a/shared/instance.go +++ b/shared/instance.go @@ -305,7 +305,7 @@ func ConfigKeyChecker(key string) (func(value string) error, error) { return validate.IsAny, nil } - if strings.HasSuffix(key, "vm.uuid") { + if strings.HasSuffix(key, ".uuid") { return validate.IsAny, nil } From 36ff22e7c69079e78d86727c23cb6de7e42791b7 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 6 Nov 2020 09:45:15 +0100 Subject: [PATCH 2/6] lxd/instance/drivers: Support vgpu in qemu template Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/instance/drivers/driver_qemu_templates.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go index c0b1cc27d8..68c1927f67 100644 --- a/lxd/instance/drivers/driver_qemu_templates.go +++ b/lxd/instance/drivers/driver_qemu_templates.go @@ -528,10 +528,14 @@ addr = "{{.devAddr}}" {{if eq .bus "ccw" -}} driver = "vfio-ccw" {{- end}} +{{- if ne .vgpu "" -}} +sysfsdev = "/sys/bus/mdev/devices/{{.vgpu}}" +{{- else}} host = "{{.pciSlotName}}" {{if .vga -}} x-vga = "on" {{- end }} +{{- end }} {{if .multifunction -}} multifunction = "on" {{- end }} From 240d577d3402de0d03dea285a3e2fa8b24dbceec Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 6 Nov 2020 09:46:46 +0100 Subject: [PATCH 3/6] lxd/instance/drivers: Support vgpu in VMs --- lxd/instance/drivers/driver_qemu.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go index 41174d770e..cdb753ebdf 100644 --- a/lxd/instance/drivers/driver_qemu.go +++ b/lxd/instance/drivers/driver_qemu.go @@ -2364,12 +2364,14 @@ func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus *qemuBus, bootIndexes m // addGPUDevConfig adds the qemu config required for adding a GPU device. func (vm *qemu) addGPUDevConfig(sb *strings.Builder, bus *qemuBus, gpuConfig []deviceConfig.RunConfigItem) error { - var devName, pciSlotName string + var devName, pciSlotName, vgpu string for _, gpuItem := range gpuConfig { if gpuItem.Key == "devName" { devName = gpuItem.Value } else if gpuItem.Key == "pciSlotName" { pciSlotName = gpuItem.Value + } else if gpuItem.Key == "vgpu" { + vgpu = gpuItem.Value } } @@ -2386,6 +2388,7 @@ func (vm *qemu) addGPUDevConfig(sb *strings.Builder, bus *qemuBus, gpuConfig []d "devName": devName, "pciSlotName": pciSlotName, "vga": vgaMode, + "vgpu": vgpu, } // Add main GPU device in VGA mode to qemu config. @@ -2394,8 +2397,14 @@ func (vm *qemu) addGPUDevConfig(sb *strings.Builder, bus *qemuBus, gpuConfig []d return err } - // Add any other related IOMMU VFs as generic PCI devices. - iommuGroupPath := filepath.Join("/sys/bus/pci/devices", pciSlotName, "iommu_group", "devices") + var iommuGroupPath string + + if vgpu != "" { + iommuGroupPath = filepath.Join("/sys/bus/mdev/devices", vgpu, "iommu_group", "devices") + } else { + // Add any other related IOMMU VFs as generic PCI devices. + iommuGroupPath = filepath.Join("/sys/bus/pci/devices", pciSlotName, "iommu_group", "devices") + } if shared.PathExists(iommuGroupPath) { // Extract parent slot name by removing any virtual function ID. From 593eb27c3a14ae7d533c3874aaeeb7b9c8e6669d Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 6 Nov 2020 09:54:36 +0100 Subject: [PATCH 4/6] lxd/device: Support virtual GPUs Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/device/gpu.go | 135 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 128 insertions(+), 7 deletions(-) diff --git a/lxd/device/gpu.go b/lxd/device/gpu.go index 9c7724077b..d2a2ecfed5 100644 --- a/lxd/device/gpu.go +++ b/lxd/device/gpu.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/google/uuid" "github.com/pkg/errors" "golang.org/x/sys/unix" @@ -17,10 +18,12 @@ import ( "github.com/lxc/lxd/lxd/instance/instancetype" "github.com/lxc/lxd/lxd/resources" "github.com/lxc/lxd/shared" + "github.com/lxc/lxd/shared/logger" "github.com/lxc/lxd/shared/validate" ) const gpuDRIDevPath = "/dev/dri" +const gpuVFIODevPath = "/dev/vfio" // Non-card devices such as {/dev/nvidiactl, /dev/nvidia-uvm, ...} type nvidiaNonCardDevice struct { @@ -47,6 +50,7 @@ func (d *gpu) validateConfig(instConf instance.ConfigReader) error { "uid": unixValidUserID, "gid": unixValidUserID, "mode": unixValidOctalFileMode, + "mdev": validate.IsAny, } err := d.config.Validate(rules) @@ -90,6 +94,62 @@ func (d *gpu) validateEnvironment() error { return nil } +func (d *gpu) createVirtualGPU() error { + gpus, err := resources.GetGPU() + if err != nil { + return err + } + + for _, gpu := range gpus.Cards { + // Skip any cards that don't match the vendorid, pci or productid settings (if specified). + if (d.config["vendorid"] != "" && gpu.VendorID != d.config["vendorid"]) || + (d.config["pci"] != "" && gpu.PCIAddress != d.config["pci"]) || + (d.config["productid"] != "" && gpu.ProductID != d.config["productid"]) { + continue + } + + foundMdev := false + + for mdev := range gpu.Mdev { + if d.config["mdev"] == mdev { + foundMdev = true + break + } + } + + if !foundMdev { + return fmt.Errorf("Invalid mdev %q", d.config["mdev"]) + } + + // Check if the vgpu exists before creating it. + v := d.volatileGet() + + if v["vgpu.uuid"] != "" && shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s/%s", gpu.PCIAddress, v["vgpu.uuid"])) { + return nil + } + + // Create the virtual gpu + devUUID, err := uuid.NewUUID() + if err != nil { + return errors.Wrap(err, "Failed to generate UUID") + } + + err = ioutil.WriteFile(filepath.Join(fmt.Sprintf("/sys/bus/pci/devices/%s/mdev_supported_types/%s/create", gpu.PCIAddress, d.config["mdev"])), []byte(devUUID.String()), 200) + if err != nil { + return errors.Wrapf(err, "Failed to create virtual gpu %q", devUUID.String()) + } + + err = d.volatileSet(map[string]string{"vgpu.uuid": devUUID.String()}) + if err != nil { + return err + } + + break + } + + return nil +} + // Start is run when the device is added to the container. func (d *gpu) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() @@ -97,6 +157,13 @@ func (d *gpu) Start() (*deviceConfig.RunConfig, error) { return nil, err } + if d.config["mdev"] != "" { + err = d.createVirtualGPU() + if err != nil { + return nil, err + } + } + if d.inst.Type() == instancetype.VM { return d.startVM() } @@ -128,7 +195,38 @@ func (d *gpu) startContainer() (*deviceConfig.RunConfig, error) { if gpu.DRM != nil && (d.config["id"] == "" || fmt.Sprintf("%d", gpu.DRM.ID) == d.config["id"]) { found = true - if gpu.DRM.CardName != "" && gpu.DRM.CardDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.CardName)) { + if d.config["mdev"] != "" { + v := d.volatileGet() + + // Get iommu group + vgpuPath := fmt.Sprintf("/sys/bus/pci/devices/%s/%s/iommu_group", gpu.PCIAddress, v["vgpu.uuid"]) + + link, err := os.Readlink(vgpuPath) + if err != nil { + return nil, err + } + + iommuGroup := filepath.Base(link) + path := filepath.Join(gpuVFIODevPath, iommuGroup) + + // Get major and minor numbers + var stat unix.Stat_t + + err = unix.Stat(path, &stat) + if err != nil { + return nil, err + } + + major := uint32(stat.Rdev / 256) + minor := uint32(stat.Rdev % 256) + + err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, path, false, &runConf) + if err != nil { + return nil, err + } + } + + if d.config["mdev"] == "" && gpu.DRM.CardName != "" && gpu.DRM.CardDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.CardName)) { path := filepath.Join(gpuDRIDevPath, gpu.DRM.CardName) major, minor, err := d.deviceNumStringToUint32(gpu.DRM.CardDevice) if err != nil { @@ -141,7 +239,7 @@ func (d *gpu) startContainer() (*deviceConfig.RunConfig, error) { } } - if gpu.DRM.RenderName != "" && gpu.DRM.RenderDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.RenderName)) { + if d.config["mdev"] == "" && gpu.DRM.RenderName != "" && gpu.DRM.RenderDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.RenderName)) { path := filepath.Join(gpuDRIDevPath, gpu.DRM.RenderName) major, minor, err := d.deviceNumStringToUint32(gpu.DRM.RenderDevice) if err != nil { @@ -154,7 +252,7 @@ func (d *gpu) startContainer() (*deviceConfig.RunConfig, error) { } } - if gpu.DRM.ControlName != "" && gpu.DRM.ControlDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.ControlName)) { + if d.config["mdev"] == "" && gpu.DRM.ControlName != "" && gpu.DRM.ControlDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.ControlName)) { path := filepath.Join(gpuDRIDevPath, gpu.DRM.ControlName) major, minor, err := d.deviceNumStringToUint32(gpu.DRM.ControlDevice) if err != nil { @@ -256,9 +354,11 @@ func (d *gpu) startVM() (*deviceConfig.RunConfig, error) { saveData["last_state.pci.slot.name"] = pciDev.SlotName saveData["last_state.pci.driver"] = pciDev.Driver - err = d.pciDeviceDriverOverrideIOMMU(pciDev, "vfio-pci", false) - if err != nil { - return nil, errors.Wrapf(err, "Failed to override IOMMU group driver") + if d.config["mdev"] == "" { + err = d.pciDeviceDriverOverrideIOMMU(pciDev, "vfio-pci", false) + if err != nil { + return nil, errors.Wrapf(err, "Failed to override IOMMU group driver") + } } runConf.GPUDevice = append(runConf.GPUDevice, @@ -267,6 +367,15 @@ func (d *gpu) startVM() (*deviceConfig.RunConfig, error) { {Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]}, }...) + if d.config["mdev"] != "" { + v := d.volatileGet() + + saveData["vgpu.uuid"] = v["vgpu.uuid"] + + runConf.GPUDevice = append(runConf.GPUDevice, + deviceConfig.RunConfigItem{Key: "vgpu", Value: v["vgpu.uuid"]}) + } + err = d.volatileSet(saveData) if err != nil { return nil, err @@ -349,10 +458,22 @@ func (d *gpu) postStop() error { defer d.volatileSet(map[string]string{ "last_state.pci.slot.name": "", "last_state.pci.driver": "", + "vgpu.uuid": "", }) v := d.volatileGet() + if v["vgpu.uuid"] != "" { + path := fmt.Sprintf("/sys/bus/mdev/devices/%s", v["vgpu.uuid"]) + + if shared.PathExists(path) { + err := ioutil.WriteFile(filepath.Join(path, "remove"), []byte("1\n"), 0200) + if err != nil { + logger.Debugf("Failed to remove vgpu %q", v["vgpu.uuid"]) + } + } + } + if d.inst.Type() == instancetype.Container { // Remove host files for this device. err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), "unix", d.name, "") @@ -362,7 +483,7 @@ func (d *gpu) postStop() error { } // If VM physical pass through, unbind from vfio-pci and bind back to host driver. - if d.inst.Type() == instancetype.VM && v["last_state.pci.slot.name"] != "" { + if d.inst.Type() == instancetype.VM && v["last_state.pci.slot.name"] != "" && v["vgpu.uuid"] == "" { pciDev := pciDevice{ Driver: "vfio-pci", SlotName: v["last_state.pci.slot.name"], From 531ca6adbb80027ff433ac6c6e4ab4c1c79622d3 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 6 Nov 2020 10:21:28 +0100 Subject: [PATCH 5/6] doc: Document mdev config key Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- doc/instances.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/instances.md b/doc/instances.md index d6809ff115..cf6b89a687 100644 --- a/doc/instances.md +++ b/doc/instances.md @@ -763,6 +763,7 @@ pci | string | - | no | The pci address of the uid | int | 0 | no | UID of the device owner in the instance (container only) gid | int | 0 | no | GID of the device owner in the instance (container only) mode | int | 0660 | no | Mode of the device in the instance (container only) +mdev | string | - | no | The mdev type to use (e.g. i915-GVTg_V5_4) ### Type: proxy From c8520ff699b1fe0ab9603e0bfa58de09a5c343dc Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 6 Nov 2020 13:45:29 +0100 Subject: [PATCH 6/6] api: Add virtual_gpu Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- doc/api-extensions.md | 4 ++++ shared/version/api.go | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index 555e83d8c6..cc42d755e0 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -1222,3 +1222,7 @@ This introduces the `tpm` device type. This introduces `rebase` as a value for zfs.clone\_copy causing LXD to track down any "image" dataset in the ancestry line and then perform send/receive on top of that. + +## virtual\_gpu +This adds support for virtual GPUs. It introduces the `mdev` config key for GPU devices which takes +a supported mdev type, e.g. i915-GVTg_V5_4. diff --git a/shared/version/api.go b/shared/version/api.go index 2c986fa5f0..d827894d2d 100644 --- a/shared/version/api.go +++ b/shared/version/api.go @@ -235,6 +235,7 @@ var APIExtensions = []string{ "network_ovn_external_routes_remove", "tpm_device_type", "storage_zfs_clone_copy_rebase", + "virtual_gpu", } // APIExtensionsCount returns the number of available API extensions.
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel