The following pull request was submitted through Github.
It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/8130
This e-mail was sent by the LXC bot, direct replies will not reach the author
unless they happen to be subscribed to this list.
=== Description (from pull-request) ===
This makes everything use the cgroup abstraction and improves some aspect of cgroup2 only systems.
From 8f6e40a18fb2715f1888d9049058ad95afc2b192 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com>
Date: Fri, 6 Nov 2020 18:04:54 -0500
Subject: [PATCH 1/8] lxd/cgroup: Add file read/writer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgra...@ubuntu.com>
---
lxd/cgroup/file.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
create mode 100644 lxd/cgroup/file.go
diff --git a/lxd/cgroup/file.go b/lxd/cgroup/file.go
new file mode 100644
index 0000000000..7ad0679624
--- /dev/null
+++ b/lxd/cgroup/file.go
@@ -0,0 +1,90 @@
+package cgroup
+
+import (
+ "fmt"
+ "io/ioutil"
+ "path/filepath"
+ "strings"
+
+ "github.com/lxc/lxd/shared"
+)
+
+// NewFileReadWriter returns a CGroup instance using the filesystem as its
backend.
+func NewFileReadWriter(pid int, unifiedCapable bool) (*CGroup, error) {
+ // Setup the read/writer struct.
+ rw := fileReadWriter{}
+
+ // Locate the base path for each controller.
+ rw.paths = map[string]string{}
+
+ controllers, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid))
+ if err != nil {
+ return nil, err
+ }
+
+ for _, line := range strings.Split(string(controllers), "\n") {
+ // Skip empty lines.
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue
+ }
+
+ // Extract the fields.
+ fields := strings.Split(line, ":")
+
+ // Determine the mount path.
+ path := filepath.Join("/sys/fs/cgroup", fields[1], fields[2])
+ if fields[0] == "0" {
+ fields[1] = "unified"
+ if shared.PathExists("/sys/fs/cgroup/unified") {
+ path = filepath.Join("/sys/fs/cgroup",
"unified", fields[2])
+ } else {
+ path = filepath.Join("/sys/fs/cgroup",
fields[2])
+ }
+
+ if fields[2] != "/init.scope" {
+ path = filepath.Dir(path)
+ }
+ }
+
+ // Add the controllers individually.
+ for _, ctrl := range strings.Split(fields[1], ",") {
+ rw.paths[ctrl] = path
+ }
+ }
+
+ cg, err := New(&rw)
+ if err != nil {
+ return nil, err
+ }
+
+ cg.UnifiedCapable = unifiedCapable
+ return cg, nil
+}
+
+type fileReadWriter struct {
+ paths map[string]string
+}
+
+func (rw *fileReadWriter) Get(version Backend, controller string, key string)
(string, error) {
+ path := filepath.Join(rw.paths[controller], key)
+ if cgLayout == CgroupsUnified {
+ path = filepath.Join(rw.paths["unified"], key)
+ }
+
+ value, err := ioutil.ReadFile(path)
+ if err != nil {
+ return "", err
+ }
+
+ return strings.TrimSpace(string(value)), nil
+}
+
+func (rw *fileReadWriter) Set(version Backend, controller string, key string,
value string) error {
+ path := filepath.Join(rw.paths[controller], key)
+ if cgLayout == CgroupsUnified {
+ path = filepath.Join(rw.paths["unified"], key)
+ }
+
+ return ioutil.WriteFile(path, []byte(value), 0600)
+}
From 71c25894921733922e305f6fdedfd18dbb26c3e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com>
Date: Fri, 6 Nov 2020 23:22:38 -0500
Subject: [PATCH 2/8] lxd/cgroup: Fix controller detection
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgra...@ubuntu.com>
---
lxd/cgroup/init.go | 69 +++++++++++++++++++++++++++-------------------
1 file changed, 40 insertions(+), 29 deletions(-)
diff --git a/lxd/cgroup/init.go b/lxd/cgroup/init.go
index 732d4afa40..7338e6d47e 100644
--- a/lxd/cgroup/init.go
+++ b/lxd/cgroup/init.go
@@ -123,39 +123,46 @@ func (info *Info) SupportsVersion(resource Resource)
(Backend, bool) {
switch resource {
case Blkio:
val, ok := cgControllers["blkio"]
- if ok && val == V1 {
- return V1, ok
+ if ok {
+ return val, ok
+ }
+
+ val, ok = cgControllers["io"]
+ if ok {
+ return val, ok
}
return Unavailable, false
case BlkioWeight:
val, ok := cgControllers["blkio.weight"]
- if ok && val == V1 {
- return V1, ok
+ if ok {
+ return val, ok
}
- return Unavailable, false
- case CPU:
- val, ok := cgControllers["cpu"]
- if ok && val == V1 {
- return V1, ok
+ val, ok = cgControllers["io"]
+ if ok {
+ return val, ok
}
return Unavailable, false
+ case CPU:
+ val, ok := cgControllers["cpu"]
+ return val, ok
case CPUAcct:
val, ok := cgControllers["cpuacct"]
- if ok && val == V1 {
- return V1, ok
+ if ok {
+ return val, ok
}
- return Unavailable, false
- case CPUSet:
- val, ok := cgControllers["cpuset"]
- if ok && val == V1 {
- return V1, ok
+ val, ok = cgControllers["cpu"]
+ if ok {
+ return val, ok
}
return Unavailable, false
+ case CPUSet:
+ val, ok := cgControllers["memory"]
+ return val, ok
case Devices:
val, ok := cgControllers["devices"]
return val, ok
@@ -185,8 +192,8 @@ func (info *Info) SupportsVersion(resource Resource)
(Backend, bool) {
return Unavailable, false
case MemorySwapMaxUsage:
val, ok := cgControllers["memory.memsw.max_usage_in_bytes"]
- if ok && val == V1 {
- return V1, ok
+ if ok {
+ return val, ok
}
return Unavailable, false
@@ -204,22 +211,18 @@ func (info *Info) SupportsVersion(resource Resource)
(Backend, bool) {
return Unavailable, false
case MemorySwappiness:
val, ok := cgControllers["memory.swappiness"]
- if ok && val == V1 {
- return V1, ok
+ if ok {
+ return val, ok
}
return Unavailable, false
case NetPrio:
val, ok := cgControllers["net_prio"]
- if ok && val == V1 {
- return V1, ok
- }
-
- return Unavailable, false
+ return val, ok
case Pids:
val, ok := cgControllers["pids"]
- if ok && val == V1 {
- return V1, ok
+ if ok {
+ return val, ok
}
return Unavailable, false
@@ -413,11 +416,11 @@ func init() {
val, ok = cgControllers["memory"]
if ok && val == V2 {
- if shared.PathExists("/sys/fs/cgroup/memory.swap.max") {
+ if
shared.PathExists("/sys/fs/cgroup/init.scope/memory.swap.max") {
cgControllers["memory.swap.max"] = V2
}
- if shared.PathExists("/sys/fs/cgroup/memory.swap.current") {
+ if
shared.PathExists("/sys/fs/cgroup/init.scope/memory.swap.current") {
cgControllers["memory.swap.current"] = V2
}
}
@@ -429,4 +432,12 @@ func init() {
} else if hasV2 {
cgLayout = CgroupsUnified
}
+
+ if cgLayout == CgroupsUnified {
+ // With Cgroup2 devices is built-in (through eBPF).
+ cgControllers["devices"] = V2
+
+ // With Cgroup2 freezer is built-in.
+ cgControllers["freezer"] = V2
+ }
}
From fa5f713cacad1e79b151b2d4f27a6858a92c5c56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com>
Date: Fri, 6 Nov 2020 18:04:44 -0500
Subject: [PATCH 3/8] lxd/cgroup: Add cpuset functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgra...@ubuntu.com>
---
lxd/cgroup/abstraction.go | 45 ++++++++++++++++++++++++++++++++++++++-
1 file changed, 44 insertions(+), 1 deletion(-)
diff --git a/lxd/cgroup/abstraction.go b/lxd/cgroup/abstraction.go
index 61b7ccf664..20d77d9306 100644
--- a/lxd/cgroup/abstraction.go
+++ b/lxd/cgroup/abstraction.go
@@ -243,7 +243,6 @@ func (cg *CGroup) SetBlkioWeight(value string) error {
return ErrControllerMissing
}
return ErrUnknownVersion
-
}
// SetCPUShare sets the weight of each group in the same hierarchy
@@ -323,3 +322,47 @@ func (cg *CGroup) SetMaxHugepages(pageType string, value
string) error {
}
return ErrUnknownVersion
}
+
+// GetEffectiveCpuset returns the current set of CPUs for the cgroup
+func (cg *CGroup) GetEffectiveCpuset() (string, error) {
+ // Confirm we have the controller
+ version := cgControllers["cpuset"]
+ switch version {
+ case Unavailable:
+ return "", ErrControllerMissing
+ case V1:
+ return cg.rw.Get(version, "cpuset", "cpuset.effective_cpus")
+ case V2:
+ return cg.rw.Get(version, "cpuset", "cpuset.cpus.effective")
+ }
+ return "", ErrUnknownVersion
+}
+
+// GetCpuset returns the current set of CPUs for the cgroup
+func (cg *CGroup) GetCpuset() (string, error) {
+ // Confirm we have the controller
+ version := cgControllers["cpuset"]
+ switch version {
+ case Unavailable:
+ return "", ErrControllerMissing
+ case V1:
+ return cg.rw.Get(version, "cpuset", "cpuset.cpus")
+ case V2:
+ return cg.rw.Get(version, "cpuset", "cpuset.cpus")
+ }
+ return "", ErrUnknownVersion
+}
+
+// SetCpuset set the currently allowed set of CPUs for the cgroups
+func (cg *CGroup) SetCpuset(value string) error {
+ version := cgControllers["cpuset"]
+ switch version {
+ case Unavailable:
+ return ErrControllerMissing
+ case V1:
+ return cg.rw.Set(version, "cpuset", "cpuset.cpus", value)
+ case V2:
+ return cg.rw.Set(version, "cpuset", "cpuset.cpus", value)
+ }
+ return ErrUnknownVersion
+}
From 57f08497dc890f8ef681190b680efabec1abce03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com>
Date: Fri, 6 Nov 2020 23:44:09 -0500
Subject: [PATCH 4/8] lxd/cgroup: Fix warning wording
---
lxd/cgroup/init.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lxd/cgroup/init.go b/lxd/cgroup/init.go
index 7338e6d47e..26c9163fdd 100644
--- a/lxd/cgroup/init.go
+++ b/lxd/cgroup/init.go
@@ -246,11 +246,11 @@ func (info *Info) Log() {
logger.Infof(" - cgroup layout: %s", info.Mode())
if !info.Supports(Blkio, nil) {
- logger.Warnf(" - Couldn't find the CGroup blkio, I/O limits
will be ignored")
+ logger.Warnf(" - Couldn't find the CGroup blkio, disk I/O
limits will be ignored")
}
if !info.Supports(BlkioWeight, nil) {
- logger.Warnf(" - Couldn't find the CGroup blkio.weight, I/O
weight limits will be ignored")
+ logger.Warnf(" - Couldn't find the CGroup blkio.weight, disk
priority will be ignored")
}
if !info.Supports(CPU, nil) {
@@ -282,7 +282,7 @@ func (info *Info) Log() {
}
if !info.Supports(NetPrio, nil) {
- logger.Warnf(" - Couldn't find the CGroup network class
controller, network limits will be ignored")
+ logger.Warnf(" - Couldn't find the CGroup network priority
controller, network priority will be ignored")
}
if !info.Supports(Pids, nil) {
From f3b1da47bb4ccab6afee795889e1b5f33c0e33a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com>
Date: Fri, 6 Nov 2020 18:14:17 -0500
Subject: [PATCH 5/8] lxd/devices: Drop old workaround
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgra...@ubuntu.com>
---
lxd/devices.go | 5 -----
1 file changed, 5 deletions(-)
diff --git a/lxd/devices.go b/lxd/devices.go
index 982b24d752..1ec90d105f 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -357,11 +357,6 @@ func deviceTaskBalance(s *state.State) {
}
effectiveCpus = strings.Join(effectiveCpusSlice, ",")
-
- err = cGroupSet("cpuset", "/lxc", "cpuset.cpus", effectiveCpus)
- if err != nil && shared.PathExists("/sys/fs/cgroup/cpuset/lxc") {
- logger.Warn("Error setting lxd's cpuset.cpus", log.Ctx{"err":
err})
- }
cpus, err := resources.ParseCpuset(effectiveCpus)
if err != nil {
logger.Error("Error parsing host's cpu set", log.Ctx{"cpuset":
effectiveCpus, "err": err})
From 995bd38b282b8f4770cfc87834d700c237822668 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com>
Date: Fri, 6 Nov 2020 18:14:25 -0500
Subject: [PATCH 6/8] lxd/devices: Port to cgroup package
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgra...@ubuntu.com>
---
lxd/cgroup.go | 61 --------------------------------------------------
lxd/devices.go | 10 +++++++--
2 files changed, 8 insertions(+), 63 deletions(-)
delete mode 100644 lxd/cgroup.go
diff --git a/lxd/cgroup.go b/lxd/cgroup.go
deleted file mode 100644
index 33f93cc9e8..0000000000
--- a/lxd/cgroup.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package main
-
-import (
- "bufio"
- "io/ioutil"
- "os"
- "path"
- "strings"
-)
-
-func getInitCgroupPath(controller string) string {
- f, err := os.Open("/proc/1/cgroup")
- if err != nil {
- return "/"
- }
- defer f.Close()
-
- scan := bufio.NewScanner(f)
- for scan.Scan() {
- line := scan.Text()
-
- fields := strings.Split(line, ":")
- if len(fields) != 3 {
- return "/"
- }
-
- if fields[2] != controller {
- continue
- }
-
- initPath := string(fields[3])
-
- // ignore trailing /init.scope if it is there
- dir, file := path.Split(initPath)
- if file == "init.scope" {
- return dir
- } else {
- return initPath
- }
- }
-
- return "/"
-}
-
-func cGroupGet(controller, cgroup, file string) (string, error) {
- initPath := getInitCgroupPath(controller)
- path := path.Join("/sys/fs/cgroup", controller, initPath, cgroup, file)
-
- contents, err := ioutil.ReadFile(path)
- if err != nil {
- return "", err
- }
- return strings.Trim(string(contents), "\n"), nil
-}
-
-func cGroupSet(controller, cgroup, file string, value string) error {
- initPath := getInitCgroupPath(controller)
- path := path.Join("/sys/fs/cgroup", controller, initPath, cgroup, file)
-
- return ioutil.WriteFile(path, []byte(value), 0755)
-}
diff --git a/lxd/devices.go b/lxd/devices.go
index 1ec90d105f..a30a8a244f 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -330,10 +330,16 @@ func deviceTaskBalance(s *state.State) {
}
// Get effective cpus list - those are all guaranteed to be online
- effectiveCpus, err := cGroupGet("cpuset", "/", "cpuset.effective_cpus")
+ cg, err := cgroup.NewFileReadWriter(1, true)
+ if err != nil {
+ logger.Errorf("Unable to load cgroup writer: %v", err)
+ return
+ }
+
+ effectiveCpus, err := cg.GetEffectiveCpuset()
if err != nil {
// Older kernel - use cpuset.cpus
- effectiveCpus, err = cGroupGet("cpuset", "/", "cpuset.cpus")
+ effectiveCpus, err = cg.GetCpuset()
if err != nil {
logger.Errorf("Error reading host's cpuset.cpus")
return
From c8e408eeb9d77405cfdf7828abbebdfe38e72b6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com>
Date: Fri, 6 Nov 2020 23:28:56 -0500
Subject: [PATCH 7/8] lxd/instance: Replace CGroupGet/CGroupSet
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgra...@ubuntu.com>
---
lxd/instance/drivers/driver_lxc.go | 4 ++++
lxd/instance/drivers/driver_qemu.go | 5 +++--
lxd/instance/instance_interface.go | 3 ++-
3 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/lxd/instance/drivers/driver_lxc.go
b/lxd/instance/drivers/driver_lxc.go
index 50404c02ef..4af9b98cfa 100644
--- a/lxd/instance/drivers/driver_lxc.go
+++ b/lxd/instance/drivers/driver_lxc.go
@@ -7050,6 +7050,10 @@ func (c *lxc) maasDelete() error {
return c.state.MAAS.DeleteContainer(c)
}
+func (c *lxc) CGroup() (*cgroup.CGroup, error) {
+ return c.cgroup(nil)
+}
+
func (c *lxc) cgroup(cc *liblxc.Container) (*cgroup.CGroup, error) {
rw := lxcCgroupReadWriter{}
if cc != nil {
diff --git a/lxd/instance/drivers/driver_qemu.go
b/lxd/instance/drivers/driver_qemu.go
index 41174d770e..bda16f90ae 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -29,6 +29,7 @@ import (
lxdClient "github.com/lxc/lxd/client"
"github.com/lxc/lxd/lxd/apparmor"
"github.com/lxc/lxd/lxd/backup"
+ "github.com/lxc/lxd/lxd/cgroup"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/db/query"
@@ -3934,8 +3935,8 @@ func (vm *qemu) Migrate(args *instance.CriuMigrationArgs)
error {
}
// CGroupSet is not implemented for VMs.
-func (vm *qemu) CGroupSet(key string, value string) error {
- return instance.ErrNotImplemented
+func (vm *qemu) CGroup() (*cgroup.CGroup, error) {
+ return nil, instance.ErrNotImplemented
}
// VolatileSet sets one or more volatile config keys.
diff --git a/lxd/instance/instance_interface.go
b/lxd/instance/instance_interface.go
index 89f628e87a..3992870f35 100644
--- a/lxd/instance/instance_interface.go
+++ b/lxd/instance/instance_interface.go
@@ -8,6 +8,7 @@ import (
liblxc "gopkg.in/lxc/go-lxc.v2"
"github.com/lxc/lxd/lxd/backup"
+ "github.com/lxc/lxd/lxd/cgroup"
"github.com/lxc/lxd/lxd/db"
deviceConfig "github.com/lxc/lxd/lxd/device/config"
"github.com/lxc/lxd/lxd/instance/instancetype"
@@ -75,7 +76,7 @@ type Instance interface {
DevPaths() []string
// Live configuration.
- CGroupSet(key string, value string) error
+ CGroup() (*cgroup.CGroup, error)
VolatileSet(changes map[string]string) error
// File handling.
From 4d982b749e8fd3fca3bcb2d358b3eb207b1ebfb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com>
Date: Fri, 6 Nov 2020 23:29:46 -0500
Subject: [PATCH 8/8] lxd/devices: Update to use cgroup abstraction
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgra...@ubuntu.com>
---
lxd/devices.go | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/lxd/devices.go b/lxd/devices.go
index a30a8a244f..e72e125fc9 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -479,7 +479,13 @@ func deviceTaskBalance(s *state.State) {
}
sort.Strings(set)
- err := ctn.CGroupSet("cpuset.cpus", strings.Join(set, ","))
+ cg, err := ctn.CGroup()
+ if err != nil {
+ logger.Error("balance: Unable to get cgroup struct",
log.Ctx{"name": ctn.Name(), "err": err, "value": strings.Join(set, ",")})
+ continue
+ }
+
+ err = cg.SetCpuset(strings.Join(set, ","))
if err != nil {
logger.Error("balance: Unable to set cpuset",
log.Ctx{"name": ctn.Name(), "err": err, "value": strings.Join(set, ",")})
}
@@ -510,7 +516,12 @@ func deviceNetworkPriority(s *state.State, netif string) {
}
// Set the value for the new interface
- c.CGroupSet("net_prio.ifpriomap", fmt.Sprintf("%s %d", netif,
networkInt))
+ cg, err := c.CGroup()
+ if err != nil {
+ continue
+ }
+
+ cg.SetNetIfPrio(fmt.Sprintf("%s %d", netif, networkInt))
}
return
_______________________________________________
lxc-devel mailing list
lxc-devel@lists.linuxcontainers.org
http://lists.linuxcontainers.org/listinfo/lxc-devel
