The following pull request was submitted through Github.
It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/7977
This e-mail was sent by the LXC bot, direct replies will not reach the author
unless they happen to be subscribed to this list.
=== Description (from pull-request) ===
Fixes #7936
This is a redone PR as in the last PR I created, I messed up commits while rebasing :(
From 34097139ae42fc62082adb6ed8e7b2415b3a0b30 Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthad...@gmail.com>
Date: Sun, 4 Oct 2020 21:40:53 +0530
Subject: [PATCH 1/4] lxd/storage: Adds rsync.compression config key
Signed-off-by: D R Siddhartha <siddharthad...@gmail.com>
---
lxd/storage/drivers/driver_btrfs.go | 10 +++++++++-
lxd/storage/drivers/driver_ceph.go | 10 +++++++++-
lxd/storage/drivers/driver_cephfs.go | 12 +++++++++++-
lxd/storage/drivers/driver_common.go | 11 ++++++++++-
lxd/storage/drivers/driver_zfs.go | 10 +++++++++-
lxd/storage/utils.go | 1 +
lxd/storage_pools_config.go | 3 +++
7 files changed, 52 insertions(+), 5 deletions(-)
diff --git a/lxd/storage/drivers/driver_btrfs.go
b/lxd/storage/drivers/driver_btrfs.go
index 988e741277..c98b8409de 100644
--- a/lxd/storage/drivers/driver_btrfs.go
+++ b/lxd/storage/drivers/driver_btrfs.go
@@ -372,9 +372,17 @@ func (d *btrfs) GetResources() (*api.ResourcesStoragePool,
error) {
// MigrationType returns the type of transfer methods to be used when doing
migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool)
[]migration.Type {
- rsyncFeatures := []string{"xattrs", "delete", "compress",
"bidirectional"}
+ var rsyncFeatures []string
btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader,
migration.BTRFSFeatureSubvolumes}
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" &&
!shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress",
"bidirectional"}
+ }
+
// Only offer rsync for refreshes or if running in an unprivileged
container.
if refresh || d.state.OS.RunningInUserNS {
var transportType migration.MigrationFSType
diff --git a/lxd/storage/drivers/driver_ceph.go
b/lxd/storage/drivers/driver_ceph.go
index f13f1a3c0e..2500bb94ac 100644
--- a/lxd/storage/drivers/driver_ceph.go
+++ b/lxd/storage/drivers/driver_ceph.go
@@ -340,7 +340,15 @@ func (d *ceph) GetResources() (*api.ResourcesStoragePool,
error) {
// MigrationType returns the type of transfer methods to be used when doing
migrations between pools in preference order.
func (d *ceph) MigrationTypes(contentType ContentType, refresh bool)
[]migration.Type {
- rsyncFeatures := []string{"delete", "compress", "bidirectional"}
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" &&
!shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"delete", "compress", "bidirectional"}
+ }
if refresh {
var transportType migration.MigrationFSType
diff --git a/lxd/storage/drivers/driver_cephfs.go
b/lxd/storage/drivers/driver_cephfs.go
index d275212c45..015da2c949 100644
--- a/lxd/storage/drivers/driver_cephfs.go
+++ b/lxd/storage/drivers/driver_cephfs.go
@@ -312,6 +312,16 @@ func (d *cephfs) GetResources()
(*api.ResourcesStoragePool, error) {
// MigrationTypes returns the supported migration types and options supported
by the driver.
func (d *cephfs) MigrationTypes(contentType ContentType, refresh bool)
[]migration.Type {
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" &&
!shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"delete", "compress", "bidirectional"}
+ }
+
if contentType != ContentTypeFS {
return nil
}
@@ -320,7 +330,7 @@ func (d *cephfs) MigrationTypes(contentType ContentType,
refresh bool) []migrati
return []migration.Type{
{
FSType: migration.MigrationFSType_RSYNC,
- Features: []string{"delete", "compress",
"bidirectional"},
+ Features: rsyncFeatures,
},
}
}
diff --git a/lxd/storage/drivers/driver_common.go
b/lxd/storage/drivers/driver_common.go
index 843f5652f0..e15950acad 100644
--- a/lxd/storage/drivers/driver_common.go
+++ b/lxd/storage/drivers/driver_common.go
@@ -132,6 +132,15 @@ func (d *common) validateVolume(vol Volume, driverRules
map[string]func(value st
// in preference order.
func (d *common) MigrationTypes(contentType ContentType, refresh bool)
[]migration.Type {
var transportType migration.MigrationFSType
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" &&
!shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress",
"bidirectional"}
+ }
if contentType == ContentTypeBlock {
transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
@@ -142,7 +151,7 @@ func (d *common) MigrationTypes(contentType ContentType,
refresh bool) []migrati
return []migration.Type{
{
FSType: transportType,
- Features: []string{"xattrs", "delete", "compress",
"bidirectional"},
+ Features: rsyncFeatures,
},
}
}
diff --git a/lxd/storage/drivers/driver_zfs.go
b/lxd/storage/drivers/driver_zfs.go
index 387a351f5f..5e290af54f 100644
--- a/lxd/storage/drivers/driver_zfs.go
+++ b/lxd/storage/drivers/driver_zfs.go
@@ -452,7 +452,15 @@ func (d *zfs) GetResources() (*api.ResourcesStoragePool,
error) {
// MigrationType returns the type of transfer methods to be used when doing
migrations between pools in preference order.
func (d *zfs) MigrationTypes(contentType ContentType, refresh bool)
[]migration.Type {
- rsyncFeatures := []string{"xattrs", "delete", "compress",
"bidirectional"}
+ var rsyncFeatures []string
+
+ // Do not pass compression argument to rsync if the associated
+ // config key, that is rsync.compression, is set to false.
+ if d.Config()["rsync.compression"] != "" &&
!shared.IsTrue(d.Config()["rsync.compression"]) {
+ rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
+ } else {
+ rsyncFeatures = []string{"xattrs", "delete", "compress",
"bidirectional"}
+ }
// When performing a refresh, always use rsync. Using zfs send/receive
// here doesn't make sense since it would need to send everything again
diff --git a/lxd/storage/utils.go b/lxd/storage/utils.go
index 93201ad46e..6f4c9e8c0b 100644
--- a/lxd/storage/utils.go
+++ b/lxd/storage/utils.go
@@ -424,6 +424,7 @@ func validatePoolCommonRules() map[string]func(string)
error {
"volume.size": validate.Optional(validate.IsSize),
"size": validate.Optional(validate.IsSize),
"rsync.bwlimit": validate.IsAny,
+ "rsync.compression": validate.Optional(validate.IsBool),
}
}
diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go
index cfcdf0b234..5c6892bd12 100644
--- a/lxd/storage_pools_config.go
+++ b/lxd/storage_pools_config.go
@@ -81,6 +81,9 @@ var storagePoolConfigKeys = map[string]func(value string)
error{
"zfs.clone_copy": validate.Optional(validate.IsBool),
"zfs.pool_name": validate.IsAny,
"rsync.bwlimit": validate.IsAny,
+
+ // valid drivers: btrfs, ceph, cephfs, zfs
+ "rsync.compression": validate.Optional(validate.IsBool),
}
func storagePoolValidateConfig(name string, driver string, config
map[string]string, oldConfig map[string]string) error {
From f371c6c931afca0a033a345c9116efc91fa8e2ae Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthad...@gmail.com>
Date: Sun, 4 Oct 2020 21:44:06 +0530
Subject: [PATCH 2/4] doc: Adds rsync.compression
Signed-off-by: D R Siddhartha <siddharthad...@gmail.com>
---
doc/storage.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/storage.md b/doc/storage.md
index 6e0dd9f5fa..67dc5786f1 100644
--- a/doc/storage.md
+++ b/doc/storage.md
@@ -27,6 +27,7 @@ lvm.vg.force\_reuse | bool | lvm driver
volume.lvm.stripes | string | lvm driver
| - | storage\_lvm\_stripes | Number of
stripes to use for new volumes (or thin pool volume).
volume.lvm.stripes.size | string | lvm driver
| - | storage\_lvm\_stripes | Size of
stripes to use (at least 4096 bytes and multiple of 512bytes).
rsync.bwlimit | string | -
| 0 (no limit) | storage\_rsync\_bwlimit | Specifies
the upper limit to be placed on the socket I/O whenever rsync has to be used to
transfer storage entities.
+rsync.compression | bool | appropriate driver
| true | storage\_rsync\_compression | Whether to
use compression while migrating storage pools.
volatile.initial\_source | string | -
| - | storage\_volatile\_initial\_source | Records
the actual source passed during creating (e.g. /dev/sdb).
volatile.pool.pristine | string | -
| true | storage\_driver\_ceph | Whether
the pool has been empty on creation time.
volume.block.filesystem | string | block based driver (lvm)
| ext4 | storage | Filesystem
to use for new volumes
From f9b21d257a4e8deb79b36d1540d151b4d14bc242 Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthad...@gmail.com>
Date: Sun, 4 Oct 2020 21:46:45 +0530
Subject: [PATCH 3/4] api: storage_rsync_compression
Signed-off-by: D R Siddhartha <siddharthad...@gmail.com>
---
doc/api-extensions.md | 4 ++++
shared/version/api.go | 1 +
2 files changed, 5 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index a0cd6ec30c..d7c233c3e6 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -1182,3 +1182,7 @@ Adds `Name` field to `InstanceBackupArgs` to allow
specifying a different instan
Adds `Name` and `PoolName` fields to `StoragePoolVolumeBackupArgs` to allow
specifying a different volume name
when restoring a custom volume backup.
+
+## storage\_rsync\_compression
+Adds `rsync.compression` config key to storage pools. This key can be used
+to disable compression in rsync while migrating storage pools.
\ No newline at end of file
diff --git a/shared/version/api.go b/shared/version/api.go
index 41a64fac67..ba9380b676 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -228,6 +228,7 @@ var APIExtensions = []string{
"projects_networks_restricted_uplinks",
"custom_volume_backup",
"backup_override_name",
+ "storage_rsync_compression",
}
// APIExtensionsCount returns the number of available API extensions.
From 1d0da811313c7c0e1ab3bd6449ea9ad591ad34d0 Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthad...@gmail.com>
Date: Sun, 4 Oct 2020 21:48:35 +0530
Subject: [PATCH 4/4] tests: Valid rsync.compression
Signed-off-by: D R Siddhartha <siddharthad...@gmail.com>
---
test/suites/migration.sh | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/test/suites/migration.sh b/test/suites/migration.sh
index 6b3896f117..f2416eed6b 100644
--- a/test/suites/migration.sh
+++ b/test/suites/migration.sh
@@ -329,6 +329,13 @@ migration() {
lxc_remote storage volume delete l2:"$remote_pool2" vol5
lxc_remote storage volume delete l2:"$remote_pool2" vol6
+ # Test migration when rsync compression is disabled
+ lxc_remote storage set l1:"$remote_pool1" rsync.compression false
+ lxc_remote storage volume create l1:"$remote_pool1" foo
+ lxc_remote storage volume copy l1:"$remote_pool1"/foo l2:"$remote_pool2"/bar
+ lxc_remote storage volume delete l1:"$remote_pool1" foo
+ lxc_remote storage volume delete l2:"$remote_pool2" bar
+
# Test some migration between projects
lxc_remote project create l1:proj -c features.images=false -c
features.profiles=false
lxc_remote project switch l1:proj
_______________________________________________
lxc-devel mailing list
lxc-devel@lists.linuxcontainers.org
http://lists.linuxcontainers.org/listinfo/lxc-devel
