[libvirt] [PULL v3 02/19] block: move bdrv_can_store_new_dirty_bitmap to block/dirty-bitmap.c

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

block/dirty-bitmap.c seems to be more appropriate for it and
bdrv_remove_persistent_dirty_bitmap already in it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190920082543.23444-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block.c  | 22 --
 block/dirty-bitmap.c | 22 ++
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/block.c b/block.c
index 1946fc6f57..d19a4781a3 100644
--- a/block.c
+++ b/block.c
@@ -6582,25 +6582,3 @@ void bdrv_del_child(BlockDriverState *parent_bs, 
BdrvChild *child, Error **errp)
 
 parent_bs->drv->bdrv_del_child(parent_bs, child, errp);
 }
-
-bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
- uint32_t granularity, Error **errp)
-{
-BlockDriver *drv = bs->drv;
-
-if (!drv) {
-error_setg_errno(errp, ENOMEDIUM,
- "Can't store persistent bitmaps to %s",
- bdrv_get_device_or_node_name(bs));
-return false;
-}
-
-if (!drv->bdrv_can_store_new_dirty_bitmap) {
-error_setg_errno(errp, ENOTSUP,
- "Can't store persistent bitmaps to %s",
- bdrv_get_device_or_node_name(bs));
-return false;
-}
-
-return drv->bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp);
-}
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 134e0c9a0c..8f42015db9 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -464,6 +464,28 @@ void bdrv_remove_persistent_dirty_bitmap(BlockDriverState 
*bs,
 }
 }
 
+bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
+ uint32_t granularity, Error **errp)
+{
+BlockDriver *drv = bs->drv;
+
+if (!drv) {
+error_setg_errno(errp, ENOMEDIUM,
+ "Can't store persistent bitmaps to %s",
+ bdrv_get_device_or_node_name(bs));
+return false;
+}
+
+if (!drv->bdrv_can_store_new_dirty_bitmap) {
+error_setg_errno(errp, ENOTSUP,
+ "Can't store persistent bitmaps to %s",
+ bdrv_get_device_or_node_name(bs));
+return false;
+}
+
+return drv->bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp);
+}
+
 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
 bdrv_dirty_bitmap_lock(bitmap);
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v3 08/19] block/dirty-bitmap: refactor bdrv_dirty_bitmap_next

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

bdrv_dirty_bitmap_next is always used in same pattern. So, split it
into _next and _first, instead of combining two functions into one and
add FOR_EACH_DIRTY_BITMAP macro.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-5-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h   |  9 +++--
 block.c|  4 +---
 block/dirty-bitmap.c   | 11 +++
 block/qcow2-bitmap.c   |  8 ++--
 migration/block-dirty-bitmap.c |  4 +---
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 2f9b088e11..257f0f6704 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -96,8 +96,13 @@ bool bdrv_dirty_bitmap_get_autoload(const BdrvDirtyBitmap 
*bitmap);
 bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap);
 bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs);
-BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap);
+
+BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs);
+BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap);
+#define FOR_EACH_DIRTY_BITMAP(bs, bitmap) \
+for (bitmap = bdrv_dirty_bitmap_first(bs); bitmap; \
+ bitmap = bdrv_dirty_bitmap_next(bitmap))
+
 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp);
 int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t offset,
 uint64_t bytes);
diff --git a/block.c b/block.c
index d19a4781a3..5721441697 100644
--- a/block.c
+++ b/block.c
@@ -5390,9 +5390,7 @@ static void coroutine_fn 
bdrv_co_invalidate_cache(BlockDriverState *bs,
 }
 }
 
-for (bm = bdrv_dirty_bitmap_next(bs, NULL); bm;
- bm = bdrv_dirty_bitmap_next(bs, bm))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bm) {
 bdrv_dirty_bitmap_skip_store(bm, false);
 }
 
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 4e5c87a907..6065db8094 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -851,11 +851,14 @@ bool bdrv_has_changed_persistent_bitmaps(BlockDriverState 
*bs)
 return false;
 }
 
-BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap)
+BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs)
 {
-return bitmap == NULL ? QLIST_FIRST(>dirty_bitmaps) :
-QLIST_NEXT(bitmap, list);
+return QLIST_FIRST(>dirty_bitmaps);
+}
+
+BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap)
+{
+return QLIST_NEXT(bitmap, list);
 }
 
 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp)
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 687087d2bc..99812b418b 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1488,9 +1488,7 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 }
 
 /* check constraints and names */
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap != NULL;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 const char *name = bdrv_dirty_bitmap_name(bitmap);
 uint32_t granularity = bdrv_dirty_bitmap_granularity(bitmap);
 Qcow2Bitmap *bm;
@@ -1610,9 +1608,7 @@ int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error 
**errp)
 return -EINVAL;
 }
 
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap != NULL;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
 bdrv_dirty_bitmap_set_readonly(bitmap, true);
 }
diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c
index 793f249aa5..7eafface61 100644
--- a/migration/block-dirty-bitmap.c
+++ b/migration/block-dirty-bitmap.c
@@ -283,9 +283,7 @@ static int init_dirty_bitmap_migration(void)
 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
 const char *name = bdrv_get_device_or_node_name(bs);
 
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 if (!bdrv_dirty_bitmap_name(bitmap)) {
 continue;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v3 11/19] iotests: add test-case to 165 to test reopening qcow2 bitmaps to RW

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Reopening bitmaps to RW was broken prior to previous commit. Check that
it works now.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-4-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 tests/qemu-iotests/165 | 57 --
 tests/qemu-iotests/165.out |  4 +--
 2 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/tests/qemu-iotests/165 b/tests/qemu-iotests/165
index 5650dc7c87..951ea011a2 100755
--- a/tests/qemu-iotests/165
+++ b/tests/qemu-iotests/165
@@ -43,10 +43,10 @@ class TestPersistentDirtyBitmap(iotests.QMPTestCase):
 os.remove(disk)
 
 def mkVm(self):
-return iotests.VM().add_drive(disk)
+return iotests.VM().add_drive(disk, opts='node-name=node0')
 
 def mkVmRo(self):
-return iotests.VM().add_drive(disk, opts='readonly=on')
+return iotests.VM().add_drive(disk, opts='readonly=on,node-name=node0')
 
 def getSha256(self):
 result = self.vm.qmp('x-debug-block-dirty-bitmap-sha256',
@@ -102,6 +102,59 @@ class TestPersistentDirtyBitmap(iotests.QMPTestCase):
 
 self.vm.shutdown()
 
+def test_reopen_rw(self):
+self.vm = self.mkVm()
+self.vm.launch()
+self.qmpAddBitmap()
+
+# Calculate hashes
+
+self.writeRegions(regions1)
+sha256_1 = self.getSha256()
+
+self.writeRegions(regions2)
+sha256_2 = self.getSha256()
+assert sha256_1 != sha256_2 # Otherwise, it's not very interesting.
+
+result = self.vm.qmp('block-dirty-bitmap-clear', node='drive0',
+ name='bitmap0')
+self.assert_qmp(result, 'return', {})
+
+# Start with regions1
+
+self.writeRegions(regions1)
+assert sha256_1 == self.getSha256()
+
+self.vm.shutdown()
+
+self.vm = self.mkVmRo()
+self.vm.launch()
+
+assert sha256_1 == self.getSha256()
+
+# Check that we are in RO mode and can't modify bitmap.
+self.writeRegions(regions2)
+assert sha256_1 == self.getSha256()
+
+# Reopen to RW
+result = self.vm.qmp('x-blockdev-reopen', **{
+'node-name': 'node0',
+'driver': iotests.imgfmt,
+'file': {
+'driver': 'file',
+'filename': disk
+},
+'read-only': False
+})
+self.assert_qmp(result, 'return', {})
+
+# Check that bitmap is reopened to RW and we can write to it.
+self.writeRegions(regions2)
+assert sha256_2 == self.getSha256()
+
+self.vm.shutdown()
+
+
 if __name__ == '__main__':
 iotests.main(supported_fmts=['qcow2'],
  supported_protocols=['file'])
diff --git a/tests/qemu-iotests/165.out b/tests/qemu-iotests/165.out
index ae1213e6f8..fbc63e62f8 100644
--- a/tests/qemu-iotests/165.out
+++ b/tests/qemu-iotests/165.out
@@ -1,5 +1,5 @@
-.
+..
 --
-Ran 1 tests
+Ran 2 tests
 
 OK
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v3 09/19] block: switch reopen queue from QSIMPLEQ to QTAILQ

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

We'll need reverse-foreach in the following commit, QTAILQ support it,
so move to QTAILQ.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Message-id: 20190927122355.7344-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/block.h |  2 +-
 block.c   | 24 
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index 792bb826db..89606bd9f8 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -195,7 +195,7 @@ typedef struct HDGeometry {
 #define BDRV_BLOCK_EOF  0x20
 #define BDRV_BLOCK_RECURSE  0x40
 
-typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) 
BlockReopenQueue;
+typedef QTAILQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
 
 typedef struct BDRVReopenState {
 BlockDriverState *bs;
diff --git a/block.c b/block.c
index 5721441697..0347632c6c 100644
--- a/block.c
+++ b/block.c
@@ -1719,7 +1719,7 @@ typedef struct BlockReopenQueueEntry {
  bool prepared;
  bool perms_checked;
  BDRVReopenState state;
- QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
+ QTAILQ_ENTRY(BlockReopenQueueEntry) entry;
 } BlockReopenQueueEntry;
 
 /*
@@ -1732,7 +1732,7 @@ static int bdrv_reopen_get_flags(BlockReopenQueue *q, 
BlockDriverState *bs)
 BlockReopenQueueEntry *entry;
 
 if (q != NULL) {
-QSIMPLEQ_FOREACH(entry, q, entry) {
+QTAILQ_FOREACH(entry, q, entry) {
 if (entry->state.bs == bs) {
 return entry->state.flags;
 }
@@ -3249,7 +3249,7 @@ static bool bdrv_recurse_has_child(BlockDriverState *bs,
  * Adds a BlockDriverState to a simple queue for an atomic, transactional
  * reopen of multiple devices.
  *
- * bs_queue can either be an existing BlockReopenQueue that has had 
QSIMPLE_INIT
+ * bs_queue can either be an existing BlockReopenQueue that has had QTAILQ_INIT
  * already performed, or alternatively may be NULL a new BlockReopenQueue will
  * be created and initialized. This newly created BlockReopenQueue should be
  * passed back in for subsequent calls that are intended to be of the same
@@ -3290,7 +3290,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 
 if (bs_queue == NULL) {
 bs_queue = g_new0(BlockReopenQueue, 1);
-QSIMPLEQ_INIT(bs_queue);
+QTAILQ_INIT(bs_queue);
 }
 
 if (!options) {
@@ -3298,7 +3298,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 }
 
 /* Check if this BlockDriverState is already in the queue */
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 if (bs == bs_entry->state.bs) {
 break;
 }
@@ -3354,7 +3354,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 
 if (!bs_entry) {
 bs_entry = g_new0(BlockReopenQueueEntry, 1);
-QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
+QTAILQ_INSERT_TAIL(bs_queue, bs_entry, entry);
 } else {
 qobject_unref(bs_entry->state.options);
 qobject_unref(bs_entry->state.explicit_options);
@@ -3455,7 +3455,7 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 
 assert(bs_queue != NULL);
 
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 assert(bs_entry->state.bs->quiesce_counter > 0);
 if (bdrv_reopen_prepare(_entry->state, bs_queue, errp)) {
 goto cleanup;
@@ -3463,7 +3463,7 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 bs_entry->prepared = true;
 }
 
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 BDRVReopenState *state = _entry->state;
 ret = bdrv_check_perm(state->bs, bs_queue, state->perm,
   state->shared_perm, NULL, NULL, errp);
@@ -3489,13 +3489,13 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 /* If we reach this point, we have success and just need to apply the
  * changes
  */
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 bdrv_reopen_commit(_entry->state);
 }
 
 ret = 0;
 cleanup_perm:
-QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
 BDRVReopenState *state = _entry->state;
 
 if (!bs_entry->perms_checked) {
@@ -3512,7 +3512,7 @@ cleanup_perm:
 }
 }
 cleanup:
-QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
 if (ret) {
 if (bs_entry->prepared) {
 bdrv_reopen_abort(_entry->s

[libvirt] [PULL v3 06/19] block/dirty-bitmap: add bs link

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Add bs field to BdrvDirtyBitmap structure. Drop BlockDriverState
parameter from bitmap APIs where possible.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-3-vsement...@virtuozzo.com
[Rebased on top of block-copy. --js]
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h   | 14 +-
 block/backup.c |  8 
 block/block-copy.c |  2 +-
 block/dirty-bitmap.c   | 24 
 block/mirror.c |  4 ++--
 block/qcow2-bitmap.c   |  6 +++---
 blockdev.c |  6 +++---
 migration/block-dirty-bitmap.c |  7 +++
 migration/block.c  |  4 ++--
 9 files changed, 35 insertions(+), 40 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 973056778a..2f9b088e11 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -18,21 +18,18 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
   uint32_t granularity,
   const char *name,
   Error **errp);
-int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
-   BdrvDirtyBitmap *bitmap,
+int bdrv_dirty_bitmap_create_successor(BdrvDirtyBitmap *bitmap,
Error **errp);
-BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BdrvDirtyBitmap *bitmap,
 Error **errp);
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
-   BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BdrvDirtyBitmap *bitmap,
Error **errp);
 void bdrv_dirty_bitmap_enable_successor(BdrvDirtyBitmap *bitmap);
 BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs,
 const char *name);
 int bdrv_dirty_bitmap_check(const BdrvDirtyBitmap *bitmap, uint32_t flags,
 Error **errp);
-void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
+void bdrv_release_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs);
 int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
 Error **errp);
@@ -106,8 +103,7 @@ int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap 
*bitmap, uint64_t offset,
 uint64_t bytes);
 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
uint64_t *offset, uint64_t *bytes);
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BlockDriverState *bs,
-  BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
   Error **errp);
 
 #endif
diff --git a/block/backup.c b/block/backup.c
index 46978c1785..dddcf77f53 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -98,13 +98,13 @@ static void backup_cleanup_sync_bitmap(BackupBlockJob *job, 
int ret)
  * We succeeded, or we always intended to sync the bitmap.
  * Delete this bitmap and install the child.
  */
-bm = bdrv_dirty_bitmap_abdicate(job->source_bs, job->sync_bitmap, 
NULL);
+bm = bdrv_dirty_bitmap_abdicate(job->sync_bitmap, NULL);
 } else {
 /*
  * We failed, or we never intended to sync the bitmap anyway.
  * Merge the successor back into the parent, keeping all data.
  */
-bm = bdrv_reclaim_dirty_bitmap(job->source_bs, job->sync_bitmap, NULL);
+bm = bdrv_reclaim_dirty_bitmap(job->sync_bitmap, NULL);
 }
 
 assert(bm);
@@ -402,7 +402,7 @@ BlockJob *backup_job_create(const char *job_id, 
BlockDriverState *bs,
 }
 
 /* Create a new bitmap, and freeze/disable this one. */
-if (bdrv_dirty_bitmap_create_successor(bs, sync_bitmap, errp) < 0) {
+if (bdrv_dirty_bitmap_create_successor(sync_bitmap, errp) < 0) {
 return NULL;
 }
 }
@@ -472,7 +472,7 @@ BlockJob *backup_job_create(const char *job_id, 
BlockDriverState *bs,
 
  error:
 if (sync_bitmap) {
-bdrv_reclaim_dirty_bitmap(bs, sync_bitmap, NULL);
+bdrv_reclaim_dirty_bitmap(sync_bitmap, NULL);
 }
 if (job) {
 backup_clean(>common.job);
diff --git a/block/block-copy.c b/block/block-copy.c
index 0f76ea1e63..066e3a7274 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -60,7 +60,7 @@ void block_copy_s

[libvirt] [PULL v3 03/19] block/dirty-bitmap: return int from bdrv_remove_persistent_dirty_bitmap

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

It's more comfortable to not deal with local_err.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190920082543.23444-3-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  5 ++---
 include/block/block_int.h|  6 +++---
 include/block/dirty-bitmap.h |  5 ++---
 block/dirty-bitmap.c |  9 +
 block/qcow2-bitmap.c | 18 ++
 blockdev.c   |  7 +++
 6 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index f51f478e34..08b4c15dc4 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -750,9 +750,8 @@ bool qcow2_can_store_new_dirty_bitmap(BlockDriverState *bs,
   const char *name,
   uint32_t granularity,
   Error **errp);
-void qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
-  const char *name,
-  Error **errp);
+int qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char 
*name,
+ Error **errp);
 
 ssize_t coroutine_fn
 qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 05056b308a..6b511dd889 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -557,9 +557,9 @@ struct BlockDriver {
 const char *name,
 uint32_t granularity,
 Error **errp);
-void (*bdrv_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
-const char *name,
-Error **errp);
+int (*bdrv_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
+   const char *name,
+   Error **errp);
 
 /**
  * Register/unregister a buffer for I/O. For example, when the driver is
diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 4b4b731b46..07503b03b5 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -37,9 +37,8 @@ int bdrv_dirty_bitmap_check(const BdrvDirtyBitmap *bitmap, 
uint32_t flags,
 Error **errp);
 void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
 void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs);
-void bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs,
- const char *name,
- Error **errp);
+int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
+Error **errp);
 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_enable_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap);
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 8f42015db9..d1ae2e1922 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -455,13 +455,14 @@ void bdrv_release_named_dirty_bitmaps(BlockDriverState 
*bs)
  * not fail.
  * This function doesn't release corresponding BdrvDirtyBitmap.
  */
-void bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs,
- const char *name,
- Error **errp)
+int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
+Error **errp)
 {
 if (bs->drv && bs->drv->bdrv_remove_persistent_dirty_bitmap) {
-bs->drv->bdrv_remove_persistent_dirty_bitmap(bs, name, errp);
+return bs->drv->bdrv_remove_persistent_dirty_bitmap(bs, name, errp);
 }
+
+return 0;
 }
 
 bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index b2487101ed..9821c1628f 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1404,9 +1404,8 @@ static Qcow2Bitmap *find_bitmap_by_name(Qcow2BitmapList 
*bm_list,
 return NULL;
 }
 
-void qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
-  const char *name,
-  Error **errp)
+int qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char 
*name,
+ Error **errp)
 {
 int ret;
 BDRVQcow2State *s = bs->opaque;
@@ -1416,18 +1415,19 @@ void 
qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
 if (s->nb_bitmaps == 0) {
 /* Absence of the bitmap is 

[libvirt] [PULL v3 18/19] MAINTAINERS: Add Vladimir as a reviewer for bitmaps

[John Snow]

I already try to make sure all bitmaps patches have been reviewed by both
Red Hat and Virtuozzo anyway, so this formalizes the arrangement.

Fam meanwhile is no longer as active, so I am removing him as a co-maintainer
simply to reflect the current practice.

Signed-off-by: John Snow 
Reviewed-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20191005194448.16629-2-js...@redhat.com
---
 MAINTAINERS | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/MAINTAINERS b/MAINTAINERS
index fe4dc51b08..250ce8e7a1 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1816,8 +1816,8 @@ F: qapi/transaction.json
 T: git https://repo.or.cz/qemu/armbru.git block-next
 
 Dirty Bitmaps
-M: Fam Zheng 
 M: John Snow 
+R: Vladimir Sementsov-Ogievskiy 
 L: qemu-bl...@nongnu.org
 S: Supported
 F: util/hbitmap.c
@@ -1826,7 +1826,6 @@ F: include/qemu/hbitmap.h
 F: include/block/dirty-bitmap.h
 F: tests/test-hbitmap.c
 F: docs/interop/bitmaps.rst
-T: git https://github.com/famz/qemu.git bitmaps
 T: git https://github.com/jnsnow/qemu.git bitmaps
 
 Character device backends
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v3 16/19] block/qcow2-bitmap: fix and improve qcow2_reopen_bitmaps_rw

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

- Correct check for write access to file child, and in correct place
  (only if we want to write).
- Support reopen rw -> rw (which will be used in following commit),
  for example, !bdrv_dirty_bitmap_readonly() is not a corruption if
  bitmap is marked IN_USE in the image.
- Consider unexpected bitmap as a corruption and check other
  combinations of in-image and in-RAM bitmaps.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-9-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2-bitmap.c | 77 +---
 1 file changed, 58 insertions(+), 19 deletions(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index f7dfb40256..98294a7696 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1108,18 +1108,14 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error 
**errp)
 Qcow2BitmapList *bm_list;
 Qcow2Bitmap *bm;
 GSList *ro_dirty_bitmaps = NULL;
-int ret = 0;
+int ret = -EINVAL;
+bool need_header_update = false;
 
 if (s->nb_bitmaps == 0) {
 /* No bitmaps - nothing to do */
 return 0;
 }
 
-if (!can_write(bs)) {
-error_setg(errp, "Can't write to the image on reopening bitmaps rw");
-return -EINVAL;
-}
-
 bm_list = bitmap_list_load(bs, s->bitmap_directory_offset,
s->bitmap_directory_size, errp);
 if (bm_list == NULL) {
@@ -1128,32 +1124,75 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error 
**errp)
 
 QSIMPLEQ_FOREACH(bm, bm_list, entry) {
 BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(bs, bm->name);
-if (bitmap == NULL) {
-continue;
-}
 
-if (!bdrv_dirty_bitmap_readonly(bitmap)) {
-error_setg(errp, "Bitmap %s was loaded prior to rw-reopen, but was 
"
-   "not marked as readonly. This is a bug, something went "
-   "wrong. All of the bitmaps may be corrupted", bm->name);
-ret = -EINVAL;
+if (!bitmap) {
+error_setg(errp, "Unexpected bitmap '%s' in image '%s'",
+   bm->name, bs->filename);
 goto out;
 }
 
-bm->flags |= BME_FLAG_IN_USE;
-ro_dirty_bitmaps = g_slist_append(ro_dirty_bitmaps, bitmap);
+if (!(bm->flags & BME_FLAG_IN_USE)) {
+if (!bdrv_dirty_bitmap_readonly(bitmap)) {
+error_setg(errp, "Corruption: bitmap '%s' is not marked IN_USE 
"
+   "in the image '%s' and not marked readonly in RAM",
+   bm->name, bs->filename);
+goto out;
+}
+if (bdrv_dirty_bitmap_inconsistent(bitmap)) {
+error_setg(errp, "Corruption: bitmap '%s' is inconsistent but "
+   "is not marked IN_USE in the image '%s'", bm->name,
+   bs->filename);
+goto out;
+}
+
+bm->flags |= BME_FLAG_IN_USE;
+need_header_update = true;
+} else {
+/*
+ * What if flags already has BME_FLAG_IN_USE ?
+ *
+ * 1. if we are reopening RW -> RW it's OK, of course.
+ * 2. if we are reopening RO -> RW:
+ *   2.1 if @bitmap is inconsistent, it's OK. It means that it was
+ *   inconsistent (IN_USE) when we loaded it
+ *   2.2 if @bitmap is not inconsistent. This seems to be 
impossible
+ *   and implies third party interaction. Let's error-out for
+ *   safety.
+ */
+if (bdrv_dirty_bitmap_readonly(bitmap) &&
+!bdrv_dirty_bitmap_inconsistent(bitmap))
+{
+error_setg(errp, "Corruption: bitmap '%s' is marked IN_USE "
+   "in the image '%s' but it is readonly and "
+   "consistent in RAM",
+   bm->name, bs->filename);
+goto out;
+}
+}
+
+if (bdrv_dirty_bitmap_readonly(bitmap)) {
+ro_dirty_bitmaps = g_slist_append(ro_dirty_bitmaps, bitmap);
+}
 }
 
-if (ro_dirty_bitmaps != NULL) {
+if (need_header_update) {
+if (!can_write(bs->file->bs) || !(bs->file->perm & BLK_PERM_WRITE)) {
+error_setg(errp, "Failed to reopen bitmaps rw: no write access "
+   "the protocol file");
+goto out;
+}
+
 /* in_use flags must be updated */
 ret = update_ext_header_and_dir_in_place(bs, bm_list);
 if (ret < 0) {
-error_setg_errn

[libvirt] [PULL v3 15/19] iotests: add test 260 to check bitmap life after snapshot + commit

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-8-vsement...@virtuozzo.com
[Maintainer edit: removed 260 from auto group per Peter Maydell. --js]
Signed-off-by: John Snow 
---
 tests/qemu-iotests/260 | 89 ++
 tests/qemu-iotests/260.out | 52 ++
 tests/qemu-iotests/group   |  1 +
 3 files changed, 142 insertions(+)
 create mode 100755 tests/qemu-iotests/260
 create mode 100644 tests/qemu-iotests/260.out

diff --git a/tests/qemu-iotests/260 b/tests/qemu-iotests/260
new file mode 100755
index 00..4f6082c9d2
--- /dev/null
+++ b/tests/qemu-iotests/260
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+#
+# Tests for temporary external snapshot when we have bitmaps.
+#
+# Copyright (c) 2019 Virtuozzo International GmbH. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+from iotests import qemu_img_create, file_path, log, filter_qmp_event
+
+iotests.verify_image_format(supported_fmts=['qcow2'])
+
+base, top = file_path('base', 'top')
+size = 64 * 1024 * 3
+
+
+def print_bitmap(msg, vm):
+result = vm.qmp('query-block')['return'][0]
+if 'dirty-bitmaps' in result:
+bitmap = result['dirty-bitmaps'][0]
+log('{}: name={} dirty-clusters={}'.format(msg, bitmap['name'],
+bitmap['count'] // 64 // 1024))
+else:
+log(msg + ': not found')
+
+
+def test(persistent, restart):
+assert persistent or not restart
+log("\nTestcase {}persistent {} restart\n".format(
+'' if persistent else 'non-', 'with' if restart else 'without'))
+
+qemu_img_create('-f', iotests.imgfmt, base, str(size))
+
+vm = iotests.VM().add_drive(base)
+vm.launch()
+
+vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap0',
+   persistent=persistent)
+vm.hmp_qemu_io('drive0', 'write 0 64K')
+print_bitmap('initial bitmap', vm)
+
+vm.qmp_log('blockdev-snapshot-sync', device='drive0', snapshot_file=top,
+   format=iotests.imgfmt, filters=[iotests.filter_qmp_testfiles])
+vm.hmp_qemu_io('drive0', 'write 64K 512')
+print_bitmap('check that no bitmaps are in snapshot', vm)
+
+if restart:
+log("... Restart ...")
+vm.shutdown()
+vm = iotests.VM().add_drive(top)
+vm.launch()
+
+vm.qmp_log('block-commit', device='drive0', top=top,
+   filters=[iotests.filter_qmp_testfiles])
+ev = vm.events_wait((('BLOCK_JOB_READY', None),
+ ('BLOCK_JOB_COMPLETED', None)))
+log(filter_qmp_event(ev))
+if (ev['event'] == 'BLOCK_JOB_COMPLETED'):
+vm.shutdown()
+log(vm.get_log())
+exit()
+
+vm.qmp_log('block-job-complete', device='drive0')
+ev = vm.event_wait('BLOCK_JOB_COMPLETED')
+log(filter_qmp_event(ev))
+print_bitmap('check bitmap after commit', vm)
+
+vm.hmp_qemu_io('drive0', 'write 128K 64K')
+print_bitmap('check updated bitmap', vm)
+
+vm.shutdown()
+
+
+test(persistent=False, restart=False)
+test(persistent=True, restart=False)
+test(persistent=True, restart=True)
diff --git a/tests/qemu-iotests/260.out b/tests/qemu-iotests/260.out
new file mode 100644
index 00..2f0d98d036
--- /dev/null
+++ b/tests/qemu-iotests/260.out
@@ -0,0 +1,52 @@
+
+Testcase non-persistent without restart
+
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap0", "node": 
"drive0", "persistent": false}}
+{"return": {}}
+initial bitmap: name=bitmap0 dirty-clusters=1
+{"execute": "blockdev-snapshot-sync", "arguments": {"device": "drive0", 
"format": "qcow2", "snapshot-file": "TEST_DIR/PID-top"}}
+{"return": {}}
+check that no bitmaps are in snapshot: not found
+{"execute": "block-commit", "arguments": {"device": "drive0", "top": 
"TEST_DIR/PID-top"}}
+{"return": {}}
+{"data": {"device": "drive0", "len": 65536, "offset": 65536, "speed": 0, 
"type": "commit"}, "event": "BLOCK_JOB_READY&

[libvirt] [PULL v3 14/19] block/qcow2-bitmap: do not remove bitmaps on reopen-ro

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

qcow2_reopen_bitmaps_ro wants to store bitmaps and then mark them all
readonly. But the latter don't work, as
qcow2_store_persistent_dirty_bitmaps removes bitmaps after storing.
It's OK for inactivation but bad idea for reopen-ro. And this leads to
the following bug:

Assume we have persistent bitmap 'bitmap0'.
Create external snapshot
  bitmap0 is stored and therefore removed
Commit snapshot
  now we have no bitmaps
Do some writes from guest (*)
  they are not marked in bitmap
Shutdown
Start
  bitmap0 is loaded as valid, but it is actually broken! It misses
  writes (*)
Incremental backup
  it will be inconsistent

So, let's stop removing bitmaps on reopen-ro. But don't rejoice:
reopening bitmaps to rw is broken too, so the whole scenario will not
work after this patch and we can't enable corresponding test cases in
260 iotests still. Reopening bitmaps rw will be fixed in the following
patches.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190927122355.7344-7-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  3 ++-
 block/qcow2-bitmap.c | 49 ++--
 block/qcow2.c|  2 +-
 3 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index 23a9898a54..5cccd87162 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -742,7 +742,8 @@ Qcow2BitmapInfoList 
*qcow2_get_bitmap_info_list(BlockDriverState *bs,
 Error **errp);
 int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp);
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp);
-void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp);
+void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
+  bool release_stored, Error **errp);
 int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error **errp);
 bool qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
  const char *name,
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index ebc1afccd3..f7dfb40256 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1440,7 +1440,32 @@ out:
 return ret;
 }
 
-void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
+/*
+ * qcow2_store_persistent_dirty_bitmaps
+ *
+ * Stores persistent BdrvDirtyBitmap objects.
+ *
+ * @release_stored: if true, release BdrvDirtyBitmap's after storing to the
+ * image. This is used in two cases, both via qcow2_inactivate:
+ * 1. bdrv_close: It's correct to remove bitmaps on close.
+ * 2. migration: If bitmaps are migrated through migration channel via
+ *'dirty-bitmaps' migration capability they are not handled by this code.
+ *Otherwise, it's OK to drop BdrvDirtyBitmap's and reload them on
+ *invalidation.
+ *
+ * Anyway, it's correct to remove BdrvDirtyBitmap's on inactivation, as
+ * inactivation means that we lose control on disk, and therefore on bitmaps,
+ * we should sync them and do not touch more.
+ *
+ * Contrariwise, we don't want to release any bitmaps on just reopen-to-ro,
+ * when we need to store them, as image is still under our control, and it's
+ * good to keep all the bitmaps in read-only mode. Moreover, keeping them
+ * read-only is correct because this is what would happen if we opened the node
+ * readonly to begin with, and whether we opened directly or reopened to that
+ * state shouldn't matter for the state we get afterward.
+ */
+void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
+  bool release_stored, Error **errp)
 {
 BdrvDirtyBitmap *bitmap;
 BDRVQcow2State *s = bs->opaque;
@@ -1551,20 +1576,14 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 g_free(tb);
 }
 
-QSIMPLEQ_FOREACH(bm, bm_list, entry) {
-/* For safety, we remove bitmap after storing.
- * We may be here in two cases:
- * 1. bdrv_close. It's ok to drop bitmap.
- * 2. inactivation. It means migration without 'dirty-bitmaps'
- *capability, so bitmaps are not marked with
- *BdrvDirtyBitmap.migration flags. It's not bad to drop them too,
- *and reload on invalidation.
- */
-if (bm->dirty_bitmap == NULL) {
-continue;
-}
+if (release_stored) {
+QSIMPLEQ_FOREACH(bm, bm_list, entry) {
+if (bm->dirty_bitmap == NULL) {
+continue;
+}
 
-bdrv_release_dirty_bitmap(bm->dirty_bitmap);
+bdrv_release_dirty_bitmap(bm->dirty_bitmap);
+}
 }
 
 success:
@@ -1592,7 +1611,7 @@ int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error 
**errp)
 BdrvDirtyBitmap *bitmap;
 Error *local_err = NULL;
 
-qcow2_store_persistent_dirty_b

[libvirt] [PULL v3 17/19] qcow2-bitmap: move bitmap reopen-rw code to qcow2_reopen_commit

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

The only reason I can imagine for this strange code at the very-end of
bdrv_reopen_commit is the fact that bs->read_only updated after
calling drv->bdrv_reopen_commit in bdrv_reopen_commit. And in the same
time, prior to previous commit, qcow2_reopen_bitmaps_rw did a wrong
check for being writable, when actually it only need writable file
child not self.

So, as it's fixed, let's move things to correct place.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Acked-by: Max Reitz 
Message-id: 20190927122355.7344-10-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/block_int.h |  6 --
 block.c   | 19 ---
 block/qcow2.c | 15 ++-
 3 files changed, 14 insertions(+), 26 deletions(-)

diff --git a/include/block/block_int.h b/include/block/block_int.h
index 32fb493cbb..ca4ccac4c1 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -547,12 +547,6 @@ struct BlockDriver {
  uint64_t parent_perm, uint64_t parent_shared,
  uint64_t *nperm, uint64_t *nshared);
 
-/**
- * Bitmaps should be marked as 'IN_USE' in the image on reopening image
- * as rw. This handler should realize it. It also should unset readonly
- * field of BlockDirtyBitmap's in case of success.
- */
-int (*bdrv_reopen_bitmaps_rw)(BlockDriverState *bs, Error **errp);
 bool (*bdrv_co_can_store_new_dirty_bitmap)(BlockDriverState *bs,
const char *name,
uint32_t granularity,
diff --git a/block.c b/block.c
index cf312258a9..dad5a3d8e0 100644
--- a/block.c
+++ b/block.c
@@ -3935,16 +3935,12 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
 BlockDriver *drv;
 BlockDriverState *bs;
 BdrvChild *child;
-bool old_can_write, new_can_write;
 
 assert(reopen_state != NULL);
 bs = reopen_state->bs;
 drv = bs->drv;
 assert(drv != NULL);
 
-old_can_write =
-!bdrv_is_read_only(bs) && !(bdrv_get_flags(bs) & BDRV_O_INACTIVE);
-
 /* If there are any driver level actions to take */
 if (drv->bdrv_reopen_commit) {
 drv->bdrv_reopen_commit(reopen_state);
@@ -3988,21 +3984,6 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
 }
 
 bdrv_refresh_limits(bs, NULL);
-
-new_can_write =
-!bdrv_is_read_only(bs) && !(bdrv_get_flags(bs) & BDRV_O_INACTIVE);
-if (!old_can_write && new_can_write && drv->bdrv_reopen_bitmaps_rw) {
-Error *local_err = NULL;
-if (drv->bdrv_reopen_bitmaps_rw(bs, _err) < 0) {
-/* This is not fatal, bitmaps just left read-only, so all following
- * writes will fail. User can remove read-only bitmaps to unblock
- * writes.
- */
-error_reportf_err(local_err,
-  "%s: Failed to make dirty bitmaps writable: ",
-  bdrv_get_node_name(bs));
-}
-}
 }
 
 /*
diff --git a/block/qcow2.c b/block/qcow2.c
index 53a025703e..8b05933565 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1835,6 +1835,20 @@ fail:
 static void qcow2_reopen_commit(BDRVReopenState *state)
 {
 qcow2_update_options_commit(state->bs, state->opaque);
+if (state->flags & BDRV_O_RDWR) {
+Error *local_err = NULL;
+
+if (qcow2_reopen_bitmaps_rw(state->bs, _err) < 0) {
+/*
+ * This is not fatal, bitmaps just left read-only, so all following
+ * writes will fail. User can remove read-only bitmaps to unblock
+ * writes or retry reopen.
+ */
+error_reportf_err(local_err,
+  "%s: Failed to make dirty bitmaps writable: ",
+  bdrv_get_node_name(state->bs));
+}
+}
 g_free(state->opaque);
 }
 
@@ -5406,7 +5420,6 @@ BlockDriver bdrv_qcow2 = {
 .bdrv_detach_aio_context  = qcow2_detach_aio_context,
 .bdrv_attach_aio_context  = qcow2_attach_aio_context,
 
-.bdrv_reopen_bitmaps_rw = qcow2_reopen_bitmaps_rw,
 .bdrv_co_can_store_new_dirty_bitmap = qcow2_co_can_store_new_dirty_bitmap,
 .bdrv_co_remove_persistent_dirty_bitmap =
 qcow2_co_remove_persistent_dirty_bitmap,
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v3 19/19] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

This parameter has been deprecated since 2.12.0 and is eligible for
removal. Remove this parameter as it is actually completely ignored;
let's not give false hope.

Signed-off-by: John Snow 
Reviewed-by: Eric Blake 
Reviewed-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20191002232411.29968-1-js...@redhat.com
---
 qemu-deprecated.texi | 20 +++-
 qapi/block-core.json |  6 +-
 blockdev.c   |  6 --
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index 01245e0b1c..7239e0959d 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -149,11 +149,6 @@ QEMU 4.1 has three options, please migrate to one of these 
three:
 
 @section QEMU Machine Protocol (QMP) commands
 
-@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
-
-"autoload" parameter is now ignored. All bitmaps are automatically loaded
-from qcow2 images.
-
 @subsection query-block result field dirty-bitmaps[i].status (since 4.0)
 
 The ``status'' field of the ``BlockDirtyInfo'' structure, returned by
@@ -356,3 +351,18 @@ existing CPU models.  Management software that needs 
runnability
 guarantees must resolve the CPU model aliases using te
 ``alias-of'' field returned by the ``query-cpu-definitions'' QMP
 command.
+
+
+@node Recently removed features
+@appendix Recently removed features
+
+What follows is a record of recently removed, formerly deprecated
+features that serves as a record for users who have encountered
+trouble after a recent upgrade.
+
+@section QEMU Machine Protocol (QMP) commands
+
+@subsection block-dirty-bitmap-add "autoload" parameter (since 4.2.0)
+
+The "autoload" parameter has been ignored since 2.12.0. All bitmaps
+are automatically loaded from qcow2 images.
diff --git a/qapi/block-core.json b/qapi/block-core.json
index f66553aac7..b274aef713 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -2052,10 +2052,6 @@
 #  Qcow2 disks support persistent bitmaps. Default is false for
 #  block-dirty-bitmap-add. (Since: 2.10)
 #
-# @autoload: ignored and deprecated since 2.12.
-#Currently, all dirty tracking bitmaps are loaded from Qcow2 on
-#open.
-#
 # @disabled: the bitmap is created in the disabled state, which means that
 #it will not track drive changes. The bitmap may be enabled with
 #block-dirty-bitmap-enable. Default is false. (Since: 4.0)
@@ -2064,7 +2060,7 @@
 ##
 { 'struct': 'BlockDirtyBitmapAdd',
   'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32',
-'*persistent': 'bool', '*autoload': 'bool', '*disabled': 'bool' } }
+'*persistent': 'bool', '*disabled': 'bool' } }
 
 ##
 # @BlockDirtyBitmapMergeSource:
diff --git a/blockdev.c b/blockdev.c
index d77e809623..03c7cd7651 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -1966,7 +1966,6 @@ static void block_dirty_bitmap_add_prepare(BlkActionState 
*common,
 qmp_block_dirty_bitmap_add(action->node, action->name,
action->has_granularity, action->granularity,
action->has_persistent, action->persistent,
-   action->has_autoload, action->autoload,
action->has_disabled, action->disabled,
_err);
 
@@ -2858,7 +2857,6 @@ out:
 void qmp_block_dirty_bitmap_add(const char *node, const char *name,
 bool has_granularity, uint32_t granularity,
 bool has_persistent, bool persistent,
-bool has_autoload, bool autoload,
 bool has_disabled, bool disabled,
 Error **errp)
 {
@@ -2890,10 +2888,6 @@ void qmp_block_dirty_bitmap_add(const char *node, const 
char *name,
 persistent = false;
 }
 
-if (has_autoload) {
-warn_report("Autoload option is deprecated and its value is ignored");
-}
-
 if (!has_disabled) {
 disabled = false;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v3 13/19] block/qcow2-bitmap: drop qcow2_reopen_bitmaps_rw_hint()

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

The function is unused, drop it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190927122355.7344-6-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  2 --
 block/qcow2-bitmap.c | 15 +--
 2 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index 0f3d9b088e..23a9898a54 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -740,8 +740,6 @@ int qcow2_check_bitmaps_refcounts(BlockDriverState *bs, 
BdrvCheckResult *res,
 bool qcow2_load_dirty_bitmaps(BlockDriverState *bs, Error **errp);
 Qcow2BitmapInfoList *qcow2_get_bitmap_info_list(BlockDriverState *bs,
 Error **errp);
-int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, bool *header_updated,
- Error **errp);
 int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp);
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp);
 void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp);
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 6dfc083548..ebc1afccd3 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1102,8 +1102,7 @@ Qcow2BitmapInfoList 
*qcow2_get_bitmap_info_list(BlockDriverState *bs,
 return list;
 }
 
-int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, bool *header_updated,
- Error **errp)
+int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)
 {
 BDRVQcow2State *s = bs->opaque;
 Qcow2BitmapList *bm_list;
@@ -,10 +1110,6 @@ int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, 
bool *header_updated,
 GSList *ro_dirty_bitmaps = NULL;
 int ret = 0;
 
-if (header_updated != NULL) {
-*header_updated = false;
-}
-
 if (s->nb_bitmaps == 0) {
 /* No bitmaps - nothing to do */
 return 0;
@@ -1156,9 +1151,6 @@ int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, 
bool *header_updated,
 error_setg_errno(errp, -ret, "Can't update bitmap directory");
 goto out;
 }
-if (header_updated != NULL) {
-*header_updated = true;
-}
 g_slist_foreach(ro_dirty_bitmaps, set_readonly_helper, false);
 }
 
@@ -1169,11 +1161,6 @@ out:
 return ret;
 }
 
-int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)
-{
-return qcow2_reopen_bitmaps_rw_hint(bs, NULL, errp);
-}
-
 /* Checks to see if it's safe to resize bitmaps */
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp)
 {
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 1/9] IDE: deprecate ide-drive

[John Snow]

It's an old compatibility shim that just delegates to ide-cd or ide-hd.
I'd like to refactor these some day, and getting rid of the super-object
will make that easier.

Either way, we don't need this.

Signed-off-by: John Snow 
Reviewed-by: Thomas Huth 
Reviewed-by: Markus Armbruster 
ACKed-by: Peter Krempa 
Message-id: 20191009224303.10232-2-js...@redhat.com
Signed-off-by: John Snow 
---
 qemu-deprecated.texi  | 5 +
 hw/ide/qdev.c | 3 +++
 tests/qemu-iotests/051.pc.out | 6 --
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index f727bd3932..296bfc93a3 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -254,6 +254,11 @@ quite a bit. It will be removed without replacement unless 
some users speaks
 up at the @email{qemu-devel@@nongnu.org} mailing list with information about
 their usecases.
 
+@subsection ide-drive (since 4.2)
+
+The 'ide-drive' device is deprecated. Users should use 'ide-hd' or
+'ide-cd' as appropriate to get an IDE hard disk or CD-ROM as needed.
+
 @section System emulator machines
 
 @subsection pc-0.12, pc-0.13, pc-0.14 and pc-0.15 (since 4.0)
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 6fba6b62b8..3666e59721 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -279,6 +279,9 @@ static void ide_drive_realize(IDEDevice *dev, Error **errp)
 {
 DriveInfo *dinfo = NULL;
 
+warn_report("'ide-drive' is deprecated, "
+"please use 'ide-hd' or 'ide-cd' instead");
+
 if (dev->conf.blk) {
 dinfo = blk_legacy_dinfo(dev->conf.blk);
 }
diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out
index 000557c7c8..34849dd172 100644
--- a/tests/qemu-iotests/051.pc.out
+++ b/tests/qemu-iotests/051.pc.out
@@ -158,7 +158,8 @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 Testing: -drive if=none,id=disk -device ide-drive,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive 
is empty
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: warning: 'ide-drive' is 
deprecated, please use 'ide-hd' or 'ide-cd' instead
+QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive is empty
 
 Testing: -drive if=none,id=disk -device ide-hd,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
@@ -228,7 +229,8 @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device 
ide-drive,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Block node is read-only
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: warning: 'ide-drive' is 
deprecated, please use 'ide-hd' or 'ide-cd' instead
+QEMU_PROG: -device ide-drive,drive=disk: Block node is read-only
 
 Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device 
ide-hd,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 0/9] Ide patches

[John Snow]

The following changes since commit 68d8ef4ec540682c3538d4963e836e43a211dd17:

  Merge remote-tracking branch 'remotes/stsquad/tags/pull-tcg-plugins-281019-4' 
into staging (2019-10-30 14:10:32 +)

are available in the Git repository at:

  https://github.com/jnsnow/qemu.git tags/ide-pull-request

for you to fetch changes up to c35564caf20e8d3431786dddf0fa513daa7d7f3c:

  hd-geo-test: Add tests for lchs override (2019-10-31 06:11:34 -0400)


Pull request



John Snow (1):
  IDE: deprecate ide-drive

Sam Eiderman (8):
  block: Refactor macros - fix tabbing
  block: Support providing LCHS from user
  bootdevice: Add interface to gather LCHS
  scsi: Propagate unrealize() callback to scsi-hd
  bootdevice: Gather LCHS from all relevant devices
  bootdevice: Refactor get_boot_devices_list
  bootdevice: FW_CFG interface for LCHS values
  hd-geo-test: Add tests for lchs override

 qemu-deprecated.texi  |   5 +
 include/hw/block/block.h  |  22 +-
 include/hw/scsi/scsi.h|   1 +
 include/sysemu/sysemu.h   |   4 +
 bootdevice.c  | 147 +++--
 hw/block/virtio-blk.c |   6 +
 hw/ide/qdev.c |  10 +-
 hw/nvram/fw_cfg.c |  14 +-
 hw/scsi/scsi-bus.c|  16 +
 hw/scsi/scsi-disk.c   |  12 +
 tests/hd-geo-test.c   | 551 ++
 tests/Makefile.include|   2 +-
 tests/qemu-iotests/051.pc.out |   6 +-
 13 files changed, 753 insertions(+), 43 deletions(-)

-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 2/9] block: Refactor macros - fix tabbing

[John Snow]

From: Sam Eiderman 

Fixing tabbing in block related macros.

Signed-off-by: Sam Eiderman 
Signed-off-by: Sam Eiderman 
Reviewed-by: Karl Heubaum 
Reviewed-by: Arbel Moshe 
Reviewed-by: Philippe Mathieu-Daudé 
Signed-off-by: John Snow 
---
 include/hw/block/block.h | 16 
 hw/ide/qdev.c|  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/include/hw/block/block.h b/include/hw/block/block.h
index 607539057a..fd55a30bca 100644
--- a/include/hw/block/block.h
+++ b/include/hw/block/block.h
@@ -50,21 +50,21 @@ static inline unsigned int get_physical_block_exp(BlockConf 
*conf)
   _conf.logical_block_size),\
 DEFINE_PROP_BLOCKSIZE("physical_block_size", _state,\
   _conf.physical_block_size),   \
-DEFINE_PROP_UINT16("min_io_size", _state, _conf.min_io_size, 0),  \
+DEFINE_PROP_UINT16("min_io_size", _state, _conf.min_io_size, 0),\
 DEFINE_PROP_UINT32("opt_io_size", _state, _conf.opt_io_size, 0),\
-DEFINE_PROP_UINT32("discard_granularity", _state, \
-   _conf.discard_granularity, -1), \
-DEFINE_PROP_ON_OFF_AUTO("write-cache", _state, _conf.wce, \
-ON_OFF_AUTO_AUTO), \
+DEFINE_PROP_UINT32("discard_granularity", _state,   \
+   _conf.discard_granularity, -1),  \
+DEFINE_PROP_ON_OFF_AUTO("write-cache", _state, _conf.wce,   \
+ON_OFF_AUTO_AUTO),  \
 DEFINE_PROP_BOOL("share-rw", _state, _conf.share_rw, false)
 
 #define DEFINE_BLOCK_PROPERTIES(_state, _conf)  \
 DEFINE_PROP_DRIVE("drive", _state, _conf.blk),  \
 DEFINE_BLOCK_PROPERTIES_BASE(_state, _conf)
 
-#define DEFINE_BLOCK_CHS_PROPERTIES(_state, _conf)  \
-DEFINE_PROP_UINT32("cyls", _state, _conf.cyls, 0),  \
-DEFINE_PROP_UINT32("heads", _state, _conf.heads, 0), \
+#define DEFINE_BLOCK_CHS_PROPERTIES(_state, _conf)  \
+DEFINE_PROP_UINT32("cyls", _state, _conf.cyls, 0),  \
+DEFINE_PROP_UINT32("heads", _state, _conf.heads, 0),\
 DEFINE_PROP_UINT32("secs", _state, _conf.secs, 0)
 
 #define DEFINE_BLOCK_ERROR_PROPERTIES(_state, _conf)\
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 3666e59721..85cca6ec38 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -293,7 +293,7 @@ static void ide_drive_realize(IDEDevice *dev, Error **errp)
 DEFINE_BLOCK_PROPERTIES(IDEDrive, dev.conf),\
 DEFINE_BLOCK_ERROR_PROPERTIES(IDEDrive, dev.conf),  \
 DEFINE_PROP_STRING("ver",  IDEDrive, dev.version),  \
-DEFINE_PROP_UINT64("wwn",  IDEDrive, dev.wwn, 0),\
+DEFINE_PROP_UINT64("wwn",  IDEDrive, dev.wwn, 0),   \
 DEFINE_PROP_STRING("serial",  IDEDrive, dev.serial),\
 DEFINE_PROP_STRING("model", IDEDrive, dev.model)
 
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 3/9] block: Support providing LCHS from user

[John Snow]

From: Sam Eiderman 

Add logical geometry variables to BlockConf.

A user can now supply "lcyls", "lheads" & "lsecs" for any HD device
that supports CHS ("cyls", "heads", "secs").

These devices include:
* ide-hd
* scsi-hd
* virtio-blk-pci

In future commits we will use the provided LCHS and pass it to the BIOS
through fw_cfg to be supplied using INT13 routines.

Reviewed-by: Karl Heubaum 
Reviewed-by: Arbel Moshe 
Signed-off-by: Sam Eiderman 
Signed-off-by: Sam Eiderman 
Reviewed-by: Philippe Mathieu-Daudé 
Tested-by: Philippe Mathieu-Daudé 
Signed-off-by: John Snow 
---
 include/hw/block/block.h | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/include/hw/block/block.h b/include/hw/block/block.h
index fd55a30bca..d7246f3862 100644
--- a/include/hw/block/block.h
+++ b/include/hw/block/block.h
@@ -26,6 +26,7 @@ typedef struct BlockConf {
 uint32_t discard_granularity;
 /* geometry, not all devices use this */
 uint32_t cyls, heads, secs;
+uint32_t lcyls, lheads, lsecs;
 OnOffAuto wce;
 bool share_rw;
 BlockdevOnError rerror;
@@ -65,7 +66,10 @@ static inline unsigned int get_physical_block_exp(BlockConf 
*conf)
 #define DEFINE_BLOCK_CHS_PROPERTIES(_state, _conf)  \
 DEFINE_PROP_UINT32("cyls", _state, _conf.cyls, 0),  \
 DEFINE_PROP_UINT32("heads", _state, _conf.heads, 0),\
-DEFINE_PROP_UINT32("secs", _state, _conf.secs, 0)
+DEFINE_PROP_UINT32("secs", _state, _conf.secs, 0),  \
+DEFINE_PROP_UINT32("lcyls", _state, _conf.lcyls, 0),\
+DEFINE_PROP_UINT32("lheads", _state, _conf.lheads, 0),  \
+DEFINE_PROP_UINT32("lsecs", _state, _conf.lsecs, 0)
 
 #define DEFINE_BLOCK_ERROR_PROPERTIES(_state, _conf)\
 DEFINE_PROP_BLOCKDEV_ON_ERROR("rerror", _state, _conf.rerror,   \
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 4/9] bootdevice: Add interface to gather LCHS

[John Snow]

From: Sam Eiderman 

Add an interface to provide direct logical CHS values for boot devices.
We will use this interface in the next commits.

Reviewed-by: Karl Heubaum 
Reviewed-by: Arbel Moshe 
Signed-off-by: Sam Eiderman 
Signed-off-by: Sam Eiderman 
Reviewed-by: Philippe Mathieu-Daudé 
Tested-by: Philippe Mathieu-Daudé 
Signed-off-by: John Snow 
---
 include/sysemu/sysemu.h |  3 +++
 bootdevice.c| 55 +
 2 files changed, 58 insertions(+)

diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 44f18eb739..5bc5c79cbc 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -103,6 +103,9 @@ void device_add_bootindex_property(Object *obj, int32_t 
*bootindex,
DeviceState *dev, Error **errp);
 void restore_boot_order(void *opaque);
 void validate_bootdevices(const char *devices, Error **errp);
+void add_boot_device_lchs(DeviceState *dev, const char *suffix,
+  uint32_t lcyls, uint32_t lheads, uint32_t lsecs);
+void del_boot_device_lchs(DeviceState *dev, const char *suffix);
 
 /* handler to set the boot_device order for a specific type of MachineClass */
 typedef void QEMUBootSetHandler(void *opaque, const char *boot_order,
diff --git a/bootdevice.c b/bootdevice.c
index 1d225202f9..bc5e1c2de4 100644
--- a/bootdevice.c
+++ b/bootdevice.c
@@ -343,3 +343,58 @@ void device_add_bootindex_property(Object *obj, int32_t 
*bootindex,
 /* initialize devices' bootindex property to -1 */
 object_property_set_int(obj, -1, name, NULL);
 }
+
+typedef struct FWLCHSEntry FWLCHSEntry;
+
+struct FWLCHSEntry {
+QTAILQ_ENTRY(FWLCHSEntry) link;
+DeviceState *dev;
+char *suffix;
+uint32_t lcyls;
+uint32_t lheads;
+uint32_t lsecs;
+};
+
+static QTAILQ_HEAD(, FWLCHSEntry) fw_lchs =
+QTAILQ_HEAD_INITIALIZER(fw_lchs);
+
+void add_boot_device_lchs(DeviceState *dev, const char *suffix,
+  uint32_t lcyls, uint32_t lheads, uint32_t lsecs)
+{
+FWLCHSEntry *node;
+
+if (!lcyls && !lheads && !lsecs) {
+return;
+}
+
+assert(dev != NULL || suffix != NULL);
+
+node = g_malloc0(sizeof(FWLCHSEntry));
+node->suffix = g_strdup(suffix);
+node->dev = dev;
+node->lcyls = lcyls;
+node->lheads = lheads;
+node->lsecs = lsecs;
+
+QTAILQ_INSERT_TAIL(_lchs, node, link);
+}
+
+void del_boot_device_lchs(DeviceState *dev, const char *suffix)
+{
+FWLCHSEntry *i;
+
+if (dev == NULL) {
+return;
+}
+
+QTAILQ_FOREACH(i, _lchs, link) {
+if ((!suffix || !g_strcmp0(i->suffix, suffix)) &&
+ i->dev == dev) {
+QTAILQ_REMOVE(_lchs, i, link);
+g_free(i->suffix);
+g_free(i);
+
+break;
+}
+}
+}
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 7/9] bootdevice: Refactor get_boot_devices_list

[John Snow]

From: Sam Eiderman 

Move device name construction to a separate function.

We will reuse this function in the following commit to pass logical CHS
parameters through fw_cfg much like we currently pass bootindex.

Reviewed-by: Karl Heubaum 
Reviewed-by: Arbel Moshe 
Reviewed-by: Philippe Mathieu-Daudé 
Signed-off-by: Sam Eiderman 
Signed-off-by: Sam Eiderman 
Tested-by: Philippe Mathieu-Daudé 
Signed-off-by: John Snow 
---
 bootdevice.c | 61 +---
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/bootdevice.c b/bootdevice.c
index bc5e1c2de4..2cf6b37c57 100644
--- a/bootdevice.c
+++ b/bootdevice.c
@@ -202,6 +202,39 @@ DeviceState *get_boot_device(uint32_t position)
 return res;
 }
 
+static char *get_boot_device_path(DeviceState *dev, bool ignore_suffixes,
+  const char *suffix)
+{
+char *devpath = NULL, *s = NULL, *d, *bootpath;
+
+if (dev) {
+devpath = qdev_get_fw_dev_path(dev);
+assert(devpath);
+}
+
+if (!ignore_suffixes) {
+if (dev) {
+d = qdev_get_own_fw_dev_path_from_handler(dev->parent_bus, dev);
+if (d) {
+assert(!suffix);
+s = d;
+} else {
+s = g_strdup(suffix);
+}
+} else {
+s = g_strdup(suffix);
+}
+}
+
+bootpath = g_strdup_printf("%s%s",
+   devpath ? devpath : "",
+   s ? s : "");
+g_free(devpath);
+g_free(s);
+
+return bootpath;
+}
+
 /*
  * This function returns null terminated string that consist of new line
  * separated device paths.
@@ -218,36 +251,10 @@ char *get_boot_devices_list(size_t *size)
 bool ignore_suffixes = mc->ignore_boot_device_suffixes;
 
 QTAILQ_FOREACH(i, _boot_order, link) {
-char *devpath = NULL,  *suffix = NULL;
 char *bootpath;
-char *d;
 size_t len;
 
-if (i->dev) {
-devpath = qdev_get_fw_dev_path(i->dev);
-assert(devpath);
-}
-
-if (!ignore_suffixes) {
-if (i->dev) {
-d = qdev_get_own_fw_dev_path_from_handler(i->dev->parent_bus,
-  i->dev);
-if (d) {
-assert(!i->suffix);
-suffix = d;
-} else {
-suffix = g_strdup(i->suffix);
-}
-} else {
-suffix = g_strdup(i->suffix);
-}
-}
-
-bootpath = g_strdup_printf("%s%s",
-   devpath ? devpath : "",
-   suffix ? suffix : "");
-g_free(devpath);
-g_free(suffix);
+bootpath = get_boot_device_path(i->dev, ignore_suffixes, i->suffix);
 
 if (total) {
 list[total-1] = '\n';
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 8/9] bootdevice: FW_CFG interface for LCHS values

[John Snow]

From: Sam Eiderman 

Using fw_cfg, supply logical CHS values directly from QEMU to the BIOS.

Non-standard logical geometries break under QEMU.

A virtual disk which contains an operating system which depends on
logical geometries (consistent values being reported from BIOS INT13
AH=08) will most likely break under QEMU/SeaBIOS if it has non-standard
logical geometries - for example 56 SPT (sectors per track).
No matter what QEMU will report - SeaBIOS, for large enough disks - will
use LBA translation, which will report 63 SPT instead.

In addition we cannot force SeaBIOS to rely on physical geometries at
all. A virtio-blk-pci virtual disk with 255 phyiscal heads cannot
report more than 16 physical heads when moved to an IDE controller,
since the ATA spec allows a maximum of 16 heads - this is an artifact of
virtualization.

By supplying the logical geometries directly we are able to support such
"exotic" disks.

We serialize this information in a similar way to the "bootorder"
interface.
The new fw_cfg entry is "bios-geometry".

Reviewed-by: Karl Heubaum 
Reviewed-by: Arbel Moshe 
Reviewed-by: Philippe Mathieu-Daudé 
Signed-off-by: Sam Eiderman 
Signed-off-by: Sam Eiderman 
Tested-by: Philippe Mathieu-Daudé 
Signed-off-by: John Snow 
---
 include/sysemu/sysemu.h |  1 +
 bootdevice.c| 31 +++
 hw/nvram/fw_cfg.c   | 14 +++---
 3 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 5bc5c79cbc..80c57fdc4e 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -106,6 +106,7 @@ void validate_bootdevices(const char *devices, Error 
**errp);
 void add_boot_device_lchs(DeviceState *dev, const char *suffix,
   uint32_t lcyls, uint32_t lheads, uint32_t lsecs);
 void del_boot_device_lchs(DeviceState *dev, const char *suffix);
+char *get_boot_devices_lchs_list(size_t *size);
 
 /* handler to set the boot_device order for a specific type of MachineClass */
 typedef void QEMUBootSetHandler(void *opaque, const char *boot_order,
diff --git a/bootdevice.c b/bootdevice.c
index 2cf6b37c57..03aaffcc8d 100644
--- a/bootdevice.c
+++ b/bootdevice.c
@@ -405,3 +405,34 @@ void del_boot_device_lchs(DeviceState *dev, const char 
*suffix)
 }
 }
 }
+
+char *get_boot_devices_lchs_list(size_t *size)
+{
+FWLCHSEntry *i;
+size_t total = 0;
+char *list = NULL;
+
+QTAILQ_FOREACH(i, _lchs, link) {
+char *bootpath;
+char *chs_string;
+size_t len;
+
+bootpath = get_boot_device_path(i->dev, false, i->suffix);
+chs_string = g_strdup_printf("%s %" PRIu32 " %" PRIu32 " %" PRIu32,
+ bootpath, i->lcyls, i->lheads, i->lsecs);
+
+if (total) {
+list[total - 1] = '\n';
+}
+len = strlen(chs_string) + 1;
+list = g_realloc(list, total + len);
+memcpy([total], chs_string, len);
+total += len;
+g_free(chs_string);
+g_free(bootpath);
+}
+
+*size = total;
+
+return list;
+}
diff --git a/hw/nvram/fw_cfg.c b/hw/nvram/fw_cfg.c
index aef1727250..44a3c19326 100644
--- a/hw/nvram/fw_cfg.c
+++ b/hw/nvram/fw_cfg.c
@@ -949,13 +949,21 @@ void *fw_cfg_modify_file(FWCfgState *s, const char 
*filename,
 
 static void fw_cfg_machine_reset(void *opaque)
 {
+MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
+FWCfgState *s = opaque;
 void *ptr;
 size_t len;
-FWCfgState *s = opaque;
-char *bootindex = get_boot_devices_list();
+char *buf;
 
-ptr = fw_cfg_modify_file(s, "bootorder", (uint8_t *)bootindex, len);
+buf = get_boot_devices_list();
+ptr = fw_cfg_modify_file(s, "bootorder", (uint8_t *)buf, len);
 g_free(ptr);
+
+if (!mc->legacy_fw_cfg_order) {
+buf = get_boot_devices_lchs_list();
+ptr = fw_cfg_modify_file(s, "bios-geometry", (uint8_t *)buf, len);
+g_free(ptr);
+}
 }
 
 static void fw_cfg_machine_ready(struct Notifier *n, void *data)
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 6/9] bootdevice: Gather LCHS from all relevant devices

[John Snow]

From: Sam Eiderman 

Relevant devices are:
* ide-hd (and ide-cd, ide-drive)
* scsi-hd (and scsi-cd, scsi-disk, scsi-block)
* virtio-blk-pci

We do not call del_boot_device_lchs() for ide-* since we don't need to -
IDE block devices do not support unplugging.

Reviewed-by: Karl Heubaum 
Reviewed-by: Arbel Moshe 
Signed-off-by: Sam Eiderman 
Signed-off-by: Sam Eiderman 
Reviewed-by: Philippe Mathieu-Daudé 
Tested-by: Philippe Mathieu-Daudé 
Signed-off-by: John Snow 
---
 hw/block/virtio-blk.c |  6 ++
 hw/ide/qdev.c |  5 +
 hw/scsi/scsi-disk.c   | 12 
 3 files changed, 23 insertions(+)

diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index 9fa2eaf890..4c357d2928 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -1200,6 +1200,11 @@ static void virtio_blk_device_realize(DeviceState *dev, 
Error **errp)
 blk_set_guest_block_size(s->blk, s->conf.conf.logical_block_size);
 
 blk_iostatus_enable(s->blk);
+
+add_boot_device_lchs(dev, "/disk@0,0",
+ conf->conf.lcyls,
+ conf->conf.lheads,
+ conf->conf.lsecs);
 }
 
 static void virtio_blk_device_unrealize(DeviceState *dev, Error **errp)
@@ -1210,6 +1215,7 @@ static void virtio_blk_device_unrealize(DeviceState *dev, 
Error **errp)
 unsigned i;
 
 blk_drain(s->blk);
+del_boot_device_lchs(dev, "/disk@0,0");
 virtio_blk_data_plane_destroy(s->dataplane);
 s->dataplane = NULL;
 for (i = 0; i < conf->num_queues; i++) {
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 85cca6ec38..374a791a45 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -220,6 +220,11 @@ static void ide_dev_initfn(IDEDevice *dev, IDEDriveKind 
kind, Error **errp)
 
 add_boot_device_path(dev->conf.bootindex, >qdev,
  dev->unit ? "/disk@1" : "/disk@0");
+
+add_boot_device_lchs(>qdev, dev->unit ? "/disk@1" : "/disk@0",
+ dev->conf.lcyls,
+ dev->conf.lheads,
+ dev->conf.lsecs);
 }
 
 static void ide_dev_get_bootindex(Object *obj, Visitor *v, const char *name,
diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c
index 68b1675fd9..07fb5ebdf1 100644
--- a/hw/scsi/scsi-disk.c
+++ b/hw/scsi/scsi-disk.c
@@ -35,6 +35,7 @@
 #include "hw/block/block.h"
 #include "hw/qdev-properties.h"
 #include "sysemu/dma.h"
+#include "sysemu/sysemu.h"
 #include "qemu/cutils.h"
 #include "trace.h"
 
@@ -2414,6 +2415,16 @@ static void scsi_realize(SCSIDevice *dev, Error **errp)
 blk_set_guest_block_size(s->qdev.conf.blk, s->qdev.blocksize);
 
 blk_iostatus_enable(s->qdev.conf.blk);
+
+add_boot_device_lchs(>qdev, NULL,
+ dev->conf.lcyls,
+ dev->conf.lheads,
+ dev->conf.lsecs);
+}
+
+static void scsi_unrealize(SCSIDevice *dev, Error **errp)
+{
+del_boot_device_lchs(>qdev, NULL);
 }
 
 static void scsi_hd_realize(SCSIDevice *dev, Error **errp)
@@ -3018,6 +3029,7 @@ static void scsi_hd_class_initfn(ObjectClass *klass, void 
*data)
 SCSIDeviceClass *sc = SCSI_DEVICE_CLASS(klass);
 
 sc->realize  = scsi_hd_realize;
+sc->unrealize= scsi_unrealize;
 sc->alloc_req= scsi_new_request;
 sc->unit_attention_reported = scsi_disk_unit_attention_reported;
 dc->desc = "virtual SCSI disk";
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 5/9] scsi: Propagate unrealize() callback to scsi-hd

[John Snow]

From: Sam Eiderman 

We will need to add LCHS removal logic to scsi-hd's unrealize() in the
next commit.

Reviewed-by: Karl Heubaum 
Reviewed-by: Arbel Moshe 
Reviewed-by: Philippe Mathieu-Daudé 
Signed-off-by: Sam Eiderman 
Signed-off-by: Sam Eiderman 
Tested-by: Philippe Mathieu-Daudé 
Signed-off-by: John Snow 
---
 include/hw/scsi/scsi.h |  1 +
 hw/scsi/scsi-bus.c | 16 
 2 files changed, 17 insertions(+)

diff --git a/include/hw/scsi/scsi.h b/include/hw/scsi/scsi.h
index d77a92361b..332ef602f4 100644
--- a/include/hw/scsi/scsi.h
+++ b/include/hw/scsi/scsi.h
@@ -59,6 +59,7 @@ struct SCSIRequest {
 typedef struct SCSIDeviceClass {
 DeviceClass parent_class;
 void (*realize)(SCSIDevice *dev, Error **errp);
+void (*unrealize)(SCSIDevice *dev, Error **errp);
 int (*parse_cdb)(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf,
  void *hba_private);
 SCSIRequest *(*alloc_req)(SCSIDevice *s, uint32_t tag, uint32_t lun,
diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c
index bccb7cc4c6..359d50d6d0 100644
--- a/hw/scsi/scsi-bus.c
+++ b/hw/scsi/scsi-bus.c
@@ -59,6 +59,14 @@ static void scsi_device_realize(SCSIDevice *s, Error **errp)
 }
 }
 
+static void scsi_device_unrealize(SCSIDevice *s, Error **errp)
+{
+SCSIDeviceClass *sc = SCSI_DEVICE_GET_CLASS(s);
+if (sc->unrealize) {
+sc->unrealize(s, errp);
+}
+}
+
 int scsi_bus_parse_cdb(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf,
void *hba_private)
 {
@@ -217,12 +225,20 @@ static void scsi_qdev_realize(DeviceState *qdev, Error 
**errp)
 static void scsi_qdev_unrealize(DeviceState *qdev, Error **errp)
 {
 SCSIDevice *dev = SCSI_DEVICE(qdev);
+Error *local_err = NULL;
 
 if (dev->vmsentry) {
 qemu_del_vm_change_state_handler(dev->vmsentry);
 }
 
 scsi_device_purge_requests(dev, SENSE_CODE(NO_SENSE));
+
+scsi_device_unrealize(dev, _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
+
 blockdev_mark_auto_del(dev->conf.blk);
 }
 
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 9/9] hd-geo-test: Add tests for lchs override

[John Snow]

From: Sam Eiderman 

Add QTest tests to check the logical geometry override option.

The tests in hd-geo-test are out of date - they only test IDE and do not
test interesting MBRs.

Creating qcow2 disks with specific size and MBR layout is currently
unused - we only use a default empty MBR.

Reviewed-by: Karl Heubaum 
Reviewed-by: Arbel Moshe 
Signed-off-by: Sam Eiderman 
Signed-off-by: Sam Eiderman 
Signed-off-by: John Snow 
---
 tests/hd-geo-test.c| 551 +
 tests/Makefile.include |   2 +-
 2 files changed, 552 insertions(+), 1 deletion(-)

diff --git a/tests/hd-geo-test.c b/tests/hd-geo-test.c
index 62eb624726..7e86c5416c 100644
--- a/tests/hd-geo-test.c
+++ b/tests/hd-geo-test.c
@@ -17,7 +17,12 @@
 
 #include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/bswap.h"
+#include "qapi/qmp/qlist.h"
 #include "libqtest.h"
+#include "libqos/fw_cfg.h"
+#include "libqos/libqos.h"
+#include "standard-headers/linux/qemu_fw_cfg.h"
 
 #define ARGV_SIZE 256
 
@@ -388,6 +393,537 @@ static void test_ide_drive_cd_0(void)
 qtest_quit(qts);
 }
 
+typedef struct {
+bool active;
+uint32_t head;
+uint32_t sector;
+uint32_t cyl;
+uint32_t end_head;
+uint32_t end_sector;
+uint32_t end_cyl;
+uint32_t start_sect;
+uint32_t nr_sects;
+} MBRpartitions[4];
+
+static MBRpartitions empty_mbr = { {false, 0, 0, 0, 0, 0, 0, 0, 0},
+   {false, 0, 0, 0, 0, 0, 0, 0, 0},
+   {false, 0, 0, 0, 0, 0, 0, 0, 0},
+   {false, 0, 0, 0, 0, 0, 0, 0, 0} };
+
+static char *create_qcow2_with_mbr(MBRpartitions mbr, uint64_t sectors)
+{
+const char *template = "/tmp/qtest.XX";
+char *raw_path = strdup(template);
+char *qcow2_path = strdup(template);
+char cmd[100 + 2 * PATH_MAX];
+uint8_t buf[512];
+int i, ret, fd, offset;
+uint64_t qcow2_size = sectors * 512;
+uint8_t status, parttype, head, sector, cyl;
+char *qemu_img_path;
+char *qemu_img_abs_path;
+
+offset = 0xbe;
+
+for (i = 0; i < 4; i++) {
+status = mbr[i].active ? 0x80 : 0x00;
+g_assert(mbr[i].head < 256);
+g_assert(mbr[i].sector < 64);
+g_assert(mbr[i].cyl < 1024);
+head = mbr[i].head;
+sector = mbr[i].sector + ((mbr[i].cyl & 0x300) >> 2);
+cyl = mbr[i].cyl & 0xff;
+
+buf[offset + 0x0] = status;
+buf[offset + 0x1] = head;
+buf[offset + 0x2] = sector;
+buf[offset + 0x3] = cyl;
+
+parttype = 0;
+g_assert(mbr[i].end_head < 256);
+g_assert(mbr[i].end_sector < 64);
+g_assert(mbr[i].end_cyl < 1024);
+head = mbr[i].end_head;
+sector = mbr[i].end_sector + ((mbr[i].end_cyl & 0x300) >> 2);
+cyl = mbr[i].end_cyl & 0xff;
+
+buf[offset + 0x4] = parttype;
+buf[offset + 0x5] = head;
+buf[offset + 0x6] = sector;
+buf[offset + 0x7] = cyl;
+
+(*(uint32_t *)[offset + 0x8]) = cpu_to_le32(mbr[i].start_sect);
+(*(uint32_t *)[offset + 0xc]) = cpu_to_le32(mbr[i].nr_sects);
+
+offset += 0x10;
+}
+
+fd = mkstemp(raw_path);
+g_assert(fd);
+close(fd);
+
+fd = open(raw_path, O_WRONLY);
+g_assert(fd >= 0);
+ret = write(fd, buf, sizeof(buf));
+g_assert(ret == sizeof(buf));
+close(fd);
+
+fd = mkstemp(qcow2_path);
+g_assert(fd);
+close(fd);
+
+qemu_img_path = getenv("QTEST_QEMU_IMG");
+g_assert(qemu_img_path);
+qemu_img_abs_path = realpath(qemu_img_path, NULL);
+g_assert(qemu_img_abs_path);
+
+ret = snprintf(cmd, sizeof(cmd),
+   "%s convert -f raw -O qcow2 %s %s > /dev/null",
+   qemu_img_abs_path,
+   raw_path, qcow2_path);
+g_assert((0 < ret) && (ret <= sizeof(cmd)));
+ret = system(cmd);
+g_assert(ret == 0);
+
+ret = snprintf(cmd, sizeof(cmd),
+   "%s resize %s %" PRIu64 " > /dev/null",
+   qemu_img_abs_path,
+   qcow2_path, qcow2_size);
+g_assert((0 < ret) && (ret <= sizeof(cmd)));
+ret = system(cmd);
+g_assert(ret == 0);
+
+free(qemu_img_abs_path);
+
+unlink(raw_path);
+free(raw_path);
+
+return qcow2_path;
+}
+
+#define BIOS_GEOMETRY_MAX_SIZE 1
+
+typedef struct {
+uint32_t c;
+uint32_t h;
+uint32_t s;
+} CHS;
+
+typedef struct {
+const char *dev_path;
+CHS chs;
+} CHSResult;
+
+static void read_bootdevices(QFWCFG *fw_cfg, CHSResult expected[])
+{
+char *buf = g_malloc0(BIOS_GEOMETRY_MAX_SIZE);
+char *cur;
+GList *results = NULL, *cur_result;
+CHSResult *r;
+int i;
+int res;
+bool found;
+
+qfw_cfg_get_file(fw_cfg,

[libvirt] [PATCH 1/1] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

This parameter has been deprecated since 2.12.0 and is eligible for
removal. Remove this parameter as it is actually completely ignored;
let's not give false hope.

Signed-off-by: John Snow 
---
 qemu-deprecated.texi | 20 +++-
 qapi/block-core.json |  6 +-
 blockdev.c   |  6 --
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index 01245e0b1c..d60246d5d6 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -149,11 +149,6 @@ QEMU 4.1 has three options, please migrate to one of these 
three:
 
 @section QEMU Machine Protocol (QMP) commands
 
-@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
-
-"autoload" parameter is now ignored. All bitmaps are automatically loaded
-from qcow2 images.
-
 @subsection query-block result field dirty-bitmaps[i].status (since 4.0)
 
 The ``status'' field of the ``BlockDirtyInfo'' structure, returned by
@@ -356,3 +351,18 @@ existing CPU models.  Management software that needs 
runnability
 guarantees must resolve the CPU model aliases using te
 ``alias-of'' field returned by the ``query-cpu-definitions'' QMP
 command.
+
+
+@node Recently removed features
+@appendix Recently removed features
+
+What follows is a record of recently removed, formerly deprecated
+features that serves as a record for users who have encountered
+trouble after a recent upgrade.
+
+@section QEMU Machine Protocol (QMP) commands
+
+@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
+
+"autoload" parameter is now ignored. All bitmaps are automatically loaded
+from qcow2 images.
diff --git a/qapi/block-core.json b/qapi/block-core.json
index e6edd641f1..e4975ece4a 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -1987,10 +1987,6 @@
 #  Qcow2 disks support persistent bitmaps. Default is false for
 #  block-dirty-bitmap-add. (Since: 2.10)
 #
-# @autoload: ignored and deprecated since 2.12.
-#Currently, all dirty tracking bitmaps are loaded from Qcow2 on
-#open.
-#
 # @disabled: the bitmap is created in the disabled state, which means that
 #it will not track drive changes. The bitmap may be enabled with
 #block-dirty-bitmap-enable. Default is false. (Since: 4.0)
@@ -1999,7 +1995,7 @@
 ##
 { 'struct': 'BlockDirtyBitmapAdd',
   'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32',
-'*persistent': 'bool', '*autoload': 'bool', '*disabled': 'bool' } }
+'*persistent': 'bool', '*disabled': 'bool' } }
 
 ##
 # @BlockDirtyBitmapMergeSource:
diff --git a/blockdev.c b/blockdev.c
index fbef6845c8..93804da840 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -1966,7 +1966,6 @@ static void block_dirty_bitmap_add_prepare(BlkActionState 
*common,
 qmp_block_dirty_bitmap_add(action->node, action->name,
action->has_granularity, action->granularity,
action->has_persistent, action->persistent,
-   action->has_autoload, action->autoload,
action->has_disabled, action->disabled,
_err);
 
@@ -2858,7 +2857,6 @@ out:
 void qmp_block_dirty_bitmap_add(const char *node, const char *name,
 bool has_granularity, uint32_t granularity,
 bool has_persistent, bool persistent,
-bool has_autoload, bool autoload,
 bool has_disabled, bool disabled,
 Error **errp)
 {
@@ -2890,10 +2888,6 @@ void qmp_block_dirty_bitmap_add(const char *node, const 
char *name,
 persistent = false;
 }
 
-if (has_autoload) {
-warn_report("Autoload option is deprecated and its value is ignored");
-}
-
 if (!has_disabled) {
 disabled = false;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PATCH 0/1] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

I'm going to be honest, here. There's actually no real reason to remove
this now, but we could, so I'm going to.

Also, in terms of the API serving as documentation, it's nicer to not
pretend this is an option that does anything, so out it goes.

This will serve as a little smoke test to see what happens if we
actually stop dropping features we claimed were deprecated.

John Snow (1):
  dirty-bitmaps: remove deprecated autoload parameter

 qemu-deprecated.texi | 20 +++-
 qapi/block-core.json |  6 +-
 blockdev.c   |  6 --
 3 files changed, 16 insertions(+), 16 deletions(-)

-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] QEMU bitmap backup usability FAQ

[John Snow]

On 8/20/19 6:25 PM, John Snow wrote:
> Hi, downstream here at Red Hat I've been fielding some questions about
> the usability and feature readiness of Bitmaps (and related features) in
> QEMU.
> 
> Here are some questions I answered internally that I am copying to the
> list for two reasons:
> 
> (1) To make sure my answers are actually correct, and
> (2) To share this pseudo-reference with the community at large.
> 
> This is long, and mostly for reference. There's a summary at the bottom
> with some todo items and observations about the usability of the feature
> as it exists in QEMU.
> 
> Before too long, I intend to send a more summarized "roadmap" mail which
> details all of the current and remaining work to be done in and around
> the bitmaps feature in QEMU.
> 
> 
> Questions:
> 
>> "What format(s) is/are required for this functionality?"
> 
> From the QEMU API, any format can be used to create and author
> incremental backups. The only known format limitations are:
> 
> 1. Persistent bitmaps cannot be created on any format except qcow2,
> although there are hooks to add support to other formats at a later date
> if desired.
> 
> DANGER CAVEAT #1: Adding bitmaps to QEMU by default creates transient
> bitmaps instead of persistent ones.
> 
> Possible TODO: Allow users to 'upgrade' transient bitmaps to persistent
> ones in case they made a mistake.
> 
> 
> 2. When using push backups (blockdev-backup, drive-backup), you may use
> any format as a target format.
> 
> DANGER CAVEAT #2: without backing file and/or filesystem-less sparse
> support, these images will be unusable.
> 
> EXAMPLE: Backing up to a raw file loses allocation information, so we
> can no longer distinguish between zeroes and unallocated regions. The
> cluster size is also lost. This file will not be usable without
> additional metadata recorded elsewhere.*
> 
> (* This is complicated, but it is in theory possible to do a push backup
> to e.g. an NBD target with custom server code that saves allocation
> information to a metadata file, which would allow you to reconstruct
> backups. For instance, recording in a .json file which extents were
> written out would allow you to -- with a custom binary -- write this
> information on top of a base file to reconstruct a backup.)
> 
> 
> 3. Any format can be used for either shared storage or live storage
> migrations. There are TWO distinct mechanisms for migrating bitmaps:
> 
> A) The bitmap is flushed to storage and re-opened on the destination.
> This is only supported for qcow2 and shared-storage migrations.
> 
> B) The bitmap is live-migrated to the destination. This is supported for
> any format and can be used for either shared storage or live storage
> migrations.
> 
> DANGER CAVEAT #3: The second bitmap migration technique there is an
> optional migration capability that must be enabled explicitly.
> Otherwise, some migration combinations may drop bitmaps.
> 
> Matrix:
> 
>> migrate = migrate_capability or (persistent and shared_storage)
> 
> Enumerated:
> 
> live storage + raw : transient + no-capability: Dropped
> live-storage + raw : transient + bm-capability: Migrated
> live-storage + qcow2 : transient + no-capability: Dropped
> live-storage + qcow2 : transient + bm-capability: Migrated
> live-storage + qcow2 : persistent + no-capability: Dropped (!)
> live-storage + qcow2 : persistent + bm-capability: Migrated
> 
> shared-storage + raw : transient - no-capability: Dropped
> shared-storage + raw : transient + bm-capability: Migrated
> shared-storage + qcow2 : transient + no-capability: Migrated
> shared-storage + qcow2 : transient + bm-capability: Migrated
> shared-storage + qcow2 : persistent + no-capability: Migrated
> shared-storage + qcow2 : persistent + bm-capability: Migrated
> 
> Enabling the bitmap migration capability will ALWAYS migrate the bitmap.
> If it's disabled, we will only migrate the bitmaps for shared storage
> migrations where the bitmap is persistent, which is a qcow2-only case.
> 
> There is no warning or error if you attempt to migrate in a manner that
> loses your bitmaps.
> 
> (I might be persuaded to add a case for when you are doing a live
> storage migration of qcow2 with persistent bitmaps, which is somewhat a
> conflicting case: you've asked for the bitmap to be persistent, but it
> seems likely that if this ever happens in practice, it's because you
> have neglected to ask for it to be migrated to the new host.)
> 
> See iotest 169 for more details on this.
> 
> At present, these are the only format limitations I am consciously aware
> of. From a management API/GUI perspective, it makes sen

Re: [libvirt] [PATCH 1/1] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

On 9/25/19 3:20 AM, Vladimir Sementsov-Ogievskiy wrote:
> 25.09.2019 2:01, John Snow wrote:
>> This parameter has been deprecated since 2.12.0 and is eligible for
>> removal. Remove this parameter as it is actually completely ignored;
>> let's not give false hope.
>>
>> Signed-off-by: John Snow 
>> ---
>>   qemu-deprecated.texi | 20 +++-
>>   qapi/block-core.json |  6 +-
>>   blockdev.c   |  6 --
>>   3 files changed, 16 insertions(+), 16 deletions(-)
>>
>> diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
>> index 01245e0b1c..d60246d5d6 100644
>> --- a/qemu-deprecated.texi
>> +++ b/qemu-deprecated.texi
>> @@ -149,11 +149,6 @@ QEMU 4.1 has three options, please migrate to one of 
>> these three:
>>   
>>   @section QEMU Machine Protocol (QMP) commands
>>   
>> -@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
>> -
>> -"autoload" parameter is now ignored. All bitmaps are automatically loaded
>> -from qcow2 images.
>> -
>>   @subsection query-block result field dirty-bitmaps[i].status (since 4.0)
>>   
>>   The ``status'' field of the ``BlockDirtyInfo'' structure, returned by
>> @@ -356,3 +351,18 @@ existing CPU models.  Management software that needs 
>> runnability
>>   guarantees must resolve the CPU model aliases using te
>>   ``alias-of'' field returned by the ``query-cpu-definitions'' QMP
>>   command.
>> +
>> +
>> +@node Recently removed features
>> +@appendix Recently removed features
>> +
>> +What follows is a record of recently removed, formerly deprecated
>> +features that serves as a record for users who have encountered
>> +trouble after a recent upgrade.
>> +
>> +@section QEMU Machine Protocol (QMP) commands
>> +
>> +@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
> 
> Agree with Eric that it should be 4.2 - as this section is about removing
> 

Yes, shame on me. I spent about three seconds on this patch and should
have spent four.

>> +
>> +"autoload" parameter is now ignored. All bitmaps are automatically loaded
>> +from qcow2 images.
> 
> Maybe, rephrase it as s/is now ignored/is now removed (ignored since 2.12.0)/ 
> ,
> so that this paragraph don't mislead without a context.
> 

Also a good idea.

'The "autoload" parameter has been ignored since 2.12.0. All bitmaps are
automatically loaded from qcow2 images.'

> Reviewed-by: Vladimir Sementsov-Ogievskiy 
> 
> (Yay, deprecation works!)
> 

Thanks, and I'll get to the rest of your pending bitmap patches and
cleanups soon.

--js

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] QEMU bitmap backup usability FAQ

[John Snow]

On 9/25/19 11:11 AM, Vladimir Sementsov-Ogievskiy wrote:
> 25.09.2019 16:52, John Snow wrote:
>>
>>
>> On 8/20/19 6:25 PM, John Snow wrote:
>>> Hi, downstream here at Red Hat I've been fielding some questions about
>>> the usability and feature readiness of Bitmaps (and related features) in
>>> QEMU.
>>>
>>> Here are some questions I answered internally that I am copying to the
>>> list for two reasons:
>>>
>>> (1) To make sure my answers are actually correct, and
>>> (2) To share this pseudo-reference with the community at large.
>>>
>>> This is long, and mostly for reference. There's a summary at the bottom
>>> with some todo items and observations about the usability of the feature
>>> as it exists in QEMU.
>>>
>>> Before too long, I intend to send a more summarized "roadmap" mail which
>>> details all of the current and remaining work to be done in and around
>>> the bitmaps feature in QEMU.
>>>
>>>
>>> Questions:
>>>
>>>> "What format(s) is/are required for this functionality?"
>>>
>>>  From the QEMU API, any format can be used to create and author
>>> incremental backups. The only known format limitations are:
>>>
>>> 1. Persistent bitmaps cannot be created on any format except qcow2,
>>> although there are hooks to add support to other formats at a later date
>>> if desired.
>>>
>>> DANGER CAVEAT #1: Adding bitmaps to QEMU by default creates transient
>>> bitmaps instead of persistent ones.
>>>
>>> Possible TODO: Allow users to 'upgrade' transient bitmaps to persistent
>>> ones in case they made a mistake.
>>>
>>>
>>> 2. When using push backups (blockdev-backup, drive-backup), you may use
>>> any format as a target format.
>>>
>>> DANGER CAVEAT #2: without backing file and/or filesystem-less sparse
>>> support, these images will be unusable.
>>>
>>> EXAMPLE: Backing up to a raw file loses allocation information, so we
>>> can no longer distinguish between zeroes and unallocated regions. The
>>> cluster size is also lost. This file will not be usable without
>>> additional metadata recorded elsewhere.*
>>>
>>> (* This is complicated, but it is in theory possible to do a push backup
>>> to e.g. an NBD target with custom server code that saves allocation
>>> information to a metadata file, which would allow you to reconstruct
>>> backups. For instance, recording in a .json file which extents were
>>> written out would allow you to -- with a custom binary -- write this
>>> information on top of a base file to reconstruct a backup.)
>>>
>>>
>>> 3. Any format can be used for either shared storage or live storage
>>> migrations. There are TWO distinct mechanisms for migrating bitmaps:
>>>
>>> A) The bitmap is flushed to storage and re-opened on the destination.
>>> This is only supported for qcow2 and shared-storage migrations.
>>>
>>> B) The bitmap is live-migrated to the destination. This is supported for
>>> any format and can be used for either shared storage or live storage
>>> migrations.
>>>
>>> DANGER CAVEAT #3: The second bitmap migration technique there is an
>>> optional migration capability that must be enabled explicitly.
>>> Otherwise, some migration combinations may drop bitmaps.
>>>
>>> Matrix:
>>>
>>>> migrate = migrate_capability or (persistent and shared_storage)
>>>
>>> Enumerated:
>>>
>>> live storage + raw : transient + no-capability: Dropped
>>> live-storage + raw : transient + bm-capability: Migrated
>>> live-storage + qcow2 : transient + no-capability: Dropped
>>> live-storage + qcow2 : transient + bm-capability: Migrated
>>> live-storage + qcow2 : persistent + no-capability: Dropped (!)
>>> live-storage + qcow2 : persistent + bm-capability: Migrated
>>>
>>> shared-storage + raw : transient - no-capability: Dropped
>>> shared-storage + raw : transient + bm-capability: Migrated
>>> shared-storage + qcow2 : transient + no-capability: Migrated
>>> shared-storage + qcow2 : transient + bm-capability: Migrated
>>> shared-storage + qcow2 : persistent + no-capability: Migrated
>>> shared-storage + qcow2 : persistent + bm-capability: Migrated
>>>
>>> Enabling the bitmap migration capability will ALWAYS migrate the bitmap.
>>> 

[libvirt] [PATCH 1/1] IDE: deprecate ide-drive

[John Snow]

It's an old compatibility shim that just delegates to ide-cd or ide-hd.
I'd like to refactor these some day, and getting rid of the super-object
will make that easier.

Either way, we don't need this.

Signed-off-by: John Snow 
---
 qemu-deprecated.texi  | 5 +
 hw/ide/qdev.c | 3 +++
 tests/qemu-iotests/051.pc.out | 6 --
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index 01245e0b1c4..f802d83983e 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -247,6 +247,11 @@ quite a bit. It will be removed without replacement unless 
some users speaks
 up at the @email{qemu-devel@@nongnu.org} mailing list with information about
 their usecases.
 
+@subsection ide-drive (since 4.2)
+
+The 'ide-drive' device is deprecated. Users should use 'ide-hd' or
+'ide-cd' as appropriate to get an IDE hard disk or CDROM as needed.
+
 @section System emulator machines
 
 @subsection pc-0.12, pc-0.13, pc-0.14 and pc-0.15 (since 4.0)
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 6fba6b62b87..9ecee4da074 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -279,6 +279,9 @@ static void ide_drive_realize(IDEDevice *dev, Error **errp)
 {
 DriveInfo *dinfo = NULL;
 
+warn_report("The 'ide-drive' device is deprecated. "
+"Use 'ide-hd' or 'ide-cd' instead");
+
 if (dev->conf.blk) {
 dinfo = blk_legacy_dinfo(dev->conf.blk);
 }
diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out
index 000557c7c83..93b9a1f82ca 100644
--- a/tests/qemu-iotests/051.pc.out
+++ b/tests/qemu-iotests/051.pc.out
@@ -158,7 +158,8 @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 Testing: -drive if=none,id=disk -device ide-drive,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive 
is empty
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: warning: The 'ide-drive' 
device is deprecated. Use 'ide-hd' or 'ide-cd' instead
+QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive is empty
 
 Testing: -drive if=none,id=disk -device ide-hd,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
@@ -228,7 +229,8 @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device 
ide-drive,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Block node is read-only
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: warning: The 'ide-drive' 
device is deprecated. Use 'ide-hd' or 'ide-cd' instead
+QEMU_PROG: -device ide-drive,drive=disk: Block node is read-only
 
 Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device 
ide-hd,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PATCH 0/1] IDE: Deprecate ide-drive

[John Snow]

John Snow (1):
  IDE: deprecate ide-drive

 qemu-deprecated.texi  | 5 +
 hw/ide/qdev.c | 3 +++
 tests/qemu-iotests/051.pc.out | 6 --
 3 files changed, 12 insertions(+), 2 deletions(-)

-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] [PATCH 1/1] IDE: deprecate ide-drive

[John Snow]

On 10/7/19 5:49 AM, Markus Armbruster wrote:
> John Snow  writes:
> 
>> It's an old compatibility shim that just delegates to ide-cd or ide-hd.
>> I'd like to refactor these some day, and getting rid of the super-object
>> will make that easier.
> 
> Device "scsi-disk" is similar.  However, it's still used by the
> scsi_bus_legacy_add_drive() magic.  Not sure that's fully deprecated,
> yet.  If / once it is, we can deprecate "scsi-disk", too.  Anyway, not
> your department.
> 

Yeah. I just want to get rid of this to allow myself to do bolder things
later on.

I have literally no time to do this and it's not really anything that
would make anyone money, but...

I want to add a few explicit devices:

ata-hd
ata-cd
sata-hd
sata-cd

With some shared state structures that implement common feature subsets,
like ata_registers, sata_registers, atapi_registers, etc.

I'd also like to separate out frontend and backend state providing a bit
of a cleaner division between device configuration (parameters on the
hardware creation itself), emulated device state (ATA register sets and
state machine), and QEMU backend state (block_backend pointers, aio
state counters, locks, etc etc etc -- Things solely purposed for
interacting with the block module.)

I'd also like to make each device type plug into ATA or SATA bus slots
explicitly -- no more magic IDE devices.

It's like the 5-year itch I can't help but want to scratch. My name's on
this code and it's UGLY UGLY UGLY!

The biggest roadblock to me actually doing this is figuring out how it
would be even vaguely possible to migrate from ide-hd or ide-cd to the
newer models -- it might be pretty complex, but maybe I can figure
something out somehow...

Well, suggestions welcome.

>> Either way, we don't need this.
>>
>> Signed-off-by: John Snow 
>> ---
>>  qemu-deprecated.texi  | 5 +
>>  hw/ide/qdev.c | 3 +++
>>  tests/qemu-iotests/051.pc.out | 6 --
>>  3 files changed, 12 insertions(+), 2 deletions(-)
>>
>> diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
>> index 01245e0b1c4..f802d83983e 100644
>> --- a/qemu-deprecated.texi
>> +++ b/qemu-deprecated.texi
>> @@ -247,6 +247,11 @@ quite a bit. It will be removed without replacement 
>> unless some users speaks
>>  up at the @email{qemu-devel@@nongnu.org} mailing list with information about
>>  their usecases.
>>  
>> +@subsection ide-drive (since 4.2)
>> +
>> +The 'ide-drive' device is deprecated. Users should use 'ide-hd' or
>> +'ide-cd' as appropriate to get an IDE hard disk or CDROM as needed.
> 
> CD-ROM
> 

>:[

>> +
>>  @section System emulator machines
>>  
>>  @subsection pc-0.12, pc-0.13, pc-0.14 and pc-0.15 (since 4.0)
>> diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
>> index 6fba6b62b87..9ecee4da074 100644
>> --- a/hw/ide/qdev.c
>> +++ b/hw/ide/qdev.c
>> @@ -279,6 +279,9 @@ static void ide_drive_realize(IDEDevice *dev, Error 
>> **errp)
>>  {
>>  DriveInfo *dinfo = NULL;
>>  
>> +warn_report("The 'ide-drive' device is deprecated. "
>> +"Use 'ide-hd' or 'ide-cd' instead");
> 
> Two sentences, where only the first one terminated with a period.
> 
> Let's say "is deprecated, please use", like we do in several other places.
> 

Alright.

>> +
>>  if (dev->conf.blk) {
>>  dinfo = blk_legacy_dinfo(dev->conf.blk);
>>  }
>> diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out
>> index 000557c7c83..93b9a1f82ca 100644
>> --- a/tests/qemu-iotests/051.pc.out
>> +++ b/tests/qemu-iotests/051.pc.out
>> @@ -158,7 +158,8 @@ QEMU X.Y.Z monitor - type 'help' for more information
>>  
>>  Testing: -drive if=none,id=disk -device ide-drive,drive=disk
>>  QEMU X.Y.Z monitor - type 'help' for more information
>> -(qemu) QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but 
>> drive is empty
>> +(qemu) QEMU_PROG: -device ide-drive,drive=disk: warning: The 'ide-drive' 
>> device is deprecated. Use 'ide-hd' or 'ide-cd' instead
>> +QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive is 
>> empty
>>  
>>  Testing: -drive if=none,id=disk -device ide-hd,drive=disk
>>  QEMU X.Y.Z monitor - type 'help' for more information
>> @@ -228,7 +229,8 @@ QEMU X.Y.Z monitor - type 'help' for more information
>>  
>>  Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device 
>> ide-drive,drive=disk
>>  QEMU X.Y.Z monitor - type 'help' for more information
>> -(qemu) QEMU_PROG: -device ide-drive,drive=dis

[libvirt] [PATCH v2 1/1] IDE: deprecate ide-drive

[John Snow]

It's an old compatibility shim that just delegates to ide-cd or ide-hd.
I'd like to refactor these some day, and getting rid of the super-object
will make that easier.

Either way, we don't need this.

Libvirt-checked-by: Peter Krempa 
Signed-off-by: John Snow 
---
 qemu-deprecated.texi  | 5 +
 hw/ide/qdev.c | 3 +++
 tests/qemu-iotests/051.pc.out | 6 --
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index 01245e0b1c4..7cd4648df3c 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -247,6 +247,11 @@ quite a bit. It will be removed without replacement unless 
some users speaks
 up at the @email{qemu-devel@@nongnu.org} mailing list with information about
 their usecases.
 
+@subsection ide-drive (since 4.2)
+
+The 'ide-drive' device is deprecated. Users should use 'ide-hd' or
+'ide-cd' as appropriate to get an IDE hard disk or CD-ROM as needed.
+
 @section System emulator machines
 
 @subsection pc-0.12, pc-0.13, pc-0.14 and pc-0.15 (since 4.0)
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 6fba6b62b87..3666e597211 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -279,6 +279,9 @@ static void ide_drive_realize(IDEDevice *dev, Error **errp)
 {
 DriveInfo *dinfo = NULL;
 
+warn_report("'ide-drive' is deprecated, "
+"please use 'ide-hd' or 'ide-cd' instead");
+
 if (dev->conf.blk) {
 dinfo = blk_legacy_dinfo(dev->conf.blk);
 }
diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out
index 000557c7c83..34849dd1720 100644
--- a/tests/qemu-iotests/051.pc.out
+++ b/tests/qemu-iotests/051.pc.out
@@ -158,7 +158,8 @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 Testing: -drive if=none,id=disk -device ide-drive,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive 
is empty
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: warning: 'ide-drive' is 
deprecated, please use 'ide-hd' or 'ide-cd' instead
+QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive is empty
 
 Testing: -drive if=none,id=disk -device ide-hd,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
@@ -228,7 +229,8 @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device 
ide-drive,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Block node is read-only
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: warning: 'ide-drive' is 
deprecated, please use 'ide-hd' or 'ide-cd' instead
+QEMU_PROG: -device ide-drive,drive=disk: Block node is read-only
 
 Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device 
ide-hd,drive=disk
 QEMU X.Y.Z monitor - type 'help' for more information
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PATCH v2 0/1] IDE: Deprecate ide-drive

[John Snow]

V2: Change phrasings and spellings as Markus suggested.

John Snow (1):
  IDE: deprecate ide-drive

 qemu-deprecated.texi  | 5 +
 hw/ide/qdev.c | 3 +++
 tests/qemu-iotests/051.pc.out | 6 --
 3 files changed, 12 insertions(+), 2 deletions(-)

-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] [PATCH 1/1] IDE: deprecate ide-drive

[John Snow]

On 10/8/19 2:51 AM, Markus Armbruster wrote:
>> I'll respin to hit the tests with a stiffer scrub-brush.
> Thanks!

051 is the only test I can find that uses ide-drive, and the non-pc
version of the test doesn't seem to use it, so this actually seems
sufficient.

I'd like to keep the test for ide-drive itself in until we actually
remove it, just to make sure it still works. No harm in keeping it, I think.

--js

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] [PATCH v2 1/1] IDE: deprecate ide-drive

[John Snow]

On 10/10/19 7:54 AM, Peter Krempa wrote:
> On Thu, Oct 10, 2019 at 13:42:26 +0200, Philippe Mathieu-Daudé wrote:
>> On 10/10/19 1:26 PM, Peter Krempa wrote:
>>> On Thu, Oct 10, 2019 at 13:22:37 +0200, Philippe Mathieu-Daudé wrote:
>>>> On 10/10/19 12:43 AM, John Snow wrote:
>>>>> It's an old compatibility shim that just delegates to ide-cd or ide-hd.
>>>>> I'd like to refactor these some day, and getting rid of the super-object
>>>>> will make that easier.
>>>>>
>>>>> Either way, we don't need this.
>>>>>
>>>>> Libvirt-checked-by: Peter Krempa 
>>>>
>>>> Peter made a comment regarding Laszlo's Regression-tested-by tag:
>>>>
>>>>[...] nobody else is using
>>>>this convention (there are exactly 0 instances of
>>>>"Regression-tested-by" in the project git log as far as
>>>>I can see), and so in practice people reading the commits
>>>>won't really know what you meant by it. Everybody else
>>>>on the project uses "Tested-by" to mean either of the
>>>>two cases you describe above, without distinction...
>>>>
>>>> It probably applies to 'Libvirt-checked-by' too.
>>>
>>> I certainly didn't test it. So feel free to drop that line altogether.
>>
>> But you reviewed it, can we use your 'Reviewed-by' instead?
> 
> To be honest, I didn't really review the code nor the documentation.
> I actually reviewed only the idea itself in the context of integration
> with libvirt and that's why I didn't go for 'Reviewed-by:'.
> 
> The gist of the citation above is that we should stick to well known
> tags with their well known meanings and I think that considering this a
> 'review' would be a stretch of the definiton.
> 

I wasn't aware that PMM wanted to avoid non-standard tags; I consider
them to be for human use, but I can change that behavior.

Peter, I'll change it to an ACK (as suggested by Kevin) is that's OK by you.

--js

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] [PATCH v3] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

On 10/2/19 7:24 PM, John Snow wrote:
> This parameter has been deprecated since 2.12.0 and is eligible for
> removal. Remove this parameter as it is actually completely ignored;
> let's not give false hope.
> 
> Signed-off-by: John Snow 
> Reviewed-by: Eric Blake 
> Reviewed-by: Vladimir Sementsov-Ogievskiy 


Thanks, applied to my bitmaps tree:

https://github.com/jnsnow/qemu/commits/bitmaps
https://github.com/jnsnow/qemu.git

--js

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PATCH v2] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

This parameter has been deprecated since 2.12.0 and is eligible for
removal. Remove this parameter as it is actually completely ignored;
let's not give false hope.

Signed-off-by: John Snow 
Reviewed-by: Eric Blake 
Reviewed-by: Vladimir Sementsov-Ogievskiy 
---
V2: Change 2.12.0 -> 4.2.0 in removed section.
Adjust phrasing to match.

 qemu-deprecated.texi | 20 +++-
 qapi/block-core.json |  6 +-
 blockdev.c   |  6 --
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index 01245e0b1c..d60246d5d6 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -149,11 +149,6 @@ QEMU 4.1 has three options, please migrate to one of these 
three:
 
 @section QEMU Machine Protocol (QMP) commands
 
-@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
-
-"autoload" parameter is now ignored. All bitmaps are automatically loaded
-from qcow2 images.
-
 @subsection query-block result field dirty-bitmaps[i].status (since 4.0)
 
 The ``status'' field of the ``BlockDirtyInfo'' structure, returned by
@@ -356,3 +351,18 @@ existing CPU models.  Management software that needs 
runnability
 guarantees must resolve the CPU model aliases using te
 ``alias-of'' field returned by the ``query-cpu-definitions'' QMP
 command.
+
+
+@node Recently removed features
+@appendix Recently removed features
+
+What follows is a record of recently removed, formerly deprecated
+features that serves as a record for users who have encountered
+trouble after a recent upgrade.
+
+@section QEMU Machine Protocol (QMP) commands
+
+@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
+
+"autoload" parameter is now ignored. All bitmaps are automatically loaded
+from qcow2 images.
diff --git a/qapi/block-core.json b/qapi/block-core.json
index e6edd641f1..e4975ece4a 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -1987,10 +1987,6 @@
 #  Qcow2 disks support persistent bitmaps. Default is false for
 #  block-dirty-bitmap-add. (Since: 2.10)
 #
-# @autoload: ignored and deprecated since 2.12.
-#Currently, all dirty tracking bitmaps are loaded from Qcow2 on
-#open.
-#
 # @disabled: the bitmap is created in the disabled state, which means that
 #it will not track drive changes. The bitmap may be enabled with
 #block-dirty-bitmap-enable. Default is false. (Since: 4.0)
@@ -1999,7 +1995,7 @@
 ##
 { 'struct': 'BlockDirtyBitmapAdd',
   'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32',
-'*persistent': 'bool', '*autoload': 'bool', '*disabled': 'bool' } }
+'*persistent': 'bool', '*disabled': 'bool' } }
 
 ##
 # @BlockDirtyBitmapMergeSource:
diff --git a/blockdev.c b/blockdev.c
index fbef6845c8..93804da840 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -1966,7 +1966,6 @@ static void block_dirty_bitmap_add_prepare(BlkActionState 
*common,
 qmp_block_dirty_bitmap_add(action->node, action->name,
action->has_granularity, action->granularity,
action->has_persistent, action->persistent,
-   action->has_autoload, action->autoload,
action->has_disabled, action->disabled,
_err);
 
@@ -2858,7 +2857,6 @@ out:
 void qmp_block_dirty_bitmap_add(const char *node, const char *name,
 bool has_granularity, uint32_t granularity,
 bool has_persistent, bool persistent,
-bool has_autoload, bool autoload,
 bool has_disabled, bool disabled,
 Error **errp)
 {
@@ -2890,10 +2888,6 @@ void qmp_block_dirty_bitmap_add(const char *node, const 
char *name,
 persistent = false;
 }
 
-if (has_autoload) {
-warn_report("Autoload option is deprecated and its value is ignored");
-}
-
 if (!has_disabled) {
 disabled = false;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] [PATCH v2] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

On 10/2/19 7:22 PM, John Snow wrote:
> This parameter has been deprecated since 2.12.0 and is eligible for
> removal. Remove this parameter as it is actually completely ignored;
> let's not give false hope.
> 
> Signed-off-by: John Snow 
> Reviewed-by: Eric Blake 
> Reviewed-by: Vladimir Sementsov-Ogievskiy 

bah, NACK; I didn't commit the changes I made.

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PATCH v3] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

This parameter has been deprecated since 2.12.0 and is eligible for
removal. Remove this parameter as it is actually completely ignored;
let's not give false hope.

Signed-off-by: John Snow 
Reviewed-by: Eric Blake 
Reviewed-by: Vladimir Sementsov-Ogievskiy 
---
V2: Change 2.12.0 -> 4.2.0 in removed section.
Adjust phrasing to match.

 qemu-deprecated.texi | 20 +++-
 qapi/block-core.json |  6 +-
 blockdev.c   |  6 --
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index 01245e0b1c..7239e0959d 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -149,11 +149,6 @@ QEMU 4.1 has three options, please migrate to one of these 
three:
 
 @section QEMU Machine Protocol (QMP) commands
 
-@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
-
-"autoload" parameter is now ignored. All bitmaps are automatically loaded
-from qcow2 images.
-
 @subsection query-block result field dirty-bitmaps[i].status (since 4.0)
 
 The ``status'' field of the ``BlockDirtyInfo'' structure, returned by
@@ -356,3 +351,18 @@ existing CPU models.  Management software that needs 
runnability
 guarantees must resolve the CPU model aliases using te
 ``alias-of'' field returned by the ``query-cpu-definitions'' QMP
 command.
+
+
+@node Recently removed features
+@appendix Recently removed features
+
+What follows is a record of recently removed, formerly deprecated
+features that serves as a record for users who have encountered
+trouble after a recent upgrade.
+
+@section QEMU Machine Protocol (QMP) commands
+
+@subsection block-dirty-bitmap-add "autoload" parameter (since 4.2.0)
+
+The "autoload" parameter has been ignored since 2.12.0. All bitmaps
+are automatically loaded from qcow2 images.
diff --git a/qapi/block-core.json b/qapi/block-core.json
index e6edd641f1..e4975ece4a 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -1987,10 +1987,6 @@
 #  Qcow2 disks support persistent bitmaps. Default is false for
 #  block-dirty-bitmap-add. (Since: 2.10)
 #
-# @autoload: ignored and deprecated since 2.12.
-#Currently, all dirty tracking bitmaps are loaded from Qcow2 on
-#open.
-#
 # @disabled: the bitmap is created in the disabled state, which means that
 #it will not track drive changes. The bitmap may be enabled with
 #block-dirty-bitmap-enable. Default is false. (Since: 4.0)
@@ -1999,7 +1995,7 @@
 ##
 { 'struct': 'BlockDirtyBitmapAdd',
   'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32',
-'*persistent': 'bool', '*autoload': 'bool', '*disabled': 'bool' } }
+'*persistent': 'bool', '*disabled': 'bool' } }
 
 ##
 # @BlockDirtyBitmapMergeSource:
diff --git a/blockdev.c b/blockdev.c
index fbef6845c8..93804da840 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -1966,7 +1966,6 @@ static void block_dirty_bitmap_add_prepare(BlkActionState 
*common,
 qmp_block_dirty_bitmap_add(action->node, action->name,
action->has_granularity, action->granularity,
action->has_persistent, action->persistent,
-   action->has_autoload, action->autoload,
action->has_disabled, action->disabled,
_err);
 
@@ -2858,7 +2857,6 @@ out:
 void qmp_block_dirty_bitmap_add(const char *node, const char *name,
 bool has_granularity, uint32_t granularity,
 bool has_persistent, bool persistent,
-bool has_autoload, bool autoload,
 bool has_disabled, bool disabled,
 Error **errp)
 {
@@ -2890,10 +2888,6 @@ void qmp_block_dirty_bitmap_add(const char *node, const 
char *name,
 persistent = false;
 }
 
-if (has_autoload) {
-warn_report("Autoload option is deprecated and its value is ignored");
-}
-
 if (!has_disabled) {
 disabled = false;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 16/19] block/qcow2-bitmap: fix and improve qcow2_reopen_bitmaps_rw

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

- Correct check for write access to file child, and in correct place
  (only if we want to write).
- Support reopen rw -> rw (which will be used in following commit),
  for example, !bdrv_dirty_bitmap_readonly() is not a corruption if
  bitmap is marked IN_USE in the image.
- Consider unexpected bitmap as a corruption and check other
  combinations of in-image and in-RAM bitmaps.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-9-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2-bitmap.c | 77 +---
 1 file changed, 58 insertions(+), 19 deletions(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index f7dfb40256e..98294a76965 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1108,18 +1108,14 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error 
**errp)
 Qcow2BitmapList *bm_list;
 Qcow2Bitmap *bm;
 GSList *ro_dirty_bitmaps = NULL;
-int ret = 0;
+int ret = -EINVAL;
+bool need_header_update = false;
 
 if (s->nb_bitmaps == 0) {
 /* No bitmaps - nothing to do */
 return 0;
 }
 
-if (!can_write(bs)) {
-error_setg(errp, "Can't write to the image on reopening bitmaps rw");
-return -EINVAL;
-}
-
 bm_list = bitmap_list_load(bs, s->bitmap_directory_offset,
s->bitmap_directory_size, errp);
 if (bm_list == NULL) {
@@ -1128,32 +1124,75 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error 
**errp)
 
 QSIMPLEQ_FOREACH(bm, bm_list, entry) {
 BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(bs, bm->name);
-if (bitmap == NULL) {
-continue;
-}
 
-if (!bdrv_dirty_bitmap_readonly(bitmap)) {
-error_setg(errp, "Bitmap %s was loaded prior to rw-reopen, but was 
"
-   "not marked as readonly. This is a bug, something went "
-   "wrong. All of the bitmaps may be corrupted", bm->name);
-ret = -EINVAL;
+if (!bitmap) {
+error_setg(errp, "Unexpected bitmap '%s' in image '%s'",
+   bm->name, bs->filename);
 goto out;
 }
 
-bm->flags |= BME_FLAG_IN_USE;
-ro_dirty_bitmaps = g_slist_append(ro_dirty_bitmaps, bitmap);
+if (!(bm->flags & BME_FLAG_IN_USE)) {
+if (!bdrv_dirty_bitmap_readonly(bitmap)) {
+error_setg(errp, "Corruption: bitmap '%s' is not marked IN_USE 
"
+   "in the image '%s' and not marked readonly in RAM",
+   bm->name, bs->filename);
+goto out;
+}
+if (bdrv_dirty_bitmap_inconsistent(bitmap)) {
+error_setg(errp, "Corruption: bitmap '%s' is inconsistent but "
+   "is not marked IN_USE in the image '%s'", bm->name,
+   bs->filename);
+goto out;
+}
+
+bm->flags |= BME_FLAG_IN_USE;
+need_header_update = true;
+} else {
+/*
+ * What if flags already has BME_FLAG_IN_USE ?
+ *
+ * 1. if we are reopening RW -> RW it's OK, of course.
+ * 2. if we are reopening RO -> RW:
+ *   2.1 if @bitmap is inconsistent, it's OK. It means that it was
+ *   inconsistent (IN_USE) when we loaded it
+ *   2.2 if @bitmap is not inconsistent. This seems to be 
impossible
+ *   and implies third party interaction. Let's error-out for
+ *   safety.
+ */
+if (bdrv_dirty_bitmap_readonly(bitmap) &&
+!bdrv_dirty_bitmap_inconsistent(bitmap))
+{
+error_setg(errp, "Corruption: bitmap '%s' is marked IN_USE "
+   "in the image '%s' but it is readonly and "
+   "consistent in RAM",
+   bm->name, bs->filename);
+goto out;
+}
+}
+
+if (bdrv_dirty_bitmap_readonly(bitmap)) {
+ro_dirty_bitmaps = g_slist_append(ro_dirty_bitmaps, bitmap);
+}
 }
 
-if (ro_dirty_bitmaps != NULL) {
+if (need_header_update) {
+if (!can_write(bs->file->bs) || !(bs->file->perm & BLK_PERM_WRITE)) {
+error_setg(errp, "Failed to reopen bitmaps rw: no write access "
+   "the protocol file");
+goto out;
+}
+
 /* in_use flags must be updated */
 ret = update_ext_header_and_dir_in_place(bs, bm_list);
 if (ret < 0) {
-error_setg_errn

[libvirt] [PULL 17/19] qcow2-bitmap: move bitmap reopen-rw code to qcow2_reopen_commit

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

The only reason I can imagine for this strange code at the very-end of
bdrv_reopen_commit is the fact that bs->read_only updated after
calling drv->bdrv_reopen_commit in bdrv_reopen_commit. And in the same
time, prior to previous commit, qcow2_reopen_bitmaps_rw did a wrong
check for being writable, when actually it only need writable file
child not self.

So, as it's fixed, let's move things to correct place.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Acked-by: Max Reitz 
Message-id: 20190927122355.7344-10-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/block_int.h |  6 --
 block.c   | 19 ---
 block/qcow2.c | 15 ++-
 3 files changed, 14 insertions(+), 26 deletions(-)

diff --git a/include/block/block_int.h b/include/block/block_int.h
index 1e54486ad14..9ceca23ef75 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -546,12 +546,6 @@ struct BlockDriver {
  uint64_t parent_perm, uint64_t parent_shared,
  uint64_t *nperm, uint64_t *nshared);
 
-/**
- * Bitmaps should be marked as 'IN_USE' in the image on reopening image
- * as rw. This handler should realize it. It also should unset readonly
- * field of BlockDirtyBitmap's in case of success.
- */
-int (*bdrv_reopen_bitmaps_rw)(BlockDriverState *bs, Error **errp);
 bool (*bdrv_co_can_store_new_dirty_bitmap)(BlockDriverState *bs,
const char *name,
uint32_t granularity,
diff --git a/block.c b/block.c
index c548885608d..ba09d97e0a2 100644
--- a/block.c
+++ b/block.c
@@ -3935,16 +3935,12 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
 BlockDriver *drv;
 BlockDriverState *bs;
 BdrvChild *child;
-bool old_can_write, new_can_write;
 
 assert(reopen_state != NULL);
 bs = reopen_state->bs;
 drv = bs->drv;
 assert(drv != NULL);
 
-old_can_write =
-!bdrv_is_read_only(bs) && !(bdrv_get_flags(bs) & BDRV_O_INACTIVE);
-
 /* If there are any driver level actions to take */
 if (drv->bdrv_reopen_commit) {
 drv->bdrv_reopen_commit(reopen_state);
@@ -3988,21 +3984,6 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
 }
 
 bdrv_refresh_limits(bs, NULL);
-
-new_can_write =
-!bdrv_is_read_only(bs) && !(bdrv_get_flags(bs) & BDRV_O_INACTIVE);
-if (!old_can_write && new_can_write && drv->bdrv_reopen_bitmaps_rw) {
-Error *local_err = NULL;
-if (drv->bdrv_reopen_bitmaps_rw(bs, _err) < 0) {
-/* This is not fatal, bitmaps just left read-only, so all following
- * writes will fail. User can remove read-only bitmaps to unblock
- * writes.
- */
-error_reportf_err(local_err,
-  "%s: Failed to make dirty bitmaps writable: ",
-  bdrv_get_node_name(bs));
-}
-}
 }
 
 /*
diff --git a/block/qcow2.c b/block/qcow2.c
index 7ab1aad9779..b652bf884e5 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1834,6 +1834,20 @@ fail:
 static void qcow2_reopen_commit(BDRVReopenState *state)
 {
 qcow2_update_options_commit(state->bs, state->opaque);
+if (state->flags & BDRV_O_RDWR) {
+Error *local_err = NULL;
+
+if (qcow2_reopen_bitmaps_rw(state->bs, _err) < 0) {
+/*
+ * This is not fatal, bitmaps just left read-only, so all following
+ * writes will fail. User can remove read-only bitmaps to unblock
+ * writes or retry reopen.
+ */
+error_reportf_err(local_err,
+  "%s: Failed to make dirty bitmaps writable: ",
+  bdrv_get_node_name(state->bs));
+}
+}
 g_free(state->opaque);
 }
 
@@ -5262,7 +5276,6 @@ BlockDriver bdrv_qcow2 = {
 .bdrv_detach_aio_context  = qcow2_detach_aio_context,
 .bdrv_attach_aio_context  = qcow2_attach_aio_context,
 
-.bdrv_reopen_bitmaps_rw = qcow2_reopen_bitmaps_rw,
 .bdrv_co_can_store_new_dirty_bitmap = qcow2_co_can_store_new_dirty_bitmap,
 .bdrv_co_remove_persistent_dirty_bitmap =
 qcow2_co_remove_persistent_dirty_bitmap,
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 06/19] block/dirty-bitmap: add bs link

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Add bs field to BdrvDirtyBitmap structure. Drop BlockDriverState
parameter from bitmap APIs where possible.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-3-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h   | 14 +-
 block/backup.c | 14 ++
 block/dirty-bitmap.c   | 24 
 block/mirror.c |  4 ++--
 block/qcow2-bitmap.c   |  6 +++---
 blockdev.c |  6 +++---
 migration/block-dirty-bitmap.c |  7 +++
 migration/block.c  |  4 ++--
 8 files changed, 36 insertions(+), 43 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 973056778aa..2f9b088e11e 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -18,21 +18,18 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
   uint32_t granularity,
   const char *name,
   Error **errp);
-int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
-   BdrvDirtyBitmap *bitmap,
+int bdrv_dirty_bitmap_create_successor(BdrvDirtyBitmap *bitmap,
Error **errp);
-BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BdrvDirtyBitmap *bitmap,
 Error **errp);
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
-   BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BdrvDirtyBitmap *bitmap,
Error **errp);
 void bdrv_dirty_bitmap_enable_successor(BdrvDirtyBitmap *bitmap);
 BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs,
 const char *name);
 int bdrv_dirty_bitmap_check(const BdrvDirtyBitmap *bitmap, uint32_t flags,
 Error **errp);
-void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
+void bdrv_release_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs);
 int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
 Error **errp);
@@ -106,8 +103,7 @@ int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap 
*bitmap, uint64_t offset,
 uint64_t bytes);
 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
uint64_t *offset, uint64_t *bytes);
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BlockDriverState *bs,
-  BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
   Error **errp);
 
 #endif
diff --git a/block/backup.c b/block/backup.c
index 763f0d7ff6d..acb67da3a7b 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -352,7 +352,6 @@ static int coroutine_fn backup_before_write_notify(
 static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)
 {
 BdrvDirtyBitmap *bm;
-BlockDriverState *bs = blk_bs(job->common.blk);
 bool sync = (((ret == 0) || (job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS)) 
\
  && (job->bitmap_mode != BITMAP_SYNC_MODE_NEVER));
 
@@ -361,13 +360,13 @@ static void backup_cleanup_sync_bitmap(BackupBlockJob 
*job, int ret)
  * We succeeded, or we always intended to sync the bitmap.
  * Delete this bitmap and install the child.
  */
-bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL);
+bm = bdrv_dirty_bitmap_abdicate(job->sync_bitmap, NULL);
 } else {
 /*
  * We failed, or we never intended to sync the bitmap anyway.
  * Merge the successor back into the parent, keeping all data.
  */
-bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL);
+bm = bdrv_reclaim_dirty_bitmap(job->sync_bitmap, NULL);
 }
 
 assert(bm);
@@ -398,10 +397,9 @@ static void backup_abort(Job *job)
 static void backup_clean(Job *job)
 {
 BackupBlockJob *s = container_of(job, BackupBlockJob, common.job);
-BlockDriverState *bs = blk_bs(s->common.blk);
 
 if (s->copy_bitmap) {
-bdrv_release_dirty_bitmap(bs, s->copy_bitmap);
+bdrv_release_dirty_bitmap(s->copy_bitmap);
 s->copy_bitmap = NULL;
 }
 
@@ -679,7 +677,7 @@ BlockJob *backup_job_create(const char *job_id, 
BlockDriverState *bs,
 }
 
 /* Crea

[libvirt] [PULL 09/19] block: switch reopen queue from QSIMPLEQ to QTAILQ

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

We'll need reverse-foreach in the following commit, QTAILQ support it,
so move to QTAILQ.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Message-id: 20190927122355.7344-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/block.h |  2 +-
 block.c   | 24 
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index 37c9de7446d..f5099435136 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -195,7 +195,7 @@ typedef struct HDGeometry {
 #define BDRV_BLOCK_EOF  0x20
 #define BDRV_BLOCK_RECURSE  0x40
 
-typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) 
BlockReopenQueue;
+typedef QTAILQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
 
 typedef struct BDRVReopenState {
 BlockDriverState *bs;
diff --git a/block.c b/block.c
index 5b5b0337acc..aaf5d796284 100644
--- a/block.c
+++ b/block.c
@@ -1719,7 +1719,7 @@ typedef struct BlockReopenQueueEntry {
  bool prepared;
  bool perms_checked;
  BDRVReopenState state;
- QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
+ QTAILQ_ENTRY(BlockReopenQueueEntry) entry;
 } BlockReopenQueueEntry;
 
 /*
@@ -1732,7 +1732,7 @@ static int bdrv_reopen_get_flags(BlockReopenQueue *q, 
BlockDriverState *bs)
 BlockReopenQueueEntry *entry;
 
 if (q != NULL) {
-QSIMPLEQ_FOREACH(entry, q, entry) {
+QTAILQ_FOREACH(entry, q, entry) {
 if (entry->state.bs == bs) {
 return entry->state.flags;
 }
@@ -3249,7 +3249,7 @@ static bool bdrv_recurse_has_child(BlockDriverState *bs,
  * Adds a BlockDriverState to a simple queue for an atomic, transactional
  * reopen of multiple devices.
  *
- * bs_queue can either be an existing BlockReopenQueue that has had 
QSIMPLE_INIT
+ * bs_queue can either be an existing BlockReopenQueue that has had QTAILQ_INIT
  * already performed, or alternatively may be NULL a new BlockReopenQueue will
  * be created and initialized. This newly created BlockReopenQueue should be
  * passed back in for subsequent calls that are intended to be of the same
@@ -3290,7 +3290,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 
 if (bs_queue == NULL) {
 bs_queue = g_new0(BlockReopenQueue, 1);
-QSIMPLEQ_INIT(bs_queue);
+QTAILQ_INIT(bs_queue);
 }
 
 if (!options) {
@@ -3298,7 +3298,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 }
 
 /* Check if this BlockDriverState is already in the queue */
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 if (bs == bs_entry->state.bs) {
 break;
 }
@@ -3354,7 +3354,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 
 if (!bs_entry) {
 bs_entry = g_new0(BlockReopenQueueEntry, 1);
-QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
+QTAILQ_INSERT_TAIL(bs_queue, bs_entry, entry);
 } else {
 qobject_unref(bs_entry->state.options);
 qobject_unref(bs_entry->state.explicit_options);
@@ -3455,7 +3455,7 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 
 assert(bs_queue != NULL);
 
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 assert(bs_entry->state.bs->quiesce_counter > 0);
 if (bdrv_reopen_prepare(_entry->state, bs_queue, errp)) {
 goto cleanup;
@@ -3463,7 +3463,7 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 bs_entry->prepared = true;
 }
 
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 BDRVReopenState *state = _entry->state;
 ret = bdrv_check_perm(state->bs, bs_queue, state->perm,
   state->shared_perm, NULL, NULL, errp);
@@ -3489,13 +3489,13 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 /* If we reach this point, we have success and just need to apply the
  * changes
  */
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 bdrv_reopen_commit(_entry->state);
 }
 
 ret = 0;
 cleanup_perm:
-QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
 BDRVReopenState *state = _entry->state;
 
 if (!bs_entry->perms_checked) {
@@ -3512,7 +3512,7 @@ cleanup_perm:
 }
 }
 cleanup:
-QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
 if (ret) {
 if (bs_entry->prepared) {
 bdrv_reopen_abort(_entry->s

[libvirt] [PULL 10/19] block: reverse order for reopen commits

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

It's needed to fix reopening qcow2 with bitmaps to RW. Currently it
can't work, as qcow2 needs write access to file child, to mark bitmaps
in-image with IN_USE flag. But usually children goes after parents in
reopen queue and file child is still RO on qcow2 reopen commit. Reverse
reopen order to fix it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Acked-by: Max Reitz 
Acked-by: John Snow 
Message-id: 20190927122355.7344-3-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block.c | 12 +---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/block.c b/block.c
index aaf5d796284..c548885608d 100644
--- a/block.c
+++ b/block.c
@@ -3486,10 +3486,16 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 bs_entry->perms_checked = true;
 }
 
-/* If we reach this point, we have success and just need to apply the
- * changes
+/*
+ * If we reach this point, we have success and just need to apply the
+ * changes.
+ *
+ * Reverse order is used to comfort qcow2 driver: on commit it need to 
write
+ * IN_USE flag to the image, to mark bitmaps in the image as invalid. But
+ * children are usually goes after parents in reopen-queue, so go from last
+ * to first element.
  */
-QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH_REVERSE(bs_entry, bs_queue, entry) {
 bdrv_reopen_commit(_entry->state);
 }
 
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 15/19] iotests: add test 260 to check bitmap life after snapshot + commit

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-8-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 tests/qemu-iotests/260 | 89 ++
 tests/qemu-iotests/260.out | 52 ++
 tests/qemu-iotests/group   |  1 +
 3 files changed, 142 insertions(+)
 create mode 100755 tests/qemu-iotests/260
 create mode 100644 tests/qemu-iotests/260.out

diff --git a/tests/qemu-iotests/260 b/tests/qemu-iotests/260
new file mode 100755
index 000..4f6082c9d22
--- /dev/null
+++ b/tests/qemu-iotests/260
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+#
+# Tests for temporary external snapshot when we have bitmaps.
+#
+# Copyright (c) 2019 Virtuozzo International GmbH. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+from iotests import qemu_img_create, file_path, log, filter_qmp_event
+
+iotests.verify_image_format(supported_fmts=['qcow2'])
+
+base, top = file_path('base', 'top')
+size = 64 * 1024 * 3
+
+
+def print_bitmap(msg, vm):
+result = vm.qmp('query-block')['return'][0]
+if 'dirty-bitmaps' in result:
+bitmap = result['dirty-bitmaps'][0]
+log('{}: name={} dirty-clusters={}'.format(msg, bitmap['name'],
+bitmap['count'] // 64 // 1024))
+else:
+log(msg + ': not found')
+
+
+def test(persistent, restart):
+assert persistent or not restart
+log("\nTestcase {}persistent {} restart\n".format(
+'' if persistent else 'non-', 'with' if restart else 'without'))
+
+qemu_img_create('-f', iotests.imgfmt, base, str(size))
+
+vm = iotests.VM().add_drive(base)
+vm.launch()
+
+vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap0',
+   persistent=persistent)
+vm.hmp_qemu_io('drive0', 'write 0 64K')
+print_bitmap('initial bitmap', vm)
+
+vm.qmp_log('blockdev-snapshot-sync', device='drive0', snapshot_file=top,
+   format=iotests.imgfmt, filters=[iotests.filter_qmp_testfiles])
+vm.hmp_qemu_io('drive0', 'write 64K 512')
+print_bitmap('check that no bitmaps are in snapshot', vm)
+
+if restart:
+log("... Restart ...")
+vm.shutdown()
+vm = iotests.VM().add_drive(top)
+vm.launch()
+
+vm.qmp_log('block-commit', device='drive0', top=top,
+   filters=[iotests.filter_qmp_testfiles])
+ev = vm.events_wait((('BLOCK_JOB_READY', None),
+ ('BLOCK_JOB_COMPLETED', None)))
+log(filter_qmp_event(ev))
+if (ev['event'] == 'BLOCK_JOB_COMPLETED'):
+vm.shutdown()
+log(vm.get_log())
+exit()
+
+vm.qmp_log('block-job-complete', device='drive0')
+ev = vm.event_wait('BLOCK_JOB_COMPLETED')
+log(filter_qmp_event(ev))
+print_bitmap('check bitmap after commit', vm)
+
+vm.hmp_qemu_io('drive0', 'write 128K 64K')
+print_bitmap('check updated bitmap', vm)
+
+vm.shutdown()
+
+
+test(persistent=False, restart=False)
+test(persistent=True, restart=False)
+test(persistent=True, restart=True)
diff --git a/tests/qemu-iotests/260.out b/tests/qemu-iotests/260.out
new file mode 100644
index 000..2f0d98d0365
--- /dev/null
+++ b/tests/qemu-iotests/260.out
@@ -0,0 +1,52 @@
+
+Testcase non-persistent without restart
+
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap0", "node": 
"drive0", "persistent": false}}
+{"return": {}}
+initial bitmap: name=bitmap0 dirty-clusters=1
+{"execute": "blockdev-snapshot-sync", "arguments": {"device": "drive0", 
"format": "qcow2", "snapshot-file": "TEST_DIR/PID-top"}}
+{"return": {}}
+check that no bitmaps are in snapshot: not found
+{"execute": "block-commit", "arguments": {"device": "drive0", "top": 
"TEST_DIR/PID-top"}}
+{"return": {}}
+{"data": {"device": "drive0", "len": 65536, "offset": 65536, "speed": 0, 
"type": "commit"}, "event": "BLOCK_JOB_READY", "timestamp": {"microseconds": 
"U

[libvirt] [PULL 13/19] block/qcow2-bitmap: drop qcow2_reopen_bitmaps_rw_hint()

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

The function is unused, drop it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190927122355.7344-6-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  2 --
 block/qcow2-bitmap.c | 15 +--
 2 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index 113d99bf520..b3398a13c20 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -737,8 +737,6 @@ int qcow2_check_bitmaps_refcounts(BlockDriverState *bs, 
BdrvCheckResult *res,
 bool qcow2_load_dirty_bitmaps(BlockDriverState *bs, Error **errp);
 Qcow2BitmapInfoList *qcow2_get_bitmap_info_list(BlockDriverState *bs,
 Error **errp);
-int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, bool *header_updated,
- Error **errp);
 int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp);
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp);
 void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp);
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 6dfc0835485..ebc1afccd3d 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1102,8 +1102,7 @@ Qcow2BitmapInfoList 
*qcow2_get_bitmap_info_list(BlockDriverState *bs,
 return list;
 }
 
-int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, bool *header_updated,
- Error **errp)
+int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)
 {
 BDRVQcow2State *s = bs->opaque;
 Qcow2BitmapList *bm_list;
@@ -,10 +1110,6 @@ int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, 
bool *header_updated,
 GSList *ro_dirty_bitmaps = NULL;
 int ret = 0;
 
-if (header_updated != NULL) {
-*header_updated = false;
-}
-
 if (s->nb_bitmaps == 0) {
 /* No bitmaps - nothing to do */
 return 0;
@@ -1156,9 +1151,6 @@ int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, 
bool *header_updated,
 error_setg_errno(errp, -ret, "Can't update bitmap directory");
 goto out;
 }
-if (header_updated != NULL) {
-*header_updated = true;
-}
 g_slist_foreach(ro_dirty_bitmaps, set_readonly_helper, false);
 }
 
@@ -1169,11 +1161,6 @@ out:
 return ret;
 }
 
-int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)
-{
-return qcow2_reopen_bitmaps_rw_hint(bs, NULL, errp);
-}
-
 /* Checks to see if it's safe to resize bitmaps */
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp)
 {
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 18/19] MAINTAINERS: Add Vladimir as a reviewer for bitmaps

[John Snow]

I already try to make sure all bitmaps patches have been reviewed by both
Red Hat and Virtuozzo anyway, so this formalizes the arrangement.

Fam meanwhile is no longer as active, so I am removing him as a co-maintainer
simply to reflect the current practice.

Signed-off-by: John Snow 
Reviewed-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20191005194448.16629-2-js...@redhat.com
---
 MAINTAINERS | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/MAINTAINERS b/MAINTAINERS
index 3ca814850e0..a08c92a4162 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1816,8 +1816,8 @@ F: qapi/transaction.json
 T: git https://repo.or.cz/qemu/armbru.git block-next
 
 Dirty Bitmaps
-M: Fam Zheng 
 M: John Snow 
+R: Vladimir Sementsov-Ogievskiy 
 L: qemu-bl...@nongnu.org
 S: Supported
 F: util/hbitmap.c
@@ -1826,7 +1826,6 @@ F: include/qemu/hbitmap.h
 F: include/block/dirty-bitmap.h
 F: tests/test-hbitmap.c
 F: docs/interop/bitmaps.rst
-T: git https://github.com/famz/qemu.git bitmaps
 T: git https://github.com/jnsnow/qemu.git bitmaps
 
 Character device backends
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 14/19] block/qcow2-bitmap: do not remove bitmaps on reopen-ro

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

qcow2_reopen_bitmaps_ro wants to store bitmaps and then mark them all
readonly. But the latter don't work, as
qcow2_store_persistent_dirty_bitmaps removes bitmaps after storing.
It's OK for inactivation but bad idea for reopen-ro. And this leads to
the following bug:

Assume we have persistent bitmap 'bitmap0'.
Create external snapshot
  bitmap0 is stored and therefore removed
Commit snapshot
  now we have no bitmaps
Do some writes from guest (*)
  they are not marked in bitmap
Shutdown
Start
  bitmap0 is loaded as valid, but it is actually broken! It misses
  writes (*)
Incremental backup
  it will be inconsistent

So, let's stop removing bitmaps on reopen-ro. But don't rejoice:
reopening bitmaps to rw is broken too, so the whole scenario will not
work after this patch and we can't enable corresponding test cases in
260 iotests still. Reopening bitmaps rw will be fixed in the following
patches.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190927122355.7344-7-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  3 ++-
 block/qcow2-bitmap.c | 49 ++--
 block/qcow2.c|  2 +-
 3 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index b3398a13c20..8d293f2b64e 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -739,7 +739,8 @@ Qcow2BitmapInfoList 
*qcow2_get_bitmap_info_list(BlockDriverState *bs,
 Error **errp);
 int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp);
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp);
-void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp);
+void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
+  bool release_stored, Error **errp);
 int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error **errp);
 bool qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
  const char *name,
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index ebc1afccd3d..f7dfb40256e 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1440,7 +1440,32 @@ out:
 return ret;
 }
 
-void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
+/*
+ * qcow2_store_persistent_dirty_bitmaps
+ *
+ * Stores persistent BdrvDirtyBitmap objects.
+ *
+ * @release_stored: if true, release BdrvDirtyBitmap's after storing to the
+ * image. This is used in two cases, both via qcow2_inactivate:
+ * 1. bdrv_close: It's correct to remove bitmaps on close.
+ * 2. migration: If bitmaps are migrated through migration channel via
+ *'dirty-bitmaps' migration capability they are not handled by this code.
+ *Otherwise, it's OK to drop BdrvDirtyBitmap's and reload them on
+ *invalidation.
+ *
+ * Anyway, it's correct to remove BdrvDirtyBitmap's on inactivation, as
+ * inactivation means that we lose control on disk, and therefore on bitmaps,
+ * we should sync them and do not touch more.
+ *
+ * Contrariwise, we don't want to release any bitmaps on just reopen-to-ro,
+ * when we need to store them, as image is still under our control, and it's
+ * good to keep all the bitmaps in read-only mode. Moreover, keeping them
+ * read-only is correct because this is what would happen if we opened the node
+ * readonly to begin with, and whether we opened directly or reopened to that
+ * state shouldn't matter for the state we get afterward.
+ */
+void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
+  bool release_stored, Error **errp)
 {
 BdrvDirtyBitmap *bitmap;
 BDRVQcow2State *s = bs->opaque;
@@ -1551,20 +1576,14 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 g_free(tb);
 }
 
-QSIMPLEQ_FOREACH(bm, bm_list, entry) {
-/* For safety, we remove bitmap after storing.
- * We may be here in two cases:
- * 1. bdrv_close. It's ok to drop bitmap.
- * 2. inactivation. It means migration without 'dirty-bitmaps'
- *capability, so bitmaps are not marked with
- *BdrvDirtyBitmap.migration flags. It's not bad to drop them too,
- *and reload on invalidation.
- */
-if (bm->dirty_bitmap == NULL) {
-continue;
-}
+if (release_stored) {
+QSIMPLEQ_FOREACH(bm, bm_list, entry) {
+if (bm->dirty_bitmap == NULL) {
+continue;
+}
 
-bdrv_release_dirty_bitmap(bm->dirty_bitmap);
+bdrv_release_dirty_bitmap(bm->dirty_bitmap);
+}
 }
 
 success:
@@ -1592,7 +1611,7 @@ int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error 
**errp)
 BdrvDirtyBitmap *bitmap;
 Error *local_err = NULL;
 
-qcow2_store_persistent_dirty_b

[libvirt] [PULL 19/19] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

This parameter has been deprecated since 2.12.0 and is eligible for
removal. Remove this parameter as it is actually completely ignored;
let's not give false hope.

Signed-off-by: John Snow 
Reviewed-by: Eric Blake 
Reviewed-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20191002232411.29968-1-js...@redhat.com
---
 qemu-deprecated.texi | 20 +++-
 qapi/block-core.json |  6 +-
 blockdev.c   |  6 --
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index 01245e0b1c4..7239e0959da 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -149,11 +149,6 @@ QEMU 4.1 has three options, please migrate to one of these 
three:
 
 @section QEMU Machine Protocol (QMP) commands
 
-@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
-
-"autoload" parameter is now ignored. All bitmaps are automatically loaded
-from qcow2 images.
-
 @subsection query-block result field dirty-bitmaps[i].status (since 4.0)
 
 The ``status'' field of the ``BlockDirtyInfo'' structure, returned by
@@ -356,3 +351,18 @@ existing CPU models.  Management software that needs 
runnability
 guarantees must resolve the CPU model aliases using te
 ``alias-of'' field returned by the ``query-cpu-definitions'' QMP
 command.
+
+
+@node Recently removed features
+@appendix Recently removed features
+
+What follows is a record of recently removed, formerly deprecated
+features that serves as a record for users who have encountered
+trouble after a recent upgrade.
+
+@section QEMU Machine Protocol (QMP) commands
+
+@subsection block-dirty-bitmap-add "autoload" parameter (since 4.2.0)
+
+The "autoload" parameter has been ignored since 2.12.0. All bitmaps
+are automatically loaded from qcow2 images.
diff --git a/qapi/block-core.json b/qapi/block-core.json
index e6edd641f18..e4975ece4ab 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -1987,10 +1987,6 @@
 #  Qcow2 disks support persistent bitmaps. Default is false for
 #  block-dirty-bitmap-add. (Since: 2.10)
 #
-# @autoload: ignored and deprecated since 2.12.
-#Currently, all dirty tracking bitmaps are loaded from Qcow2 on
-#open.
-#
 # @disabled: the bitmap is created in the disabled state, which means that
 #it will not track drive changes. The bitmap may be enabled with
 #block-dirty-bitmap-enable. Default is false. (Since: 4.0)
@@ -1999,7 +1995,7 @@
 ##
 { 'struct': 'BlockDirtyBitmapAdd',
   'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32',
-'*persistent': 'bool', '*autoload': 'bool', '*disabled': 'bool' } }
+'*persistent': 'bool', '*disabled': 'bool' } }
 
 ##
 # @BlockDirtyBitmapMergeSource:
diff --git a/blockdev.c b/blockdev.c
index 9b6eca66430..873aba3c27f 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -1966,7 +1966,6 @@ static void block_dirty_bitmap_add_prepare(BlkActionState 
*common,
 qmp_block_dirty_bitmap_add(action->node, action->name,
action->has_granularity, action->granularity,
action->has_persistent, action->persistent,
-   action->has_autoload, action->autoload,
action->has_disabled, action->disabled,
_err);
 
@@ -2858,7 +2857,6 @@ out:
 void qmp_block_dirty_bitmap_add(const char *node, const char *name,
 bool has_granularity, uint32_t granularity,
 bool has_persistent, bool persistent,
-bool has_autoload, bool autoload,
 bool has_disabled, bool disabled,
 Error **errp)
 {
@@ -2890,10 +2888,6 @@ void qmp_block_dirty_bitmap_add(const char *node, const 
char *name,
 persistent = false;
 }
 
-if (has_autoload) {
-warn_report("Autoload option is deprecated and its value is ignored");
-}
-
 if (!has_disabled) {
 disabled = false;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 12/19] block/qcow2-bitmap: get rid of bdrv_has_changed_persistent_bitmaps

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Firstly, no reason to optimize failure path. Then, function name is
ambiguous: it checks for readonly and similar things, but someone may
think that it will ignore normal bitmaps which was just unchanged, and
this is in bad relation with the fact that we should drop IN_USE flag
for unchanged bitmaps in the image.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190927122355.7344-5-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h |  1 -
 block/dirty-bitmap.c | 12 
 block/qcow2-bitmap.c | 23 +--
 3 files changed, 13 insertions(+), 23 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 257f0f67046..958e7474fb5 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -95,7 +95,6 @@ bool bdrv_has_readonly_bitmaps(BlockDriverState *bs);
 bool bdrv_dirty_bitmap_get_autoload(const BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap);
-bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs);
 
 BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs);
 BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap);
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 6065db80949..4bbb251b2c9 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -839,18 +839,6 @@ bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap 
*bitmap)
 return bitmap->inconsistent;
 }
 
-bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs)
-{
-BdrvDirtyBitmap *bm;
-QLIST_FOREACH(bm, >dirty_bitmaps, list) {
-if (bm->persistent && !bm->readonly && !bm->skip_store) {
-return true;
-}
-}
-
-return false;
-}
-
 BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs)
 {
 return QLIST_FIRST(>dirty_bitmaps);
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 99812b418b8..6dfc0835485 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1464,16 +1464,7 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 Qcow2Bitmap *bm;
 QSIMPLEQ_HEAD(, Qcow2BitmapTable) drop_tables;
 Qcow2BitmapTable *tb, *tb_next;
-
-if (!bdrv_has_changed_persistent_bitmaps(bs)) {
-/* nothing to do */
-return;
-}
-
-if (!can_write(bs)) {
-error_setg(errp, "No write access");
-return;
-}
+bool need_write = false;
 
 QSIMPLEQ_INIT(_tables);
 
@@ -1499,6 +1490,8 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 continue;
 }
 
+need_write = true;
+
 if (check_constraints_on_bitmap(bs, name, granularity, errp) < 0) {
 error_prepend(errp, "Bitmap '%s' doesn't satisfy the constraints: 
",
   name);
@@ -1537,6 +1530,15 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 bm->dirty_bitmap = bitmap;
 }
 
+if (!need_write) {
+goto success;
+}
+
+if (!can_write(bs)) {
+error_setg(errp, "No write access");
+goto fail;
+}
+
 /* allocate clusters and store bitmaps */
 QSIMPLEQ_FOREACH(bm, bm_list, entry) {
 if (bm->dirty_bitmap == NULL) {
@@ -1578,6 +1580,7 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 bdrv_release_dirty_bitmap(bm->dirty_bitmap);
 }
 
+success:
 bitmap_list_free(bm_list);
 return;
 
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] [PATCH v2 1/1] IDE: deprecate ide-drive

[John Snow]

On 10/11/19 5:12 AM, Peter Krempa wrote:
> On Thu, Oct 10, 2019 at 14:08:12 -0400, John Snow wrote:
>> On 10/10/19 7:54 AM, Peter Krempa wrote:
>>> On Thu, Oct 10, 2019 at 13:42:26 +0200, Philippe Mathieu-Daudé wrote:
>>>> On 10/10/19 1:26 PM, Peter Krempa wrote:
> 
> [...]
> 
>>> To be honest, I didn't really review the code nor the documentation.
>>> I actually reviewed only the idea itself in the context of integration
>>> with libvirt and that's why I didn't go for 'Reviewed-by:'.
>>>
>>> The gist of the citation above is that we should stick to well known
>>> tags with their well known meanings and I think that considering this a
>>> 'review' would be a stretch of the definiton.
>>>
>>
>> I wasn't aware that PMM wanted to avoid non-standard tags; I consider
>> them to be for human use, but I can change that behavior.
>>
>> Peter, I'll change it to an ACK (as suggested by Kevin) is that's OK by you.
> 
> Sure! I'll spell it out even for compliance:
> 
> ACKed-by: Peter Krempa 
> 

Thanks, applied to my IDE tree:

https://github.com/jnsnow/qemu/commits/ide
https://github.com/jnsnow/qemu.git

--js

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 00/19] Bitmaps patches

[John Snow]

The following changes since commit 98b2e3c9ab3abfe476a2b02f8f51813edb90e72d:

  Merge remote-tracking branch 'remotes/stefanha/tags/block-pull-request' into 
staging (2019-10-08 16:08:35 +0100)

are available in the Git repository at:

  https://github.com/jnsnow/qemu.git tags/bitmaps-pull-request

for you to fetch changes up to b97d9a1014b61dd0980e7f4a0c9ca1e3b0aaa761:

  dirty-bitmaps: remove deprecated autoload parameter (2019-10-09 17:02:45 
-0400)


Pull request



John Snow (2):
  MAINTAINERS: Add Vladimir as a reviewer for bitmaps
  dirty-bitmaps: remove deprecated autoload parameter

Vladimir Sementsov-Ogievskiy (17):
  util/hbitmap: strict hbitmap_reset
  block: move bdrv_can_store_new_dirty_bitmap to block/dirty-bitmap.c
  block/dirty-bitmap: return int from
bdrv_remove_persistent_dirty_bitmap
  block/qcow2: proper locking on bitmap add/remove paths
  block/dirty-bitmap: drop meta
  block/dirty-bitmap: add bs link
  block/dirty-bitmap: drop BdrvDirtyBitmap.mutex
  block/dirty-bitmap: refactor bdrv_dirty_bitmap_next
  block: switch reopen queue from QSIMPLEQ to QTAILQ
  block: reverse order for reopen commits
  iotests: add test-case to 165 to test reopening qcow2 bitmaps to RW
  block/qcow2-bitmap: get rid of bdrv_has_changed_persistent_bitmaps
  block/qcow2-bitmap: drop qcow2_reopen_bitmaps_rw_hint()
  block/qcow2-bitmap: do not remove bitmaps on reopen-ro
  iotests: add test 260 to check bitmap life after snapshot + commit
  block/qcow2-bitmap: fix and improve qcow2_reopen_bitmaps_rw
  qcow2-bitmap: move bitmap reopen-rw code to qcow2_reopen_commit

 qemu-deprecated.texi   |  20 ++-
 qapi/block-core.json   |   6 +-
 block/qcow2.h  |  19 +--
 include/block/block.h  |   2 +-
 include/block/block_int.h  |  20 +--
 include/block/dirty-bitmap.h   |  34 ++--
 include/qemu/hbitmap.h |   5 +
 block.c|  79 +++--
 block/backup.c |  14 +-
 block/dirty-bitmap.c   | 290 +++--
 block/mirror.c |   4 +-
 block/qcow2-bitmap.c   | 212 +++-
 block/qcow2.c  |  22 ++-
 blockdev.c |  40 ++---
 migration/block-dirty-bitmap.c |  11 +-
 migration/block.c  |   4 +-
 tests/test-hbitmap.c   |   2 +-
 util/hbitmap.c |   4 +
 MAINTAINERS|   3 +-
 tests/qemu-iotests/165 |  57 ++-
 tests/qemu-iotests/165.out |   4 +-
 tests/qemu-iotests/260 |  89 ++
 tests/qemu-iotests/260.out |  52 ++
 tests/qemu-iotests/group   |   1 +
 24 files changed, 624 insertions(+), 370 deletions(-)
 create mode 100755 tests/qemu-iotests/260
 create mode 100644 tests/qemu-iotests/260.out

-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 02/19] block: move bdrv_can_store_new_dirty_bitmap to block/dirty-bitmap.c

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

block/dirty-bitmap.c seems to be more appropriate for it and
bdrv_remove_persistent_dirty_bitmap already in it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190920082543.23444-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block.c  | 22 --
 block/dirty-bitmap.c | 22 ++
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/block.c b/block.c
index 59441248451..bea03cfcc92 100644
--- a/block.c
+++ b/block.c
@@ -6555,25 +6555,3 @@ void bdrv_del_child(BlockDriverState *parent_bs, 
BdrvChild *child, Error **errp)
 
 parent_bs->drv->bdrv_del_child(parent_bs, child, errp);
 }
-
-bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
- uint32_t granularity, Error **errp)
-{
-BlockDriver *drv = bs->drv;
-
-if (!drv) {
-error_setg_errno(errp, ENOMEDIUM,
- "Can't store persistent bitmaps to %s",
- bdrv_get_device_or_node_name(bs));
-return false;
-}
-
-if (!drv->bdrv_can_store_new_dirty_bitmap) {
-error_setg_errno(errp, ENOTSUP,
- "Can't store persistent bitmaps to %s",
- bdrv_get_device_or_node_name(bs));
-return false;
-}
-
-return drv->bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp);
-}
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 134e0c9a0c8..8f42015db95 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -464,6 +464,28 @@ void bdrv_remove_persistent_dirty_bitmap(BlockDriverState 
*bs,
 }
 }
 
+bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
+ uint32_t granularity, Error **errp)
+{
+BlockDriver *drv = bs->drv;
+
+if (!drv) {
+error_setg_errno(errp, ENOMEDIUM,
+ "Can't store persistent bitmaps to %s",
+ bdrv_get_device_or_node_name(bs));
+return false;
+}
+
+if (!drv->bdrv_can_store_new_dirty_bitmap) {
+error_setg_errno(errp, ENOTSUP,
+ "Can't store persistent bitmaps to %s",
+ bdrv_get_device_or_node_name(bs));
+return false;
+}
+
+return drv->bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp);
+}
+
 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
 bdrv_dirty_bitmap_lock(bitmap);
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 07/19] block/dirty-bitmap: drop BdrvDirtyBitmap.mutex

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

mutex field is just a pointer to bs->dirty_bitmap_mutex, so no needs
to store it in BdrvDirtyBitmap when we have bs pointer in it (since
previous patch).

Drop mutex field. Constantly use bdrv_dirty_bitmaps_lock/unlock in
block/dirty-bitmap.c to make it more obvious that it's not per-bitmap
lock. Still, for simplicity, leave bdrv_dirty_bitmap_lock/unlock
functions as an external API.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-4-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/dirty-bitmap.c | 84 +---
 1 file changed, 41 insertions(+), 43 deletions(-)

diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 44453ff8241..4e5c87a907f 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -29,7 +29,6 @@
 #include "qemu/main-loop.h"
 
 struct BdrvDirtyBitmap {
-QemuMutex *mutex;
 BlockDriverState *bs;
 HBitmap *bitmap;/* Dirty bitmap implementation */
 bool busy;  /* Bitmap is busy, it can't be used via QMP */
@@ -72,12 +71,12 @@ static inline void 
bdrv_dirty_bitmaps_unlock(BlockDriverState *bs)
 
 void bdrv_dirty_bitmap_lock(BdrvDirtyBitmap *bitmap)
 {
-qemu_mutex_lock(bitmap->mutex);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 }
 
 void bdrv_dirty_bitmap_unlock(BdrvDirtyBitmap *bitmap)
 {
-qemu_mutex_unlock(bitmap->mutex);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 /* Called with BQL or dirty_bitmap lock taken.  */
@@ -117,7 +116,6 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
 }
 bitmap = g_new0(BdrvDirtyBitmap, 1);
 bitmap->bs = bs;
-bitmap->mutex = >dirty_bitmap_mutex;
 bitmap->bitmap = hbitmap_alloc(bitmap_size, ctz32(granularity));
 bitmap->size = bitmap_size;
 bitmap->name = g_strdup(name);
@@ -151,9 +149,9 @@ static bool bdrv_dirty_bitmap_busy(const BdrvDirtyBitmap 
*bitmap)
 
 void bdrv_dirty_bitmap_set_busy(BdrvDirtyBitmap *bitmap, bool busy)
 {
-qemu_mutex_lock(bitmap->mutex);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bitmap->busy = busy;
-qemu_mutex_unlock(bitmap->mutex);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 /* Called with BQL taken.  */
@@ -278,10 +276,10 @@ void bdrv_enable_dirty_bitmap_locked(BdrvDirtyBitmap 
*bitmap)
 /* Called with BQL taken. */
 void bdrv_dirty_bitmap_enable_successor(BdrvDirtyBitmap *bitmap)
 {
-assert(bitmap->mutex == bitmap->successor->mutex);
-qemu_mutex_lock(bitmap->mutex);
+assert(bitmap->bs == bitmap->successor->bs);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bdrv_enable_dirty_bitmap_locked(bitmap->successor);
-qemu_mutex_unlock(bitmap->mutex);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 /* Called within bdrv_dirty_bitmap_lock..unlock and with BQL taken.  */
@@ -361,9 +359,9 @@ BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BdrvDirtyBitmap 
*parent,
 {
 BdrvDirtyBitmap *ret;
 
-qemu_mutex_lock(parent->mutex);
+bdrv_dirty_bitmaps_lock(parent->bs);
 ret = bdrv_reclaim_dirty_bitmap_locked(parent, errp);
-qemu_mutex_unlock(parent->mutex);
+bdrv_dirty_bitmaps_unlock(parent->bs);
 
 return ret;
 }
@@ -543,16 +541,16 @@ bool bdrv_can_store_new_dirty_bitmap(BlockDriverState 
*bs, const char *name,
 
 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
-bdrv_dirty_bitmap_lock(bitmap);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bitmap->disabled = true;
-bdrv_dirty_bitmap_unlock(bitmap);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
-bdrv_dirty_bitmap_lock(bitmap);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bdrv_enable_dirty_bitmap_locked(bitmap);
-bdrv_dirty_bitmap_unlock(bitmap);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
@@ -593,9 +591,9 @@ bool bdrv_dirty_bitmap_get_locked(BdrvDirtyBitmap *bitmap, 
int64_t offset)
 bool bdrv_dirty_bitmap_get(BdrvDirtyBitmap *bitmap, int64_t offset)
 {
 bool ret;
-bdrv_dirty_bitmap_lock(bitmap);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 ret = bdrv_dirty_bitmap_get_locked(bitmap, offset);
-bdrv_dirty_bitmap_unlock(bitmap);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 
 return ret;
 }
@@ -660,9 +658,9 @@ void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
 void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
int64_t offset, int64_t bytes)
 {
-bdrv_dirty_bitmap_lock(bitmap);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bdrv_set_dirty_bitmap_locked(bitmap, offset, bytes);
-bdrv_dirty_bitmap_unlock(bitmap);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 /* Called within bdrv_dirty_bitmap_lock..unlock */
@@ -676

[libvirt] [PULL 05/19] block/dirty-bitmap: drop meta

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Drop meta bitmaps, as they are unused.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h |  5 
 block/dirty-bitmap.c | 46 
 2 files changed, 51 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 07503b03b53..973056778aa 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -18,9 +18,6 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
   uint32_t granularity,
   const char *name,
   Error **errp);
-void bdrv_create_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap,
-   int chunk_size);
-void bdrv_release_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
BdrvDirtyBitmap *bitmap,
Error **errp);
@@ -54,7 +51,6 @@ void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
int64_t offset, int64_t bytes);
 void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
  int64_t offset, int64_t bytes);
-BdrvDirtyBitmapIter *bdrv_dirty_meta_iter_new(BdrvDirtyBitmap *bitmap);
 BdrvDirtyBitmapIter *bdrv_dirty_iter_new(BdrvDirtyBitmap *bitmap);
 void bdrv_dirty_iter_free(BdrvDirtyBitmapIter *iter);
 
@@ -96,7 +92,6 @@ void bdrv_reset_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
 int64_t bdrv_dirty_iter_next(BdrvDirtyBitmapIter *iter);
 void bdrv_set_dirty_iter(BdrvDirtyBitmapIter *hbi, int64_t offset);
 int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap);
-int64_t bdrv_get_meta_dirty_count(BdrvDirtyBitmap *bitmap);
 void bdrv_dirty_bitmap_truncate(BlockDriverState *bs, int64_t bytes);
 bool bdrv_dirty_bitmap_readonly(const BdrvDirtyBitmap *bitmap);
 bool bdrv_has_readonly_bitmaps(BlockDriverState *bs);
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 03e0872b972..4ecf18d5df7 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -31,7 +31,6 @@
 struct BdrvDirtyBitmap {
 QemuMutex *mutex;
 HBitmap *bitmap;/* Dirty bitmap implementation */
-HBitmap *meta;  /* Meta dirty bitmap */
 bool busy;  /* Bitmap is busy, it can't be used via QMP */
 BdrvDirtyBitmap *successor; /* Anonymous child, if any. */
 char *name; /* Optional non-empty unique ID */
@@ -127,36 +126,6 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
 return bitmap;
 }
 
-/* bdrv_create_meta_dirty_bitmap
- *
- * Create a meta dirty bitmap that tracks the changes of bits in @bitmap. I.e.
- * when a dirty status bit in @bitmap is changed (either from reset to set or
- * the other way around), its respective meta dirty bitmap bit will be marked
- * dirty as well.
- *
- * @bitmap: the block dirty bitmap for which to create a meta dirty bitmap.
- * @chunk_size: how many bytes of bitmap data does each bit in the meta bitmap
- * track.
- */
-void bdrv_create_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap,
-   int chunk_size)
-{
-assert(!bitmap->meta);
-qemu_mutex_lock(bitmap->mutex);
-bitmap->meta = hbitmap_create_meta(bitmap->bitmap,
-   chunk_size * BITS_PER_BYTE);
-qemu_mutex_unlock(bitmap->mutex);
-}
-
-void bdrv_release_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap)
-{
-assert(bitmap->meta);
-qemu_mutex_lock(bitmap->mutex);
-hbitmap_free_meta(bitmap->bitmap);
-bitmap->meta = NULL;
-qemu_mutex_unlock(bitmap->mutex);
-}
-
 int64_t bdrv_dirty_bitmap_size(const BdrvDirtyBitmap *bitmap)
 {
 return bitmap->size;
@@ -320,7 +289,6 @@ static void 
bdrv_release_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)
 assert(!bitmap->active_iterators);
 assert(!bdrv_dirty_bitmap_busy(bitmap));
 assert(!bdrv_dirty_bitmap_has_successor(bitmap));
-assert(!bitmap->meta);
 QLIST_REMOVE(bitmap, list);
 hbitmap_free(bitmap->bitmap);
 g_free(bitmap->name);
@@ -666,15 +634,6 @@ BdrvDirtyBitmapIter *bdrv_dirty_iter_new(BdrvDirtyBitmap 
*bitmap)
 return iter;
 }
 
-BdrvDirtyBitmapIter *bdrv_dirty_meta_iter_new(BdrvDirtyBitmap *bitmap)
-{
-BdrvDirtyBitmapIter *iter = g_new(BdrvDirtyBitmapIter, 1);
-hbitmap_iter_init(>hbi, bitmap->meta, 0);
-iter->bitmap = bitmap;
-bitmap->active_iterators++;
-return iter;
-}
-
 void bdrv_dirty_iter_free(BdrvDirtyBitmapIter *iter)
 {
 if (!iter) {
@@ -821,11 +780,6 @@ int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap)
 return hbitmap_count(bitmap->bitmap);
 }
 
-int64_t bdrv_get_meta_dirty_count(

[libvirt] [PULL 03/19] block/dirty-bitmap: return int from bdrv_remove_persistent_dirty_bitmap

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

It's more comfortable to not deal with local_err.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190920082543.23444-3-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  5 ++---
 include/block/block_int.h|  6 +++---
 include/block/dirty-bitmap.h |  5 ++---
 block/dirty-bitmap.c |  9 +
 block/qcow2-bitmap.c | 18 ++
 blockdev.c   |  7 +++
 6 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index a488d761ff0..2ed54821635 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -747,9 +747,8 @@ bool qcow2_can_store_new_dirty_bitmap(BlockDriverState *bs,
   const char *name,
   uint32_t granularity,
   Error **errp);
-void qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
-  const char *name,
-  Error **errp);
+int qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char 
*name,
+ Error **errp);
 
 ssize_t coroutine_fn
 qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 0422acdf1c4..503ac9e3cd2 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -556,9 +556,9 @@ struct BlockDriver {
 const char *name,
 uint32_t granularity,
 Error **errp);
-void (*bdrv_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
-const char *name,
-Error **errp);
+int (*bdrv_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
+   const char *name,
+   Error **errp);
 
 /**
  * Register/unregister a buffer for I/O. For example, when the driver is
diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 4b4b731b469..07503b03b53 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -37,9 +37,8 @@ int bdrv_dirty_bitmap_check(const BdrvDirtyBitmap *bitmap, 
uint32_t flags,
 Error **errp);
 void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
 void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs);
-void bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs,
- const char *name,
- Error **errp);
+int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
+Error **errp);
 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_enable_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap);
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 8f42015db95..d1ae2e19229 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -455,13 +455,14 @@ void bdrv_release_named_dirty_bitmaps(BlockDriverState 
*bs)
  * not fail.
  * This function doesn't release corresponding BdrvDirtyBitmap.
  */
-void bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs,
- const char *name,
- Error **errp)
+int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
+Error **errp)
 {
 if (bs->drv && bs->drv->bdrv_remove_persistent_dirty_bitmap) {
-bs->drv->bdrv_remove_persistent_dirty_bitmap(bs, name, errp);
+return bs->drv->bdrv_remove_persistent_dirty_bitmap(bs, name, errp);
 }
+
+return 0;
 }
 
 bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index b2487101ede..9821c1628f5 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1404,9 +1404,8 @@ static Qcow2Bitmap *find_bitmap_by_name(Qcow2BitmapList 
*bm_list,
 return NULL;
 }
 
-void qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
-  const char *name,
-  Error **errp)
+int qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char 
*name,
+ Error **errp)
 {
 int ret;
 BDRVQcow2State *s = bs->opaque;
@@ -1416,18 +1415,19 @@ void 
qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
 if (s->nb_bitmaps == 0) {
 /* Absence of the bitmap is 

[libvirt] [PULL 04/19] block/qcow2: proper locking on bitmap add/remove paths

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

qmp_block_dirty_bitmap_add and do_block_dirty_bitmap_remove do acquire
aio context since 0a6c86d024c52b. But this is not enough: we also must
lock qcow2 mutex when access in-image metadata. Especially it concerns
freeing qcow2 clusters.

To achieve this, move qcow2_can_store_new_dirty_bitmap and
qcow2_remove_persistent_dirty_bitmap to coroutine context.

Since we work in coroutines in correct aio context, we don't need
context acquiring in blockdev.c anymore, drop it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190920082543.23444-4-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h |  11 ++--
 include/block/block_int.h |  10 ++--
 block/dirty-bitmap.c  | 102 +++---
 block/qcow2-bitmap.c  |  24 ++---
 block/qcow2.c |   5 +-
 blockdev.c|  27 +++---
 6 files changed, 131 insertions(+), 48 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index 2ed54821635..113d99bf520 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -743,12 +743,13 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error 
**errp);
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp);
 void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp);
 int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error **errp);
-bool qcow2_can_store_new_dirty_bitmap(BlockDriverState *bs,
-  const char *name,
-  uint32_t granularity,
-  Error **errp);
-int qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char 
*name,
+bool qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
+ const char *name,
+ uint32_t granularity,
  Error **errp);
+int qcow2_co_remove_persistent_dirty_bitmap(BlockDriverState *bs,
+const char *name,
+Error **errp);
 
 ssize_t coroutine_fn
 qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 503ac9e3cd2..1e54486ad14 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -552,13 +552,13 @@ struct BlockDriver {
  * field of BlockDirtyBitmap's in case of success.
  */
 int (*bdrv_reopen_bitmaps_rw)(BlockDriverState *bs, Error **errp);
-bool (*bdrv_can_store_new_dirty_bitmap)(BlockDriverState *bs,
-const char *name,
-uint32_t granularity,
-Error **errp);
-int (*bdrv_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
+bool (*bdrv_co_can_store_new_dirty_bitmap)(BlockDriverState *bs,
const char *name,
+   uint32_t granularity,
Error **errp);
+int (*bdrv_co_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
+  const char *name,
+  Error **errp);
 
 /**
  * Register/unregister a buffer for I/O. For example, when the driver is
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index d1ae2e19229..03e0872b972 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -26,6 +26,7 @@
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qemu/main-loop.h"
 
 struct BdrvDirtyBitmap {
 QemuMutex *mutex;
@@ -455,18 +456,59 @@ void bdrv_release_named_dirty_bitmaps(BlockDriverState 
*bs)
  * not fail.
  * This function doesn't release corresponding BdrvDirtyBitmap.
  */
+static int coroutine_fn
+bdrv_co_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
+   Error **errp)
+{
+if (bs->drv && bs->drv->bdrv_co_remove_persistent_dirty_bitmap) {
+return bs->drv->bdrv_co_remove_persistent_dirty_bitmap(bs, name, errp);
+}
+
+return 0;
+}
+
+typedef struct BdrvRemovePersistentDirtyBitmapCo {
+BlockDriverState *bs;
+const char *name;
+Error **errp;
+int ret;
+} BdrvRemovePersistentDirtyBitmapCo;
+
+static void coroutine_fn
+bdrv_co_remove_persistent_dirty_bitmap_entry(void *opaque)
+{
+BdrvRemovePersistentDirtyBitmapCo *s = opaque;
+
+s->ret = bdrv_co_remove_persistent_dirty_bitmap(s->bs, s->name, s->errp);
+aio_wait_kick();
+}
+
 int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
 Error **errp)
 {
-if (bs

[libvirt] [PULL 01/19] util/hbitmap: strict hbitmap_reset

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

hbitmap_reset has an unobvious property: it rounds requested region up.
It may provoke bugs, like in recently fixed write-blocking mode of
mirror: user calls reset on unaligned region, not keeping in mind that
there are possible unrelated dirty bytes, covered by rounded-up region
and information of this unrelated "dirtiness" will be lost.

Make hbitmap_reset strict: assert that arguments are aligned, allowing
only one exception when @start + @count == hb->orig_size. It's needed
to comfort users of hbitmap_next_dirty_area, which cares about
hb->orig_size.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Message-Id: <20190806152611.280389-1-vsement...@virtuozzo.com>
[Maintainer edit: Max's suggestions from on-list. --js]
Signed-off-by: John Snow 
---
 include/qemu/hbitmap.h | 5 +
 tests/test-hbitmap.c   | 2 +-
 util/hbitmap.c | 4 
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 4afbe6292e3..1bf944ca3d1 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -132,6 +132,11 @@ void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t 
count);
  * @count: Number of bits to reset.
  *
  * Reset a consecutive range of bits in an HBitmap.
+ * @start and @count must be aligned to bitmap granularity. The only exception
+ * is resetting the tail of the bitmap: @count may be equal to hb->orig_size -
+ * @start, in this case @count may be not aligned. The sum of @start + @count 
is
+ * allowed to be greater than hb->orig_size, but only if @start < hb->orig_size
+ * and @start + @count = ALIGN_UP(hb->orig_size, granularity).
  */
 void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count);
 
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
index eed5d288cbc..e1f867085f4 100644
--- a/tests/test-hbitmap.c
+++ b/tests/test-hbitmap.c
@@ -423,7 +423,7 @@ static void test_hbitmap_granularity(TestHBitmapData *data,
 hbitmap_test_check(data, 0);
 hbitmap_test_set(data, 0, 3);
 g_assert_cmpint(hbitmap_count(data->hb), ==, 4);
-hbitmap_test_reset(data, 0, 1);
+hbitmap_test_reset(data, 0, 2);
 g_assert_cmpint(hbitmap_count(data->hb), ==, 2);
 }
 
diff --git a/util/hbitmap.c b/util/hbitmap.c
index fd44c897ab0..757d39e360a 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -476,6 +476,10 @@ void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t 
count)
 /* Compute range in the last layer.  */
 uint64_t first;
 uint64_t last = start + count - 1;
+uint64_t gran = 1ULL << hb->granularity;
+
+assert(!(start & (gran - 1)));
+assert(!(count & (gran - 1)) || (start + count == hb->orig_size));
 
 trace_hbitmap_reset(hb, start, count,
 start >> hb->granularity, last >> hb->granularity);
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 11/19] iotests: add test-case to 165 to test reopening qcow2 bitmaps to RW

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Reopening bitmaps to RW was broken prior to previous commit. Check that
it works now.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-4-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 tests/qemu-iotests/165 | 57 --
 tests/qemu-iotests/165.out |  4 +--
 2 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/tests/qemu-iotests/165 b/tests/qemu-iotests/165
index 5650dc7c874..951ea011a27 100755
--- a/tests/qemu-iotests/165
+++ b/tests/qemu-iotests/165
@@ -43,10 +43,10 @@ class TestPersistentDirtyBitmap(iotests.QMPTestCase):
 os.remove(disk)
 
 def mkVm(self):
-return iotests.VM().add_drive(disk)
+return iotests.VM().add_drive(disk, opts='node-name=node0')
 
 def mkVmRo(self):
-return iotests.VM().add_drive(disk, opts='readonly=on')
+return iotests.VM().add_drive(disk, opts='readonly=on,node-name=node0')
 
 def getSha256(self):
 result = self.vm.qmp('x-debug-block-dirty-bitmap-sha256',
@@ -102,6 +102,59 @@ class TestPersistentDirtyBitmap(iotests.QMPTestCase):
 
 self.vm.shutdown()
 
+def test_reopen_rw(self):
+self.vm = self.mkVm()
+self.vm.launch()
+self.qmpAddBitmap()
+
+# Calculate hashes
+
+self.writeRegions(regions1)
+sha256_1 = self.getSha256()
+
+self.writeRegions(regions2)
+sha256_2 = self.getSha256()
+assert sha256_1 != sha256_2 # Otherwise, it's not very interesting.
+
+result = self.vm.qmp('block-dirty-bitmap-clear', node='drive0',
+ name='bitmap0')
+self.assert_qmp(result, 'return', {})
+
+# Start with regions1
+
+self.writeRegions(regions1)
+assert sha256_1 == self.getSha256()
+
+self.vm.shutdown()
+
+self.vm = self.mkVmRo()
+self.vm.launch()
+
+assert sha256_1 == self.getSha256()
+
+# Check that we are in RO mode and can't modify bitmap.
+self.writeRegions(regions2)
+assert sha256_1 == self.getSha256()
+
+# Reopen to RW
+result = self.vm.qmp('x-blockdev-reopen', **{
+'node-name': 'node0',
+'driver': iotests.imgfmt,
+'file': {
+'driver': 'file',
+'filename': disk
+},
+'read-only': False
+})
+self.assert_qmp(result, 'return', {})
+
+# Check that bitmap is reopened to RW and we can write to it.
+self.writeRegions(regions2)
+assert sha256_2 == self.getSha256()
+
+self.vm.shutdown()
+
+
 if __name__ == '__main__':
 iotests.main(supported_fmts=['qcow2'],
  supported_protocols=['file'])
diff --git a/tests/qemu-iotests/165.out b/tests/qemu-iotests/165.out
index ae1213e6f86..fbc63e62f88 100644
--- a/tests/qemu-iotests/165.out
+++ b/tests/qemu-iotests/165.out
@@ -1,5 +1,5 @@
-.
+..
 --
-Ran 1 tests
+Ran 2 tests
 
 OK
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL 08/19] block/dirty-bitmap: refactor bdrv_dirty_bitmap_next

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

bdrv_dirty_bitmap_next is always used in same pattern. So, split it
into _next and _first, instead of combining two functions into one and
add FOR_EACH_DIRTY_BITMAP macro.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-5-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h   |  9 +++--
 block.c|  4 +---
 block/dirty-bitmap.c   | 11 +++
 block/qcow2-bitmap.c   |  8 ++--
 migration/block-dirty-bitmap.c |  4 +---
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 2f9b088e11e..257f0f67046 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -96,8 +96,13 @@ bool bdrv_dirty_bitmap_get_autoload(const BdrvDirtyBitmap 
*bitmap);
 bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap);
 bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs);
-BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap);
+
+BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs);
+BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap);
+#define FOR_EACH_DIRTY_BITMAP(bs, bitmap) \
+for (bitmap = bdrv_dirty_bitmap_first(bs); bitmap; \
+ bitmap = bdrv_dirty_bitmap_next(bitmap))
+
 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp);
 int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t offset,
 uint64_t bytes);
diff --git a/block.c b/block.c
index bea03cfcc92..5b5b0337acc 100644
--- a/block.c
+++ b/block.c
@@ -5363,9 +5363,7 @@ static void coroutine_fn 
bdrv_co_invalidate_cache(BlockDriverState *bs,
 }
 }
 
-for (bm = bdrv_dirty_bitmap_next(bs, NULL); bm;
- bm = bdrv_dirty_bitmap_next(bs, bm))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bm) {
 bdrv_dirty_bitmap_skip_store(bm, false);
 }
 
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 4e5c87a907f..6065db80949 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -851,11 +851,14 @@ bool bdrv_has_changed_persistent_bitmaps(BlockDriverState 
*bs)
 return false;
 }
 
-BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap)
+BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs)
 {
-return bitmap == NULL ? QLIST_FIRST(>dirty_bitmaps) :
-QLIST_NEXT(bitmap, list);
+return QLIST_FIRST(>dirty_bitmaps);
+}
+
+BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap)
+{
+return QLIST_NEXT(bitmap, list);
 }
 
 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp)
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 687087d2bc2..99812b418b8 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1488,9 +1488,7 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 }
 
 /* check constraints and names */
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap != NULL;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 const char *name = bdrv_dirty_bitmap_name(bitmap);
 uint32_t granularity = bdrv_dirty_bitmap_granularity(bitmap);
 Qcow2Bitmap *bm;
@@ -1610,9 +1608,7 @@ int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error 
**errp)
 return -EINVAL;
 }
 
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap != NULL;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
 bdrv_dirty_bitmap_set_readonly(bitmap, true);
 }
diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c
index 793f249aa5b..7eafface614 100644
--- a/migration/block-dirty-bitmap.c
+++ b/migration/block-dirty-bitmap.c
@@ -283,9 +283,7 @@ static int init_dirty_bitmap_migration(void)
 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
 const char *name = bdrv_get_device_or_node_name(bs);
 
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 if (!bdrv_dirty_bitmap_name(bitmap)) {
 continue;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] [PULL 01/19] util/hbitmap: strict hbitmap_reset

[John Snow]

On 10/11/19 5:48 PM, Eric Blake wrote:
> On 10/11/19 4:25 PM, John Snow wrote:
>> From: Vladimir Sementsov-Ogievskiy 
>>
>> hbitmap_reset has an unobvious property: it rounds requested region up.
>> It may provoke bugs, like in recently fixed write-blocking mode of
>> mirror: user calls reset on unaligned region, not keeping in mind that
>> there are possible unrelated dirty bytes, covered by rounded-up region
>> and information of this unrelated "dirtiness" will be lost.
>>
>> Make hbitmap_reset strict: assert that arguments are aligned, allowing
>> only one exception when @start + @count == hb->orig_size. It's needed
>> to comfort users of hbitmap_next_dirty_area, which cares about
>> hb->orig_size.
>>
>> Signed-off-by: Vladimir Sementsov-Ogievskiy 
>> Reviewed-by: Max Reitz 
>> Message-Id: <20190806152611.280389-1-vsement...@virtuozzo.com>
>> [Maintainer edit: Max's suggestions from on-list. --js]
>> Signed-off-by: John Snow 
>> ---
>>   include/qemu/hbitmap.h | 5 +
>>   tests/test-hbitmap.c   | 2 +-
>>   util/hbitmap.c | 4 
>>   3 files changed, 10 insertions(+), 1 deletion(-)
>>
> 
>> +++ b/util/hbitmap.c
>> @@ -476,6 +476,10 @@ void hbitmap_reset(HBitmap *hb, uint64_t start,
>> uint64_t count)
>>   /* Compute range in the last layer.  */
>>   uint64_t first;
>>   uint64_t last = start + count - 1;
>> +    uint64_t gran = 1ULL << hb->granularity;
>> +
>> +    assert(!(start & (gran - 1)));
>> +    assert(!(count & (gran - 1)) || (start + count == hb->orig_size));
> 
> I know I'm replying a bit late (since this is now a pull request), but
> would it be worth using the dedicated macro:
> 
> assert(QEMU_IS_ALIGNED(start, gran));
> assert(QEMU_IS_ALIGNED(count, gran) || start + count == hb->orig_size);
> 
> instead of open-coding it?  (I would also drop the extra () around the
> right half of ||). If we want it, that would now be a followup patch.
> 

If the PR doesn't make it for some reason, I can amend a cleanup patch
for the next PR.

--js

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 04/19] block/qcow2: proper locking on bitmap add/remove paths

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

qmp_block_dirty_bitmap_add and do_block_dirty_bitmap_remove do acquire
aio context since 0a6c86d024c52b. But this is not enough: we also must
lock qcow2 mutex when access in-image metadata. Especially it concerns
freeing qcow2 clusters.

To achieve this, move qcow2_can_store_new_dirty_bitmap and
qcow2_remove_persistent_dirty_bitmap to coroutine context.

Since we work in coroutines in correct aio context, we don't need
context acquiring in blockdev.c anymore, drop it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190920082543.23444-4-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h |  11 ++--
 include/block/block_int.h |  10 ++--
 block/dirty-bitmap.c  | 102 +++---
 block/qcow2-bitmap.c  |  24 ++---
 block/qcow2.c |   5 +-
 blockdev.c|  27 +++---
 6 files changed, 131 insertions(+), 48 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index 08b4c15dc4..0f3d9b088e 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -746,12 +746,13 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error 
**errp);
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp);
 void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp);
 int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error **errp);
-bool qcow2_can_store_new_dirty_bitmap(BlockDriverState *bs,
-  const char *name,
-  uint32_t granularity,
-  Error **errp);
-int qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char 
*name,
+bool qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
+ const char *name,
+ uint32_t granularity,
  Error **errp);
+int qcow2_co_remove_persistent_dirty_bitmap(BlockDriverState *bs,
+const char *name,
+Error **errp);
 
 ssize_t coroutine_fn
 qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 6b511dd889..32fb493cbb 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -553,13 +553,13 @@ struct BlockDriver {
  * field of BlockDirtyBitmap's in case of success.
  */
 int (*bdrv_reopen_bitmaps_rw)(BlockDriverState *bs, Error **errp);
-bool (*bdrv_can_store_new_dirty_bitmap)(BlockDriverState *bs,
-const char *name,
-uint32_t granularity,
-Error **errp);
-int (*bdrv_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
+bool (*bdrv_co_can_store_new_dirty_bitmap)(BlockDriverState *bs,
const char *name,
+   uint32_t granularity,
Error **errp);
+int (*bdrv_co_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
+  const char *name,
+  Error **errp);
 
 /**
  * Register/unregister a buffer for I/O. For example, when the driver is
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index d1ae2e1922..03e0872b97 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -26,6 +26,7 @@
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qemu/main-loop.h"
 
 struct BdrvDirtyBitmap {
 QemuMutex *mutex;
@@ -455,18 +456,59 @@ void bdrv_release_named_dirty_bitmaps(BlockDriverState 
*bs)
  * not fail.
  * This function doesn't release corresponding BdrvDirtyBitmap.
  */
+static int coroutine_fn
+bdrv_co_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
+   Error **errp)
+{
+if (bs->drv && bs->drv->bdrv_co_remove_persistent_dirty_bitmap) {
+return bs->drv->bdrv_co_remove_persistent_dirty_bitmap(bs, name, errp);
+}
+
+return 0;
+}
+
+typedef struct BdrvRemovePersistentDirtyBitmapCo {
+BlockDriverState *bs;
+const char *name;
+Error **errp;
+int ret;
+} BdrvRemovePersistentDirtyBitmapCo;
+
+static void coroutine_fn
+bdrv_co_remove_persistent_dirty_bitmap_entry(void *opaque)
+{
+BdrvRemovePersistentDirtyBitmapCo *s = opaque;
+
+s->ret = bdrv_co_remove_persistent_dirty_bitmap(s->bs, s->name, s->errp);
+aio_wait_kick();
+}
+
 int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
 Error **errp)
 {
-if (bs->

[libvirt] [PULL v2 06/19] block/dirty-bitmap: add bs link

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Add bs field to BdrvDirtyBitmap structure. Drop BlockDriverState
parameter from bitmap APIs where possible.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-3-vsement...@virtuozzo.com
[Rebased on top of block-copy. --js]
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h   | 14 +-
 block/backup.c |  8 
 block/block-copy.c |  2 +-
 block/dirty-bitmap.c   | 24 
 block/mirror.c |  4 ++--
 block/qcow2-bitmap.c   |  6 +++---
 blockdev.c |  6 +++---
 migration/block-dirty-bitmap.c |  7 +++
 migration/block.c  |  4 ++--
 9 files changed, 35 insertions(+), 40 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 973056778a..2f9b088e11 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -18,21 +18,18 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
   uint32_t granularity,
   const char *name,
   Error **errp);
-int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
-   BdrvDirtyBitmap *bitmap,
+int bdrv_dirty_bitmap_create_successor(BdrvDirtyBitmap *bitmap,
Error **errp);
-BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BdrvDirtyBitmap *bitmap,
 Error **errp);
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
-   BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BdrvDirtyBitmap *bitmap,
Error **errp);
 void bdrv_dirty_bitmap_enable_successor(BdrvDirtyBitmap *bitmap);
 BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs,
 const char *name);
 int bdrv_dirty_bitmap_check(const BdrvDirtyBitmap *bitmap, uint32_t flags,
 Error **errp);
-void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
+void bdrv_release_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs);
 int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
 Error **errp);
@@ -106,8 +103,7 @@ int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap 
*bitmap, uint64_t offset,
 uint64_t bytes);
 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
uint64_t *offset, uint64_t *bytes);
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BlockDriverState *bs,
-  BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
   Error **errp);
 
 #endif
diff --git a/block/backup.c b/block/backup.c
index 46978c1785..dddcf77f53 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -98,13 +98,13 @@ static void backup_cleanup_sync_bitmap(BackupBlockJob *job, 
int ret)
  * We succeeded, or we always intended to sync the bitmap.
  * Delete this bitmap and install the child.
  */
-bm = bdrv_dirty_bitmap_abdicate(job->source_bs, job->sync_bitmap, 
NULL);
+bm = bdrv_dirty_bitmap_abdicate(job->sync_bitmap, NULL);
 } else {
 /*
  * We failed, or we never intended to sync the bitmap anyway.
  * Merge the successor back into the parent, keeping all data.
  */
-bm = bdrv_reclaim_dirty_bitmap(job->source_bs, job->sync_bitmap, NULL);
+bm = bdrv_reclaim_dirty_bitmap(job->sync_bitmap, NULL);
 }
 
 assert(bm);
@@ -402,7 +402,7 @@ BlockJob *backup_job_create(const char *job_id, 
BlockDriverState *bs,
 }
 
 /* Create a new bitmap, and freeze/disable this one. */
-if (bdrv_dirty_bitmap_create_successor(bs, sync_bitmap, errp) < 0) {
+if (bdrv_dirty_bitmap_create_successor(sync_bitmap, errp) < 0) {
 return NULL;
 }
 }
@@ -472,7 +472,7 @@ BlockJob *backup_job_create(const char *job_id, 
BlockDriverState *bs,
 
  error:
 if (sync_bitmap) {
-bdrv_reclaim_dirty_bitmap(bs, sync_bitmap, NULL);
+bdrv_reclaim_dirty_bitmap(sync_bitmap, NULL);
 }
 if (job) {
 backup_clean(>common.job);
diff --git a/block/block-copy.c b/block/block-copy.c
index 0f76ea1e63..066e3a7274 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -60,7 +60,7 @@ void block_copy_s

[libvirt] [PULL v2 11/19] iotests: add test-case to 165 to test reopening qcow2 bitmaps to RW

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Reopening bitmaps to RW was broken prior to previous commit. Check that
it works now.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-4-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 tests/qemu-iotests/165 | 57 --
 tests/qemu-iotests/165.out |  4 +--
 2 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/tests/qemu-iotests/165 b/tests/qemu-iotests/165
index 5650dc7c87..951ea011a2 100755
--- a/tests/qemu-iotests/165
+++ b/tests/qemu-iotests/165
@@ -43,10 +43,10 @@ class TestPersistentDirtyBitmap(iotests.QMPTestCase):
 os.remove(disk)
 
 def mkVm(self):
-return iotests.VM().add_drive(disk)
+return iotests.VM().add_drive(disk, opts='node-name=node0')
 
 def mkVmRo(self):
-return iotests.VM().add_drive(disk, opts='readonly=on')
+return iotests.VM().add_drive(disk, opts='readonly=on,node-name=node0')
 
 def getSha256(self):
 result = self.vm.qmp('x-debug-block-dirty-bitmap-sha256',
@@ -102,6 +102,59 @@ class TestPersistentDirtyBitmap(iotests.QMPTestCase):
 
 self.vm.shutdown()
 
+def test_reopen_rw(self):
+self.vm = self.mkVm()
+self.vm.launch()
+self.qmpAddBitmap()
+
+# Calculate hashes
+
+self.writeRegions(regions1)
+sha256_1 = self.getSha256()
+
+self.writeRegions(regions2)
+sha256_2 = self.getSha256()
+assert sha256_1 != sha256_2 # Otherwise, it's not very interesting.
+
+result = self.vm.qmp('block-dirty-bitmap-clear', node='drive0',
+ name='bitmap0')
+self.assert_qmp(result, 'return', {})
+
+# Start with regions1
+
+self.writeRegions(regions1)
+assert sha256_1 == self.getSha256()
+
+self.vm.shutdown()
+
+self.vm = self.mkVmRo()
+self.vm.launch()
+
+assert sha256_1 == self.getSha256()
+
+# Check that we are in RO mode and can't modify bitmap.
+self.writeRegions(regions2)
+assert sha256_1 == self.getSha256()
+
+# Reopen to RW
+result = self.vm.qmp('x-blockdev-reopen', **{
+'node-name': 'node0',
+'driver': iotests.imgfmt,
+'file': {
+'driver': 'file',
+'filename': disk
+},
+'read-only': False
+})
+self.assert_qmp(result, 'return', {})
+
+# Check that bitmap is reopened to RW and we can write to it.
+self.writeRegions(regions2)
+assert sha256_2 == self.getSha256()
+
+self.vm.shutdown()
+
+
 if __name__ == '__main__':
 iotests.main(supported_fmts=['qcow2'],
  supported_protocols=['file'])
diff --git a/tests/qemu-iotests/165.out b/tests/qemu-iotests/165.out
index ae1213e6f8..fbc63e62f8 100644
--- a/tests/qemu-iotests/165.out
+++ b/tests/qemu-iotests/165.out
@@ -1,5 +1,5 @@
-.
+..
 --
-Ran 1 tests
+Ran 2 tests
 
 OK
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 18/19] MAINTAINERS: Add Vladimir as a reviewer for bitmaps

[John Snow]

I already try to make sure all bitmaps patches have been reviewed by both
Red Hat and Virtuozzo anyway, so this formalizes the arrangement.

Fam meanwhile is no longer as active, so I am removing him as a co-maintainer
simply to reflect the current practice.

Signed-off-by: John Snow 
Reviewed-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20191005194448.16629-2-js...@redhat.com
---
 MAINTAINERS | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/MAINTAINERS b/MAINTAINERS
index fe4dc51b08..250ce8e7a1 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1816,8 +1816,8 @@ F: qapi/transaction.json
 T: git https://repo.or.cz/qemu/armbru.git block-next
 
 Dirty Bitmaps
-M: Fam Zheng 
 M: John Snow 
+R: Vladimir Sementsov-Ogievskiy 
 L: qemu-bl...@nongnu.org
 S: Supported
 F: util/hbitmap.c
@@ -1826,7 +1826,6 @@ F: include/qemu/hbitmap.h
 F: include/block/dirty-bitmap.h
 F: tests/test-hbitmap.c
 F: docs/interop/bitmaps.rst
-T: git https://github.com/famz/qemu.git bitmaps
 T: git https://github.com/jnsnow/qemu.git bitmaps
 
 Character device backends
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 09/19] block: switch reopen queue from QSIMPLEQ to QTAILQ

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

We'll need reverse-foreach in the following commit, QTAILQ support it,
so move to QTAILQ.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Message-id: 20190927122355.7344-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/block.h |  2 +-
 block.c   | 24 
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index 792bb826db..89606bd9f8 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -195,7 +195,7 @@ typedef struct HDGeometry {
 #define BDRV_BLOCK_EOF  0x20
 #define BDRV_BLOCK_RECURSE  0x40
 
-typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) 
BlockReopenQueue;
+typedef QTAILQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
 
 typedef struct BDRVReopenState {
 BlockDriverState *bs;
diff --git a/block.c b/block.c
index 5721441697..0347632c6c 100644
--- a/block.c
+++ b/block.c
@@ -1719,7 +1719,7 @@ typedef struct BlockReopenQueueEntry {
  bool prepared;
  bool perms_checked;
  BDRVReopenState state;
- QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
+ QTAILQ_ENTRY(BlockReopenQueueEntry) entry;
 } BlockReopenQueueEntry;
 
 /*
@@ -1732,7 +1732,7 @@ static int bdrv_reopen_get_flags(BlockReopenQueue *q, 
BlockDriverState *bs)
 BlockReopenQueueEntry *entry;
 
 if (q != NULL) {
-QSIMPLEQ_FOREACH(entry, q, entry) {
+QTAILQ_FOREACH(entry, q, entry) {
 if (entry->state.bs == bs) {
 return entry->state.flags;
 }
@@ -3249,7 +3249,7 @@ static bool bdrv_recurse_has_child(BlockDriverState *bs,
  * Adds a BlockDriverState to a simple queue for an atomic, transactional
  * reopen of multiple devices.
  *
- * bs_queue can either be an existing BlockReopenQueue that has had 
QSIMPLE_INIT
+ * bs_queue can either be an existing BlockReopenQueue that has had QTAILQ_INIT
  * already performed, or alternatively may be NULL a new BlockReopenQueue will
  * be created and initialized. This newly created BlockReopenQueue should be
  * passed back in for subsequent calls that are intended to be of the same
@@ -3290,7 +3290,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 
 if (bs_queue == NULL) {
 bs_queue = g_new0(BlockReopenQueue, 1);
-QSIMPLEQ_INIT(bs_queue);
+QTAILQ_INIT(bs_queue);
 }
 
 if (!options) {
@@ -3298,7 +3298,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 }
 
 /* Check if this BlockDriverState is already in the queue */
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 if (bs == bs_entry->state.bs) {
 break;
 }
@@ -3354,7 +3354,7 @@ static BlockReopenQueue 
*bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
 
 if (!bs_entry) {
 bs_entry = g_new0(BlockReopenQueueEntry, 1);
-QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
+QTAILQ_INSERT_TAIL(bs_queue, bs_entry, entry);
 } else {
 qobject_unref(bs_entry->state.options);
 qobject_unref(bs_entry->state.explicit_options);
@@ -3455,7 +3455,7 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 
 assert(bs_queue != NULL);
 
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 assert(bs_entry->state.bs->quiesce_counter > 0);
 if (bdrv_reopen_prepare(_entry->state, bs_queue, errp)) {
 goto cleanup;
@@ -3463,7 +3463,7 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 bs_entry->prepared = true;
 }
 
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 BDRVReopenState *state = _entry->state;
 ret = bdrv_check_perm(state->bs, bs_queue, state->perm,
   state->shared_perm, NULL, NULL, errp);
@@ -3489,13 +3489,13 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 /* If we reach this point, we have success and just need to apply the
  * changes
  */
-QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
 bdrv_reopen_commit(_entry->state);
 }
 
 ret = 0;
 cleanup_perm:
-QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
 BDRVReopenState *state = _entry->state;
 
 if (!bs_entry->perms_checked) {
@@ -3512,7 +3512,7 @@ cleanup_perm:
 }
 }
 cleanup:
-QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
 if (ret) {
 if (bs_entry->prepared) {
 bdrv_reopen_abort(_entry->s

[libvirt] [PULL v2 16/19] block/qcow2-bitmap: fix and improve qcow2_reopen_bitmaps_rw

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

- Correct check for write access to file child, and in correct place
  (only if we want to write).
- Support reopen rw -> rw (which will be used in following commit),
  for example, !bdrv_dirty_bitmap_readonly() is not a corruption if
  bitmap is marked IN_USE in the image.
- Consider unexpected bitmap as a corruption and check other
  combinations of in-image and in-RAM bitmaps.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-9-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2-bitmap.c | 77 +---
 1 file changed, 58 insertions(+), 19 deletions(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index f7dfb40256..98294a7696 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1108,18 +1108,14 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error 
**errp)
 Qcow2BitmapList *bm_list;
 Qcow2Bitmap *bm;
 GSList *ro_dirty_bitmaps = NULL;
-int ret = 0;
+int ret = -EINVAL;
+bool need_header_update = false;
 
 if (s->nb_bitmaps == 0) {
 /* No bitmaps - nothing to do */
 return 0;
 }
 
-if (!can_write(bs)) {
-error_setg(errp, "Can't write to the image on reopening bitmaps rw");
-return -EINVAL;
-}
-
 bm_list = bitmap_list_load(bs, s->bitmap_directory_offset,
s->bitmap_directory_size, errp);
 if (bm_list == NULL) {
@@ -1128,32 +1124,75 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error 
**errp)
 
 QSIMPLEQ_FOREACH(bm, bm_list, entry) {
 BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(bs, bm->name);
-if (bitmap == NULL) {
-continue;
-}
 
-if (!bdrv_dirty_bitmap_readonly(bitmap)) {
-error_setg(errp, "Bitmap %s was loaded prior to rw-reopen, but was 
"
-   "not marked as readonly. This is a bug, something went "
-   "wrong. All of the bitmaps may be corrupted", bm->name);
-ret = -EINVAL;
+if (!bitmap) {
+error_setg(errp, "Unexpected bitmap '%s' in image '%s'",
+   bm->name, bs->filename);
 goto out;
 }
 
-bm->flags |= BME_FLAG_IN_USE;
-ro_dirty_bitmaps = g_slist_append(ro_dirty_bitmaps, bitmap);
+if (!(bm->flags & BME_FLAG_IN_USE)) {
+if (!bdrv_dirty_bitmap_readonly(bitmap)) {
+error_setg(errp, "Corruption: bitmap '%s' is not marked IN_USE 
"
+   "in the image '%s' and not marked readonly in RAM",
+   bm->name, bs->filename);
+goto out;
+}
+if (bdrv_dirty_bitmap_inconsistent(bitmap)) {
+error_setg(errp, "Corruption: bitmap '%s' is inconsistent but "
+   "is not marked IN_USE in the image '%s'", bm->name,
+   bs->filename);
+goto out;
+}
+
+bm->flags |= BME_FLAG_IN_USE;
+need_header_update = true;
+} else {
+/*
+ * What if flags already has BME_FLAG_IN_USE ?
+ *
+ * 1. if we are reopening RW -> RW it's OK, of course.
+ * 2. if we are reopening RO -> RW:
+ *   2.1 if @bitmap is inconsistent, it's OK. It means that it was
+ *   inconsistent (IN_USE) when we loaded it
+ *   2.2 if @bitmap is not inconsistent. This seems to be 
impossible
+ *   and implies third party interaction. Let's error-out for
+ *   safety.
+ */
+if (bdrv_dirty_bitmap_readonly(bitmap) &&
+!bdrv_dirty_bitmap_inconsistent(bitmap))
+{
+error_setg(errp, "Corruption: bitmap '%s' is marked IN_USE "
+   "in the image '%s' but it is readonly and "
+   "consistent in RAM",
+   bm->name, bs->filename);
+goto out;
+}
+}
+
+if (bdrv_dirty_bitmap_readonly(bitmap)) {
+ro_dirty_bitmaps = g_slist_append(ro_dirty_bitmaps, bitmap);
+}
 }
 
-if (ro_dirty_bitmaps != NULL) {
+if (need_header_update) {
+if (!can_write(bs->file->bs) || !(bs->file->perm & BLK_PERM_WRITE)) {
+error_setg(errp, "Failed to reopen bitmaps rw: no write access "
+   "the protocol file");
+goto out;
+}
+
 /* in_use flags must be updated */
 ret = update_ext_header_and_dir_in_place(bs, bm_list);
 if (ret < 0) {
-error_setg_errn

[libvirt] [PULL v2 17/19] qcow2-bitmap: move bitmap reopen-rw code to qcow2_reopen_commit

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

The only reason I can imagine for this strange code at the very-end of
bdrv_reopen_commit is the fact that bs->read_only updated after
calling drv->bdrv_reopen_commit in bdrv_reopen_commit. And in the same
time, prior to previous commit, qcow2_reopen_bitmaps_rw did a wrong
check for being writable, when actually it only need writable file
child not self.

So, as it's fixed, let's move things to correct place.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Acked-by: Max Reitz 
Message-id: 20190927122355.7344-10-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/block_int.h |  6 --
 block.c   | 19 ---
 block/qcow2.c | 15 ++-
 3 files changed, 14 insertions(+), 26 deletions(-)

diff --git a/include/block/block_int.h b/include/block/block_int.h
index 32fb493cbb..ca4ccac4c1 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -547,12 +547,6 @@ struct BlockDriver {
  uint64_t parent_perm, uint64_t parent_shared,
  uint64_t *nperm, uint64_t *nshared);
 
-/**
- * Bitmaps should be marked as 'IN_USE' in the image on reopening image
- * as rw. This handler should realize it. It also should unset readonly
- * field of BlockDirtyBitmap's in case of success.
- */
-int (*bdrv_reopen_bitmaps_rw)(BlockDriverState *bs, Error **errp);
 bool (*bdrv_co_can_store_new_dirty_bitmap)(BlockDriverState *bs,
const char *name,
uint32_t granularity,
diff --git a/block.c b/block.c
index cf312258a9..dad5a3d8e0 100644
--- a/block.c
+++ b/block.c
@@ -3935,16 +3935,12 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
 BlockDriver *drv;
 BlockDriverState *bs;
 BdrvChild *child;
-bool old_can_write, new_can_write;
 
 assert(reopen_state != NULL);
 bs = reopen_state->bs;
 drv = bs->drv;
 assert(drv != NULL);
 
-old_can_write =
-!bdrv_is_read_only(bs) && !(bdrv_get_flags(bs) & BDRV_O_INACTIVE);
-
 /* If there are any driver level actions to take */
 if (drv->bdrv_reopen_commit) {
 drv->bdrv_reopen_commit(reopen_state);
@@ -3988,21 +3984,6 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
 }
 
 bdrv_refresh_limits(bs, NULL);
-
-new_can_write =
-!bdrv_is_read_only(bs) && !(bdrv_get_flags(bs) & BDRV_O_INACTIVE);
-if (!old_can_write && new_can_write && drv->bdrv_reopen_bitmaps_rw) {
-Error *local_err = NULL;
-if (drv->bdrv_reopen_bitmaps_rw(bs, _err) < 0) {
-/* This is not fatal, bitmaps just left read-only, so all following
- * writes will fail. User can remove read-only bitmaps to unblock
- * writes.
- */
-error_reportf_err(local_err,
-  "%s: Failed to make dirty bitmaps writable: ",
-  bdrv_get_node_name(bs));
-}
-}
 }
 
 /*
diff --git a/block/qcow2.c b/block/qcow2.c
index 53a025703e..8b05933565 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1835,6 +1835,20 @@ fail:
 static void qcow2_reopen_commit(BDRVReopenState *state)
 {
 qcow2_update_options_commit(state->bs, state->opaque);
+if (state->flags & BDRV_O_RDWR) {
+Error *local_err = NULL;
+
+if (qcow2_reopen_bitmaps_rw(state->bs, _err) < 0) {
+/*
+ * This is not fatal, bitmaps just left read-only, so all following
+ * writes will fail. User can remove read-only bitmaps to unblock
+ * writes or retry reopen.
+ */
+error_reportf_err(local_err,
+  "%s: Failed to make dirty bitmaps writable: ",
+  bdrv_get_node_name(state->bs));
+}
+}
 g_free(state->opaque);
 }
 
@@ -5406,7 +5420,6 @@ BlockDriver bdrv_qcow2 = {
 .bdrv_detach_aio_context  = qcow2_detach_aio_context,
 .bdrv_attach_aio_context  = qcow2_attach_aio_context,
 
-.bdrv_reopen_bitmaps_rw = qcow2_reopen_bitmaps_rw,
 .bdrv_co_can_store_new_dirty_bitmap = qcow2_co_can_store_new_dirty_bitmap,
 .bdrv_co_remove_persistent_dirty_bitmap =
 qcow2_co_remove_persistent_dirty_bitmap,
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 15/19] iotests: add test 260 to check bitmap life after snapshot + commit

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20190927122355.7344-8-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 tests/qemu-iotests/260 | 89 ++
 tests/qemu-iotests/260.out | 52 ++
 tests/qemu-iotests/group   |  1 +
 3 files changed, 142 insertions(+)
 create mode 100755 tests/qemu-iotests/260
 create mode 100644 tests/qemu-iotests/260.out

diff --git a/tests/qemu-iotests/260 b/tests/qemu-iotests/260
new file mode 100755
index 00..4f6082c9d2
--- /dev/null
+++ b/tests/qemu-iotests/260
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+#
+# Tests for temporary external snapshot when we have bitmaps.
+#
+# Copyright (c) 2019 Virtuozzo International GmbH. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+from iotests import qemu_img_create, file_path, log, filter_qmp_event
+
+iotests.verify_image_format(supported_fmts=['qcow2'])
+
+base, top = file_path('base', 'top')
+size = 64 * 1024 * 3
+
+
+def print_bitmap(msg, vm):
+result = vm.qmp('query-block')['return'][0]
+if 'dirty-bitmaps' in result:
+bitmap = result['dirty-bitmaps'][0]
+log('{}: name={} dirty-clusters={}'.format(msg, bitmap['name'],
+bitmap['count'] // 64 // 1024))
+else:
+log(msg + ': not found')
+
+
+def test(persistent, restart):
+assert persistent or not restart
+log("\nTestcase {}persistent {} restart\n".format(
+'' if persistent else 'non-', 'with' if restart else 'without'))
+
+qemu_img_create('-f', iotests.imgfmt, base, str(size))
+
+vm = iotests.VM().add_drive(base)
+vm.launch()
+
+vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap0',
+   persistent=persistent)
+vm.hmp_qemu_io('drive0', 'write 0 64K')
+print_bitmap('initial bitmap', vm)
+
+vm.qmp_log('blockdev-snapshot-sync', device='drive0', snapshot_file=top,
+   format=iotests.imgfmt, filters=[iotests.filter_qmp_testfiles])
+vm.hmp_qemu_io('drive0', 'write 64K 512')
+print_bitmap('check that no bitmaps are in snapshot', vm)
+
+if restart:
+log("... Restart ...")
+vm.shutdown()
+vm = iotests.VM().add_drive(top)
+vm.launch()
+
+vm.qmp_log('block-commit', device='drive0', top=top,
+   filters=[iotests.filter_qmp_testfiles])
+ev = vm.events_wait((('BLOCK_JOB_READY', None),
+ ('BLOCK_JOB_COMPLETED', None)))
+log(filter_qmp_event(ev))
+if (ev['event'] == 'BLOCK_JOB_COMPLETED'):
+vm.shutdown()
+log(vm.get_log())
+exit()
+
+vm.qmp_log('block-job-complete', device='drive0')
+ev = vm.event_wait('BLOCK_JOB_COMPLETED')
+log(filter_qmp_event(ev))
+print_bitmap('check bitmap after commit', vm)
+
+vm.hmp_qemu_io('drive0', 'write 128K 64K')
+print_bitmap('check updated bitmap', vm)
+
+vm.shutdown()
+
+
+test(persistent=False, restart=False)
+test(persistent=True, restart=False)
+test(persistent=True, restart=True)
diff --git a/tests/qemu-iotests/260.out b/tests/qemu-iotests/260.out
new file mode 100644
index 00..2f0d98d036
--- /dev/null
+++ b/tests/qemu-iotests/260.out
@@ -0,0 +1,52 @@
+
+Testcase non-persistent without restart
+
+{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap0", "node": 
"drive0", "persistent": false}}
+{"return": {}}
+initial bitmap: name=bitmap0 dirty-clusters=1
+{"execute": "blockdev-snapshot-sync", "arguments": {"device": "drive0", 
"format": "qcow2", "snapshot-file": "TEST_DIR/PID-top"}}
+{"return": {}}
+check that no bitmaps are in snapshot: not found
+{"execute": "block-commit", "arguments": {"device": "drive0", "top": 
"TEST_DIR/PID-top"}}
+{"return": {}}
+{"data": {"device": "drive0", "len": 65536, "offset": 65536, "speed": 0, 
"type": "commit"}, "event": "BLOCK_JOB_READY", "timestamp": {"microseconds": 
"U

[libvirt] [PULL v2 13/19] block/qcow2-bitmap: drop qcow2_reopen_bitmaps_rw_hint()

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

The function is unused, drop it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190927122355.7344-6-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  2 --
 block/qcow2-bitmap.c | 15 +--
 2 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index 0f3d9b088e..23a9898a54 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -740,8 +740,6 @@ int qcow2_check_bitmaps_refcounts(BlockDriverState *bs, 
BdrvCheckResult *res,
 bool qcow2_load_dirty_bitmaps(BlockDriverState *bs, Error **errp);
 Qcow2BitmapInfoList *qcow2_get_bitmap_info_list(BlockDriverState *bs,
 Error **errp);
-int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, bool *header_updated,
- Error **errp);
 int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp);
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp);
 void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp);
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 6dfc083548..ebc1afccd3 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1102,8 +1102,7 @@ Qcow2BitmapInfoList 
*qcow2_get_bitmap_info_list(BlockDriverState *bs,
 return list;
 }
 
-int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, bool *header_updated,
- Error **errp)
+int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)
 {
 BDRVQcow2State *s = bs->opaque;
 Qcow2BitmapList *bm_list;
@@ -,10 +1110,6 @@ int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, 
bool *header_updated,
 GSList *ro_dirty_bitmaps = NULL;
 int ret = 0;
 
-if (header_updated != NULL) {
-*header_updated = false;
-}
-
 if (s->nb_bitmaps == 0) {
 /* No bitmaps - nothing to do */
 return 0;
@@ -1156,9 +1151,6 @@ int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, 
bool *header_updated,
 error_setg_errno(errp, -ret, "Can't update bitmap directory");
 goto out;
 }
-if (header_updated != NULL) {
-*header_updated = true;
-}
 g_slist_foreach(ro_dirty_bitmaps, set_readonly_helper, false);
 }
 
@@ -1169,11 +1161,6 @@ out:
 return ret;
 }
 
-int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)
-{
-return qcow2_reopen_bitmaps_rw_hint(bs, NULL, errp);
-}
-
 /* Checks to see if it's safe to resize bitmaps */
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp)
 {
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 03/19] block/dirty-bitmap: return int from bdrv_remove_persistent_dirty_bitmap

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

It's more comfortable to not deal with local_err.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190920082543.23444-3-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  5 ++---
 include/block/block_int.h|  6 +++---
 include/block/dirty-bitmap.h |  5 ++---
 block/dirty-bitmap.c |  9 +
 block/qcow2-bitmap.c | 18 ++
 blockdev.c   |  7 +++
 6 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index f51f478e34..08b4c15dc4 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -750,9 +750,8 @@ bool qcow2_can_store_new_dirty_bitmap(BlockDriverState *bs,
   const char *name,
   uint32_t granularity,
   Error **errp);
-void qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
-  const char *name,
-  Error **errp);
+int qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char 
*name,
+ Error **errp);
 
 ssize_t coroutine_fn
 qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 05056b308a..6b511dd889 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -557,9 +557,9 @@ struct BlockDriver {
 const char *name,
 uint32_t granularity,
 Error **errp);
-void (*bdrv_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
-const char *name,
-Error **errp);
+int (*bdrv_remove_persistent_dirty_bitmap)(BlockDriverState *bs,
+   const char *name,
+   Error **errp);
 
 /**
  * Register/unregister a buffer for I/O. For example, when the driver is
diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 4b4b731b46..07503b03b5 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -37,9 +37,8 @@ int bdrv_dirty_bitmap_check(const BdrvDirtyBitmap *bitmap, 
uint32_t flags,
 Error **errp);
 void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
 void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs);
-void bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs,
- const char *name,
- Error **errp);
+int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
+Error **errp);
 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_enable_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap);
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 8f42015db9..d1ae2e1922 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -455,13 +455,14 @@ void bdrv_release_named_dirty_bitmaps(BlockDriverState 
*bs)
  * not fail.
  * This function doesn't release corresponding BdrvDirtyBitmap.
  */
-void bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs,
- const char *name,
- Error **errp)
+int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
+Error **errp)
 {
 if (bs->drv && bs->drv->bdrv_remove_persistent_dirty_bitmap) {
-bs->drv->bdrv_remove_persistent_dirty_bitmap(bs, name, errp);
+return bs->drv->bdrv_remove_persistent_dirty_bitmap(bs, name, errp);
 }
+
+return 0;
 }
 
 bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index b2487101ed..9821c1628f 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1404,9 +1404,8 @@ static Qcow2Bitmap *find_bitmap_by_name(Qcow2BitmapList 
*bm_list,
 return NULL;
 }
 
-void qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
-  const char *name,
-  Error **errp)
+int qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char 
*name,
+ Error **errp)
 {
 int ret;
 BDRVQcow2State *s = bs->opaque;
@@ -1416,18 +1415,19 @@ void 
qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
 if (s->nb_bitmaps == 0) {
 /* Absence of the bitmap is 

[libvirt] [PULL v2 10/19] block: reverse order for reopen commits

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

It's needed to fix reopening qcow2 with bitmaps to RW. Currently it
can't work, as qcow2 needs write access to file child, to mark bitmaps
in-image with IN_USE flag. But usually children goes after parents in
reopen queue and file child is still RO on qcow2 reopen commit. Reverse
reopen order to fix it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Acked-by: Max Reitz 
Acked-by: John Snow 
Message-id: 20190927122355.7344-3-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block.c | 12 +---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/block.c b/block.c
index 0347632c6c..cf312258a9 100644
--- a/block.c
+++ b/block.c
@@ -3486,10 +3486,16 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, 
Error **errp)
 bs_entry->perms_checked = true;
 }
 
-/* If we reach this point, we have success and just need to apply the
- * changes
+/*
+ * If we reach this point, we have success and just need to apply the
+ * changes.
+ *
+ * Reverse order is used to comfort qcow2 driver: on commit it need to 
write
+ * IN_USE flag to the image, to mark bitmaps in the image as invalid. But
+ * children are usually goes after parents in reopen-queue, so go from last
+ * to first element.
  */
-QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
+QTAILQ_FOREACH_REVERSE(bs_entry, bs_queue, entry) {
 bdrv_reopen_commit(_entry->state);
 }
 
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 02/19] block: move bdrv_can_store_new_dirty_bitmap to block/dirty-bitmap.c

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

block/dirty-bitmap.c seems to be more appropriate for it and
bdrv_remove_persistent_dirty_bitmap already in it.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190920082543.23444-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block.c  | 22 --
 block/dirty-bitmap.c | 22 ++
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/block.c b/block.c
index 1946fc6f57..d19a4781a3 100644
--- a/block.c
+++ b/block.c
@@ -6582,25 +6582,3 @@ void bdrv_del_child(BlockDriverState *parent_bs, 
BdrvChild *child, Error **errp)
 
 parent_bs->drv->bdrv_del_child(parent_bs, child, errp);
 }
-
-bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
- uint32_t granularity, Error **errp)
-{
-BlockDriver *drv = bs->drv;
-
-if (!drv) {
-error_setg_errno(errp, ENOMEDIUM,
- "Can't store persistent bitmaps to %s",
- bdrv_get_device_or_node_name(bs));
-return false;
-}
-
-if (!drv->bdrv_can_store_new_dirty_bitmap) {
-error_setg_errno(errp, ENOTSUP,
- "Can't store persistent bitmaps to %s",
- bdrv_get_device_or_node_name(bs));
-return false;
-}
-
-return drv->bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp);
-}
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 134e0c9a0c..8f42015db9 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -464,6 +464,28 @@ void bdrv_remove_persistent_dirty_bitmap(BlockDriverState 
*bs,
 }
 }
 
+bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
+ uint32_t granularity, Error **errp)
+{
+BlockDriver *drv = bs->drv;
+
+if (!drv) {
+error_setg_errno(errp, ENOMEDIUM,
+ "Can't store persistent bitmaps to %s",
+ bdrv_get_device_or_node_name(bs));
+return false;
+}
+
+if (!drv->bdrv_can_store_new_dirty_bitmap) {
+error_setg_errno(errp, ENOTSUP,
+ "Can't store persistent bitmaps to %s",
+ bdrv_get_device_or_node_name(bs));
+return false;
+}
+
+return drv->bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp);
+}
+
 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
 bdrv_dirty_bitmap_lock(bitmap);
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 07/19] block/dirty-bitmap: drop BdrvDirtyBitmap.mutex

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

mutex field is just a pointer to bs->dirty_bitmap_mutex, so no needs
to store it in BdrvDirtyBitmap when we have bs pointer in it (since
previous patch).

Drop mutex field. Constantly use bdrv_dirty_bitmaps_lock/unlock in
block/dirty-bitmap.c to make it more obvious that it's not per-bitmap
lock. Still, for simplicity, leave bdrv_dirty_bitmap_lock/unlock
functions as an external API.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-4-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/dirty-bitmap.c | 84 +---
 1 file changed, 41 insertions(+), 43 deletions(-)

diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 44453ff824..4e5c87a907 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -29,7 +29,6 @@
 #include "qemu/main-loop.h"
 
 struct BdrvDirtyBitmap {
-QemuMutex *mutex;
 BlockDriverState *bs;
 HBitmap *bitmap;/* Dirty bitmap implementation */
 bool busy;  /* Bitmap is busy, it can't be used via QMP */
@@ -72,12 +71,12 @@ static inline void 
bdrv_dirty_bitmaps_unlock(BlockDriverState *bs)
 
 void bdrv_dirty_bitmap_lock(BdrvDirtyBitmap *bitmap)
 {
-qemu_mutex_lock(bitmap->mutex);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 }
 
 void bdrv_dirty_bitmap_unlock(BdrvDirtyBitmap *bitmap)
 {
-qemu_mutex_unlock(bitmap->mutex);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 /* Called with BQL or dirty_bitmap lock taken.  */
@@ -117,7 +116,6 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
 }
 bitmap = g_new0(BdrvDirtyBitmap, 1);
 bitmap->bs = bs;
-bitmap->mutex = >dirty_bitmap_mutex;
 bitmap->bitmap = hbitmap_alloc(bitmap_size, ctz32(granularity));
 bitmap->size = bitmap_size;
 bitmap->name = g_strdup(name);
@@ -151,9 +149,9 @@ static bool bdrv_dirty_bitmap_busy(const BdrvDirtyBitmap 
*bitmap)
 
 void bdrv_dirty_bitmap_set_busy(BdrvDirtyBitmap *bitmap, bool busy)
 {
-qemu_mutex_lock(bitmap->mutex);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bitmap->busy = busy;
-qemu_mutex_unlock(bitmap->mutex);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 /* Called with BQL taken.  */
@@ -278,10 +276,10 @@ void bdrv_enable_dirty_bitmap_locked(BdrvDirtyBitmap 
*bitmap)
 /* Called with BQL taken. */
 void bdrv_dirty_bitmap_enable_successor(BdrvDirtyBitmap *bitmap)
 {
-assert(bitmap->mutex == bitmap->successor->mutex);
-qemu_mutex_lock(bitmap->mutex);
+assert(bitmap->bs == bitmap->successor->bs);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bdrv_enable_dirty_bitmap_locked(bitmap->successor);
-qemu_mutex_unlock(bitmap->mutex);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 /* Called within bdrv_dirty_bitmap_lock..unlock and with BQL taken.  */
@@ -361,9 +359,9 @@ BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BdrvDirtyBitmap 
*parent,
 {
 BdrvDirtyBitmap *ret;
 
-qemu_mutex_lock(parent->mutex);
+bdrv_dirty_bitmaps_lock(parent->bs);
 ret = bdrv_reclaim_dirty_bitmap_locked(parent, errp);
-qemu_mutex_unlock(parent->mutex);
+bdrv_dirty_bitmaps_unlock(parent->bs);
 
 return ret;
 }
@@ -543,16 +541,16 @@ bool bdrv_can_store_new_dirty_bitmap(BlockDriverState 
*bs, const char *name,
 
 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
-bdrv_dirty_bitmap_lock(bitmap);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bitmap->disabled = true;
-bdrv_dirty_bitmap_unlock(bitmap);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
-bdrv_dirty_bitmap_lock(bitmap);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bdrv_enable_dirty_bitmap_locked(bitmap);
-bdrv_dirty_bitmap_unlock(bitmap);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
@@ -593,9 +591,9 @@ bool bdrv_dirty_bitmap_get_locked(BdrvDirtyBitmap *bitmap, 
int64_t offset)
 bool bdrv_dirty_bitmap_get(BdrvDirtyBitmap *bitmap, int64_t offset)
 {
 bool ret;
-bdrv_dirty_bitmap_lock(bitmap);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 ret = bdrv_dirty_bitmap_get_locked(bitmap, offset);
-bdrv_dirty_bitmap_unlock(bitmap);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 
 return ret;
 }
@@ -660,9 +658,9 @@ void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
 void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
int64_t offset, int64_t bytes)
 {
-bdrv_dirty_bitmap_lock(bitmap);
+bdrv_dirty_bitmaps_lock(bitmap->bs);
 bdrv_set_dirty_bitmap_locked(bitmap, offset, bytes);
-bdrv_dirty_bitmap_unlock(bitmap);
+bdrv_dirty_bitmaps_unlock(bitmap->bs);
 }
 
 /* Called within bdrv_dirty_bitmap_lock..unlock */
@@ -676

[libvirt] [PULL v2 05/19] block/dirty-bitmap: drop meta

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Drop meta bitmaps, as they are unused.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h |  5 
 block/dirty-bitmap.c | 46 
 2 files changed, 51 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 07503b03b5..973056778a 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -18,9 +18,6 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
   uint32_t granularity,
   const char *name,
   Error **errp);
-void bdrv_create_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap,
-   int chunk_size);
-void bdrv_release_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
BdrvDirtyBitmap *bitmap,
Error **errp);
@@ -54,7 +51,6 @@ void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
int64_t offset, int64_t bytes);
 void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
  int64_t offset, int64_t bytes);
-BdrvDirtyBitmapIter *bdrv_dirty_meta_iter_new(BdrvDirtyBitmap *bitmap);
 BdrvDirtyBitmapIter *bdrv_dirty_iter_new(BdrvDirtyBitmap *bitmap);
 void bdrv_dirty_iter_free(BdrvDirtyBitmapIter *iter);
 
@@ -96,7 +92,6 @@ void bdrv_reset_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
 int64_t bdrv_dirty_iter_next(BdrvDirtyBitmapIter *iter);
 void bdrv_set_dirty_iter(BdrvDirtyBitmapIter *hbi, int64_t offset);
 int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap);
-int64_t bdrv_get_meta_dirty_count(BdrvDirtyBitmap *bitmap);
 void bdrv_dirty_bitmap_truncate(BlockDriverState *bs, int64_t bytes);
 bool bdrv_dirty_bitmap_readonly(const BdrvDirtyBitmap *bitmap);
 bool bdrv_has_readonly_bitmaps(BlockDriverState *bs);
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 03e0872b97..4ecf18d5df 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -31,7 +31,6 @@
 struct BdrvDirtyBitmap {
 QemuMutex *mutex;
 HBitmap *bitmap;/* Dirty bitmap implementation */
-HBitmap *meta;  /* Meta dirty bitmap */
 bool busy;  /* Bitmap is busy, it can't be used via QMP */
 BdrvDirtyBitmap *successor; /* Anonymous child, if any. */
 char *name; /* Optional non-empty unique ID */
@@ -127,36 +126,6 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState 
*bs,
 return bitmap;
 }
 
-/* bdrv_create_meta_dirty_bitmap
- *
- * Create a meta dirty bitmap that tracks the changes of bits in @bitmap. I.e.
- * when a dirty status bit in @bitmap is changed (either from reset to set or
- * the other way around), its respective meta dirty bitmap bit will be marked
- * dirty as well.
- *
- * @bitmap: the block dirty bitmap for which to create a meta dirty bitmap.
- * @chunk_size: how many bytes of bitmap data does each bit in the meta bitmap
- * track.
- */
-void bdrv_create_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap,
-   int chunk_size)
-{
-assert(!bitmap->meta);
-qemu_mutex_lock(bitmap->mutex);
-bitmap->meta = hbitmap_create_meta(bitmap->bitmap,
-   chunk_size * BITS_PER_BYTE);
-qemu_mutex_unlock(bitmap->mutex);
-}
-
-void bdrv_release_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap)
-{
-assert(bitmap->meta);
-qemu_mutex_lock(bitmap->mutex);
-hbitmap_free_meta(bitmap->bitmap);
-bitmap->meta = NULL;
-qemu_mutex_unlock(bitmap->mutex);
-}
-
 int64_t bdrv_dirty_bitmap_size(const BdrvDirtyBitmap *bitmap)
 {
 return bitmap->size;
@@ -320,7 +289,6 @@ static void 
bdrv_release_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)
 assert(!bitmap->active_iterators);
 assert(!bdrv_dirty_bitmap_busy(bitmap));
 assert(!bdrv_dirty_bitmap_has_successor(bitmap));
-assert(!bitmap->meta);
 QLIST_REMOVE(bitmap, list);
 hbitmap_free(bitmap->bitmap);
 g_free(bitmap->name);
@@ -666,15 +634,6 @@ BdrvDirtyBitmapIter *bdrv_dirty_iter_new(BdrvDirtyBitmap 
*bitmap)
 return iter;
 }
 
-BdrvDirtyBitmapIter *bdrv_dirty_meta_iter_new(BdrvDirtyBitmap *bitmap)
-{
-BdrvDirtyBitmapIter *iter = g_new(BdrvDirtyBitmapIter, 1);
-hbitmap_iter_init(>hbi, bitmap->meta, 0);
-iter->bitmap = bitmap;
-bitmap->active_iterators++;
-return iter;
-}
-
 void bdrv_dirty_iter_free(BdrvDirtyBitmapIter *iter)
 {
 if (!iter) {
@@ -821,11 +780,6 @@ int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap)
 return hbitmap_count(bitmap->bitmap);
 }
 
-int64_t bdrv_get_meta_dirty_count(BdrvDi

[libvirt] [PULL v2 01/19] util/hbitmap: strict hbitmap_reset

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

hbitmap_reset has an unobvious property: it rounds requested region up.
It may provoke bugs, like in recently fixed write-blocking mode of
mirror: user calls reset on unaligned region, not keeping in mind that
there are possible unrelated dirty bytes, covered by rounded-up region
and information of this unrelated "dirtiness" will be lost.

Make hbitmap_reset strict: assert that arguments are aligned, allowing
only one exception when @start + @count == hb->orig_size. It's needed
to comfort users of hbitmap_next_dirty_area, which cares about
hb->orig_size.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Message-Id: <20190806152611.280389-1-vsement...@virtuozzo.com>
[Maintainer edit: Max's suggestions from on-list. --js]
[Maintainer edit: Eric's suggestion for aligned macro. --js]
Signed-off-by: John Snow 
---
 include/qemu/hbitmap.h | 5 +
 tests/test-hbitmap.c   | 2 +-
 util/hbitmap.c | 4 
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 4afbe6292e..1bf944ca3d 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -132,6 +132,11 @@ void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t 
count);
  * @count: Number of bits to reset.
  *
  * Reset a consecutive range of bits in an HBitmap.
+ * @start and @count must be aligned to bitmap granularity. The only exception
+ * is resetting the tail of the bitmap: @count may be equal to hb->orig_size -
+ * @start, in this case @count may be not aligned. The sum of @start + @count 
is
+ * allowed to be greater than hb->orig_size, but only if @start < hb->orig_size
+ * and @start + @count = ALIGN_UP(hb->orig_size, granularity).
  */
 void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count);
 
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
index eed5d288cb..e1f867085f 100644
--- a/tests/test-hbitmap.c
+++ b/tests/test-hbitmap.c
@@ -423,7 +423,7 @@ static void test_hbitmap_granularity(TestHBitmapData *data,
 hbitmap_test_check(data, 0);
 hbitmap_test_set(data, 0, 3);
 g_assert_cmpint(hbitmap_count(data->hb), ==, 4);
-hbitmap_test_reset(data, 0, 1);
+hbitmap_test_reset(data, 0, 2);
 g_assert_cmpint(hbitmap_count(data->hb), ==, 2);
 }
 
diff --git a/util/hbitmap.c b/util/hbitmap.c
index fd44c897ab..66db87c6ff 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -476,6 +476,10 @@ void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t 
count)
 /* Compute range in the last layer.  */
 uint64_t first;
 uint64_t last = start + count - 1;
+uint64_t gran = 1ULL << hb->granularity;
+
+assert(QEMU_IS_ALIGNED(start, gran));
+assert(QEMU_IS_ALIGNED(count, gran) || (start + count == hb->orig_size));
 
 trace_hbitmap_reset(hb, start, count,
 start >> hb->granularity, last >> hb->granularity);
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 00/19] Bitmaps patches

[John Snow]

The following changes since commit c760cb77e511eb05094df67c1b30029a952efa35:

  Merge remote-tracking branch 'remotes/dgilbert/tags/pull-migration-20191011a' 
into staging (2019-10-14 16:09:52 +0100)

are available in the Git repository at:

  https://github.com/jnsnow/qemu.git tags/bitmaps-pull-request

for you to fetch changes up to b2ca29ee390743c42a6062d44ee3b10fb51f9fa6:

  dirty-bitmaps: remove deprecated autoload parameter (2019-10-14 15:28:17 
-0400)


Pull request



John Snow (2):
  MAINTAINERS: Add Vladimir as a reviewer for bitmaps
  dirty-bitmaps: remove deprecated autoload parameter

Vladimir Sementsov-Ogievskiy (17):
  util/hbitmap: strict hbitmap_reset
  block: move bdrv_can_store_new_dirty_bitmap to block/dirty-bitmap.c
  block/dirty-bitmap: return int from
bdrv_remove_persistent_dirty_bitmap
  block/qcow2: proper locking on bitmap add/remove paths
  block/dirty-bitmap: drop meta
  block/dirty-bitmap: add bs link
  block/dirty-bitmap: drop BdrvDirtyBitmap.mutex
  block/dirty-bitmap: refactor bdrv_dirty_bitmap_next
  block: switch reopen queue from QSIMPLEQ to QTAILQ
  block: reverse order for reopen commits
  iotests: add test-case to 165 to test reopening qcow2 bitmaps to RW
  block/qcow2-bitmap: get rid of bdrv_has_changed_persistent_bitmaps
  block/qcow2-bitmap: drop qcow2_reopen_bitmaps_rw_hint()
  block/qcow2-bitmap: do not remove bitmaps on reopen-ro
  iotests: add test 260 to check bitmap life after snapshot + commit
  block/qcow2-bitmap: fix and improve qcow2_reopen_bitmaps_rw
  qcow2-bitmap: move bitmap reopen-rw code to qcow2_reopen_commit

 qemu-deprecated.texi   |  20 ++-
 qapi/block-core.json   |   6 +-
 block/qcow2.h  |  19 +--
 include/block/block.h  |   2 +-
 include/block/block_int.h  |  20 +--
 include/block/dirty-bitmap.h   |  34 ++--
 include/qemu/hbitmap.h |   5 +
 block.c|  79 +++--
 block/backup.c |   8 +-
 block/block-copy.c |   2 +-
 block/dirty-bitmap.c   | 290 +++--
 block/mirror.c |   4 +-
 block/qcow2-bitmap.c   | 212 +++-
 block/qcow2.c  |  22 ++-
 blockdev.c |  40 ++---
 migration/block-dirty-bitmap.c |  11 +-
 migration/block.c  |   4 +-
 tests/test-hbitmap.c   |   2 +-
 util/hbitmap.c |   4 +
 MAINTAINERS|   3 +-
 tests/qemu-iotests/165 |  57 ++-
 tests/qemu-iotests/165.out |   4 +-
 tests/qemu-iotests/260 |  89 ++
 tests/qemu-iotests/260.out |  52 ++
 tests/qemu-iotests/group   |   1 +
 25 files changed, 623 insertions(+), 367 deletions(-)
 create mode 100755 tests/qemu-iotests/260
 create mode 100644 tests/qemu-iotests/260.out

-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

Re: [libvirt] [PULL 01/19] util/hbitmap: strict hbitmap_reset

[John Snow]

On 10/11/19 7:18 PM, John Snow wrote:
> 
> 
> On 10/11/19 5:48 PM, Eric Blake wrote:
>> On 10/11/19 4:25 PM, John Snow wrote:
>>> From: Vladimir Sementsov-Ogievskiy 
>>>
>>> hbitmap_reset has an unobvious property: it rounds requested region up.
>>> It may provoke bugs, like in recently fixed write-blocking mode of
>>> mirror: user calls reset on unaligned region, not keeping in mind that
>>> there are possible unrelated dirty bytes, covered by rounded-up region
>>> and information of this unrelated "dirtiness" will be lost.
>>>
>>> Make hbitmap_reset strict: assert that arguments are aligned, allowing
>>> only one exception when @start + @count == hb->orig_size. It's needed
>>> to comfort users of hbitmap_next_dirty_area, which cares about
>>> hb->orig_size.
>>>
>>> Signed-off-by: Vladimir Sementsov-Ogievskiy 
>>> Reviewed-by: Max Reitz 
>>> Message-Id: <20190806152611.280389-1-vsement...@virtuozzo.com>
>>> [Maintainer edit: Max's suggestions from on-list. --js]
>>> Signed-off-by: John Snow 
>>> ---
>>>   include/qemu/hbitmap.h | 5 +
>>>   tests/test-hbitmap.c   | 2 +-
>>>   util/hbitmap.c | 4 
>>>   3 files changed, 10 insertions(+), 1 deletion(-)
>>>
>>
>>> +++ b/util/hbitmap.c
>>> @@ -476,6 +476,10 @@ void hbitmap_reset(HBitmap *hb, uint64_t start,
>>> uint64_t count)
>>>   /* Compute range in the last layer.  */
>>>   uint64_t first;
>>>   uint64_t last = start + count - 1;
>>> +    uint64_t gran = 1ULL << hb->granularity;
>>> +
>>> +    assert(!(start & (gran - 1)));
>>> +    assert(!(count & (gran - 1)) || (start + count == hb->orig_size));
>>
>> I know I'm replying a bit late (since this is now a pull request), but
>> would it be worth using the dedicated macro:
>>
>> assert(QEMU_IS_ALIGNED(start, gran));
>> assert(QEMU_IS_ALIGNED(count, gran) || start + count == hb->orig_size);
>>
>> instead of open-coding it?  (I would also drop the extra () around the
>> right half of ||). If we want it, that would now be a followup patch.

I've noticed that seasoned C programmers hate extra parentheses a lot.
I've noticed that I cannot remember operator precedence enough to ever
feel like this is actually an improvement.

Something about a nice weighted tree of ((expr1) || (expr2)) feels
soothing to my weary eyes. So, if it's not terribly important, I'd
prefer to leave it as-is.

(You may feel free to counter-educate me as desired.)

>>
> 
> If the PR doesn't make it for some reason, I can amend a cleanup patch
> for the next PR.
> 

by the way: GOOD NEWS! ...

--js

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 12/19] block/qcow2-bitmap: get rid of bdrv_has_changed_persistent_bitmaps

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Firstly, no reason to optimize failure path. Then, function name is
ambiguous: it checks for readonly and similar things, but someone may
think that it will ignore normal bitmaps which was just unchanged, and
this is in bad relation with the fact that we should drop IN_USE flag
for unchanged bitmaps in the image.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190927122355.7344-5-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h |  1 -
 block/dirty-bitmap.c | 12 
 block/qcow2-bitmap.c | 23 +--
 3 files changed, 13 insertions(+), 23 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 257f0f6704..958e7474fb 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -95,7 +95,6 @@ bool bdrv_has_readonly_bitmaps(BlockDriverState *bs);
 bool bdrv_dirty_bitmap_get_autoload(const BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap);
-bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs);
 
 BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs);
 BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap);
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 6065db8094..4bbb251b2c 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -839,18 +839,6 @@ bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap 
*bitmap)
 return bitmap->inconsistent;
 }
 
-bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs)
-{
-BdrvDirtyBitmap *bm;
-QLIST_FOREACH(bm, >dirty_bitmaps, list) {
-if (bm->persistent && !bm->readonly && !bm->skip_store) {
-return true;
-}
-}
-
-return false;
-}
-
 BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs)
 {
 return QLIST_FIRST(>dirty_bitmaps);
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 99812b418b..6dfc083548 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1464,16 +1464,7 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 Qcow2Bitmap *bm;
 QSIMPLEQ_HEAD(, Qcow2BitmapTable) drop_tables;
 Qcow2BitmapTable *tb, *tb_next;
-
-if (!bdrv_has_changed_persistent_bitmaps(bs)) {
-/* nothing to do */
-return;
-}
-
-if (!can_write(bs)) {
-error_setg(errp, "No write access");
-return;
-}
+bool need_write = false;
 
 QSIMPLEQ_INIT(_tables);
 
@@ -1499,6 +1490,8 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 continue;
 }
 
+need_write = true;
+
 if (check_constraints_on_bitmap(bs, name, granularity, errp) < 0) {
 error_prepend(errp, "Bitmap '%s' doesn't satisfy the constraints: 
",
   name);
@@ -1537,6 +1530,15 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 bm->dirty_bitmap = bitmap;
 }
 
+if (!need_write) {
+goto success;
+}
+
+if (!can_write(bs)) {
+error_setg(errp, "No write access");
+goto fail;
+}
+
 /* allocate clusters and store bitmaps */
 QSIMPLEQ_FOREACH(bm, bm_list, entry) {
 if (bm->dirty_bitmap == NULL) {
@@ -1578,6 +1580,7 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 bdrv_release_dirty_bitmap(bm->dirty_bitmap);
 }
 
+success:
 bitmap_list_free(bm_list);
 return;
 
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 19/19] dirty-bitmaps: remove deprecated autoload parameter

[John Snow]

This parameter has been deprecated since 2.12.0 and is eligible for
removal. Remove this parameter as it is actually completely ignored;
let's not give false hope.

Signed-off-by: John Snow 
Reviewed-by: Eric Blake 
Reviewed-by: Vladimir Sementsov-Ogievskiy 
Message-id: 20191002232411.29968-1-js...@redhat.com
---
 qemu-deprecated.texi | 20 +++-
 qapi/block-core.json |  6 +-
 blockdev.c   |  6 --
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi
index 01245e0b1c..7239e0959d 100644
--- a/qemu-deprecated.texi
+++ b/qemu-deprecated.texi
@@ -149,11 +149,6 @@ QEMU 4.1 has three options, please migrate to one of these 
three:
 
 @section QEMU Machine Protocol (QMP) commands
 
-@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
-
-"autoload" parameter is now ignored. All bitmaps are automatically loaded
-from qcow2 images.
-
 @subsection query-block result field dirty-bitmaps[i].status (since 4.0)
 
 The ``status'' field of the ``BlockDirtyInfo'' structure, returned by
@@ -356,3 +351,18 @@ existing CPU models.  Management software that needs 
runnability
 guarantees must resolve the CPU model aliases using te
 ``alias-of'' field returned by the ``query-cpu-definitions'' QMP
 command.
+
+
+@node Recently removed features
+@appendix Recently removed features
+
+What follows is a record of recently removed, formerly deprecated
+features that serves as a record for users who have encountered
+trouble after a recent upgrade.
+
+@section QEMU Machine Protocol (QMP) commands
+
+@subsection block-dirty-bitmap-add "autoload" parameter (since 4.2.0)
+
+The "autoload" parameter has been ignored since 2.12.0. All bitmaps
+are automatically loaded from qcow2 images.
diff --git a/qapi/block-core.json b/qapi/block-core.json
index f66553aac7..b274aef713 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -2052,10 +2052,6 @@
 #  Qcow2 disks support persistent bitmaps. Default is false for
 #  block-dirty-bitmap-add. (Since: 2.10)
 #
-# @autoload: ignored and deprecated since 2.12.
-#Currently, all dirty tracking bitmaps are loaded from Qcow2 on
-#open.
-#
 # @disabled: the bitmap is created in the disabled state, which means that
 #it will not track drive changes. The bitmap may be enabled with
 #block-dirty-bitmap-enable. Default is false. (Since: 4.0)
@@ -2064,7 +2060,7 @@
 ##
 { 'struct': 'BlockDirtyBitmapAdd',
   'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32',
-'*persistent': 'bool', '*autoload': 'bool', '*disabled': 'bool' } }
+'*persistent': 'bool', '*disabled': 'bool' } }
 
 ##
 # @BlockDirtyBitmapMergeSource:
diff --git a/blockdev.c b/blockdev.c
index d77e809623..03c7cd7651 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -1966,7 +1966,6 @@ static void block_dirty_bitmap_add_prepare(BlkActionState 
*common,
 qmp_block_dirty_bitmap_add(action->node, action->name,
action->has_granularity, action->granularity,
action->has_persistent, action->persistent,
-   action->has_autoload, action->autoload,
action->has_disabled, action->disabled,
_err);
 
@@ -2858,7 +2857,6 @@ out:
 void qmp_block_dirty_bitmap_add(const char *node, const char *name,
 bool has_granularity, uint32_t granularity,
 bool has_persistent, bool persistent,
-bool has_autoload, bool autoload,
 bool has_disabled, bool disabled,
 Error **errp)
 {
@@ -2890,10 +2888,6 @@ void qmp_block_dirty_bitmap_add(const char *node, const 
char *name,
 persistent = false;
 }
 
-if (has_autoload) {
-warn_report("Autoload option is deprecated and its value is ignored");
-}
-
 if (!has_disabled) {
 disabled = false;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 08/19] block/dirty-bitmap: refactor bdrv_dirty_bitmap_next

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

bdrv_dirty_bitmap_next is always used in same pattern. So, split it
into _next and _first, instead of combining two functions into one and
add FOR_EACH_DIRTY_BITMAP macro.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190916141911.5255-5-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h   |  9 +++--
 block.c|  4 +---
 block/dirty-bitmap.c   | 11 +++
 block/qcow2-bitmap.c   |  8 ++--
 migration/block-dirty-bitmap.c |  4 +---
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 2f9b088e11..257f0f6704 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -96,8 +96,13 @@ bool bdrv_dirty_bitmap_get_autoload(const BdrvDirtyBitmap 
*bitmap);
 bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap);
 bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs);
-BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap);
+
+BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs);
+BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap);
+#define FOR_EACH_DIRTY_BITMAP(bs, bitmap) \
+for (bitmap = bdrv_dirty_bitmap_first(bs); bitmap; \
+ bitmap = bdrv_dirty_bitmap_next(bitmap))
+
 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp);
 int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t offset,
 uint64_t bytes);
diff --git a/block.c b/block.c
index d19a4781a3..5721441697 100644
--- a/block.c
+++ b/block.c
@@ -5390,9 +5390,7 @@ static void coroutine_fn 
bdrv_co_invalidate_cache(BlockDriverState *bs,
 }
 }
 
-for (bm = bdrv_dirty_bitmap_next(bs, NULL); bm;
- bm = bdrv_dirty_bitmap_next(bs, bm))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bm) {
 bdrv_dirty_bitmap_skip_store(bm, false);
 }
 
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 4e5c87a907..6065db8094 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -851,11 +851,14 @@ bool bdrv_has_changed_persistent_bitmaps(BlockDriverState 
*bs)
 return false;
 }
 
-BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs,
-BdrvDirtyBitmap *bitmap)
+BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs)
 {
-return bitmap == NULL ? QLIST_FIRST(>dirty_bitmaps) :
-QLIST_NEXT(bitmap, list);
+return QLIST_FIRST(>dirty_bitmaps);
+}
+
+BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap)
+{
+return QLIST_NEXT(bitmap, list);
 }
 
 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp)
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 687087d2bc..99812b418b 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1488,9 +1488,7 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 }
 
 /* check constraints and names */
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap != NULL;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 const char *name = bdrv_dirty_bitmap_name(bitmap);
 uint32_t granularity = bdrv_dirty_bitmap_granularity(bitmap);
 Qcow2Bitmap *bm;
@@ -1610,9 +1608,7 @@ int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error 
**errp)
 return -EINVAL;
 }
 
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap != NULL;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
 bdrv_dirty_bitmap_set_readonly(bitmap, true);
 }
diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c
index 793f249aa5..7eafface61 100644
--- a/migration/block-dirty-bitmap.c
+++ b/migration/block-dirty-bitmap.c
@@ -283,9 +283,7 @@ static int init_dirty_bitmap_migration(void)
 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
 const char *name = bdrv_get_device_or_node_name(bs);
 
-for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap;
- bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
-{
+FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 if (!bdrv_dirty_bitmap_name(bitmap)) {
 continue;
 }
-- 
2.21.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

[libvirt] [PULL v2 14/19] block/qcow2-bitmap: do not remove bitmaps on reopen-ro

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

qcow2_reopen_bitmaps_ro wants to store bitmaps and then mark them all
readonly. But the latter don't work, as
qcow2_store_persistent_dirty_bitmaps removes bitmaps after storing.
It's OK for inactivation but bad idea for reopen-ro. And this leads to
the following bug:

Assume we have persistent bitmap 'bitmap0'.
Create external snapshot
  bitmap0 is stored and therefore removed
Commit snapshot
  now we have no bitmaps
Do some writes from guest (*)
  they are not marked in bitmap
Shutdown
Start
  bitmap0 is loaded as valid, but it is actually broken! It misses
  writes (*)
Incremental backup
  it will be inconsistent

So, let's stop removing bitmaps on reopen-ro. But don't rejoice:
reopening bitmaps to rw is broken too, so the whole scenario will not
work after this patch and we can't enable corresponding test cases in
260 iotests still. Reopening bitmaps rw will be fixed in the following
patches.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: John Snow 
Message-id: 20190927122355.7344-7-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2.h|  3 ++-
 block/qcow2-bitmap.c | 49 ++--
 block/qcow2.c|  2 +-
 3 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index 23a9898a54..5cccd87162 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -742,7 +742,8 @@ Qcow2BitmapInfoList 
*qcow2_get_bitmap_info_list(BlockDriverState *bs,
 Error **errp);
 int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp);
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp);
-void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp);
+void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
+  bool release_stored, Error **errp);
 int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error **errp);
 bool qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
  const char *name,
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index ebc1afccd3..f7dfb40256 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1440,7 +1440,32 @@ out:
 return ret;
 }
 
-void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
+/*
+ * qcow2_store_persistent_dirty_bitmaps
+ *
+ * Stores persistent BdrvDirtyBitmap objects.
+ *
+ * @release_stored: if true, release BdrvDirtyBitmap's after storing to the
+ * image. This is used in two cases, both via qcow2_inactivate:
+ * 1. bdrv_close: It's correct to remove bitmaps on close.
+ * 2. migration: If bitmaps are migrated through migration channel via
+ *'dirty-bitmaps' migration capability they are not handled by this code.
+ *Otherwise, it's OK to drop BdrvDirtyBitmap's and reload them on
+ *invalidation.
+ *
+ * Anyway, it's correct to remove BdrvDirtyBitmap's on inactivation, as
+ * inactivation means that we lose control on disk, and therefore on bitmaps,
+ * we should sync them and do not touch more.
+ *
+ * Contrariwise, we don't want to release any bitmaps on just reopen-to-ro,
+ * when we need to store them, as image is still under our control, and it's
+ * good to keep all the bitmaps in read-only mode. Moreover, keeping them
+ * read-only is correct because this is what would happen if we opened the node
+ * readonly to begin with, and whether we opened directly or reopened to that
+ * state shouldn't matter for the state we get afterward.
+ */
+void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
+  bool release_stored, Error **errp)
 {
 BdrvDirtyBitmap *bitmap;
 BDRVQcow2State *s = bs->opaque;
@@ -1551,20 +1576,14 @@ void 
qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 g_free(tb);
 }
 
-QSIMPLEQ_FOREACH(bm, bm_list, entry) {
-/* For safety, we remove bitmap after storing.
- * We may be here in two cases:
- * 1. bdrv_close. It's ok to drop bitmap.
- * 2. inactivation. It means migration without 'dirty-bitmaps'
- *capability, so bitmaps are not marked with
- *BdrvDirtyBitmap.migration flags. It's not bad to drop them too,
- *and reload on invalidation.
- */
-if (bm->dirty_bitmap == NULL) {
-continue;
-}
+if (release_stored) {
+QSIMPLEQ_FOREACH(bm, bm_list, entry) {
+if (bm->dirty_bitmap == NULL) {
+continue;
+}
 
-bdrv_release_dirty_bitmap(bm->dirty_bitmap);
+bdrv_release_dirty_bitmap(bm->dirty_bitmap);
+}
 }
 
 success:
@@ -1592,7 +1611,7 @@ int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error 
**errp)
 BdrvDirtyBitmap *bitmap;
 Error *local_err = NULL;
 
-qcow2_store_persistent_dirty_b

[PULL v2 10/11] nbd/server: use bdrv_dirty_bitmap_next_dirty_area

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Use bdrv_dirty_bitmap_next_dirty_area for bitmap_to_extents. Since
bdrv_dirty_bitmap_next_dirty_area is very accurate in its interface,
we'll never exceed requested region with last chunk. So, we don't need
dont_fragment, and bitmap_to_extents() interface becomes clean enough
to not require any comment.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Eric Blake 
Message-id: 20200205112041.6003-10-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 nbd/server.c | 59 +---
 1 file changed, 19 insertions(+), 40 deletions(-)

diff --git a/nbd/server.c b/nbd/server.c
index f90bb33a75..02b1ed0801 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -2068,57 +2068,36 @@ static int nbd_co_send_block_status(NBDClient *client, 
uint64_t handle,
 return nbd_co_send_extents(client, handle, ea, last, context_id, errp);
 }
 
-/*
- * Populate @ea from a dirty bitmap. Unless @dont_fragment, the
- * final extent may exceed the original @length.
- */
+/* Populate @ea from a dirty bitmap. */
 static void bitmap_to_extents(BdrvDirtyBitmap *bitmap,
   uint64_t offset, uint64_t length,
-  NBDExtentArray *ea, bool dont_fragment)
+  NBDExtentArray *es)
 {
-uint64_t begin = offset, end = offset;
-uint64_t overall_end = offset + length;
-BdrvDirtyBitmapIter *it;
-bool dirty;
+int64_t start, dirty_start, dirty_count;
+int64_t end = offset + length;
+bool full = false;
 
 bdrv_dirty_bitmap_lock(bitmap);
 
-it = bdrv_dirty_iter_new(bitmap);
-dirty = bdrv_dirty_bitmap_get_locked(bitmap, offset);
-
-while (begin < overall_end) {
-bool next_dirty = !dirty;
-
-if (dirty) {
-end = bdrv_dirty_bitmap_next_zero(bitmap, begin, INT64_MAX);
-} else {
-bdrv_set_dirty_iter(it, begin);
-end = bdrv_dirty_iter_next(it);
-}
-if (end == -1 || end - begin > UINT32_MAX) {
-/* Cap to an aligned value < 4G beyond begin. */
-end = MIN(bdrv_dirty_bitmap_size(bitmap),
-  begin + UINT32_MAX + 1 -
-  bdrv_dirty_bitmap_granularity(bitmap));
-next_dirty = dirty;
-}
-if (dont_fragment && end > overall_end) {
-end = overall_end;
-}
-
-if (nbd_extent_array_add(ea, end - begin,
- dirty ? NBD_STATE_DIRTY : 0) < 0) {
+for (start = offset;
+ bdrv_dirty_bitmap_next_dirty_area(bitmap, start, end, INT32_MAX,
+   _start, _count);
+ start = dirty_start + dirty_count)
+{
+if ((nbd_extent_array_add(es, dirty_start - start, 0) < 0) ||
+(nbd_extent_array_add(es, dirty_count, NBD_STATE_DIRTY) < 0))
+{
+full = true;
 break;
 }
-begin = end;
-dirty = next_dirty;
 }
 
-bdrv_dirty_iter_free(it);
+if (!full) {
+/* last non dirty extent */
+nbd_extent_array_add(es, end - start, 0);
+}
 
 bdrv_dirty_bitmap_unlock(bitmap);
-
-assert(offset < end);
 }
 
 static int nbd_co_send_bitmap(NBDClient *client, uint64_t handle,
@@ -2129,7 +2108,7 @@ static int nbd_co_send_bitmap(NBDClient *client, uint64_t 
handle,
 unsigned int nb_extents = dont_fragment ? 1 : NBD_MAX_BLOCK_STATUS_EXTENTS;
 g_autoptr(NBDExtentArray) ea = nbd_extent_array_new(nb_extents);
 
-bitmap_to_extents(bitmap, offset, length, ea, dont_fragment);
+bitmap_to_extents(bitmap, offset, length, ea);
 
 return nbd_co_send_extents(client, handle, ea, last, context_id, errp);
 }
-- 
2.21.1

[PULL v2 11/11] block/qcow2-bitmap: use bdrv_dirty_bitmap_next_dirty

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

store_bitmap_data() loop does bdrv_set_dirty_iter() on each iteration,
which means that we actually don't need iterator itself and we can use
simpler bitmap API.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-11-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2-bitmap.c | 15 +--
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 82c9f3..cb06954b4a 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1288,7 +1288,6 @@ static uint64_t *store_bitmap_data(BlockDriverState *bs,
 uint64_t bm_size = bdrv_dirty_bitmap_size(bitmap);
 const char *bm_name = bdrv_dirty_bitmap_name(bitmap);
 uint8_t *buf = NULL;
-BdrvDirtyBitmapIter *dbi;
 uint64_t *tb;
 uint64_t tb_size =
 size_to_clusters(s,
@@ -1307,12 +1306,14 @@ static uint64_t *store_bitmap_data(BlockDriverState *bs,
 return NULL;
 }
 
-dbi = bdrv_dirty_iter_new(bitmap);
 buf = g_malloc(s->cluster_size);
 limit = bytes_covered_by_bitmap_cluster(s, bitmap);
 assert(DIV_ROUND_UP(bm_size, limit) == tb_size);
 
-while ((offset = bdrv_dirty_iter_next(dbi)) >= 0) {
+offset = 0;
+while ((offset = bdrv_dirty_bitmap_next_dirty(bitmap, offset, INT64_MAX))
+   >= 0)
+{
 uint64_t cluster = offset / limit;
 uint64_t end, write_size;
 int64_t off;
@@ -1355,23 +1356,17 @@ static uint64_t *store_bitmap_data(BlockDriverState *bs,
 goto fail;
 }
 
-if (end >= bm_size) {
-break;
-}
-
-bdrv_set_dirty_iter(dbi, end);
+offset = end;
 }
 
 *bitmap_table_size = tb_size;
 g_free(buf);
-bdrv_dirty_iter_free(dbi);
 
 return tb;
 
 fail:
 clear_bitmap_table(bs, tb, tb_size);
 g_free(buf);
-bdrv_dirty_iter_free(dbi);
 g_free(tb);
 
 return NULL;
-- 
2.21.1

[PULL v2 09/11] nbd/server: introduce NBDExtentArray

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Introduce NBDExtentArray class, to handle extents list creation in more
controlled way and with fewer OUT parameters in functions.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Eric Blake 
Message-id: 20200205112041.6003-9-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 nbd/server.c | 210 +--
 1 file changed, 118 insertions(+), 92 deletions(-)

diff --git a/nbd/server.c b/nbd/server.c
index 3106aaf3b4..f90bb33a75 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -1909,27 +1909,98 @@ static int coroutine_fn 
nbd_co_send_sparse_read(NBDClient *client,
 return ret;
 }
 
+typedef struct NBDExtentArray {
+NBDExtent *extents;
+unsigned int nb_alloc;
+unsigned int count;
+uint64_t total_length;
+bool can_add;
+bool converted_to_be;
+} NBDExtentArray;
+
+static NBDExtentArray *nbd_extent_array_new(unsigned int nb_alloc)
+{
+NBDExtentArray *ea = g_new0(NBDExtentArray, 1);
+
+ea->nb_alloc = nb_alloc;
+ea->extents = g_new(NBDExtent, nb_alloc);
+ea->can_add = true;
+
+return ea;
+}
+
+static void nbd_extent_array_free(NBDExtentArray *ea)
+{
+g_free(ea->extents);
+g_free(ea);
+}
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(NBDExtentArray, nbd_extent_array_free);
+
+/* Further modifications of the array after conversion are abandoned */
+static void nbd_extent_array_convert_to_be(NBDExtentArray *ea)
+{
+int i;
+
+assert(!ea->converted_to_be);
+ea->can_add = false;
+ea->converted_to_be = true;
+
+for (i = 0; i < ea->count; i++) {
+ea->extents[i].flags = cpu_to_be32(ea->extents[i].flags);
+ea->extents[i].length = cpu_to_be32(ea->extents[i].length);
+}
+}
+
 /*
- * Populate @extents from block status. Update @bytes to be the actual
- * length encoded (which may be smaller than the original), and update
- * @nb_extents to the number of extents used.
- *
- * Returns zero on success and -errno on bdrv_block_status_above failure.
+ * Add extent to NBDExtentArray. If extent can't be added (no available space),
+ * return -1.
+ * For safety, when returning -1 for the first time, .can_add is set to false,
+ * further call to nbd_extent_array_add() will crash.
+ * (to avoid the situation, when after failing to add an extent (returned -1),
+ * user miss this failure and add another extent, which is successfully added
+ * (array is full, but new extent may be squashed into the last one), then we
+ * have invalid array with skipped extent)
  */
+static int nbd_extent_array_add(NBDExtentArray *ea,
+uint32_t length, uint32_t flags)
+{
+assert(ea->can_add);
+
+if (!length) {
+return 0;
+}
+
+/* Extend previous extent if flags are the same */
+if (ea->count > 0 && flags == ea->extents[ea->count - 1].flags) {
+uint64_t sum = (uint64_t)length + ea->extents[ea->count - 1].length;
+
+if (sum <= UINT32_MAX) {
+ea->extents[ea->count - 1].length = sum;
+ea->total_length += length;
+return 0;
+}
+}
+
+if (ea->count >= ea->nb_alloc) {
+ea->can_add = false;
+return -1;
+}
+
+ea->total_length += length;
+ea->extents[ea->count] = (NBDExtent) {.length = length, .flags = flags};
+ea->count++;
+
+return 0;
+}
+
 static int blockstatus_to_extents(BlockDriverState *bs, uint64_t offset,
-  uint64_t *bytes, NBDExtent *extents,
-  unsigned int *nb_extents)
+  uint64_t bytes, NBDExtentArray *ea)
 {
-uint64_t remaining_bytes = *bytes;
-NBDExtent *extent = extents, *extents_end = extents + *nb_extents;
-bool first_extent = true;
-
-assert(*nb_extents);
-while (remaining_bytes) {
+while (bytes) {
 uint32_t flags;
 int64_t num;
-int ret = bdrv_block_status_above(bs, NULL, offset, remaining_bytes,
-  , NULL, NULL);
+int ret = bdrv_block_status_above(bs, NULL, offset, bytes, ,
+  NULL, NULL);
 
 if (ret < 0) {
 return ret;
@@ -1938,60 +2009,37 @@ static int blockstatus_to_extents(BlockDriverState *bs, 
uint64_t offset,
 flags = (ret & BDRV_BLOCK_ALLOCATED ? 0 : NBD_STATE_HOLE) |
 (ret & BDRV_BLOCK_ZERO  ? NBD_STATE_ZERO : 0);
 
-if (first_extent) {
-extent->flags = flags;
-extent->length = num;
-first_extent = false;
-} else if (flags == extent->flags) {
-/* extend current extent */
-extent->length += num;
-} else {
-if (extent + 1 == extents_end) {
-break;
-}
-
-/* start new extent */

[PULL v2 02/11] hbitmap: assert that we don't create bitmap larger than INT64_MAX

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

We have APIs which returns signed int64_t, to be able to return error.
Therefore we can't handle bitmaps with absolute size larger than
(INT64_MAX+1). Still, keep maximum to be INT64_MAX which is a bit
safer.

Note, that bitmaps are used to represent disk images, which can't
exceed INT64_MAX anyway.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: Eric Blake 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-2-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 util/hbitmap.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/util/hbitmap.c b/util/hbitmap.c
index 242c6e519c..7f9b3e0cd7 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -716,6 +716,7 @@ HBitmap *hbitmap_alloc(uint64_t size, int granularity)
 HBitmap *hb = g_new0(struct HBitmap, 1);
 unsigned i;
 
+assert(size <= INT64_MAX);
 hb->orig_size = size;
 
 assert(granularity >= 0 && granularity < 64);
@@ -746,6 +747,7 @@ void hbitmap_truncate(HBitmap *hb, uint64_t size)
 uint64_t num_elements = size;
 uint64_t old;
 
+assert(size <= INT64_MAX);
 hb->orig_size = size;
 
 /* Size comes in as logical elements, adjust for granularity. */
-- 
2.21.1

[PULL v2 03/11] hbitmap: move hbitmap_iter_next_word to hbitmap.c

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

The function is definitely internal (it's not used by third party and
it has complicated interface). Move it to .c file.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-3-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/qemu/hbitmap.h | 30 --
 util/hbitmap.c | 29 +
 2 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 1bf944ca3d..ab227b117f 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -362,34 +362,4 @@ void hbitmap_free_meta(HBitmap *hb);
  */
 int64_t hbitmap_iter_next(HBitmapIter *hbi);
 
-/**
- * hbitmap_iter_next_word:
- * @hbi: HBitmapIter to operate on.
- * @p_cur: Location where to store the next non-zero word.
- *
- * Return the index of the next nonzero word that is set in @hbi's
- * associated HBitmap, and set *p_cur to the content of that word
- * (bits before the index that was passed to hbitmap_iter_init are
- * trimmed on the first call).  Return -1, and set *p_cur to zero,
- * if all remaining words are zero.
- */
-static inline size_t hbitmap_iter_next_word(HBitmapIter *hbi, unsigned long 
*p_cur)
-{
-unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
-
-if (cur == 0) {
-cur = hbitmap_iter_skip_words(hbi);
-if (cur == 0) {
-*p_cur = 0;
-return -1;
-}
-}
-
-/* The next call will resume work from the next word.  */
-hbi->cur[HBITMAP_LEVELS - 1] = 0;
-*p_cur = cur;
-return hbi->pos;
-}
-
-
 #endif
diff --git a/util/hbitmap.c b/util/hbitmap.c
index 7f9b3e0cd7..a368dc5ef7 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -298,6 +298,35 @@ uint64_t hbitmap_count(const HBitmap *hb)
 return hb->count << hb->granularity;
 }
 
+/**
+ * hbitmap_iter_next_word:
+ * @hbi: HBitmapIter to operate on.
+ * @p_cur: Location where to store the next non-zero word.
+ *
+ * Return the index of the next nonzero word that is set in @hbi's
+ * associated HBitmap, and set *p_cur to the content of that word
+ * (bits before the index that was passed to hbitmap_iter_init are
+ * trimmed on the first call).  Return -1, and set *p_cur to zero,
+ * if all remaining words are zero.
+ */
+static size_t hbitmap_iter_next_word(HBitmapIter *hbi, unsigned long *p_cur)
+{
+unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
+
+if (cur == 0) {
+cur = hbitmap_iter_skip_words(hbi);
+if (cur == 0) {
+*p_cur = 0;
+return -1;
+}
+}
+
+/* The next call will resume work from the next word.  */
+hbi->cur[HBITMAP_LEVELS - 1] = 0;
+*p_cur = cur;
+return hbi->pos;
+}
+
 /* Count the number of set bits between start and end, not accounting for
  * the granularity.  Also an example of how to use hbitmap_iter_next_word.
  */
-- 
2.21.1

[PULL v2 05/11] hbitmap: drop meta bitmaps as they are unused

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-5-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/qemu/hbitmap.h |  21 
 tests/test-hbitmap.c   | 115 -
 util/hbitmap.c |  16 --
 3 files changed, 152 deletions(-)

diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 15837a0e2d..df922d8517 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -325,27 +325,6 @@ int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t 
start, uint64_t count);
 bool hbitmap_next_dirty_area(const HBitmap *hb, uint64_t *start,
  uint64_t *count);
 
-/* hbitmap_create_meta:
- * Create a "meta" hbitmap to track dirtiness of the bits in this HBitmap.
- * The caller owns the created bitmap and must call hbitmap_free_meta(hb) to
- * free it.
- *
- * Currently, we only guarantee that if a bit in the hbitmap is changed it
- * will be reflected in the meta bitmap, but we do not yet guarantee the
- * opposite.
- *
- * @hb: The HBitmap to operate on.
- * @chunk_size: How many bits in @hb does one bit in the meta track.
- */
-HBitmap *hbitmap_create_meta(HBitmap *hb, int chunk_size);
-
-/* hbitmap_free_meta:
- * Free the meta bitmap of @hb.
- *
- * @hb: The HBitmap whose meta bitmap should be freed.
- */
-void hbitmap_free_meta(HBitmap *hb);
-
 /**
  * hbitmap_iter_next:
  * @hbi: HBitmapIter to operate on.
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
index e1f867085f..aeaa0b3f22 100644
--- a/tests/test-hbitmap.c
+++ b/tests/test-hbitmap.c
@@ -22,7 +22,6 @@
 
 typedef struct TestHBitmapData {
 HBitmap   *hb;
-HBitmap   *meta;
 unsigned long *bits;
 size_t size;
 size_t old_size;
@@ -94,14 +93,6 @@ static void hbitmap_test_init(TestHBitmapData *data,
 }
 }
 
-static void hbitmap_test_init_meta(TestHBitmapData *data,
-   uint64_t size, int granularity,
-   int meta_chunk)
-{
-hbitmap_test_init(data, size, granularity);
-data->meta = hbitmap_create_meta(data->hb, meta_chunk);
-}
-
 static inline size_t hbitmap_test_array_size(size_t bits)
 {
 size_t n = DIV_ROUND_UP(bits, BITS_PER_LONG);
@@ -144,9 +135,6 @@ static void hbitmap_test_teardown(TestHBitmapData *data,
   const void *unused)
 {
 if (data->hb) {
-if (data->meta) {
-hbitmap_free_meta(data->hb);
-}
 hbitmap_free(data->hb);
 data->hb = NULL;
 }
@@ -648,96 +636,6 @@ static void 
test_hbitmap_truncate_shrink_large(TestHBitmapData *data,
 hbitmap_test_truncate(data, size, -diff, 0);
 }
 
-static void hbitmap_check_meta(TestHBitmapData *data,
-   int64_t start, int count)
-{
-int64_t i;
-
-for (i = 0; i < data->size; i++) {
-if (i >= start && i < start + count) {
-g_assert(hbitmap_get(data->meta, i));
-} else {
-g_assert(!hbitmap_get(data->meta, i));
-}
-}
-}
-
-static void hbitmap_test_meta(TestHBitmapData *data,
-  int64_t start, int count,
-  int64_t check_start, int check_count)
-{
-hbitmap_reset_all(data->hb);
-hbitmap_reset_all(data->meta);
-
-/* Test "unset" -> "unset" will not update meta. */
-hbitmap_reset(data->hb, start, count);
-hbitmap_check_meta(data, 0, 0);
-
-/* Test "unset" -> "set" will update meta */
-hbitmap_set(data->hb, start, count);
-hbitmap_check_meta(data, check_start, check_count);
-
-/* Test "set" -> "set" will not update meta */
-hbitmap_reset_all(data->meta);
-hbitmap_set(data->hb, start, count);
-hbitmap_check_meta(data, 0, 0);
-
-/* Test "set" -> "unset" will update meta */
-hbitmap_reset_all(data->meta);
-hbitmap_reset(data->hb, start, count);
-hbitmap_check_meta(data, check_start, check_count);
-}
-
-static void hbitmap_test_meta_do(TestHBitmapData *data, int chunk_size)
-{
-uint64_t size = chunk_size * 100;
-hbitmap_test_init_meta(data, size, 0, chunk_size);
-
-hbitmap_test_meta(data, 0, 1, 0, chunk_size);
-hbitmap_test_meta(data, 0, chunk_size, 0, chunk_size);
-hbitmap_test_meta(data, chunk_size - 1, 1, 0, chunk_size);
-hbitmap_test_meta(data, chunk_size - 1, 2, 0, chunk_size * 2);
-hbitmap_test_meta(data, chunk_size - 1, chunk_size + 1, 0, chunk_size * 2);
-hbitmap_test_meta(data, chunk_size - 1, chunk_size + 2, 0, chunk_size * 3);
-hbitmap_test_meta(data, 7 * chunk_size - 1, chunk_size + 2,
-  6 * chunk_size, chunk_size * 3);
-hbitmap_test_meta(data, si

[PULL v2 01/11] build: Silence clang warning on older glib autoptr usage

[John Snow]

From: Eric Blake 

glib's G_DEFINE_AUTOPTR_CLEANUP_FUNC() macro defines several static
inline functions, often with some of them unused, but prior to 2.57.2
did not mark the functions as such.  As a result, clang (but not gcc)
fails to build with older glib unless -Wno-unused-function is enabled.

Reported-by: Peter Maydell 
Signed-off-by: Eric Blake 
Reviewed-by: John Snow 
Message-id: 20200317175534.196295-1-ebl...@redhat.com
Signed-off-by: John Snow 
---
 configure | 20 
 1 file changed, 20 insertions(+)

diff --git a/configure b/configure
index 06fcd070fb..479336bf6e 100755
--- a/configure
+++ b/configure
@@ -3851,6 +3851,26 @@ if ! compile_prog "$glib_cflags -Werror" "$glib_libs" ; 
then
 fi
 fi
 
+# Silence clang warnings triggered by glib < 2.57.2
+cat > $TMPC << EOF
+#include 
+typedef struct Foo {
+int i;
+} Foo;
+static void foo_free(Foo *f)
+{
+g_free(f);
+}
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(Foo, foo_free);
+int main(void) { return 0; }
+EOF
+if ! compile_prog "$glib_cflags -Werror" "$glib_libs" ; then
+if cc_has_warning_flag "-Wno-unused-function"; then
+glib_cflags="$glib_cflags -Wno-unused-function"
+CFLAGS="$CFLAGS -Wno-unused-function"
+fi
+fi
+
 #
 # zlib check
 
-- 
2.21.1

[PULL v2 07/11] block/dirty-bitmap: add _next_dirty API

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

We have bdrv_dirty_bitmap_next_zero, let's add corresponding
bdrv_dirty_bitmap_next_dirty, which is more comfortable to use than
bitmap iterators in some cases.

For test modify test_hbitmap_next_zero_check_range to check both
next_zero and next_dirty and add some new checks.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-7-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h |   2 +
 include/qemu/hbitmap.h   |  13 
 block/dirty-bitmap.c |   6 ++
 tests/test-hbitmap.c | 130 ---
 util/hbitmap.c   |  60 
 5 files changed, 126 insertions(+), 85 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 27c72cc56a..b1f0de12db 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -105,6 +105,8 @@ for (bitmap = bdrv_dirty_bitmap_first(bs); bitmap; \
  bitmap = bdrv_dirty_bitmap_next(bitmap))
 
 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp);
+int64_t bdrv_dirty_bitmap_next_dirty(BdrvDirtyBitmap *bitmap, int64_t offset,
+ int64_t bytes);
 int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, int64_t offset,
 int64_t bytes);
 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index b6e85f3d5d..6e9ae51ed3 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -297,6 +297,19 @@ void hbitmap_free(HBitmap *hb);
  */
 void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first);
 
+/*
+ * hbitmap_next_dirty:
+ *
+ * Find next dirty bit within selected range. If not found, return -1.
+ *
+ * @hb: The HBitmap to operate on
+ * @start: The bit to start from.
+ * @count: Number of bits to proceed. If @start+@count > bitmap size, the whole
+ * bitmap is looked through. You can use INT64_MAX as @count to search up to
+ * the bitmap end.
+ */
+int64_t hbitmap_next_dirty(const HBitmap *hb, int64_t start, int64_t count);
+
 /* hbitmap_next_zero:
  *
  * Find next not dirty bit within selected range. If not found, return -1.
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index af9f5411a6..1b14c8eb26 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -860,6 +860,12 @@ char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap 
*bitmap, Error **errp)
 return hbitmap_sha256(bitmap->bitmap, errp);
 }
 
+int64_t bdrv_dirty_bitmap_next_dirty(BdrvDirtyBitmap *bitmap, int64_t offset,
+ int64_t bytes)
+{
+return hbitmap_next_dirty(bitmap->bitmap, offset, bytes);
+}
+
 int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, int64_t offset,
 int64_t bytes)
 {
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
index 9d210dc18c..8905b8a351 100644
--- a/tests/test-hbitmap.c
+++ b/tests/test-hbitmap.c
@@ -816,92 +816,108 @@ static void test_hbitmap_iter_and_reset(TestHBitmapData 
*data,
 hbitmap_iter_next();
 }
 
-static void test_hbitmap_next_zero_check_range(TestHBitmapData *data,
-   int64_t start,
-   int64_t count)
+static void test_hbitmap_next_x_check_range(TestHBitmapData *data,
+int64_t start,
+int64_t count)
 {
-int64_t ret1 = hbitmap_next_zero(data->hb, start, count);
-int64_t ret2 = start;
+int64_t next_zero = hbitmap_next_zero(data->hb, start, count);
+int64_t next_dirty = hbitmap_next_dirty(data->hb, start, count);
+int64_t next;
 int64_t end = start >= data->size || data->size - start < count ?
 data->size : start + count;
+bool first_bit = hbitmap_get(data->hb, start);
 
-for ( ; ret2 < end && hbitmap_get(data->hb, ret2); ret2++) {
+for (next = start;
+ next < end && hbitmap_get(data->hb, next) == first_bit;
+ next++)
+{
 ;
 }
-if (ret2 == end) {
-ret2 = -1;
+
+if (next == end) {
+next = -1;
 }
 
-g_assert_cmpint(ret1, ==, ret2);
+g_assert_cmpint(next_dirty, ==, first_bit ? start : next);
+g_assert_cmpint(next_zero, ==, first_bit ? next : start);
 }
 
-static void test_hbitmap_next_zero_check(TestHBitmapData *data, int64_t start)
+static void test_hbitmap_next_x_check(TestHBitmapData *data, int64_t start)
 {
-test_hbitmap_next_zero_check_range(data, start, INT64_MAX);
+test_hbitmap_next_x_check_range(data, start, INT64_MAX);
 }
 
-static void test_hbitmap_next_zero_do(TestHBitmapData *data, int granularity)
+static void test_hbitmap_next_x_do(TestHBitma

[PULL v2 00/11] Bitmaps patches

[John Snow]

The following changes since commit d649689a8ecb2e276cc20d3af6d416e3c299cb17:

  Merge remote-tracking branch 'remotes/bonzini/tags/for-upstream' into staging 
(2020-03-17 18:33:05 +)

are available in the Git repository at:

  https://github.com/jnsnow/qemu.git tags/bitmaps-pull-request

for you to fetch changes up to 2d00cbd8e222a4adc08f415c399e84590ee8ff9a:

  block/qcow2-bitmap: use bdrv_dirty_bitmap_next_dirty (2020-03-18 14:03:46 
-0400)


Pull request



Eric Blake (1):
  build: Silence clang warning on older glib autoptr usage

Vladimir Sementsov-Ogievskiy (10):
  hbitmap: assert that we don't create bitmap larger than INT64_MAX
  hbitmap: move hbitmap_iter_next_word to hbitmap.c
  hbitmap: unpublish hbitmap_iter_skip_words
  hbitmap: drop meta bitmaps as they are unused
  block/dirty-bitmap: switch _next_dirty_area and _next_zero to int64_t
  block/dirty-bitmap: add _next_dirty API
  block/dirty-bitmap: improve _next_dirty_area API
  nbd/server: introduce NBDExtentArray
  nbd/server: use bdrv_dirty_bitmap_next_dirty_area
  block/qcow2-bitmap: use bdrv_dirty_bitmap_next_dirty

 configure|  20 +++
 include/block/dirty-bitmap.h |   9 +-
 include/qemu/hbitmap.h   |  95 +++
 block/dirty-bitmap.c |  16 +-
 block/qcow2-bitmap.c |  15 +-
 nbd/server.c | 251 ++--
 tests/test-hbitmap.c | 316 +--
 util/hbitmap.c   | 134 +--
 8 files changed, 395 insertions(+), 461 deletions(-)

-- 
2.21.1

[PULL v2 06/11] block/dirty-bitmap: switch _next_dirty_area and _next_zero to int64_t

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

We are going to introduce bdrv_dirty_bitmap_next_dirty so that same
variable may be used to store its return value and to be its parameter,
so it would int64_t.

Similarly, we are going to refactor hbitmap_next_dirty_area to use
hbitmap_next_dirty together with hbitmap_next_zero, therefore we want
hbitmap_next_zero parameter type to be int64_t too.

So, for convenience update all parameters of *_next_zero and
*_next_dirty_area to be int64_t.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-6-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h |  6 +++---
 include/qemu/hbitmap.h   |  7 +++
 block/dirty-bitmap.c |  6 +++---
 nbd/server.c |  2 +-
 tests/test-hbitmap.c | 36 ++--
 util/hbitmap.c   | 13 -
 6 files changed, 36 insertions(+), 34 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index e2b20ecab9..27c72cc56a 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -105,10 +105,10 @@ for (bitmap = bdrv_dirty_bitmap_first(bs); bitmap; \
  bitmap = bdrv_dirty_bitmap_next(bitmap))
 
 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp);
-int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t offset,
-uint64_t bytes);
+int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, int64_t offset,
+int64_t bytes);
 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
-   uint64_t *offset, uint64_t *bytes);
+   int64_t *offset, int64_t *bytes);
 BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
   Error **errp);
 
diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index df922d8517..b6e85f3d5d 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -304,10 +304,10 @@ void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap 
*hb, uint64_t first);
  * @hb: The HBitmap to operate on
  * @start: The bit to start from.
  * @count: Number of bits to proceed. If @start+@count > bitmap size, the whole
- * bitmap is looked through. You can use UINT64_MAX as @count to search up to
+ * bitmap is looked through. You can use INT64_MAX as @count to search up to
  * the bitmap end.
  */
-int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t start, uint64_t count);
+int64_t hbitmap_next_zero(const HBitmap *hb, int64_t start, int64_t count);
 
 /* hbitmap_next_dirty_area:
  * @hb: The HBitmap to operate on
@@ -322,8 +322,7 @@ int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t 
start, uint64_t count);
  * @offset and @bytes appropriately. Otherwise returns false and leaves @offset
  * and @bytes unchanged.
  */
-bool hbitmap_next_dirty_area(const HBitmap *hb, uint64_t *start,
- uint64_t *count);
+bool hbitmap_next_dirty_area(const HBitmap *hb, int64_t *start, int64_t 
*count);
 
 /**
  * hbitmap_iter_next:
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 7039e82520..af9f5411a6 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -860,14 +860,14 @@ char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap 
*bitmap, Error **errp)
 return hbitmap_sha256(bitmap->bitmap, errp);
 }
 
-int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, uint64_t offset,
-uint64_t bytes)
+int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, int64_t offset,
+int64_t bytes)
 {
 return hbitmap_next_zero(bitmap->bitmap, offset, bytes);
 }
 
 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
-   uint64_t *offset, uint64_t *bytes)
+   int64_t *offset, int64_t *bytes)
 {
 return hbitmap_next_dirty_area(bitmap->bitmap, offset, bytes);
 }
diff --git a/nbd/server.c b/nbd/server.c
index 11a31094ff..3106aaf3b4 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -2055,7 +2055,7 @@ static unsigned int bitmap_to_extents(BdrvDirtyBitmap 
*bitmap, uint64_t offset,
 bool next_dirty = !dirty;
 
 if (dirty) {
-end = bdrv_dirty_bitmap_next_zero(bitmap, begin, UINT64_MAX);
+end = bdrv_dirty_bitmap_next_zero(bitmap, begin, INT64_MAX);
 } else {
 bdrv_set_dirty_iter(it, begin);
 end = bdrv_dirty_iter_next(it);
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
index aeaa0b3f22..9d210dc18c 100644
--- a/tests/test-hbitmap.c
+++ b/tests/test-hbitmap.c
@@ -817,8 +817,8 @@ static void test_hbitmap_iter_and_reset(TestHBitmapData 
*data,
 }
 
 static void test_hbitmap_next_zero

[PULL v2 08/11] block/dirty-bitmap: improve _next_dirty_area API

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Firstly, _next_dirty_area is for scenarios when we may contiguously
search for next dirty area inside some limited region, so it is more
comfortable to specify "end" which should not be recalculated on each
iteration.

Secondly, let's add a possibility to limit resulting area size, not
limiting searching area. This will be used in NBD code in further
commit. (Note that now bdrv_dirty_bitmap_next_dirty_area is unused)

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-8-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/block/dirty-bitmap.h |  3 ++-
 include/qemu/hbitmap.h   | 23 ++
 block/dirty-bitmap.c |  6 +++--
 tests/test-hbitmap.c | 45 +++-
 util/hbitmap.c   | 44 +--
 5 files changed, 75 insertions(+), 46 deletions(-)

diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index b1f0de12db..8a10029418 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -110,7 +110,8 @@ int64_t bdrv_dirty_bitmap_next_dirty(BdrvDirtyBitmap 
*bitmap, int64_t offset,
 int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, int64_t offset,
 int64_t bytes);
 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
-   int64_t *offset, int64_t *bytes);
+int64_t start, int64_t end, int64_t max_dirty_count,
+int64_t *dirty_start, int64_t *dirty_count);
 BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
   Error **errp);
 
diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 6e9ae51ed3..5e71b6d6f7 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -324,18 +324,21 @@ int64_t hbitmap_next_zero(const HBitmap *hb, int64_t 
start, int64_t count);
 
 /* hbitmap_next_dirty_area:
  * @hb: The HBitmap to operate on
- * @start: in-out parameter.
- * in: the offset to start from
- * out: (if area found) start of found area
- * @count: in-out parameter.
- * in: length of requested region
- * out: length of found area
+ * @start: the offset to start from
+ * @end: end of requested area
+ * @max_dirty_count: limit for out parameter dirty_count
+ * @dirty_start: on success: start of found area
+ * @dirty_count: on success: length of found area
  *
- * If dirty area found within [@start, @start + @count), returns true and sets
- * @offset and @bytes appropriately. Otherwise returns false and leaves @offset
- * and @bytes unchanged.
+ * If dirty area found within [@start, @end), returns true and sets
+ * @dirty_start and @dirty_count appropriately. @dirty_count will not exceed
+ * @max_dirty_count.
+ * If dirty area was not found, returns false and leaves @dirty_start and
+ * @dirty_count unchanged.
  */
-bool hbitmap_next_dirty_area(const HBitmap *hb, int64_t *start, int64_t 
*count);
+bool hbitmap_next_dirty_area(const HBitmap *hb, int64_t start, int64_t end,
+ int64_t max_dirty_count,
+ int64_t *dirty_start, int64_t *dirty_count);
 
 /**
  * hbitmap_iter_next:
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 1b14c8eb26..063793e316 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -873,9 +873,11 @@ int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap 
*bitmap, int64_t offset,
 }
 
 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
-   int64_t *offset, int64_t *bytes)
+int64_t start, int64_t end, int64_t max_dirty_count,
+int64_t *dirty_start, int64_t *dirty_count)
 {
-return hbitmap_next_dirty_area(bitmap->bitmap, offset, bytes);
+return hbitmap_next_dirty_area(bitmap->bitmap, start, end, max_dirty_count,
+   dirty_start, dirty_count);
 }
 
 /**
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
index 8905b8a351..b6726cf76b 100644
--- a/tests/test-hbitmap.c
+++ b/tests/test-hbitmap.c
@@ -920,18 +920,19 @@ static void 
test_hbitmap_next_x_after_truncate(TestHBitmapData *data,
 test_hbitmap_next_x_check(data, 0);
 }
 
-static void test_hbitmap_next_dirty_area_check(TestHBitmapData *data,
-   int64_t offset,
-   int64_t count)
+static void test_hbitmap_next_dirty_area_check_limited(TestHBitmapData *data,
+   int64_t offset,
+   int64_t count,
+   int64_t max_dirty)
 {
 int64_t off1, off2;
 int64_t len1 = 0, len2;
 bool ret1, ret2;
 int64_t end;
 
-off1 = offset;
-

[PULL v2 04/11] hbitmap: unpublish hbitmap_iter_skip_words

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Function is internal and even commented as internal. Drop its
definition from .h file.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-4-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/qemu/hbitmap.h | 7 ---
 util/hbitmap.c | 2 +-
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index ab227b117f..15837a0e2d 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -297,13 +297,6 @@ void hbitmap_free(HBitmap *hb);
  */
 void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first);
 
-/* hbitmap_iter_skip_words:
- * @hbi: HBitmapIter to operate on.
- *
- * Internal function used by hbitmap_iter_next and hbitmap_iter_next_word.
- */
-unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi);
-
 /* hbitmap_next_zero:
  *
  * Find next not dirty bit within selected range. If not found, return -1.
diff --git a/util/hbitmap.c b/util/hbitmap.c
index a368dc5ef7..26145d4b9e 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -104,7 +104,7 @@ struct HBitmap {
 /* Advance hbi to the next nonzero word and return it.  hbi->pos
  * is updated.  Returns zero if we reach the end of the bitmap.
  */
-unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi)
+static unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi)
 {
 size_t pos = hbi->pos;
 const HBitmap *hb = hbi->hb;
-- 
2.21.1

Re: [PULL v2 00/11] Bitmaps patches

[John Snow]

On 3/19/20 1:57 PM, Peter Maydell wrote:
> On Wed, 18 Mar 2020 at 20:24, John Snow  wrote:
>>
>> The following changes since commit d649689a8ecb2e276cc20d3af6d416e3c299cb17:
>>
>>   Merge remote-tracking branch 'remotes/bonzini/tags/for-upstream' into 
>> staging (2020-03-17 18:33:05 +)
>>
>> are available in the Git repository at:
>>
>>   https://github.com/jnsnow/qemu.git tags/bitmaps-pull-request
>>
>> for you to fetch changes up to 2d00cbd8e222a4adc08f415c399e84590ee8ff9a:
>>
>>   block/qcow2-bitmap: use bdrv_dirty_bitmap_next_dirty (2020-03-18 14:03:46 
>> -0400)
>>
>> 
>> Pull request
>>
>> 
> 
> 
> Applied, thanks.
> 

Wonderful, thanks!

> Please update the changelog at https://wiki.qemu.org/ChangeLog/5.0
> for any user-visible changes.
> 
> -- PMM

Will do.

[PULL 00/10] Bitmaps patches

[John Snow]

The following changes since commit 6e8a73e911f066527e775e04b98f31ebd19db600:

  Merge remote-tracking branch 'remotes/stefanha/tags/block-pull-request' into 
staging (2020-03-11 14:41:27 +)

are available in the Git repository at:

  https://github.com/jnsnow/qemu.git tags/bitmaps-pull-request

for you to fetch changes up to 34b456d485a4df3a88116fb5ef0c418f2f12990d:

  block/qcow2-bitmap: use bdrv_dirty_bitmap_next_dirty (2020-03-12 16:36:46 
-0400)


Pull request



Vladimir Sementsov-Ogievskiy (10):
  hbitmap: assert that we don't create bitmap larger than INT64_MAX
  hbitmap: move hbitmap_iter_next_word to hbitmap.c
  hbitmap: unpublish hbitmap_iter_skip_words
  hbitmap: drop meta bitmaps as they are unused
  block/dirty-bitmap: switch _next_dirty_area and _next_zero to int64_t
  block/dirty-bitmap: add _next_dirty API
  block/dirty-bitmap: improve _next_dirty_area API
  nbd/server: introduce NBDExtentArray
  nbd/server: use bdrv_dirty_bitmap_next_dirty_area
  block/qcow2-bitmap: use bdrv_dirty_bitmap_next_dirty

 include/block/dirty-bitmap.h |   9 +-
 include/qemu/hbitmap.h   |  95 +++
 block/dirty-bitmap.c |  16 +-
 block/qcow2-bitmap.c |  15 +-
 nbd/server.c | 251 ++--
 tests/test-hbitmap.c | 316 +--
 util/hbitmap.c   | 134 +--
 7 files changed, 375 insertions(+), 461 deletions(-)

-- 
2.21.1

[PULL 08/10] nbd/server: introduce NBDExtentArray

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Introduce NBDExtentArray class, to handle extents list creation in more
controlled way and with fewer OUT parameters in functions.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Eric Blake 
Message-id: 20200205112041.6003-9-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 nbd/server.c | 210 +--
 1 file changed, 118 insertions(+), 92 deletions(-)

diff --git a/nbd/server.c b/nbd/server.c
index 3106aaf3b4..f90bb33a75 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -1909,27 +1909,98 @@ static int coroutine_fn 
nbd_co_send_sparse_read(NBDClient *client,
 return ret;
 }
 
+typedef struct NBDExtentArray {
+NBDExtent *extents;
+unsigned int nb_alloc;
+unsigned int count;
+uint64_t total_length;
+bool can_add;
+bool converted_to_be;
+} NBDExtentArray;
+
+static NBDExtentArray *nbd_extent_array_new(unsigned int nb_alloc)
+{
+NBDExtentArray *ea = g_new0(NBDExtentArray, 1);
+
+ea->nb_alloc = nb_alloc;
+ea->extents = g_new(NBDExtent, nb_alloc);
+ea->can_add = true;
+
+return ea;
+}
+
+static void nbd_extent_array_free(NBDExtentArray *ea)
+{
+g_free(ea->extents);
+g_free(ea);
+}
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(NBDExtentArray, nbd_extent_array_free);
+
+/* Further modifications of the array after conversion are abandoned */
+static void nbd_extent_array_convert_to_be(NBDExtentArray *ea)
+{
+int i;
+
+assert(!ea->converted_to_be);
+ea->can_add = false;
+ea->converted_to_be = true;
+
+for (i = 0; i < ea->count; i++) {
+ea->extents[i].flags = cpu_to_be32(ea->extents[i].flags);
+ea->extents[i].length = cpu_to_be32(ea->extents[i].length);
+}
+}
+
 /*
- * Populate @extents from block status. Update @bytes to be the actual
- * length encoded (which may be smaller than the original), and update
- * @nb_extents to the number of extents used.
- *
- * Returns zero on success and -errno on bdrv_block_status_above failure.
+ * Add extent to NBDExtentArray. If extent can't be added (no available space),
+ * return -1.
+ * For safety, when returning -1 for the first time, .can_add is set to false,
+ * further call to nbd_extent_array_add() will crash.
+ * (to avoid the situation, when after failing to add an extent (returned -1),
+ * user miss this failure and add another extent, which is successfully added
+ * (array is full, but new extent may be squashed into the last one), then we
+ * have invalid array with skipped extent)
  */
+static int nbd_extent_array_add(NBDExtentArray *ea,
+uint32_t length, uint32_t flags)
+{
+assert(ea->can_add);
+
+if (!length) {
+return 0;
+}
+
+/* Extend previous extent if flags are the same */
+if (ea->count > 0 && flags == ea->extents[ea->count - 1].flags) {
+uint64_t sum = (uint64_t)length + ea->extents[ea->count - 1].length;
+
+if (sum <= UINT32_MAX) {
+ea->extents[ea->count - 1].length = sum;
+ea->total_length += length;
+return 0;
+}
+}
+
+if (ea->count >= ea->nb_alloc) {
+ea->can_add = false;
+return -1;
+}
+
+ea->total_length += length;
+ea->extents[ea->count] = (NBDExtent) {.length = length, .flags = flags};
+ea->count++;
+
+return 0;
+}
+
 static int blockstatus_to_extents(BlockDriverState *bs, uint64_t offset,
-  uint64_t *bytes, NBDExtent *extents,
-  unsigned int *nb_extents)
+  uint64_t bytes, NBDExtentArray *ea)
 {
-uint64_t remaining_bytes = *bytes;
-NBDExtent *extent = extents, *extents_end = extents + *nb_extents;
-bool first_extent = true;
-
-assert(*nb_extents);
-while (remaining_bytes) {
+while (bytes) {
 uint32_t flags;
 int64_t num;
-int ret = bdrv_block_status_above(bs, NULL, offset, remaining_bytes,
-  , NULL, NULL);
+int ret = bdrv_block_status_above(bs, NULL, offset, bytes, ,
+  NULL, NULL);
 
 if (ret < 0) {
 return ret;
@@ -1938,60 +2009,37 @@ static int blockstatus_to_extents(BlockDriverState *bs, 
uint64_t offset,
 flags = (ret & BDRV_BLOCK_ALLOCATED ? 0 : NBD_STATE_HOLE) |
 (ret & BDRV_BLOCK_ZERO  ? NBD_STATE_ZERO : 0);
 
-if (first_extent) {
-extent->flags = flags;
-extent->length = num;
-first_extent = false;
-} else if (flags == extent->flags) {
-/* extend current extent */
-extent->length += num;
-} else {
-if (extent + 1 == extents_end) {
-break;
-}
-
-/* start new extent */

[PULL 10/10] block/qcow2-bitmap: use bdrv_dirty_bitmap_next_dirty

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

store_bitmap_data() loop does bdrv_set_dirty_iter() on each iteration,
which means that we actually don't need iterator itself and we can use
simpler bitmap API.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-11-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 block/qcow2-bitmap.c | 15 +--
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 82c9f3..cb06954b4a 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1288,7 +1288,6 @@ static uint64_t *store_bitmap_data(BlockDriverState *bs,
 uint64_t bm_size = bdrv_dirty_bitmap_size(bitmap);
 const char *bm_name = bdrv_dirty_bitmap_name(bitmap);
 uint8_t *buf = NULL;
-BdrvDirtyBitmapIter *dbi;
 uint64_t *tb;
 uint64_t tb_size =
 size_to_clusters(s,
@@ -1307,12 +1306,14 @@ static uint64_t *store_bitmap_data(BlockDriverState *bs,
 return NULL;
 }
 
-dbi = bdrv_dirty_iter_new(bitmap);
 buf = g_malloc(s->cluster_size);
 limit = bytes_covered_by_bitmap_cluster(s, bitmap);
 assert(DIV_ROUND_UP(bm_size, limit) == tb_size);
 
-while ((offset = bdrv_dirty_iter_next(dbi)) >= 0) {
+offset = 0;
+while ((offset = bdrv_dirty_bitmap_next_dirty(bitmap, offset, INT64_MAX))
+   >= 0)
+{
 uint64_t cluster = offset / limit;
 uint64_t end, write_size;
 int64_t off;
@@ -1355,23 +1356,17 @@ static uint64_t *store_bitmap_data(BlockDriverState *bs,
 goto fail;
 }
 
-if (end >= bm_size) {
-break;
-}
-
-bdrv_set_dirty_iter(dbi, end);
+offset = end;
 }
 
 *bitmap_table_size = tb_size;
 g_free(buf);
-bdrv_dirty_iter_free(dbi);
 
 return tb;
 
 fail:
 clear_bitmap_table(bs, tb, tb_size);
 g_free(buf);
-bdrv_dirty_iter_free(dbi);
 g_free(tb);
 
 return NULL;
-- 
2.21.1

[PULL 02/10] hbitmap: move hbitmap_iter_next_word to hbitmap.c

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

The function is definitely internal (it's not used by third party and
it has complicated interface). Move it to .c file.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Max Reitz 
Reviewed-by: John Snow 
Message-id: 20200205112041.6003-3-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 include/qemu/hbitmap.h | 30 --
 util/hbitmap.c | 29 +
 2 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 1bf944ca3d..ab227b117f 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -362,34 +362,4 @@ void hbitmap_free_meta(HBitmap *hb);
  */
 int64_t hbitmap_iter_next(HBitmapIter *hbi);
 
-/**
- * hbitmap_iter_next_word:
- * @hbi: HBitmapIter to operate on.
- * @p_cur: Location where to store the next non-zero word.
- *
- * Return the index of the next nonzero word that is set in @hbi's
- * associated HBitmap, and set *p_cur to the content of that word
- * (bits before the index that was passed to hbitmap_iter_init are
- * trimmed on the first call).  Return -1, and set *p_cur to zero,
- * if all remaining words are zero.
- */
-static inline size_t hbitmap_iter_next_word(HBitmapIter *hbi, unsigned long 
*p_cur)
-{
-unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
-
-if (cur == 0) {
-cur = hbitmap_iter_skip_words(hbi);
-if (cur == 0) {
-*p_cur = 0;
-return -1;
-}
-}
-
-/* The next call will resume work from the next word.  */
-hbi->cur[HBITMAP_LEVELS - 1] = 0;
-*p_cur = cur;
-return hbi->pos;
-}
-
-
 #endif
diff --git a/util/hbitmap.c b/util/hbitmap.c
index 7f9b3e0cd7..a368dc5ef7 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -298,6 +298,35 @@ uint64_t hbitmap_count(const HBitmap *hb)
 return hb->count << hb->granularity;
 }
 
+/**
+ * hbitmap_iter_next_word:
+ * @hbi: HBitmapIter to operate on.
+ * @p_cur: Location where to store the next non-zero word.
+ *
+ * Return the index of the next nonzero word that is set in @hbi's
+ * associated HBitmap, and set *p_cur to the content of that word
+ * (bits before the index that was passed to hbitmap_iter_init are
+ * trimmed on the first call).  Return -1, and set *p_cur to zero,
+ * if all remaining words are zero.
+ */
+static size_t hbitmap_iter_next_word(HBitmapIter *hbi, unsigned long *p_cur)
+{
+unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
+
+if (cur == 0) {
+cur = hbitmap_iter_skip_words(hbi);
+if (cur == 0) {
+*p_cur = 0;
+return -1;
+}
+}
+
+/* The next call will resume work from the next word.  */
+hbi->cur[HBITMAP_LEVELS - 1] = 0;
+*p_cur = cur;
+return hbi->pos;
+}
+
 /* Count the number of set bits between start and end, not accounting for
  * the granularity.  Also an example of how to use hbitmap_iter_next_word.
  */
-- 
2.21.1

[PULL 09/10] nbd/server: use bdrv_dirty_bitmap_next_dirty_area

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Use bdrv_dirty_bitmap_next_dirty_area for bitmap_to_extents. Since
bdrv_dirty_bitmap_next_dirty_area is very accurate in its interface,
we'll never exceed requested region with last chunk. So, we don't need
dont_fragment, and bitmap_to_extents() interface becomes clean enough
to not require any comment.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Reviewed-by: Eric Blake 
Message-id: 20200205112041.6003-10-vsement...@virtuozzo.com
Signed-off-by: John Snow 
---
 nbd/server.c | 59 +---
 1 file changed, 19 insertions(+), 40 deletions(-)

diff --git a/nbd/server.c b/nbd/server.c
index f90bb33a75..02b1ed0801 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -2068,57 +2068,36 @@ static int nbd_co_send_block_status(NBDClient *client, 
uint64_t handle,
 return nbd_co_send_extents(client, handle, ea, last, context_id, errp);
 }
 
-/*
- * Populate @ea from a dirty bitmap. Unless @dont_fragment, the
- * final extent may exceed the original @length.
- */
+/* Populate @ea from a dirty bitmap. */
 static void bitmap_to_extents(BdrvDirtyBitmap *bitmap,
   uint64_t offset, uint64_t length,
-  NBDExtentArray *ea, bool dont_fragment)
+  NBDExtentArray *es)
 {
-uint64_t begin = offset, end = offset;
-uint64_t overall_end = offset + length;
-BdrvDirtyBitmapIter *it;
-bool dirty;
+int64_t start, dirty_start, dirty_count;
+int64_t end = offset + length;
+bool full = false;
 
 bdrv_dirty_bitmap_lock(bitmap);
 
-it = bdrv_dirty_iter_new(bitmap);
-dirty = bdrv_dirty_bitmap_get_locked(bitmap, offset);
-
-while (begin < overall_end) {
-bool next_dirty = !dirty;
-
-if (dirty) {
-end = bdrv_dirty_bitmap_next_zero(bitmap, begin, INT64_MAX);
-} else {
-bdrv_set_dirty_iter(it, begin);
-end = bdrv_dirty_iter_next(it);
-}
-if (end == -1 || end - begin > UINT32_MAX) {
-/* Cap to an aligned value < 4G beyond begin. */
-end = MIN(bdrv_dirty_bitmap_size(bitmap),
-  begin + UINT32_MAX + 1 -
-  bdrv_dirty_bitmap_granularity(bitmap));
-next_dirty = dirty;
-}
-if (dont_fragment && end > overall_end) {
-end = overall_end;
-}
-
-if (nbd_extent_array_add(ea, end - begin,
- dirty ? NBD_STATE_DIRTY : 0) < 0) {
+for (start = offset;
+ bdrv_dirty_bitmap_next_dirty_area(bitmap, start, end, INT32_MAX,
+   _start, _count);
+ start = dirty_start + dirty_count)
+{
+if ((nbd_extent_array_add(es, dirty_start - start, 0) < 0) ||
+(nbd_extent_array_add(es, dirty_count, NBD_STATE_DIRTY) < 0))
+{
+full = true;
 break;
 }
-begin = end;
-dirty = next_dirty;
 }
 
-bdrv_dirty_iter_free(it);
+if (!full) {
+/* last non dirty extent */
+nbd_extent_array_add(es, end - start, 0);
+}
 
 bdrv_dirty_bitmap_unlock(bitmap);
-
-assert(offset < end);
 }
 
 static int nbd_co_send_bitmap(NBDClient *client, uint64_t handle,
@@ -2129,7 +2108,7 @@ static int nbd_co_send_bitmap(NBDClient *client, uint64_t 
handle,
 unsigned int nb_extents = dont_fragment ? 1 : NBD_MAX_BLOCK_STATUS_EXTENTS;
 g_autoptr(NBDExtentArray) ea = nbd_extent_array_new(nb_extents);
 
-bitmap_to_extents(bitmap, offset, length, ea, dont_fragment);
+bitmap_to_extents(bitmap, offset, length, ea);
 
 return nbd_co_send_extents(client, handle, ea, last, context_id, errp);
 }
-- 
2.21.1