Re: [Qemu-block] [Qemu-devel] [PATCH v2 00/11] blockjobs: Fix transactional race condition

[no-reply]

Hi,

Your series seems to have some coding style problems. See output below for
more information:

Type: series
Message-id: 1475272849-19990-1-git-send-email-js...@redhat.com
Subject: [Qemu-devel] [PATCH v2 00/11] blockjobs: Fix transactional race 
condition

=== TEST SCRIPT BEGIN ===
#!/bin/bash

BASE=base
n=1
total=$(git log --oneline $BASE.. | wc -l)
failed=0

# Useful git options
git config --local diff.renamelimit 0
git config --local diff.renames True

commits="$(git log --format=%H --reverse $BASE..)"
for c in $commits; do
echo "Checking PATCH $n/$total: $(git show --no-patch --format=%s $c)..."
if ! git show $c --format=email | ./scripts/checkpatch.pl --mailback -; then
failed=1
echo
fi
n=$((n+1))
done

exit $failed
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
From https://github.com/patchew-project/qemu
 * [new tag] patchew/1475272849-19990-1-git-send-email-js...@redhat.com 
-> patchew/1475272849-19990-1-git-send-email-js...@redhat.com
Switched to a new branch 'test'
d3bf4cf iotests: add transactional failure race test
ea3adad blockjob: refactor backup_start as backup_job_create
6b5e7c6 blockjob: add block_job_start
fc02e4d blockjob: add .start field
f34ba3c blockjob: add .clean property
6dc649d blockjobs: fix documentation
3e472b6 blockjobs: split interface into public/private
6cbfc4c blockjobs: Always use block_job_get_aio_context
b2ec7f6 Blockjobs: Internalize user_pause logic
e76a966 blockjob: centralize QMP event emissions
cb59e5e blockjob: fix dead pointer in txn list

=== OUTPUT BEGIN ===
Checking PATCH 1/11: blockjob: fix dead pointer in txn list...
Checking PATCH 2/11: blockjob: centralize QMP event emissions...
Checking PATCH 3/11: Blockjobs: Internalize user_pause logic...
Checking PATCH 4/11: blockjobs: Always use block_job_get_aio_context...
Checking PATCH 5/11: blockjobs: split interface into public/private...
ERROR: struct BlockJobDriver should normally be const
#301: FILE: include/block/blockjob.h:31:
+typedef struct BlockJobDriver BlockJobDriver;

ERROR: struct BlockJobDriver should normally be const
#535: FILE: include/block/blockjob_int.h:37:
+struct BlockJobDriver {

ERROR: space prohibited between function name and open parenthesis '('
#579: FILE: include/block/blockjob_int.h:81:
+void coroutine_fn (*pause)(BlockJob *job);

ERROR: space prohibited between function name and open parenthesis '('
#586: FILE: include/block/blockjob_int.h:88:
+void coroutine_fn (*resume)(BlockJob *job);

total: 4 errors, 0 warnings, 805 lines checked

Your patch has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.

Checking PATCH 6/11: blockjobs: fix documentation...
Checking PATCH 7/11: blockjob: add .clean property...
Checking PATCH 8/11: blockjob: add .start field...
Checking PATCH 9/11: blockjob: add block_job_start...
Checking PATCH 10/11: blockjob: refactor backup_start as backup_job_create...
Checking PATCH 11/11: iotests: add transactional failure race test...
=== OUTPUT END ===

Test command exited with code: 1


---
Email generated automatically by Patchew [http://patchew.org/].
Please send your feedback to patchew-de...@freelists.org

[Qemu-block] [PATCH v2 11/11] iotests: add transactional failure race test

[John Snow]

Add a regression test for the case found by Vladimir.

Reported-by: Vladimir Sementsov-Ogievskiy 
Signed-off-by: John Snow 
---
 tests/qemu-iotests/124 | 91 ++
 tests/qemu-iotests/124.out |  4 +-
 2 files changed, 93 insertions(+), 2 deletions(-)

diff --git a/tests/qemu-iotests/124 b/tests/qemu-iotests/124
index 2f0bc24..b8f7dad 100644
--- a/tests/qemu-iotests/124
+++ b/tests/qemu-iotests/124
@@ -512,6 +512,97 @@ class TestIncrementalBackup(TestIncrementalBackupBase):
 self.check_backups()
 
 
+def test_transaction_failure_race(self):
+'''Test: Verify that transactions with jobs that have no data to
+transfer do not cause race conditions in the cancellation of the entire
+transaction job group.
+'''
+
+# Create a second drive, with pattern:
+drive1 = self.add_node('drive1')
+self.img_create(drive1['file'], drive1['fmt'])
+io_write_patterns(drive1['file'], (('0x14', 0, 512),
+   ('0x5d', '1M', '32k'),
+   ('0xcd', '32M', '124k')))
+
+# Create a blkdebug interface to this img as 'drive1'
+result = self.vm.qmp('blockdev-add', options={
+'node-name': drive1['id'],
+'driver': drive1['fmt'],
+'file': {
+'driver': 'blkdebug',
+'image': {
+'driver': 'file',
+'filename': drive1['file']
+},
+'set-state': [{
+'event': 'flush_to_disk',
+'state': 1,
+'new_state': 2
+}],
+'inject-error': [{
+'event': 'read_aio',
+'errno': 5,
+'state': 2,
+'immediately': False,
+'once': True
+}],
+}
+})
+self.assert_qmp(result, 'return', {})
+
+# Create bitmaps and full backups for both drives
+drive0 = self.drives[0]
+dr0bm0 = self.add_bitmap('bitmap0', drive0)
+dr1bm0 = self.add_bitmap('bitmap0', drive1)
+self.create_anchor_backup(drive0)
+self.create_anchor_backup(drive1)
+self.assert_no_active_block_jobs()
+self.assertFalse(self.vm.get_qmp_events(wait=False))
+
+# Emulate some writes
+self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
+  ('0xef', '16M', '256k'),
+  ('0x46', '32736k', '64k')))
+
+# Create incremental backup targets
+target0 = self.prepare_backup(dr0bm0)
+target1 = self.prepare_backup(dr1bm0)
+
+# Ask for a new incremental backup per-each drive, expecting drive1's
+# backup to fail and attempt to cancel the empty drive0 job.
+transaction = [
+transaction_drive_backup(drive0['id'], target0, sync='incremental',
+ format=drive0['fmt'], mode='existing',
+ bitmap=dr0bm0.name),
+transaction_drive_backup(drive1['id'], target1, sync='incremental',
+ format=drive1['fmt'], mode='existing',
+ bitmap=dr1bm0.name)
+]
+result = self.vm.qmp('transaction', actions=transaction,
+ properties={'completion-mode': 'grouped'} )
+self.assert_qmp(result, 'return', {})
+
+# Observe that drive0's backup is cancelled and drive1 completes with
+# an error.
+self.wait_qmp_backup_cancelled(drive0['id'])
+self.assertFalse(self.wait_qmp_backup(drive1['id']))
+error = self.vm.event_wait('BLOCK_JOB_ERROR')
+self.assert_qmp(error, 'data', {'device': drive1['id'],
+'action': 'report',
+'operation': 'read'})
+self.assertFalse(self.vm.get_qmp_events(wait=False))
+self.assert_no_active_block_jobs()
+
+# Delete drive0's successful target and eliminate our record of the
+# unsuccessful drive1 target.
+dr0bm0.del_target()
+dr1bm0.del_target()
+
+self.vm.shutdown()
+
+
+
 def test_sync_dirty_bitmap_missing(self):
 self.assert_no_active_block_jobs()
 self.files.append(self.err_img)
diff --git a/tests/qemu-iotests/124.out b/tests/qemu-iotests/124.out
index 36376be..e56cae0 100644
--- a/tests/qemu-iotests/124.out
+++ b/tests/qemu-iotests/124.out
@@ -1,5 +1,5 @@
-..
+...
 --
-Ran 10 tests
+Ran 11 tests
 
 OK
-- 
2.7.4

[Qemu-block] [PATCH v2 08/11] blockjob: add .start field

[John Snow]

Add an explicit start field to specify the entrypoint. We already have
ownership of the coroutine itself AND managing the lifetime of the
coroutine, let's take control of creation of the coroutine, too.

This will allow us to delay creation of the actual coroutine until we
know we'll actually start a BlockJob in block_job_start. This avoids
the sticky question of how to "un-create" a Coroutine that hasn't been
started yet.

Signed-off-by: John Snow 
---
 block/backup.c   | 23 ---
 block/commit.c   |  3 ++-
 block/mirror.c   |  4 +++-
 block/stream.c   |  3 ++-
 include/block/blockjob_int.h |  3 +++
 5 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/block/backup.c b/block/backup.c
index 42ff4c0..58dfc58 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -307,16 +307,6 @@ void backup_cow_request_end(CowRequest *req)
 cow_request_end(req);
 }
 
-static const BlockJobDriver backup_job_driver = {
-.instance_size  = sizeof(BackupBlockJob),
-.job_type   = BLOCK_JOB_TYPE_BACKUP,
-.set_speed  = backup_set_speed,
-.commit = backup_commit,
-.abort  = backup_abort,
-.clean  = backup_clean,
-.attached_aio_context   = backup_attached_aio_context,
-};
-
 static BlockErrorAction backup_error_action(BackupBlockJob *job,
 bool read, int error)
 {
@@ -526,6 +516,17 @@ static void coroutine_fn backup_run(void *opaque)
 block_job_defer_to_main_loop(>common, backup_complete, data);
 }
 
+static const BlockJobDriver backup_job_driver = {
+.instance_size  = sizeof(BackupBlockJob),
+.job_type   = BLOCK_JOB_TYPE_BACKUP,
+.start  = backup_run,
+.set_speed  = backup_set_speed,
+.commit = backup_commit,
+.abort  = backup_abort,
+.clean  = backup_clean,
+.attached_aio_context   = backup_attached_aio_context,
+};
+
 void backup_start(const char *job_id, BlockDriverState *bs,
   BlockDriverState *target, int64_t speed,
   MirrorSyncMode sync_mode, BdrvDirtyBitmap *sync_bitmap,
@@ -636,7 +637,7 @@ void backup_start(const char *job_id, BlockDriverState *bs,
 
 bdrv_op_block_all(target, job->common.blocker);
 job->common.len = len;
-job->common.co = qemu_coroutine_create(backup_run, job);
+job->common.co = qemu_coroutine_create(job->common.driver->start, job);
 block_job_txn_add_job(txn, >common);
 qemu_coroutine_enter(job->common.co);
 return;
diff --git a/block/commit.c b/block/commit.c
index fb5bede..dbaf39e 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -205,6 +205,7 @@ static const BlockJobDriver commit_job_driver = {
 .instance_size = sizeof(CommitBlockJob),
 .job_type  = BLOCK_JOB_TYPE_COMMIT,
 .set_speed = commit_set_speed,
+.start = commit_run,
 };
 
 void commit_start(const char *job_id, BlockDriverState *bs,
@@ -274,7 +275,7 @@ void commit_start(const char *job_id, BlockDriverState *bs,
 s->backing_file_str = g_strdup(backing_file_str);
 
 s->on_error = on_error;
-s->common.co = qemu_coroutine_create(commit_run, s);
+s->common.co = qemu_coroutine_create(s->common.driver->start, s);
 
 trace_commit_start(bs, base, top, s, s->common.co, opaque);
 qemu_coroutine_enter(s->common.co);
diff --git a/block/mirror.c b/block/mirror.c
index cc62fb0..ef54e5b 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -891,6 +891,7 @@ static const BlockJobDriver mirror_job_driver = {
 .instance_size  = sizeof(MirrorBlockJob),
 .job_type   = BLOCK_JOB_TYPE_MIRROR,
 .set_speed  = mirror_set_speed,
+.start  = mirror_run,
 .complete   = mirror_complete,
 .pause  = mirror_pause,
 .attached_aio_context   = mirror_attached_aio_context,
@@ -900,6 +901,7 @@ static const BlockJobDriver commit_active_job_driver = {
 .instance_size  = sizeof(MirrorBlockJob),
 .job_type   = BLOCK_JOB_TYPE_COMMIT,
 .set_speed  = mirror_set_speed,
+.start  = mirror_run,
 .complete   = mirror_complete,
 .pause  = mirror_pause,
 .attached_aio_context   = mirror_attached_aio_context,
@@ -967,7 +969,7 @@ static void mirror_start_job(const char *job_id, 
BlockDriverState *bs,
 
 bdrv_op_block_all(target, s->common.blocker);
 
-s->common.co = qemu_coroutine_create(mirror_run, s);
+s->common.co = qemu_coroutine_create(s->common.driver->start, s);
 trace_mirror_start(bs, s, s->common.co, opaque);
 qemu_coroutine_enter(s->common.co);
 }
diff --git a/block/stream.c b/block/stream.c
index 71d0e7a..2a1c814 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -212,6 

[Qemu-block] [PATCH v2 09/11] blockjob: add block_job_start

[John Snow]

Instead of automatically starting jobs at creation time via backup_start
et al, we'd like to return a job object pointer that can be started
manually at later point in time.

For now, add the block_job_start mechanism and start the jobs
automatically as we have been doing, with conversions job-by-job coming
in later patches.

Of note: cancellation of unstarted jobs will perform all the normal
cleanup as if the job had started, particularly abort and clean. The
only difference is that we will not emit any events, because the job
never actually started.

Signed-off-by: John Snow 
---
 block/backup.c|  3 +--
 block/commit.c|  3 +--
 block/mirror.c|  3 +--
 block/stream.c|  3 +--
 blockjob.c| 48 +++
 include/block/block_int.h |  8 
 tests/test-blockjob-txn.c | 12 ++--
 7 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/block/backup.c b/block/backup.c
index 58dfc58..7294169 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -637,9 +637,8 @@ void backup_start(const char *job_id, BlockDriverState *bs,
 
 bdrv_op_block_all(target, job->common.blocker);
 job->common.len = len;
-job->common.co = qemu_coroutine_create(job->common.driver->start, job);
 block_job_txn_add_job(txn, >common);
-qemu_coroutine_enter(job->common.co);
+block_job_start(>common);
 return;
 
  error:
diff --git a/block/commit.c b/block/commit.c
index dbaf39e..e64fdf3 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -275,10 +275,9 @@ void commit_start(const char *job_id, BlockDriverState *bs,
 s->backing_file_str = g_strdup(backing_file_str);
 
 s->on_error = on_error;
-s->common.co = qemu_coroutine_create(s->common.driver->start, s);
 
 trace_commit_start(bs, base, top, s, s->common.co, opaque);
-qemu_coroutine_enter(s->common.co);
+block_job_start(>common);
 }
 
 
diff --git a/block/mirror.c b/block/mirror.c
index ef54e5b..2a6a662 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -969,9 +969,8 @@ static void mirror_start_job(const char *job_id, 
BlockDriverState *bs,
 
 bdrv_op_block_all(target, s->common.blocker);
 
-s->common.co = qemu_coroutine_create(s->common.driver->start, s);
 trace_mirror_start(bs, s, s->common.co, opaque);
-qemu_coroutine_enter(s->common.co);
+block_job_start(>common);
 }
 
 void mirror_start(const char *job_id, BlockDriverState *bs,
diff --git a/block/stream.c b/block/stream.c
index 2a1c814..55cbbe7 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -232,7 +232,6 @@ void stream_start(const char *job_id, BlockDriverState *bs,
 s->backing_file_str = g_strdup(backing_file_str);
 
 s->on_error = on_error;
-s->common.co = qemu_coroutine_create(s->common.driver->start, s);
 trace_stream_start(bs, base, s, s->common.co, opaque);
-qemu_coroutine_enter(s->common.co);
+block_job_start(>common);
 }
diff --git a/blockjob.c b/blockjob.c
index 44cbf6c..ac9a68d 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -161,7 +161,8 @@ void *block_job_create(const char *job_id, const 
BlockJobDriver *driver,
 job->blk   = blk;
 job->cb= cb;
 job->opaque= opaque;
-job->busy  = true;
+job->busy  = false;
+job->paused= true;
 job->refcnt= 1;
 bs->job = job;
 
@@ -184,6 +185,15 @@ void *block_job_create(const char *job_id, const 
BlockJobDriver *driver,
 return job;
 }
 
+void block_job_start(BlockJob *job)
+{
+assert(job && !job->co && job->paused && !job->busy && job->driver->start);
+job->paused = false;
+job->busy = true;
+job->co = qemu_coroutine_create(job->driver->start, job);
+qemu_coroutine_enter(job->co);
+}
+
 void block_job_ref(BlockJob *job)
 {
 ++job->refcnt;
@@ -220,18 +230,24 @@ static void block_job_completed_single(BlockJob *job)
 if (job->driver->clean) {
 job->driver->clean(job);
 }
+
 if (job->cb) {
 job->cb(job->opaque, job->ret);
 }
-if (block_job_is_cancelled(job)) {
-block_job_event_cancelled(job);
-} else {
-const char *msg = NULL;
-if (job->ret < 0) {
-msg = strerror(-job->ret);
+
+/* Emit events only if we actually started */
+if (job->co) {
+if (block_job_is_cancelled(job)) {
+block_job_event_cancelled(job);
+} else {
+const char *msg = NULL;
+if (job->ret < 0) {
+msg = strerror(-job->ret);
+}
+block_job_event_completed(job, msg);
 }
-block_job_event_completed(job, msg);
 }
+
 if (job->txn) {
 QLIST_REMOVE(job, txn_list);
 block_job_txn_unref(job->txn);
@@ -335,7 +351,8 @@ void block_job_set_speed(BlockJob *job, int64_t speed, 
Error **errp)
 
 void block_job_complete(BlockJob *job, Error **errp)
 {
-if (job->pause_count || 

[Qemu-block] [PATCH v2 05/11] blockjobs: split interface into public/private

[John Snow]

To make it a little more obvious which functions are intended to be
public interface and which are intended to be for use only by jobs
themselves, split the interface into "public" and "private" files.

Convert blockjobs (e.g. block/backup) to using the private interface.
Leave blockdev and others on the public interface.

Give up and let qemu-img use the internal interface, though it doesn't
strictly need to be using it.

As a side-effect, hide the BlockJob and BlockJobDriver implementation
from most of the QEMU codebase.

Signed-off-by: John Snow 
---
 block/backup.c   |   2 +-
 block/commit.c   |   2 +-
 block/mirror.c   |   2 +-
 block/replication.c  |   2 +-
 block/stream.c   |   2 +-
 blockjob.c   |   2 +-
 include/block/block.h|   3 +-
 include/block/blockjob.h | 325 +--
 include/block/blockjob_int.h | 355 +++
 qemu-img.c   |   2 +-
 tests/test-blockjob-txn.c|   2 +-
 tests/test-blockjob.c|   2 +-
 12 files changed, 368 insertions(+), 333 deletions(-)
 create mode 100644 include/block/blockjob_int.h

diff --git a/block/backup.c b/block/backup.c
index 582bd0f..d667482 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -16,7 +16,7 @@
 #include "trace.h"
 #include "block/block.h"
 #include "block/block_int.h"
-#include "block/blockjob.h"
+#include "block/blockjob_int.h"
 #include "block/block_backup.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
diff --git a/block/commit.c b/block/commit.c
index 9f67a8b..fb5bede 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -15,7 +15,7 @@
 #include "qemu/osdep.h"
 #include "trace.h"
 #include "block/block_int.h"
-#include "block/blockjob.h"
+#include "block/blockjob_int.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
diff --git a/block/mirror.c b/block/mirror.c
index f9d1fec..cc62fb0 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -13,7 +13,7 @@
 
 #include "qemu/osdep.h"
 #include "trace.h"
-#include "block/blockjob.h"
+#include "block/blockjob_int.h"
 #include "block/block_int.h"
 #include "sysemu/block-backend.h"
 #include "qapi/error.h"
diff --git a/block/replication.c b/block/replication.c
index 3bd1cf1..b604b93 100644
--- a/block/replication.c
+++ b/block/replication.c
@@ -15,7 +15,7 @@
 #include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/nbd.h"
-#include "block/blockjob.h"
+#include "block/blockjob_int.h"
 #include "block/block_int.h"
 #include "block/block_backup.h"
 #include "sysemu/block-backend.h"
diff --git a/block/stream.c b/block/stream.c
index 3187481..71d0e7a 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -14,7 +14,7 @@
 #include "qemu/osdep.h"
 #include "trace.h"
 #include "block/block_int.h"
-#include "block/blockjob.h"
+#include "block/blockjob_int.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
diff --git a/blockjob.c b/blockjob.c
index 073d9ce..09fb602 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -27,7 +27,7 @@
 #include "qemu-common.h"
 #include "trace.h"
 #include "block/block.h"
-#include "block/blockjob.h"
+#include "block/blockjob_int.h"
 #include "block/block_int.h"
 #include "sysemu/block-backend.h"
 #include "qapi/qmp/qerror.h"
diff --git a/include/block/block.h b/include/block/block.h
index e18233a..f556345 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -7,16 +7,15 @@
 #include "qemu/coroutine.h"
 #include "block/accounting.h"
 #include "block/dirty-bitmap.h"
+#include "block/blockjob.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi-types.h"
 #include "qemu/hbitmap.h"
 
 /* block.c */
 typedef struct BlockDriver BlockDriver;
-typedef struct BlockJob BlockJob;
 typedef struct BdrvChild BdrvChild;
 typedef struct BdrvChildRole BdrvChildRole;
-typedef struct BlockJobTxn BlockJobTxn;
 
 typedef struct BlockDriverInfo {
 /* in bytes, 0 if irrelevant */
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index 6f28c73..efae26b 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -28,177 +28,9 @@
 
 #include "block/block.h"
 
-/**
- * BlockJobDriver:
- *
- * A class type for block job driver.
- */
-typedef struct BlockJobDriver {
-/** Derived BlockJob struct size */
-size_t instance_size;
-
-/** String describing the operation, part of query-block-jobs QMP API */
-BlockJobType job_type;
-
-/** Optional callback for job types that support setting a speed limit */
-void (*set_speed)(BlockJob *job, int64_t speed, Error **errp);
-
-/** Optional callback for job types that need to forward I/O status reset 
*/
-void (*iostatus_reset)(BlockJob *job);
-
-/**
- * Optional callback for job types whose completion must be triggered
- * manually.
- */
-void (*complete)(BlockJob *job, Error **errp);
-
-/**
- * If the callback 

[Qemu-block] [PATCH v2 06/11] blockjobs: fix documentation

[John Snow]

Wrong function names in documentation.

Signed-off-by: John Snow 
---
 include/block/blockjob_int.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/block/blockjob_int.h b/include/block/blockjob_int.h
index 0a2d41e..c6da7e4 100644
--- a/include/block/blockjob_int.h
+++ b/include/block/blockjob_int.h
@@ -305,7 +305,7 @@ void block_job_enter(BlockJob *job);
 void block_job_event_cancelled(BlockJob *job);
 
 /**
- * block_job_ready:
+ * block_job_event_completed:
  * @job: The job which is now ready to complete.
  * @msg: Error message. Only present on failure.
  *
@@ -314,8 +314,8 @@ void block_job_event_cancelled(BlockJob *job);
 void block_job_event_completed(BlockJob *job, const char *msg);
 
 /**
- * block_job_ready:
- * @job: The job which is now ready to complete.
+ * block_job_event_ready:
+ * @job: The job which is now ready to be completed.
  *
  * Send a BLOCK_JOB_READY event for the specified job.
  */
-- 
2.7.4

[Qemu-block] [PATCH v2 07/11] blockjob: add .clean property

[John Snow]

Cleaning up after we have deferred to the main thread but before the
transaction has converged can be dangerous and result in deadlocks
if the job cleanup invokes any BH polling loops.

A job may attempt to begin cleaning up, but may induce another job to
enter its cleanup routine. The second job, part of our same transaction,
will block waiting for the first job to finish, so neither job may now
make progress.

To rectify this, allow jobs to register a cleanup operation that will
always run regardless of if the job was in a transaction or not, and
if the transaction job group completed successfully or not.

Move sensitive cleanup to this callback instead which is guaranteed to
be run only after the transaction has converged, which removes sensitive
timing constraints from said cleanup.

Furthermore, in future patches these cleanup operations will be performed
regardless of whether or not we actually started the job. Therefore,
cleanup callbacks should essentially confine themselves to undoing create
operations, e.g. setup actions taken in what is now backup_run.

Reported-by: Vladimir Sementsov-Ogievskiy 
Signed-off-by: John Snow 
---
 block/backup.c   | 11 ---
 blockjob.c   |  5 +++--
 include/block/blockjob_int.h |  8 
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/block/backup.c b/block/backup.c
index d667482..42ff4c0 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -242,6 +242,13 @@ static void backup_abort(BlockJob *job)
 }
 }
 
+static void backup_clean(BlockJob *job)
+{
+BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+assert(s->target);
+blk_unref(s->target);
+}
+
 static void backup_attached_aio_context(BlockJob *job, AioContext *aio_context)
 {
 BackupBlockJob *s = container_of(job, BackupBlockJob, common);
@@ -306,6 +313,7 @@ static const BlockJobDriver backup_job_driver = {
 .set_speed  = backup_set_speed,
 .commit = backup_commit,
 .abort  = backup_abort,
+.clean  = backup_clean,
 .attached_aio_context   = backup_attached_aio_context,
 };
 
@@ -327,11 +335,8 @@ typedef struct {
 
 static void backup_complete(BlockJob *job, void *opaque)
 {
-BackupBlockJob *s = container_of(job, BackupBlockJob, common);
 BackupCompleteData *data = opaque;
 
-blk_unref(s->target);
-
 block_job_completed(job, data->ret);
 g_free(data);
 }
diff --git a/blockjob.c b/blockjob.c
index 09fb602..44cbf6c 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -217,7 +217,9 @@ static void block_job_completed_single(BlockJob *job)
 job->driver->abort(job);
 }
 }
-
+if (job->driver->clean) {
+job->driver->clean(job);
+}
 if (job->cb) {
 job->cb(job->opaque, job->ret);
 }
@@ -230,7 +232,6 @@ static void block_job_completed_single(BlockJob *job)
 }
 block_job_event_completed(job, msg);
 }
-
 if (job->txn) {
 QLIST_REMOVE(job, txn_list);
 block_job_txn_unref(job->txn);
diff --git a/include/block/blockjob_int.h b/include/block/blockjob_int.h
index c6da7e4..b7aeaef 100644
--- a/include/block/blockjob_int.h
+++ b/include/block/blockjob_int.h
@@ -74,6 +74,14 @@ struct BlockJobDriver {
 void (*abort)(BlockJob *job);
 
 /**
+ * If the callback is not NULL, it will be invoked after a call to either
+ * .commit() or .abort(). Regardless of which callback is invoked after
+ * completion, .clean() will always be called, even if the job does not
+ * belong to a transaction group.
+ */
+void (*clean)(BlockJob *job);
+
+/**
  * If the callback is not NULL, it will be invoked when the job transitions
  * into the paused state.  Paused jobs must not perform any asynchronous
  * I/O or event loop activity.  This callback is used to quiesce jobs.
-- 
2.7.4

[Qemu-block] [PATCH v2 04/11] blockjobs: Always use block_job_get_aio_context

[John Snow]

There are a few places where we're fishing it out for ourselves.
Let's not do that and instead use the helper.

Signed-off-by: John Snow 
---
 block/io.c   | 4 ++--
 blockdev.c   | 4 ++--
 blockjob.c   | 2 +-
 include/block/blockjob.h | 9 +
 qemu-img.c   | 2 +-
 5 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/block/io.c b/block/io.c
index 868b065..f26a503 100644
--- a/block/io.c
+++ b/block/io.c
@@ -288,7 +288,7 @@ void bdrv_drain_all(void)
 GSList *aio_ctxs = NULL, *ctx;
 
 while ((job = block_job_next(job))) {
-AioContext *aio_context = blk_get_aio_context(job->blk);
+AioContext *aio_context = block_job_get_aio_context(job);
 
 aio_context_acquire(aio_context);
 block_job_pause(job, false);
@@ -347,7 +347,7 @@ void bdrv_drain_all(void)
 
 job = NULL;
 while ((job = block_job_next(job))) {
-AioContext *aio_context = blk_get_aio_context(job->blk);
+AioContext *aio_context = block_job_get_aio_context(job);
 
 aio_context_acquire(aio_context);
 block_job_resume(job);
diff --git a/blockdev.c b/blockdev.c
index 268452f..0ac507f 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -3596,7 +3596,7 @@ static BlockJob *find_block_job(const char *id, 
AioContext **aio_context,
 return NULL;
 }
 
-*aio_context = blk_get_aio_context(job->blk);
+*aio_context = block_job_get_aio_context(job);
 aio_context_acquire(*aio_context);
 
 return job;
@@ -3956,7 +3956,7 @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp)
 
 for (job = block_job_next(NULL); job; job = block_job_next(job)) {
 BlockJobInfoList *elem = g_new0(BlockJobInfoList, 1);
-AioContext *aio_context = blk_get_aio_context(job->blk);
+AioContext *aio_context = block_job_get_aio_context(job);
 
 aio_context_acquire(aio_context);
 elem->value = block_job_query(job);
diff --git a/blockjob.c b/blockjob.c
index 2a35f50..073d9ce 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -78,7 +78,7 @@ BlockJob *block_job_get(const char *id)
  * block_job_defer_to_main_loop() where it runs in the QEMU main loop.  Code
  * that supports both cases uses this helper function.
  */
-static AioContext *block_job_get_aio_context(BlockJob *job)
+AioContext *block_job_get_aio_context(BlockJob *job)
 {
 return job->deferred_to_main_loop ?
qemu_get_aio_context() :
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index 081f6c2..6f28c73 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -513,4 +513,13 @@ void block_job_txn_unref(BlockJobTxn *txn);
  */
 void block_job_txn_add_job(BlockJobTxn *txn, BlockJob *job);
 
+/**
+ * block_job_get_aio_context:
+ * @job: Job to get the aio_context for
+ *
+ * Fetch the current context for the given BlockJob. May be the main loop if
+ * the job has already deferred to main for final cleanup.
+ */
+AioContext *block_job_get_aio_context(BlockJob *job);
+
 #endif
diff --git a/qemu-img.c b/qemu-img.c
index ceffefe..204fa9c 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -793,7 +793,7 @@ static void common_block_job_cb(void *opaque, int ret)
 
 static void run_block_job(BlockJob *job, Error **errp)
 {
-AioContext *aio_context = blk_get_aio_context(job->blk);
+AioContext *aio_context = block_job_get_aio_context(job);
 
 do {
 aio_poll(aio_context, true);
-- 
2.7.4

[Qemu-block] [PATCH v2 01/11] blockjob: fix dead pointer in txn list

[John Snow]

From: Vladimir Sementsov-Ogievskiy 

Though it is not intended to be reached through normal circumstances,
if we do not gracefully deconstruct the transaction QLIST, we may wind
up with stale pointers in the list.

The rest of this series attempts to address the underlying issues,
but this should fix list inconsistencies.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Tested-by: John Snow 
Reviewed-by: John Snow 
[Rewrote commit message. --js]
Signed-off-by: John Snow 
Reviewed-by: Eric Blake 

Signed-off-by: John Snow 
---
 blockjob.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/blockjob.c b/blockjob.c
index a167f96..13e7134 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -220,6 +220,7 @@ static void block_job_completed_single(BlockJob *job)
 }
 job->cb(job->opaque, job->ret);
 if (job->txn) {
+QLIST_REMOVE(job, txn_list);
 block_job_txn_unref(job->txn);
 }
 block_job_unref(job);
-- 
2.7.4

[Qemu-block] [PATCH v2 10/11] blockjob: refactor backup_start as backup_job_create

[John Snow]

Refactor backup_start as backup_job_create, which only creates the job,
but does not automatically start it. The old interface, 'backup_start',
is not kept in favor of limiting the number of nearly-identical iterfaces
that would have to be edited to keep up with QAPI changes in the future.

Callers that wish to synchronously start the backup_block_job can
instead just call block_job_start immediately after calling
backup_job_create.

Transactions are updated to use the new interface, calling block_job_start
only during the .commit phase, which helps prevent race conditions where
jobs may finish before we even finish building the transaction. This may
happen, for instance, during empty block backup jobs.

Reported-by: Vladimir Sementsov-Ogievskiy 
Signed-off-by: John Snow 
---
 block/backup.c| 26 ---
 block/replication.c   | 11 ---
 blockdev.c| 81 +++
 include/block/block_int.h | 21 ++--
 4 files changed, 86 insertions(+), 53 deletions(-)

diff --git a/block/backup.c b/block/backup.c
index 7294169..aad69eb 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -527,7 +527,7 @@ static const BlockJobDriver backup_job_driver = {
 .attached_aio_context   = backup_attached_aio_context,
 };
 
-void backup_start(const char *job_id, BlockDriverState *bs,
+BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
   BlockDriverState *target, int64_t speed,
   MirrorSyncMode sync_mode, BdrvDirtyBitmap *sync_bitmap,
   bool compress,
@@ -546,52 +546,52 @@ void backup_start(const char *job_id, BlockDriverState 
*bs,
 
 if (bs == target) {
 error_setg(errp, "Source and target cannot be the same");
-return;
+return NULL;
 }
 
 if (!bdrv_is_inserted(bs)) {
 error_setg(errp, "Device is not inserted: %s",
bdrv_get_device_name(bs));
-return;
+return NULL;
 }
 
 if (!bdrv_is_inserted(target)) {
 error_setg(errp, "Device is not inserted: %s",
bdrv_get_device_name(target));
-return;
+return NULL;
 }
 
 if (compress && target->drv->bdrv_co_pwritev_compressed == NULL) {
 error_setg(errp, "Compression is not supported for this drive %s",
bdrv_get_device_name(target));
-return;
+return NULL;
 }
 
 if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP_SOURCE, errp)) {
-return;
+return NULL;
 }
 
 if (bdrv_op_is_blocked(target, BLOCK_OP_TYPE_BACKUP_TARGET, errp)) {
-return;
+return NULL;
 }
 
 if (sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) {
 if (!sync_bitmap) {
 error_setg(errp, "must provide a valid bitmap name for "
  "\"incremental\" sync mode");
-return;
+return NULL;
 }
 
 /* Create a new bitmap, and freeze/disable this one. */
 if (bdrv_dirty_bitmap_create_successor(bs, sync_bitmap, errp) < 0) {
-return;
+return NULL;
 }
 } else if (sync_bitmap) {
 error_setg(errp,
"a sync_bitmap was provided to backup_run, "
"but received an incompatible sync_mode (%s)",
MirrorSyncMode_lookup[sync_mode]);
-return;
+return NULL;
 }
 
 len = bdrv_getlength(bs);
@@ -638,8 +638,8 @@ void backup_start(const char *job_id, BlockDriverState *bs,
 bdrv_op_block_all(target, job->common.blocker);
 job->common.len = len;
 block_job_txn_add_job(txn, >common);
-block_job_start(>common);
-return;
+
+return >common;
 
  error:
 if (sync_bitmap) {
@@ -649,4 +649,6 @@ void backup_start(const char *job_id, BlockDriverState *bs,
 blk_unref(job->target);
 block_job_unref(>common);
 }
+
+return NULL;
 }
diff --git a/block/replication.c b/block/replication.c
index b604b93..d9cdc36 100644
--- a/block/replication.c
+++ b/block/replication.c
@@ -409,6 +409,7 @@ static void replication_start(ReplicationState *rs, 
ReplicationMode mode,
 int64_t active_length, hidden_length, disk_length;
 AioContext *aio_context;
 Error *local_err = NULL;
+BlockJob *job;
 
 aio_context = bdrv_get_aio_context(bs);
 aio_context_acquire(aio_context);
@@ -496,16 +497,18 @@ static void replication_start(ReplicationState *rs, 
ReplicationMode mode,
 bdrv_op_block_all(top_bs, s->blocker);
 bdrv_op_unblock(top_bs, BLOCK_OP_TYPE_DATAPLANE, s->blocker);
 
-backup_start("replication-backup", s->secondary_disk->bs,
- s->hidden_disk->bs, 0, MIRROR_SYNC_MODE_NONE, NULL, false,
- BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
- backup_job_completed, s, NULL, _err);
+job = 

[Qemu-block] [PATCH v2 02/11] blockjob: centralize QMP event emissions

[John Snow]

There's no reason to leave this to blockdev; we can do it in blockjobs
directly and get rid of an extra callback for most users.

Signed-off-by: John Snow 
---
 blockdev.c | 37 ++---
 blockjob.c | 16 ++--
 2 files changed, 20 insertions(+), 33 deletions(-)

diff --git a/blockdev.c b/blockdev.c
index 29c6561..03200e7 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -2957,31 +2957,6 @@ out:
 aio_context_release(aio_context);
 }
 
-static void block_job_cb(void *opaque, int ret)
-{
-/* Note that this function may be executed from another AioContext besides
- * the QEMU main loop.  If you need to access anything that assumes the
- * QEMU global mutex, use a BH or introduce a mutex.
- */
-
-BlockDriverState *bs = opaque;
-const char *msg = NULL;
-
-trace_block_job_cb(bs, bs->job, ret);
-
-assert(bs->job);
-
-if (ret < 0) {
-msg = strerror(-ret);
-}
-
-if (block_job_is_cancelled(bs->job)) {
-block_job_event_cancelled(bs->job);
-} else {
-block_job_event_completed(bs->job, msg);
-}
-}
-
 void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
   bool has_base, const char *base,
   bool has_backing_file, const char *backing_file,
@@ -3033,7 +3008,7 @@ void qmp_block_stream(bool has_job_id, const char 
*job_id, const char *device,
 base_name = has_backing_file ? backing_file : base_name;
 
 stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name,
- has_speed ? speed : 0, on_error, block_job_cb, bs, 
_err);
+ has_speed ? speed : 0, on_error, NULL, bs, _err);
 if (local_err) {
 error_propagate(errp, local_err);
 goto out;
@@ -3136,10 +3111,10 @@ void qmp_block_commit(bool has_job_id, const char 
*job_id, const char *device,
 goto out;
 }
 commit_active_start(has_job_id ? job_id : NULL, bs, base_bs, speed,
-on_error, block_job_cb, bs, _err, false);
+on_error, NULL, bs, _err, false);
 } else {
 commit_start(has_job_id ? job_id : NULL, bs, base_bs, top_bs, speed,
- on_error, block_job_cb, bs,
+ on_error, NULL, bs,
  has_backing_file ? backing_file : NULL, _err);
 }
 if (local_err != NULL) {
@@ -3260,7 +3235,7 @@ static void do_drive_backup(DriveBackup *backup, 
BlockJobTxn *txn, Error **errp)
 
 backup_start(backup->job_id, bs, target_bs, backup->speed, backup->sync,
  bmap, backup->compress, backup->on_source_error,
- backup->on_target_error, block_job_cb, bs, txn, _err);
+ backup->on_target_error, NULL, bs, txn, _err);
 bdrv_unref(target_bs);
 if (local_err != NULL) {
 error_propagate(errp, local_err);
@@ -3330,7 +3305,7 @@ void do_blockdev_backup(BlockdevBackup *backup, 
BlockJobTxn *txn, Error **errp)
 }
 backup_start(backup->job_id, bs, target_bs, backup->speed, backup->sync,
  NULL, backup->compress, backup->on_source_error,
- backup->on_target_error, block_job_cb, bs, txn, _err);
+ backup->on_target_error, NULL, bs, txn, _err);
 if (local_err != NULL) {
 error_propagate(errp, local_err);
 }
@@ -3410,7 +3385,7 @@ static void blockdev_mirror_common(const char *job_id, 
BlockDriverState *bs,
  has_replaces ? replaces : NULL,
  speed, granularity, buf_size, sync, backing_mode,
  on_source_error, on_target_error, unmap,
- block_job_cb, bs, errp);
+ NULL, bs, errp);
 }
 
 void qmp_drive_mirror(DriveMirror *arg, Error **errp)
diff --git a/blockjob.c b/blockjob.c
index 13e7134..6a300ba 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -124,7 +124,6 @@ void *block_job_create(const char *job_id, const 
BlockJobDriver *driver,
 BlockBackend *blk;
 BlockJob *job;
 
-assert(cb);
 if (bs->job) {
 error_setg(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
 return NULL;
@@ -218,7 +217,20 @@ static void block_job_completed_single(BlockJob *job)
 job->driver->abort(job);
 }
 }
-job->cb(job->opaque, job->ret);
+
+if (job->cb) {
+job->cb(job->opaque, job->ret);
+}
+if (block_job_is_cancelled(job)) {
+block_job_event_cancelled(job);
+} else {
+const char *msg = NULL;
+if (job->ret < 0) {
+msg = strerror(-job->ret);
+}
+block_job_event_completed(job, msg);
+}
+
 if (job->txn) {
 QLIST_REMOVE(job, txn_list);
 block_job_txn_unref(job->txn);
-- 
2.7.4

[Qemu-block] [PATCH v2 00/11] blockjobs: Fix transactional race condition

[John Snow]

There are a few problems with transactional job completion right now.

First, if jobs complete so quickly they complete before remaining jobs
get a chance to join the transaction, the completion mode can leave well
known state and the QLIST can get corrupted and the transactional jobs
can complete in batches or phases instead of all together.

Second, if two or more jobs defer to the main loop at roughly the same
time, it's possible for one job's cleanup to directly invoke the other
job's cleanup from within the same thread, leading to a situation that
will deadlock the entire transaction.

Thanks to Vladimir for pointing out these modes of failure.

This series also does a little digging into refactoring Jobs into public
and private interfaces. It's somewhat unrelated, but it was easier to
include this with this series than separate it out and send it later.
This comprises patches 2-6. The actual fixes here are in patches 1 and
7-10. A new test to catch Vladimir's failure scenario is in patch 11.

v2:
 - Lots of differences in patches 2-9.
 - Cancel should now work on an "unstarted" blockjob.
 - New refactoring patches.
 - Added "start" property for BlockJob Drivers.



For convenience, this branch is available at:
https://github.com/jnsnow/qemu.git branch job-manual-start
https://github.com/jnsnow/qemu/tree/job-manual-start

This version is tagged job-manual-start-v2:
https://github.com/jnsnow/qemu/releases/tag/job-manual-start-v2

John Snow (10):
  blockjob: centralize QMP event emissions
  Blockjobs: Internalize user_pause logic
  blockjobs: Always use block_job_get_aio_context
  blockjobs: split interface into public/private
  blockjobs: fix documentation
  blockjob: add .clean property
  blockjob: add .start field
  blockjob: add block_job_start
  blockjob: refactor backup_start as backup_job_create
  iotests: add transactional failure race test

Vladimir Sementsov-Ogievskiy (1):
  blockjob: fix dead pointer in txn list

 block/backup.c   |  59 ---
 block/commit.c   |   6 +-
 block/io.c   |   6 +-
 block/mirror.c   |   7 +-
 block/replication.c  |  13 +-
 block/stream.c   |   6 +-
 blockdev.c   | 128 +++
 blockjob.c   |  72 +++--
 include/block/block.h|   3 +-
 include/block/block_int.h|  29 ++--
 include/block/blockjob.h | 345 +++-
 include/block/blockjob_int.h | 366 +++
 qemu-img.c   |   4 +-
 tests/qemu-iotests/124   |  91 +++
 tests/qemu-iotests/124.out   |   4 +-
 tests/test-blockjob-txn.c|  14 +-
 tests/test-blockjob.c|   2 +-
 17 files changed, 688 insertions(+), 467 deletions(-)
 create mode 100644 include/block/blockjob_int.h

-- 
2.7.4

[Qemu-block] [PATCH v2 03/11] Blockjobs: Internalize user_pause logic

[John Snow]

BlockJobs will begin hiding their state in preparation for some
refactorings anyway, so let's internalize the user_pause mechanism
instead of leaving it to callers to correctly manage.

Signed-off-by: John Snow 
---
 block/io.c   |  2 +-
 blockdev.c   | 10 --
 blockjob.c   | 16 
 include/block/blockjob.h | 11 ++-
 4 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/block/io.c b/block/io.c
index fdf7080..868b065 100644
--- a/block/io.c
+++ b/block/io.c
@@ -291,7 +291,7 @@ void bdrv_drain_all(void)
 AioContext *aio_context = blk_get_aio_context(job->blk);
 
 aio_context_acquire(aio_context);
-block_job_pause(job);
+block_job_pause(job, false);
 aio_context_release(aio_context);
 }
 
diff --git a/blockdev.c b/blockdev.c
index 03200e7..268452f 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -3629,7 +3629,7 @@ void qmp_block_job_cancel(const char *device,
 force = false;
 }
 
-if (job->user_paused && !force) {
+if (block_job_paused(job) && !force) {
 error_setg(errp, "The block job for device '%s' is currently paused",
device);
 goto out;
@@ -3646,13 +3646,12 @@ void qmp_block_job_pause(const char *device, Error 
**errp)
 AioContext *aio_context;
 BlockJob *job = find_block_job(device, _context, errp);
 
-if (!job || job->user_paused) {
+if (!job || block_job_paused(job)) {
 return;
 }
 
-job->user_paused = true;
 trace_qmp_block_job_pause(job);
-block_job_pause(job);
+block_job_pause(job, true);
 aio_context_release(aio_context);
 }
 
@@ -3661,11 +3660,10 @@ void qmp_block_job_resume(const char *device, Error 
**errp)
 AioContext *aio_context;
 BlockJob *job = find_block_job(device, _context, errp);
 
-if (!job || !job->user_paused) {
+if (!job || !block_job_paused(job)) {
 return;
 }
 
-job->user_paused = false;
 trace_qmp_block_job_resume(job);
 block_job_iostatus_reset(job);
 block_job_resume(job);
diff --git a/blockjob.c b/blockjob.c
index 6a300ba..2a35f50 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -104,7 +104,7 @@ static void block_job_detach_aio_context(void *opaque)
 /* In case the job terminates during aio_poll()... */
 block_job_ref(job);
 
-block_job_pause(job);
+block_job_pause(job, false);
 
 if (!job->paused) {
 /* If job is !job->busy this kicks it into the next pause point. */
@@ -343,9 +343,12 @@ void block_job_complete(BlockJob *job, Error **errp)
 job->driver->complete(job, errp);
 }
 
-void block_job_pause(BlockJob *job)
+void block_job_pause(BlockJob *job, bool user)
 {
 job->pause_count++;
+if (user) {
+job->user_paused = true;
+}
 }
 
 static bool block_job_should_pause(BlockJob *job)
@@ -353,6 +356,11 @@ static bool block_job_should_pause(BlockJob *job)
 return job->pause_count > 0;
 }
 
+bool block_job_paused(BlockJob *job)
+{
+return job ? job->user_paused : 0;
+}
+
 void coroutine_fn block_job_pause_point(BlockJob *job)
 {
 if (!block_job_should_pause(job)) {
@@ -386,6 +394,7 @@ void block_job_resume(BlockJob *job)
 if (job->pause_count) {
 return;
 }
+job->user_paused = false;
 block_job_enter(job);
 }
 
@@ -592,8 +601,7 @@ BlockErrorAction block_job_error_action(BlockJob *job, 
BlockdevOnError on_err,
 action, _abort);
 if (action == BLOCK_ERROR_ACTION_STOP) {
 /* make the pause user visible, which will be resumed from QMP. */
-job->user_paused = true;
-block_job_pause(job);
+block_job_pause(job, true);
 block_job_iostatus_set_err(job, error);
 }
 return action;
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index 4ddb4ae..081f6c2 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -347,10 +347,19 @@ void coroutine_fn block_job_pause_point(BlockJob *job);
 /**
  * block_job_pause:
  * @job: The job to be paused.
+ * @user: Requested explicitly via user?
  *
  * Asynchronously pause the specified job.
  */
-void block_job_pause(BlockJob *job);
+void block_job_pause(BlockJob *job, bool user);
+
+/**
+ * block_job_paused:
+ * @job: The job to query.
+ *
+ * Returns true if the job is user-paused.
+ */
+bool block_job_paused(BlockJob *job);
 
 /**
  * block_job_resume:
-- 
2.7.4

Re: [Qemu-block] [PATCH v14 12/21] option: allow qemu_opts_to_qdict to merge repeated options

[Eric Blake]

On 09/30/2016 09:45 AM, Daniel P. Berrange wrote:
> If given an option string such as
> 
>   size=1024,nodes=10,nodes=4-5,nodes=1-2,policy=bind
> 
> the qemu_opts_to_qdict() method will currently overwrite
> the values for repeated option keys, so only the last
> value is in the returned dict:
> 
> size=QString("1024")
> nodes=QString("1-2")
> policy=QString("bind")
> 
> With this change the caller can optionally ask for all
> the repeated values to be stored in a QList. In the
> above example that would result in 'nodes' being a
> QList, so the returned dict would contain
> 
> size=QString("1024")
> nodes=QList([QString("10"),
>  QString("4-5"),
>  QString("1-2")])
> policy=QString("bind")

I think the last time around you were converting keys (turning "nodes"
into "nodes.0"; I think your idea here of keeping a single key tied to
QList makes more sense.

> 
> Note that the conversion has no way of knowing whether
> any given key is expected to be a list upfront - it can
> only figure that out when seeing the first duplicated
> key. Thus the caller has to be prepared to deal with the
> fact that if a key 'foo' is a list, then the returned
> qdict may contain either a QString or a QList for the
> key 'foo'.
> 
> In a third mode, it is possible to ask for repeated
> options to be reported as an error, rather than silently
> dropping all but the last one.
> 
> All existing callers are all converted to explicitly
> request the historical behaviour of only reporting the
> last key. Later patches will make use of the new modes.
> 
> Signed-off-by: Daniel P. Berrange 
> ---

> -QDict *qemu_opts_to_qdict(const QemuOpts *opts, QDict *qdict);
> +typedef enum {
> +/* Last occurrence of a duplicate option silently replaces earlier */
> +QEMU_OPTS_REPEAT_POLICY_LAST,
> +/* Each occurrence of a duplicate option converts value to a QList */
> +QEMU_OPTS_REPEAT_POLICY_ALL,
> +/* First occurrence of a duplicate option causes an error */
> +QEMU_OPTS_REPEAT_POLICY_ERROR,
> +} QemuOptsRepeatPolicy;
> +

Nicer.  Thanks!

> +++ b/tests/test-qemu-opts.c
> @@ -421,6 +421,130 @@ static void test_qemu_opts_set(void)
>  g_assert(opts == NULL);
>  }
>  

> +static void test_qemu_opts_to_qdict_repeat_all(void)
> +{
> +QemuOpts *opts;
> +QDict *dict;
> +QList *list;
> +const QListEntry *ent;
> +QString *str;
> +
> +/* dynamically initialized (parsed) opts */
> +opts = qemu_opts_parse(_list_03,
> +   
> "size=1024,nodes=10,nodes=4-5,nodes=1-2,policy=bind",
> +   false, NULL);
> +g_assert(opts);
> +
> +dict = qemu_opts_to_qdict(opts, NULL, QEMU_OPTS_REPEAT_POLICY_ALL,
> +  _abort);
> +g_assert(dict);
> +
> +g_assert_cmpstr(qdict_get_str(dict, "size"), ==, "1024");
> +g_assert(qdict_haskey(dict, "nodes"));
> +list = qobject_to_qlist(qdict_get(dict, "nodes"));
> +g_assert(list);
> +
> +ent = qlist_first(list);
> +g_assert(ent);
> +str = qobject_to_qstring(ent->value);
> +g_assert(str);
> +g_assert_cmpstr(qstring_get_str(str), ==, "10");
> +
> +ent = qlist_next(ent);
> +g_assert(ent);
> +str = qobject_to_qstring(ent->value);
> +g_assert(str);
> +g_assert_cmpstr(qstring_get_str(str), ==, "4-5");
> +
> +ent = qlist_next(ent);
> +g_assert(ent);
> +str = qobject_to_qstring(ent->value);
> +g_assert(str);
> +g_assert_cmpstr(qstring_get_str(str), ==, "1-2");

Yes, this is different from v13, but does look nicer with the value
turning into QList instead of the key being rewritten into dotted form.

> +++ b/util/qemu-option.c
> @@ -1057,23 +1057,73 @@ void qemu_opts_absorb_qdict(QemuOpts *opts, QDict 
> *qdict, Error **errp)
>   * The QDict values are of type QString.
>   * TODO We'll want to use types appropriate for opt->desc->type, but
>   * this is enough for now.
> + *
> + * If the @opts contains multiple occurrences of the same key,
> + * then the @repeatPolicy parameter determines how they are to
> + * be handled. Traditional behaviour was to only store the
> + * last occurrence, but if @repeatPolicy is set to
> + * QEMU_OPTS_REPEAT_POLICY_ALL, then a QList will be returned
> + * containing all values, for any key with multiple occurrences.
> + * The QEMU_OPTS_REPEAT_POLICY_ERROR value can be used to fail
> + * conversion with an error if multiple occurrences of a key
> + * are seen.
>   */
> -QDict *qemu_opts_to_qdict(const QemuOpts *opts, QDict *qdict)
> +QDict *qemu_opts_to_qdict(const QemuOpts *opts, QDict *qdict,
> +  QemuOptsRepeatPolicy repeatPolicy,
> +  Error **errp)

Reviewed-by: Eric Blake 

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

[Qemu-block] [PATCH v2 4/4] qemu-iotests: Test creating floppy drives

[Kevin Wolf]

This tests the different supported methods to create floppy drives and
how they interact.

Signed-off-by: Kevin Wolf 
---
 tests/qemu-iotests/172 |  242 +
 tests/qemu-iotests/172.out | 1205 
 tests/qemu-iotests/group   |1 +
 3 files changed, 1448 insertions(+)
 create mode 100755 tests/qemu-iotests/172
 create mode 100644 tests/qemu-iotests/172.out

diff --git a/tests/qemu-iotests/172 b/tests/qemu-iotests/172
new file mode 100755
index 000..8bb6443
--- /dev/null
+++ b/tests/qemu-iotests/172
@@ -0,0 +1,242 @@
+#!/bin/bash
+#
+# Test floppy configuration
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+#
+
+# creator
+owner=kw...@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1   # failure is the default!
+
+_cleanup()
+{
+   _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+if [ "$QEMU_DEFAULT_MACHINE" != "pc" ]; then
+_notrun "Requires a PC machine"
+fi
+
+function do_run_qemu()
+{
+(
+if ! test -t 0; then
+while read cmd; do
+echo $cmd
+done
+fi
+echo quit
+) | $QEMU -nographic -monitor stdio -serial none "$@"
+echo
+}
+
+function check_floppy_qtree()
+{
+echo
+echo Testing: "$@" | _filter_testdir
+
+# QEMU_OPTIONS contains -nodefaults, we don't want that here because the
+# defaults are part of what should be checked here
+echo "info qtree" |
+QEMU_OPTIONS="" do_run_qemu "$@" | _filter_win32 |
+grep -zo '[[:cntrl:]]\( *\)dev: isa-fdc.*\([[:cntrl:]]\1 .*\)*[[:cntrl:]] 
*dev:'
+}
+
+function check_cache_mode()
+{
+echo "info block none0" |
+QEMU_OPTIONS="" do_run_qemu -drive if=none,file="$TEST_IMG" "$@" |
+_filter_win32 | grep "Cache mode"
+}
+
+
+size=720k
+
+_make_test_img $size
+
+# Default drive semantics:
+#
+# By default you get a single empty floppy drive. You can override it with
+# -drive and using the same index, but if you use -drive to add a floppy to a
+# different index, you get both of them. However, as soon as you use any
+# '-device floppy', even to a different slot, the default drive is disabled.
+
+echo
+echo
+echo === Default ===
+
+check_floppy_qtree
+
+echo
+echo
+echo === Using -fda/-fdb options ===
+
+check_floppy_qtree -fda "$TEST_IMG"
+check_floppy_qtree -fdb "$TEST_IMG"
+check_floppy_qtree -fda "$TEST_IMG" -fdb "$TEST_IMG"
+
+
+echo
+echo
+echo === Using -drive options ===
+
+check_floppy_qtree -drive if=floppy,file="$TEST_IMG"
+check_floppy_qtree -drive if=floppy,file="$TEST_IMG",index=1
+check_floppy_qtree -drive if=floppy,file="$TEST_IMG" -drive 
if=floppy,file="$TEST_IMG",index=1
+
+echo
+echo
+echo === Using -drive if=none and -global ===
+
+check_floppy_qtree -drive if=none,file="$TEST_IMG" -global isa-fdc.driveA=none0
+check_floppy_qtree -drive if=none,file="$TEST_IMG" -global isa-fdc.driveB=none0
+check_floppy_qtree -drive if=none,file="$TEST_IMG" -drive 
if=none,file="$TEST_IMG" \
+   -global isa-fdc.driveA=none0 -global isa-fdc.driveB=none1
+
+echo
+echo
+echo === Using -drive if=none and -device ===
+
+check_floppy_qtree -drive if=none,file="$TEST_IMG" -device floppy,drive=none0
+check_floppy_qtree -drive if=none,file="$TEST_IMG" -device 
floppy,drive=none0,unit=1
+check_floppy_qtree -drive if=none,file="$TEST_IMG" -drive 
if=none,file="$TEST_IMG" \
+   -device floppy,drive=none0 -device floppy,drive=none1,unit=1
+
+echo
+echo
+echo === Mixing -fdX and -global ===
+
+# Working
+check_floppy_qtree -fda "$TEST_IMG" -drive if=none,file="$TEST_IMG" -global 
isa-fdc.driveB=none0
+check_floppy_qtree -fdb "$TEST_IMG" -drive if=none,file="$TEST_IMG" -global 
isa-fdc.driveA=none0
+
+# Conflicting (-fdX wins)
+check_floppy_qtree -fda "$TEST_IMG" -drive if=none,file="$TEST_IMG" -global 
isa-fdc.driveA=none0
+check_floppy_qtree -fdb "$TEST_IMG" -drive if=none,file="$TEST_IMG" -global 
isa-fdc.driveB=none0
+
+echo
+echo
+echo === Mixing -fdX and -device ===
+
+# Working
+check_floppy_qtree -fda "$TEST_IMG" -drive if=none,file="$TEST_IMG" -device 
floppy,drive=none0
+check_floppy_qtree -fda "$TEST_IMG" -drive 

Re: [Qemu-block] [PATCH v14 11/21] qapi: add integer range support for QObjectInputVisitor

[Eric Blake]

On 09/30/2016 09:45 AM, Daniel P. Berrange wrote:
> The traditional CLI arg syntax allows two ways to specify
> integer lists, either one value per key, or a range of
> values per key. eg the following are identical:
> 
>   -arg foo=5,foo=6,foo=7
>   -arg foo=5-7
> 
> This extends the QObjectInputVisitor so that it is able
> to parse ranges and turn them into distinct list entries.
> 
> This means that
> 
>   -arg foo=5-7
> 
> is treated as equivalent to
> 
>   -arg foo.0=5,foo.1=6,foo.2=7
> 
> Edge case tests are copied from test-opts-visitor to
> ensure identical behaviour when parsing.
> 
> Signed-off-by: Daniel P. Berrange 
> ---

> @@ -71,6 +77,19 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool 
> strict);
>   * The value given determines how many levels of structs are allowed to
>   * be flattened in this way.
>   *
> + * If @permit_int_ranges is true, then when visiting a list of integers,
> + * the integer value strings may encode ranges eg a single element

My usual complaint that 'e.g.' is usually spelled with dots :)

> + * containing "5-7" is treated as if there were three elements "5", "6",
> + * "7". This should only be used if compatibility is required with the
> + * OptsVisitor which would allow integer ranges. e.g. set this to true

Ah. Here you DID use dots, but used the wrong abbreviation.
Substituting "For example" in this location does not make as much sense
as "i.e." would.  Or go for simplicity; it reads just fine as:

"...would allow integer ranges; so only set this to true if..."

> + * if you have compatibility requirements for
> + *
> + *   -arg val=5-8
> + *
> + * to be treated as equivalent to the preferred syntax:
> + *
> + *   -arg val.0=5,val.1=6,val.2=7,val.3=8
> + *

> +
> +/*
> + * If we have string that represents an integer range (5-24),
> + * parse the end of the range and set things up so we'll process
> + * the rest of the range before consuming another element
> + * from the QList.
> + */
> +if (end && *end) {
> +if (!qiv->permit_int_ranges) {
> +error_setg(errp,
> +   "Integer ranges are not permitted here");
> +return;
> +}
> +if (!inlist) {
> +error_setg(errp,
> +   "Integer ranges are only permitted when "
> +   "visiting list parameters");
> +return;
> +}
> +if (*end != '-') {
> +error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name,
> +   "a number range");
> +return;
> +}
> +end++;
> +if (qemu_strtoll(end, NULL, 0, ) < 0) {

Signed parse...

> +error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
> +return;
> +}
> +if (*obj > ret) {

...and signed comparison...

> +error_setg(errp,
> +   "Parameter '%s' range start %" PRIu64
> +   " must be less than (or equal to) %" PRIu64,

...but unsigned numbers in the error message.  Looks a bit odd (but not
necessarily worse than what our OptsVisitor was already doing).

> +   name, *obj, ret);
> +return;
> +}
> +
> +if ((*obj <= (INT64_MAX - QIV_RANGE_MAX)) &&
> +(ret >= (*obj + QIV_RANGE_MAX))) {
> +error_setg(errp,
> +   "Parameter '%s' range must be less than %d",
> +   name, QIV_RANGE_MAX);
> +return;
> +}
> +if (*obj != ret) {
> +tos->range_val = *obj;
> +tos->range_limit = ret;
> +}
> +}
>  }

Overall, I found this a bit easier to read than the OptsVisitor state
machine.  Which makes me worried that we may have some subtle
translation bugs; but I hope it's the same end result.

My consolation is that if there are any differences, they would be in
the corner cases, where a user is probably going to get surprising
behavior from the command line anyway, and shouldn't have been
requesting ranges in that corner case.

>  
>  static void qobject_input_type_uint64(Visitor *v, const char *name,
> @@ -366,21 +438,85 @@ static void qobject_input_type_uint64_autocast(Visitor 
> *v, const char *name,
> uint64_t *obj, Error **errp)
>  {
>  QObjectInputVisitor *qiv = to_qiv(v);

Feels like a lot of code duplication; but I don't know if there's any
sane way to share more of the code while handling both signed and
unsigned types.

The testsuite additions are reassuring; I confirmed that they match
test-opts-visitor.c in coverage.

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

[Qemu-block] [PATCH v2 3/4] fdc: Move qdev properties to FloppyDrive

[Kevin Wolf]

This makes the FloppyDrive qdev object actually useful: Now that it has
all properties that don't belong to the controller, you can actually
use '-device floppy' and get a working result.

Command line semantics is consistent with CD-ROM drives: By default you
get a single empty floppy drive. You can override it with -drive and
using the same index, but if you use -drive to add a floppy to a
different index, you get both of them. However, as soon as you use any
'-device floppy', even to a different slot, the default drive is
disabled.

Using '-device floppy' without specifying the unit will choose the first
free slot on the controller.

Signed-off-by: Kevin Wolf 
---
 hw/block/fdc.c | 112 ++---
 vl.c   |   1 +
 2 files changed, 85 insertions(+), 28 deletions(-)

diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index 5aa8e52..00c0ec6 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -35,6 +35,7 @@
 #include "qemu/timer.h"
 #include "hw/isa/isa.h"
 #include "hw/sysbus.h"
+#include "hw/block/block.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
@@ -487,12 +488,18 @@ static const BlockDevOps fd_block_ops = {
  OBJECT_CHECK(FloppyDrive, (obj), TYPE_FLOPPY_DRIVE)
 
 typedef struct FloppyDrive {
-DeviceState qdev;
-uint32_tunit;
+DeviceState qdev;
+uint32_tunit;
+BlockConf   conf;
+FloppyDriveType type;
 } FloppyDrive;
 
 static Property floppy_drive_properties[] = {
 DEFINE_PROP_UINT32("unit", FloppyDrive, unit, -1),
+DEFINE_BLOCK_PROPERTIES(FloppyDrive, conf),
+DEFINE_PROP_DEFAULT("drive-type", FloppyDrive, type,
+FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+FloppyDriveType),
 DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -501,6 +508,7 @@ static int floppy_drive_init(DeviceState *qdev)
 FloppyDrive *dev = FLOPPY_DRIVE(qdev);
 FloppyBus *bus = DO_UPCAST(FloppyBus, bus, dev->qdev.parent_bus);
 FDrive *drive;
+int ret;
 
 if (dev->unit == -1) {
 for (dev->unit = 0; dev->unit < MAX_FD; dev->unit++) {
@@ -517,29 +525,57 @@ static int floppy_drive_init(DeviceState *qdev)
 return -1;
 }
 
-/* TODO Check whether unit is in use */
-
 drive = get_drv(bus->fdc, dev->unit);
-
 if (drive->blk) {
-if (blk_get_on_error(drive->blk, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
-error_report("fdc doesn't support drive option werror");
-return -1;
-}
-if (blk_get_on_error(drive->blk, 1) != BLOCKDEV_ON_ERROR_REPORT) {
-error_report("fdc doesn't support drive option rerror");
-return -1;
-}
-} else {
+error_report("Floppy unit %d is in use", dev->unit);
+return -1;
+}
+
+if (!dev->conf.blk) {
 /* Anonymous BlockBackend for an empty drive */
-drive->blk = blk_new();
+dev->conf.blk = blk_new();
+ret = blk_attach_dev(dev->conf.blk, dev);
+assert(ret == 0);
 }
 
-fd_init(drive);
-if (drive->blk) {
-blk_set_dev_ops(drive->blk, _block_ops, drive);
-pick_drive_type(drive);
+blkconf_blocksizes(>conf);
+if (dev->conf.logical_block_size != 512 ||
+dev->conf.physical_block_size != 512)
+{
+error_report("Physical and logical block size must be 512 for floppy");
+return -1;
+}
+
+/* rerror/werror aren't supported by fdc and therefore not even registered
+ * with qdev. So set the defaults manually before they are used in
+ * blkconf_apply_backend_options(). */
+dev->conf.rerror = BLOCKDEV_ON_ERROR_AUTO;
+dev->conf.werror = BLOCKDEV_ON_ERROR_AUTO;
+blkconf_apply_backend_options(>conf);
+
+/* 'enospc' is the default for -drive, 'report' is what blk_new() gives us
+ * for empty drives. */
+if (blk_get_on_error(dev->conf.blk, 0) != BLOCKDEV_ON_ERROR_ENOSPC &&
+blk_get_on_error(dev->conf.blk, 0) != BLOCKDEV_ON_ERROR_REPORT) {
+error_report("fdc doesn't support drive option werror");
+return -1;
 }
+if (blk_get_on_error(dev->conf.blk, 1) != BLOCKDEV_ON_ERROR_REPORT) {
+error_report("fdc doesn't support drive option rerror");
+return -1;
+}
+
+drive->blk = dev->conf.blk;
+drive->fdctrl = bus->fdc;
+
+fd_init(drive);
+blk_set_dev_ops(drive->blk, _block_ops, drive);
+
+/* Keep 'type' qdev property and FDrive->drive in sync */
+drive->drive = dev->type;
+pick_drive_type(drive);
+dev->type = drive->drive;
+
 fd_revalidate(drive);
 
 return 0;
@@ -808,6 +844,10 @@ struct FDCtrl {
 FloppyBus bus;
 uint8_t num_floppies;
 FDrive drives[MAX_FD];
+struct {
+BlockBackend *blk;
+FloppyDriveType type;
+} qdev_for_drives[MAX_FD];
 int reset_sensei;
 uint32_t check_media_rate;
 FloppyDriveType fallback; /* 

[Qemu-block] [PATCH v2 0/4] fdc: Use separate qdev device for drives

[Kevin Wolf]

We have been complaining for a long time about how the floppy controller and
floppy drives are combined in a single qdev device and how this makes the
device awkward to work with because it behaves different from all other block
devices.

The latest reason to complain was when I noticed that using qdev device names
in QMP commands (e.g. for media change) doesn't really work when only the
controller is a qdev device, but the drives aren't.

So I decided to have a go at it, and this is the result.

It doesn't actually change any of the inner workings of the floppy controller,
but it wires things up differently on the qdev layer so that a floppy
controller now exposes a bus on which the floppy drives sit. This results in a
structure that is similar to IDE where the actual drive state is still in the
controller and the qdev device basically just contains the qdev properties -
not pretty, but quite workable.

The commit message of patch 3 explains how to use it. In short, there is a
'-device floppy' now and it does what you would expect if you ever used ide-cd.

The other problem is old command lines, especially those using things like
'-global isa-fdc,driveA=...'. In order to keep them working, we need to forward
the property to an internally created floppy drive device. This is a bit like
usb-storage, which we know is ugly, but works well enough in practice. The good
news here is that in contrast to usb-storage, the floppy controller only does
the forwarding for legacy configurations; as soon as you start using '-device
floppy', it doesn't happen any more.

So as you may have expected, this conversion doesn't result in a perfect
device, but I think it's definitely an improvement over the old state. I hope
you like it despite the warts. :-)

v2:
- Added patch 4 (qemu-iotests case for floppy config on the command line)
- Patch 2: Create a floppy device only if a BlockBackend exists instead of
  always creating two of them
- Patch 2: Initialise drive->fdctrl even if no drive is attached, it is
  accessed anyway during migration
- Patch 3: Keep 'type' qdev property and FDrive->drive in sync
- Patch 3: Removed if with condition that is always true

Kevin Wolf (4):
  fdc: Add a floppy qbus
  fdc: Add a floppy drive qdev
  fdc: Move qdev properties to FloppyDrive
  qemu-iotests: Test creating floppy drives

 hw/block/fdc.c |  263 --
 tests/qemu-iotests/172 |  242 +
 tests/qemu-iotests/172.out | 1205 
 tests/qemu-iotests/group   |1 +
 vl.c   |1 +
 5 files changed, 1668 insertions(+), 44 deletions(-)
 create mode 100755 tests/qemu-iotests/172
 create mode 100644 tests/qemu-iotests/172.out

-- 
1.8.3.1

[Qemu-block] [PATCH v2 2/4] fdc: Add a floppy drive qdev

[Kevin Wolf]

Floppy controllers automatically create two floppy drive devices in qdev
now. (They always created two drives, but managed them only internally.)

Signed-off-by: Kevin Wolf 
---
 hw/block/fdc.c | 151 +
 1 file changed, 120 insertions(+), 31 deletions(-)

diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index a3afb62..5aa8e52 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -60,6 +60,8 @@
 #define FLOPPY_BUS(obj) OBJECT_CHECK(FloppyBus, (obj), TYPE_FLOPPY_BUS)
 
 typedef struct FDCtrl FDCtrl;
+typedef struct FDrive FDrive;
+static FDrive *get_drv(FDCtrl *fdctrl, int unit);
 
 typedef struct FloppyBus {
 BusState bus;
@@ -180,7 +182,7 @@ typedef enum FDiskFlags {
 FDISK_DBL_SIDES  = 0x01,
 } FDiskFlags;
 
-typedef struct FDrive {
+struct FDrive {
 FDCtrl *fdctrl;
 BlockBackend *blk;
 /* Drive status */
@@ -201,7 +203,7 @@ typedef struct FDrive {
 uint8_t media_rate;   /* Data rate of medium*/
 
 bool media_validated; /* Have we validated the media? */
-} FDrive;
+};
 
 
 static FloppyDriveType get_fallback_drive_type(FDrive *drv);
@@ -466,6 +468,100 @@ static void fd_revalidate(FDrive *drv)
 }
 }
 
+static void fd_change_cb(void *opaque, bool load)
+{
+FDrive *drive = opaque;
+
+drive->media_changed = 1;
+drive->media_validated = false;
+fd_revalidate(drive);
+}
+
+static const BlockDevOps fd_block_ops = {
+.change_media_cb = fd_change_cb,
+};
+
+
+#define TYPE_FLOPPY_DRIVE "floppy"
+#define FLOPPY_DRIVE(obj) \
+ OBJECT_CHECK(FloppyDrive, (obj), TYPE_FLOPPY_DRIVE)
+
+typedef struct FloppyDrive {
+DeviceState qdev;
+uint32_tunit;
+} FloppyDrive;
+
+static Property floppy_drive_properties[] = {
+DEFINE_PROP_UINT32("unit", FloppyDrive, unit, -1),
+DEFINE_PROP_END_OF_LIST(),
+};
+
+static int floppy_drive_init(DeviceState *qdev)
+{
+FloppyDrive *dev = FLOPPY_DRIVE(qdev);
+FloppyBus *bus = DO_UPCAST(FloppyBus, bus, dev->qdev.parent_bus);
+FDrive *drive;
+
+if (dev->unit == -1) {
+for (dev->unit = 0; dev->unit < MAX_FD; dev->unit++) {
+drive = get_drv(bus->fdc, dev->unit);
+if (!drive->blk) {
+break;
+}
+}
+}
+
+if (dev->unit >= MAX_FD) {
+error_report("Can't create floppy unit %d, bus supports only %d units",
+ dev->unit, MAX_FD);
+return -1;
+}
+
+/* TODO Check whether unit is in use */
+
+drive = get_drv(bus->fdc, dev->unit);
+
+if (drive->blk) {
+if (blk_get_on_error(drive->blk, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
+error_report("fdc doesn't support drive option werror");
+return -1;
+}
+if (blk_get_on_error(drive->blk, 1) != BLOCKDEV_ON_ERROR_REPORT) {
+error_report("fdc doesn't support drive option rerror");
+return -1;
+}
+} else {
+/* Anonymous BlockBackend for an empty drive */
+drive->blk = blk_new();
+}
+
+fd_init(drive);
+if (drive->blk) {
+blk_set_dev_ops(drive->blk, _block_ops, drive);
+pick_drive_type(drive);
+}
+fd_revalidate(drive);
+
+return 0;
+}
+
+static void floppy_drive_class_init(ObjectClass *klass, void *data)
+{
+DeviceClass *k = DEVICE_CLASS(klass);
+k->init = floppy_drive_init;
+set_bit(DEVICE_CATEGORY_STORAGE, k->categories);
+k->bus_type = TYPE_FLOPPY_BUS;
+k->props = floppy_drive_properties;
+k->desc = "virtual floppy drive";
+}
+
+static const TypeInfo floppy_drive_info = {
+.name = TYPE_FLOPPY_DRIVE,
+.parent = TYPE_DEVICE,
+.instance_size = sizeof(FloppyDrive),
+.class_init = floppy_drive_class_init,
+};
+
 //
 /* Intel 82078 floppy disk controller emulation  */
 
@@ -1185,9 +1281,9 @@ static inline FDrive *drv3(FDCtrl *fdctrl)
 }
 #endif
 
-static FDrive *get_cur_drv(FDCtrl *fdctrl)
+static FDrive *get_drv(FDCtrl *fdctrl, int unit)
 {
-switch (fdctrl->cur_drv) {
+switch (unit) {
 case 0: return drv0(fdctrl);
 case 1: return drv1(fdctrl);
 #if MAX_FD == 4
@@ -1198,6 +1294,11 @@ static FDrive *get_cur_drv(FDCtrl *fdctrl)
 }
 }
 
+static FDrive *get_cur_drv(FDCtrl *fdctrl)
+{
+return get_drv(fdctrl, fdctrl->cur_drv);
+}
+
 /* Status A register : 0x00 (read-only) */
 static uint32_t fdctrl_read_statusA(FDCtrl *fdctrl)
 {
@@ -2357,46 +2458,33 @@ static void fdctrl_result_timer(void *opaque)
 }
 }
 
-static void fdctrl_change_cb(void *opaque, bool load)
-{
-FDrive *drive = opaque;
-
-drive->media_changed = 1;
-drive->media_validated = false;
-fd_revalidate(drive);
-}
-
-static const BlockDevOps fdctrl_block_ops = {
-.change_media_cb = fdctrl_change_cb,
-};
-
 /* Init functions */
 static void fdctrl_connect_drives(FDCtrl *fdctrl, Error **errp)
 {
 unsigned int i;
 FDrive *drive;
+

[Qemu-block] [PATCH v2 1/4] fdc: Add a floppy qbus

[Kevin Wolf]

This adds a qbus to the floppy controller that should contain the floppy
drives eventually. At the moment it just exists and is empty.

Signed-off-by: Kevin Wolf 
---
 hw/block/fdc.c | 40 +++-
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index b79873a..a3afb62 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -52,6 +52,33 @@
 }   \
 } while (0)
 
+
+//
+/* qdev floppy bus  */
+
+#define TYPE_FLOPPY_BUS "Floppy"
+#define FLOPPY_BUS(obj) OBJECT_CHECK(FloppyBus, (obj), TYPE_FLOPPY_BUS)
+
+typedef struct FDCtrl FDCtrl;
+
+typedef struct FloppyBus {
+BusState bus;
+FDCtrl *fdc;
+} FloppyBus;
+
+static const TypeInfo floppy_bus_info = {
+.name = TYPE_FLOPPY_BUS,
+.parent = TYPE_BUS,
+.instance_size = sizeof(FloppyBus),
+};
+
+static void floppy_bus_create(FDCtrl *fdc, FloppyBus *bus, DeviceState *dev)
+{
+qbus_create_inplace(bus, sizeof(FloppyBus), TYPE_FLOPPY_BUS, dev, NULL);
+bus->fdc = fdc;
+}
+
+
 //
 /* Floppy drive emulation   */
 
@@ -148,8 +175,6 @@ static FDriveSize drive_size(FloppyDriveType drive)
 #define FD_SECTOR_SC   2   /* Sector size code */
 #define FD_RESET_SENSEI_COUNT  4   /* Number of sense interrupts on RESET */
 
-typedef struct FDCtrl FDCtrl;
-
 /* Floppy disk drive emulation */
 typedef enum FDiskFlags {
 FDISK_DBL_SIDES  = 0x01,
@@ -684,6 +709,7 @@ struct FDCtrl {
 /* Power down config (also with status regB access mode */
 uint8_t pwrd;
 /* Floppy drives */
+FloppyBus bus;
 uint8_t num_floppies;
 FDrive drives[MAX_FD];
 int reset_sensei;
@@ -2442,7 +2468,8 @@ void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
 *fdc_tc = qdev_get_gpio_in(dev, 0);
 }
 
-static void fdctrl_realize_common(FDCtrl *fdctrl, Error **errp)
+static void fdctrl_realize_common(DeviceState *dev, FDCtrl *fdctrl,
+  Error **errp)
 {
 int i, j;
 static int command_tables_inited = 0;
@@ -2480,6 +2507,8 @@ static void fdctrl_realize_common(FDCtrl *fdctrl, Error 
**errp)
 k->register_channel(fdctrl->dma, fdctrl->dma_chann,
 _transfer_handler, fdctrl);
 }
+
+floppy_bus_create(fdctrl, >bus, dev);
 fdctrl_connect_drives(fdctrl, errp);
 }
 
@@ -2508,7 +2537,7 @@ static void isabus_fdc_realize(DeviceState *dev, Error 
**errp)
 }
 
 qdev_set_legacy_instance_id(dev, isa->iobase, 2);
-fdctrl_realize_common(fdctrl, );
+fdctrl_realize_common(dev, fdctrl, );
 if (err != NULL) {
 error_propagate(errp, err);
 return;
@@ -2559,7 +2588,7 @@ static void sysbus_fdc_common_realize(DeviceState *dev, 
Error **errp)
 FDCtrlSysBus *sys = SYSBUS_FDC(dev);
 FDCtrl *fdctrl = >state;
 
-fdctrl_realize_common(fdctrl, errp);
+fdctrl_realize_common(dev, fdctrl, errp);
 }
 
 FloppyDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i)
@@ -2744,6 +2773,7 @@ static void fdc_register_types(void)
 type_register_static(_fdc_type_info);
 type_register_static(_fdc_info);
 type_register_static(_fdc_info);
+type_register_static(_bus_info);
 }
 
 type_init(fdc_register_types)
-- 
1.8.3.1

Re: [Qemu-block] backup notifier fail policy

[Vladimir Sementsov-Ogievskiy]

On 30.09.2016 20:11, Vladimir Sementsov-Ogievskiy wrote:

Hi all!

Please, can somebody explain me, why we fail guest request in case of 
io error in write notifier? I think guest consistency is more 
important than success of unfinished backup. Or, what am I missing?


I'm saying about this code:

static int coroutine_fn backup_before_write_notify(
NotifierWithReturn *notifier,
void *opaque)
{
BackupBlockJob *job = container_of(notifier, BackupBlockJob, 
before_write);

BdrvTrackedRequest *req = opaque;
int64_t sector_num = req->offset >> BDRV_SECTOR_BITS;
int nb_sectors = req->bytes >> BDRV_SECTOR_BITS;

assert(req->bs == blk_bs(job->common.blk));
assert((req->offset & (BDRV_SECTOR_SIZE - 1)) == 0);
assert((req->bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

return backup_do_cow(job, sector_num, nb_sectors, NULL, true);
}

So, what about something like

ret = backup_do_cow(job, ...
if (ret < 0 && job->notif_ret == 0) {
   job->notif_ret = ret;
}

return 0;

and fail block job if notif_ret < 0 in other places of backup code?



And second question about notifiers in backup block job. If block job is 
paused, notifiers still works and can copy data. Is it ok? So, user 
thinks that job is paused, so he can do something with target disk.. But 
really, this 'something' will race with write-notifiers. So, what 
assumptions may user actually have about paused backup job? Is there any 
agreements? Also, on query-block-jobs we will see job.busy = false, when 
actually copy-on-write may be in flight..


--
Best regards,
Vladimir

Re: [Qemu-block] [Qemu-devel] [PATCH v14 00/21] QAPI/QOM work for non-scalar object properties

[Eric Blake]

On 09/30/2016 10:45 AM,
no-re...@ec2-52-6-146-230.compute-1.amazonaws.com wrote:
> Hi,
> 
> Your series failed automatic build test. Please find the testing commands and
> their output below. If you have docker installed, you can probably reproduce 
> it
> locally.
> 

>   CCqapi/string-output-visitor.o
> /tmp/qemu-test/src/qapi/qobject-input-visitor.c: In function 
> ‘qobject_input_get_object’:
> /tmp/qemu-test/src/qapi/qobject-input-visitor.c:97: warning: implicit 
> declaration of function ‘g_hash_table_contains’
> /tmp/qemu-test/src/qapi/qobject-input-visitor.c:97: warning: nested extern 
> declaration of ‘g_hash_table_contains’

> 
> tests/docker/Makefile.include:107: recipe for target 
> 'docker-run-test-quick@centos6' failed
> make: *** [docker-run-test-quick@centos6] Error 2

Cool - the autobuilder is now detecting use of too-new glib functions :)

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

Re: [Qemu-block] [PATCH v4 02/12] block/nbd: Reject port parameter without host

[Eric Blake]

On 09/28/2016 03:55 PM, Max Reitz wrote:
> Currently, a port that is passed along with a UNIX socket path is
> silently ignored. That is not exactly ideal, it should be an error
> instead.
> 
> Signed-off-by: Max Reitz 
> ---
>  block/nbd.c | 7 +--
>  1 file changed, 5 insertions(+), 2 deletions(-)
> 

Reviewed-by: Eric Blake 


-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

Re: [Qemu-block] Transactions, Jobs, and Cancellation

[John Snow]

On 09/30/2016 02:30 PM, Eric Blake wrote:

On 09/30/2016 11:35 AM, John Snow wrote:

Hi Eric (as a proxy for Libvirt);

I want to make a change to transactions such that they do not actually
start the jobs until the entire transaction is error-checked for validity.

This would be a change from the current setup where:

- Some jobs are started
- One job cannot start
- Existing jobs are cancelled, emitting job events
- QMP transaction returns failure.


In this case, the number of events emitted is less than the number of
events comprising the overall transaction, right?



Yes, because some may fail to execute %jobtype%_start(), and the 
transaction halts and returns ERROR. Some jobs may have already started, 
and those are canceled, producing an event.




to something more like:

- Some jobs are queued to start
- One job cannot start
- Existing queued jobs are un-created
- No events are emitted, but the QMP transaction fails.


Is the failure guaranteed to be synchronous to the QMP command that
requested the 'transaction' command?  If so, I think you're fine: in
general, libvirt is looking for events (and presumably N events for a
'transaction' with N components) _only_ if the 'transaction' command
itself succeeded; but if the 'transaction' command fails, then there is
no expectation of events.



Yep, the synchronous case -- before we return any info on the QMP 
transaction at all.



If the failure is asynchronous (and can happen even after 'transaction'
returns success), then it gets a bit weirder; but libvirt still tries to
operate on the principle that events are best-effort notifications and
may be missed, so as long as it is always possible to poll QMP and learn
the same information as would have been presented in the omitted events,
we are probably still okay.



Yeah, no changes to the async case, regardless of the value of the 
trasnactional-cancel property.




If I understand correctly, it's possible for the transaction to fail
even after it has started several jobs because they begin operating
during the prepare phase.

Then, because the transaction preparation has failed, QEMU will cancel
the transaction instead of proceeding, which will generate some
BLOCK_JOB_CANCELLED events.

This may affect libvirt and others if I change these semantics.

Does that sound appropriate to you, or would you from a libvirt
perspective RATHER get events for "uncreated" jobs like you do now? I
could make it do either, but I'd rather prefer to simply not emit events
for jobs that didn't truly never start.


Avoiding events for jobs that never start makes sense if the transaction
itself returns failure.



I can also begin emitting events for jobs that *actually* start if it
would help to disambiguate the cases between old and new transactions.

Note: This has nothing to do with the transactional-cancel property,
which only impacts what happens when jobs created by a transaction fail
AFTER a successful return from qmp_transaction.


Ah, so it sounds like this is ALL about the synchronous case (that is,
the old behavior was that _even_ when 'transaction' fails, some events
were possibly leaked for < N portions of the transaction, if the earlier
portions were started to the point that they could emit a cancelled
event as a result of the later portions never starting).  So I think
your idea makes sense.



Great!

--js

Re: [Qemu-block] Transactions, Jobs, and Cancellation

[Eric Blake]

On 09/30/2016 11:35 AM, John Snow wrote:
> Hi Eric (as a proxy for Libvirt);
> 
> I want to make a change to transactions such that they do not actually
> start the jobs until the entire transaction is error-checked for validity.
> 
> This would be a change from the current setup where:
> 
> - Some jobs are started
> - One job cannot start
> - Existing jobs are cancelled, emitting job events
> - QMP transaction returns failure.

In this case, the number of events emitted is less than the number of
events comprising the overall transaction, right?

> 
> to something more like:
> 
> - Some jobs are queued to start
> - One job cannot start
> - Existing queued jobs are un-created
> - No events are emitted, but the QMP transaction fails.

Is the failure guaranteed to be synchronous to the QMP command that
requested the 'transaction' command?  If so, I think you're fine: in
general, libvirt is looking for events (and presumably N events for a
'transaction' with N components) _only_ if the 'transaction' command
itself succeeded; but if the 'transaction' command fails, then there is
no expectation of events.

If the failure is asynchronous (and can happen even after 'transaction'
returns success), then it gets a bit weirder; but libvirt still tries to
operate on the principle that events are best-effort notifications and
may be missed, so as long as it is always possible to poll QMP and learn
the same information as would have been presented in the omitted events,
we are probably still okay.

> 
> If I understand correctly, it's possible for the transaction to fail
> even after it has started several jobs because they begin operating
> during the prepare phase.
> 
> Then, because the transaction preparation has failed, QEMU will cancel
> the transaction instead of proceeding, which will generate some
> BLOCK_JOB_CANCELLED events.
> 
> This may affect libvirt and others if I change these semantics.
> 
> Does that sound appropriate to you, or would you from a libvirt
> perspective RATHER get events for "uncreated" jobs like you do now? I
> could make it do either, but I'd rather prefer to simply not emit events
> for jobs that didn't truly never start.

Avoiding events for jobs that never start makes sense if the transaction
itself returns failure.

> 
> I can also begin emitting events for jobs that *actually* start if it
> would help to disambiguate the cases between old and new transactions.
> 
> Note: This has nothing to do with the transactional-cancel property,
> which only impacts what happens when jobs created by a transaction fail
> AFTER a successful return from qmp_transaction.

Ah, so it sounds like this is ALL about the synchronous case (that is,
the old behavior was that _even_ when 'transaction' fails, some events
were possibly leaked for < N portions of the transaction, if the earlier
portions were started to the point that they could emit a cancelled
event as a result of the later portions never starting).  So I think
your idea makes sense.

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

Re: [Qemu-block] [PATCH v14 10/21] qapi: permit auto-creating nested structs

[Eric Blake]

On 09/30/2016 09:45 AM, Daniel P. Berrange wrote:
> Some of the historical command line opts that had their
> keys in in a completely flat namespace are now represented
> by QAPI schemas that use a nested structs. When converting

singular/plural mismatch; either s/a // or s/structs/struct/

> the QemuOpts to QObject, there is no information about
> compound types available, so the QObject will be completely
> flat, even after the qdict_crumple() call. So when starting
> a struct, we may not have a QDict available in the input
> data, so we auto-create a QDict containing all the currently
> unvisited input data keys. Not all historical command line
> opts require this, so the behaviour is opt-in, by specifying
> how many levels of structs are permitted to be auto-created.
> 
> Note that this only works if the child struct is the last
> field to the visited in the parent struct. This is always
> the case for currently existing legacy command line options.
> 
> The example is the NetLegacy type which has 3 levels of
> structs. The modern way to represent this in QemuOpts would
> be the dot-separated component approach
> 
>   -net vlan=1,id=foo,name=bar,opts.type=tap,\
>opts.data.fd=3,opts.data.script=ifup
> 
> The legacy syntax will just be presenting
> 
>   -net vlan=1,id=foo,name=bar,type=tap,fd=3,script=ifup

So basically, any key=values that were not consumed as known keys in the
parent struct are reparsed as the (presumably lone) remaining QDict
child, recursing as needed to follow the QAPI nesting.

Does this still work if the command line is presented in a different order:

-net script=ifup,vlan=1,type=tap,fd=3,id=foo,name=bar

My guess is yes, but if the testsuite doesn't cover it, you may want to
add a test.

> 
> So we need to auto-create 3 levels of struct when visiting.
> 
> The implementation here will enable visiting in both the
> modern and legacy syntax, compared to OptsVisitor which
> only allows the legacy syntax.
> 
> Signed-off-by: Daniel P. Berrange 
> ---
>  include/qapi/qobject-input-visitor.h |  22 +-
>  qapi/qobject-input-visitor.c |  59 ++--
>  tests/test-qobject-input-visitor.c   | 130 
> ---
>  3 files changed, 194 insertions(+), 17 deletions(-)
> 
> diff --git a/include/qapi/qobject-input-visitor.h 
> b/include/qapi/qobject-input-visitor.h
> index 1809f48..94051f3 100644
> --- a/include/qapi/qobject-input-visitor.h
> +++ b/include/qapi/qobject-input-visitor.h
> @@ -45,7 +45,7 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool 
> strict);
>   * If @autocreate_list is true, then as an alternative to a normal QList,
>   * list values can be stored as a QString or QDict instead, which will
>   * be interpreted as representing single element lists. This should only
> - * by used if compatibility is required with the OptsVisitor which allowed
> + * be used if compatibility is required with the OptsVisitor which allowed

This typo fix should be hoisted earlier in the series when it was
introduced.

>   * repeated keys, without list indexes, to represent lists. e.g. set this
>   * to true if you have compatibility requirements for
>   *
> @@ -55,6 +55,22 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool 
> strict);
>   *
>   *   -arg foo.0=hello,foo.1=world
>   *
> + * If @autocreate_struct_levels is non-zero, then when visiting structs,
> + * if the corresponding QDict is not found, it will automatically create
> + * a QDict containing all remaining unvisited options. This should only
> + * be used if compatibility is required with the OptsVisitor which flatten
> + * structs so that all keys were at the same level. e.g. set this to a
> + * non-zero number if you compatibility requirements for
> + *
> + *   -arg type=person,surname=blogs,forename=fred
> + *
> + * to be treated as equivalent to the perferred syntax

s/perferred/preferred/


> +static void test_visitor_in_struct_autocreate(TestInputVisitorData *data,
> +  const void *unused)
> +{
> +Visitor *v;
> +int64_t vlan;
> +char *id = NULL;
> +char *type;
> +int64_t fd;
> +char *script = NULL;
> +
> +v = visitor_input_test_init_internal(
> +data, true, true, false, 3,
> +"{ 'vlan': '1', 'id': 'foo', 'type': 'tap', 'fd': '3', "
> +"'script': 'ifup' }", NULL);
> +

This is where intentionally mixing up the parameter orders might be
worthwhile.

Overall, the idea looks reasonable.

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

Re: [Qemu-block] [PATCH v14 09/21] qapi: permit auto-creating single element lists

[Eric Blake]

On 09/30/2016 09:45 AM, Daniel P. Berrange wrote:
> When converting QemuOpts to a QObject, there is no information
> about compound types available, so when visiting a list, the
> corresponding QObject is not guaranteed to be a QList. We
> therefore need to be able to auto-create a single element QList
> from whatever type we find.
> 
> This mode should only be enabled if you have compatibility
> requirements for
> 
>-arg foo=hello,foo=world
> 
> to be treated as equivalent to the preferred syntax:
> 
>-arg foo.0=hello,foo.1=world
> 
> Signed-off-by: Daniel P. Berrange 
> ---
>  include/qapi/qobject-input-visitor.h | 20 +++-
>  qapi/qobject-input-visitor.c | 27 +--
>  tests/test-qobject-input-visitor.c   | 88 
> +++-
>  3 files changed, 117 insertions(+), 18 deletions(-)
> 

Reviewed-by: Eric Blake 

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

Re: [Qemu-block] [PATCH v14 08/21] qapi: allow QObjectInputVisitor to be created with QemuOpts

[Eric Blake]

On 09/30/2016 09:45 AM, Daniel P. Berrange wrote:
> Instead of requiring all callers to go through the mutli-step
> process of turning QemuOpts into a suitable QObject for visiting,
> add a new constructor that encapsulates this logic. This will
> allow QObjectInputVisitor to be a drop-in replacement for the
> existing OptsVisitor with minimal code changes for callers.
> 
> NB, at this point it is only supporting opts syntax which
> explicitly matches the QAPI schema structure, so is not yet
> a true drop-in replacement for OptsVisitor. The patches that
> follow will add the special cases requird for full backwards

s/requird/required/

> compatibility with OptsVisitor.
> 
> Signed-off-by: Daniel P. Berrange 
> ---

> +++ b/include/qemu/option.h
> @@ -125,7 +125,7 @@ void qemu_opts_set_defaults(QemuOptsList *list, const 
> char *params,
>  int permit_abbrev);
>  QemuOpts *qemu_opts_from_qdict(QemuOptsList *list, const QDict *qdict,
> Error **errp);
> -QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict);
> +QDict *qemu_opts_to_qdict(const QemuOpts *opts, QDict *qdict);
>  void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, Error **errp);

Should the const-correctness of this parameter be hoisted to any earlier
patch?  But here is fine, as the first place where the compiler requires it.

Reviewed-by: Eric Blake 

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

Re: [Qemu-block] [PATCH v14 06/21] qapi: don't pass two copies of TestInputVisitorData to tests

[Eric Blake]

On 09/30/2016 09:45 AM, Daniel P. Berrange wrote:
> The input_visitor_test_add() method was accepting an instance
> of 'TestInputVisitorData' and passing it as the 'user_data'
> parameter to test functions. The main 'TestInputVisitorData'
> instance that was actually used, was meanwhile being allocated
> automatically by the test framework fixture setup.
> 
> The 'user_data' parameter is going to be needed for tests
> added in later patches, so getting rid of the current mistaken
> usage now allows this.
> 
> Signed-off-by: Daniel P. Berrange 
> ---
>  tests/test-qobject-input-visitor.c | 76 
> --
>  1 file changed, 32 insertions(+), 44 deletions(-)

Reviewed-by: Eric Blake 

The improved commit message makes the difference :)

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

Re: [Qemu-block] [PATCH v4 01/12] block/nbd: Drop trailing "." in error messages

[Eric Blake]

On 09/28/2016 03:55 PM, Max Reitz wrote:
> Signed-off-by: Max Reitz 
> ---
>  block/nbd.c   | 4 ++--
>  tests/qemu-iotests/051.out| 4 ++--
>  tests/qemu-iotests/051.pc.out | 4 ++--
>  3 files changed, 6 insertions(+), 6 deletions(-)
> 

Reviewed-by: Eric Blake 

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

[Qemu-block] backup notifier fail policy

[Vladimir Sementsov-Ogievskiy]

Hi all!

Please, can somebody explain me, why we fail guest request in case of io 
error in write notifier? I think guest consistency is more important 
than success of unfinished backup. Or, what am I missing?


I'm saying about this code:

static int coroutine_fn backup_before_write_notify(
NotifierWithReturn *notifier,
void *opaque)
{
BackupBlockJob *job = container_of(notifier, BackupBlockJob, 
before_write);

BdrvTrackedRequest *req = opaque;
int64_t sector_num = req->offset >> BDRV_SECTOR_BITS;
int nb_sectors = req->bytes >> BDRV_SECTOR_BITS;

assert(req->bs == blk_bs(job->common.blk));
assert((req->offset & (BDRV_SECTOR_SIZE - 1)) == 0);
assert((req->bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

return backup_do_cow(job, sector_num, nb_sectors, NULL, true);
}

So, what about something like

ret = backup_do_cow(job, ...
if (ret < 0 && job->notif_ret == 0) {
   job->notif_ret = ret;
}

return 0;

and fail block job if notif_ret < 0 in other places of backup code?

--
Best regards,
Vladimir

[Qemu-block] Transactions, Jobs, and Cancellation

[John Snow]

Hi Eric (as a proxy for Libvirt);

I want to make a change to transactions such that they do not actually 
start the jobs until the entire transaction is error-checked for validity.


This would be a change from the current setup where:

- Some jobs are started
- One job cannot start
- Existing jobs are cancelled, emitting job events
- QMP transaction returns failure.

to something more like:

- Some jobs are queued to start
- One job cannot start
- Existing queued jobs are un-created
- No events are emitted, but the QMP transaction fails.

If I understand correctly, it's possible for the transaction to fail 
even after it has started several jobs because they begin operating 
during the prepare phase.


Then, because the transaction preparation has failed, QEMU will cancel 
the transaction instead of proceeding, which will generate some 
BLOCK_JOB_CANCELLED events.


This may affect libvirt and others if I change these semantics.

Does that sound appropriate to you, or would you from a libvirt 
perspective RATHER get events for "uncreated" jobs like you do now? I 
could make it do either, but I'd rather prefer to simply not emit events 
for jobs that didn't truly never start.


I can also begin emitting events for jobs that *actually* start if it 
would help to disambiguate the cases between old and new transactions.


Note: This has nothing to do with the transactional-cancel property, 
which only impacts what happens when jobs created by a transaction fail 
AFTER a successful return from qmp_transaction.


--js

Re: [Qemu-block] [PATCH v14 03/21] qapi: add trace events for visitor

[Eric Blake]

On 09/30/2016 09:45 AM, Daniel P. Berrange wrote:
> Allow tracing of the operation of visitors
> 
> Signed-off-by: Daniel P. Berrange 
> ---
>  Makefile.objs  |  1 +
>  qapi/qapi-visit-core.c | 27 +++
>  qapi/trace-events  | 33 +
>  3 files changed, 61 insertions(+)
>  create mode 100644 qapi/trace-events

Reviewed-by: Eric Blake 

> +
> +visit_start_alternate(void *v, const char *name, void *obj, size_t size, 
> bool promote_int) "v=%p name=%s obj=%p size=%zu promote_int=%d"

Unrelated to your patch, but should the trace engines be taught how to
honor line wrapping in the trace files?

-- 
Eric Blake   eblake redhat com+1-919-301-3266
Libvirt virtualization library http://libvirt.org



signature.asc
Description: OpenPGP digital signature

Re: [Qemu-block] [Qemu-devel] [PATCH v14 00/21] QAPI/QOM work for non-scalar object properties

[no-reply]

Hi,

Your series failed automatic build test. Please find the testing commands and
their output below. If you have docker installed, you can probably reproduce it
locally.

Type: series
Message-id: 1475246744-29302-1-git-send-email-berra...@redhat.com
Subject: [Qemu-devel] [PATCH v14 00/21] QAPI/QOM work for non-scalar object 
properties

=== TEST SCRIPT BEGIN ===
#!/bin/bash
set -e
git submodule update --init dtc
# Let docker tests dump environment info
export SHOW_ENV=1
make J=8 docker-test-quick@centos6
make J=8 docker-test-mingw@fedora
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
From https://github.com/patchew-project/qemu
 - [tag update]  
patchew/1475202478-20716-1-git-send-email-zhangchen.f...@cn.fujitsu.com -> 
patchew/1475202478-20716-1-git-send-email-zhangchen.f...@cn.fujitsu.com
Switched to a new branch 'test'
9b89b2d qapi: delete unused OptsVisitor code
7d7d027 net: convert to QObjectInputVisitor for -net/-netdev parsing
f7fd6c1 acpi: convert to QObjectInputVisitor for -acpi parsing
fecaabf block: convert crypto driver to use QObjectInputVisitor
7941b3f numa: convert to use QObjectInputVisitor for -numa
3c6762d hmp: support non-scalar properties with object_add
efb01de qom: support non-scalar properties with -object
3c90b78 qapi: allow repeated opts with qobject_input_visitor_new_opts
81f728f qdict: allow qdict_crumple to accept compound types as values
8353c22 option: allow qemu_opts_to_qdict to merge repeated options
0948fcd qapi: add integer range support for QObjectInputVisitor
6da7117 qapi: permit auto-creating nested structs
1170f4b qapi: permit auto-creating single element lists
2ec3eac qapi: allow QObjectInputVisitor to be created with QemuOpts
b56363f qapi: permit scalar type conversions in QObjectInputVisitor
6fa7850 qapi: don't pass two copies of TestInputVisitorData to tests
d1b24d3 qapi: rename QmpOutputVisitor to QObjectOutputVisitor
794fec6 qapi: rename QmpInputVisitor to QObjectInputVisitor
43b5ede qapi: add trace events for visitor
587d218 qdict: implement a qdict_crumple method for un-flattening a dict
67cbb95 option: make parse_option_bool/number non-static

=== OUTPUT BEGIN ===
Submodule 'dtc' (git://git.qemu-project.org/dtc.git) registered for path 'dtc'
Cloning into 'dtc'...
Submodule path 'dtc': checked out '65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf'
  BUILD centos6
  ARCHIVE qemu.tgz
  ARCHIVE dtc.tgz
  COPY RUNNER
  RUN test-quick in centos6
Packages installed:
SDL-devel-1.2.14-7.el6_7.1.x86_64
ccache-3.1.6-2.el6.x86_64
epel-release-6-8.noarch
gcc-4.4.7-17.el6.x86_64
git-1.7.1-4.el6_7.1.x86_64
glib2-devel-2.28.8-5.el6.x86_64
libfdt-devel-1.4.0-1.el6.x86_64
make-3.81-23.el6.x86_64
package g++ is not installed
pixman-devel-0.32.8-1.el6.x86_64
tar-1.23-15.el6_8.x86_64
zlib-devel-1.2.3-29.el6.x86_64

Environment variables:
PACKAGES=libfdt-devel ccache tar git make gcc g++ zlib-devel 
glib2-devel SDL-devel pixman-devel epel-release
HOSTNAME=9320e4514cd6
TERM=xterm
MAKEFLAGS= -j8
HISTSIZE=1000
J=8
USER=root
CCACHE_DIR=/var/tmp/ccache
EXTRA_CONFIGURE_OPTS=
V=
SHOW_ENV=1
MAIL=/var/spool/mail/root
PATH=/usr/lib/ccache:/usr/lib64/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
PWD=/
LANG=en_US.UTF-8
TARGET_LIST=
HISTCONTROL=ignoredups
SHLVL=1
HOME=/root
TEST_DIR=/tmp/qemu-test
LOGNAME=root
LESSOPEN=||/usr/bin/lesspipe.sh %s
FEATURES= dtc
DEBUG=
G_BROKEN_FILENAMES=1
CCACHE_HASHDIR=
_=/usr/bin/env

Configure options:
--enable-werror --target-list=x86_64-softmmu,aarch64-softmmu 
--prefix=/tmp/qemu-test/src/tests/docker/install
No C++ compiler available; disabling C++ specific optional code
Install prefix/tmp/qemu-test/src/tests/docker/install
BIOS directory/tmp/qemu-test/src/tests/docker/install/share/qemu
binary directory  /tmp/qemu-test/src/tests/docker/install/bin
library directory /tmp/qemu-test/src/tests/docker/install/lib
module directory  /tmp/qemu-test/src/tests/docker/install/lib/qemu
libexec directory /tmp/qemu-test/src/tests/docker/install/libexec
include directory /tmp/qemu-test/src/tests/docker/install/include
config directory  /tmp/qemu-test/src/tests/docker/install/etc
local state directory   /tmp/qemu-test/src/tests/docker/install/var
Manual directory  /tmp/qemu-test/src/tests/docker/install/share/man
ELF interp prefix /usr/gnemul/qemu-%M
Source path   /tmp/qemu-test/src
C compilercc
Host C compiler   cc
C++ compiler  
Objective-C compiler cc
ARFLAGS   rv
CFLAGS-O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -g 
QEMU_CFLAGS   -I/usr/include/pixman-1-pthread -I/usr/include/glib-2.0 
-I/usr/lib64/glib-2.0/include   -fPIE -DPIE -m64 -D_GNU_SOURCE 
-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -Wstrict-prototypes 
-Wredundant-decls -Wall -Wundef -Wwrite-strings -Wmissing-prototypes 
-fno-strict-aliasing -fno-common -fwrapv  -Wendif-labels -Wmissing-include-dirs 
-Wempty-body -Wnested-externs -Wformat-security -Wformat-y2k -Winit-self 
-Wignored-qualifiers 

Re: [Qemu-block] [PATCH v5] qemu-img: change opening method for the output in dd

[Stefan Hajnoczi]

On Tue, Sep 13, 2016 at 4:35 PM, Stefan Hajnoczi  wrote:
> On Fri, Aug 26, 2016 at 09:19:40AM +0200, Reda Sallahi wrote:
>> -blk2 = img_open(image_opts, out.filename, out_fmt, BDRV_O_RDWR,
>> -false, false);
>> +ret = access(out_filename, F_OK); /* Check if file exists */
>
> The QEMU block layer does not use POSIX file I/O because it needs to
> support protocols like iSCSI and ssh.
>
> The only way to test for the existence of a disk image is by opening it
> with block layer APIs.
>
> Can you just move the img_open() call from the else statement up here
> instead?

Ping

[Qemu-block] [PATCH v14 21/21] qapi: delete unused OptsVisitor code

[Daniel P. Berrange]

Now that all code has been converted to QObjectInputVisitor's
QemuOpts compatibility mode, there is no longer any reason
to keep OptsVisitor alive.

Signed-off-by: Daniel P. Berrange 
---
 include/qapi/opts-visitor.h |  40 
 qapi/Makefile.objs  |   2 +-
 qapi/opts-visitor.c | 544 
 tests/Makefile.include  |   5 +-
 tests/test-opts-visitor.c   | 268 --
 vl.c|   1 -
 6 files changed, 2 insertions(+), 858 deletions(-)
 delete mode 100644 include/qapi/opts-visitor.h
 delete mode 100644 qapi/opts-visitor.c
 delete mode 100644 tests/test-opts-visitor.c

diff --git a/include/qapi/opts-visitor.h b/include/qapi/opts-visitor.h
deleted file mode 100644
index 6462c96..000
--- a/include/qapi/opts-visitor.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Options Visitor
- *
- * Copyright Red Hat, Inc. 2012
- *
- * Author: Laszlo Ersek 
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef OPTS_VISITOR_H
-#define OPTS_VISITOR_H
-
-#include "qapi/visitor.h"
-#include "qemu/option.h"
-
-/* Inclusive upper bound on the size of any flattened range. This is a safety
- * (= anti-annoyance) measure; wrong ranges should not cause long startup
- * delays nor exhaust virtual memory.
- */
-#define OPTS_VISITOR_RANGE_MAX 65536
-
-typedef struct OptsVisitor OptsVisitor;
-
-/* Contrarily to qemu-option.c::parse_option_number(), OptsVisitor's "int"
- * parser relies on strtoll() instead of strtoull(). Consequences:
- * - string representations of negative numbers yield negative values,
- * - values below INT64_MIN or LLONG_MIN are rejected,
- * - values above INT64_MAX or LLONG_MAX are rejected.
- *
- * The Opts input visitor does not implement support for visiting QAPI
- * alternates, numbers (other than integers), null, or arbitrary
- * QTypes.  It also requires a non-null list argument to
- * visit_start_list().
- */
-Visitor *opts_visitor_new(const QemuOpts *opts);
-
-#endif
diff --git a/qapi/Makefile.objs b/qapi/Makefile.objs
index 33906ff..4b820a7 100644
--- a/qapi/Makefile.objs
+++ b/qapi/Makefile.objs
@@ -1,6 +1,6 @@
 util-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qobject-input-visitor.o
 util-obj-y += qobject-output-visitor.o qmp-registry.o qmp-dispatch.o
 util-obj-y += string-input-visitor.o string-output-visitor.o
-util-obj-y += opts-visitor.o qapi-clone-visitor.o
+util-obj-y += qapi-clone-visitor.o
 util-obj-y += qmp-event.o
 util-obj-y += qapi-util.o
diff --git a/qapi/opts-visitor.c b/qapi/opts-visitor.c
deleted file mode 100644
index 084f7cc..000
--- a/qapi/opts-visitor.c
+++ /dev/null
@@ -1,544 +0,0 @@
-/*
- * Options Visitor
- *
- * Copyright Red Hat, Inc. 2012-2016
- *
- * Author: Laszlo Ersek 
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#include "qemu/osdep.h"
-#include "qapi/error.h"
-#include "qemu/cutils.h"
-#include "qapi/qmp/qerror.h"
-#include "qapi/opts-visitor.h"
-#include "qemu/queue.h"
-#include "qemu/option_int.h"
-#include "qapi/visitor-impl.h"
-
-
-enum ListMode
-{
-LM_NONE, /* not traversing a list of repeated options */
-
-LM_IN_PROGRESS,  /* opts_next_list() ready to be called.
-  *
-  * Generating the next list link will consume the most
-  * recently parsed QemuOpt instance of the repeated
-  * option.
-  *
-  * Parsing a value into the list link will examine the
-  * next QemuOpt instance of the repeated option, and
-  * possibly enter LM_SIGNED_INTERVAL or
-  * LM_UNSIGNED_INTERVAL.
-  */
-
-LM_SIGNED_INTERVAL,  /* opts_next_list() has been called.
-  *
-  * Generating the next list link will consume the most
-  * recently stored element from the signed interval,
-  * parsed from the most recent QemuOpt instance of the
-  * repeated option. This may consume QemuOpt itself
-  * and return to LM_IN_PROGRESS.
-  *
-  * Parsing a value into the list link will store the
-  * next element of the signed interval.
-  */
-
-LM_UNSIGNED_INTERVAL /* Same as above, only for an unsigned interval. */
-};
-
-typedef enum ListMode ListMode;
-
-struct OptsVisitor
-{
-Visitor visitor;
-
-/* Ownership remains with opts_visitor_new()'s caller. */
-const QemuOpts *opts_root;
-
-unsigned depth;
-
-/* Non-null 

[Qemu-block] [PATCH v14 19/21] acpi: convert to QObjectInputVisitor for -acpi parsing

[Daniel P. Berrange]

The -acpi command line option parsing uses the OptsVisitor
currently. This is easily replaced by the QObjectInputVisitor
instead. There is no need to enable any of the compatibility
options, since the AcpiTableOptions QAPI struct only contains
scalar properties.

Signed-off-by: Daniel P. Berrange 
---
 hw/acpi/core.c | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/hw/acpi/core.c b/hw/acpi/core.c
index e890a5d..67c0abe 100644
--- a/hw/acpi/core.c
+++ b/hw/acpi/core.c
@@ -25,7 +25,7 @@
 #include "hw/acpi/acpi.h"
 #include "hw/nvram/fw_cfg.h"
 #include "qemu/config-file.h"
-#include "qapi/opts-visitor.h"
+#include "qapi/qobject-input-visitor.h"
 #include "qapi-visit.h"
 #include "qapi-event.h"
 
@@ -237,14 +237,14 @@ void acpi_table_add(const QemuOpts *opts, Error **errp)
 char **cur;
 size_t bloblen = 0;
 char unsigned *blob = NULL;
+Visitor *v;
 
-{
-Visitor *v;
-
-v = opts_visitor_new(opts);
-visit_type_AcpiTableOptions(v, NULL, , );
-visit_free(v);
+v = qobject_input_visitor_new_opts(opts, false, 0, false, false, );
+if (err) {
+goto out;
 }
+visit_type_AcpiTableOptions(v, NULL, , );
+visit_free(v);
 
 if (err) {
 goto out;
-- 
2.7.4

[Qemu-block] [PATCH v14 18/21] block: convert crypto driver to use QObjectInputVisitor

[Daniel P. Berrange]

The crypto block driver currently uses OptsVisitor to
convert from the block driver open/create options into
QCryptoBlockOpenOptions/QCryptoBlockCreateOptions. This
is easily replaced by use of QObjectInputVisitor with
no need to enable any compatibility options, since the
structs dealt with contain only scalars.

Signed-off-by: Daniel P. Berrange 
---
 block/crypto.c | 14 +++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/block/crypto.c b/block/crypto.c
index 7aa7eb5..12b0b9f 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -23,7 +23,7 @@
 #include "block/block_int.h"
 #include "sysemu/block-backend.h"
 #include "crypto/block.h"
-#include "qapi/opts-visitor.h"
+#include "qapi/qobject-input-visitor.h"
 #include "qapi-visit.h"
 #include "qapi/error.h"
 
@@ -206,7 +206,11 @@ block_crypto_open_opts_init(QCryptoBlockFormat format,
 ret = g_new0(QCryptoBlockOpenOptions, 1);
 ret->format = format;
 
-v = opts_visitor_new(opts);
+v = qobject_input_visitor_new_opts(opts, false, 0, false, false,
+   _err);
+if (local_err) {
+goto out;
+}
 
 visit_start_struct(v, NULL, NULL, 0, _err);
 if (local_err) {
@@ -252,7 +256,11 @@ block_crypto_create_opts_init(QCryptoBlockFormat format,
 ret = g_new0(QCryptoBlockCreateOptions, 1);
 ret->format = format;
 
-v = opts_visitor_new(opts);
+v = qobject_input_visitor_new_opts(opts, false, 0, false, false,
+   _err);
+if (local_err) {
+goto out;
+}
 
 visit_start_struct(v, NULL, NULL, 0, _err);
 if (local_err) {
-- 
2.7.4

[Qemu-block] [PATCH v14 17/21] numa: convert to use QObjectInputVisitor for -numa

[Daniel P. Berrange]

Switch away from using OptsVisitor to parse the -numa
argument processing. This enables use of the modern
list syntax for specifying CPUs. e.g. the old syntax

  -numa node,nodeid=0,cpus=0-3,cpus=8-11,mem=107

is equivalent to

  -numa node,nodeid=0,cpus.0=0,cpus.1=1,cpus.2=2,cpus.3=3,\
cpus.4=8,cpus.5=9,cpus.6=10,cpus.7=11,mem=107

Furthermore, the cli arg can now follow the QAPI schema
nesting, so the above is equivalent to the canonical
syntax:

  -numa type=node,data.nodeid=0,data.cpus.0=0,data.cpus.1=1,\
data.cpus.2=2,data.cpus.3=3,data.cpus.4=8,data.cpus.5=9,\
data.cpus.6=10,data.cpus.7=11,data.mem=107

A test case is added to cover argument parsing to validate
that both the old and new syntax is correctly handled.

Signed-off-by: Daniel P. Berrange 
---
 include/sysemu/numa_int.h |  11 +
 numa.c|  36 +-
 stubs/Makefile.objs   |   5 ++
 stubs/exec.c  |   6 +++
 stubs/hostmem.c   |  14 ++
 stubs/memory.c|  41 
 stubs/qdev.c  |   8 
 stubs/vl.c|   8 
 stubs/vmstate.c   |   4 ++
 tests/Makefile.include|   2 +
 tests/test-numa.c | 116 ++
 11 files changed, 240 insertions(+), 11 deletions(-)
 create mode 100644 include/sysemu/numa_int.h
 create mode 100644 stubs/exec.c
 create mode 100644 stubs/hostmem.c
 create mode 100644 stubs/memory.c
 create mode 100644 stubs/qdev.c
 create mode 100644 stubs/vl.c
 create mode 100644 tests/test-numa.c

diff --git a/include/sysemu/numa_int.h b/include/sysemu/numa_int.h
new file mode 100644
index 000..93160da
--- /dev/null
+++ b/include/sysemu/numa_int.h
@@ -0,0 +1,11 @@
+#ifndef SYSEMU_NUMA_INT_H
+#define SYSEMU_NUMA_INT_H
+
+#include "sysemu/numa.h"
+
+extern int have_memdevs;
+extern int max_numa_nodeid;
+
+int parse_numa(void *opaque, QemuOpts *opts, Error **errp);
+
+#endif
diff --git a/numa.c b/numa.c
index 6289f46..653ebf1 100644
--- a/numa.c
+++ b/numa.c
@@ -23,14 +23,14 @@
  */
 
 #include "qemu/osdep.h"
-#include "sysemu/numa.h"
+#include "sysemu/numa_int.h"
 #include "exec/cpu-common.h"
 #include "qemu/bitmap.h"
 #include "qom/cpu.h"
 #include "qemu/error-report.h"
 #include "include/exec/cpu-common.h" /* for RAM_ADDR_FMT */
 #include "qapi-visit.h"
-#include "qapi/opts-visitor.h"
+#include "qapi/qobject-input-visitor.h"
 #include "hw/boards.h"
 #include "sysemu/hostmem.h"
 #include "qmp-commands.h"
@@ -45,10 +45,10 @@ QemuOptsList qemu_numa_opts = {
 .desc = { { 0 } } /* validated with OptsVisitor */
 };
 
-static int have_memdevs = -1;
-static int max_numa_nodeid; /* Highest specified NUMA node ID, plus one.
- * For all nodes, nodeid < max_numa_nodeid
- */
+int have_memdevs = -1;
+int max_numa_nodeid; /* Highest specified NUMA node ID, plus one.
+  * For all nodes, nodeid < max_numa_nodeid
+  */
 int nb_numa_nodes;
 NodeInfo numa_info[MAX_NODES];
 
@@ -189,6 +189,9 @@ static void numa_node_parse(NumaNodeOptions *node, QemuOpts 
*opts, Error **errp)
 if (node->has_mem) {
 uint64_t mem_size = node->mem;
 const char *mem_str = qemu_opt_get(opts, "mem");
+if (!mem_str) {
+mem_str = qemu_opt_get(opts, "data.mem");
+}
 /* Fix up legacy suffix-less format */
 if (g_ascii_isdigit(mem_str[strlen(mem_str) - 1])) {
 mem_size <<= 20;
@@ -211,16 +214,27 @@ static void numa_node_parse(NumaNodeOptions *node, 
QemuOpts *opts, Error **errp)
 max_numa_nodeid = MAX(max_numa_nodeid, nodenr + 1);
 }
 
-static int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
+int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
 {
 NumaOptions *object = NULL;
 Error *err = NULL;
+Visitor *v;
 
-{
-Visitor *v = opts_visitor_new(opts);
-visit_type_NumaOptions(v, NULL, , );
-visit_free(v);
+/*
+ * Needs autocreate_list=true, permit_int_ranges=true and
+ * permit_repeated_opts=true in order to support existing
+ * syntax for 'cpus' parameter which is an int list.
+ *
+ * Needs autocreate_struct_levels=1, because existing syntax
+ * used a nested struct in the QMP schema with a flat namespace
+ * in the CLI args.
+ */
+v = qobject_input_visitor_new_opts(opts, true, 1, true, true, );
+if (err) {
+goto end;
 }
+visit_type_NumaOptions(v, NULL, , );
+visit_free(v);
 
 if (err) {
 goto end;
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
index c5850e8..661b48a 100644
--- a/stubs/Makefile.objs
+++ b/stubs/Makefile.objs
@@ -48,3 +48,8 @@ stub-obj-y += iohandler.o
 stub-obj-y += smbios_type_38.o
 stub-obj-y += ipmi.o
 stub-obj-y += pc_madt_cpu_entry.o
+stub-obj-y += vl.o
+stub-obj-y += exec.o
+stub-obj-y += memory.o
+stub-obj-y += hostmem.o
+stub-obj-y += 

[Qemu-block] [PATCH v14 15/21] qom: support non-scalar properties with -object

[Daniel P. Berrange]

The current -object command line syntax only allows for
creation of objects with scalar properties, or a list
with a fixed scalar element type. Objects which have
properties that are represented as structs in the QAPI
schema cannot be created using -object.

This is a design limitation of the way the OptsVisitor
is written. It simply iterates over the QemuOpts values
as a flat list. The support for lists is enabled by
allowing the same key to be repeated in the opts string.

The QObjectInputVisitor now has functionality that allows
it to replace OptsVisitor, maintaining full backwards
compatibility for previous CLI syntax, while also allowing
use of new syntax for structs.

Thus -object can now support non-scalar properties,
for example the QMP object:

  {
"execute": "object-add",
"arguments": {
  "qom-type": "demo",
  "id": "demo0",
  "parameters": {
"foo": [
  { "bar": "one", "wizz": "1" },
  { "bar": "two", "wizz": "2" }
]
  }
}
  }

Would be creatable via the CLI now using

$QEMU \
  -object demo,id=demo0,\
  foo.0.bar=one,foo.0.wizz=1,\
  foo.1.bar=two,foo.1.wizz=2

Notice that this syntax is intentionally compatible
with that currently used by block drivers. NB the
indentation seen here after line continuations should
not be used in reality, it is just for clarity in this
commit message.

Signed-off-by: Daniel P. Berrange 
---
 qapi/qobject-input-visitor.c |   2 +-
 qom/object_interfaces.c  |  37 -
 tests/check-qom-proplist.c   | 367 ++-
 3 files changed, 397 insertions(+), 9 deletions(-)

diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c
index 5a3872c..f1030d5 100644
--- a/qapi/qobject-input-visitor.c
+++ b/qapi/qobject-input-visitor.c
@@ -204,7 +204,7 @@ static void qobject_input_start_struct(Visitor *v, const 
char *name, void **obj,
 *obj = NULL;
 }
 
-if (!qobj && (qiv->struct_level < qiv->autocreate_struct_levels)) {
+if (!qobj && (qiv->struct_level <= qiv->autocreate_struct_levels)) {
 /* Create a new dict that contains all the currently
  * unvisited items */
 if (tos) {
diff --git a/qom/object_interfaces.c b/qom/object_interfaces.c
index fdc406b..88a1e88 100644
--- a/qom/object_interfaces.c
+++ b/qom/object_interfaces.c
@@ -4,7 +4,8 @@
 #include "qemu/module.h"
 #include "qapi-visit.h"
 #include "qapi/qobject-output-visitor.h"
-#include "qapi/opts-visitor.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qemu/option.h"
 
 void user_creatable_complete(Object *obj, Error **errp)
 {
@@ -63,12 +64,16 @@ Object *user_creatable_add(const QDict *qdict,
 if (local_err) {
 goto out_visit;
 }
-visit_check_struct(v, _err);
+
+obj = user_creatable_add_type(type, id, pdict, v, _err);
 if (local_err) {
 goto out_visit;
 }
 
-obj = user_creatable_add_type(type, id, pdict, v, _err);
+visit_check_struct(v, _err);
+if (local_err) {
+goto out_visit;
+}
 
 out_visit:
 visit_end_struct(v, NULL);
@@ -114,7 +119,7 @@ Object *user_creatable_add_type(const char *type, const 
char *id,
 
 assert(qdict);
 obj = object_new(type);
-visit_start_struct(v, NULL, NULL, 0, _err);
+visit_start_struct(v, "props", NULL, 0, _err);
 if (local_err) {
 goto out;
 }
@@ -158,14 +163,32 @@ Object *user_creatable_add_opts(QemuOpts *opts, Error 
**errp)
 {
 Visitor *v;
 QDict *pdict;
+QObject *pobj;
 Object *obj = NULL;
 
-v = opts_visitor_new(opts);
-pdict = qemu_opts_to_qdict(opts, NULL, QEMU_OPTS_REPEAT_POLICY_LAST,
+pdict = qemu_opts_to_qdict(opts, NULL,
+   QEMU_OPTS_REPEAT_POLICY_ALL,
_abort);
 
-obj = user_creatable_add(pdict, v, errp);
+pobj = qdict_crumple(pdict, true, errp);
+if (!pobj) {
+goto cleanup;
+}
+/*
+ * We need autocreate_list=true + permit_int_ranges=true
+ * in order to maintain compat with OptsVisitor creation
+ * of the 'numa' object which had an int16List property.
+ *
+ * We need autocreate_structs=1, because at the CLI level
+ * we have the object type + properties in the same flat
+ * struct, even though at the QMP level they are nested.
+ */
+v = qobject_input_visitor_new_autocast(pobj, true, 1, true);
+
+obj = user_creatable_add(qobject_to_qdict(pobj), v, errp);
 visit_free(v);
+qobject_decref(pobj);
+ cleanup:
 QDECREF(pdict);
 return obj;
 }
diff --git a/tests/check-qom-proplist.c b/tests/check-qom-proplist.c
index a16cefc..20eb1d1 100644
--- a/tests/check-qom-proplist.c
+++ b/tests/check-qom-proplist.c
@@ -22,7 +22,16 @@
 
 #include "qapi/error.h"
 #include "qom/object.h"
+#include "qom/object_interfaces.h"
 #include "qemu/module.h"
+#include "qapi/visitor.h"
+#include "qom/object_interfaces.h"
+#include 

[Qemu-block] [PATCH v14 14/21] qapi: allow repeated opts with qobject_input_visitor_new_opts

[Daniel P. Berrange]

The qobject_input_visitor_new_opts() method gains a new
parameter to control whether it allows repeated option
keys in the input QemuOpts or not.

Signed-off-by: Daniel P. Berrange 
---
 include/qapi/qobject-input-visitor.h | 6 ++
 qapi/qobject-input-visitor.c | 5 -
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/include/qapi/qobject-input-visitor.h 
b/include/qapi/qobject-input-visitor.h
index 242b767..bc5062a 100644
--- a/include/qapi/qobject-input-visitor.h
+++ b/include/qapi/qobject-input-visitor.h
@@ -112,6 +112,11 @@ Visitor *qobject_input_visitor_new_autocast(QObject *obj,
  * qobject_input_visitor_new_autocast() method. See the docs
  * of that method for further details on processing behaviour.
  *
+ * If the @permit_repeated_opts parameter is true, then the input
+ * @opts is allowed to contain repeated keys and they will be
+ * turned into a QList. If it is false, then repeated keys will
+ * result in an error being reported.
+ *
  * The returned input visitor should be released by calling
  * visit_free() when no longer required.
  */
@@ -119,6 +124,7 @@ Visitor *qobject_input_visitor_new_opts(const QemuOpts 
*opts,
 bool autocreate_list,
 size_t autocreate_struct_levels,
 bool permit_int_ranges,
+bool permit_repeated_opts,
 Error **errp);
 
 #endif
diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c
index 2287d11..5a3872c 100644
--- a/qapi/qobject-input-visitor.c
+++ b/qapi/qobject-input-visitor.c
@@ -753,6 +753,7 @@ Visitor *qobject_input_visitor_new_opts(const QemuOpts 
*opts,
 bool autocreate_list,
 size_t autocreate_struct_levels,
 bool permit_int_ranges,
+bool permit_repeated_opts,
 Error **errp)
 {
 QDict *pdict;
@@ -760,7 +761,9 @@ Visitor *qobject_input_visitor_new_opts(const QemuOpts 
*opts,
 Visitor *v = NULL;
 
 pdict = qemu_opts_to_qdict(opts, NULL,
-   QEMU_OPTS_REPEAT_POLICY_LAST,
+   permit_repeated_opts ?
+   QEMU_OPTS_REPEAT_POLICY_ALL :
+   QEMU_OPTS_REPEAT_POLICY_ERROR,
errp);
 if (!pdict) {
 goto cleanup;
-- 
2.7.4

[Qemu-block] [PATCH v14 20/21] net: convert to QObjectInputVisitor for -net/-netdev parsing

[Daniel P. Berrange]

The -net/-netdev command line parsing code uses OptsVisitor
for parsing options to populate NetLegacy or NetDev struct
respectively. Although those structs have nesting, the
OptsVisitor flattens them, so we must enable compatibility
options to auto-create structs. This allows the legacy
syntax

  -net tap,id=net0,vlan=3,fd=3,script=/bin/ifup-qemu

to be treated as equivalent to the modern QAPI based
syntax

  -net 
id=net0,vlan=3,opts.type=tap,opts.data.fd=3,opts.data.script=/bin/ifup-qemu

Signed-off-by: Daniel P. Berrange 
---
 net/net.c | 18 --
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/net/net.c b/net/net.c
index ec984bf..de6bf8e 100644
--- a/net/net.c
+++ b/net/net.c
@@ -43,7 +43,7 @@
 #include "qemu/iov.h"
 #include "qemu/main-loop.h"
 #include "qapi-visit.h"
-#include "qapi/opts-visitor.h"
+#include "qapi/qobject-input-visitor.h"
 #include "sysemu/sysemu.h"
 #include "net/filter.h"
 #include "qapi/string-output-visitor.h"
@@ -1069,7 +1069,21 @@ int net_client_init(QemuOpts *opts, bool is_netdev, 
Error **errp)
 void *object = NULL;
 Error *err = NULL;
 int ret = -1;
-Visitor *v = opts_visitor_new(opts);
+/*
+ * Needs autocreate_lists=true in order support existing
+ * syntax for list options where the bare key is repeated
+ *
+ * Needs autocreate_struct_levels=3 in order to deal with
+ * 3 level nesting in NetLegacy option args, which was
+ * exposed as a flat namespace with OptVisitor
+ */
+Visitor *v = qobject_input_visitor_new_opts(opts, true, 3, false, true,
+);
+
+if (err) {
+error_propagate(errp, err);
+return -1;
+}
 
 {
 /* Parse convenience option format ip6-net=fec0::0[/64] */
-- 
2.7.4

[Qemu-block] [PATCH v14 05/21] qapi: rename QmpOutputVisitor to QObjectOutputVisitor

[Daniel P. Berrange]

The QmpOutputVisitor has no direct dependency on QMP. It is
valid to use it anywhere that one wants a QObject. Rename it
to better reflect its functionality as a generic QAPI
to QObject converter.

Reviewed-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Signed-off-by: Daniel P. Berrange 
---
 block/qapi.c   |   4 +-
 blockdev.c |   4 +-
 docs/qapi-code-gen.txt |   2 +-
 ...p-output-visitor.h => qobject-output-visitor.h} |  10 +-
 qapi/Makefile.objs |   2 +-
 qapi/qapi-clone-visitor.c  |   2 +-
 qapi/qmp-output-visitor.c  | 256 -
 qapi/qobject-output-visitor.c  | 254 
 qemu-img.c |   8 +-
 qom/object_interfaces.c|   2 +-
 qom/qom-qobject.c  |   4 +-
 scripts/qapi-commands.py   |   4 +-
 scripts/qapi-event.py  |   4 +-
 tests/.gitignore   |   2 +-
 tests/Makefile.include |   8 +-
 tests/check-qnull.c|   4 +-
 ...put-visitor.c => test-qobject-output-visitor.c} |   6 +-
 tests/test-string-output-visitor.c |   2 +-
 tests/test-visitor-serialization.c |   4 +-
 util/qemu-sockets.c|   2 +-
 20 files changed, 291 insertions(+), 293 deletions(-)
 rename include/qapi/{qmp-output-visitor.h => qobject-output-visitor.h} (66%)
 delete mode 100644 qapi/qmp-output-visitor.c
 create mode 100644 qapi/qobject-output-visitor.c
 rename tests/{test-qmp-output-visitor.c => test-qobject-output-visitor.c} (99%)

diff --git a/block/qapi.c b/block/qapi.c
index 6f947e3..f35c89f 100644
--- a/block/qapi.c
+++ b/block/qapi.c
@@ -29,7 +29,7 @@
 #include "block/write-threshold.h"
 #include "qmp-commands.h"
 #include "qapi-visit.h"
-#include "qapi/qmp-output-visitor.h"
+#include "qapi/qobject-output-visitor.h"
 #include "qapi/qmp/types.h"
 #include "sysemu/block-backend.h"
 #include "qemu/cutils.h"
@@ -691,7 +691,7 @@ void bdrv_image_info_specific_dump(fprintf_function 
func_fprintf, void *f,
ImageInfoSpecific *info_spec)
 {
 QObject *obj, *data;
-Visitor *v = qmp_output_visitor_new();
+Visitor *v = qobject_output_visitor_new();
 
 visit_type_ImageInfoSpecific(v, NULL, _spec, _abort);
 visit_complete(v, );
diff --git a/blockdev.c b/blockdev.c
index 29c6561..814d49f 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -43,7 +43,7 @@
 #include "qapi/qmp/types.h"
 #include "qapi-visit.h"
 #include "qapi/qmp/qerror.h"
-#include "qapi/qmp-output-visitor.h"
+#include "qapi/qobject-output-visitor.h"
 #include "qapi/util.h"
 #include "sysemu/sysemu.h"
 #include "block/block_int.h"
@@ -3828,7 +3828,7 @@ void qmp_blockdev_add(BlockdevOptions *options, Error 
**errp)
 {
 BlockDriverState *bs;
 QObject *obj;
-Visitor *v = qmp_output_visitor_new();
+Visitor *v = qobject_output_visitor_new();
 QDict *qdict;
 Error *local_err = NULL;
 
diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt
index d2604b6..2841c51 100644
--- a/docs/qapi-code-gen.txt
+++ b/docs/qapi-code-gen.txt
@@ -1005,7 +1005,7 @@ Example:
 Error *err = NULL;
 Visitor *v;
 
-v = qmp_output_visitor_new(ret_out);
+v = qobject_output_visitor_new(ret_out);
 visit_type_UserDefOne(v, "unused", _in, );
 if (!err) {
 visit_complete(v, ret_out);
diff --git a/include/qapi/qmp-output-visitor.h 
b/include/qapi/qobject-output-visitor.h
similarity index 66%
rename from include/qapi/qmp-output-visitor.h
rename to include/qapi/qobject-output-visitor.h
index 040fdda..358c959 100644
--- a/include/qapi/qmp-output-visitor.h
+++ b/include/qapi/qobject-output-visitor.h
@@ -11,20 +11,20 @@
  *
  */
 
-#ifndef QMP_OUTPUT_VISITOR_H
-#define QMP_OUTPUT_VISITOR_H
+#ifndef QOBJECT_OUTPUT_VISITOR_H
+#define QOBJECT_OUTPUT_VISITOR_H
 
 #include "qapi/visitor.h"
 #include "qapi/qmp/qobject.h"
 
-typedef struct QmpOutputVisitor QmpOutputVisitor;
+typedef struct QObjectOutputVisitor QObjectOutputVisitor;
 
 /*
- * Create a new QMP output visitor.
+ * Create a new QOBJECT output visitor.
  *
  * If everything else succeeds, pass @result to visit_complete() to
  * collect the result of the visit.
  */
-Visitor *qmp_output_visitor_new(QObject **result);
+Visitor *qobject_output_visitor_new(QObject **result);
 
 #endif
diff --git a/qapi/Makefile.objs b/qapi/Makefile.objs
index 6ec7bdc..33906ff 100644
--- a/qapi/Makefile.objs
+++ b/qapi/Makefile.objs
@@ -1,5 +1,5 @@
 util-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qobject-input-visitor.o
-util-obj-y += qmp-output-visitor.o qmp-registry.o 

[Qemu-block] [PATCH v14 16/21] hmp: support non-scalar properties with object_add

[Daniel P. Berrange]

The current object_add HMP syntax only allows for
creation of objects with scalar properties, or a list
with a fixed scalar element type. Objects which have
properties that are represented as structs in the QAPI
schema cannot be created using -object.

This is a design limitation of the way the OptsVisitor
is written. It simply iterates over the QemuOpts values
as a flat list. The support for lists is enabled by
allowing the same key to be repeated in the opts string.

The QObjectInputVisitor now has functionality that allows
it to replace OptsVisitor, maintaining full backwards
compatibility for previous CLI syntax, while also allowing
use of new syntax for structs.

Thus -object can now support non-scalar properties,
for example the QMP object:

  {
"execute": "object-add",
"arguments": {
  "qom-type": "demo",
  "id": "demo0",
  "parameters": {
"foo": [
  { "bar": "one", "wizz": "1" },
  { "bar": "two", "wizz": "2" }
]
  }
}
  }

Would be creatable via the HMP now using

   object_add demo,id=demo0,\
  foo.0.bar=one,foo.0.wizz=1,\
  foo.1.bar=two,foo.1.wizz=2

Notice that this syntax is intentionally compatible
with that currently used by block drivers. NB the
indentation seen here after line continuations should
not be used in reality, it is just for clarity in this
commit message.

Signed-off-by: Daniel P. Berrange 
---
 hmp.c | 25 +
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/hmp.c b/hmp.c
index 336e7bf..b32d8c8 100644
--- a/hmp.c
+++ b/hmp.c
@@ -25,7 +25,7 @@
 #include "qemu/sockets.h"
 #include "monitor/monitor.h"
 #include "monitor/qdev.h"
-#include "qapi/opts-visitor.h"
+#include "qapi/qobject-input-visitor.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/string-output-visitor.h"
 #include "qapi/util.h"
@@ -1696,21 +1696,30 @@ void hmp_netdev_del(Monitor *mon, const QDict *qdict)
 void hmp_object_add(Monitor *mon, const QDict *qdict)
 {
 Error *err = NULL;
-QemuOpts *opts;
 Visitor *v;
 Object *obj = NULL;
+QObject *pobj;
 
-opts = qemu_opts_from_qdict(qemu_find_opts("object"), qdict, );
+pobj = qdict_crumple(qdict, true, );
 if (err) {
-hmp_handle_error(mon, );
-return;
+goto cleanup;
 }
 
-v = opts_visitor_new(opts);
-obj = user_creatable_add(qdict, v, );
+/*
+ * We need autocreate_list=true + permit_int_ranges=true
+ * in order to maintain compat with OptsVisitor creation
+ * of the 'numa' object which had an int16List property.
+ *
+ * We need autocreate_structs=1, because at the CLI level
+ * we have the object type + properties in the same flat
+ * struct, even though at the QMP level they are nested.
+ */
+v = qobject_input_visitor_new_autocast(pobj, true, 1, true);
+obj = user_creatable_add(qobject_to_qdict(pobj), v, );
 visit_free(v);
-qemu_opts_del(opts);
 
+ cleanup:
+qobject_decref(pobj);
 if (err) {
 hmp_handle_error(mon, );
 }
-- 
2.7.4

[Qemu-block] [PATCH v14 11/21] qapi: add integer range support for QObjectInputVisitor

[Daniel P. Berrange]

The traditional CLI arg syntax allows two ways to specify
integer lists, either one value per key, or a range of
values per key. eg the following are identical:

  -arg foo=5,foo=6,foo=7
  -arg foo=5-7

This extends the QObjectInputVisitor so that it is able
to parse ranges and turn them into distinct list entries.

This means that

  -arg foo=5-7

is treated as equivalent to

  -arg foo.0=5,foo.1=6,foo.2=7

Edge case tests are copied from test-opts-visitor to
ensure identical behaviour when parsing.

Signed-off-by: Daniel P. Berrange 
---
 include/qapi/qobject-input-visitor.h |  23 -
 qapi/qobject-input-visitor.c | 158 ++--
 tests/test-qobject-input-visitor.c   | 195 +--
 3 files changed, 360 insertions(+), 16 deletions(-)

diff --git a/include/qapi/qobject-input-visitor.h 
b/include/qapi/qobject-input-visitor.h
index 94051f3..242b767 100644
--- a/include/qapi/qobject-input-visitor.h
+++ b/include/qapi/qobject-input-visitor.h
@@ -19,6 +19,12 @@
 
 typedef struct QObjectInputVisitor QObjectInputVisitor;
 
+/* Inclusive upper bound on the size of any flattened range. This is a safety
+ * (= anti-annoyance) measure; wrong ranges should not cause long startup
+ * delays nor exhaust virtual memory.
+ */
+#define QIV_RANGE_MAX 65536
+
 /**
  * Create a new input visitor that converts @obj to a QAPI object.
  *
@@ -71,6 +77,19 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool 
strict);
  * The value given determines how many levels of structs are allowed to
  * be flattened in this way.
  *
+ * If @permit_int_ranges is true, then when visiting a list of integers,
+ * the integer value strings may encode ranges eg a single element
+ * containing "5-7" is treated as if there were three elements "5", "6",
+ * "7". This should only be used if compatibility is required with the
+ * OptsVisitor which would allow integer ranges. e.g. set this to true
+ * if you have compatibility requirements for
+ *
+ *   -arg val=5-8
+ *
+ * to be treated as equivalent to the preferred syntax:
+ *
+ *   -arg val.0=5,val.1=6,val.2=7,val.3=8
+ *
  * The visitor always operates in strict mode, requiring all dict keys
  * to be consumed during visitation. An error will be reported if this
  * does not happen.
@@ -80,7 +99,8 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool strict);
  */
 Visitor *qobject_input_visitor_new_autocast(QObject *obj,
 bool autocreate_list,
-size_t autocreate_struct_levels);
+size_t autocreate_struct_levels,
+bool permit_int_ranges);
 
 
 /**
@@ -98,6 +118,7 @@ Visitor *qobject_input_visitor_new_autocast(QObject *obj,
 Visitor *qobject_input_visitor_new_opts(const QemuOpts *opts,
 bool autocreate_list,
 size_t autocreate_struct_levels,
+bool permit_int_ranges,
 Error **errp);
 
 #endif
diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c
index 1be4865..6b3d0f2 100644
--- a/qapi/qobject-input-visitor.c
+++ b/qapi/qobject-input-visitor.c
@@ -31,6 +31,8 @@ typedef struct StackObject
 
 GHashTable *h;   /* If obj is dict: unvisited keys */
 const QListEntry *entry; /* If obj is list: unvisited tail */
+uint64_t range_val;
+uint64_t range_limit;
 
 QSLIST_ENTRY(StackObject) node;
 } StackObject;
@@ -60,6 +62,10 @@ struct QObjectInputVisitor
  * consider auto-creating a struct containing
  * remaining unvisited items */
 size_t autocreate_struct_levels;
+
+/* Whether int lists can have single values representing
+ * ranges of values */
+bool permit_int_ranges;
 };
 
 static QObjectInputVisitor *to_qiv(Visitor *v)
@@ -282,7 +288,7 @@ static GenericList *qobject_input_next_list(Visitor *v, 
GenericList *tail,
 QObjectInputVisitor *qiv = to_qiv(v);
 StackObject *so = QSLIST_FIRST(>stack);
 
-if (!so->entry) {
+if ((so->range_val == so->range_limit) && !so->entry) {
 return NULL;
 }
 tail->next = g_malloc0(size);
@@ -329,21 +335,87 @@ static void qobject_input_type_int64_autocast(Visitor *v, 
const char *name,
   int64_t *obj, Error **errp)
 {
 QObjectInputVisitor *qiv = to_qiv(v);
-QString *qstr = qobject_to_qstring(qobject_input_get_object(qiv, name,
-true));
+QString *qstr;
 int64_t ret;
+const char *end = NULL;
+StackObject *tos;
+bool inlist = false;
+
+/* Preferentially generate values from a range, before
+ * trying to consume another QList element */
+tos = QSLIST_FIRST(>stack);
+if (tos) {
+if ((int64_t)tos->range_val < 

[Qemu-block] [PATCH v14 10/21] qapi: permit auto-creating nested structs

[Daniel P. Berrange]

Some of the historical command line opts that had their
keys in in a completely flat namespace are now represented
by QAPI schemas that use a nested structs. When converting
the QemuOpts to QObject, there is no information about
compound types available, so the QObject will be completely
flat, even after the qdict_crumple() call. So when starting
a struct, we may not have a QDict available in the input
data, so we auto-create a QDict containing all the currently
unvisited input data keys. Not all historical command line
opts require this, so the behaviour is opt-in, by specifying
how many levels of structs are permitted to be auto-created.

Note that this only works if the child struct is the last
field to the visited in the parent struct. This is always
the case for currently existing legacy command line options.

The example is the NetLegacy type which has 3 levels of
structs. The modern way to represent this in QemuOpts would
be the dot-separated component approach

  -net vlan=1,id=foo,name=bar,opts.type=tap,\
   opts.data.fd=3,opts.data.script=ifup

The legacy syntax will just be presenting

  -net vlan=1,id=foo,name=bar,type=tap,fd=3,script=ifup

So we need to auto-create 3 levels of struct when visiting.

The implementation here will enable visiting in both the
modern and legacy syntax, compared to OptsVisitor which
only allows the legacy syntax.

Signed-off-by: Daniel P. Berrange 
---
 include/qapi/qobject-input-visitor.h |  22 +-
 qapi/qobject-input-visitor.c |  59 ++--
 tests/test-qobject-input-visitor.c   | 130 ---
 3 files changed, 194 insertions(+), 17 deletions(-)

diff --git a/include/qapi/qobject-input-visitor.h 
b/include/qapi/qobject-input-visitor.h
index 1809f48..94051f3 100644
--- a/include/qapi/qobject-input-visitor.h
+++ b/include/qapi/qobject-input-visitor.h
@@ -45,7 +45,7 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool strict);
  * If @autocreate_list is true, then as an alternative to a normal QList,
  * list values can be stored as a QString or QDict instead, which will
  * be interpreted as representing single element lists. This should only
- * by used if compatibility is required with the OptsVisitor which allowed
+ * be used if compatibility is required with the OptsVisitor which allowed
  * repeated keys, without list indexes, to represent lists. e.g. set this
  * to true if you have compatibility requirements for
  *
@@ -55,6 +55,22 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool 
strict);
  *
  *   -arg foo.0=hello,foo.1=world
  *
+ * If @autocreate_struct_levels is non-zero, then when visiting structs,
+ * if the corresponding QDict is not found, it will automatically create
+ * a QDict containing all remaining unvisited options. This should only
+ * be used if compatibility is required with the OptsVisitor which flatten
+ * structs so that all keys were at the same level. e.g. set this to a
+ * non-zero number if you compatibility requirements for
+ *
+ *   -arg type=person,surname=blogs,forename=fred
+ *
+ * to be treated as equivalent to the perferred syntax
+ *
+ *   -arg type=person,data.surname=blogs,data.forename=fred
+ *
+ * The value given determines how many levels of structs are allowed to
+ * be flattened in this way.
+ *
  * The visitor always operates in strict mode, requiring all dict keys
  * to be consumed during visitation. An error will be reported if this
  * does not happen.
@@ -63,7 +79,8 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool strict);
  * visit_free() when no longer required.
  */
 Visitor *qobject_input_visitor_new_autocast(QObject *obj,
-bool autocreate_list);
+bool autocreate_list,
+size_t autocreate_struct_levels);
 
 
 /**
@@ -80,6 +97,7 @@ Visitor *qobject_input_visitor_new_autocast(QObject *obj,
  */
 Visitor *qobject_input_visitor_new_opts(const QemuOpts *opts,
 bool autocreate_list,
+size_t autocreate_struct_levels,
 Error **errp);
 
 #endif
diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c
index d88e9f9..1be4865 100644
--- a/qapi/qobject-input-visitor.c
+++ b/qapi/qobject-input-visitor.c
@@ -52,6 +52,14 @@ struct QObjectInputVisitor
 /* Whether we can auto-create single element lists when
  * encountering a non-QList type */
 bool autocreate_list;
+
+/* Current depth of recursion into structs */
+size_t struct_level;
+
+/* Numbers of levels at which we will
+ * consider auto-creating a struct containing
+ * remaining unvisited items */
+size_t autocreate_struct_levels;
 };
 
 static QObjectInputVisitor *to_qiv(Visitor *v)
@@ -79,7 +87,12 @@ static QObject *qobject_input_get_object(QObjectInputVisitor 
*qiv,
 
 if 

[Qemu-block] [PATCH v14 07/21] qapi: permit scalar type conversions in QObjectInputVisitor

[Daniel P. Berrange]

Currently the QObjectInputVisitor assumes that all scalar
values are directly represented as the final types declared
by the thing being visited. ie it assumes an 'int' is using
QInt, and a 'bool' is using QBool, etc.  This is good when
QObjectInputVisitor is fed a QObject that came from a JSON
document on the QMP monitor, as it will strictly validate
correctness.

To allow QObjectInputVisitor to be reused for visiting
a QObject originating from QemuOpts, an alternative mode
is needed where all the scalars types are represented as
QString and converted on the fly to the final desired
type.

Reviewed-by: Kevin Wolf 
Reviewed-by: Marc-André Lureau 
Signed-off-by: Daniel P. Berrange 
---
 include/qapi/qobject-input-visitor.h |  32 +-
 qapi/qobject-input-visitor.c | 132 
 tests/test-qobject-input-visitor.c   | 194 ++-
 3 files changed, 350 insertions(+), 8 deletions(-)

diff --git a/include/qapi/qobject-input-visitor.h 
b/include/qapi/qobject-input-visitor.h
index cde328d..5022297 100644
--- a/include/qapi/qobject-input-visitor.h
+++ b/include/qapi/qobject-input-visitor.h
@@ -19,12 +19,36 @@
 
 typedef struct QObjectInputVisitor QObjectInputVisitor;
 
-/*
- * Return a new input visitor that converts a QObject to a QAPI object.
+/**
+ * Create a new input visitor that converts @obj to a QAPI object.
+ *
+ * Any scalar values in the @obj input data structure should be in the
+ * required type already. i.e. if visiting a bool, the value should
+ * already be a QBool instance.
  *
- * Set @strict to reject a parse that doesn't consume all keys of a
- * dictionary; otherwise excess input is ignored.
+ * If @strict is set to true, then an error will be reported if any
+ * dict keys are not consumed during visitation. If @strict is false
+ * then extra dict keys are silently ignored.
+ *
+ * The returned input visitor should be released by calling
+ * visit_free() when no longer required.
  */
 Visitor *qobject_input_visitor_new(QObject *obj, bool strict);
 
+/**
+ * Create a new input visitor that converts @obj to a QAPI object.
+ *
+ * Any scalar values in the @obj input data structure should always be
+ * represented as strings. i.e. if visiting a boolean, the value should
+ * be a QString whose contents represent a valid boolean.
+ *
+ * The visitor always operates in strict mode, requiring all dict keys
+ * to be consumed during visitation. An error will be reported if this
+ * does not happen.
+ *
+ * The returned input visitor should be released by calling
+ * visit_free() when no longer required.
+ */
+Visitor *qobject_input_visitor_new_autocast(QObject *obj);
+
 #endif
diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c
index 5ff3db3..cf41df6 100644
--- a/qapi/qobject-input-visitor.c
+++ b/qapi/qobject-input-visitor.c
@@ -20,6 +20,7 @@
 #include "qemu-common.h"
 #include "qapi/qmp/types.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/cutils.h"
 
 #define QIV_STACK_SIZE 1024
 
@@ -263,6 +264,28 @@ static void qobject_input_type_int64(Visitor *v, const 
char *name, int64_t *obj,
 *obj = qint_get_int(qint);
 }
 
+
+static void qobject_input_type_int64_autocast(Visitor *v, const char *name,
+  int64_t *obj, Error **errp)
+{
+QObjectInputVisitor *qiv = to_qiv(v);
+QString *qstr = qobject_to_qstring(qobject_input_get_object(qiv, name,
+true));
+int64_t ret;
+
+if (!qstr || !qstr->string) {
+error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+   "string");
+return;
+}
+
+if (qemu_strtoll(qstr->string, NULL, 0, ) < 0) {
+error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
+return;
+}
+*obj = ret;
+}
+
 static void qobject_input_type_uint64(Visitor *v, const char *name,
   uint64_t *obj, Error **errp)
 {
@@ -279,6 +302,27 @@ static void qobject_input_type_uint64(Visitor *v, const 
char *name,
 *obj = qint_get_int(qint);
 }
 
+static void qobject_input_type_uint64_autocast(Visitor *v, const char *name,
+   uint64_t *obj, Error **errp)
+{
+QObjectInputVisitor *qiv = to_qiv(v);
+QString *qstr = qobject_to_qstring(qobject_input_get_object(qiv, name,
+true));
+unsigned long long ret;
+
+if (!qstr || !qstr->string) {
+error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+   "string");
+return;
+}
+
+if (parse_uint_full(qstr->string, , 0) < 0) {
+error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
+return;
+}
+*obj = ret;
+}
+
 static void qobject_input_type_bool(Visitor *v, const char *name, bool 

[Qemu-block] [PATCH v14 02/21] qdict: implement a qdict_crumple method for un-flattening a dict

[Daniel P. Berrange]

The qdict_flatten() method will take a dict whose elements are
further nested dicts/lists and flatten them by concatenating
keys.

The qdict_crumple() method aims to do the reverse, taking a flat
qdict, and turning it into a set of nested dicts/lists. It will
apply nesting based on the key name, with a '.' indicating a
new level in the hierarchy. If the keys in the nested structure
are all numeric, it will create a list, otherwise it will create
a dict.

If the keys are a mixture of numeric and non-numeric, or the
numeric keys are not in strictly ascending order, an error will
be reported.

As an example, a flat dict containing

 {
   'foo.0.bar': 'one',
   'foo.0.wizz': '1',
   'foo.1.bar': 'two',
   'foo.1.wizz': '2'
 }

will get turned into a dict with one element 'foo' whose
value is a list. The list elements will each in turn be
dicts.

 {
   'foo': [
 { 'bar': 'one', 'wizz': '1' },
 { 'bar': 'two', 'wizz': '2' }
   ],
 }

If the key is intended to contain a literal '.', then it must
be escaped as '..'. ie a flat dict

  {
 'foo..bar': 'wizz',
 'bar.foo..bar': 'eek',
 'bar.hello': 'world'
  }

Will end up as

  {
 'foo.bar': 'wizz',
 'bar': {
'foo.bar': 'eek',
'hello': 'world'
 }
  }

The intent of this function is that it allows a set of QemuOpts
to be turned into a nested data structure that mirrors the nesting
used when the same object is defined over QMP.

Reviewed-by: Eric Blake 
Reviewed-by: Kevin Wolf 
Reviewed-by: Marc-André Lureau 
Signed-off-by: Daniel P. Berrange 
---
 include/qapi/qmp/qdict.h |   1 +
 qobject/qdict.c  | 289 +++
 tests/check-qdict.c  | 261 ++
 3 files changed, 551 insertions(+)

diff --git a/include/qapi/qmp/qdict.h b/include/qapi/qmp/qdict.h
index 71b8eb0..e0d24e1 100644
--- a/include/qapi/qmp/qdict.h
+++ b/include/qapi/qmp/qdict.h
@@ -73,6 +73,7 @@ void qdict_flatten(QDict *qdict);
 void qdict_extract_subqdict(QDict *src, QDict **dst, const char *start);
 void qdict_array_split(QDict *src, QList **dst);
 int qdict_array_entries(QDict *src, const char *subqdict);
+QObject *qdict_crumple(const QDict *src, bool recursive, Error **errp);
 
 void qdict_join(QDict *dest, QDict *src, bool overwrite);
 
diff --git a/qobject/qdict.c b/qobject/qdict.c
index 60f158c..c38e90e 100644
--- a/qobject/qdict.c
+++ b/qobject/qdict.c
@@ -17,6 +17,7 @@
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qobject.h"
+#include "qapi/error.h"
 #include "qemu/queue.h"
 #include "qemu-common.h"
 #include "qemu/cutils.h"
@@ -683,6 +684,294 @@ void qdict_array_split(QDict *src, QList **dst)
 }
 }
 
+
+/**
+ * qdict_split_flat_key:
+ * @key: the key string to split
+ * @prefix: non-NULL pointer to hold extracted prefix
+ * @suffix: non-NULL pointer to remaining suffix
+ *
+ * Given a flattened key such as 'foo.0.bar', split it into two parts
+ * at the first '.' separator. Allows double dot ('..') to escape the
+ * normal separator.
+ *
+ * e.g.
+ *'foo.0.bar' -> prefix='foo' and suffix='0.bar'
+ *'foo..0.bar' -> prefix='foo.0' and suffix='bar'
+ *
+ * The '..' sequence will be unescaped in the returned 'prefix'
+ * string. The 'suffix' string will be left in escaped format, so it
+ * can be fed back into the qdict_split_flat_key() key as the input
+ * later.
+ *
+ * The caller is responsible for freeing the string returned in @prefix
+ * using g_free().
+ */
+static void qdict_split_flat_key(const char *key, char **prefix,
+ const char **suffix)
+{
+const char *separator;
+size_t i, j;
+
+/* Find first '.' separator, but if there is a pair '..'
+ * that acts as an escape, so skip over '..' */
+separator = NULL;
+do {
+if (separator) {
+separator += 2;
+} else {
+separator = key;
+}
+separator = strchr(separator, '.');
+} while (separator && separator[1] == '.');
+
+if (separator) {
+*prefix = g_strndup(key, separator - key);
+*suffix = separator + 1;
+} else {
+*prefix = g_strdup(key);
+*suffix = NULL;
+}
+
+/* Unescape the '..' sequence into '.' */
+for (i = 0, j = 0; (*prefix)[i] != '\0'; i++, j++) {
+if ((*prefix)[i] == '.') {
+assert((*prefix)[i + 1] == '.');
+i++;
+}
+(*prefix)[j] = (*prefix)[i];
+}
+(*prefix)[j] = '\0';
+}
+
+
+/**
+ * qdict_is_list:
+ * @maybe_list: dict to check if keys represent list elements.
+ *
+ * Determine whether all keys in @maybe_list are valid list elements.
+ * If @maybe_list is non-zero in length and all the keys look like
+ * valid list indexes, this will return 1. If @maybe_list is zero
+ * length or all keys are non-numeric then it will return 0 to indicate
+ * it is a 

[Qemu-block] [PATCH v14 08/21] qapi: allow QObjectInputVisitor to be created with QemuOpts

[Daniel P. Berrange]

Instead of requiring all callers to go through the mutli-step
process of turning QemuOpts into a suitable QObject for visiting,
add a new constructor that encapsulates this logic. This will
allow QObjectInputVisitor to be a drop-in replacement for the
existing OptsVisitor with minimal code changes for callers.

NB, at this point it is only supporting opts syntax which
explicitly matches the QAPI schema structure, so is not yet
a true drop-in replacement for OptsVisitor. The patches that
follow will add the special cases requird for full backwards
compatibility with OptsVisitor.

Signed-off-by: Daniel P. Berrange 
---
 include/qapi/qobject-input-visitor.h | 15 +++
 include/qemu/option.h|  2 +-
 qapi/qobject-input-visitor.c | 25 +
 util/qemu-option.c   |  2 +-
 4 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/include/qapi/qobject-input-visitor.h 
b/include/qapi/qobject-input-visitor.h
index 5022297..f134d90 100644
--- a/include/qapi/qobject-input-visitor.h
+++ b/include/qapi/qobject-input-visitor.h
@@ -51,4 +51,19 @@ Visitor *qobject_input_visitor_new(QObject *obj, bool 
strict);
  */
 Visitor *qobject_input_visitor_new_autocast(QObject *obj);
 
+
+/**
+ * Create a new input visitor that converts @opts to a QAPI object.
+ *
+ * The QemuOpts will be converted into a QObject using the
+ * qdict_crumple() method to automatically create structs
+ * and lists. The resulting QDict will then be passed to the
+ * qobject_input_visitor_new_autocast() method. See the docs
+ * of that method for further details on processing behaviour.
+ *
+ * The returned input visitor should be released by calling
+ * visit_free() when no longer required.
+ */
+Visitor *qobject_input_visitor_new_opts(const QemuOpts *opts, Error **errp);
+
 #endif
diff --git a/include/qemu/option.h b/include/qemu/option.h
index 2a5266f..29f3f18 100644
--- a/include/qemu/option.h
+++ b/include/qemu/option.h
@@ -125,7 +125,7 @@ void qemu_opts_set_defaults(QemuOptsList *list, const char 
*params,
 int permit_abbrev);
 QemuOpts *qemu_opts_from_qdict(QemuOptsList *list, const QDict *qdict,
Error **errp);
-QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict);
+QDict *qemu_opts_to_qdict(const QemuOpts *opts, QDict *qdict);
 void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, Error **errp);
 
 typedef int (*qemu_opts_loopfunc)(void *opaque, QemuOpts *opts, Error **errp);
diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c
index cf41df6..d9269c9 100644
--- a/qapi/qobject-input-visitor.c
+++ b/qapi/qobject-input-visitor.c
@@ -545,3 +545,28 @@ Visitor *qobject_input_visitor_new_autocast(QObject *obj)
 
 return >visitor;
 }
+
+
+Visitor *qobject_input_visitor_new_opts(const QemuOpts *opts,
+Error **errp)
+{
+QDict *pdict;
+QObject *pobj = NULL;
+Visitor *v = NULL;
+
+pdict = qemu_opts_to_qdict(opts, NULL);
+if (!pdict) {
+goto cleanup;
+}
+
+pobj = qdict_crumple(pdict, true, errp);
+if (!pobj) {
+goto cleanup;
+}
+
+v = qobject_input_visitor_new_autocast(pobj);
+ cleanup:
+qobject_decref(pobj);
+QDECREF(pdict);
+return v;
+}
diff --git a/util/qemu-option.c b/util/qemu-option.c
index 41b356c..418cbb6 100644
--- a/util/qemu-option.c
+++ b/util/qemu-option.c
@@ -1058,7 +1058,7 @@ void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, 
Error **errp)
  * TODO We'll want to use types appropriate for opt->desc->type, but
  * this is enough for now.
  */
-QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict)
+QDict *qemu_opts_to_qdict(const QemuOpts *opts, QDict *qdict)
 {
 QemuOpt *opt;
 QObject *val;
-- 
2.7.4

[Qemu-block] [PATCH v14 12/21] option: allow qemu_opts_to_qdict to merge repeated options

[Daniel P. Berrange]

If given an option string such as

  size=1024,nodes=10,nodes=4-5,nodes=1-2,policy=bind

the qemu_opts_to_qdict() method will currently overwrite
the values for repeated option keys, so only the last
value is in the returned dict:

size=QString("1024")
nodes=QString("1-2")
policy=QString("bind")

With this change the caller can optionally ask for all
the repeated values to be stored in a QList. In the
above example that would result in 'nodes' being a
QList, so the returned dict would contain

size=QString("1024")
nodes=QList([QString("10"),
 QString("4-5"),
 QString("1-2")])
policy=QString("bind")

Note that the conversion has no way of knowing whether
any given key is expected to be a list upfront - it can
only figure that out when seeing the first duplicated
key. Thus the caller has to be prepared to deal with the
fact that if a key 'foo' is a list, then the returned
qdict may contain either a QString or a QList for the
key 'foo'.

In a third mode, it is possible to ask for repeated
options to be reported as an error, rather than silently
dropping all but the last one.

All existing callers are all converted to explicitly
request the historical behaviour of only reporting the
last key. Later patches will make use of the new modes.

Signed-off-by: Daniel P. Berrange 
---
 blockdev.c   |   7 ++-
 include/qemu/option.h|  13 -
 monitor.c|   3 +-
 qapi/qobject-input-visitor.c |   4 +-
 qemu-img.c   |   4 +-
 qemu-io-cmds.c   |   3 +-
 qemu-io.c|   6 +-
 qemu-nbd.c   |   3 +-
 qom/object_interfaces.c  |   3 +-
 tests/test-qemu-opts.c   | 132 +++
 tests/test-replication.c |   9 ++-
 util/qemu-option.c   |  64 ++---
 12 files changed, 229 insertions(+), 22 deletions(-)

diff --git a/blockdev.c b/blockdev.c
index 814d49f..a999d46 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -914,7 +914,8 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType 
block_default_type)
 
 /* Get a QDict for processing the options */
 bs_opts = qdict_new();
-qemu_opts_to_qdict(all_opts, bs_opts);
+qemu_opts_to_qdict(all_opts, bs_opts, QEMU_OPTS_REPEAT_POLICY_LAST,
+   _abort);
 
 legacy_opts = qemu_opts_create(_legacy_drive_opts, NULL, 0,
_abort);
@@ -3804,8 +3805,8 @@ void hmp_drive_add_node(Monitor *mon, const char *optstr)
 return;
 }
 
-qdict = qemu_opts_to_qdict(opts, NULL);
-
+qdict = qemu_opts_to_qdict(opts, NULL, QEMU_OPTS_REPEAT_POLICY_LAST,
+   _abort);
 if (!qdict_get_try_str(qdict, "node-name")) {
 QDECREF(qdict);
 error_report("'node-name' needs to be specified");
diff --git a/include/qemu/option.h b/include/qemu/option.h
index 29f3f18..ffd841d 100644
--- a/include/qemu/option.h
+++ b/include/qemu/option.h
@@ -125,7 +125,18 @@ void qemu_opts_set_defaults(QemuOptsList *list, const char 
*params,
 int permit_abbrev);
 QemuOpts *qemu_opts_from_qdict(QemuOptsList *list, const QDict *qdict,
Error **errp);
-QDict *qemu_opts_to_qdict(const QemuOpts *opts, QDict *qdict);
+typedef enum {
+/* Last occurrence of a duplicate option silently replaces earlier */
+QEMU_OPTS_REPEAT_POLICY_LAST,
+/* Each occurrence of a duplicate option converts value to a QList */
+QEMU_OPTS_REPEAT_POLICY_ALL,
+/* First occurrence of a duplicate option causes an error */
+QEMU_OPTS_REPEAT_POLICY_ERROR,
+} QemuOptsRepeatPolicy;
+
+QDict *qemu_opts_to_qdict(const QemuOpts *opts, QDict *qdict,
+  QemuOptsRepeatPolicy repeatPolicy,
+  Error **errp);
 void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, Error **errp);
 
 typedef int (*qemu_opts_loopfunc)(void *opaque, QemuOpts *opts, Error **errp);
diff --git a/monitor.c b/monitor.c
index 14089cc..84e79d4 100644
--- a/monitor.c
+++ b/monitor.c
@@ -2642,7 +2642,8 @@ static QDict *monitor_parse_arguments(Monitor *mon,
 if (!opts) {
 goto fail;
 }
-qemu_opts_to_qdict(opts, qdict);
+qemu_opts_to_qdict(opts, qdict, QEMU_OPTS_REPEAT_POLICY_LAST,
+   _abort);
 qemu_opts_del(opts);
 }
 break;
diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c
index 6b3d0f2..2287d11 100644
--- a/qapi/qobject-input-visitor.c
+++ b/qapi/qobject-input-visitor.c
@@ -759,7 +759,9 @@ Visitor *qobject_input_visitor_new_opts(const QemuOpts 
*opts,
 QObject *pobj = NULL;
 Visitor *v = NULL;
 
-pdict = qemu_opts_to_qdict(opts, NULL);
+pdict = qemu_opts_to_qdict(opts, NULL,
+   

[Qemu-block] [PATCH v14 13/21] qdict: allow qdict_crumple to accept compound types as values

[Daniel P. Berrange]

Currently qdict_crumple requires a totally flat QDict as its
input. i.e. all values in the QDict must be scalar types.

In order to have backwards compatibility with the OptsVisitor,
qemu_opt_to_qdict() has a new mode where it may return a QList
for values in the QDict, if there was a repeated key. We thus
need to allow compound types to appear as values in the input
dict given to qdict_crumple().

To avoid confusion, we sanity check that the user has not mixed
the old and new syntax at the same time. e.g. these are allowed

   foo=hello,foo=world,foo=wibble
   foo.0=hello,foo.1=world,foo.2=wibble

but this is forbidden

   foo=hello,foo=world,foo.2=wibble

Signed-off-by: Daniel P. Berrange 
---
 qobject/qdict.c |  45 +-
 tests/check-qdict.c | 175 
 2 files changed, 204 insertions(+), 16 deletions(-)

diff --git a/qobject/qdict.c b/qobject/qdict.c
index c38e90e..83384b2 100644
--- a/qobject/qdict.c
+++ b/qobject/qdict.c
@@ -814,15 +814,16 @@ static int qdict_is_list(QDict *maybe_list, Error **errp)
 
 /**
  * qdict_crumple:
- * @src: the original flat dictionary (only scalar values) to crumple
+ * @src: the original dictionary to crumple
  * @recursive: true to recursively crumple nested dictionaries
  *
- * Takes a flat dictionary whose keys use '.' separator to indicate
- * nesting, and values are scalars, and crumples it into a nested
- * structure. If the @recursive parameter is false, then only the
- * first level of structure implied by the keys will be crumpled. If
- * @recursive is true, then the input will be recursively crumpled to
- * expand all levels of structure in the keys.
+ * Takes a dictionary whose keys use '.' separator to indicate
+ * nesting, crumples it into a nested structure. The values in
+ * @src are permitted to be scalars or compound typee. If the
+ * @recursive parameter is false, then only the first level of
+ * structure implied by the keys will be crumpled. If @recursive
+ * is true, then the input will be recursively crumpled to expand
+ * all levels of structure in the keys.
  *
  * To include a literal '.' in a key name, it must be escaped as '..'
  *
@@ -843,7 +844,10 @@ static int qdict_is_list(QDict *maybe_list, Error **errp)
  * The following scenarios in the input dict will result in an
  * error being returned:
  *
- *  - Any values in @src are non-scalar types
+ *  - If a key in @src has a value that is non-scalar, and
+ *a later key implies creation of a compound type. e.g.
+ *if 'foo' point to a 'QList' or 'QDict', then it is
+ *not permitted to also have 'foo.NNN' keys later.
  *  - If keys in @src imply that a particular level is both a
  *list and a dict. e.g., "foo.0.bar" and "foo.eek.bar".
  *  - If keys in @src imply that a particular level is a list,
@@ -864,21 +868,26 @@ QObject *qdict_crumple(const QDict *src, bool recursive, 
Error **errp)
 char *prefix = NULL;
 const char *suffix = NULL;
 int is_list;
+QDict *compound;
 
+compound = qdict_new();
 two_level = qdict_new();
 
 /* Step 1: split our totally flat dict into a two level dict */
 for (ent = qdict_first(src); ent != NULL; ent = qdict_next(src, ent)) {
-if (qobject_type(ent->value) == QTYPE_QDICT ||
-qobject_type(ent->value) == QTYPE_QLIST) {
-error_setg(errp, "Value %s is not a scalar",
-   ent->key);
-goto error;
-}
-
 qdict_split_flat_key(ent->key, , );
 
 child = qdict_get(two_level, prefix);
+
+if (qdict_haskey(compound, prefix)) {
+error_setg(errp, "Key %s is already set as a list/dict", prefix);
+goto error;
+}
+if (qobject_type(ent->value) == QTYPE_QLIST ||
+qobject_type(ent->value) == QTYPE_QDICT) {
+qobject_incref(ent->value);
+qdict_put_obj(compound, prefix, ent->value);
+}
 if (suffix) {
 if (child) {
 if (qobject_type(child) != QTYPE_QDICT) {
@@ -913,7 +922,8 @@ QObject *qdict_crumple(const QDict *src, bool recursive, 
Error **errp)
 for (ent = qdict_first(two_level); ent != NULL;
  ent = qdict_next(two_level, ent)) {
 
-if (qobject_type(ent->value) == QTYPE_QDICT) {
+if (qobject_type(ent->value) == QTYPE_QDICT &&
+!qdict_haskey(compound, ent->key)) {
 child = qdict_crumple(qobject_to_qdict(ent->value),
   recursive, errp);
 if (!child) {
@@ -961,10 +971,13 @@ QObject *qdict_crumple(const QDict *src, bool recursive, 
Error **errp)
 dst = QOBJECT(multi_level);
 }
 
+QDECREF(compound);
+
 return dst;
 
  error:
 g_free(prefix);
+QDECREF(compound);
 QDECREF(multi_level);
 QDECREF(two_level);
 qobject_decref(dst);
diff --git a/tests/check-qdict.c b/tests/check-qdict.c
index 

[Qemu-block] [PATCH v14 01/21] option: make parse_option_bool/number non-static

[Daniel P. Berrange]

The opts-visitor.c opts_type_bool() method has code for
parsing a string to set a bool value, as does the
qemu-option.c parse_option_bool() method, except it
handles fewer cases.

To enable consistency across the codebase, extend
parse_option_bool() to handle "yes", "no", "y" and
"n", and make it non-static. Convert the opts
visitor to call this method directly.

Also make parse_option_number() non-static to allow
for similar reuse later.

Reviewed-by: Kevin Wolf 
Reviewed-by: Marc-André Lureau 
Reviewed-by: Eric Blake 
Reviewed-by: Markus Armbruster 
Signed-off-by: Daniel P. Berrange 
---
 include/qemu/option.h |  4 
 qapi/opts-visitor.c   | 19 +--
 tests/qemu-iotests/051.out|  6 +++---
 tests/qemu-iotests/051.pc.out |  6 +++---
 tests/qemu-iotests/137.out|  4 ++--
 util/qemu-option.c| 27 ---
 6 files changed, 29 insertions(+), 37 deletions(-)

diff --git a/include/qemu/option.h b/include/qemu/option.h
index 1f9e3f9..2a5266f 100644
--- a/include/qemu/option.h
+++ b/include/qemu/option.h
@@ -37,6 +37,10 @@ int get_param_value(char *buf, int buf_size,
 const char *tag, const char *str);
 
 
+void parse_option_bool(const char *name, const char *value, bool *ret,
+   Error **errp);
+void parse_option_number(const char *name, const char *value,
+ uint64_t *ret, Error **errp);
 void parse_option_size(const char *name, const char *value,
uint64_t *ret, Error **errp);
 bool has_help_option(const char *param);
diff --git a/qapi/opts-visitor.c b/qapi/opts-visitor.c
index 1048bbc..084f7cc 100644
--- a/qapi/opts-visitor.c
+++ b/qapi/opts-visitor.c
@@ -334,7 +334,6 @@ opts_type_str(Visitor *v, const char *name, char **obj, 
Error **errp)
 }
 
 
-/* mimics qemu-option.c::parse_option_bool() */
 static void
 opts_type_bool(Visitor *v, const char *name, bool *obj, Error **errp)
 {
@@ -346,23 +345,7 @@ opts_type_bool(Visitor *v, const char *name, bool *obj, 
Error **errp)
 return;
 }
 
-if (opt->str) {
-if (strcmp(opt->str, "on") == 0 ||
-strcmp(opt->str, "yes") == 0 ||
-strcmp(opt->str, "y") == 0) {
-*obj = true;
-} else if (strcmp(opt->str, "off") == 0 ||
-strcmp(opt->str, "no") == 0 ||
-strcmp(opt->str, "n") == 0) {
-*obj = false;
-} else {
-error_setg(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
-   "on|yes|y|off|no|n");
-return;
-}
-} else {
-*obj = true;
-}
+parse_option_bool(opt->name, opt->str, obj, errp);
 
 processed(ov, name);
 }
diff --git a/tests/qemu-iotests/051.out b/tests/qemu-iotests/051.out
index 408d613..ffdc0b9 100644
--- a/tests/qemu-iotests/051.out
+++ b/tests/qemu-iotests/051.out
@@ -86,13 +86,13 @@ QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: 
Parameter 'lazy-refcounts' expects 'on' or 'off'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: 
Parameter 'lazy-refcounts' expects 'on', 'yes', 'y', 'off', 'no' or 'n'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: 
Parameter 'lazy-refcounts' expects 'on' or 'off'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: 
Parameter 'lazy-refcounts' expects 'on', 'yes', 'y', 'off', 'no' or 'n'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: 
Parameter 'lazy-refcounts' expects 'on' or 'off'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: 
Parameter 'lazy-refcounts' expects 'on', 'yes', 'y', 'off', 'no' or 'n'
 
 
 === With version 2 images enabling lazy refcounts must fail ===
diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out
index ec6d222..6abf1ab 100644
--- a/tests/qemu-iotests/051.pc.out
+++ b/tests/qemu-iotests/051.pc.out
@@ -86,13 +86,13 @@ QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: 
Parameter 'lazy-refcounts' expects 'on' or 'off'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: 
Parameter 'lazy-refcounts' expects 'on', 'yes', 'y', 'off', 'no' or 'n'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: 
Parameter 

[Qemu-block] [PATCH v14 04/21] qapi: rename QmpInputVisitor to QObjectInputVisitor

[Daniel P. Berrange]

The QmpInputVisitor has no direct dependency on QMP. It is
valid to use it anywhere that one has a QObject. Rename it
to better reflect its functionality as a generic QObject
to QAPI converter.

Reviewed-by: Kevin Wolf 
Reviewed-by: Eric Blake 
Signed-off-by: Daniel P. Berrange 
---
 docs/qapi-code-gen.txt |   2 +-
 ...qmp-input-visitor.h => qobject-input-visitor.h} |  10 +-
 include/qapi/visitor.h |   6 +-
 monitor.c  |   2 +-
 qapi/Makefile.objs |   2 +-
 ...qmp-input-visitor.c => qobject-input-visitor.c} | 171 +++--
 qmp.c  |   4 +-
 qom/qom-qobject.c  |   4 +-
 scripts/qapi-commands.py   |   4 +-
 target-s390x/cpu_models.c  |   4 +-
 tests/.gitignore   |   4 +-
 tests/Makefile.include |  12 +-
 tests/check-qnull.c|   4 +-
 tests/test-qmp-commands.c  |   4 +-
 ...-input-strict.c => test-qobject-input-strict.c} |   6 +-
 ...nput-visitor.c => test-qobject-input-visitor.c} |   6 +-
 tests/test-string-input-visitor.c  |   2 +-
 tests/test-visitor-serialization.c |   4 +-
 util/qemu-sockets.c|   2 +-
 19 files changed, 128 insertions(+), 125 deletions(-)
 rename include/qapi/{qmp-input-visitor.h => qobject-input-visitor.h} (63%)
 rename qapi/{qmp-input-visitor.c => qobject-input-visitor.c} (56%)
 rename tests/{test-qmp-input-strict.c => test-qobject-input-strict.c} (98%)
 rename tests/{test-qmp-input-visitor.c => test-qobject-input-visitor.c} (99%)

diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt
index 5d4c2cd..d2604b6 100644
--- a/docs/qapi-code-gen.txt
+++ b/docs/qapi-code-gen.txt
@@ -1024,7 +1024,7 @@ Example:
 Visitor *v;
 UserDefOneList *arg1 = NULL;
 
-v = qmp_input_visitor_new(QOBJECT(args), true);
+v = qobject_input_visitor_new(QOBJECT(args), true);
 visit_start_struct(v, NULL, NULL, 0, );
 if (err) {
 goto out;
diff --git a/include/qapi/qmp-input-visitor.h 
b/include/qapi/qobject-input-visitor.h
similarity index 63%
rename from include/qapi/qmp-input-visitor.h
rename to include/qapi/qobject-input-visitor.h
index f3ff5f3..cde328d 100644
--- a/include/qapi/qmp-input-visitor.h
+++ b/include/qapi/qobject-input-visitor.h
@@ -11,20 +11,20 @@
  *
  */
 
-#ifndef QMP_INPUT_VISITOR_H
-#define QMP_INPUT_VISITOR_H
+#ifndef QOBJECT_INPUT_VISITOR_H
+#define QOBJECT_INPUT_VISITOR_H
 
 #include "qapi/visitor.h"
 #include "qapi/qmp/qobject.h"
 
-typedef struct QmpInputVisitor QmpInputVisitor;
+typedef struct QObjectInputVisitor QObjectInputVisitor;
 
 /*
- * Return a new input visitor that converts QMP to QAPI.
+ * Return a new input visitor that converts a QObject to a QAPI object.
  *
  * Set @strict to reject a parse that doesn't consume all keys of a
  * dictionary; otherwise excess input is ignored.
  */
-Visitor *qmp_input_visitor_new(QObject *obj, bool strict);
+Visitor *qobject_input_visitor_new(QObject *obj, bool strict);
 
 #endif
diff --git a/include/qapi/visitor.h b/include/qapi/visitor.h
index 6c77a91..9bb6cba 100644
--- a/include/qapi/visitor.h
+++ b/include/qapi/visitor.h
@@ -25,14 +25,14 @@
  * for doing work at each node of a QAPI graph; it can also be used
  * for a virtual walk, where there is no actual QAPI C struct.
  *
- * There are four kinds of visitor classes: input visitors (QMP,
+ * There are four kinds of visitor classes: input visitors (QObject,
  * string, and QemuOpts) parse an external representation and build
- * the corresponding QAPI graph, output visitors (QMP and string) take
+ * the corresponding QAPI graph, output visitors (QObject and string) take
  * a completed QAPI graph and generate an external representation, the
  * dealloc visitor can take a QAPI graph (possibly partially
  * constructed) and recursively free its resources, and the clone
  * visitor performs a deep clone of one QAPI object to another.  While
- * the dealloc and QMP input/output visitors are general, the string,
+ * the dealloc and QObject input/output visitors are general, the string,
  * QemuOpts, and clone visitors have some implementation limitations;
  * see the documentation for each visitor for more details on what it
  * supports.  Also, see visitor-impl.h for the callback contracts
diff --git a/monitor.c b/monitor.c
index 83c4edf..14089cc 100644
--- a/monitor.c
+++ b/monitor.c
@@ -957,7 +957,7 @@ EventInfoList *qmp_query_events(Error **errp)
  * directly into QObject instead of first parsing it with
  * visit_type_SchemaInfoList() into a SchemaInfoList, then marshal it
  * to QObject with generated output marshallers, every 

[Qemu-block] [PATCH v14 03/21] qapi: add trace events for visitor

[Daniel P. Berrange]

Allow tracing of the operation of visitors

Signed-off-by: Daniel P. Berrange 
---
 Makefile.objs  |  1 +
 qapi/qapi-visit-core.c | 27 +++
 qapi/trace-events  | 33 +
 3 files changed, 61 insertions(+)
 create mode 100644 qapi/trace-events

diff --git a/Makefile.objs b/Makefile.objs
index a8e0224..b3e8aef 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -160,3 +160,4 @@ trace-events-y += target-s390x/trace-events
 trace-events-y += target-ppc/trace-events
 trace-events-y += qom/trace-events
 trace-events-y += linux-user/trace-events
+trace-events-y += qapi/trace-events
diff --git a/qapi/qapi-visit-core.c b/qapi/qapi-visit-core.c
index 55f5876..bfcb276 100644
--- a/qapi/qapi-visit-core.c
+++ b/qapi/qapi-visit-core.c
@@ -19,10 +19,12 @@
 #include "qapi/qmp/qerror.h"
 #include "qapi/visitor.h"
 #include "qapi/visitor-impl.h"
+#include "trace.h"
 
 void visit_complete(Visitor *v, void *opaque)
 {
 assert(v->type != VISITOR_OUTPUT || v->complete);
+trace_visit_complete(v, opaque);
 if (v->complete) {
 v->complete(v, opaque);
 }
@@ -30,6 +32,7 @@ void visit_complete(Visitor *v, void *opaque)
 
 void visit_free(Visitor *v)
 {
+trace_visit_free(v);
 if (v) {
 v->free(v);
 }
@@ -40,6 +43,7 @@ void visit_start_struct(Visitor *v, const char *name, void 
**obj,
 {
 Error *err = NULL;
 
+trace_visit_start_struct(v, name, obj, size);
 if (obj) {
 assert(size);
 assert(!(v->type & VISITOR_OUTPUT) || *obj);
@@ -53,6 +57,7 @@ void visit_start_struct(Visitor *v, const char *name, void 
**obj,
 
 void visit_check_struct(Visitor *v, Error **errp)
 {
+trace_visit_check_struct(v);
 if (v->check_struct) {
 v->check_struct(v, errp);
 }
@@ -60,6 +65,7 @@ void visit_check_struct(Visitor *v, Error **errp)
 
 void visit_end_struct(Visitor *v, void **obj)
 {
+trace_visit_end_struct(v, obj);
 v->end_struct(v, obj);
 }
 
@@ -69,6 +75,7 @@ void visit_start_list(Visitor *v, const char *name, 
GenericList **list,
 Error *err = NULL;
 
 assert(!list || size >= sizeof(GenericList));
+trace_visit_start_list(v, name, list, size);
 v->start_list(v, name, list, size, );
 if (list && (v->type & VISITOR_INPUT)) {
 assert(!(err && *list));
@@ -79,11 +86,13 @@ void visit_start_list(Visitor *v, const char *name, 
GenericList **list,
 GenericList *visit_next_list(Visitor *v, GenericList *tail, size_t size)
 {
 assert(tail && size >= sizeof(GenericList));
+trace_visit_next_list(v, tail, size);
 return v->next_list(v, tail, size);
 }
 
 void visit_end_list(Visitor *v, void **obj)
 {
+trace_visit_end_list(v, obj);
 v->end_list(v, obj);
 }
 
@@ -95,6 +104,7 @@ void visit_start_alternate(Visitor *v, const char *name,
 
 assert(obj && size >= sizeof(GenericAlternate));
 assert(!(v->type & VISITOR_OUTPUT) || *obj);
+trace_visit_start_alternate(v, name, obj, size, promote_int);
 if (v->start_alternate) {
 v->start_alternate(v, name, obj, size, promote_int, );
 }
@@ -106,6 +116,7 @@ void visit_start_alternate(Visitor *v, const char *name,
 
 void visit_end_alternate(Visitor *v, void **obj)
 {
+trace_visit_end_alternate(v, obj);
 if (v->end_alternate) {
 v->end_alternate(v, obj);
 }
@@ -113,6 +124,7 @@ void visit_end_alternate(Visitor *v, void **obj)
 
 bool visit_optional(Visitor *v, const char *name, bool *present)
 {
+trace_visit_optional(v, name, present);
 if (v->optional) {
 v->optional(v, name, present);
 }
@@ -127,6 +139,7 @@ bool visit_is_input(Visitor *v)
 void visit_type_int(Visitor *v, const char *name, int64_t *obj, Error **errp)
 {
 assert(obj);
+trace_visit_type_int(v, name, obj);
 v->type_int64(v, name, obj, errp);
 }
 
@@ -151,6 +164,7 @@ void visit_type_uint8(Visitor *v, const char *name, uint8_t 
*obj,
   Error **errp)
 {
 uint64_t value = *obj;
+trace_visit_type_uint8(v, name, obj);
 visit_type_uintN(v, , name, UINT8_MAX, "uint8_t", errp);
 *obj = value;
 }
@@ -159,6 +173,7 @@ void visit_type_uint16(Visitor *v, const char *name, 
uint16_t *obj,
Error **errp)
 {
 uint64_t value = *obj;
+trace_visit_type_uint16(v, name, obj);
 visit_type_uintN(v, , name, UINT16_MAX, "uint16_t", errp);
 *obj = value;
 }
@@ -167,6 +182,7 @@ void visit_type_uint32(Visitor *v, const char *name, 
uint32_t *obj,
Error **errp)
 {
 uint64_t value = *obj;
+trace_visit_type_uint32(v, name, obj);
 visit_type_uintN(v, , name, UINT32_MAX, "uint32_t", errp);
 *obj = value;
 }
@@ -175,6 +191,7 @@ void visit_type_uint64(Visitor *v, const char *name, 
uint64_t *obj,
Error **errp)
 {
 assert(obj);
+trace_visit_type_uint64(v, name, obj);
 v->type_uint64(v, name, obj, errp);
 }
 
@@ -199,6 +216,7 @@ static void 

[Qemu-block] [PATCH v14 00/21] QAPI/QOM work for non-scalar object properties

[Daniel P. Berrange]

An update of a series previously posted

 v1: https://lists.gnu.org/archive/html/qemu-devel/2016-02/msg04618.html
 v2: https://lists.gnu.org/archive/html/qemu-devel/2016-03/msg01454.html
 v3: https://lists.gnu.org/archive/html/qemu-devel/2016-03/msg02498.html
 v4: https://lists.gnu.org/archive/html/qemu-devel/2016-05/msg01661.html
 v5: https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg00485.html
 v6: https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg03876.html
 v7: https://lists.gnu.org/archive/html/qemu-devel/2016-07/msg00919.html
 v8: https://lists.gnu.org/archive/html/qemu-devel/2016-07/msg03115.html
 v9: https://lists.gnu.org/archive/html/qemu-devel/2016-08/msg02653.html
 v10: https://lists.gnu.org/archive/html/qemu-devel/2016-08/msg02694.html
 v11: https://lists.gnu.org/archive/html/qemu-devel/2016-09/msg00652.html
 v12: https://lists.gnu.org/archive/html/qemu-devel/2016-09/msg03559.html
 v13: https://lists.gnu.org/archive/html/qemu-devel/2016-09/msg04212.html
 v14: https://lists.gnu.org/archive/html/qemu-devel/2016-09/msg07037.html

This series provides the infrastructure to allow use of non-scalar
properties with the -object CLI arg, and object_add monitor commands.
eg a property which is a list of structs. The syntax used for this is
intentionally compatible with the syntax used by the block layer. This
will allow the qdict_crumple method to be used by the block layer to
convert from QemuOpts into structured QAPI block layer structs at a
future date. It is already used by one of Max's patch series, by
Kevin's -blockdev series, and by my own ACL series.

This patch series has grown a fair bit larger since  previous
postings (19^H^H21 patches instead of 6). This was to avoid trying
to squish too much into one patch, to make it easier to review.

It does not neccessarily need to be merged in one go. There are
three logical blocks of patches here.

Patches 1-8 are the core support for visiting QemuOpts using the
QObjectInputVisitor class. Merging this would unblock at least
Kevin's blockdev series, and probably Max's series too.

The patches 9-14 add enhancements to the code to support various
special cases / edge cases needed to provide fully semantic
replacement of the OptsVisitor

Patches 15-21 deal with converting code over to use the new visitor
logic.

Once this is merged, there is also scope to further enhance the
QObjectInputVisitor to deal with visiting single-properties,
which would let us delete StringInputVisitor too. This would
mean we have a single input visitor class, instead of the
current three.

Changed in v15:

 - Re-arranged patch series to allow easier incremental merge
   of core functionality
 - Extra error checking of some scenarios to be avoided (Eric)
 - Misc typos (Eric)
 - Rename QEMU_OPTS_REPEAT_POLICY_LIST to
   QEMU_OPTS_REPEAT_POLICY_ALL, and add QEMU_OPTS_REPEAT_POLICY_ERROR
   (Eric)

Changed in v14:

 - Extend qemu_opts_to_qdict to optionally handle case of
   repeated keys without dropping them.
 - Support parsing of integer ranges for "list of int" visits
   for back compat with OptsVisitor
 - Support visiting nested structs which are represented as
   a flat keyspace for back compat with OptsVisitor
 - Add trace events for visitor functions to facilitate
   debugging
 - Add a QObjectInputVisitor constructor which takes a
   QemuOpts object, to avoid repeated code patterns in callers
 - Greater test coverage of -object to validate correct handling
   of "list of ints" legacy OptsVisitor syntax
 - Convert all other use of OptsVisitor to QObjectInputVisitor
 - Delete OptsVisitor

Changed in v13:

 - Fix typos (Kevin)
 - Remove unneeded line breaks (Kevin)

Changed in v12:

 - More user friendly error message for mixing dict/list
   keys (Kevin)
 - Report error instead of assert for non-contiguous list
   keys (Kevin)
 - Fix tests for non-contiguous list keys (Kevin)
 - Add tests for escaping of '.' when crumpling (Kevin)
 - Fix remaining references to Qmp(In|Out)putVisitor (Markus)
 - Misc typos / whitespace fixes (Eric, Kevin)
 - Avoid touching 'ret' when parsing int64 fails (Eric)
 - Testing of more edge cases in QObjectInputVisitor (Eric)
 - Simplify API doc format (Markus)
 - Use parse_option_size instead of qemu_strtosz_suffix
   for consistency (Kevin)
 - Use safer qobject_to_qdict casts (Eric)

Changed in v11:

 - Split QAPI/QOM patches off from the access control patches

Changed in v10:

 - Fixed stupid build mistake

Changed in v9:

 - Rename QmpInputVisitor -> QObjectInputVisitor (Markus/Eric)
 - Rename QmpOutputVisitor -> QObjectOutputVisitor (Markus/Eric)
 - Drop "strict" param from qobject_string_visitor_new() (Marus)
 - Misc docs typos
 - Add a visitor able to use strict or string types (for Eric's
   netdev series)
 - Add a authorization API implementation that uses PAM

Changed in v8:

 - Rebase due to merge of Visitor API changes (Eric)

Changed in v7:

 - Misc typos in API docs (Marc-André)
 - Fix parsing of properties using type_size 

Re: [Qemu-block] [PATCH v3] block: Turn on "unmap" in active commit

[Kevin Wolf]

Am 30.09.2016 um 04:12 hat Fam Zheng geschrieben:
> On Thu, 09/29 13:41, Stefan Hajnoczi wrote:
> > On Tue, Sep 27, 2016 at 07:14:52PM +0800, Fam Zheng wrote:
> > > We already specified BDRV_O_UNMAP when opening images in 'qemu-img
> > > commit', but didn't turn on the "unmap" in the active commit job. This
> > > patch fixes that so that zeroed clusters in top image can be discarded
> > > which is desired in the virt-sparsify use case, where a temporary
> > > overlay is created and fstrim'ed before commiting back, to free space in
> > > the original image.
> > > 
> > > This also enables it for block-commit.
> > > 
> > > Signed-off-by: Fam Zheng 
> > > ---
> > > v3: Change the right parameter.
> > > v2: Add "unmap" to block-commit as well. [Kevin]
> > > ---
> > >  block/mirror.c | 2 +-
> > >  1 file changed, 1 insertion(+), 1 deletion(-)
> > > 
> > > diff --git a/block/mirror.c b/block/mirror.c
> > > index f9d1fec..8847ec5 100644
> > > --- a/block/mirror.c
> > > +++ b/block/mirror.c
> > > @@ -1042,7 +1042,7 @@ void commit_active_start(const char *job_id, 
> > > BlockDriverState *bs,
> > >  
> > >  mirror_start_job(job_id, bs, base, NULL, speed, 0, 0,
> > >   MIRROR_LEAVE_BACKING_CHAIN,
> > > - on_error, on_error, false, cb, opaque, _err,
> > > + on_error, on_error, true, cb, opaque, _err,
> > >   _active_job_driver, false, base, 
> > > auto_complete);
> > >  if (local_err) {
> > >  error_propagate(errp, local_err);
> > 
> > Why is unmap an option at all?
> > 
> > What's wrong with using BDRV_REQ_MAY_UNMAP on all
> > blk_aio_pwrite_zeroes() calls?
> 
> Because unmap is an QMP option of drive-backup. I think in the drive-mirror
> context, it mitigates the limitation that we have no control over target's
> BDRV_O_UNMAP (always inherited from source).

Wouldn't the more straightforward implementation then be if
qmp_drive_mirror() set BDRV_O_UNMAP for the target depending on the flag
rather than passing the flag down to the mirror job?

Hm... And should BDRV_O_UNMAP really be a BlockBackend option rather
than a BDS one? We already enable it unconditionally on non-root nodes
and it seems to make sense to me to allow discard e.g. from a block job,
but not from the guest.

Kevin

Re: [Qemu-block] [PATCH v3] block: Turn on "unmap" in active commit

[Stefan Hajnoczi]

On Fri, Sep 30, 2016 at 10:12:05AM +0800, Fam Zheng wrote:
> On Thu, 09/29 13:41, Stefan Hajnoczi wrote:
> > On Tue, Sep 27, 2016 at 07:14:52PM +0800, Fam Zheng wrote:
> > > We already specified BDRV_O_UNMAP when opening images in 'qemu-img
> > > commit', but didn't turn on the "unmap" in the active commit job. This
> > > patch fixes that so that zeroed clusters in top image can be discarded
> > > which is desired in the virt-sparsify use case, where a temporary
> > > overlay is created and fstrim'ed before commiting back, to free space in
> > > the original image.
> > > 
> > > This also enables it for block-commit.
> > > 
> > > Signed-off-by: Fam Zheng 
> > > ---
> > > v3: Change the right parameter.
> > > v2: Add "unmap" to block-commit as well. [Kevin]
> > > ---
> > >  block/mirror.c | 2 +-
> > >  1 file changed, 1 insertion(+), 1 deletion(-)
> > > 
> > > diff --git a/block/mirror.c b/block/mirror.c
> > > index f9d1fec..8847ec5 100644
> > > --- a/block/mirror.c
> > > +++ b/block/mirror.c
> > > @@ -1042,7 +1042,7 @@ void commit_active_start(const char *job_id, 
> > > BlockDriverState *bs,
> > >  
> > >  mirror_start_job(job_id, bs, base, NULL, speed, 0, 0,
> > >   MIRROR_LEAVE_BACKING_CHAIN,
> > > - on_error, on_error, false, cb, opaque, _err,
> > > + on_error, on_error, true, cb, opaque, _err,
> > >   _active_job_driver, false, base, 
> > > auto_complete);
> > >  if (local_err) {
> > >  error_propagate(errp, local_err);
> > 
> > Why is unmap an option at all?
> > 
> > What's wrong with using BDRV_REQ_MAY_UNMAP on all
> > blk_aio_pwrite_zeroes() calls?
> 
> Because unmap is an QMP option of drive-backup. I think in the drive-mirror
> context, it mitigates the limitation that we have no control over target's
> BDRV_O_UNMAP (always inherited from source).

That doesn't explain why unmap is an option.

Is it because the user might not want to unmap the target and would
rather writes zeroes?

For example in qcow2 they might want to leave the data clusters
allocated and simply mark their contents zeroed:

"Standard Cluster Descriptor:

Bit   0:If set to 1, the cluster reads as all zeros. The host
cluster offset can be used to describe a preallocation,
but it won't be used for reading data from this cluster,
nor is data read from the backing file if the cluster is
unallocated."


signature.asc
Description: PGP signature

[Qemu-block] [PATCH v8 36/36] sd: Add 'lock-mode' property

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/sd/sd.c | 8 
 1 file changed, 8 insertions(+)

diff --git a/hw/sd/sd.c b/hw/sd/sd.c
index 8e88e83..6c18fb8 100644
--- a/hw/sd/sd.c
+++ b/hw/sd/sd.c
@@ -124,6 +124,7 @@ struct SDState {
 qemu_irq readonly_cb;
 qemu_irq inserted_cb;
 BlockBackend *blk;
+ImageLockMode lock_mode;
 
 bool enable;
 };
@@ -1886,6 +1887,7 @@ static void sd_instance_finalize(Object *obj)
 
 static void sd_realize(DeviceState *dev, Error **errp)
 {
+Error *local_err = NULL;
 SDState *sd = SD_CARD(dev);
 
 if (sd->blk && blk_is_read_only(sd->blk)) {
@@ -1894,12 +1896,18 @@ static void sd_realize(DeviceState *dev, Error **errp)
 }
 
 if (sd->blk) {
+blk_lock_image(sd->blk, sd->lock_mode, _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
 blk_set_dev_ops(sd->blk, _block_ops, sd);
 }
 }
 
 static Property sd_properties[] = {
 DEFINE_PROP_DRIVE("drive", SDState, blk),
+DEFINE_PROP_LOCK_MODE("lock-mode", SDState, lock_mode),
 /* We do not model the chip select pin, so allow the board to select
  * whether card should be in SSI or MMC/SD mode.  It is also up to the
  * board to ensure that ssi transfers only occur when the chip select
-- 
2.7.4

[Qemu-block] [PATCH v8 34/36] onenand: Add 'lock-mode' property

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/block/onenand.c | 7 +++
 1 file changed, 7 insertions(+)

diff --git a/hw/block/onenand.c b/hw/block/onenand.c
index 8d84227..9b058e8 100644
--- a/hw/block/onenand.c
+++ b/hw/block/onenand.c
@@ -778,6 +778,7 @@ static int onenand_initfn(SysBusDevice *sbd)
 OneNANDState *s = ONE_NAND(dev);
 uint32_t size = 1 << (24 + ((s->id.dev >> 4) & 7));
 void *ram;
+Error *local_err = NULL;
 
 s->base = (hwaddr)-1;
 s->rdy = NULL;
@@ -796,6 +797,11 @@ static int onenand_initfn(SysBusDevice *sbd)
 error_report("Can't use a read-only drive");
 return -1;
 }
+blk_lock_image(s->blk, s->lock_mode, _err);
+if (local_err) {
+error_report_err(local_err);
+return -1;
+}
 s->blk_cur = s->blk;
 }
 s->otp = memset(g_malloc((64 + 2) << PAGE_SHIFT),
@@ -828,6 +834,7 @@ static Property onenand_properties[] = {
 DEFINE_PROP_UINT16("version_id", OneNANDState, id.ver, 0),
 DEFINE_PROP_INT32("shift", OneNANDState, shift, 0),
 DEFINE_PROP_DRIVE("drive", OneNANDState, blk),
+DEFINE_PROP_LOCK_MODE("lock-mode", OneNANDState, lock_mode),
 DEFINE_PROP_END_OF_LIST(),
 };
 
-- 
2.7.4

[Qemu-block] [PATCH v8 27/36] tests: Use null-co:// instead of /dev/null

[Fam Zheng]

With image locking, opening /dev/null can fail when multiple tests run
in parallel (make -j2, for example). Use null-co:// as the null protocol
doesn't do image locking.

While it's arguable we could special-case /dev/null, /dev/zero,
/dev/urandom etc in raw-posix driver, it is not really necessary because
user can always specify lock-mode=off when it is appropriate. So let's
write sensible testing code too.

Signed-off-by: Fam Zheng 
Reviewed-by: Max Reitz 
---
 tests/drive_del-test.c| 2 +-
 tests/nvme-test.c | 2 +-
 tests/usb-hcd-uhci-test.c | 2 +-
 tests/usb-hcd-xhci-test.c | 2 +-
 tests/virtio-blk-test.c   | 2 +-
 tests/virtio-scsi-test.c  | 4 ++--
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/drive_del-test.c b/tests/drive_del-test.c
index 121b9c9..2175139 100644
--- a/tests/drive_del-test.c
+++ b/tests/drive_del-test.c
@@ -92,7 +92,7 @@ static void test_after_failed_device_add(void)
 static void test_drive_del_device_del(void)
 {
 /* Start with a drive used by a device that unplugs instantaneously */
-qtest_start("-drive if=none,id=drive0,file=/dev/null,format=raw"
+qtest_start("-drive if=none,id=drive0,file=null-co://,format=raw"
 " -device virtio-scsi-pci"
 " -device scsi-hd,drive=drive0,id=dev0");
 
diff --git a/tests/nvme-test.c b/tests/nvme-test.c
index c8bece4..7674a44 100644
--- a/tests/nvme-test.c
+++ b/tests/nvme-test.c
@@ -22,7 +22,7 @@ int main(int argc, char **argv)
 g_test_init(, , NULL);
 qtest_add_func("/nvme/nop", nop);
 
-qtest_start("-drive id=drv0,if=none,file=/dev/null,format=raw "
+qtest_start("-drive id=drv0,if=none,file=null-co://,format=raw "
 "-device nvme,drive=drv0,serial=foo");
 ret = g_test_run();
 
diff --git a/tests/usb-hcd-uhci-test.c b/tests/usb-hcd-uhci-test.c
index 5cd59ad..3684503 100644
--- a/tests/usb-hcd-uhci-test.c
+++ b/tests/usb-hcd-uhci-test.c
@@ -85,7 +85,7 @@ int main(int argc, char **argv)
 qtest_add_func("/uhci/pci/hotplug/usb-storage", test_usb_storage_hotplug);
 
 qtest_start("-device piix3-usb-uhci,id=uhci,addr=1d.0"
-" -drive id=drive0,if=none,file=/dev/null,format=raw"
+" -drive id=drive0,if=none,file=null-co://,format=raw"
 " -device usb-tablet,bus=uhci.0,port=1");
 ret = g_test_run();
 qtest_end();
diff --git a/tests/usb-hcd-xhci-test.c b/tests/usb-hcd-xhci-test.c
index 22513e9..031764d 100644
--- a/tests/usb-hcd-xhci-test.c
+++ b/tests/usb-hcd-xhci-test.c
@@ -89,7 +89,7 @@ int main(int argc, char **argv)
 qtest_add_func("/xhci/pci/hotplug/usb-uas", test_usb_uas_hotplug);
 
 qtest_start("-device nec-usb-xhci,id=xhci"
-" -drive id=drive0,if=none,file=/dev/null,format=raw");
+" -drive id=drive0,if=none,file=null-co://,format=raw");
 ret = g_test_run();
 qtest_end();
 
diff --git a/tests/virtio-blk-test.c b/tests/virtio-blk-test.c
index 811cf75..5dba567 100644
--- a/tests/virtio-blk-test.c
+++ b/tests/virtio-blk-test.c
@@ -66,7 +66,7 @@ static QPCIBus *pci_test_start(void)
 tmp_path = drive_create();
 
 cmdline = g_strdup_printf("-drive if=none,id=drive0,file=%s,format=raw "
-"-drive if=none,id=drive1,file=/dev/null,format=raw "
+"-drive if=none,id=drive1,file=null-co://,format=raw "
 "-device virtio-blk-pci,id=drv0,drive=drive0,"
 "addr=%x.%x",
 tmp_path, PCI_SLOT, PCI_FN);
diff --git a/tests/virtio-scsi-test.c b/tests/virtio-scsi-test.c
index f1489e6..3c2f5df 100644
--- a/tests/virtio-scsi-test.c
+++ b/tests/virtio-scsi-test.c
@@ -40,7 +40,7 @@ static void qvirtio_scsi_start(const char *extra_opts)
 char *cmdline;
 
 cmdline = g_strdup_printf(
-"-drive id=drv0,if=none,file=/dev/null,format=raw "
+"-drive id=drv0,if=none,file=null-co://,format=raw "
 "-device virtio-scsi-pci,id=vs0 "
 "-device scsi-hd,bus=vs0.0,drive=drv0 %s",
 extra_opts ? : "");
@@ -192,7 +192,7 @@ static void hotplug(void)
 {
 QDict *response;
 
-qvirtio_scsi_start("-drive id=drv1,if=none,file=/dev/null,format=raw");
+qvirtio_scsi_start("-drive id=drv1,if=none,file=null-co://,format=raw");
 response = qmp("{\"execute\": \"device_add\","
" \"arguments\": {"
"   \"driver\": \"scsi-hd\","
-- 
2.7.4

[Qemu-block] [PATCH v8 32/36] m25p80: Add 'lock-mode' property

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/block/m25p80.c | 8 
 1 file changed, 8 insertions(+)

diff --git a/hw/block/m25p80.c b/hw/block/m25p80.c
index d29ff4c..3c1765a 100644
--- a/hw/block/m25p80.c
+++ b/hw/block/m25p80.c
@@ -380,6 +380,7 @@ typedef struct Flash {
 SSISlave parent_obj;
 
 BlockBackend *blk;
+ImageLockMode lock_mode;
 
 uint8_t *storage;
 uint32_t size;
@@ -1144,6 +1145,7 @@ static void m25p80_realize(SSISlave *ss, Error **errp)
 {
 Flash *s = M25P80(ss);
 M25P80Class *mc = M25P80_GET_CLASS(s);
+Error *local_err = NULL;
 
 s->pi = mc->pi;
 
@@ -1151,6 +1153,11 @@ static void m25p80_realize(SSISlave *ss, Error **errp)
 s->dirty_page = -1;
 
 if (s->blk) {
+blk_lock_image(s->blk, s->lock_mode, _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
 DB_PRINT_L(0, "Binding to IF_MTD drive\n");
 s->storage = blk_blockalign(s->blk, s->size);
 
@@ -1185,6 +1192,7 @@ static Property m25p80_properties[] = {
 DEFINE_PROP_UINT8("spansion-cr3nv", Flash, spansion_cr3nv, 0x2),
 DEFINE_PROP_UINT8("spansion-cr4nv", Flash, spansion_cr4nv, 0x10),
 DEFINE_PROP_DRIVE("drive", Flash, blk),
+DEFINE_PROP_LOCK_MODE("lock-mode", Flash, lock_mode),
 DEFINE_PROP_END_OF_LIST(),
 };
 
-- 
2.7.4

[Qemu-block] [PATCH v8 31/36] fdc: Add lock-mode qdev properties

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/block/fdc.c | 19 ++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index b79873a..20684b8 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -158,6 +158,7 @@ typedef enum FDiskFlags {
 typedef struct FDrive {
 FDCtrl *fdctrl;
 BlockBackend *blk;
+ImageLockMode lock_mode;
 /* Drive status */
 FloppyDriveType drive;/* CMOS drive type*/
 uint8_t perpendicular;/* 2.88 MB access mode*/
@@ -2446,11 +2447,23 @@ static void fdctrl_realize_common(FDCtrl *fdctrl, Error 
**errp)
 {
 int i, j;
 static int command_tables_inited = 0;
+Error *local_err = NULL;
 
 if (fdctrl->fallback == FLOPPY_DRIVE_TYPE_AUTO) {
 error_setg(errp, "Cannot choose a fallback FDrive type of 'auto'");
 }
 
+for (i = 0; i < 2; ++i) {
+if (fdctrl->drives[i].blk) {
+blk_lock_image(fdctrl->drives[i].blk, fdctrl->drives[i].lock_mode,
+   _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
+}
+}
+
 /* Fill 'command_to_handler' lookup table */
 if (!command_tables_inited) {
 command_tables_inited = 1;
@@ -2495,7 +2508,6 @@ static void isabus_fdc_realize(DeviceState *dev, Error 
**errp)
 FDCtrlISABus *isa = ISA_FDC(dev);
 FDCtrl *fdctrl = >state;
 Error *err = NULL;
-
 isa_register_portio_list(isadev, >portio_list,
  isa->iobase, fdc_portio_list, fdctrl,
  "fdc");
@@ -2608,6 +2620,8 @@ static Property isa_fdc_properties[] = {
 DEFINE_PROP_UINT32("dma", FDCtrlISABus, dma, 2),
 DEFINE_PROP_DRIVE("driveA", FDCtrlISABus, state.drives[0].blk),
 DEFINE_PROP_DRIVE("driveB", FDCtrlISABus, state.drives[1].blk),
+DEFINE_PROP_LOCK_MODE("lock-modeA", FDCtrlISABus, 
state.drives[0].lock_mode),
+DEFINE_PROP_LOCK_MODE("lock-modeB", FDCtrlISABus, 
state.drives[1].lock_mode),
 DEFINE_PROP_BIT("check_media_rate", FDCtrlISABus, state.check_media_rate,
 0, true),
 DEFINE_PROP_DEFAULT("fdtypeA", FDCtrlISABus, state.drives[0].drive,
@@ -2667,6 +2681,8 @@ static const VMStateDescription vmstate_sysbus_fdc ={
 static Property sysbus_fdc_properties[] = {
 DEFINE_PROP_DRIVE("driveA", FDCtrlSysBus, state.drives[0].blk),
 DEFINE_PROP_DRIVE("driveB", FDCtrlSysBus, state.drives[1].blk),
+DEFINE_PROP_LOCK_MODE("lock-modeA", FDCtrlISABus, 
state.drives[0].lock_mode),
+DEFINE_PROP_LOCK_MODE("lock-modeB", FDCtrlISABus, 
state.drives[1].lock_mode),
 DEFINE_PROP_DEFAULT("fdtypeA", FDCtrlSysBus, state.drives[0].drive,
 FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
 FloppyDriveType),
@@ -2696,6 +2712,7 @@ static const TypeInfo sysbus_fdc_info = {
 
 static Property sun4m_fdc_properties[] = {
 DEFINE_PROP_DRIVE("drive", FDCtrlSysBus, state.drives[0].blk),
+DEFINE_PROP_LOCK_MODE("lock-mode", FDCtrlISABus, 
state.drives[0].lock_mode),
 DEFINE_PROP_DEFAULT("fdtype", FDCtrlSysBus, state.drives[0].drive,
 FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
 FloppyDriveType),
-- 
2.7.4

[Qemu-block] [PATCH v8 23/36] iotests: 087: Disable image locking in cases where file is shared

[Fam Zheng]

Otherwise the error handling we are expecting will be masked by the
preceding image locking check, and is going to be indistinguishable
because the error messages are all the same.

Signed-off-by: Fam Zheng 
Reviewed-by: Max Reitz 
---
 tests/qemu-iotests/087 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/qemu-iotests/087 b/tests/qemu-iotests/087
index 5c04577..91e4c71 100755
--- a/tests/qemu-iotests/087
+++ b/tests/qemu-iotests/087
@@ -84,6 +84,7 @@ run_qemu -drive 
driver=$IMGFMT,id=disk,node-name=test-node,file="$TEST_IMG" <

[Qemu-block] [PATCH v8 25/36] iotests: 130: Check image info locklessly

[Fam Zheng]

By the time _img_info is run, QEMU process's resources may still be on
its way being cleaned up, asynchronously, even though the process itself
is already gone after the "kill -KILL" and "wait" commands in
_cleanup_qemu.

Change the last HMP command to 'q' to ensure the locks are released.

Signed-off-by: Fam Zheng 
---
 tests/qemu-iotests/130 | 4 ++--
 tests/qemu-iotests/130.out | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/qemu-iotests/130 b/tests/qemu-iotests/130
index ecc8a5b..f14a04f 100755
--- a/tests/qemu-iotests/130
+++ b/tests/qemu-iotests/130
@@ -60,7 +60,7 @@ echo
 # Test that a backing file isn't written
 _launch_qemu -drive 
id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base"
 _send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)"
-_send_qemu_cmd $QEMU_HANDLE '' '(qemu)'
+_send_qemu_cmd $QEMU_HANDLE 'q' '(qemu)'
 _cleanup_qemu
 _img_info | _filter_img_info
 
@@ -69,7 +69,7 @@ _img_info | _filter_img_info
 _make_test_img -F raw -b "$TEST_IMG.orig" 64M
 _launch_qemu -drive 
id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base",backing.driver=$IMGFMT
 _send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)"
-_send_qemu_cmd $QEMU_HANDLE '' '(qemu)'
+_send_qemu_cmd $QEMU_HANDLE 'q' '(qemu)'
 _cleanup_qemu
 _img_info | _filter_img_info
 
diff --git a/tests/qemu-iotests/130.out b/tests/qemu-iotests/130.out
index ae95b50..2422329 100644
--- a/tests/qemu-iotests/130.out
+++ b/tests/qemu-iotests/130.out
@@ -10,14 +10,14 @@ virtual size: 64M (67108864 bytes)
 
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) 
ccocomcommcommicommitcommit
 commit tcommit 
tecommit 
tescommit 
testcommit 
testdcommit 
testdicommit 
testdiscommit testdisk
-(qemu) 
+(qemu) q
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
backing_file=TEST_DIR/t.IMGFMT.orig backing_fmt=raw
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) 
ccocomcommcommicommitcommit
 commit tcommit 
tecommit 
tescommit 
testcommit 
testdcommit 
testdicommit 
testdiscommit testdisk
-(qemu) 
+(qemu) q
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
-- 
2.7.4

Re: [Qemu-block] [Qemu-devel] [PATCH] qcow2: Support BDRV_REQ_MAY_UNMAP

[Kevin Wolf]

Am 30.09.2016 um 04:04 hat Fam Zheng geschrieben:
> On Thu, 09/29 12:39, Kevin Wolf wrote:
> > Am 29.09.2016 um 11:55 hat Fam Zheng geschrieben:
> > > On Thu, 09/29 11:29, Kevin Wolf wrote:
> > > > Am 28.09.2016 um 09:04 hat Fam Zheng geschrieben:
> > > > > Handling this is similar to what is done to the L2 entry in the case 
> > > > > of
> > > > > compressed clusters.
> > > > > 
> > > > > Signed-off-by: Fam Zheng 
> > > > > ---
> > > > >  block/qcow2-cluster.c | 9 +
> > > > >  block/qcow2.c | 3 ++-
> > > > >  block/qcow2.h | 3 ++-
> > > > >  3 files changed, 9 insertions(+), 6 deletions(-)
> > > > > 
> > > > > diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
> > > > > index 61d1ffd..928c1e2 100644
> > > > > --- a/block/qcow2-cluster.c
> > > > > +++ b/block/qcow2-cluster.c
> > > > > @@ -1558,7 +1558,7 @@ fail:
> > > > >   * clusters.
> > > > >   */
> > > > >  static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
> > > > > -  uint64_t nb_clusters)
> > > > > +  uint64_t nb_clusters, int flags)
> > > > >  {
> > > > >  BDRVQcow2State *s = bs->opaque;
> > > > >  uint64_t *l2_table;
> > > > > @@ -1582,7 +1582,7 @@ static int zero_single_l2(BlockDriverState *bs, 
> > > > > uint64_t offset,
> > > > >  
> > > > >  /* Update L2 entries */
> > > > >  qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, 
> > > > > l2_table);
> > > > > -if (old_offset & QCOW_OFLAG_COMPRESSED) {
> > > > > +if (old_offset & QCOW_OFLAG_COMPRESSED || flags & 
> > > > > BDRV_REQ_MAY_UNMAP) {
> > > > >  l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);
> > > > >  qcow2_free_any_clusters(bs, old_offset, 1, 
> > > > > QCOW2_DISCARD_REQUEST);
> > > > >  } else {
> > > > 
> > > > I don't think we should always do this for BDRV_REQ_MAY_UNMAP. The user
> > > > may want to keep the image fully allocated even if the guest (or block
> > > > job etc.) thinks this can be discarded.
> > > > 
> > > > When we discussed this in another email thread (?), I think I suggested
> > > > checking whether the image was opened with pass-discard-request=on. If
> > > > so, go ahead and deallocate the cluster, otherwise keep it allocated.
> > > > 
> > > > In those cases where pass-discard-request=off, it doesn't make a lot of
> > > > sense to deallocate the cluster anyway because it won't shrink the file
> > > > size.
> > > 
> > > I think this patch does what you mean:
> > > 
> > > $ strace -f -e fallocate qemu-io \
> > > -c 'open -o pass-discard-request=on /var/tmp/test' \
> > > -c 'write 0 1M' \
> > > -c 'write -u -z 0 1M' \
> > > 2>&1 >/dev/null | \
> > > grep fallocate
> > > [pid 17548] fallocate(17, FALLOC_FL_KEEP_SIZE|FALLOC_FL_PUNCH_HOLE, 
> > > 327680, 1048576) = 0
> > > $ strace -f -e fallocate qemu-io \
> > > -c 'open -o pass-discard-request=off /var/tmp/test' \
> > > -c 'write 0 1M' \
> > > -c 'write -u -z 0 1M' \
> > > 2>&1 >/dev/null | \
> > > grep fallocate
> > > 
> > > Because there is another check of pass-discard-request value in
> > > update_refcount:
> > > 
> > > if (refcount == 0 && s->discard_passthrough[type]) {
> > > update_refcount_discard(bs, cluster_offset, s->cluster_size);
> > > }
> > 
> > What I mean is that in the second case, you're still uselessly
> > deallocating the cluster on the qcow2 level while you can't reclaim it
> > on the filesystem level. So it would be better to leave it allocated in
> > qcow2, too, so that you don't get an expensive reallocation the next
> > time you write to it.
> > 
> 
> Like this?

That's what I had in mind, but actually I think Paolo is right and the
BDRV_O_UNMAP check in bdrv_co_pwrite_zeroes() already does all that is
needed. So the approach of the original patch should be okay.

Kevin

> ---
> 
> diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
> index 928c1e2..3d3cea4 100644
> --- a/block/qcow2-cluster.c
> +++ b/block/qcow2-cluster.c
> @@ -1582,7 +1582,9 @@ static int zero_single_l2(BlockDriverState *bs, 
> uint64_t offset,
> 
>  /* Update L2 entries */
>  qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);
> -if (old_offset & QCOW_OFLAG_COMPRESSED || flags & 
> BDRV_REQ_MAY_UNMAP) {
> +if (old_offset & QCOW_OFLAG_COMPRESSED ||
> +((flags & BDRV_REQ_MAY_UNMAP) &&
> + s->discard_passthrough[QCOW2_DISCARD_REQUEST])) {
>  l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);
>  qcow2_free_any_clusters(bs, old_offset, 1, 
> QCOW2_DISCARD_REQUEST);
>  } else {

[Qemu-block] [PATCH v8 33/36] nand: Add 'lock-mode' property

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/block/nand.c | 8 
 1 file changed, 8 insertions(+)

diff --git a/hw/block/nand.c b/hw/block/nand.c
index c69e675..75b5a68 100644
--- a/hw/block/nand.c
+++ b/hw/block/nand.c
@@ -64,6 +64,7 @@ struct NANDFlashState {
 int page_shift, oob_shift, erase_shift, addr_shift;
 uint8_t *storage;
 BlockBackend *blk;
+ImageLockMode lock_mode;
 int mem_oob;
 
 uint8_t cle, ale, ce, wp, gnd;
@@ -373,6 +374,7 @@ static void nand_realize(DeviceState *dev, Error **errp)
 {
 int pagesize;
 NANDFlashState *s = NAND(dev);
+Error *local_err = NULL;
 
 s->buswidth = nand_flash_ids[s->chip_id].width >> 3;
 s->size = nand_flash_ids[s->chip_id].size << 20;
@@ -407,6 +409,11 @@ static void nand_realize(DeviceState *dev, Error **errp)
 error_setg(errp, "Can't use a read-only drive");
 return;
 }
+blk_lock_image(s->blk, s->lock_mode, _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
 if (blk_getlength(s->blk) >=
 (s->pages << s->page_shift) + (s->pages << s->oob_shift)) {
 pagesize = 0;
@@ -427,6 +434,7 @@ static Property nand_properties[] = {
 DEFINE_PROP_UINT8("manufacturer_id", NANDFlashState, manf_id, 0),
 DEFINE_PROP_UINT8("chip_id", NANDFlashState, chip_id, 0),
 DEFINE_PROP_DRIVE("drive", NANDFlashState, blk),
+DEFINE_PROP_LOCK_MODE("lock-mode", NANDFlashState, lock_mode),
 DEFINE_PROP_END_OF_LIST(),
 };
 
-- 
2.7.4

[Qemu-block] [PATCH v8 29/36] ahci: Use shared lock for shared storage migration

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 tests/ahci-test.c | 27 +--
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/tests/ahci-test.c b/tests/ahci-test.c
index 9c0adce..838ff45 100644
--- a/tests/ahci-test.c
+++ b/tests/ahci-test.c
@@ -1132,10 +1132,14 @@ static void test_migrate_sanity(void)
 char *uri = g_strdup_printf("unix:%s", mig_socket);
 
 src = ahci_boot("-m 1024 -M q35 "
-"-drive if=ide,file=%s,format=%s ", tmp_path, imgfmt);
+"-drive if=none,id=drive0,file=%s,format=%s "
+"-device ide-hd,drive=drive0,lock-mode=shared ",
+tmp_path, imgfmt);
 dst = ahci_boot("-m 1024 -M q35 "
-"-drive if=ide,file=%s,format=%s "
-"-incoming %s", tmp_path, imgfmt, uri);
+"-drive if=none,id=drive0,file=%s,format=%s "
+"-device ide-hd,drive=drive0,lock-mode=shared "
+"-incoming %s",
+tmp_path, imgfmt, uri);
 
 ahci_migrate(src, dst, uri);
 
@@ -1157,11 +1161,14 @@ static void ahci_migrate_simple(uint8_t cmd_read, 
uint8_t cmd_write)
 char *uri = g_strdup_printf("unix:%s", mig_socket);
 
 src = ahci_boot_and_enable("-m 1024 -M q35 "
-   "-drive if=ide,format=%s,file=%s ",
+   "-drive if=none,format=%s,file=%s,id=drive0 "
+   "-device ide-hd,drive=drive0,lock-mode=shared ",
imgfmt, tmp_path);
 dst = ahci_boot("-m 1024 -M q35 "
-"-drive if=ide,format=%s,file=%s "
-"-incoming %s", imgfmt, tmp_path, uri);
+"-drive if=none,format=%s,file=%s,id=drive0 "
+"-device ide-hd,drive=drive0,lock-mode=shared "
+"-incoming %s",
+imgfmt, tmp_path, uri);
 
 set_context(src->parent);
 
@@ -1286,7 +1293,7 @@ static void ahci_migrate_halted_io(uint8_t cmd_read, 
uint8_t cmd_write)
"format=%s,cache=writeback,"
"rerror=stop,werror=stop "
"-M q35 "
-   "-device ide-hd,drive=drive0 ",
+   "-device ide-hd,drive=drive0,lock-mode=shared ",
debug_path,
tmp_path, imgfmt);
 
@@ -1294,7 +1301,7 @@ static void ahci_migrate_halted_io(uint8_t cmd_read, 
uint8_t cmd_write)
 "format=%s,cache=writeback,"
 "rerror=stop,werror=stop "
 "-M q35 "
-"-device ide-hd,drive=drive0 "
+"-device ide-hd,drive=drive0,lock-mode=shared "
 "-incoming %s",
 tmp_path, imgfmt, uri);
 
@@ -1358,13 +1365,13 @@ static void test_flush_migrate(void)
"cache=writeback,rerror=stop,werror=stop,"
"format=%s "
"-M q35 "
-   "-device ide-hd,drive=drive0 ",
+   "-device ide-hd,drive=drive0,lock-mode=shared ",
debug_path, tmp_path, imgfmt);
 dst = ahci_boot("-drive file=%s,if=none,id=drive0,"
 "cache=writeback,rerror=stop,werror=stop,"
 "format=%s "
 "-M q35 "
-"-device ide-hd,drive=drive0 "
+"-device ide-hd,drive=drive0,lock-mode=shared "
 "-incoming %s", tmp_path, imgfmt, uri);
 
 set_context(src->parent);
-- 
2.7.4

[Qemu-block] [PATCH v8 30/36] tests/postcopy: Use shared lock for images

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 tests/postcopy-test.c | 9 ++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/postcopy-test.c b/tests/postcopy-test.c
index 41ed1a9..3ec4d01 100644
--- a/tests/postcopy-test.c
+++ b/tests/postcopy-test.c
@@ -371,12 +371,14 @@ static void test_migrate(void)
 cmd_src = g_strdup_printf("-machine accel=kvm:tcg -m 150M"
   " -name pcsource,debug-threads=on"
   " -serial file:%s/src_serial"
-  " -drive file=%s,format=raw",
+  " -drive 
file=%s,format=raw,if=none,id=drive1"
+  " -device 
ide-hd,drive=drive1,lock-mode=shared",
   tmpfs, bootpath);
 cmd_dst = g_strdup_printf("-machine accel=kvm:tcg -m 150M"
   " -name pcdest,debug-threads=on"
   " -serial file:%s/dest_serial"
-  " -drive file=%s,format=raw"
+  " -drive 
file=%s,format=raw,if=none,id=drive2"
+  " -device 
ide-hd,drive=drive2,lock-mode=shared"
   " -incoming %s",
   tmpfs, bootpath, uri);
 } else if (strcmp(arch, "ppc64") == 0) {
@@ -384,7 +386,8 @@ static void test_migrate(void)
 cmd_src = g_strdup_printf("-machine accel=kvm:tcg -m 256M"
   " -name pcsource,debug-threads=on"
   " -serial file:%s/src_serial"
-  " -drive file=%s,if=pflash,format=raw",
+  " -drive 
file=%s,if=none,format=raw,id=drive3"
+  " -device 
pflash,drive=drive3,lock-mode=shared",
   tmpfs, bootpath);
 cmd_dst = g_strdup_printf("-machine accel=kvm:tcg -m 256M"
   " -name pcdest,debug-threads=on"
-- 
2.7.4

[Qemu-block] [PATCH v8 26/36] iotests: Disable image locking in 085

[Fam Zheng]

The cases is about live snapshot features. Disable image locking because
otherwise a few tests are going to fail because we reuse the same images
at blockdev-add.

Signed-off-by: Fam Zheng 
---
 tests/qemu-iotests/085 | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/qemu-iotests/085 b/tests/qemu-iotests/085
index aa77eca..59bfd0e 100755
--- a/tests/qemu-iotests/085
+++ b/tests/qemu-iotests/085
@@ -102,6 +102,7 @@ function add_snapshot_image()
 cmd="{ 'execute': 'blockdev-add', 'arguments':
{ 'options':
  { 'driver': 'qcow2', 'node-name': 'snap_${1}', ${extra_params}
+   'read-only': true,
'file':
{ 'driver': 'file', 'filename': '${snapshot_file}',
  'node-name': 'file_${1}' } } } }"
@@ -130,7 +131,10 @@ echo === Running QEMU ===
 echo
 
 qemu_comm_method="qmp"
-_launch_qemu -drive file="${TEST_IMG}.1",if=virtio -drive 
file="${TEST_IMG}.2",if=virtio
+_launch_qemu -drive file="${TEST_IMG}.1",if=none,id=virtio0 \
+ -device virtio-blk,drive=virtio0,lock-mode=nolock \
+ -drive file="${TEST_IMG}.2",if=none,id=virtio1 \
+ -device virtio-blk,drive=virtio1,lock-mode=nolock
 h=$QEMU_HANDLE
 
 echo
-- 
2.7.4

[Qemu-block] [PATCH v8 28/36] qemu-iotests: Add test case 153 for image locking

[Fam Zheng]

Signed-off-by: Fam Zheng 
Reviewed-by: Max Reitz 
---
 tests/qemu-iotests/153 | 197 +
 tests/qemu-iotests/153.out | 426 +
 tests/qemu-iotests/group   |   1 +
 3 files changed, 624 insertions(+)
 create mode 100755 tests/qemu-iotests/153
 create mode 100644 tests/qemu-iotests/153.out

diff --git a/tests/qemu-iotests/153 b/tests/qemu-iotests/153
new file mode 100755
index 000..68770ec
--- /dev/null
+++ b/tests/qemu-iotests/153
@@ -0,0 +1,197 @@
+#!/bin/bash
+#
+# Test image locking
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+#
+
+# creator
+owner=f...@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1   # failure is the default!
+
+_cleanup()
+{
+_cleanup_test_img
+rm -f "${TEST_IMG}.base"
+rm -f "${TEST_IMG}.convert"
+rm -f "${TEST_IMG}.a"
+rm -f "${TEST_IMG}.b"
+rm -f "${TEST_IMG}.lnk"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+size=32M
+
+_run_cmd()
+{
+echo
+(echo "$@"; "$@" 2>&1 1>/dev/null) | _filter_testdir
+}
+
+function _do_run_qemu()
+{
+(
+if ! test -t 0; then
+while read cmd; do
+echo $cmd
+done
+fi
+echo quit
+) | $QEMU -nographic -monitor stdio -serial none "$@" 1>/dev/null
+}
+
+function _run_qemu_with_images()
+{
+_do_run_qemu \
+$(for i in $@; do echo "-drive if=none,file=$i"; done) 2>&1 \
+| _filter_testdir | _filter_qemu
+}
+
+for opts1 in "" "readonly=on" "lock-mode=off" "lock-mode=shared"; do
+echo
+echo "== Creating base image =="
+TEST_IMG="${TEST_IMG}.base" _make_test_img $size
+
+echo
+echo "== Creating test image =="
+$QEMU_IMG create -f $IMGFMT "${TEST_IMG}" -b ${TEST_IMG}.base | 
_filter_img_create
+
+echo
+echo "== Launching QEMU $opts1 =="
+_launch_qemu -drive file="${TEST_IMG}",if=none,$opts1
+h=$QEMU_HANDLE
+
+for opts2 in "" "lock-mode=auto" "lock-mode=shared" \
+ "lock-mode=off" "lock-mode=auto,readonly=on"; do
+echo
+echo "== Launching another QEMU $opts2 =="
+echo "quit" | \
+$QEMU -nographic -monitor stdio \
+-drive file="${TEST_IMG}",if=none,$opts2 2>&1 1>/dev/null | \
+_filter_testdir | _filter_qemu
+done
+
+for L in "" "-L"; do
+
+echo
+echo "== Running utility commands $(test -n "$L" && echo "(lock 
disabled)") =="
+_run_cmd $QEMU_IO $L -c "read 0 512" "${TEST_IMG}"
+_run_cmd $QEMU_IO $L -r -c "read 0 512" "${TEST_IMG}"
+_run_cmd $QEMU_IMG info$L "${TEST_IMG}"
+_run_cmd $QEMU_IMG check   $L "${TEST_IMG}"
+_run_cmd $QEMU_IMG compare $L "${TEST_IMG}" "${TEST_IMG}"
+_run_cmd $QEMU_IMG map $L "${TEST_IMG}"
+_run_cmd $QEMU_IMG amend -o "" $L "${TEST_IMG}"
+_run_cmd $QEMU_IMG commit  $L "${TEST_IMG}"
+_run_cmd $QEMU_IMG resize  $L "${TEST_IMG}" $size
+_run_cmd $QEMU_IMG rebase  $L "${TEST_IMG}" -b "${TEST_IMG}.base"
+_run_cmd $QEMU_IMG snapshot -l $L "${TEST_IMG}"
+_run_cmd $QEMU_IMG convert $L "${TEST_IMG}" "${TEST_IMG}.convert"
+done
+_send_qemu_cmd $h "{ 'execute': 'quit', }" ""
+echo
+echo "Round done"
+_cleanup_qemu
+done
+
+function _enum_opts()
+{
+echo lock-mode={shared,auto,off},readonly={on,off}
+}
+
+for opt1 in $(_enum_opts); do
+for opt2 in $(_enum_opts); do
+echo
+echo "== Two devices with the same image ($opt1 - $opt2) =="
+_run_qemu_with_images "${TEST_IMG},$opt1" "${TEST_IMG},$opt2"
+done
+done
+
+(
+$QEMU_IMG create -f qcow2 "${TEST_IMG}.a" -b "${TEST_IMG}"
+$QEMU_IMG create -f qcow2 "${TEST_IMG}.b" -b "${TEST_IMG}"
+$QEMU_IMG create -f qcow2 "${TEST_IMG}.c" -b "${TEST_IMG}.b"
+) | _filter_img_create
+
+echo
+echo "== Two devices sharing the same file in backing chain =="
+_run_qemu_with_images "${TEST_IMG}.a" "${TEST_IMG}.b"
+_run_qemu_with_images "${TEST_IMG}.a" "${TEST_IMG}.c"
+
+echo
+echo "== Backing image also as an 

[Qemu-block] [PATCH v8 22/36] qemu-iotests: 030: Disable image locking when checking test image

[Fam Zheng]

The VM is running, qemu-io would fail the lock acquisition.

Signed-off-by: Fam Zheng 
Reviewed-by: Max Reitz 
---
 tests/qemu-iotests/030 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
index 107049b..acae67e 100755
--- a/tests/qemu-iotests/030
+++ b/tests/qemu-iotests/030
@@ -95,7 +95,7 @@ class TestSingleDrive(iotests.QMPTestCase):
 self.assert_no_active_block_jobs()
 
 # The image map is empty before the operation
-empty_map = qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img)
+empty_map = qemu_io('-L', '-f', iotests.imgfmt, '-c', 'map', test_img)
 
 # This is a no-op: no data should ever be copied from the base image
 result = self.vm.qmp('block-stream', device='drive0', base=mid_img)
-- 
2.7.4

[Qemu-block] [PATCH v8 19/36] pflash: Add "lock-mode" property

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/block/pflash_cfi01.c | 10 ++
 hw/block/pflash_cfi02.c |  9 +
 2 files changed, 19 insertions(+)

diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c
index 62d7a56..df82471 100644
--- a/hw/block/pflash_cfi01.c
+++ b/hw/block/pflash_cfi01.c
@@ -76,6 +76,7 @@ struct pflash_t {
 /*< public >*/
 
 BlockBackend *blk;
+ImageLockMode lock_mode;
 uint32_t nb_blocs;
 uint64_t sector_len;
 uint8_t bank_width;
@@ -738,6 +739,14 @@ static void pflash_cfi01_realize(DeviceState *dev, Error 
**errp)
 sysbus_init_mmio(SYS_BUS_DEVICE(dev), >mem);
 
 if (pfl->blk) {
+Error *local_err = NULL;
+
+blk_lock_image(pfl->blk, pfl->lock_mode, _err);
+if (local_err) {
+vmstate_unregister_ram(>mem, DEVICE(pfl));
+error_propagate(errp, local_err);
+return;
+}
 /* read the initial flash content */
 ret = blk_pread(pfl->blk, 0, pfl->storage, total_len);
 
@@ -852,6 +861,7 @@ static void pflash_cfi01_realize(DeviceState *dev, Error 
**errp)
 
 static Property pflash_cfi01_properties[] = {
 DEFINE_PROP_DRIVE("drive", struct pflash_t, blk),
+DEFINE_PROP_LOCK_MODE("lock-mode", struct pflash_t, lock_mode),
 /* num-blocks is the number of blocks actually visible to the guest,
  * ie the total size of the device divided by the sector length.
  * If we're emulating flash devices wired in parallel the actual
diff --git a/hw/block/pflash_cfi02.c b/hw/block/pflash_cfi02.c
index 4f6105c..ff6a9fb 100644
--- a/hw/block/pflash_cfi02.c
+++ b/hw/block/pflash_cfi02.c
@@ -620,6 +620,14 @@ static void pflash_cfi02_realize(DeviceState *dev, Error 
**errp)
 pfl->storage = memory_region_get_ram_ptr(>orig_mem);
 pfl->chip_len = chip_len;
 if (pfl->blk) {
+Error *local_err = NULL;
+
+blk_lock_image(pfl->blk, pfl->lock_mode, _err);
+if (local_err) {
+vmstate_unregister_ram(>mem, DEVICE(pfl));
+error_propagate(errp, local_err);
+return;
+}
 /* read the initial flash content */
 ret = blk_pread(pfl->blk, 0, pfl->storage, chip_len);
 if (ret < 0) {
@@ -724,6 +732,7 @@ static void pflash_cfi02_realize(DeviceState *dev, Error 
**errp)
 
 static Property pflash_cfi02_properties[] = {
 DEFINE_PROP_DRIVE("drive", struct pflash_t, blk),
+DEFINE_PROP_LOCK_MODE("lock-mode", struct pflash_t, lock_mode),
 DEFINE_PROP_UINT32("num-blocks", struct pflash_t, nb_blocs, 0),
 DEFINE_PROP_UINT32("sector-length", struct pflash_t, sector_len, 0),
 DEFINE_PROP_UINT8("width", struct pflash_t, width, 0),
-- 
2.7.4

[Qemu-block] [PATCH v8 16/36] ide: Apply lock-mode when initialize

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/ide/core.c | 10 +-
 hw/ide/qdev.c |  2 +-
 include/hw/ide/internal.h |  3 ++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/hw/ide/core.c b/hw/ide/core.c
index b0e42a6..a426baf 100644
--- a/hw/ide/core.c
+++ b/hw/ide/core.c
@@ -28,6 +28,7 @@
 #include "hw/pci/pci.h"
 #include "hw/isa/isa.h"
 #include "qemu/error-report.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
@@ -2394,17 +2395,24 @@ static const BlockDevOps ide_hd_block_ops = {
 .resize_cb = ide_resize_cb,
 };
 
-int ide_init_drive(IDEState *s, BlockBackend *blk, IDEDriveKind kind,
+int ide_init_drive(IDEState *s, BlockBackend *blk, ImageLockMode lock_mode,
+   IDEDriveKind kind,
const char *version, const char *serial, const char *model,
uint64_t wwn,
uint32_t cylinders, uint32_t heads, uint32_t secs,
int chs_trans)
 {
+Error *local_err = NULL;
 uint64_t nb_sectors;
 
 s->blk = blk;
 s->drive_kind = kind;
 
+blk_lock_image(blk, lock_mode, _err);
+if (local_err) {
+error_report_err(local_err);
+return -1;
+}
 blk_get_geometry(blk, _sectors);
 s->cylinders = cylinders;
 s->heads = heads;
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 2eb055a..c5b88fb 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -188,7 +188,7 @@ static int ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind)
 }
 blkconf_apply_backend_options(>conf);
 
-if (ide_init_drive(s, dev->conf.blk, kind,
+if (ide_init_drive(s, dev->conf.blk, dev->conf.lock_mode, kind,
dev->version, dev->serial, dev->model, dev->wwn,
dev->conf.cyls, dev->conf.heads, dev->conf.secs,
dev->chs_trans) < 0) {
diff --git a/include/hw/ide/internal.h b/include/hw/ide/internal.h
index a6dd2c3..828f88c 100644
--- a/include/hw/ide/internal.h
+++ b/include/hw/ide/internal.h
@@ -600,7 +600,8 @@ uint32_t ide_data_readw(void *opaque, uint32_t addr);
 void ide_data_writel(void *opaque, uint32_t addr, uint32_t val);
 uint32_t ide_data_readl(void *opaque, uint32_t addr);
 
-int ide_init_drive(IDEState *s, BlockBackend *blk, IDEDriveKind kind,
+int ide_init_drive(IDEState *s, BlockBackend *blk, ImageLockMode lock_mode,
+   IDEDriveKind kind,
const char *version, const char *serial, const char *model,
uint64_t wwn,
uint32_t cylinders, uint32_t heads, uint32_t secs,
-- 
2.7.4

[Qemu-block] [PATCH v8 17/36] nvme: Apply lock-mode when initialize

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/block/nvme.c | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index cef3bb4..318dc94 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -829,7 +829,7 @@ static int nvme_init(PCIDevice *pci_dev)
 {
 NvmeCtrl *n = NVME(pci_dev);
 NvmeIdCtrl *id = >id_ctrl;
-
+Error *local_err = NULL;
 int i;
 int64_t bs_size;
 uint8_t *pci_conf;
@@ -837,6 +837,10 @@ static int nvme_init(PCIDevice *pci_dev)
 if (!n->conf.blk) {
 return -1;
 }
+blk_lock_image(n->conf.blk, n->conf.lock_mode, _err);
+if (local_err) {
+return -1;
+}
 
 bs_size = blk_getlength(n->conf.blk);
 if (bs_size < 0) {
-- 
2.7.4

[Qemu-block] [PATCH v8 11/36] block: Add blk_lock_image

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 block/block-backend.c  | 18 ++
 include/sysemu/block-backend.h |  2 ++
 2 files changed, 20 insertions(+)

diff --git a/block/block-backend.c b/block/block-backend.c
index 0bd19ab..dc1ad36 100644
--- a/block/block-backend.c
+++ b/block/block-backend.c
@@ -61,6 +61,7 @@ struct BlockBackend {
 bool allow_write_beyond_eof;
 
 NotifierList remove_bs_notifiers, insert_bs_notifiers;
+ImageLockMode lock_mode;
 };
 
 typedef struct BlockBackendAIOCB {
@@ -1720,3 +1721,20 @@ static void blk_root_drained_end(BdrvChild *child)
 assert(blk->public.io_limits_disabled);
 --blk->public.io_limits_disabled;
 }
+
+void blk_lock_image(BlockBackend *blk, ImageLockMode mode, Error **errp)
+{
+int r;
+BlockDriverState *bs = blk_bs(blk);
+
+if (!bs) {
+blk->lock_mode = mode;
+return;
+}
+r = bdrv_set_lock_mode(bs, mode);
+if (r) {
+error_setg(errp, "Failed to lock image");
+} else {
+blk->lock_mode = mode;
+}
+}
diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h
index 3b29317..78f9da5 100644
--- a/include/sysemu/block-backend.h
+++ b/include/sysemu/block-backend.h
@@ -224,4 +224,6 @@ void blk_io_limits_disable(BlockBackend *blk);
 void blk_io_limits_enable(BlockBackend *blk, const char *group);
 void blk_io_limits_update_group(BlockBackend *blk, const char *group);
 
+void blk_lock_image(BlockBackend *blk, ImageLockMode mode, Error **errp);
+
 #endif
-- 
2.7.4

[Qemu-block] [PATCH v8 12/36] virtio-blk: Apply lock-mode when realize

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/block/virtio-blk.c | 5 +
 1 file changed, 5 insertions(+)

diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index 3a6112f..ce65615 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -896,6 +896,11 @@ static void virtio_blk_device_realize(DeviceState *dev, 
Error **errp)
 error_setg(errp, "num-queues property must be larger than 0");
 return;
 }
+blk_lock_image(conf->conf.blk, conf->conf.lock_mode, );
+if (err) {
+error_propagate(errp, err);
+return;
+}
 
 blkconf_serial(>conf, >serial);
 blkconf_apply_backend_options(>conf);
-- 
2.7.4

[Qemu-block] [PATCH v8 24/36] iotests: 087: Disable image locking in cases where file is shared

[Fam Zheng]

Otherwise the error handling we are expecting will be masked by the
preceding image locking check, and is going to be indistinguishable
because the error messages are all the same.

Signed-off-by: Fam Zheng 
---
 tests/qemu-iotests/087 | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/qemu-iotests/087 b/tests/qemu-iotests/087
index 91e4c71..5c04577 100755
--- a/tests/qemu-iotests/087
+++ b/tests/qemu-iotests/087
@@ -84,7 +84,6 @@ run_qemu -drive 
driver=$IMGFMT,id=disk,node-name=test-node,file="$TEST_IMG" <

[Qemu-block] [PATCH v8 18/36] usb-storage: Apply lock-mode when realize

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/usb/dev-storage.c | 5 +
 1 file changed, 5 insertions(+)

diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index c607f76..6d0c00e 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -600,6 +600,11 @@ static void usb_msd_realize_storage(USBDevice *dev, Error 
**errp)
 error_setg(errp, "drive property not set");
 return;
 }
+blk_lock_image(blk, s->conf.lock_mode, );
+if (err) {
+error_propagate(errp, err);
+return;
+}
 
 blkconf_serial(>conf, >serial);
 blkconf_blocksizes(>conf);
-- 
2.7.4

[Qemu-block] [PATCH v8 14/36] scsi-generic: Apply lock-mode when realize

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 hw/scsi/scsi-generic.c | 7 +++
 1 file changed, 7 insertions(+)

diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c
index 7a588a7..4f131e8 100644
--- a/hw/scsi/scsi-generic.c
+++ b/hw/scsi/scsi-generic.c
@@ -502,12 +502,19 @@ static void scsi_generic_realize(SCSIDevice *s, Error 
**errp)
 int rc;
 int sg_version;
 struct sg_scsi_id scsiid;
+Error *err = NULL;
 
 if (!s->conf.blk) {
 error_setg(errp, "drive property not set");
 return;
 }
 
+blk_lock_image(s->conf.blk, s->conf.lock_mode, );
+if (err) {
+error_propagate(errp, err);
+return;
+}
+
 if (blk_get_on_error(s->conf.blk, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
 error_setg(errp, "Device doesn't support drive option werror");
 return;
-- 
2.7.4

[Qemu-block] [PATCH v8 10/36] block: Don't lock drive-backup target image in none mode

[Fam Zheng]

As a very special case, in sync=none mode, the source is the backing image of
the target, which will be RO opened again. This won't work with image locking
because the first open could be exclusive.

Signed-off-by: Fam Zheng 
Reviewed-by: Max Reitz 
---
 blockdev.c | 5 +
 1 file changed, 5 insertions(+)

diff --git a/blockdev.c b/blockdev.c
index 29c6561..a9a7dee 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -3211,6 +3211,11 @@ static void do_drive_backup(DriveBackup *backup, 
BlockJobTxn *txn, Error **errp)
 }
 }
 if (backup->sync == MIRROR_SYNC_MODE_NONE) {
+/* XXX: bs will be open second time as the backing file of target,
+ * disable image locking. Once block layer allows sharing backing BDS,
+ * change below to BDRV_O_NO_BACKING and assign it after bdrv_open().
+ **/
+flags |= BDRV_O_NO_LOCK;
 source = bs;
 }
 
-- 
2.7.4

[Qemu-block] [PATCH v8 09/36] qemu-nbd: Add "--no-lock/-L" option

[Fam Zheng]

Signed-off-by: Fam Zheng 
Reviewed-by: Max Reitz 
---
 qemu-nbd.c| 7 ++-
 qemu-nbd.texi | 2 ++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/qemu-nbd.c b/qemu-nbd.c
index 99297a5..6585b2c 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -102,6 +102,7 @@ static void usage(const char *name)
 "Block device options:\n"
 "  -f, --format=FORMAT   set image format (raw, qcow2, ...)\n"
 "  -r, --read-only   export read-only\n"
+"  -L, --no-lock disable image locking\n"
 "  -s, --snapshotuse FILE as an external snapshot, create a 
temporary\n"
 "file with backing_file=FILE, redirect the write 
to\n"
 "the temporary one\n"
@@ -474,7 +475,7 @@ int main(int argc, char **argv)
 off_t fd_size;
 QemuOpts *sn_opts = NULL;
 const char *sn_id_or_name = NULL;
-const char *sopt = "hVb:o:p:rsnP:c:dvk:e:f:tl:x:T:";
+const char *sopt = "hVb:o:p:rsnLP:c:dvk:e:f:tl:x:T:";
 struct option lopt[] = {
 { "help", no_argument, NULL, 'h' },
 { "version", no_argument, NULL, 'V' },
@@ -483,6 +484,7 @@ int main(int argc, char **argv)
 { "socket", required_argument, NULL, 'k' },
 { "offset", required_argument, NULL, 'o' },
 { "read-only", no_argument, NULL, 'r' },
+{ "no-lock", no_argument, NULL, 'L' },
 { "partition", required_argument, NULL, 'P' },
 { "connect", required_argument, NULL, 'c' },
 { "disconnect", no_argument, NULL, 'd' },
@@ -638,6 +640,9 @@ int main(int argc, char **argv)
 nbdflags |= NBD_FLAG_READ_ONLY;
 flags &= ~BDRV_O_RDWR;
 break;
+case 'L':
+flags |= BDRV_O_NO_LOCK;
+break;
 case 'P':
 partition = strtol(optarg, , 0);
 if (*end) {
diff --git a/qemu-nbd.texi b/qemu-nbd.texi
index 91ebf04..5936b37 100644
--- a/qemu-nbd.texi
+++ b/qemu-nbd.texi
@@ -43,6 +43,8 @@ Force the use of the block driver for format @var{fmt} 
instead of
 auto-detecting
 @item -r, --read-only
 Export the disk as read-only
+@item -L, --no-lock
+Disable image locking
 @item -P, --partition=@var{num}
 Only expose partition @var{num}
 @item -s, --snapshot
-- 
2.7.4

[Qemu-block] [PATCH v8 05/36] raw-posix: Add image locking support

[Fam Zheng]

virtlockd in libvirt locks the first byte, we lock byte 1 to avoid
the intervene.

Both file and host device protocols are covered.

The complication is with reopen. We have three different locking states,
namely "unlocked", "shared locked" and "exclusively locked".

When we reopen, the new fd may need a new locking mode. Moving away to or from
exclusive is a bit tricky because we cannot do it atomically. This patch solves
it by dup() s->fd to s->lock_fd and avoid close(), so that there isn't a racy
window where we drop the lock on one fd before acquiring the exclusive lock on
the other.

To make the logic easier to manage, and allow better reuse, the code is
internally organized by state transition table (old_lock -> new_lock).

Signed-off-by: Fam Zheng 
---
 block/raw-posix.c | 318 +-
 1 file changed, 317 insertions(+), 1 deletion(-)

diff --git a/block/raw-posix.c b/block/raw-posix.c
index 6ed7547..22de242 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -133,6 +133,7 @@ do { \
 
 typedef struct BDRVRawState {
 int fd;
+int lock_fd;
 int type;
 int open_flags;
 size_t buf_align;
@@ -149,6 +150,7 @@ typedef struct BDRVRawState {
 
 typedef struct BDRVRawReopenState {
 int fd;
+int lock_fd;
 int open_flags;
 } BDRVRawReopenState;
 
@@ -367,6 +369,43 @@ static void raw_parse_flags(int bdrv_flags, int 
*open_flags)
 }
 }
 
+static int raw_lock_fd(int fd, ImageLockMode mode)
+{
+assert(fd >= 0);
+/* Locking byte 1 avoids interfereing with virtlockd. */
+switch (mode) {
+case IMAGE_LOCK_MODE_EXCLUSIVE:
+return qemu_lock_fd(fd, 1, 1, true);
+case IMAGE_LOCK_MODE_SHARED:
+return qemu_lock_fd(fd, 1, 1, false);
+case IMAGE_LOCK_MODE_NOLOCK:
+return qemu_unlock_fd(fd, 1, 1);
+default:
+abort();
+}
+}
+
+static int raw_lockf(BlockDriverState *bs, ImageLockMode mode)
+{
+BDRVRawState *s = bs->opaque;
+
+if (s->lock_fd < 0) {
+if (mode == IMAGE_LOCK_MODE_NOLOCK) {
+return 0;
+}
+s->lock_fd = qemu_dup(s->fd);
+if (s->lock_fd < 0) {
+return s->lock_fd;
+}
+}
+if (mode == IMAGE_LOCK_MODE_AUTO) {
+mode = bdrv_get_flags(bs) & BDRV_O_RDWR ?
+   IMAGE_LOCK_MODE_EXCLUSIVE :
+   IMAGE_LOCK_MODE_SHARED;
+}
+return raw_lock_fd(s->lock_fd, mode);
+}
+
 #ifdef CONFIG_LINUX_AIO
 static bool raw_use_aio(int bdrv_flags)
 {
@@ -433,6 +472,7 @@ static int raw_open_common(BlockDriverState *bs, QDict 
*options,
 raw_parse_flags(bdrv_flags, >open_flags);
 
 s->fd = -1;
+s->lock_fd = -1;
 fd = qemu_open(filename, s->open_flags, 0644);
 if (fd < 0) {
 ret = -errno;
@@ -529,6 +569,268 @@ static int raw_open(BlockDriverState *bs, QDict *options, 
int flags,
 return raw_open_common(bs, options, flags, 0, errp);
 }
 
+typedef enum {
+RAW_REOPEN_PREPARE,
+RAW_REOPEN_COMMIT,
+RAW_REOPEN_ABORT
+} RawReopenOperation;
+
+typedef int (*RawReopenFunc)(BDRVReopenState *state,
+ RawReopenOperation op,
+ ImageLockMode old_lock,
+ ImageLockMode new_lock,
+ Error **errp);
+
+static int
+raw_reopen_identical(BDRVReopenState *state,
+ RawReopenOperation op,
+ ImageLockMode old_lock,
+ ImageLockMode new_lock,
+ Error **errp)
+{
+assert(old_lock == new_lock);
+return 0;
+}
+
+static int
+raw_reopen_from_unlock(BDRVReopenState *state,
+   RawReopenOperation op,
+   ImageLockMode old_lock,
+   ImageLockMode new_lock,
+   Error **errp)
+{
+BDRVRawReopenState *raw_s = state->opaque;
+int ret = 0;
+
+assert(old_lock != new_lock);
+assert(old_lock == IMAGE_LOCK_MODE_NOLOCK);
+switch (op) {
+case RAW_REOPEN_PREPARE:
+ret = raw_lock_fd(raw_s->lock_fd, new_lock);
+if (ret) {
+error_setg_errno(errp, -ret, "Failed to lock new fd %d", 
raw_s->lock_fd);
+}
+break;
+case RAW_REOPEN_COMMIT:
+case RAW_REOPEN_ABORT:
+break;
+}
+
+return ret;
+}
+
+static int
+raw_reopen_to_unlock(BDRVReopenState *state,
+ RawReopenOperation op,
+ ImageLockMode old_lock,
+ ImageLockMode new_lock,
+ Error **errp)
+{
+BDRVRawState *s = state->bs->opaque;
+int ret = 0;
+
+assert(old_lock != new_lock);
+assert(new_lock == IMAGE_LOCK_MODE_NOLOCK);
+switch (op) {
+case RAW_REOPEN_PREPARE:
+break;
+case RAW_REOPEN_COMMIT:
+if (s->lock_fd >= 0) {
+qemu_close(s->lock_fd);
+s->lock_fd = -1;
+}
+break;
+case RAW_REOPEN_ABORT:
+

[Qemu-block] [PATCH v8 06/36] qemu-io: Add "-L" option for BDRV_O_NO_LOCK

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 qemu-io.c | 24 +---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/qemu-io.c b/qemu-io.c
index db129ea..5c14eba 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -108,6 +108,7 @@ static void open_help(void)
 " -r, -- open file read-only\n"
 " -s, -- use snapshot file\n"
 " -n, -- disable host cache, short for -t none\n"
+" -L, -- disable image locking\n"
 " -k, -- use kernel AIO implementation (on Linux only)\n"
 " -t, -- use the given cache mode for the image\n"
 " -d, -- use the given discard mode for the image\n"
@@ -124,7 +125,7 @@ static const cmdinfo_t open_cmd = {
 .argmin = 1,
 .argmax = -1,
 .flags  = CMD_NOFILE_OK,
-.args   = "[-rsnk] [-t cache] [-d discard] [-o options] [path]",
+.args   = "[-rsnLk] [-t cache] [-d discard] [-o options] [path]",
 .oneline= "open the file specified by path",
 .help   = open_help,
 };
@@ -143,12 +144,13 @@ static int open_f(BlockBackend *blk, int argc, char 
**argv)
 {
 int flags = BDRV_O_UNMAP;
 int readonly = 0;
+bool nolock = false;
 bool writethrough = true;
 int c;
 QemuOpts *qopts;
 QDict *opts;
 
-while ((c = getopt(argc, argv, "snro:kt:d:")) != -1) {
+while ((c = getopt(argc, argv, "snrLo:kt:d:")) != -1) {
 switch (c) {
 case 's':
 flags |= BDRV_O_SNAPSHOT;
@@ -177,6 +179,9 @@ static int open_f(BlockBackend *blk, int argc, char **argv)
 return 0;
 }
 break;
+case 'L':
+nolock = true;
+break;
 case 'o':
 if (imageOpts) {
 printf("--image-opts and 'open -o' are mutually exclusive\n");
@@ -198,6 +203,10 @@ static int open_f(BlockBackend *blk, int argc, char **argv)
 flags |= BDRV_O_RDWR;
 }
 
+if (nolock) {
+flags |= BDRV_O_NO_LOCK;
+}
+
 if (imageOpts && (optind == argc - 1)) {
 if (!qemu_opts_parse_noisily(_opts, argv[optind], false)) {
 qemu_opts_reset(_opts);
@@ -436,13 +445,15 @@ static QemuOptsList file_opts = {
 int main(int argc, char **argv)
 {
 int readonly = 0;
-const char *sopt = "hVc:d:f:rsnmkt:T:";
+const char *sopt = "hVc:d:f:rLsnmkt:T:";
+bool nolock = false;
 const struct option lopt[] = {
 { "help", no_argument, NULL, 'h' },
 { "version", no_argument, NULL, 'V' },
 { "cmd", required_argument, NULL, 'c' },
 { "format", required_argument, NULL, 'f' },
 { "read-only", no_argument, NULL, 'r' },
+{ "no-lock", no_argument, NULL, 'L' },
 { "snapshot", no_argument, NULL, 's' },
 { "nocache", no_argument, NULL, 'n' },
 { "misalign", no_argument, NULL, 'm' },
@@ -501,6 +512,9 @@ int main(int argc, char **argv)
 case 'r':
 readonly = 1;
 break;
+case 'L':
+nolock = true;
+break;
 case 'm':
 qemuio_misalign = true;
 break;
@@ -586,6 +600,10 @@ int main(int argc, char **argv)
 flags |= BDRV_O_RDWR;
 }
 
+if (nolock) {
+flags |= BDRV_O_NO_LOCK;
+}
+
 if ((argc - optind) == 1) {
 if (imageOpts) {
 QemuOpts *qopts = NULL;
-- 
2.7.4

[Qemu-block] [PATCH v8 02/36] qapi: Add ImageLockMode

[Fam Zheng]

Signed-off-by: Fam Zheng 
---
 qapi/block-core.json | 18 ++
 1 file changed, 18 insertions(+)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 92193ab..22e8d04 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -2754,3 +2754,21 @@
   'data' : { 'parent': 'str',
  '*child': 'str',
  '*node': 'str' } }
+
+##
+# @ImageLockMode:
+#
+# @auto: defer to the block driver to use the least strict mode, based on
+#the nature of format and read-only flag, and the supported locking
+#operations of the protocol.
+#
+# @exclusive: always exclusively lock the image.
+#
+# @shared: use a shared lock mode.
+#
+# @nolock: don't lock the image.
+#
+# Since: 2.8
+##
+{ 'enum': 'ImageLockMode',
+  'data': ['auto', 'exclusive', 'shared', 'nolock'] }
-- 
2.7.4

[Qemu-block] [PATCH v8 04/36] osdep: Add qemu_lock_fd and qemu_unlock_fd

[Fam Zheng]

They are wrappers of POSIX fcntl "file private locking".

Signed-off-by: Fam Zheng 
---
 include/qemu/osdep.h |  2 ++
 util/osdep.c | 29 +
 2 files changed, 31 insertions(+)

diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h
index 9e9fa61..f773f49 100644
--- a/include/qemu/osdep.h
+++ b/include/qemu/osdep.h
@@ -286,6 +286,8 @@ int qemu_close(int fd);
 #ifndef _WIN32
 int qemu_dup(int fd);
 #endif
+int qemu_lock_fd(int fd, int64_t start, int64_t len, bool exclusive);
+int qemu_unlock_fd(int fd, int64_t start, int64_t len);
 
 #if defined(__HAIKU__) && defined(__i386__)
 #define FMT_pid "%ld"
diff --git a/util/osdep.c b/util/osdep.c
index 06fb1cf..b85a490 100644
--- a/util/osdep.c
+++ b/util/osdep.c
@@ -140,6 +140,35 @@ static int qemu_parse_fdset(const char *param)
 {
 return qemu_parse_fd(param);
 }
+
+static int qemu_lock_fcntl(int fd, int64_t start, int64_t len, int fl_type)
+{
+#ifdef F_OFD_SETLK
+int ret;
+struct flock fl = {
+.l_whence = SEEK_SET,
+.l_start  = start,
+.l_len= len,
+.l_type   = fl_type,
+};
+do {
+ret = fcntl(fd, F_OFD_SETLK, );
+} while (ret == -1 && errno == EINTR);
+return ret == -1 ? -errno : 0;
+#else
+return -ENOTSUP;
+#endif
+}
+
+int qemu_lock_fd(int fd, int64_t start, int64_t len, bool exclusive)
+{
+return qemu_lock_fcntl(fd, start, len, exclusive ? F_WRLCK : F_RDLCK);
+}
+
+int qemu_unlock_fd(int fd, int64_t start, int64_t len)
+{
+return qemu_lock_fcntl(fd, start, len, F_UNLCK);
+}
 #endif
 
 /*
-- 
2.7.4

[Qemu-block] [PATCH v8 03/36] block: Introduce image file locking

[Fam Zheng]

Block drivers can implement this new operation .bdrv_lockf to actually lock the
image in the protocol specific way.

Signed-off-by: Fam Zheng 
---
 block.c   | 52 +++
 include/block/block.h |  4 +++-
 include/block/block_int.h |  5 +
 include/hw/block/block.h  |  2 ++
 4 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/block.c b/block.c
index 493ecf3..9d600df 100644
--- a/block.c
+++ b/block.c
@@ -235,6 +235,7 @@ BlockDriverState *bdrv_new(void)
 notifier_with_return_list_init(>before_write_notifiers);
 bs->refcnt = 1;
 bs->aio_context = qemu_get_aio_context();
+bs->cur_lock = IMAGE_LOCK_MODE__MAX;
 
 qemu_co_queue_init(>flush_queue);
 
@@ -925,6 +926,48 @@ out:
 g_free(gen_node_name);
 }
 
+ImageLockMode bdrv_lock_mode_from_flags(int flags)
+{
+if (flags & BDRV_O_NO_LOCK) {
+return IMAGE_LOCK_MODE_NOLOCK;
+} else if (flags & BDRV_O_SHARED_LOCK) {
+return IMAGE_LOCK_MODE_SHARED;
+} else if (flags & BDRV_O_EXCLUSIVE_LOCK) {
+return IMAGE_LOCK_MODE_EXCLUSIVE;
+} else {
+return IMAGE_LOCK_MODE_AUTO;
+}
+}
+
+ImageLockMode bdrv_get_lock_mode(BlockDriverState *bs)
+{
+return bs->cur_lock;
+}
+
+int bdrv_set_lock_mode(BlockDriverState *bs, ImageLockMode mode)
+{
+int ret;
+
+if (bs->cur_lock == mode) {
+return 0;
+} else if (!bs->drv) {
+return -ENOMEDIUM;
+} else if (!bs->drv->bdrv_lockf) {
+if (bs->file) {
+return bdrv_set_lock_mode(bs->file->bs, mode);
+}
+return 0;
+}
+ret = bs->drv->bdrv_lockf(bs, mode);
+if (ret == -ENOTSUP) {
+/* Handle it the same way as !bs->drv->bdrv_lockf */
+ret = 0;
+} else if (ret == 0) {
+bs->cur_lock = mode;
+}
+return ret;
+}
+
 static QemuOptsList bdrv_runtime_opts = {
 .name = "bdrv_common",
 .head = QTAILQ_HEAD_INITIALIZER(bdrv_runtime_opts.head),
@@ -1076,6 +1119,10 @@ static int bdrv_open_common(BlockDriverState *bs, 
BdrvChild *file,
 goto free_and_fail;
 }
 
+if (open_flags & BDRV_O_INACTIVE) {
+open_flags = (open_flags & ~BDRV_O_LOCK_MASK) & BDRV_O_NO_LOCK;
+}
+
 ret = refresh_total_sectors(bs, bs->total_sectors);
 if (ret < 0) {
 error_setg_errno(errp, -ret, "Could not refresh total sector count");
@@ -2273,6 +2320,7 @@ static void bdrv_close(BlockDriverState *bs)
 if (bs->drv) {
 BdrvChild *child, *next;
 
+bdrv_set_lock_mode(bs, IMAGE_LOCK_MODE_NOLOCK);
 bs->drv->bdrv_close(bs);
 bs->drv = NULL;
 
@@ -3188,6 +3236,9 @@ void bdrv_invalidate_cache(BlockDriverState *bs, Error 
**errp)
 error_setg_errno(errp, -ret, "Could not refresh total sector count");
 return;
 }
+if (bs->cur_lock != IMAGE_LOCK_MODE__MAX) {
+bdrv_set_lock_mode(bs, bs->cur_lock);
+}
 }
 
 void bdrv_invalidate_cache_all(Error **errp)
@@ -3230,6 +3281,7 @@ static int bdrv_inactivate_recurse(BlockDriverState *bs,
 }
 
 if (setting_flag) {
+ret = bdrv_set_lock_mode(bs, IMAGE_LOCK_MODE_NOLOCK);
 bs->open_flags |= BDRV_O_INACTIVE;
 }
 return 0;
diff --git a/include/block/block.h b/include/block/block.h
index cecab7a..828ab14 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -272,7 +272,8 @@ int bdrv_drop_intermediate(BlockDriverState *active, 
BlockDriverState *top,
 BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
 BlockDriverState *bs);
 BlockDriverState *bdrv_find_base(BlockDriverState *bs);
-
+ImageLockMode bdrv_lock_mode_from_flags(int flags);
+ImageLockMode bdrv_get_lock_mode(BlockDriverState *bs);
 
 typedef struct BdrvCheckResult {
 int corruptions;
@@ -529,5 +530,6 @@ void bdrv_drained_end(BlockDriverState *bs);
 void bdrv_add_child(BlockDriverState *parent, BlockDriverState *child,
 Error **errp);
 void bdrv_del_child(BlockDriverState *parent, BdrvChild *child, Error **errp);
+int bdrv_set_lock_mode(BlockDriverState *bs, ImageLockMode mode);
 
 #endif
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 3e79228..486e1ea 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -319,6 +319,10 @@ struct BlockDriver {
Error **errp);
 void (*bdrv_del_child)(BlockDriverState *parent, BdrvChild *child,
Error **errp);
+/**
+ * Lock/unlock the image.
+ */
+int (*bdrv_lockf)(BlockDriverState *bs, ImageLockMode mode);
 
 QLIST_ENTRY(BlockDriver) list;
 };
@@ -528,6 +532,7 @@ struct BlockDriverState {
 unsigned io_plug_disabled;
 
 int quiesce_counter;
+ImageLockMode cur_lock;
 };
 
 struct BlockBackendRootState {
diff --git a/include/hw/block/block.h b/include/hw/block/block.h
index df9d207..8c04469 100644
--- a/include/hw/block/block.h
+++ 

[Qemu-block] [PATCH v8 01/36] block: Add flag bits for image locking

[Fam Zheng]

Later the block layer will automatically lock the images to avoid unexpected
concurrent accesses to the same image, which will easily corrupt the metadata
or user data, unless in some very special cases, like migration.

The exceptional cases like shared storage migration and testing should
set BDRV_O_SHARED_LOCK or BDRV_O_NO_LOCK to advise an appropriate
locking mode.

Signed-off-by: Fam Zheng 
Reviewed-by: Max Reitz 
---
 include/block/block.h | 4 
 1 file changed, 4 insertions(+)

diff --git a/include/block/block.h b/include/block/block.h
index e18233a..cecab7a 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -98,7 +98,11 @@ typedef struct HDGeometry {
   select an appropriate protocol driver,
   ignoring the format layer */
 #define BDRV_O_NO_IO   0x1 /* don't initialize for I/O */
+#define BDRV_O_NO_LOCK 0x2 /* don't lock image file */
+#define BDRV_O_SHARED_LOCK 0x4 /* lock the image file in shared mode */
+#define BDRV_O_EXCLUSIVE_LOCK 0x8 /* lock the image file in exclusive mode 
*/
 
+#define BDRV_O_LOCK_MASK   (BDRV_O_NO_LOCK | BDRV_O_SHARED_LOCK | 
BDRV_O_EXCLUSIVE_LOCK)
 #define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_NO_FLUSH)
 
 
-- 
2.7.4

[Qemu-block] [PATCH v8 00/36] block: Image locking series

[Fam Zheng]

Hi all,

I wanted to post something before the long holiday as promised, but I couldn't
refine or test enough due to limited time.  Please take this as an RFC and do a
high level review. Thanks.

v8: Move user interface option from block device to qdev. [Kevin]
Add "exclusive" back. [Kevin]

Note: limited by the qdev interface, until blk_lock_image() is called, the
images are not locked by block layer at open, because before device tells
us what to do, block layer cannot figure out the correct lock mode any
more.

TODO 1: Lock explicitly in utils (qemu-img, qemu-io, etc) after opening
image.

TODO 2: The image locking test case 153 patch is not updated thus broken
because of the moved option.

TODO 3: Are the open flags unnecessary because we already have
ImageLockMode? If so, how to converge them?

Fam Zheng (36):
  block: Add flag bits for image locking
  qapi: Add ImageLockMode
  block: Introduce image file locking
  osdep: Add qemu_lock_fd and qemu_unlock_fd
  raw-posix: Add image locking support
  qemu-io: Add "-L" option for BDRV_O_NO_LOCK
  qemu-img: Add "-L" option to sub commands
  qemu-img: Update documentation of "-L" option
  qemu-nbd: Add "--no-lock/-L" option
  block: Don't lock drive-backup target image in none mode
  block: Add blk_lock_image
  virtio-blk: Apply lock-mode when realize
  scsi-disk: Apply lock-mode when realize
  scsi-generic: Apply lock-mode when realize
  qdev: Add "lock-mode" to block device options
  ide: Apply lock-mode when initialize
  nvme: Apply lock-mode when initialize
  usb-storage: Apply lock-mode when realize
  pflash: Add "lock-mode" property
  qemu-iotests: 046: Move version detection out from verify_io
  qemu-iotests: 091: Prepare for image lock
  qemu-iotests: 030: Disable image locking when checking test image
  iotests: 087: Disable image locking in cases where file is shared
  iotests: 087: Disable image locking in cases where file is shared
  iotests: 130: Check image info locklessly
  iotests: Disable image locking in 085
  tests: Use null-co:// instead of /dev/null
  qemu-iotests: Add test case 153 for image locking
  ahci: Use shared lock for shared storage migration
  tests/postcopy: Use shared lock for images
  fdc: Add lock-mode qdev properties
  m25p80: Add 'lock-mode' property
  nand: Add 'lock-mode' property
  onenand: Add 'lock-mode' property
  spapr_nvram: Add 'lock-mode' property
  sd: Add 'lock-mode' property

 block.c|  52 +
 block/block-backend.c  |  18 ++
 block/raw-posix.c  | 318 +-
 blockdev.c |   5 +
 hw/block/fdc.c |  19 +-
 hw/block/m25p80.c  |   8 +
 hw/block/nand.c|   8 +
 hw/block/nvme.c|   6 +-
 hw/block/onenand.c |   7 +
 hw/block/pflash_cfi01.c|  10 +
 hw/block/pflash_cfi02.c|   9 +
 hw/block/virtio-blk.c  |   5 +
 hw/core/qdev-properties.c  |  10 +
 hw/ide/core.c  |  10 +-
 hw/ide/qdev.c  |   2 +-
 hw/nvram/spapr_nvram.c |   8 +
 hw/scsi/scsi-disk.c|   6 +
 hw/scsi/scsi-generic.c |   7 +
 hw/sd/sd.c |   8 +
 hw/usb/dev-storage.c   |   5 +
 include/block/block.h  |   8 +-
 include/block/block_int.h  |   5 +
 include/hw/block/block.h   |   3 +
 include/hw/ide/internal.h  |   3 +-
 include/hw/qdev-properties.h   |   3 +
 include/qemu/osdep.h   |   2 +
 include/sysemu/block-backend.h |   2 +
 qapi/block-core.json   |  18 ++
 qemu-img-cmds.hx   |  44 ++---
 qemu-img.c |  92 +++--
 qemu-img.texi  |   3 +
 qemu-io.c  |  24 ++-
 qemu-nbd.c |   7 +-
 qemu-nbd.texi  |   2 +
 tests/ahci-test.c  |  27 ++-
 tests/drive_del-test.c |   2 +-
 tests/nvme-test.c  |   2 +-
 tests/postcopy-test.c  |   9 +-
 tests/qemu-iotests/030 |   2 +-
 tests/qemu-iotests/046 |  22 ++-
 tests/qemu-iotests/085 |   6 +-
 tests/qemu-iotests/091 |   9 +-
 tests/qemu-iotests/091.out |   1 +
 tests/qemu-iotests/130 |   4 +-
 tests/qemu-iotests/130.out |   4 +-
 tests/qemu-iotests/153 | 197 +++
 tests/qemu-iotests/153.out | 426 +
 tests/qemu-iotests/group   |   1 +
 tests/usb-hcd-uhci-test.c  |   2 +-
 tests/usb-hcd-xhci-test.c  |   2 +-
 tests/virtio-blk-test.c|   2 +-
 tests/virtio-scsi-test.c   |   4 +-
 util/osdep.c   |  29 +++
 53 files changed, 1398 insertions(+), 90 deletions(-)
 create mode 100755 tests/qemu-iotests/153
 create mode 100644 tests/qemu-iotests/153.out

-- 
2.7.4

[Qemu-block] [PATCH 08/22] block/dirty-bitmap: add autoload field to BdrvDirtyBitmap

[Vladimir Sementsov-Ogievskiy]

Mirror AUTO flag from Qcow2 bitmap in BdrvDirtyBitmap. This will be
needed in future, to save this flag back to Qcow2 for persistent
bitmaps.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/dirty-bitmap.c | 15 +++
 block/qcow2-bitmap.c |  2 ++
 include/block/dirty-bitmap.h |  2 ++
 3 files changed, 19 insertions(+)

diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 90af372..623e1d1 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -44,6 +44,7 @@ struct BdrvDirtyBitmap {
 int64_t size;   /* Size of the bitmap (Number of sectors) */
 bool disabled;  /* Bitmap is read-only */
 int active_iterators;   /* How many iterators are active */
+bool autoload;  /* bitmap must be autoloaded on image opening 
*/
 QLIST_ENTRY(BdrvDirtyBitmap) list;
 };
 
@@ -70,6 +71,8 @@ void bdrv_dirty_bitmap_make_anon(BdrvDirtyBitmap *bitmap)
 assert(!bdrv_dirty_bitmap_frozen(bitmap));
 g_free(bitmap->name);
 bitmap->name = NULL;
+
+bitmap->autoload = false;
 }
 
 BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
@@ -238,6 +241,8 @@ BdrvDirtyBitmap 
*bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
 bitmap->name = NULL;
 successor->name = name;
 bitmap->successor = NULL;
+successor->autoload = bitmap->autoload;
+bitmap->autoload = false;
 bdrv_release_dirty_bitmap(bs, bitmap);
 
 return successor;
@@ -540,3 +545,13 @@ int64_t bdrv_get_meta_dirty_count(BdrvDirtyBitmap *bitmap)
 {
 return hbitmap_count(bitmap->meta);
 }
+
+void bdrv_dirty_bitmap_set_autoload(BdrvDirtyBitmap *bitmap, bool autoload)
+{
+bitmap->autoload = autoload;
+}
+
+bool bdrv_dirty_bitmap_get_autoload(const BdrvDirtyBitmap *bitmap)
+{
+return bitmap->autoload;
+}
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 760a047..c086436 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -529,6 +529,8 @@ int qcow2_read_bitmaps(BlockDriverState *bs, Error **errp)
 ret = -EINVAL;
 goto out;
 }
+
+bdrv_dirty_bitmap_set_autoload(bitmap, true);
 } else {
 /* leave bitmap in the image */
 new_pos += copy_dir_entry(new_pos, e);
diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 1e17729..45a389a 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -75,4 +75,6 @@ void bdrv_dirty_bitmap_deserialize_ones(BdrvDirtyBitmap 
*bitmap,
 bool finish);
 void bdrv_dirty_bitmap_deserialize_finish(BdrvDirtyBitmap *bitmap);
 
+void bdrv_dirty_bitmap_set_autoload(BdrvDirtyBitmap *bitmap, bool autoload);
+bool bdrv_dirty_bitmap_get_autoload(const BdrvDirtyBitmap *bitmap);
 #endif
-- 
1.8.3.1

Re: [Qemu-block] [PATCH 00/18] Dirty bitmaps postcopy migration

[Vladimir Sementsov-Ogievskiy]

ping

For now there are some notes mostly about accessory patches. What about 
migration itself?


On 16.08.2016 13:25, Vladimir Sementsov-Ogievskiy wrote:

v2:
some bugs fixed, iotests a bit changed and merged into one test.
based on block-next (https://github.com/XanClic/qemu/commits/block-next)
clone: tag postcopy-v2 from https://src.openvz.org/scm/~vsementsov/qemu.git
online: 
https://src.openvz.org/users/vsementsov/repos/qemu/browse?at=refs%2Ftags%2Fpostcopy-v2

v1:

These series are derived from my 'Dirty bitmaps migration' series. The
core idea is switch to postcopy migration and drop usage of meta
bitmaps.

These patches provide dirty bitmap postcopy migration feature. Only
named dirty bitmaps are to be migrated. Migration may be enabled using
migration capabilities.

The overall method (thanks to John Snow):

1. migrate bitmaps meta data in .save_live_setup
- create/find related bitmaps on target
- disable them
- create successors (anonimous children) only for enabled migrated
  bitmaps
2. do nothing in precopy stage
3. just before target vm start: enable successors, created in (1)
4. migrate bitmap data
5. reclaime bitmaps (merge successors to their parents)
6. enable bitmaps (only bitmaps, which was enabled in source)


Some patches are unchnaged from (v7) of 'Dirty bitmaps migration'
(DBMv7). I've left Reviewed-by's for them, if you don't like it, say me
and I'll drop them in the following version.

So, relatively to last DBMv7:

01-04: new patches, splitting common postcopy migration out of ram
postcopy migration
05: equal to DBMv7.05
06: new
07: equal to DBMv7.06
08: new
09: equal to DBMv7.07
10: new
11: derived from DBMv7.08, see below
12-15: equal to DBMv7.09-12
16: derived from DBMv7.13
- switch from fifo to socket, as postcopy don't work with fifo
  for now
- change parameters: size, granularity, regions
- add time.sleep, to wait for postcopy migration phase (bad
  temporary solution.
- drop Reviewed-by
17: new

11: the core patch of the series, it is derived from
[DBMv7.08: migration: add migration_block-dirty-bitmap.c]
There are a lot of changes related to switching from precopy to
postcopy, but functions related to migration stream itself
(structs, send/load sequences) are mostly unchnaged.

So, changes, to switch from precopy to postcopy:
- removed all staff related to meta bitmaps and dirty phase!!!
- add dirty_bitmap_mig_enable_successors, and call it before
  target vm start in loadvm_postcopy_handle_run
- add enabled_bitmaps list of bitmaps for
  dirty_bitmap_mig_enable_successors

- enabled flag is send with start bitmap chunk instead of
  completion chunk
- sectors_per_chunk is calculated directly from CHUNK_SIZE, not
  using meta bitmap granularity

- dirty_bitmap_save_iterate: remove dirty_phase, move bulk_phase
  to postcopy stage
- dirty_bitmap_save_pending: remove dirty phase related pending,
  switch pending to non-postcopyable
- dirty_bitmap_load_start: get enabled flag and prepare
  successors for enabled bitmaps, also add them to
  enabled_bitmaps list
- dirty_bitmap_load_complete: for enabled bitmaps: merge them
  with successors and enable

- savevm handlers:
  * remove separate savevm_dirty_bitmap_live_iterate_handlers state
(it was bad idea, any way), and move its save_live_iterate to
savevm_dirty_bitmap_handlers
  * add is_active_iterate savevm handler, which allows iterations
only in postcopy stage (after stopping source vm)
  * add has_postcopy savevm handler. (ofcourse, just returning true)
  * use save_live_complete_postcopy instead of
save_live_complete_precopy

Other changes:
- some debug output changed
- remove MIN_LIVE_SIZE, is_live_iterative and related staff (it
  was needed to omit iterations if bitmap data is small, possibly
  this should be reimplemented)

Vladimir Sementsov-Ogievskiy (18):
   migration: add has_postcopy savevm handler
   migration: fix ram_save_pending
   migration: split common postcopy out of ram postcopy
   migration: introduce postcopy-only pending
   block: add bdrv_next_dirty_bitmap()
   block: add bdrv_dirty_bitmap_enable_successor()
   qapi: add dirty-bitmaps migration capability
   block/dirty-bitmap: add bdrv_dirty_bitmap_release_successor
   migration: include migrate_dirty_bitmaps in migrate_postcopy
   migration/qemu-file: add qemu_put_counted_string()
   migration: add is_active_iterate handler
   migration: add postcopy migration of dirty bitmaps
   iotests: maintain several vms in test
   iotests: add add_incoming_migration to VM class
   qapi: add md5 checksum of last dirty bitmap level to 

[Qemu-block] [PATCH 05/22] qcow2-bitmap: structs and consts

[Vladimir Sementsov-Ogievskiy]

Create block/qcow2-bitmap.c
Add data structures and constraints accordingly to docs/specs/qcow2.txt

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/Makefile.objs  |  2 +-
 block/qcow2-bitmap.c | 47 +++
 block/qcow2.h| 29 +
 3 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 block/qcow2-bitmap.c

diff --git a/block/Makefile.objs b/block/Makefile.objs
index fa4d8b8..0f661bb 100644
--- a/block/Makefile.objs
+++ b/block/Makefile.objs
@@ -1,5 +1,5 @@
 block-obj-y += raw_bsd.o qcow.o vdi.o vmdk.o cloop.o bochs.o vpc.o vvfat.o 
dmg.o
-block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o 
qcow2-cache.o
+block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o 
qcow2-cache.o qcow2-bitmap.o
 block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o
 block-obj-y += qed-check.o
 block-obj-$(CONFIG_VHDX) += vhdx.o vhdx-endian.o vhdx-log.o
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
new file mode 100644
index 000..cd18b07
--- /dev/null
+++ b/block/qcow2-bitmap.c
@@ -0,0 +1,47 @@
+/*
+ * Bitmaps for the QCOW version 2 format
+ *
+ * Copyright (c) 2014-2016 Vladimir Sementsov-Ogievskiy
+ *
+ * This file is derived from qcow2-snapshot.c, original copyright:
+ * Copyright (c) 2004-2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to 
deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/* NOTICE: BME here means Bitmaps Extension and used as a namespace for
+ * _internal_ constants. Please do not use this _internal_ abbreviation for
+ * other needs and/or outside of this file. */
+
+/* Bitmap directory entry constraints */
+#define BME_MAX_TABLE_SIZE 0x800
+#define BME_MAX_PHYS_SIZE 0x2000 /* 512 mb */
+#define BME_MAX_GRANULARITY_BITS 31
+#define BME_MIN_GRANULARITY_BITS 9
+#define BME_MAX_NAME_SIZE 1023
+
+/* Bitmap directory entry flags */
+#define BME_RESERVED_FLAGS 0x
+
+/* bits [1, 8] U [56, 63] are reserved */
+#define BME_TABLE_ENTRY_RESERVED_MASK 0xff0001fe
+
+typedef enum BitmapType {
+BT_DIRTY_TRACKING_BITMAP = 1
+} BitmapType;
diff --git a/block/qcow2.h b/block/qcow2.h
index b36a7bf..0480b8b 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -52,6 +52,10 @@
  * space for snapshot names and IDs */
 #define QCOW_MAX_SNAPSHOTS_SIZE (1024 * QCOW_MAX_SNAPSHOTS)
 
+/* Bitmap header extension constraints */
+#define QCOW_MAX_DIRTY_BITMAPS 65535
+#define QCOW_MAX_DIRTY_BITMAP_DIRECTORY_SIZE (1024 * QCOW_MAX_DIRTY_BITMAPS)
+
 /* indicate that the refcount of the referenced cluster is exactly one. */
 #define QCOW_OFLAG_COPIED (1ULL << 63)
 /* indicate that the cluster is compressed (they never have the copied flag) */
@@ -142,6 +146,22 @@ typedef struct QEMU_PACKED QCowSnapshotHeader {
 /* name follows  */
 } QCowSnapshotHeader;
 
+/* Qcow2BitmapDirEntry is actually a bitmap directory entry */
+typedef struct QEMU_PACKED Qcow2BitmapDirEntry {
+/* header is 8 byte aligned */
+uint64_t bitmap_table_offset;
+
+uint32_t bitmap_table_size;
+uint32_t flags;
+
+uint8_t type;
+uint8_t granularity_bits;
+uint16_t name_size;
+uint32_t extra_data_size;
+/* extra data follows  */
+/* name follows  */
+} Qcow2BitmapDirEntry;
+
 typedef struct QEMU_PACKED QCowSnapshotExtraData {
 uint64_t vm_state_size_large;
 uint64_t disk_size;
@@ -222,6 +242,15 @@ typedef uint64_t Qcow2GetRefcountFunc(const void 
*refcount_array,
 typedef void Qcow2SetRefcountFunc(void *refcount_array,
   uint64_t index, uint64_t value);
 
+/* Be careful, Qcow2BitmapHeaderExt is not an extension of 
Qcow2BitmapDirEntry, it
+ * is Qcow2 header extension */
+typedef struct Qcow2BitmapHeaderExt {
+uint32_t nb_bitmaps;
+uint32_t reserved32;
+uint64_t bitmap_directory_size;
+uint64_t bitmap_directory_offset;
+} QEMU_PACKED Qcow2BitmapHeaderExt;
+

[Qemu-block] [PATCH 13/22] qcow2-bitmap: check constraints

[Vladimir Sementsov-Ogievskiy]

Check bitmap header constraints as specified in docs/specs/qcow2.txt

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/qcow2-bitmap.c | 39 +++
 1 file changed, 39 insertions(+)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 8cf40f0..1c3abea 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -154,6 +154,34 @@ static inline void bitmap_directory_to_be(uint8_t *dir, 
size_t size)
 }
 }
 
+static int check_constraints(BlockDriverState *bs, Qcow2BitmapDirEntry *h)
+{
+BDRVQcow2State *s = bs->opaque;
+uint64_t phys_bitmap_bytes =
+(uint64_t)h->bitmap_table_size * s->cluster_size;
+uint64_t max_virtual_bits = (phys_bitmap_bytes * 8) << h->granularity_bits;
+int64_t nb_sectors = bdrv_nb_sectors(bs);
+
+if (nb_sectors < 0) {
+return nb_sectors;
+}
+
+int fail =
+((h->bitmap_table_size == 0) != (h->bitmap_table_offset == 0)) ||
+(h->bitmap_table_offset % s->cluster_size) ||
+(h->bitmap_table_size > BME_MAX_TABLE_SIZE) ||
+(phys_bitmap_bytes > BME_MAX_PHYS_SIZE) ||
+(h->bitmap_table_offset != 0 &&
+(nb_sectors << BDRV_SECTOR_BITS) > max_virtual_bits) ||
+(h->granularity_bits > BME_MAX_GRANULARITY_BITS) ||
+(h->granularity_bits < BME_MIN_GRANULARITY_BITS) ||
+(h->flags & BME_RESERVED_FLAGS) ||
+(h->name_size > BME_MAX_NAME_SIZE) ||
+(h->type != BT_DIRTY_TRACKING_BITMAP);
+
+return fail ? -EINVAL : 0;
+}
+
 static void clear_bitmap_table(BlockDriverState *bs, uint64_t *bitmap_table,
uint32_t bitmap_table_size)
 {
@@ -372,6 +400,12 @@ static uint8_t *directory_read(BlockDriverState *bs,
bdrv_get_device_or_node_name(bs));
 goto fail;
 }
+
+ret = check_constraints(bs, e);
+if (ret < 0) {
+error_setg(errp, "Bitmap doesn't satisfy the constraints.");
+goto fail;
+}
 }
 
 assert((uint8_t *)e == dir_end);
@@ -713,6 +747,11 @@ static int store_bitmap(BlockDriverState *bs,
 entry->extra_data_size = 0;
 memcpy(entry + 1, bm_name, entry->name_size);
 
+ret = check_constraints(bs, entry);
+if (ret < 0) {
+goto fail;
+}
+
 return 0;
 
 fail:
-- 
1.8.3.1

[Qemu-block] [PATCH 18/22] qapi: add md5 checksum of last dirty bitmap level to query-block

[Vladimir Sementsov-Ogievskiy]

Reviewed-by: John Snow 
Reviewed-by: Eric Blake 
Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/dirty-bitmap.c   | 1 +
 include/qemu/hbitmap.h | 8 
 qapi/block-core.json   | 5 -
 util/hbitmap.c | 8 
 4 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 392d660..d0fe382 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -359,6 +359,7 @@ BlockDirtyInfoList 
*bdrv_query_dirty_bitmaps(BlockDriverState *bs)
 info->has_name = !!bm->name;
 info->name = g_strdup(bm->name);
 info->status = bdrv_dirty_bitmap_status(bm);
+info->md5 = hbitmap_md5(bm->bitmap);
 entry->value = info;
 *plist = entry;
 plist = >next;
diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 64e5963..a47ce6a 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -240,6 +240,14 @@ void hbitmap_deserialize_ones(HBitmap *hb, uint64_t start, 
uint64_t count,
 void hbitmap_deserialize_finish(HBitmap *hb);
 
 /**
+ * hbitmap_md5:
+ * @bitmap: HBitmap to operate on.
+ *
+ * Returns md5 checksum of the last level.
+ */
+char *hbitmap_md5(const HBitmap *bitmap);
+
+/**
  * hbitmap_free:
  * @hb: HBitmap to operate on.
  *
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 087a681..8387e9c 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -419,11 +419,14 @@
 #
 # @status: current status of the dirty bitmap (since 2.4)
 #
+# @md5: md5 checksum (as a hexadecimal string) of the last bitmap level
+#   (since 2.8)
+#
 # Since: 1.3
 ##
 { 'struct': 'BlockDirtyInfo',
   'data': {'*name': 'str', 'count': 'int', 'granularity': 'uint32',
-   'status': 'DirtyBitmapStatus'} }
+   'status': 'DirtyBitmapStatus', 'md5': 'str'} }
 
 ##
 # @BlockInfo:
diff --git a/util/hbitmap.c b/util/hbitmap.c
index f49520f..1f0c417 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -707,3 +707,11 @@ void hbitmap_free_meta(HBitmap *hb)
 hbitmap_free(hb->meta);
 hb->meta = NULL;
 }
+
+char *hbitmap_md5(const HBitmap *bitmap)
+{
+uint64_t size =
+MAX((bitmap->size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
+const guchar *data = (const guchar *)bitmap->levels[HBITMAP_LEVELS - 1];
+return g_compute_checksum_for_data(G_CHECKSUM_MD5, data, size);
+}
-- 
1.8.3.1

[Qemu-block] [PATCH 04/22] block/dirty-bitmap: add deserialize_ones func

[Vladimir Sementsov-Ogievskiy]

Add bdrv_dirty_bitmap_deserialize_ones() function, which is needed for
qcow2 bitmap loading, to handle unallocated bitmap parts, marked as
all-ones.

Reviewed-by: Kevin Wolf 
Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/dirty-bitmap.c |  7 +++
 include/block/dirty-bitmap.h |  3 +++
 include/qemu/hbitmap.h   | 15 +++
 util/hbitmap.c   | 17 +
 4 files changed, 42 insertions(+)

diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 186941c..90af372 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -499,6 +499,13 @@ void bdrv_dirty_bitmap_deserialize_zeroes(BdrvDirtyBitmap 
*bitmap,
 hbitmap_deserialize_zeroes(bitmap->bitmap, start, count, finish);
 }
 
+void bdrv_dirty_bitmap_deserialize_ones(BdrvDirtyBitmap *bitmap,
+uint64_t start, uint64_t count,
+bool finish)
+{
+hbitmap_deserialize_ones(bitmap->bitmap, start, count, finish);
+}
+
 void bdrv_dirty_bitmap_deserialize_finish(BdrvDirtyBitmap *bitmap)
 {
 hbitmap_deserialize_finish(bitmap->bitmap);
diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 7cbe623..1e17729 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -70,6 +70,9 @@ void bdrv_dirty_bitmap_deserialize_part(BdrvDirtyBitmap 
*bitmap,
 void bdrv_dirty_bitmap_deserialize_zeroes(BdrvDirtyBitmap *bitmap,
   uint64_t start, uint64_t count,
   bool finish);
+void bdrv_dirty_bitmap_deserialize_ones(BdrvDirtyBitmap *bitmap,
+uint64_t start, uint64_t count,
+bool finish);
 void bdrv_dirty_bitmap_deserialize_finish(BdrvDirtyBitmap *bitmap);
 
 #endif
diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 9aa86c1..64e5963 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -216,6 +216,21 @@ void hbitmap_deserialize_zeroes(HBitmap *hb, uint64_t 
start, uint64_t count,
 bool finish);
 
 /**
+ * hbitmap_deserialize_ones
+ * @hb: HBitmap to operate on.
+ * @start: First bit to restore.
+ * @count: Number of bits to restore.
+ * @finish: Whether to call hbitmap_deserialize_finish automatically.
+ *
+ * Fills the bitmap with ones.
+ *
+ * If @finish is false, caller must call hbitmap_serialize_finish before using
+ * the bitmap.
+ */
+void hbitmap_deserialize_ones(HBitmap *hb, uint64_t start, uint64_t count,
+  bool finish);
+
+/**
  * hbitmap_deserialize_finish
  * @hb: HBitmap to operate on.
  *
diff --git a/util/hbitmap.c b/util/hbitmap.c
index 4f5cf96..f49520f 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -531,6 +531,23 @@ void hbitmap_deserialize_zeroes(HBitmap *hb, uint64_t 
start, uint64_t count,
 }
 }
 
+void hbitmap_deserialize_ones(HBitmap *hb, uint64_t start, uint64_t count,
+  bool finish)
+{
+uint64_t el_count;
+unsigned long *first;
+
+if (!count) {
+return;
+}
+serialization_chunk(hb, start, count, , _count);
+
+memset(first, 0xff, el_count * sizeof(unsigned long));
+if (finish) {
+hbitmap_deserialize_finish(hb);
+}
+}
+
 void hbitmap_deserialize_finish(HBitmap *bitmap)
 {
 int64_t i, size, prev_size;
-- 
1.8.3.1

[Qemu-block] [PATCH 14/22] qcow2: delete bitmaps on truncate

[Vladimir Sementsov-Ogievskiy]

This will not touch loaded bitmaps, as they are alredy removed from the
image.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/qcow2-bitmap.c | 38 ++
 block/qcow2.c|  9 ++---
 block/qcow2.h|  2 ++
 3 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 1c3abea..2642afe 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -624,6 +624,44 @@ out:
 return ret;
 }
 
+
+int qcow2_delete_bitmaps(BlockDriverState *bs)
+{
+int ret;
+BDRVQcow2State *s = bs->opaque;
+uint8_t *dir;
+uint64_t dir_size;
+Qcow2BitmapDirEntry *e;
+
+if (s->nb_bitmaps == 0) {
+/* No bitmaps - nothing to do */
+return 0;
+}
+
+dir_size = s->bitmap_directory_size;
+dir = directory_read(bs, s->bitmap_directory_offset,
+ s->bitmap_directory_size, NULL);
+if (dir == NULL) {
+ret = -EINVAL;
+goto out;
+}
+
+ret = directory_update(bs, NULL, 0, 0);
+if (ret < 0) {
+goto out;
+}
+
+/* to be consistent, free bitmap only after successfull directory update */
+for_each_bitmap_dir_entry(e, dir, dir_size) {
+free_bitmap_clusters(bs, e);
+}
+
+out:
+g_free(dir);
+
+return ret;
+}
+
 /* store_bitmap_data()
  * Store bitmap to image, filling bitamp table accordingly.
  */
diff --git a/block/qcow2.c b/block/qcow2.c
index 8238205..aa967ed 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -2592,9 +2592,12 @@ static int qcow2_truncate(BlockDriverState *bs, int64_t 
offset)
 
 /* cannot proceed if image has bitmaps */
 if (s->nb_bitmaps) {
-/* FIXME */
-error_report("Can't resize an image which has bitmaps");
-return -ENOTSUP;
+/* FIXME: not loaded bitmaps will be lost */
+ret = qcow2_delete_bitmaps(bs);
+if (ret < 0) {
+error_report("Can't remove bitmaps from qcow2 on truncate");
+return ret;
+}
 }
 
 /* shrinking is currently not supported */
diff --git a/block/qcow2.h b/block/qcow2.h
index dfcf4c6..af18efc 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -606,6 +606,8 @@ int qcow2_read_snapshots(BlockDriverState *bs);
 /* qcow2-bitmap.c functions */
 int qcow2_read_bitmaps(BlockDriverState *bs, Error **errp);
 
+int qcow2_delete_bitmaps(BlockDriverState *bs);
+
 /* qcow2-cache.c functions */
 Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables);
 int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c);
-- 
1.8.3.1

[Qemu-block] [PATCH 03/22] block: fix bdrv_dirty_bitmap_granularity signature

[Vladimir Sementsov-Ogievskiy]

Make getter signature const-correct. This allows other functions with
const dirty bitmap parameter use bdrv_dirty_bitmap_granularity().

Reviewed-by: Eric Blake 
Reviewed-by: John Snow 
Reviewed-by: Kevin Wolf 
Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/dirty-bitmap.c | 2 +-
 include/block/dirty-bitmap.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index 519737c..186941c 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -388,7 +388,7 @@ uint32_t 
bdrv_get_default_bitmap_granularity(BlockDriverState *bs)
 return granularity;
 }
 
-uint32_t bdrv_dirty_bitmap_granularity(BdrvDirtyBitmap *bitmap)
+uint32_t bdrv_dirty_bitmap_granularity(const BdrvDirtyBitmap *bitmap)
 {
 return BDRV_SECTOR_SIZE << hbitmap_granularity(bitmap->bitmap);
 }
diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index 9dea14b..7cbe623 100644
--- a/include/block/dirty-bitmap.h
+++ b/include/block/dirty-bitmap.h
@@ -29,7 +29,7 @@ void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
 BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs);
 uint32_t bdrv_get_default_bitmap_granularity(BlockDriverState *bs);
-uint32_t bdrv_dirty_bitmap_granularity(BdrvDirtyBitmap *bitmap);
+uint32_t bdrv_dirty_bitmap_granularity(const BdrvDirtyBitmap *bitmap);
 uint32_t bdrv_dirty_bitmap_meta_granularity(BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap);
 bool bdrv_dirty_bitmap_frozen(BdrvDirtyBitmap *bitmap);
-- 
1.8.3.1

[Qemu-block] [PATCH v7 00/22] qcow2: persistent dirty bitmaps

[Vladimir Sementsov-Ogievskiy]

v7:
https://src.openvz.org/users/vsementsov/repos/qemu/browse?at=refs%2Ftags%2Fqcow2-bitmap-v7
based on block-next (https://github.com/XanClic/qemu/commits/block-next)

- a lot of refactoring and reordering of patches.
- dead code removed (bdrv_dirty_bitmap_load, etc.)
- do not maintain extra data for now
- do not store dirty bitmap directory in memory
  (as we use it seldom, we can just reread if needed)

By Kevin's review:
01 - commit message changed: fix->improvement (as it was not a bug)
03 - r-b
04 - r-b
05 - add 21 patch to fix spec, also, removed all (I hope) mentions of
 "Bitmap Header", switch to one unified name for it - "Bitmap
 Directory Entry", to avoid misunderstanding with Qcow2 header.
 (also, add patch 22, to fix it in spec)
v6.06 - improve for_each_dir_entry loop, reorder patches, other small fixes
v6.07 ~> v7.09 - dead code removed, I've moved to one function 
.bdrv_store_persistent_bitmaps and have wrapper and callback in one
patch (with also some other staff. If it not ok, I can split them)
v6.08 - about keeping bitmap directory instead of bitmap list: no I don't keep
it at all.
v6.16 - old bdrv_ bitmap-storing related functions are removed. The new one is
bdrv_store_persistent_bitmaps.

v6:
https://src.openvz.org/users/vsementsov/repos/qemu/browse?at=refs%2Ftags%2Fqcow2-bitmap-v6
based on block-next (https://github.com/XanClic/qemu/commits/block-next)

There are a lot of changes, reorderings and additions in comparement with v5.
One principal thing: now bitmaps are removed from image after loading instead
of marking them in_use. It is simpler and we do not need to store superfluous 
data.
Also, we are no more interested in command line interface to dirty bitmaps.
So it is dropped.  If someone needs it I can add it later.

Vladimir Sementsov-Ogievskiy (22):
  hbitmap: improve dirty iter
  tests: add hbitmap iter test
  block: fix bdrv_dirty_bitmap_granularity signature
  block/dirty-bitmap: add deserialize_ones func
  qcow2-bitmap: structs and consts
  qcow2: add dirty bitmaps extension
  qcow2-bitmap: introduce auto-loading bitmaps
  block/dirty-bitmap: add autoload field to BdrvDirtyBitmap
  block: introduce persistent dirty bitmaps
  block/dirty-bitmap: add bdrv_dirty_bitmap_next()
  qcow2-bitmap: add qcow2_store_persistent_bitmaps()
  qcow2-bitmap: add IN_USE flag
  qcow2-bitmap: check constraints
  qcow2: delete bitmaps on truncate
  qcow2-bitmap: add autoclear bit
  qmp: add persistent flag to block-dirty-bitmap-add
  qmp: add autoload parameter to block-dirty-bitmap-add
  qapi: add md5 checksum of last dirty bitmap level to query-block
  iotests: test qcow2 persistent dirty bitmap
  qcow2-dirty-bitmap: refcounts
  specs/qcow2: fix bitmap granularity qemu-specific note
  specs/qcow2: do not use wording 'bitmap header'

 block.c  |  30 ++
 block/Makefile.objs  |   2 +-
 block/dirty-bitmap.c |  59 ++-
 block/qcow2-bitmap.c | 963 +++
 block/qcow2-refcount.c   |  14 +-
 block/qcow2.c| 105 -
 block/qcow2.h|  54 +++
 blockdev.c   |  30 +-
 docs/specs/qcow2.txt |   8 +-
 include/block/block.h|   2 +
 include/block/block_int.h|   2 +
 include/block/dirty-bitmap.h |  16 +-
 include/qemu/hbitmap.h   |  47 ++-
 qapi/block-core.json |  17 +-
 qmp-commands.hx  |   8 +-
 tests/qemu-iotests/165   |  87 
 tests/qemu-iotests/165.out   |   5 +
 tests/qemu-iotests/group |   1 +
 tests/test-hbitmap.c |  19 +
 util/hbitmap.c   |  48 ++-
 20 files changed, 1475 insertions(+), 42 deletions(-)
 create mode 100644 block/qcow2-bitmap.c
 create mode 100755 tests/qemu-iotests/165
 create mode 100644 tests/qemu-iotests/165.out

-- 
1.8.3.1

[Qemu-block] [PATCH 12/22] qcow2-bitmap: add IN_USE flag

[Vladimir Sementsov-Ogievskiy]

This flag means that the bitmap is now in use by the software or was not
successfully saved. In any way, with this flag set the bitmap data must
be considered inconsistent and should not be loaded.

With current implementation this flag is never set, as we just remove
bitmaps from the image after loading. But it defined in qcow2 spec and
must be handled. Also, it can be used in future, if async schemes of
bitmap loading/saving are implemented.

We also remove in-use bitmaps from the image on open.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/qcow2-bitmap.c | 17 -
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index a5be25a..8cf40f0 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -28,6 +28,7 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "qemu/cutils.h"
+#include "exec/log.h"
 
 #include "block/block_int.h"
 #include "block/qcow2.h"
@@ -44,7 +45,8 @@
 #define BME_MAX_NAME_SIZE 1023
 
 /* Bitmap directory entry flags */
-#define BME_RESERVED_FLAGS 0xfffd
+#define BME_RESERVED_FLAGS 0xfffc
+#define BME_FLAG_IN_USE 1
 #define BME_FLAG_AUTO   (1U << 1)
 
 /* bits [1, 8] U [56, 63] are reserved */
@@ -287,6 +289,11 @@ static BdrvDirtyBitmap *load_bitmap(BlockDriverState *bs,
 BdrvDirtyBitmap *bitmap = NULL;
 char *name = g_strndup(dir_entry_name_notcstr(entry), entry->name_size);
 
+if (entry->flags & BME_FLAG_IN_USE) {
+error_setg(errp, "Bitmap '%s' is in use", name);
+goto fail;
+}
+
 ret = bitmap_table_load(bs, entry, _table);
 if (ret < 0) {
 error_setg_errno(errp, -ret,
@@ -533,6 +540,14 @@ int qcow2_read_bitmaps(BlockDriverState *bs, Error **errp)
 for_each_bitmap_dir_entry(e, dir, dir_size) {
 uint32_t fl = e->flags;
 
+if (fl & BME_FLAG_IN_USE) {
+qemu_log("qcow2 warning: "
+ "removing in_use bitmap '%.*s' for image %s.\n",
+ e->name_size, (char *)(e + 1), 
bdrv_get_device_or_node_name(bs));
+
+continue;
+}
+
 if (fl & BME_FLAG_AUTO) {
 BdrvDirtyBitmap *bitmap = load_bitmap(bs, e, errp);
 if (bitmap == NULL) {
-- 
1.8.3.1

[Qemu-block] [PATCH 16/22] qmp: add persistent flag to block-dirty-bitmap-add

[Vladimir Sementsov-Ogievskiy]

Add optional 'persistent' flag to qmp command block-dirty-bitmap-add.
Default is false.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
Signed-off-by: Denis V. Lunev 
---
 blockdev.c   | 12 +++-
 qapi/block-core.json |  7 ++-
 qmp-commands.hx  |  5 -
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/blockdev.c b/blockdev.c
index 97062e3..ec0ec75 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -1991,6 +1991,7 @@ static void block_dirty_bitmap_add_prepare(BlkActionState 
*common,
 /* AIO context taken and released within qmp_block_dirty_bitmap_add */
 qmp_block_dirty_bitmap_add(action->node, action->name,
action->has_granularity, action->granularity,
+   action->has_persistent, action->persistent,
_err);
 
 if (!local_err) {
@@ -2694,10 +2695,12 @@ out:
 
 void qmp_block_dirty_bitmap_add(const char *node, const char *name,
 bool has_granularity, uint32_t granularity,
+bool has_persistent, bool persistent,
 Error **errp)
 {
 AioContext *aio_context;
 BlockDriverState *bs;
+BdrvDirtyBitmap *bitmap;
 
 if (!name || name[0] == '\0') {
 error_setg(errp, "Bitmap name cannot be empty");
@@ -2723,7 +2726,14 @@ void qmp_block_dirty_bitmap_add(const char *node, const 
char *name,
 granularity = bdrv_get_default_bitmap_granularity(bs);
 }
 
-bdrv_create_dirty_bitmap(bs, granularity, name, errp);
+if (!has_persistent) {
+persistent = false;
+}
+
+bitmap = bdrv_create_dirty_bitmap(bs, granularity, name, errp);
+if (bitmap != NULL) {
+bdrv_dirty_bitmap_set_persistance(bitmap, persistent);
+}
 
  out:
 aio_context_release(aio_context);
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 31f9990..2bf56cd 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -1235,10 +1235,15 @@
 # @granularity: #optional the bitmap granularity, default is 64k for
 #   block-dirty-bitmap-add
 #
+# @persistent: #optional the bitmap is persistent, i.e. it will be saved to
+#  corresponding block device on it's close. Default is false.
+#  For block-dirty-bitmap-add. (Since 2.8)
+#
 # Since 2.4
 ##
 { 'struct': 'BlockDirtyBitmapAdd',
-  'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32' } }
+  'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32',
+  '*persistent': 'bool' } }
 
 ##
 # @block-dirty-bitmap-add
diff --git a/qmp-commands.hx b/qmp-commands.hx
index ba2a916..434b418 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -1441,7 +1441,7 @@ EQMP
 
 {
 .name   = "block-dirty-bitmap-add",
-.args_type  = "node:B,name:s,granularity:i?",
+.args_type  = "node:B,name:s,granularity:i?,persistent:b?",
 .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_add,
 },
 
@@ -1458,6 +1458,9 @@ Arguments:
 - "node": device/node on which to create dirty bitmap (json-string)
 - "name": name of the new dirty bitmap (json-string)
 - "granularity": granularity to track writes with (int, optional)
+- "persistent": bitmap will be saved to corresponding block device
+on it's close. Block driver should maintain persistent bitmaps
+(json-bool, optional, default false) (Since 2.8)
 
 Example:
 
-- 
1.8.3.1

[Qemu-block] [PATCH 06/22] qcow2: add dirty bitmaps extension

[Vladimir Sementsov-Ogievskiy]

Add dirty bitmap extension as specified in docs/specs/qcow2.txt.
For now, just mirror extension header into Qcow2 state and check
constraints.

For now, disable image resize if it has bitmaps. It will be fixed later.

Signed-off-by: Vladimir Sementsov-Ogievskiy 
---
 block/qcow2.c | 83 +++
 block/qcow2.h |  4 +++
 2 files changed, 87 insertions(+)

diff --git a/block/qcow2.c b/block/qcow2.c
index c079aa8..08c4ef9 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -63,6 +63,7 @@ typedef struct {
 #define  QCOW2_EXT_MAGIC_END 0
 #define  QCOW2_EXT_MAGIC_BACKING_FORMAT 0xE2792ACA
 #define  QCOW2_EXT_MAGIC_FEATURE_TABLE 0x6803f857
+#define  QCOW2_EXT_MAGIC_DIRTY_BITMAPS 0x23852875
 
 static int qcow2_probe(const uint8_t *buf, int buf_size, const char *filename)
 {
@@ -92,6 +93,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, 
uint64_t start_offset,
 QCowExtension ext;
 uint64_t offset;
 int ret;
+Qcow2BitmapHeaderExt bitmaps_ext;
 
 #ifdef DEBUG_EXT
 printf("qcow2_read_extensions: start=%ld end=%ld\n", start_offset, 
end_offset);
@@ -162,6 +164,62 @@ static int qcow2_read_extensions(BlockDriverState *bs, 
uint64_t start_offset,
 }
 break;
 
+case QCOW2_EXT_MAGIC_DIRTY_BITMAPS:
+ret = bdrv_pread(bs->file, offset, _ext, ext.len);
+if (ret < 0) {
+error_setg_errno(errp, -ret, "ERROR: bitmaps_ext: "
+ "Could not read ext header");
+return ret;
+}
+
+if (bitmaps_ext.reserved32 != 0) {
+error_setg_errno(errp, -ret, "ERROR: bitmaps_ext: "
+ "Reserved field is not zero.");
+return -EINVAL;
+}
+
+be32_to_cpus(_ext.nb_bitmaps);
+be64_to_cpus(_ext.bitmap_directory_size);
+be64_to_cpus(_ext.bitmap_directory_offset);
+
+if (bitmaps_ext.nb_bitmaps > QCOW_MAX_DIRTY_BITMAPS) {
+error_setg(errp, "ERROR: bitmaps_ext: "
+ "too many dirty bitmaps");
+return -EINVAL;
+}
+
+if (bitmaps_ext.nb_bitmaps == 0) {
+error_setg(errp, "ERROR: bitmaps_ext: "
+ "found bitmaps extension with zero bitmaps");
+return -EINVAL;
+}
+
+if (bitmaps_ext.bitmap_directory_offset & (s->cluster_size - 1)) {
+error_setg(errp, "ERROR: bitmaps_ext: "
+ "wrong dirty bitmap directory offset");
+return -EINVAL;
+}
+
+if (bitmaps_ext.bitmap_directory_size >
+QCOW_MAX_DIRTY_BITMAP_DIRECTORY_SIZE) {
+error_setg(errp, "ERROR: bitmaps_ext: "
+ "too large dirty bitmap directory");
+return -EINVAL;
+}
+
+s->nb_bitmaps = bitmaps_ext.nb_bitmaps;
+s->bitmap_directory_offset =
+bitmaps_ext.bitmap_directory_offset;
+s->bitmap_directory_size =
+bitmaps_ext.bitmap_directory_size;
+
+#ifdef DEBUG_EXT
+printf("Qcow2: Got dirty bitmaps extension:"
+   " offset=%" PRIu64 " nb_bitmaps=%" PRIu32 "\n",
+   s->bitmap_directory_offset, s->nb_bitmaps);
+#endif
+break;
+
 default:
 /* unknown magic - save it in case we need to rewrite the header */
 {
@@ -1939,6 +1997,24 @@ int qcow2_update_header(BlockDriverState *bs)
 buflen -= ret;
 }
 
+if (s->nb_bitmaps > 0) {
+Qcow2BitmapHeaderExt bitmaps_header = {
+.nb_bitmaps = cpu_to_be32(s->nb_bitmaps),
+.bitmap_directory_size =
+cpu_to_be64(s->bitmap_directory_size),
+.bitmap_directory_offset =
+cpu_to_be64(s->bitmap_directory_offset)
+};
+ret = header_ext_add(buf, QCOW2_EXT_MAGIC_DIRTY_BITMAPS,
+ _header, sizeof(bitmaps_header),
+ buflen);
+if (ret < 0) {
+goto fail;
+}
+buf += ret;
+buflen -= ret;
+}
+
 /* Keep unknown header extensions */
 QLIST_FOREACH(uext, >unknown_header_ext, next) {
 ret = header_ext_add(buf, uext->magic, uext->data, uext->len, buflen);
@@ -2509,6 +2585,13 @@ static int qcow2_truncate(BlockDriverState *bs, int64_t 
offset)
 return -ENOTSUP;
 }
 
+/* cannot proceed if image has bitmaps */
+if (s->nb_bitmaps) {
+/* FIXME */
+error_report("Can't resize an image which has bitmaps");
+return -ENOTSUP;
+}
+
 /* shrinking is currently not supported */
 if (offset < bs->total_sectors * 512) {
 error_report("qcow2 doesn't support shrinking images