[Qemu-devel] [RFC V4 01/30] qcow2: Add deduplication to the qcow2 specification.

Signed-off-by: Benoit Canet --- docs/specs/qcow2.txt | 100 +- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt index 36a559d..c9c0d47 100644 --- a/docs/specs/qcow2.txt +++ b/docs/specs/qcow2

[Qemu-devel] [RFC V4 04/30] qcow2: Make update_refcount public.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c |6 +- block/qcow2.h |2 ++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index 6a95aa6..e014b0e 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refco

[Qemu-devel] [RFC V4 10/30] qcow2: Add qcow2_dedup_grow_table and use it.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 44 +++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 7adaaba..b998a2d 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -38

[Qemu-devel] [RFC V4 15/30] qcow2: Add qcow2_dedup_init and qcow2_dedup_close.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 16 block/qcow2.h |2 ++ 2 files changed, 18 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 4c391e5..12a2dad 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -986,3 +986,19

[Qemu-devel] [RFC V4 17/30] block: Add qemu-img dedup create option.

Signed-off-by: Benoit Canet --- block/qcow2.c | 113 +++-- block/qcow2.h |2 + include/block/block_int.h |1 + 3 files changed, 103 insertions(+), 13 deletions(-) diff --git a/block/qcow2.c b/block/qcow2.c index ad399c8..91

[Qemu-devel] [RFC V4 21/30] qcow2: Add verification of dedup table.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c |8 1 file changed, 8 insertions(+) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index aef280d..7e6d02f 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcount.c @@ -1156,6 +1156,14 @@ int qcow2_check_refc

[Qemu-devel] [RFC V4 19/30] qcow2: Integrate deduplication in qcow2_co_writev loop.

Signed-off-by: Benoit Canet --- block/qcow2.c | 85 +++-- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/block/qcow2.c b/block/qcow2.c index 9130638..54c8847 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -328,6 +328,7 @@ st

[Qemu-devel] [RFC V4 25/30] qcow2: Integrate SKEIN hash algorithm in deduplication.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 14 ++ block/qcow2.c |5 + configure | 33 + 3 files changed, 52 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 28001c6..bd8397e 100644 --- a/

[Qemu-devel] [RFC V4 20/30] qcow2: Serialize write requests when deduplication is activated.

This fix the sub cluster sized writes race conditions while waiting for a more faster solution. Signed-off-by: Benoit Canet --- block/qcow2.c |9 + block/qcow2.h |1 + 2 files changed, 10 insertions(+) diff --git a/block/qcow2.c b/block/qcow2.c index 54c8847..13f6a5c 100644 ---

[Qemu-devel] [RFC V4 29/30] qcow2: init and cleanup deduplication.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 78 +++ block/qcow2.c | 17 --- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index bd8397e..da1a668 100644 --- a/blo

[Qemu-devel] [RFC V4 22/30] qcow2: Adapt checking of QCOW_OFLAG_COPIED for dedup.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c |9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index 7e6d02f..9aef608 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcount.c @@ -1001,7 +1001,14 @@ stat

[Qemu-devel] [RFC V4 30/30] qemu-iotests: Filter dedup=on/off so existing tests don't break.

Signed-off-by: Benoit Canet --- tests/qemu-iotests/common.rc |3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index aef5f52..72e746d 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@

[Qemu-devel] [RFC V4 28/30] qcow: Set dedup cluster block size to 64KB.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c |4 ++-- block/qcow2.c |1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index 092546d..3f3efd8 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcount

[Qemu-devel] [RFC V4 14/30] qcow2-cache: Allow to choose table size at creation.

Signed-off-by: Benoit Canet --- block/qcow2-cache.c | 12 +++- block/qcow2.c |5 +++-- block/qcow2.h |3 ++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c index 2f3114e..83f2814 100644 --- a/block/qcow2-cach

[Qemu-devel] [RFC V4 27/30] qcow2: Use large L2 table for deduplication.

Signed-off-by: Benoit Canet --- block/qcow2-cluster.c |2 +- block/qcow2-refcount.c | 22 +++--- block/qcow2.c |8 ++-- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c index 07037a0..d69af17 100

[Qemu-devel] [RFC V4 09/30] qcow2: Extract qcow2_dedup_grow_table

Signed-off-by: Benoit Canet --- block/qcow2-cluster.c | 102 +++-- block/qcow2-dedup.c |3 +- block/qcow2.h |6 +++ 3 files changed, 71 insertions(+), 40 deletions(-) diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c index 6

[Qemu-devel] [RFC V4 12/30] qcow2: Load and save deduplication table header extension.

Signed-off-by: Benoit Canet --- block/qcow2.c | 38 ++ 1 file changed, 38 insertions(+) diff --git a/block/qcow2.c b/block/qcow2.c index 410d3c1..9a7177b 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -53,9 +53,16 @@ typedef struct { uint32_t len; }

[Qemu-devel] [RFC V4 05/30] qcow2: Create a way to link to l2 tables when deduplicating.

Signed-off-by: Benoit Canet --- block/qcow2-cluster.c |8 ++-- block/qcow2.h |9 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c index 56fccf9..63a7241 100644 --- a/block/qcow2-cluster.c +++ b/block/qcow2-

[Qemu-devel] [RFC V4 16/30] qcow2: Extract qcow2_add_feature and qcow2_remove_feature.

Signed-off-by: Benoit Canet --- block/qcow2.c | 49 ++--- block/qcow2.h |4 ++-- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/block/qcow2.c b/block/qcow2.c index 499e939..ad399c8 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @

[Qemu-devel] [RFC V4 08/30] qcow2: Implement qcow2_compute_cluster_hash.

Add detection of libgnutls used to compute SHA256 hashes Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 13 - configure | 22 ++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 2a4

[Qemu-devel] [RFC V4 06/30] qcow2: Add qcow2_dedup and related functions

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 436 +++ block/qcow2.h |5 + 2 files changed, 441 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 4e99eb1..5901749 100644 --- a/block/qcow2-dedup.c +++ b/bl

[Qemu-devel] [RFC V4 03/30] qcow2: Add qcow2_dedup_read_missing_and_concatenate

@@ +/* + * Deduplication for the QCOW2 format + * + * Copyright (C) Nodalink, SARL. 2012-2013 + * + * Author: + * Benoît Canet + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to d

[Qemu-devel] [RFC V4 07/30] qcow2: Add qcow2_dedup_store_new_hashes.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 315 ++- block/qcow2.h |5 + 2 files changed, 319 insertions(+), 1 deletion(-) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 5901749..2a444f5 100644 --- a/block/qcow2-d

[Qemu-devel] [RFC V4 24/30] qcow2: Do not overwrite existing entries with QCOW_OFLAG_COPIED.

In the case of a race condition between two writes a l2 entry can be written without QCOW_OFLAG_COPIED before the first write fill it. This patch simply check if the l2 entry has the correct offset without QCOW_OFLAG_COPIED and do nothing. Signed-off-by: Benoit Canet --- block/qcow2-cluster.c |

[Qemu-devel] [RFC V4 26/30] qcow2: Add lazy refcounts to deduplication to prevent qcow2_cache_set_dependency loops

Signed-off-by: Benoit Canet --- block/qcow2.c |1 + 1 file changed, 1 insertion(+) diff --git a/block/qcow2.c b/block/qcow2.c index 0154d50..f66e67d 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -1606,6 +1606,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *option

Re: [Qemu-devel] [RFC V4 30/30] qemu-iotests: Filter dedup=on/off so existing tests don't break.

Ack. There is more than one patch to move. I'll do in for the next RFC. Regards Benoît Le Wednesday 02 Jan 2013 à 09:42:06 (-0700), Eric Blake a écrit : > On 01/02/2013 09:16 AM, Benoît Canet wrote: > > Signed-off-by: Benoit Canet > > --- > > tests/qemu-iotests/c

[Qemu-devel] [RFC V4 18/30] qcow2: Behave correctly when refcount reach 0 or 2^16.

When refcount reach zero we destroy the hash on disk and remove it from GTree. When refcount is at it's maximum value we mark the hash so it won't be loaded at next startup and remove it from GTree. Signed-off-by: Benoit Canet --- block/qcow2-dedup.c| 79 +++

[Qemu-devel] [RFC V4 11/30] qcow2: create function to load deduplication hashes at startup.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 68 +++ block/qcow2.h |1 + 2 files changed, 69 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index b998a2d..4c391e5 100644 --- a/block/qcow2-dedup.c +++ b/blo

[Qemu-devel] [RFC V4 13/30] qcow2: Extract qcow2_do_table_init.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c | 43 ++- block/qcow2.h |5 + 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index e014b0e..75c2bde 100644 --- a/bloc

Re: [Qemu-devel] [RFC V4 00/30] QCOW2 deduplication

> How does this code handle hash collisions, and do you have some regression > tests that purposefully create a dedup hash collision, and verify that the > 'right thing' happens? The two hash function that can be used are cryptographics and not broken yet. So nobody knows how to generate a collisi

[Qemu-devel] [RFC V4 02/30] qcow2: Add deduplication structures and fields.

Signed-off-by: Benoit Canet --- block/qcow2.h | 57 - 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/block/qcow2.h b/block/qcow2.h index 718b52b..637c86a 100644 --- a/block/qcow2.h +++ b/block/qcow2.h @@ -58,6 +58,50 @@ #

[Qemu-devel] [RFC V4 23/30] qcow2: Add check_dedup_l2 in order to check l2 of dedup table.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c | 65 +--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index 9aef608..0c6e75a 100644 --- a/block/qcow2-refcount.c +++ b/block/qco

Re: [Qemu-devel] [RFC V4 00/30] QCOW2 deduplication

13 à 11:01:04 (-0700), Eric Blake a écrit : > On 01/02/2013 10:33 AM, Benoît Canet wrote: > >> How does this code handle hash collisions, and do you have some regression > >> tests that purposefully create a dedup hash collision, and verify that the > >> 'right thing&#

Re: [Qemu-devel] [RFC V4 00/30] QCOW2 deduplication

Le Wednesday 02 Jan 2013 à 12:26:37 (-0600), Troy Benjegerdes a écrit : > The probability may be 'low' but it is not zero. Just because it's > hard to calculate the hash doesn't mean you can't do it. If your > input data is not random the probability of a hash collision is > going to get scewed. >

Re: [Qemu-devel] [RFC V4 00/30] QCOW2 deduplication

n." True, that's more elegant. The user would still need to specify the verify option at creation and it would require to do a read before verify but it would not make the qcow2 format uglier. > > > On Wed, Jan 2, 2013 at 10:40 AM, Benoît Canet > wrote: > > Le W

Re: [Qemu-devel] [RFC V4 00/30] QCOW2 deduplication

for this ? Best regards Benoît > Le Wednesday 02 Jan 2013 à 17:16:03 (+0100), Benoît Canet a écrit : > This patchset is a cleanup of the previous QCOW2 deduplication rfc. > > One can compile and install https://github.com/wernerd/Skein3Fish and use the > --enable-skein-dedup conf

Re: [Qemu-devel] [RFC V4 01/30] qcow2: Add deduplication to the qcow2 specification.

> > +L2 tables size 64KB is different from > > +cluster size 4KB. > > Umm, doesn't the cluster_bits (bytes 20-23 of the header) determine the > size of a cluster, rather than assuming a cluster is always 4KB? And > later on, the spec

[Qemu-devel] QCOW2 deduplication design

Hello, Here is a mail to open a discussion on QCOW2 deduplication design and performance. The actual deduplication strategy is RAM based. One of the goal of the project is to plan and implement an alternative way to do the lookups from disk for bigger images. I will in a first section enumerate

Re: [Qemu-devel] QCOW2 deduplication design

> > What is the GTree indexed by physical offset used for? It's used for two things: deletion and loading of the hashes. -Deletion is a hook in the refcount code that trigger when zero is reached. the only information the code got is the physical offset of the yet to discard cluster. The hash m

Re: [Qemu-devel] QCOW2 deduplication design

> > Two GTrees are used to give access to the hashes : one indexed by hash and > > one other indexed by physical offset. > > What is the GTree indexed by physical offset used for? I think I can get rid of the second GTree for ram based deduplication. It need to: -Start qcow2 with the deduplicati

Re: [Qemu-devel] QCOW2 deduplication design

> Now I understand. This case covers overwriting existing data with new > contents. That is common :). > > But are you seeing a cluster with refcount > 1 being overwritten > often? If so, it's worth looking into why that happens. It may be a > common pattern for certain file systems or applica

Re: [Qemu-devel] [RFC V4 01/30] qcow2: Add deduplication to the qcow2 specification.

> I'd write this as CEIL(dedup_block_size / (32 + 8)) to make it clear > that it rounds up... Isn't it FLOOR instead of CEIL ? (off by one error) ? Benoît

[Qemu-devel] [RFC V5 00/62] QCOW2 deduplication

tibility] Use lazy refcounts to avoid qcow2_cache_set_dependency loops resultings in frequent caches flushes Do not create and load dedup RAM structures when bdrs->read_only is true v3: make it work barely replace kernel red black trees by gtree. Benoît Canet (62):

[Qemu-devel] [RFC V5 02/62] qcow2: Add deduplication structures and fields.

Signed-off-by: Benoit Canet --- block/qcow2.h | 72 - 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/block/qcow2.h b/block/qcow2.h index 718b52b..b31b64e 100644 --- a/block/qcow2.h +++ b/block/qcow2.h @@ -43,6 +43,10 @@ #de

[Qemu-devel] [RFC V5 01/62] qcow2: Add deduplication to the qcow2 specification.

Signed-off-by: Benoit Canet --- docs/specs/qcow2.txt | 104 +- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt index 36a559d..d5f8072 100644 --- a/docs/specs/qcow2.txt +++ b/docs/specs/qco

[Qemu-devel] [RFC V5 06/62] qcow2: Add qcow2_dedup and related functions

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 436 +++ block/qcow2.h |5 + 2 files changed, 441 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 4e99eb1..5901749 100644 --- a/block/qcow2-dedup.c +++ b/bl

[Qemu-devel] [RFC V5 08/62] qcow2: Implement qcow2_compute_cluster_hash.

Add detection of libgnutls used to compute SHA256 hashes Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 13 - configure | 22 ++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index a42

[Qemu-devel] [RFC V5 19/62] qcow2: Add a deduplication boolean to update_refcount.

This is needed for next commit which handle the deduplication refcount overflow case. Signed-off-by: Benoit Canet --- block/qcow2-dedup.c|2 +- block/qcow2-refcount.c | 20 +++- block/qcow2.h |2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff -

[Qemu-devel] [RFC V5 11/62] qcow2: Makes qcow2_alloc_cluster_link_l2 mark to deduplicate clusters.

--- block/qcow2-cluster.c |8 ++-- block/qcow2-dedup.c |7 +++ block/qcow2.h |3 +++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c index dbcb6d2..ef91216 100644 --- a/block/qcow2-cluster.c +++ b/block/qco

[Qemu-devel] [RFC V5 18/62] block: Add qemu-img dedup create option.

Signed-off-by: Benoit Canet --- block/qcow2.c | 113 +++-- block/qcow2.h |2 + include/block/block_int.h |1 + 3 files changed, 103 insertions(+), 13 deletions(-) diff --git a/block/qcow2.c b/block/qcow2.c index f046a77..83

[Qemu-devel] [RFC V5 33/62] qemu-iotests: Filter dedup=on/off so existing tests don't break.

Signed-off-by: Benoit Canet --- tests/qemu-iotests/common.rc |3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index aef5f52..72e746d 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@

[Qemu-devel] [RFC V5 32/62] qcow: Set large dedup hash block size.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c |4 ++-- block/qcow2.c |2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index f305510..348342a 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcoun

[Qemu-devel] [RFC V5 45/62] qcow2: Count deduplication refcount overflow metric.

--- block/qcow2-dedup.c |1 + 1 file changed, 1 insertion(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 4a1b184..db23b71 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -1024,6 +1024,7 @@ void qcow2_dedup_refcount_half_max_reached(BlockDriverState *bs,

[Qemu-devel] [RFC V5 34/62] qcow2: Add qcow2_dedup_init and qcow2_dedup_close.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 97 +++ block/qcow2.h |2 ++ 2 files changed, 99 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 50ffa54..35fcc01 100644 --- a/block/qcow2-dedup.c +++ b/bl

[Qemu-devel] [RFC V5 49/62] qcow2: returns deduplication metrics and status via bdrv_get_info()

--- block/qcow2.c |4 1 file changed, 4 insertions(+) diff --git a/block/qcow2.c b/block/qcow2.c index 753fce0..e442268 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -1868,6 +1868,10 @@ static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi) BDRVQcowState *s = bs-

[Qemu-devel] [RFC V5 47/62] block: Add deduplication metrics to BlockDriverInfo.

--- include/block/block.h |3 +++ 1 file changed, 3 insertions(+) diff --git a/include/block/block.h b/include/block/block.h index 16e1cf1..2043560 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -29,6 +29,9 @@ typedef struct BlockDriverInfo { /* offset at which the VM

[Qemu-devel] [RFC V5 46/62] qapi: Add support for deduplication infos in qapi-schema.json.

--- qapi-schema.json | 40 +++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/qapi-schema.json b/qapi-schema.json index 5dfa052..1a5014c 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -720,6 +720,40 @@ { 'command': 'query-block', 'ret

[Qemu-devel] [RFC V5 61/62] qmp: Add block-pause-dedup.

--- blockdev.c | 18 ++ qapi-schema.json | 18 ++ qmp-commands.hx | 23 +++ 3 files changed, 59 insertions(+) diff --git a/blockdev.c b/blockdev.c index d724e2d..4c5f954 100644 --- a/blockdev.c +++ b/blockdev.c @@ -896,6 +896,24 @@

[Qemu-devel] [RFC V5 52/62] qcow2: Add code to deduplicate cluster flagged with QCOW_OFLAG_TO_DEDUP.

--- block/qcow2-dedup.c | 126 +++ 1 file changed, 126 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 4305746..dd320bf 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -1166,6 +1166,130 @@ static int qcow2_d

[Qemu-devel] [RFC V5 54/62] block: Add bdrv_is_dedup_running.

--- block.c | 19 +++ include/block/block.h |1 + 2 files changed, 20 insertions(+) diff --git a/block.c b/block.c index aee33e0..e83666a 100644 --- a/block.c +++ b/block.c @@ -4342,6 +4342,25 @@ bool bdrv_has_dedup(BlockDriverState *bs) return bdi.has_ded

[Qemu-devel] [RFC V5 41/62] qcow2: Collect undeduplicated cluster metric.

--- block/qcow2-dedup.c |2 ++ 1 file changed, 2 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 716371c..0f095a9 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -524,6 +524,7 @@ static int qcow2_count_next_non_dedupable_clusters(BlockDriverState *bs,

[Qemu-devel] [RFC V5 00/36] QCOW2 deduplication core functionality

lack trees by gtree. Benoît Canet (36): qcow2: Add deduplication to the qcow2 specification. qcow2: Add deduplication structures and fields. qcow2: Add qcow2_dedup_read_missing_and_concatenate qcow2: Make update_refcount public. qcow2: Create a way to link to l2 tables when deduplicating.

[Qemu-devel] [RFC V5 10/36] qcow2: Add qcow2_dedup_grow_table and use it.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 44 +++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index de1b366..de6e3a3 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -38

[Qemu-devel] [RFC V5 12/36] qcow2: make the deduplication forget a cluster hash when a cluster is to dedupe

Signed-off-by: Benoit Canet --- block/qcow2-cluster.c | 11 +-- block/qcow2-dedup.c |8 +++- block/qcow2.h |2 ++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c index ef91216..5b1d20d 100644 --- a/block

[Qemu-devel] [RFC V5 13/36] qcow2: Create qcow2_is_cluster_to_dedup.

Signed-off-by: Benoit Canet --- block/qcow2-cluster.c | 52 + block/qcow2.h |4 2 files changed, 56 insertions(+) diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c index 5b1d20d..fedcf57 100644 --- a/block/qcow2-cluster.

[Qemu-devel] [RFC V5 16/36] qcow2-cache: Allow to choose table size at creation.

Signed-off-by: Benoit Canet --- block/qcow2-cache.c | 12 +++- block/qcow2.c |5 +++-- block/qcow2.h |3 ++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c index 2f3114e..83f2814 100644 --- a/block/qcow2-cach

[Qemu-devel] [RFC V5 21/36] qcow2: Remove hash when cluster is deleted.

--- block/qcow2-dedup.c| 26 ++ block/qcow2-refcount.c |3 +++ block/qcow2.h |2 ++ 3 files changed, 31 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 9eba773..8b51dda 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-ded

[Qemu-devel] [RFC V5 25/36] qcow2: Add verification of dedup table.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c |8 1 file changed, 8 insertions(+) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index 6a6719f..34a6a04 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcount.c @@ -1158,6 +1158,14 @@ int qcow2_check_refc

[Qemu-devel] [RFC V5 30/36] qcow2: Add lazy refcounts to deduplication to prevent qcow2_cache_set_dependency loops

Signed-off-by: Benoit Canet --- block/qcow2.c |1 + 1 file changed, 1 insertion(+) diff --git a/block/qcow2.c b/block/qcow2.c index e742e02..7ef9170 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -1616,6 +1616,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *option

[Qemu-devel] [RFC V5 03/62] qcow2: Add qcow2_dedup_read_missing_and_concatenate

@@ +/* + * Deduplication for the QCOW2 format + * + * Copyright (C) Nodalink, SARL. 2012-2013 + * + * Author: + * Benoît Canet + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to d

[Qemu-devel] [RFC V1 03/14] qcow2: Collect unaligned writes missing data reads metric.

--- block/qcow2-dedup.c |2 ++ 1 file changed, 2 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 997714b..e4920d4 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -130,6 +130,7 @@ int qcow2_dedup_read_missing_and_concatenate(BlockDriverState *bs,

[Qemu-devel] [RFC V1 02/14] qcow2: Initialize deduplication metrics.

--- block/qcow2-dedup.c |2 ++ 1 file changed, 2 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 6cd1af4..997714b 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -1272,6 +1272,8 @@ int qcow2_dedup_init(BlockDriverState *bs) s->has_dedup = true;

[Qemu-devel] [RFC V1 14/14] qapi: Return virtual block device deduplication metrics in QMP

--- block.c | 36 1 file changed, 36 insertions(+) diff --git a/block.c b/block.c index 4e28c55..a245653 100644 --- a/block.c +++ b/block.c @@ -2921,6 +2921,40 @@ BlockInfoList *qmp_query_block(Error **errp) return head; } +static void bdrv_get_dedup

[Qemu-devel] [RFC V1 07/14] qcow2: Count QCowHashNode removal from tree for metrics.

--- block/qcow2-dedup.c |1 + 1 file changed, 1 insertion(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index d22e2a4..64e5a13 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -841,6 +841,7 @@ static void qcow2_remove_hash_node(BlockDriverState *bs, BDRVQcowState

[Qemu-devel] [RFC V1 07/12] qcow2: Add qcow2_pause_dedup.

--- block/qcow2-dedup.c | 19 +++ block/qcow2.c |2 ++ block/qcow2.h |1 + 3 files changed, 22 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index dd320bf..e007387 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -1278,15 +

[Qemu-devel] [RFC V1 00/12] QCOW2 asynchronous deduplication

This patchset implements asynchronous deduplication in QCOW2 on top of the "QCOW2 deduplication metrics" patchset. It's a work in progress posted so reviewers can see where the feature is heading. Benoît Canet (12): block: Add BlockDriver function prototype to p

[Qemu-devel] [RFC V1 01/12] block: Add BlockDriver function prototype to pause and resume deduplication.

--- include/block/block_int.h |4 1 file changed, 4 insertions(+) diff --git a/include/block/block_int.h b/include/block/block_int.h index b7ed3e6..bb35df9 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -203,6 +203,10 @@ struct BlockDriver { */ int (

[Qemu-devel] [RFC V1 02/12] qcow2: Add code to deduplicate cluster flagged with QCOW_OFLAG_TO_DEDUP.

--- block/qcow2-dedup.c | 126 +++ 1 file changed, 126 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 4305746..dd320bf 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -1166,6 +1166,130 @@ static int qcow2_d

[Qemu-devel] [RFC V5 50/62] qapi: Return virtual block device deduplication metrics in QMP

--- block.c | 36 1 file changed, 36 insertions(+) diff --git a/block.c b/block.c index 4e28c55..a245653 100644 --- a/block.c +++ b/block.c @@ -2921,6 +2921,40 @@ BlockInfoList *qmp_query_block(Error **errp) return head; } +static void bdrv_get_dedup

[Qemu-devel] [RFC V5 25/62] qcow2: Add verification of dedup table.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c |8 1 file changed, 8 insertions(+) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index 6a6719f..34a6a04 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcount.c @@ -1158,6 +1158,14 @@ int qcow2_check_refc

[Qemu-devel] [RFC V5 15/36] qcow2: Extract qcow2_do_table_init.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c | 43 ++- block/qcow2.h |5 + 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index e014b0e..75c2bde 100644 --- a/bloc

[Qemu-devel] [RFC V5 39/62] qcow2: Collect unaligned writes missing data reads metric.

--- block/qcow2-dedup.c |2 ++ 1 file changed, 2 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 997714b..e4920d4 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -130,6 +130,7 @@ int qcow2_dedup_read_missing_and_concatenate(BlockDriverState *bs,

[Qemu-devel] [RFC V5 07/62] qcow2: Add qcow2_dedup_store_new_hashes.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 325 ++- block/qcow2.h |5 + 2 files changed, 329 insertions(+), 1 deletion(-) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 5901749..a424af8 100644 --- a/block/qcow2-d

[Qemu-devel] [RFC V5 59/62] qcow2: Make dedup status persists.

--- block/qcow2-dedup.c |7 +++ block/qcow2.c |5 - block/qcow2.h |1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 93545af..85ef66f 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -132

[Qemu-devel] [RFC V5 22/36] qcow2: Add qcow2_dedup_is_running to probe if dedup is running.

--- block/qcow2-dedup.c |6 ++ block/qcow2.h |1 + 2 files changed, 7 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 8b51dda..cc99e27 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-dedup.c @@ -999,3 +999,9 @@ void qcow2_dedup_refcount_half_max_rea

[Qemu-devel] [RFC V5 51/62] block: Add BlockDriver function prototype to pause and resume deduplication.

--- include/block/block_int.h |4 1 file changed, 4 insertions(+) diff --git a/include/block/block_int.h b/include/block/block_int.h index b7ed3e6..bb35df9 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -203,6 +203,10 @@ struct BlockDriver { */ int (

[Qemu-devel] [RFC V1 13/14] qcow2: returns deduplication metrics and status via bdrv_get_info()

--- block/qcow2.c |4 1 file changed, 4 insertions(+) diff --git a/block/qcow2.c b/block/qcow2.c index 753fce0..e442268 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -1868,6 +1868,10 @@ static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi) BDRVQcowState *s = bs-

[Qemu-devel] [RFC V5 36/62] qcow2: Enable the deduplication feature.

--- block/qcow2.c | 17 + 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/block/qcow2.c b/block/qcow2.c index bd7579a..753fce0 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -542,6 +542,13 @@ static int qcow2_open(BlockDriverState *bs, int flags) }

[Qemu-devel] [RFC V5 37/62] qcow2: Add deduplication metrics structures.

--- block/qcow2.h |3 ++- include/block/block.h | 11 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/block/qcow2.h b/block/qcow2.h index 29267a9..0729ff2 100644 --- a/block/qcow2.h +++ b/block/qcow2.h @@ -237,9 +237,10 @@ typedef struct BDRVQcowState {

[Qemu-devel] [RFC V5 27/62] qcow2: Add check_dedup_l2 in order to check l2 of dedup table.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c | 65 +--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index f7a283a..3077a9f 100644 --- a/block/qcow2-refcount.c +++ b/block/qco

[Qemu-devel] [RFC V5 18/36] block: Add qemu-img dedup create option.

Signed-off-by: Benoit Canet --- block/qcow2.c | 113 +++-- block/qcow2.h |2 + include/block/block_int.h |1 + 3 files changed, 103 insertions(+), 13 deletions(-) diff --git a/block/qcow2.c b/block/qcow2.c index f046a77..83

[Qemu-devel] [RFC V5 24/62] qcow2: Serialize write requests when deduplication is activated.

This fix the sub cluster sized writes race conditions while waiting for a more faster solution. Signed-off-by: Benoit Canet --- block/qcow2.c | 14 +- block/qcow2.h |1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/block/qcow2.c b/block/qcow2.c index 6b8f85f.

[Qemu-devel] [RFC V5 29/36] qcow2: Integrate SKEIN hash algorithm in deduplication.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 14 ++ block/qcow2.c |5 + configure | 33 + 3 files changed, 52 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index cc99e27..50ffa54 100644 --- a/

[Qemu-devel] [RFC V5 06/36] qcow2: Add qcow2_dedup and related functions

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 436 +++ block/qcow2.h |5 + 2 files changed, 441 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 4e99eb1..5901749 100644 --- a/block/qcow2-dedup.c +++ b/bl

[Qemu-devel] [RFC V5 29/62] qcow2: Integrate SKEIN hash algorithm in deduplication.

Signed-off-by: Benoit Canet --- block/qcow2-dedup.c | 14 ++ block/qcow2.c |5 + configure | 33 + 3 files changed, 52 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index cc99e27..50ffa54 100644 --- a/

[Qemu-devel] [RFC V5 26/62] qcow2: Adapt checking of QCOW_OFLAG_COPIED for dedup.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c |9 - 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index 34a6a04..f7a283a 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcount.c @@ -1003,7 +1003,14 @@ stat

[Qemu-devel] [RFC V5 27/36] qcow2: Add check_dedup_l2 in order to check l2 of dedup table.

Signed-off-by: Benoit Canet --- block/qcow2-refcount.c | 65 +--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index f7a283a..3077a9f 100644 --- a/block/qcow2-refcount.c +++ b/block/qco

[Qemu-devel] [RFC V1 06/12] block: Add bdrv_pause_dedup.

--- block.c | 19 +++ include/block/block.h |1 + 2 files changed, 20 insertions(+) diff --git a/block.c b/block.c index 4e80da8..8c527b6 100644 --- a/block.c +++ b/block.c @@ -4380,6 +4380,25 @@ int bdrv_resume_dedup(BlockDriverState *bs) return 0; } +

[Qemu-devel] [RFC V5 21/62] qcow2: Remove hash when cluster is deleted.

--- block/qcow2-dedup.c| 26 ++ block/qcow2-refcount.c |3 +++ block/qcow2.h |2 ++ 3 files changed, 31 insertions(+) diff --git a/block/qcow2-dedup.c b/block/qcow2-dedup.c index 9eba773..8b51dda 100644 --- a/block/qcow2-dedup.c +++ b/block/qcow2-ded

[Qemu-devel] [RFC V5 55/62] block: Add bdrv_resume_dedup.

--- block.c | 19 +++ include/block/block.h |1 + 2 files changed, 20 insertions(+) diff --git a/block.c b/block.c index e83666a..4e80da8 100644 --- a/block.c +++ b/block.c @@ -4361,6 +4361,25 @@ bool bdrv_is_dedup_running(BlockDriverState *bs) return bdi.

<    3   4   5   6   7   8   9   10   11   12   >