[PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-10-27 Thread Liu Bo 
This comes from one of btrfs's project ideas,
As we defragment files, we break any sharing from other snapshots.
The balancing code will preserve the sharing, and defrag needs to grow this
as well.

Now we're able to fill the blank with this patch, in which we make full use of
backref walking stuff.

Here is the basic idea,
o  set the writeback ranges started by defragment with flag EXTENT_DEFRAG
o  at endio, after we finish updating fs tree, we use backref walking to find
   all parents of the ranges and re-link them with the new COWed file layout by
   adding corresponding backrefs.

Originally patch by Li Zefan 
Signed-off-by: Liu Bo 
---
v3->v4:
  - fix duplicated refs bugs detected by mounting with autodefrag, thanks
for the bug report from Mitch and Chris.

 fs/btrfs/inode.c |  609 ++
 1 files changed, 609 insertions(+), 0 deletions(-)

diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 85a1e50..35e6993 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -54,6 +54,7 @@
 #include "locking.h"
 #include "free-space-cache.h"
 #include "inode-map.h"
+#include "backref.h"
 
 struct btrfs_iget_args {
u64 ino;
@@ -1839,6 +1840,600 @@ out:
return ret;
 }
 
+/* snapshot-aware defrag */
+struct sa_defrag_extent_backref {
+   struct rb_node node;
+   struct old_sa_defrag_extent *old;
+   u64 root_id;
+   u64 inum;
+   u64 file_pos;
+   u64 extent_offset;
+   u64 num_bytes;
+   u64 generation;
+};
+
+struct old_sa_defrag_extent {
+   struct list_head list;
+   struct new_sa_defrag_extent *new;
+
+   u64 extent_offset;
+   u64 bytenr;
+   u64 offset;
+   u64 len;
+   int count;
+};
+
+struct new_sa_defrag_extent {
+   struct rb_root root;
+   struct list_head head;
+   struct btrfs_path *path;
+   struct inode *inode;
+   u64 file_pos;
+   u64 len;
+   u64 bytenr;
+   u64 disk_len;
+   u8 compress_type;
+};
+
+static int backref_comp(struct sa_defrag_extent_backref *b1,
+   struct sa_defrag_extent_backref *b2)
+{
+   if (b1->root_id < b2->root_id)
+   return -1;
+   else if (b1->root_id > b2->root_id)
+   return 1;
+
+   if (b1->inum < b2->inum)
+   return -1;
+   else if (b1->inum > b2->inum)
+   return 1;
+
+   if (b1->file_pos < b2->file_pos)
+   return -1;
+   else if (b1->file_pos > b2->file_pos)
+   return 1;
+
+   return 0;
+}
+
+static void backref_insert(struct rb_root *root,
+  struct sa_defrag_extent_backref *backref)
+{
+   struct rb_node **p = &root->rb_node;
+   struct rb_node *parent = NULL;
+   struct sa_defrag_extent_backref *entry;
+   int ret;
+
+   while (*p) {
+   parent = *p;
+   entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
+
+   ret = backref_comp(backref, entry);
+   if (ret < 0)
+   p = &(*p)->rb_left;
+   else
+   /*
+* Since space can be shared, so there can be
+* some backrefs(extent tree to fs/file tree)
+* whoes fs/file extents map to the same address.
+* If so, we just put it after what we've found.
+*/
+   p = &(*p)->rb_right;
+   }
+
+   rb_link_node(&backref->node, parent, p);
+   rb_insert_color(&backref->node, root);
+}
+
+/*
+ * Note the backref might has changed, and in this case we just return 0.
+ */
+static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
+  void *ctx)
+{
+   struct btrfs_file_extent_item *extent;
+   struct btrfs_fs_info *fs_info;
+   struct old_sa_defrag_extent *old = ctx;
+   struct new_sa_defrag_extent *new = old->new;
+   struct btrfs_path *path = new->path;
+   struct btrfs_key key;
+   struct btrfs_root *root;
+   struct sa_defrag_extent_backref *backref;
+   struct extent_buffer *leaf;
+   struct inode *inode = new->inode;
+   int slot;
+   int ret;
+   u64 extent_offset;
+   u64 num_bytes;
+
+   if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
+   inum == btrfs_ino(inode))
+   return 0;
+
+   key.objectid = root_id;
+   key.type = BTRFS_ROOT_ITEM_KEY;
+   key.offset = (u64)-1;
+
+   fs_info = BTRFS_I(inode)->root->fs_info;
+   root = btrfs_read_fs_root_no_name(fs_info, &key);
+   if (IS_ERR(root)) {
+   if (PTR_ERR(root) == -ENOENT)
+   return 0;
+   WARN_ON(1);
+   pr_debug("inum=%llu, offset=%llu, root_id=%llu\n",
+inum, offset, root_id);
+   return PTR_ERR(root);
+   }
+
+

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-10-29 Thread Mitch Harder 
On Sat, Oct 27, 2012 at 5:28 AM, Liu Bo  wrote:
> This comes from one of btrfs's project ideas,
> As we defragment files, we break any sharing from other snapshots.
> The balancing code will preserve the sharing, and defrag needs to grow this
> as well.
>
> Now we're able to fill the blank with this patch, in which we make full use of
> backref walking stuff.
>
> Here is the basic idea,
> o  set the writeback ranges started by defragment with flag EXTENT_DEFRAG
> o  at endio, after we finish updating fs tree, we use backref walking to find
>all parents of the ranges and re-link them with the new COWed file layout 
> by
>adding corresponding backrefs.
>
> Originally patch by Li Zefan 
> Signed-off-by: Liu Bo 
> ---
> v3->v4:
>   - fix duplicated refs bugs detected by mounting with autodefrag, thanks
> for the bug report from Mitch and Chris.
>

I'm picking up many WARN_ON messages while testing this patch.

I'm testing a snapshot script that uses kernel git sources along with
some git manipulations.

The kernel is a 3.6.4 kernel merged with the latest for-linus branch.

I mounted with -o compress-force=lzo,autodefrag.

I also have the second patch in this set (Btrfs: make snapshot-aware
defrag as a mount option).  However, I did not mount with
'snap_aware_defrag'.

I did not find any corrupted data, and the partition passes a btrfsck
without error after these warnings were observed.

Here's a summary of the WARN_ON messages:

$ cat local/dmesg-3.6.4-x+ | grep "WARNING:"
[  610.407561] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[  610.407757] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[  610.407929] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[  661.211849] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[  661.212004] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[  661.212236] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[  719.882942] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[  719.883112] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[  719.883232] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[  786.978869] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[  786.979003] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[  786.979140] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[  845.605176] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[  845.605323] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[  845.605445] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[  912.300307] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[  912.300454] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[  912.300577] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[  968.835873] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[  968.836032] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[  968.836156] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[ 1023.778160] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[ 1023.778316] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[ 1023.778435] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[ 1064.342768] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[ 1064.342914] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[ 1064.343112] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[ 1177.892047] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[ 1177.892189] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[ 1177.892312] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[ 1281.951715] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[ 1281.951857] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[ 1281.951978] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[ 1282.804376] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[ 1282.804524] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[ 1282.804645] WARNING: at fs/btrfs/inode.c:7782
btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
[ 1351.187114] WARNING: at fs/btrfs/inode.c:7779
btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
[ 1351.187263] WARNING: at fs/btrfs/inode.c:7780
btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
[ 1351.187391] WARNING: at

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-10-29 Thread Liu Bo 
On 10/30/2012 04:06 AM, Mitch Harder wrote:
> On Sat, Oct 27, 2012 at 5:28 AM, Liu Bo  wrote:
>> This comes from one of btrfs's project ideas,
>> As we defragment files, we break any sharing from other snapshots.
>> The balancing code will preserve the sharing, and defrag needs to grow this
>> as well.
>>
>> Now we're able to fill the blank with this patch, in which we make full use 
>> of
>> backref walking stuff.
>>
>> Here is the basic idea,
>> o  set the writeback ranges started by defragment with flag EXTENT_DEFRAG
>> o  at endio, after we finish updating fs tree, we use backref walking to find
>>all parents of the ranges and re-link them with the new COWed file layout 
>> by
>>adding corresponding backrefs.
>>
>> Originally patch by Li Zefan 
>> Signed-off-by: Liu Bo 
>> ---
>> v3->v4:
>>   - fix duplicated refs bugs detected by mounting with autodefrag, thanks
>> for the bug report from Mitch and Chris.
>>
> 
> I'm picking up many WARN_ON messages while testing this patch.
> 
> I'm testing a snapshot script that uses kernel git sources along with
> some git manipulations.
> 
> The kernel is a 3.6.4 kernel merged with the latest for-linus branch.
> 
> I mounted with -o compress-force=lzo,autodefrag.
> 
> I also have the second patch in this set (Btrfs: make snapshot-aware
> defrag as a mount option).  However, I did not mount with
> 'snap_aware_defrag'.
> 
> I did not find any corrupted data, and the partition passes a btrfsck
> without error after these warnings were observed.
> 

Hi Mitch,

Well, good report, but I don't think it has anything to do with this 
patch(since you
didn't mount with 'snap_aware_defrag' :)

After going through the below messages, the bug comes from the space side where 
we
must have mis-used our reservation somehow.

So can you show me your script so that I can give it a shot to reproduce 
locally?

thanks,
liubo


> Here's a summary of the WARN_ON messages:
> 
> $ cat local/dmesg-3.6.4-x+ | grep "WARNING:"
> [  610.407561] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [  610.407757] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [  610.407929] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [  661.211849] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [  661.212004] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [  661.212236] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [  719.882942] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [  719.883112] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [  719.883232] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [  786.978869] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [  786.979003] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [  786.979140] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [  845.605176] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [  845.605323] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [  845.605445] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [  912.300307] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [  912.300454] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [  912.300577] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [  968.835873] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [  968.836032] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [  968.836156] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [ 1023.778160] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [ 1023.778316] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [ 1023.778435] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [ 1064.342768] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [ 1064.342914] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [ 1064.343112] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [ 1177.892047] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [ 1177.892189] WARNING: at fs/btrfs/inode.c:7780
> btrfs_destroy_inode+0x296/0x2e6 [btrfs]()
> [ 1177.892312] WARNING: at fs/btrfs/inode.c:7782
> btrfs_destroy_inode+0x26a/0x2e6 [btrfs]()
> [ 1281.951715] WARNING: at fs/btrfs/inode.c:7779
> btrfs_destroy_inode+0x2ac/0x2e6 [btrfs]()
> [ 1281.951857] WARNING: at fs/btrfs/inode.c:7780
> btrfs_

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-10-30 Thread Mitch Harder 
On Mon, Oct 29, 2012 at 8:20 PM, Liu Bo  wrote:
> On 10/30/2012 04:06 AM, Mitch Harder wrote:
>> On Sat, Oct 27, 2012 at 5:28 AM, Liu Bo  wrote:
>>> This comes from one of btrfs's project ideas,
>>> As we defragment files, we break any sharing from other snapshots.
>>> The balancing code will preserve the sharing, and defrag needs to grow this
>>> as well.
>>>
>>> Now we're able to fill the blank with this patch, in which we make full use 
>>> of
>>> backref walking stuff.
>>>
>>> Here is the basic idea,
>>> o  set the writeback ranges started by defragment with flag EXTENT_DEFRAG
>>> o  at endio, after we finish updating fs tree, we use backref walking to 
>>> find
>>>all parents of the ranges and re-link them with the new COWed file 
>>> layout by
>>>adding corresponding backrefs.
>>>
>>> Originally patch by Li Zefan 
>>> Signed-off-by: Liu Bo 
>>> ---
>>> v3->v4:
>>>   - fix duplicated refs bugs detected by mounting with autodefrag, 
>>> thanks
>>> for the bug report from Mitch and Chris.
>>>
>>
>> I'm picking up many WARN_ON messages while testing this patch.
>>
>> I'm testing a snapshot script that uses kernel git sources along with
>> some git manipulations.
>>
>> The kernel is a 3.6.4 kernel merged with the latest for-linus branch.
>>
>> I mounted with -o compress-force=lzo,autodefrag.
>>
>> I also have the second patch in this set (Btrfs: make snapshot-aware
>> defrag as a mount option).  However, I did not mount with
>> 'snap_aware_defrag'.
>>
>> I did not find any corrupted data, and the partition passes a btrfsck
>> without error after these warnings were observed.
>>
>
> Hi Mitch,
>
> Well, good report, but I don't think it has anything to do with this 
> patch(since you
> didn't mount with 'snap_aware_defrag' :)
>

I've re-run my my testing script with a combination of no compression
and lzo compression, combined with no further options, only -o
autodefrag, and -o autodefrag,snap_aware_defrag.

I only get the WARN_ONs when I run with autodefrag only (no snap_aware_defrag).

My logs are clean when I avoid all defrag options, or use both
autodefrag and snap_aware_defrag.

> After going through the below messages, the bug comes from the space side 
> where we
> must have mis-used our reservation somehow.
>
> So can you show me your script so that I can give it a shot to reproduce 
> locally?
>
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-10-31 Thread Itaru Kitayama 
Hi LiuBo:

I am seeing another warning with your patch applied btrfs-next.

[ 5224.531560] [ cut here ]
[ 5224.531565] WARNING: at fs/btrfs/inode.c:2054
record_extent_backrefs+0x87/0xe0()
[ 5224.531567] Hardware name: Bochs
[ 5224.531568] Modules linked in: microcode ppdev psmouse nfsd nfs_acl
auth_rpcgss serio_raw nfs fscache lockd binfmt_misc sunrpc cirrus
parport_pc ttm drm_kms_helper drm sysimgblt i2c_piix4 sysfillrect
syscopyarea i2c_core lp parport floppy
[ 5224.531591] Pid: 2485, comm: btrfs-endio-wri Tainted: GW
3.7.0-rc1-v11+ #53
[ 5224.531592] Call Trace:
[ 5224.531598]  [] warn_slowpath_common+0x93/0xc0
[ 5224.531600]  [] warn_slowpath_null+0x1a/0x20
[ 5224.531603]  [] record_extent_backrefs+0x87/0xe0
[ 5224.531606]  [] btrfs_finish_ordered_io+0x8bb/0xa80
[ 5224.531611]  [] ? trace_hardirqs_off_caller+0xb0/0x140
[ 5224.531614]  [] finish_ordered_fn+0x15/0x20
[ 5224.531617]  [] worker_loop+0x157/0x580
[ 5224.531620]  [] ? btrfs_queue_worker+0x2f0/0x2f0
[ 5224.531624]  [] kthread+0xe8/0xf0
[ 5224.531627]  [] ? get_lock_stats+0x22/0x70
[ 5224.531630]  [] ? kthread_create_on_node+0x160/0x160
[ 5224.531634]  [] ret_from_fork+0x7c/0xb0
[ 5224.531636]  [] ? kthread_create_on_node+0x160/0x160
[ 5224.531638] ---[ end trace 0256d2b5a195208c ]---

I've compared some of the old extents logical addresses with the corresponding
object ids and offsets from the extent tree; some are just 8k off from
the found extents
and some keys are totally off.

Itaru

On Sat, Oct 27, 2012 at 7:28 PM, Liu Bo  wrote:
> This comes from one of btrfs's project ideas,
> As we defragment files, we break any sharing from other snapshots.
> The balancing code will preserve the sharing, and defrag needs to grow this
> as well.
>
> Now we're able to fill the blank with this patch, in which we make full use of
> backref walking stuff.
>
> Here is the basic idea,
> o  set the writeback ranges started by defragment with flag EXTENT_DEFRAG
> o  at endio, after we finish updating fs tree, we use backref walking to find
>all parents of the ranges and re-link them with the new COWed file layout 
> by
>adding corresponding backrefs.
>
> Originally patch by Li Zefan 
> Signed-off-by: Liu Bo 
> ---
> v3->v4:
>   - fix duplicated refs bugs detected by mounting with autodefrag, thanks
> for the bug report from Mitch and Chris.
>
>  fs/btrfs/inode.c |  609 
> ++
>  1 files changed, 609 insertions(+), 0 deletions(-)
>
> diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
> index 85a1e50..35e6993 100644
> --- a/fs/btrfs/inode.c
> +++ b/fs/btrfs/inode.c
> @@ -54,6 +54,7 @@
>  #include "locking.h"
>  #include "free-space-cache.h"
>  #include "inode-map.h"
> +#include "backref.h"
>
>  struct btrfs_iget_args {
> u64 ino;
> @@ -1839,6 +1840,600 @@ out:
> return ret;
>  }
>
> +/* snapshot-aware defrag */
> +struct sa_defrag_extent_backref {
> +   struct rb_node node;
> +   struct old_sa_defrag_extent *old;
> +   u64 root_id;
> +   u64 inum;
> +   u64 file_pos;
> +   u64 extent_offset;
> +   u64 num_bytes;
> +   u64 generation;
> +};
> +
> +struct old_sa_defrag_extent {
> +   struct list_head list;
> +   struct new_sa_defrag_extent *new;
> +
> +   u64 extent_offset;
> +   u64 bytenr;
> +   u64 offset;
> +   u64 len;
> +   int count;
> +};
> +
> +struct new_sa_defrag_extent {
> +   struct rb_root root;
> +   struct list_head head;
> +   struct btrfs_path *path;
> +   struct inode *inode;
> +   u64 file_pos;
> +   u64 len;
> +   u64 bytenr;
> +   u64 disk_len;
> +   u8 compress_type;
> +};
> +
> +static int backref_comp(struct sa_defrag_extent_backref *b1,
> +   struct sa_defrag_extent_backref *b2)
> +{
> +   if (b1->root_id < b2->root_id)
> +   return -1;
> +   else if (b1->root_id > b2->root_id)
> +   return 1;
> +
> +   if (b1->inum < b2->inum)
> +   return -1;
> +   else if (b1->inum > b2->inum)
> +   return 1;
> +
> +   if (b1->file_pos < b2->file_pos)
> +   return -1;
> +   else if (b1->file_pos > b2->file_pos)
> +   return 1;
> +
> +   return 0;
> +}
> +
> +static void backref_insert(struct rb_root *root,
> +  struct sa_defrag_extent_backref *backref)
> +{
> +   struct rb_node **p = &root->rb_node;
> +   struct rb_node *parent = NULL;
> +   struct sa_defrag_extent_backref *entry;
> +   int ret;
> +
> +   while (*p) {
> +   parent = *p;
> +   entry = rb_entry(parent, struct sa_defrag_extent_backref, 
> node);
> +
> +   ret = backref_comp(backref, entry);
> +   if (ret < 0)
> +   p = &(*p)->rb_left;
> +   else
> +   /*
> +* Since space can be shared, so there ca

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-10-31 Thread Liu Bo 
On 10/31/2012 08:13 PM, Itaru Kitayama wrote:
> Hi LiuBo:
> 
> I am seeing another warning with your patch applied btrfs-next.
> 

Hi Itaru,

Thanks for testing, you seems to be using an old version, since in the new 
version
record_extent_backrefs() does not own a WARN_ON().

Could you please test it again with the new patches applied?

thanks,
liubo


> [ 5224.531560] [ cut here ]
> [ 5224.531565] WARNING: at fs/btrfs/inode.c:2054
> record_extent_backrefs+0x87/0xe0()
> [ 5224.531567] Hardware name: Bochs
> [ 5224.531568] Modules linked in: microcode ppdev psmouse nfsd nfs_acl
> auth_rpcgss serio_raw nfs fscache lockd binfmt_misc sunrpc cirrus
> parport_pc ttm drm_kms_helper drm sysimgblt i2c_piix4 sysfillrect
> syscopyarea i2c_core lp parport floppy
> [ 5224.531591] Pid: 2485, comm: btrfs-endio-wri Tainted: GW
> 3.7.0-rc1-v11+ #53
> [ 5224.531592] Call Trace:
> [ 5224.531598]  [] warn_slowpath_common+0x93/0xc0
> [ 5224.531600]  [] warn_slowpath_null+0x1a/0x20
> [ 5224.531603]  [] record_extent_backrefs+0x87/0xe0
> [ 5224.531606]  [] btrfs_finish_ordered_io+0x8bb/0xa80
> [ 5224.531611]  [] ? trace_hardirqs_off_caller+0xb0/0x140
> [ 5224.531614]  [] finish_ordered_fn+0x15/0x20
> [ 5224.531617]  [] worker_loop+0x157/0x580
> [ 5224.531620]  [] ? btrfs_queue_worker+0x2f0/0x2f0
> [ 5224.531624]  [] kthread+0xe8/0xf0
> [ 5224.531627]  [] ? get_lock_stats+0x22/0x70
> [ 5224.531630]  [] ? kthread_create_on_node+0x160/0x160
> [ 5224.531634]  [] ret_from_fork+0x7c/0xb0
> [ 5224.531636]  [] ? kthread_create_on_node+0x160/0x160
> [ 5224.531638] ---[ end trace 0256d2b5a195208c ]---
> 
> I've compared some of the old extents logical addresses with the corresponding
> object ids and offsets from the extent tree; some are just 8k off from
> the found extents
> and some keys are totally off.
> 
> Itaru
> 
> On Sat, Oct 27, 2012 at 7:28 PM, Liu Bo  wrote:
>> This comes from one of btrfs's project ideas,
>> As we defragment files, we break any sharing from other snapshots.
>> The balancing code will preserve the sharing, and defrag needs to grow this
>> as well.
>>
>> Now we're able to fill the blank with this patch, in which we make full use 
>> of
>> backref walking stuff.
>>
>> Here is the basic idea,
>> o  set the writeback ranges started by defragment with flag EXTENT_DEFRAG
>> o  at endio, after we finish updating fs tree, we use backref walking to find
>>all parents of the ranges and re-link them with the new COWed file layout 
>> by
>>adding corresponding backrefs.
>>
>> Originally patch by Li Zefan 
>> Signed-off-by: Liu Bo 
>> ---
>> v3->v4:
>>   - fix duplicated refs bugs detected by mounting with autodefrag, thanks
>> for the bug report from Mitch and Chris.
>>
>>  fs/btrfs/inode.c |  609 
>> ++
>>  1 files changed, 609 insertions(+), 0 deletions(-)
>>
>> diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
>> index 85a1e50..35e6993 100644
>> --- a/fs/btrfs/inode.c
>> +++ b/fs/btrfs/inode.c
>> @@ -54,6 +54,7 @@
>>  #include "locking.h"
>>  #include "free-space-cache.h"
>>  #include "inode-map.h"
>> +#include "backref.h"
>>
>>  struct btrfs_iget_args {
>> u64 ino;
>> @@ -1839,6 +1840,600 @@ out:
>> return ret;
>>  }
>>
>> +/* snapshot-aware defrag */
>> +struct sa_defrag_extent_backref {
>> +   struct rb_node node;
>> +   struct old_sa_defrag_extent *old;
>> +   u64 root_id;
>> +   u64 inum;
>> +   u64 file_pos;
>> +   u64 extent_offset;
>> +   u64 num_bytes;
>> +   u64 generation;
>> +};
>> +
>> +struct old_sa_defrag_extent {
>> +   struct list_head list;
>> +   struct new_sa_defrag_extent *new;
>> +
>> +   u64 extent_offset;
>> +   u64 bytenr;
>> +   u64 offset;
>> +   u64 len;
>> +   int count;
>> +};
>> +
>> +struct new_sa_defrag_extent {
>> +   struct rb_root root;
>> +   struct list_head head;
>> +   struct btrfs_path *path;
>> +   struct inode *inode;
>> +   u64 file_pos;
>> +   u64 len;
>> +   u64 bytenr;
>> +   u64 disk_len;
>> +   u8 compress_type;
>> +};
>> +
>> +static int backref_comp(struct sa_defrag_extent_backref *b1,
>> +   struct sa_defrag_extent_backref *b2)
>> +{
>> +   if (b1->root_id < b2->root_id)
>> +   return -1;
>> +   else if (b1->root_id > b2->root_id)
>> +   return 1;
>> +
>> +   if (b1->inum < b2->inum)
>> +   return -1;
>> +   else if (b1->inum > b2->inum)
>> +   return 1;
>> +
>> +   if (b1->file_pos < b2->file_pos)
>> +   return -1;
>> +   else if (b1->file_pos > b2->file_pos)
>> +   return 1;
>> +
>> +   return 0;
>> +}
>> +
>> +static void backref_insert(struct rb_root *root,
>> +  struct sa_defrag_extent_backref *backref)
>> +{
>> +   struct rb_node **p = &root->rb_node;
>> +   struct rb_node *parent = NULL;
>> +   s

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-11-01 Thread Itaru Kitayama 
Hi Liubo,

I couldn't apply your V4 patch against the btrfs-next HEAD. Do you have
a github branch which I can checkout?

Thanks,

Itaru

On Wed, Oct 31, 2012 at 9:55 PM, Liu Bo  wrote:
> On 10/31/2012 08:13 PM, Itaru Kitayama wrote:
>> Hi LiuBo:
>>
>> I am seeing another warning with your patch applied btrfs-next.
>>
>
> Hi Itaru,
>
> Thanks for testing, you seems to be using an old version, since in the new 
> version
> record_extent_backrefs() does not own a WARN_ON().
>
> Could you please test it again with the new patches applied?
>
> thanks,
> liubo
>
>
>> [ 5224.531560] [ cut here ]
>> [ 5224.531565] WARNING: at fs/btrfs/inode.c:2054
>> record_extent_backrefs+0x87/0xe0()
>> [ 5224.531567] Hardware name: Bochs
>> [ 5224.531568] Modules linked in: microcode ppdev psmouse nfsd nfs_acl
>> auth_rpcgss serio_raw nfs fscache lockd binfmt_misc sunrpc cirrus
>> parport_pc ttm drm_kms_helper drm sysimgblt i2c_piix4 sysfillrect
>> syscopyarea i2c_core lp parport floppy
>> [ 5224.531591] Pid: 2485, comm: btrfs-endio-wri Tainted: GW
>> 3.7.0-rc1-v11+ #53
>> [ 5224.531592] Call Trace:
>> [ 5224.531598]  [] warn_slowpath_common+0x93/0xc0
>> [ 5224.531600]  [] warn_slowpath_null+0x1a/0x20
>> [ 5224.531603]  [] record_extent_backrefs+0x87/0xe0
>> [ 5224.531606]  [] btrfs_finish_ordered_io+0x8bb/0xa80
>> [ 5224.531611]  [] ? trace_hardirqs_off_caller+0xb0/0x140
>> [ 5224.531614]  [] finish_ordered_fn+0x15/0x20
>> [ 5224.531617]  [] worker_loop+0x157/0x580
>> [ 5224.531620]  [] ? btrfs_queue_worker+0x2f0/0x2f0
>> [ 5224.531624]  [] kthread+0xe8/0xf0
>> [ 5224.531627]  [] ? get_lock_stats+0x22/0x70
>> [ 5224.531630]  [] ? kthread_create_on_node+0x160/0x160
>> [ 5224.531634]  [] ret_from_fork+0x7c/0xb0
>> [ 5224.531636]  [] ? kthread_create_on_node+0x160/0x160
>> [ 5224.531638] ---[ end trace 0256d2b5a195208c ]---
>>
>> I've compared some of the old extents logical addresses with the 
>> corresponding
>> object ids and offsets from the extent tree; some are just 8k off from
>> the found extents
>> and some keys are totally off.
>>
>> Itaru
>>
>> On Sat, Oct 27, 2012 at 7:28 PM, Liu Bo  wrote:
>>> This comes from one of btrfs's project ideas,
>>> As we defragment files, we break any sharing from other snapshots.
>>> The balancing code will preserve the sharing, and defrag needs to grow this
>>> as well.
>>>
>>> Now we're able to fill the blank with this patch, in which we make full use 
>>> of
>>> backref walking stuff.
>>>
>>> Here is the basic idea,
>>> o  set the writeback ranges started by defragment with flag EXTENT_DEFRAG
>>> o  at endio, after we finish updating fs tree, we use backref walking to 
>>> find
>>>all parents of the ranges and re-link them with the new COWed file 
>>> layout by
>>>adding corresponding backrefs.
>>>
>>> Originally patch by Li Zefan 
>>> Signed-off-by: Liu Bo 
>>> ---
>>> v3->v4:
>>>   - fix duplicated refs bugs detected by mounting with autodefrag, 
>>> thanks
>>> for the bug report from Mitch and Chris.
>>>
>>>  fs/btrfs/inode.c |  609 
>>> ++
>>>  1 files changed, 609 insertions(+), 0 deletions(-)
>>>
>>> diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
>>> index 85a1e50..35e6993 100644
>>> --- a/fs/btrfs/inode.c
>>> +++ b/fs/btrfs/inode.c
>>> @@ -54,6 +54,7 @@
>>>  #include "locking.h"
>>>  #include "free-space-cache.h"
>>>  #include "inode-map.h"
>>> +#include "backref.h"
>>>
>>>  struct btrfs_iget_args {
>>> u64 ino;
>>> @@ -1839,6 +1840,600 @@ out:
>>> return ret;
>>>  }
>>>
>>> +/* snapshot-aware defrag */
>>> +struct sa_defrag_extent_backref {
>>> +   struct rb_node node;
>>> +   struct old_sa_defrag_extent *old;
>>> +   u64 root_id;
>>> +   u64 inum;
>>> +   u64 file_pos;
>>> +   u64 extent_offset;
>>> +   u64 num_bytes;
>>> +   u64 generation;
>>> +};
>>> +
>>> +struct old_sa_defrag_extent {
>>> +   struct list_head list;
>>> +   struct new_sa_defrag_extent *new;
>>> +
>>> +   u64 extent_offset;
>>> +   u64 bytenr;
>>> +   u64 offset;
>>> +   u64 len;
>>> +   int count;
>>> +};
>>> +
>>> +struct new_sa_defrag_extent {
>>> +   struct rb_root root;
>>> +   struct list_head head;
>>> +   struct btrfs_path *path;
>>> +   struct inode *inode;
>>> +   u64 file_pos;
>>> +   u64 len;
>>> +   u64 bytenr;
>>> +   u64 disk_len;
>>> +   u8 compress_type;
>>> +};
>>> +
>>> +static int backref_comp(struct sa_defrag_extent_backref *b1,
>>> +   struct sa_defrag_extent_backref *b2)
>>> +{
>>> +   if (b1->root_id < b2->root_id)
>>> +   return -1;
>>> +   else if (b1->root_id > b2->root_id)
>>> +   return 1;
>>> +
>>> +   if (b1->inum < b2->inum)
>>> +   return -1;
>>> +   else if (b1->inum > b2->inum)
>>> +   return 1;
>>> +
>>> +   if (b1->file_pos < b2->file_pos)
>>> +   return -1;

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-11-01 Thread Liu Bo 
On Thu, Nov 01, 2012 at 08:08:52PM +0900, Itaru Kitayama wrote:
> Hi Liubo,
> 
> I couldn't apply your V4 patch against the btrfs-next HEAD. Do you have
> a github branch which I can checkout?
> 

The current btrfs-next HEAD actually have included this v4 patch, so
just pull btrfs-next and give it a shot :)

thanks,
liubo

> Thanks,
> 
> Itaru
> 
> On Wed, Oct 31, 2012 at 9:55 PM, Liu Bo  wrote:
> > On 10/31/2012 08:13 PM, Itaru Kitayama wrote:
> >> Hi LiuBo:
> >>
> >> I am seeing another warning with your patch applied btrfs-next.
> >>
> >
> > Hi Itaru,
> >
> > Thanks for testing, you seems to be using an old version, since in the new 
> > version
> > record_extent_backrefs() does not own a WARN_ON().
> >
> > Could you please test it again with the new patches applied?
> >
> > thanks,
> > liubo
> >
> >
> >> [ 5224.531560] [ cut here ]
> >> [ 5224.531565] WARNING: at fs/btrfs/inode.c:2054
> >> record_extent_backrefs+0x87/0xe0()
> >> [ 5224.531567] Hardware name: Bochs
> >> [ 5224.531568] Modules linked in: microcode ppdev psmouse nfsd nfs_acl
> >> auth_rpcgss serio_raw nfs fscache lockd binfmt_misc sunrpc cirrus
> >> parport_pc ttm drm_kms_helper drm sysimgblt i2c_piix4 sysfillrect
> >> syscopyarea i2c_core lp parport floppy
> >> [ 5224.531591] Pid: 2485, comm: btrfs-endio-wri Tainted: GW
> >> 3.7.0-rc1-v11+ #53
> >> [ 5224.531592] Call Trace:
> >> [ 5224.531598]  [] warn_slowpath_common+0x93/0xc0
> >> [ 5224.531600]  [] warn_slowpath_null+0x1a/0x20
> >> [ 5224.531603]  [] record_extent_backrefs+0x87/0xe0
> >> [ 5224.531606]  [] btrfs_finish_ordered_io+0x8bb/0xa80
> >> [ 5224.531611]  [] ? trace_hardirqs_off_caller+0xb0/0x140
> >> [ 5224.531614]  [] finish_ordered_fn+0x15/0x20
> >> [ 5224.531617]  [] worker_loop+0x157/0x580
> >> [ 5224.531620]  [] ? btrfs_queue_worker+0x2f0/0x2f0
> >> [ 5224.531624]  [] kthread+0xe8/0xf0
> >> [ 5224.531627]  [] ? get_lock_stats+0x22/0x70
> >> [ 5224.531630]  [] ? kthread_create_on_node+0x160/0x160
> >> [ 5224.531634]  [] ret_from_fork+0x7c/0xb0
> >> [ 5224.531636]  [] ? kthread_create_on_node+0x160/0x160
> >> [ 5224.531638] ---[ end trace 0256d2b5a195208c ]---
> >>
> >> I've compared some of the old extents logical addresses with the 
> >> corresponding
> >> object ids and offsets from the extent tree; some are just 8k off from
> >> the found extents
> >> and some keys are totally off.
> >>
> >> Itaru
> >>
> >> On Sat, Oct 27, 2012 at 7:28 PM, Liu Bo  wrote:
> >>> This comes from one of btrfs's project ideas,
> >>> As we defragment files, we break any sharing from other snapshots.
> >>> The balancing code will preserve the sharing, and defrag needs to grow 
> >>> this
> >>> as well.
> >>>
> >>> Now we're able to fill the blank with this patch, in which we make full 
> >>> use of
> >>> backref walking stuff.
> >>>
> >>> Here is the basic idea,
> >>> o  set the writeback ranges started by defragment with flag EXTENT_DEFRAG
> >>> o  at endio, after we finish updating fs tree, we use backref walking to 
> >>> find
> >>>all parents of the ranges and re-link them with the new COWed file 
> >>> layout by
> >>>adding corresponding backrefs.
> >>>
> >>> Originally patch by Li Zefan 
> >>> Signed-off-by: Liu Bo 
> >>> ---
> >>> v3->v4:
> >>>   - fix duplicated refs bugs detected by mounting with autodefrag, 
> >>> thanks
> >>> for the bug report from Mitch and Chris.
> >>>
> >>>  fs/btrfs/inode.c |  609 
> >>> ++
> >>>  1 files changed, 609 insertions(+), 0 deletions(-)
> >>>
> >>> diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
> >>> index 85a1e50..35e6993 100644
> >>> --- a/fs/btrfs/inode.c
> >>> +++ b/fs/btrfs/inode.c
> >>> @@ -54,6 +54,7 @@
> >>>  #include "locking.h"
> >>>  #include "free-space-cache.h"
> >>>  #include "inode-map.h"
> >>> +#include "backref.h"
> >>>
> >>>  struct btrfs_iget_args {
> >>> u64 ino;
> >>> @@ -1839,6 +1840,600 @@ out:
> >>> return ret;
> >>>  }
> >>>
> >>> +/* snapshot-aware defrag */
> >>> +struct sa_defrag_extent_backref {
> >>> +   struct rb_node node;
> >>> +   struct old_sa_defrag_extent *old;
> >>> +   u64 root_id;
> >>> +   u64 inum;
> >>> +   u64 file_pos;
> >>> +   u64 extent_offset;
> >>> +   u64 num_bytes;
> >>> +   u64 generation;
> >>> +};
> >>> +
> >>> +struct old_sa_defrag_extent {
> >>> +   struct list_head list;
> >>> +   struct new_sa_defrag_extent *new;
> >>> +
> >>> +   u64 extent_offset;
> >>> +   u64 bytenr;
> >>> +   u64 offset;
> >>> +   u64 len;
> >>> +   int count;
> >>> +};
> >>> +
> >>> +struct new_sa_defrag_extent {
> >>> +   struct rb_root root;
> >>> +   struct list_head head;
> >>> +   struct btrfs_path *path;
> >>> +   struct inode *inode;
> >>> +   u64 file_pos;
> >>> +   u64 len;
> >>> +   u64 bytenr;
> >>> +   u64 disk_len;
> >>> +   u8 compress_type;
> >>> +};
> >>> +
> >>> +static int backref_comp(s

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-11-01 Thread Itaru Kitayama 
Hi Liubo:

The V4 leaves only warnings from btrfs_destroy_inode(). So, you think
it's normal
an "old" extent recorded can be removed from the extent tree by the time
relink_file_extents() invoked?

Itaru

On Thu, Nov 1, 2012 at 8:21 PM, Liu Bo  wrote:

> The current btrfs-next HEAD actually have included this v4 patch, so
> just pull btrfs-next and give it a shot :)
>
> thanks,
> liubo
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-11-01 Thread Liu Bo 
On 11/01/2012 10:05 PM, Itaru Kitayama wrote:
> Hi Liubo:
> 
> The V4 leaves only warnings from btrfs_destroy_inode(). So, you think
> it's normal
> an "old" extent recorded can be removed from the extent tree by the time
> relink_file_extents() invoked?
> 

Yeah, it could be if only we run delayed refs in time.

But I don't think that often happens since we run delayed refs when the amount
reaches its limit(64).

thanks,
liubo

> Itaru
> 
> On Thu, Nov 1, 2012 at 8:21 PM, Liu Bo  wrote:
> 
>> The current btrfs-next HEAD actually have included this v4 patch, so
>> just pull btrfs-next and give it a shot :)
>>
>> thanks,
>> liubo
> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-11-26 Thread Liu Bo 
On Sun, Nov 25, 2012 at 12:48:33PM +0900, Itaru Kitayama wrote:
> Hi Liubo:
> 
> At the relinking stage, in some situations, looking up the extent tree with
> the extent logical start objectid key from one of the old extents fails,
> but the found previous item's objectid + offset is exactly the original
> search objectid. As you say, usually old extents are retrieved from the
> extent tree without a problem, but if the Delayed or some other mechanisms
> change extents layout in the finish ordered IO function, we need to take
> care of those rejected extents, not simply returning ENOENT. What do you
> think?

Hi Itaru san,

Thanks for the suggestion.

It's all right to ignore ENOENT.

An extent stands for a range of space, which owns an item in extent tree.

If old extents' item is not found in the extent tree, i.e. rejected extents,
it means that this range of space is not shared by any snapshots(the extent 
refs is now 0),
and so it's safe.

So the worst case is that due to COW, a file and its snapshots share nothing
after some time, the defrag on the file won't change its snapshots, but this
can be solved when we always with autodefrag and snapshot-aware defrag.

thanks,
liubo
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-12-12 Thread Mitch Harder 
On Thu, Nov 1, 2012 at 6:21 AM, Liu Bo  wrote:
> On Thu, Nov 01, 2012 at 08:08:52PM +0900, Itaru Kitayama wrote:
>> Hi Liubo,
>>
>> I couldn't apply your V4 patch against the btrfs-next HEAD. Do you have
>> a github branch which I can checkout?
>>
>
> The current btrfs-next HEAD actually have included this v4 patch, so
> just pull btrfs-next and give it a shot :)
>

I'm still seeing similar issues using Josef's current btrfs-next
branch (which still includes the v4 version of the snapshot-aware
defrag patches).

[44507.850693] [ cut here ]
[44507.850728] WARNING: at fs/btrfs/inode.c:7755
btrfs_destroy_inode+0x231/0x2c4 [btrfs]()
[44507.850732] Hardware name: OptiPlex 745
[44507.850735] Modules linked in: iTCO_wdt iTCO_vendor_support lpc_ich
mfd_core lrw xts gf128mul ablk_helper cryptd aes_x86_64 sha256_generic
btrfs libcrc32c
[44507.850753] Pid: 15719, comm: umount Tainted: GW
3.7.0-btrfs-next+ #1
[44507.850756] Call Trace:
[44507.850766]  [] warn_slowpath_common+0x74/0xa2
[44507.850770]  [] warn_slowpath_null+0x1a/0x1c
[44507.850787]  [] btrfs_destroy_inode+0x231/0x2c4 [btrfs]
[44507.850793]  [] destroy_inode+0x3c/0x5f
[44507.850797]  [] evict+0x122/0x1ac
[44507.850800]  [] iput+0xed/0x169
[44507.850816]  [] btrfs_run_delayed_iputs+0xd6/0xf6 [btrfs]
[44507.850831]  [] btrfs_commit_super+0x2c/0xfd [btrfs]
[44507.850845]  [] close_ctree+0x2c1/0x300 [btrfs]
[44507.850850]  [] ? evict_inodes+0x106/0x115
[44507.850861]  [] btrfs_put_super+0x19/0x1b [btrfs]
[44507.850866]  [] generic_shutdown_super+0x5b/0xdc
[44507.850869]  [] kill_anon_super+0x16/0x24
[44507.850880]  [] btrfs_kill_super+0x1a/0x8f [btrfs]
[44507.850884]  [] deactivate_locked_super+0x33/0x6c
[44507.850887]  [] deactivate_super+0x4e/0x66
[44507.850892]  [] mntput_no_expire+0xf7/0x14d
[44507.850896]  [] sys_umount+0x63/0x37a
[44507.850901]  [] system_call_fastpath+0x16/0x1b
[44507.850905] ---[ end trace ba14fbf3de68a237 ]---
[44507.850907] [ cut here ]
[44507.850924] WARNING: at fs/btrfs/inode.c:7756
btrfs_destroy_inode+0x2b9/0x2c4 [btrfs]()
[44507.850927] Hardware name: OptiPlex 745
[44507.850930] Modules linked in: iTCO_wdt iTCO_vendor_support lpc_ich
mfd_core lrw xts gf128mul ablk_helper cryptd aes_x86_64 sha256_generic
btrfs libcrc32c
[44507.850947] Pid: 15719, comm: umount Tainted: GW
3.7.0-btrfs-next+ #1
[44507.850949] Call Trace:
[44507.850956]  [] warn_slowpath_common+0x74/0xa2
[44507.850961]  [] warn_slowpath_null+0x1a/0x1c
[44507.850978]  [] btrfs_destroy_inode+0x2b9/0x2c4 [btrfs]
[44507.850982]  [] destroy_inode+0x3c/0x5f
[44507.850986]  [] evict+0x122/0x1ac
[44507.850990]  [] iput+0xed/0x169
[44507.851003]  [] btrfs_run_delayed_iputs+0xd6/0xf6 [btrfs]
[44507.851033]  [] btrfs_commit_super+0x2c/0xfd [btrfs]
[44507.851048]  [] close_ctree+0x2c1/0x300 [btrfs]
[44507.851052]  [] ? evict_inodes+0x106/0x115
[44507.851063]  [] btrfs_put_super+0x19/0x1b [btrfs]
[44507.851066]  [] generic_shutdown_super+0x5b/0xdc
[44507.851070]  [] kill_anon_super+0x16/0x24
[44507.851080]  [] btrfs_kill_super+0x1a/0x8f [btrfs]
[44507.851084]  [] deactivate_locked_super+0x33/0x6c
[44507.851087]  [] deactivate_super+0x4e/0x66
[44507.851091]  [] mntput_no_expire+0xf7/0x14d
[44507.851095]  [] sys_umount+0x63/0x37a
[44507.851099]  [] system_call_fastpath+0x16/0x1b
[44507.851101] ---[ end trace ba14fbf3de68a238 ]---
[44507.851104] [ cut here ]
[44507.851121] WARNING: at fs/btrfs/inode.c:7758
btrfs_destroy_inode+0x28d/0x2c4 [btrfs]()
[44507.851123] Hardware name: OptiPlex 745
[44507.851124] Modules linked in: iTCO_wdt iTCO_vendor_support lpc_ich
mfd_core lrw xts gf128mul ablk_helper cryptd aes_x86_64 sha256_generic
btrfs libcrc32c
[44507.851140] Pid: 15719, comm: umount Tainted: GW
3.7.0-btrfs-next+ #1
[44507.851142] Call Trace:
[44507.851148]  [] warn_slowpath_common+0x74/0xa2
[44507.851152]  [] warn_slowpath_null+0x1a/0x1c
[44507.851168]  [] btrfs_destroy_inode+0x28d/0x2c4 [btrfs]
[44507.851172]  [] destroy_inode+0x3c/0x5f
[44507.851176]  [] evict+0x122/0x1ac
[44507.851180]  [] iput+0xed/0x169
[44507.851195]  [] btrfs_run_delayed_iputs+0xd6/0xf6 [btrfs]
[44507.851209]  [] btrfs_commit_super+0x2c/0xfd [btrfs]
[44507.851223]  [] close_ctree+0x2c1/0x300 [btrfs]
[44507.851227]  [] ? evict_inodes+0x106/0x115
[44507.851237]  [] btrfs_put_super+0x19/0x1b [btrfs]
[44507.851241]  [] generic_shutdown_super+0x5b/0xdc
[44507.851245]  [] kill_anon_super+0x16/0x24
[44507.851255]  [] btrfs_kill_super+0x1a/0x8f [btrfs]
[44507.851259]  [] deactivate_locked_super+0x33/0x6c
[44507.851263]  [] deactivate_super+0x4e/0x66
[44507.851266]  [] mntput_no_expire+0xf7/0x14d
[44507.851270]  [] sys_umount+0x63/0x37a
[44507.851274]  [] system_call_fastpath+0x16/0x1b
[44507.851277] ---[ end trace ba14fbf3de68a239 ]---
[44507.969039] [ cut here ]
[44507.969069] WARNING: at fs/btrfs/extent-tree.c:4351
btrfs_free_block_groups+0x2ad/0x34c [btrfs]()
[44507.969074] Hardware name: OptiP

Re: [PATCH 1/2 v4] Btrfs: snapshot-aware defrag

2012-12-12 Thread Liu Bo 
On Wed, Dec 12, 2012 at 01:37:21PM -0600, Mitch Harder wrote:
> On Thu, Nov 1, 2012 at 6:21 AM, Liu Bo  wrote:
> > On Thu, Nov 01, 2012 at 08:08:52PM +0900, Itaru Kitayama wrote:
> >> Hi Liubo,
> >>
> >> I couldn't apply your V4 patch against the btrfs-next HEAD. Do you have
> >> a github branch which I can checkout?
> >>
> >
> > The current btrfs-next HEAD actually have included this v4 patch, so
> > just pull btrfs-next and give it a shot :)
> >
> 
> I'm still seeing similar issues using Josef's current btrfs-next
> branch (which still includes the v4 version of the snapshot-aware
> defrag patches).

Well, it seems that there is something wrong about delalloc reservation.

thanks,
liubo

> 
> [44507.850693] [ cut here ]
> [44507.850728] WARNING: at fs/btrfs/inode.c:7755
> btrfs_destroy_inode+0x231/0x2c4 [btrfs]()
> [44507.850732] Hardware name: OptiPlex 745
> [44507.850735] Modules linked in: iTCO_wdt iTCO_vendor_support lpc_ich
> mfd_core lrw xts gf128mul ablk_helper cryptd aes_x86_64 sha256_generic
> btrfs libcrc32c
> [44507.850753] Pid: 15719, comm: umount Tainted: GW
> 3.7.0-btrfs-next+ #1
> [44507.850756] Call Trace:
> [44507.850766]  [] warn_slowpath_common+0x74/0xa2
> [44507.850770]  [] warn_slowpath_null+0x1a/0x1c
> [44507.850787]  [] btrfs_destroy_inode+0x231/0x2c4 [btrfs]
> [44507.850793]  [] destroy_inode+0x3c/0x5f
> [44507.850797]  [] evict+0x122/0x1ac
> [44507.850800]  [] iput+0xed/0x169
> [44507.850816]  [] btrfs_run_delayed_iputs+0xd6/0xf6 [btrfs]
> [44507.850831]  [] btrfs_commit_super+0x2c/0xfd [btrfs]
> [44507.850845]  [] close_ctree+0x2c1/0x300 [btrfs]
> [44507.850850]  [] ? evict_inodes+0x106/0x115
> [44507.850861]  [] btrfs_put_super+0x19/0x1b [btrfs]
> [44507.850866]  [] generic_shutdown_super+0x5b/0xdc
> [44507.850869]  [] kill_anon_super+0x16/0x24
> [44507.850880]  [] btrfs_kill_super+0x1a/0x8f [btrfs]
> [44507.850884]  [] deactivate_locked_super+0x33/0x6c
> [44507.850887]  [] deactivate_super+0x4e/0x66
> [44507.850892]  [] mntput_no_expire+0xf7/0x14d
> [44507.850896]  [] sys_umount+0x63/0x37a
> [44507.850901]  [] system_call_fastpath+0x16/0x1b
> [44507.850905] ---[ end trace ba14fbf3de68a237 ]---
> [44507.850907] [ cut here ]
> [44507.850924] WARNING: at fs/btrfs/inode.c:7756
> btrfs_destroy_inode+0x2b9/0x2c4 [btrfs]()
> [44507.850927] Hardware name: OptiPlex 745
> [44507.850930] Modules linked in: iTCO_wdt iTCO_vendor_support lpc_ich
> mfd_core lrw xts gf128mul ablk_helper cryptd aes_x86_64 sha256_generic
> btrfs libcrc32c
> [44507.850947] Pid: 15719, comm: umount Tainted: GW
> 3.7.0-btrfs-next+ #1
> [44507.850949] Call Trace:
> [44507.850956]  [] warn_slowpath_common+0x74/0xa2
> [44507.850961]  [] warn_slowpath_null+0x1a/0x1c
> [44507.850978]  [] btrfs_destroy_inode+0x2b9/0x2c4 [btrfs]
> [44507.850982]  [] destroy_inode+0x3c/0x5f
> [44507.850986]  [] evict+0x122/0x1ac
> [44507.850990]  [] iput+0xed/0x169
> [44507.851003]  [] btrfs_run_delayed_iputs+0xd6/0xf6 [btrfs]
> [44507.851033]  [] btrfs_commit_super+0x2c/0xfd [btrfs]
> [44507.851048]  [] close_ctree+0x2c1/0x300 [btrfs]
> [44507.851052]  [] ? evict_inodes+0x106/0x115
> [44507.851063]  [] btrfs_put_super+0x19/0x1b [btrfs]
> [44507.851066]  [] generic_shutdown_super+0x5b/0xdc
> [44507.851070]  [] kill_anon_super+0x16/0x24
> [44507.851080]  [] btrfs_kill_super+0x1a/0x8f [btrfs]
> [44507.851084]  [] deactivate_locked_super+0x33/0x6c
> [44507.851087]  [] deactivate_super+0x4e/0x66
> [44507.851091]  [] mntput_no_expire+0xf7/0x14d
> [44507.851095]  [] sys_umount+0x63/0x37a
> [44507.851099]  [] system_call_fastpath+0x16/0x1b
> [44507.851101] ---[ end trace ba14fbf3de68a238 ]---
> [44507.851104] [ cut here ]
> [44507.851121] WARNING: at fs/btrfs/inode.c:7758
> btrfs_destroy_inode+0x28d/0x2c4 [btrfs]()
> [44507.851123] Hardware name: OptiPlex 745
> [44507.851124] Modules linked in: iTCO_wdt iTCO_vendor_support lpc_ich
> mfd_core lrw xts gf128mul ablk_helper cryptd aes_x86_64 sha256_generic
> btrfs libcrc32c
> [44507.851140] Pid: 15719, comm: umount Tainted: GW
> 3.7.0-btrfs-next+ #1
> [44507.851142] Call Trace:
> [44507.851148]  [] warn_slowpath_common+0x74/0xa2
> [44507.851152]  [] warn_slowpath_null+0x1a/0x1c
> [44507.851168]  [] btrfs_destroy_inode+0x28d/0x2c4 [btrfs]
> [44507.851172]  [] destroy_inode+0x3c/0x5f
> [44507.851176]  [] evict+0x122/0x1ac
> [44507.851180]  [] iput+0xed/0x169
> [44507.851195]  [] btrfs_run_delayed_iputs+0xd6/0xf6 [btrfs]
> [44507.851209]  [] btrfs_commit_super+0x2c/0xfd [btrfs]
> [44507.851223]  [] close_ctree+0x2c1/0x300 [btrfs]
> [44507.851227]  [] ? evict_inodes+0x106/0x115
> [44507.851237]  [] btrfs_put_super+0x19/0x1b [btrfs]
> [44507.851241]  [] generic_shutdown_super+0x5b/0xdc
> [44507.851245]  [] kill_anon_super+0x16/0x24
> [44507.851255]  [] btrfs_kill_super+0x1a/0x8f [btrfs]
> [44507.851259]  [] deactivate_locked_super+0x33/0x6c
> [44507.851263]  [] deactivate_super+0x4e/0x66
> [44507.851266]  [] m