[PATCH v2 6/6] btrfs-progs: add chattr support for send/receive

[Howard McLauchlan]

From: Howard McLauchlan <hmclauch...@fb.com>

Presently, btrfs send/receive does not propagate inode attribute flags;
all chattr operations are effectively discarded upon transmission.

This patch adds userspace support for inode attribute flags. Kernel
support can be found under the commit:

btrfs: add chattr support for send/receive

An associated xfstest can also be found at:

btrfs: verify chattr support for send/receive test

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 cmds-receive.c | 37 +
 send-dump.c|  6 ++
 send-stream.c  |  5 +
 send-stream.h  |  1 +
 4 files changed, 49 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index 20e593f7..2a841bfc 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -40,6 +40,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -1201,6 +1202,41 @@ out:
return ret;
 }
 
+static int process_chattr(const char *path, u64 flags, void *user)
+{
+   int ret = 0;
+   int fd = 0;
+   int _flags = flags;
+   struct btrfs_receive *rctx = user;
+   char full_path[PATH_MAX];
+
+   ret = path_cat_out(full_path, rctx->full_subvol_path, path);
+   if (ret < 0) {
+   error("chattr: path invalid: %s", path);
+   goto out;
+   }
+
+   if (g_verbose >= 2)
+   fprintf(stderr, "chattr %s - flags=0%o\n", path, (int)flags);
+
+   fd = open(full_path, O_RDONLY);
+   if (fd < 0) {
+   ret = -errno;
+   error("cannot open %s: %s", path, strerror(-ret));
+   goto out;
+   }
+
+   ret = ioctl(fd, FS_IOC_SETFLAGS, &_flags);
+   if (ret < 0) {
+   ret = -errno;
+   error("chattr %s failed: %s", path, strerror(-ret));
+   goto out;
+   }
+
+out:
+   return ret;
+}
+
 static struct btrfs_send_ops send_ops = {
.subvol = process_subvol,
.snapshot = process_snapshot,
@@ -1225,6 +1261,7 @@ static struct btrfs_send_ops send_ops = {
.update_extent = process_update_extent,
.total_data_size = process_total_data_size,
.fallocate = process_fallocate,
+   .chattr = process_chattr,
 };
 
 static int do_receive(struct btrfs_receive *rctx, const char *tomnt,
diff --git a/send-dump.c b/send-dump.c
index c5a695a2..15aea402 100644
--- a/send-dump.c
+++ b/send-dump.c
@@ -331,6 +331,11 @@ static int print_fallocate(const char *path, u32 flags, 
u64 offset, u64 len,
  len);
 }
 
+static int print_chattr(const char *path, u64 flags, void *user)
+{
+   return PRINT_DUMP(user, path, "chattr", "flags=%llu", flags);
+}
+
 struct btrfs_send_ops btrfs_print_send_ops = {
.subvol = print_subvol,
.snapshot = print_snapshot,
@@ -355,4 +360,5 @@ struct btrfs_send_ops btrfs_print_send_ops = {
.update_extent = print_update_extent,
.total_data_size = print_total_data_size,
.fallocate = print_fallocate,
+   .chattr = print_chattr,
 };
diff --git a/send-stream.c b/send-stream.c
index 74ec37dd..4f26fae3 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -470,6 +470,11 @@ static int read_and_process_cmd(struct btrfs_send_stream 
*sctx)
sctx->user);
}
break;
+   case BTRFS_SEND_C_CHATTR:
+   TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, );
+   TLV_GET_U64(sctx, BTRFS_SEND_A_CHATTR, );
+   ret = sctx->ops->chattr(path, tmp, sctx->user);
+   break;
case BTRFS_SEND_C_END:
ret = 1;
break;
diff --git a/send-stream.h b/send-stream.h
index 89e64043..a9f08d52 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -69,6 +69,7 @@ struct btrfs_send_ops {
int (*total_data_size)(u64 size, void *user);
int (*fallocate)(const char *path, u32 flags, u64 offset,
 u64 len, void *user);
+   int (*chattr)(const char *path, u64 flags, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2 1/6] Btrfs-progs: send, bump stream version

[Howard McLauchlan]

From: Filipe Manana 

This increases the send stream version from version 1 to version 2, adding
new commands:

1) total data size - used to tell the receiver how much file data the stream
   will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files;

3) inode set flags (chattr);

4) set inode otime;

5) sending compressed writes

This is preparation work for subsequent changes that implement the new features.

This doesn't break compatibility with older kernels or clients. In order to get
a version 2 send stream, new flags must be passed to the send ioctl.

Signed-off-by: Filipe David Borba Manana 
---
 cmds-send.c   | 61 ---
 ioctl.h   | 15 +
 send-stream.c |  2 +-
 send.h| 24 +++-
 4 files changed, 82 insertions(+), 20 deletions(-)

diff --git a/cmds-send.c b/cmds-send.c
index c5ecdaa1..0ec557c7 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -52,6 +52,7 @@
  * the 'At subvol' message.
  */
 static int g_verbose = 1;
+static int g_stream_version = BTRFS_SEND_STREAM_VERSION_1;
 
 struct btrfs_send {
int send_fd;
@@ -343,6 +344,8 @@ static int do_send(struct btrfs_send *send, u64 
parent_root_id,
io_send.flags |= BTRFS_SEND_FLAG_OMIT_STREAM_HEADER;
if (!is_last_subvol)
io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
+   if (g_stream_version == BTRFS_SEND_STREAM_VERSION_2)
+   io_send.flags |= BTRFS_SEND_FLAG_STREAM_V2;
ret = ioctl(subvol_fd, BTRFS_IOC_SEND, _send);
if (ret < 0) {
ret = -errno;
@@ -513,7 +516,8 @@ int cmd_send(int argc, char **argv)
static const struct option long_options[] = {
{ "verbose", no_argument, NULL, 'v' },
{ "quiet", no_argument, NULL, 'q' },
-   { "no-data", no_argument, NULL, GETOPT_VAL_SEND_NO_DATA 
}
+   { "no-data", no_argument, NULL, GETOPT_VAL_SEND_NO_DATA 
},
+   { "stream-version", 1, NULL, 'V' },
};
int c = getopt_long(argc, argv, "vqec:f:i:p:", long_options, 
NULL);
 
@@ -597,6 +601,24 @@ int cmd_send(int argc, char **argv)
error("option -i was removed, use -c instead");
ret = 1;
goto out;
+   case 'V':
+   if (sscanf(optarg, "%d", _stream_version) != 1) {
+   fprintf(stderr,
+   "ERROR: invalid value for stream 
version: %s\n",
+   optarg);
+   ret = 1;
+   goto out;
+   }
+   if (g_stream_version <= 0 ||
+   g_stream_version > BTRFS_SEND_STREAM_VERSION_MAX) {
+   fprintf(stderr,
+   "ERROR: unsupported stream version %d, 
minimum: 1, maximum: %d\n",
+   g_stream_version,
+   BTRFS_SEND_STREAM_VERSION_MAX);
+   ret = 1;
+   goto out;
+   }
+   break;
case GETOPT_VAL_SEND_NO_DATA:
send_flags |= BTRFS_SEND_FLAG_NO_FILE_DATA;
break;
@@ -776,7 +798,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-   "btrfs send [-ve] [-p ] [-c ] [-f ] 
 [...]",
+   "btrfs send [-ve] [--stream-version ] [-p ] [-c 
] [-f ]  [...]",
"Send the subvolume(s) to stdout.",
"Sends the subvolume(s) specified by  to stdout.",
" should be read-only here.",
@@ -790,21 +812,24 @@ const char * const cmd_send_usage[] = {
"which case 'btrfs send' will determine a suitable parent among the",
"clone sources itself.",
"\n",
-   "-e   If sending multiple subvols at once, use the new",
-   " format and omit the end-cmd between the subvols.",
-   "-p   Send an incremental stream from  to",
-   " .",
-   "-cUse this snapshot as a clone source for an ",
-   " incremental send (multiple allowed)",
-   "-f  Output is normally written to stdout. To write to",
-   " a file, use this option. An alternative would be to",
-   " use pipes.",
-   "--no-datasend in NO_FILE_DATA mode, Note: the output stream",
-   " does not contain any file data and thus cannot be 
used",
-   " to transfer changes. This mode is faster and useful 
to",
-   " show the differences in metadata.",
-   "-v|--verbose enable verbose output to 

[PATCH v2 4/6] Btrfs-progs: add write and clone commands debug info to receive

[Howard McLauchlan]

From: Filipe Manana 

When specifying -vv print information about received write and clone commands 
too,
as we do this for other commands already and it's very useful for debugging and
troubleshooting.

Signed-off-by: Filipe David Borba Manana 
---
 cmds-receive.c | 9 +
 1 file changed, 9 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index 510b6bc8..20e593f7 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -790,6 +790,10 @@ static int process_write(const char *path, const void 
*data, u64 offset,
u64 pos = 0;
int w;
 
+   if (g_verbose >= 2)
+   fprintf(stderr, "write %s, offset %llu, len %llu\n",
+   path, offset, len);
+
ret = path_cat_out(full_path, rctx->full_subvol_path, path);
if (ret < 0) {
error("write: path invalid: %s", path);
@@ -831,6 +835,11 @@ static int process_clone(const char *path, u64 offset, u64 
len,
char full_clone_path[PATH_MAX];
int clone_fd = -1;
 
+   if (g_verbose >= 2)
+   fprintf(stderr,
+   "clone %s, offset %llu, len %llu, clone path %s, clone 
offset %llu\n",
+   path, offset, len, clone_path, clone_offset);
+
ret = path_cat_out(full_path, rctx->full_subvol_path, path);
if (ret < 0) {
error("clone: source path invalid: %s", path);
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH v2 3/6] btrfs: send, use fallocate command to punch holes

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

Instead of sending a write command with a data buffer filled with 0 value bytes,
use the fallocate command, introduced in the send stream version 2, to tell the
receiver to punch a file hole using the fallocate system call.

[Howard: rebased on 4.17-rc7]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
---
 fs/btrfs/send.c | 54 ++---
 fs/btrfs/send.h |  4 
 2 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index 7b184831812b..328c7a2857ae 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -581,6 +581,7 @@ static int tlv_put(struct send_ctx *sctx, u16 attr, const 
void *data, int len)
return tlv_put(sctx, attr, &__tmp, sizeof(__tmp));  \
}
 
+TLV_PUT_DEFINE_INT(32)
 TLV_PUT_DEFINE_INT(64)
 
 static int tlv_put_string(struct send_ctx *sctx, u16 attr,
@@ -5047,17 +5048,57 @@ static int send_update_extent(struct send_ctx *sctx,
return ret;
 }
 
+static int send_fallocate(struct send_ctx *sctx, u32 flags,
+ u64 offset, u64 len)
+{
+   struct fs_path *p = NULL;
+   int ret = 0;
+
+   ASSERT(sctx->flags & BTRFS_SEND_FLAG_STREAM_V2);
+
+   if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+   sctx->total_data_size += len;
+   return 0;
+   }
+
+   p = fs_path_alloc();
+   if (!p)
+   return -ENOMEM;
+   ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
+   if (ret < 0)
+   goto out;
+
+   ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE);
+   if (ret < 0)
+   goto out;
+   TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
+   TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_FLAGS, flags);
+   TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
+   TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
+   ret = send_cmd(sctx);
+
+tlv_put_failure:
+out:
+   fs_path_free(p);
+   return ret;
+}
+
+
 static int send_hole(struct send_ctx *sctx, u64 end)
 {
struct fs_path *p = NULL;
u64 offset = sctx->cur_inode_last_extent;
-   u64 len;
+   u64 len = end - offset;
int ret = 0;
 
if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-   sctx->total_data_size += end - offset;
+   sctx->total_data_size += len;
return 0;
}
+   if (sctx->flags & BTRFS_SEND_FLAG_STREAM_V2)
+   return send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+ offset, len);
+
if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
return send_update_extent(sctx, offset, end - offset);
 
@@ -5304,7 +5345,8 @@ static int send_write_or_clone(struct send_ctx *sctx,
ret = 0;
goto out;
}
-   if (offset + len > sctx->cur_inode_size)
+   if (offset < sctx->cur_inode_size &&
+   offset + len > sctx->cur_inode_size)
len = sctx->cur_inode_size - offset;
if (len == 0) {
ret = 0;
@@ -5325,6 +5367,12 @@ static int send_write_or_clone(struct send_ctx *sctx,
data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
ret = clone_range(sctx, clone_root, disk_byte, data_offset,
  offset, len);
+   } else if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0 &&
+type != BTRFS_FILE_EXTENT_INLINE &&
+(sctx->flags & BTRFS_SEND_FLAG_STREAM_V2) &&
+offset < sctx->cur_inode_size) {
+   ret = send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+offset, len);
} else {
ret = send_extent_data(sctx, offset, len);
}
diff --git a/fs/btrfs/send.h b/fs/btrfs/send.h
index 152180304078..b6e9281f171a 100644
--- a/fs/btrfs/send.h
+++ b/fs/btrfs/send.h
@@ -139,6 +139,10 @@ enum {
 #define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
 #define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
 
+#define BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS\
+   (BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE |  \
+BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args 
*arg);
 #endif
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH v2 4/6] btrfs: send, use fallocate command to allocate extents

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

The send stream version 2 adds the fallocate command, which can be used to
allocate extents for a file or punch holes in a file. Previously we were
ignoring file prealloc extents or treating them as extents filled with 0
bytes and sending a regular write command to the stream.

After this change, together with my previous change titled:

"Btrfs: send, use fallocate command to punch holes"

an incremental send preserves the hole and data structure of files, which can
be seen via calls to lseek with the whence parameter set to SEEK_DATA or 
SEEK_HOLE,
as the example below shows:

mkfs.btrfs -f /dev/sdc
mount /dev/sdc /mnt
xfs_io -f -c "pwrite -S 0x01 -b 30 0 30" /mnt/foo
btrfs subvolume snapshot -r /mnt /mnt/mysnap1

xfs_io -c "fpunch 10 5" /mnt/foo
xfs_io -c "falloc 10 5" /mnt/foo
xfs_io -c "pwrite -S 0xff -b 1000 12 1000" /mnt/foo
xfs_io -c "fpunch 25 2" /mnt/foo

# prealloc extents that start beyond the inode's size
xfs_io -c "falloc -k 30 100" /mnt/foo
xfs_io -c "falloc -k 900 200" /mnt/foo

btrfs subvolume snapshot -r /mnt /mnt/mysnap2

btrfs send /mnt/mysnap1 -f /tmp/1.snap
btrfs send -p /mnt/mysnap1 /mnt/mysnap2 -f /tmp/2.snap

mkfs.btrfs -f /dev/sdd
mount /dev/sdd /mnt2
btrfs receive /mnt2 -f /tmp/1.snap
btrfs receive /mnt2 -f /tmp/2.snap

Before this change the hole/data structure differed between both filesystems:

$ xfs_io -r -c 'seek -r -a 0' /mnt/mysnap2/foo
Whence  Result
DATA0
HOLE102400
DATA118784
HOLE122880
DATA147456
HOLE253952
DATA266240
HOLE30

$ xfs_io -r -c 'seek -r -a 0' /mnt2/mysnap2/foo
Whence  Result
DATA0
HOLE30

After this change the second filesystem (/dev/sdd) ends up with the same 
hole/data
structure as the first filesystem.

Also, after this change, prealloc extents that lie beyond the inode's size (were
allocated with fallocate + keep size flag) are also replicated by an incremental
send. For the above test, it can be observed via fiemap (or btrfs-debug-tree):

$ xfs_io -r -c 'fiemap -l' /mnt2/mysnap2/foo
0: [0..191]: 25096..25287 192 blocks
1: [192..199]: 24672..24679 8 blocks
2: [200..231]: 24584..24615 32 blocks
3: [232..239]: 24680..24687 8 blocks
4: [240..287]: 24616..24663 48 blocks
5: [288..295]: 24688..24695 8 blocks
6: [296..487]: 25392..25583 192 blocks
7: [488..495]: 24696..24703 8 blocks
8: [496..519]: hole 24 blocks
9: [520..527]: 24704..24711 8 blocks
10: [528..583]: 25624..25679 56 blocks
11: [584..591]: 24712..24719 8 blocks
12: [592..2543]: 26192..28143 1952 blocks
13: [2544..17575]: hole 15032 blocks
14: [17576..21487]: 28144..32055 3912 blocks

The proposed xfstest can be found at:

xfstests: btrfs, test send's ability to punch holes and prealloc extents

This test verifies that send-stream version 2 does space pre-allocation
and hole punching.

[Howard: rebased on 4.17-rc7]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
---
 fs/btrfs/send.c | 70 -
 1 file changed, 52 insertions(+), 18 deletions(-)

diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index 328c7a2857ae..84dacb20d832 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -98,9 +98,10 @@ struct send_ctx {
 */
u64 cur_ino;
u64 cur_inode_gen;
-   int cur_inode_new;
-   int cur_inode_new_gen;
-   int cur_inode_deleted;
+   u8 cur_inode_new:1;
+   u8 cur_inode_new_gen:1;
+   u8 cur_inode_skip_truncate:1;
+   u8 cur_inode_deleted:1;
u64 cur_inode_size;
u64 cur_inode_mode;
u64 cur_inode_rdev;
@@ -5313,6 +5314,19 @@ static int clone_range(struct send_ctx *sctx,
return ret;
 }
 
+static int truncate_before_falloc(struct send_ctx *sctx)
+{
+   int ret = 0;
+
+   if (!sctx->cur_inode_skip_truncate) {
+   ret = send_truncate(sctx, sctx->cur_ino,
+   sctx->cur_inode_gen,
+   sctx->cur_inode_size);
+   sctx->cur_inode_skip_truncate = 1;
+   }
+   return ret;
+}
+
 static int send_write_or_clone(struct send_ctx *sctx,
   struct btrfs_path *path,
   struct btrfs_key *key,
@@ -5354,8 +5368,7 @@ static int send_write_or_clone(struct send_ctx *sctx,
}
 
if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-   if (offset < sctx->cur_inode_size)
-   sctx->total_data_size 

[PATCH v2 5/6] btrfs-progs: add total data size, fallocate to dump

[Howard McLauchlan]

From: Howard McLauchlan <hmclauch...@fb.com>

Adding entries to dump for new commands (total data size, fallocate).

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 send-dump.c | 19 ++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/send-dump.c b/send-dump.c
index 1591e0cc..c5a695a2 100644
--- a/send-dump.c
+++ b/send-dump.c
@@ -316,6 +316,21 @@ static int print_update_extent(const char *path, u64 
offset, u64 len,
  offset, len);
 }
 
+static int print_total_data_size(u64 size, void *user)
+{
+   char path;
+
+   return PRINT_DUMP(user, , "total_data_size", "size=%llu", size);
+}
+
+static int print_fallocate(const char *path, u32 flags, u64 offset, u64 len,
+  void *user)
+{
+   return PRINT_DUMP(user, path, "fallocate",
+ "flags=%u offset=%llu len=%llu", flags, offset,
+ len);
+}
+
 struct btrfs_send_ops btrfs_print_send_ops = {
.subvol = print_subvol,
.snapshot = print_snapshot,
@@ -337,5 +352,7 @@ struct btrfs_send_ops btrfs_print_send_ops = {
.chmod = print_chmod,
.chown = print_chown,
.utimes = print_utimes,
-   .update_extent = print_update_extent
+   .update_extent = print_update_extent,
+   .total_data_size = print_total_data_size,
+   .fallocate = print_fallocate,
 };
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH v2 1/6] btrfs: send, bump stream version

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

This increases the send stream version from version 1 to version 2, adding
new commands:

1) total data size - used to tell the receiver how much file data the stream
   will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files;

3) inode set flags (chattr);

4) set inode otime;

5) sending compressed writes.

This is preparation work for subsequent changes that implement the new features.

A version 2 stream is only produced if the send ioctl caller passes in one of 
the
new flags (BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | BTRFS_SEND_FLAG_STREAM_V2), 
meaning
old clients are unaffected.

[Howard: rebased on 4.17-rc7]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
Reviewed-by: Omar Sandoval <osan...@fb.com>
---
 fs/btrfs/send.c|  7 ++-
 fs/btrfs/send.h| 22 +-
 include/uapi/linux/btrfs.h | 21 -
 3 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index c0074d2d7d6d..eccd69387065 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -649,7 +649,10 @@ static int send_header(struct send_ctx *sctx)
struct btrfs_stream_header hdr;
 
strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
-   hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION);
+   if (sctx->flags & BTRFS_SEND_FLAG_STREAM_V2)
+   hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_2);
+   else
+   hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_1);
 
return write_buf(sctx->send_filp, , sizeof(hdr),
>send_off);
@@ -6535,6 +6538,8 @@ long btrfs_ioctl_send(struct file *mnt_file, struct 
btrfs_ioctl_send_args *arg)
INIT_LIST_HEAD(>name_cache_list);
 
sctx->flags = arg->flags;
+   if (sctx->flags & BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE)
+   sctx->flags |= BTRFS_SEND_FLAG_STREAM_V2;
 
sctx->send_filp = fget(arg->send_fd);
if (!sctx->send_filp) {
diff --git a/fs/btrfs/send.h b/fs/btrfs/send.h
index ead397f7034f..152180304078 100644
--- a/fs/btrfs/send.h
+++ b/fs/btrfs/send.h
@@ -10,7 +10,8 @@
 #include "ctree.h"
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION_1 1
+#define BTRFS_SEND_STREAM_VERSION_2 2
 
 #define BTRFS_SEND_BUF_SIZE SZ_64K
 #define BTRFS_SEND_READ_SIZE (48 * SZ_1K)
@@ -77,6 +78,16 @@ enum btrfs_send_cmd {
 
BTRFS_SEND_C_END,
BTRFS_SEND_C_UPDATE_EXTENT,
+
+   /*
+* The following commands were added in stream version 2.
+*/
+   BTRFS_SEND_C_TOTAL_DATA_SIZE,
+   BTRFS_SEND_C_FALLOCATE,
+   BTRFS_SEND_C_CHATTR,
+   BTRFS_SEND_C_UTIMES2, /* Same as UTIMES, but it includes OTIME too. */
+   BTRFS_SEND_C_WRITE_COMPRESSED, /* to be implemented */
+
__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -115,10 +126,19 @@ enum {
BTRFS_SEND_A_CLONE_OFFSET,
BTRFS_SEND_A_CLONE_LEN,
 
+   /*
+* The following attributes were added in stream version 2.
+*/
+   BTRFS_SEND_A_FALLOCATE_FLAGS,
+   BTRFS_SEND_A_CHATTR,
+
__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args 
*arg);
 #endif
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
index c8d99b9ca550..ed63176660d2 100644
--- a/include/uapi/linux/btrfs.h
+++ b/include/uapi/linux/btrfs.h
@@ -711,10 +711,29 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD   0x4
 
+/*
+ * Calculate the amount (in bytes) of new file data between the send and
+ * parent snapshots, or in case of a full send, the total amount of file data
+ * we will send.
+ * This corresponds to the sum of the data lengths of each write, clone and
+ * fallocate commands that are sent through the send stream. The receiving end
+ * can use this information to compute progress.
+ *
+ * Added in send stream version 2, and implies producing a version 2 stream.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE0x8
+
+/*
+ * Used by a client to request a version 2 of the send stream.
+ */
+#define BTRFS_SEND_FLAG_STREAM_V2  0x10
+
 #define BTRFS_SEND_FLAG_MASK \
(BTRFS_SEND_FLAG_NO_FILE_DATA | \
 BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \
-BTRFS_SEND_FLAG_OMIT_END_CMD)
+BTRFS_SEND_FLAG_OMIT_END_CMD | \
+BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | \
+BTRFS_SEND_FLAG_STREAM_

[PATCH v2 2/6] Btrfs-progs: send, implement total data size callback and progress report

[Howard McLauchlan]

From: Filipe Manana 

This is a followup to the kernel patch titled:

Btrfs: send, implement total data size command to allow for progress 
estimation

This makes the btrfs send and receive commands aware of the new send flag,
named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data
that is new between the parent and send snapshots/roots. As this command
immediately follows the commands to start a snapshot/subvolume, it can be
used to report and compute progress, by keeping a counter that is incremented
with the data length of each write, clone and fallocate command that is received
from the stream.

Example:

$ btrfs send -s --stream-version 2 /mnt/sdd/snap_base | btrfs receive 
/mnt/sdc
At subvol /mnt/sdd/snap_base
At subvol snap_base
About to receive 9212392667 bytes
Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 
44.07%, 40.32MB/s

$ btrfs send -s --stream-version 2 -p /mnt/sdd/snap_base /mnt/sdd/snap_incr 
| btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_incr
At subvol snap_incr
About to receive 9571342213 bytes
Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 
68.51%, 51.04MB/s

At the moment progress is only reported by btrfs-receive, but it is possible 
and simple
to do it for btrfs-send too, so that we can get progress report when not piping 
btrfs-send
output to btrfs-receive (directly to a file).

Signed-off-by: Filipe David Borba Manana 
---
 cmds-receive.c | 91 ++
 cmds-send.c| 23 +++--
 send-stream.c  |  4 +++
 send-stream.h  |  1 +
 4 files changed, 117 insertions(+), 2 deletions(-)

diff --git a/cmds-receive.c b/cmds-receive.c
index 68123a31..d8ff5194 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -30,6 +30,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -79,6 +80,14 @@ struct btrfs_receive
 
int honor_end_cmd;
 
+   /* For the subvolume/snapshot we're currently receiving. */
+   u64 total_data_size;
+   u64 bytes_received;
+   time_t last_progress_update;
+   u64 bytes_received_last_update;
+   float progress;
+   const char *target;
+
/*
 * Buffer to store capabilities from security.capabilities xattr,
 * usually 20 bytes, but make same room for potentially larger
@@ -156,6 +165,16 @@ out:
return ret;
 }
 
+static void reset_progress(struct btrfs_receive *rctx, const char *dest)
+{
+   rctx->total_data_size = 0;
+   rctx->bytes_received = 0;
+   rctx->progress = 0.0;
+   rctx->last_progress_update = 0;
+   rctx->bytes_received_last_update = 0;
+   rctx->target = dest;
+}
+
 static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
  void *user)
 {
@@ -180,6 +199,7 @@ static int process_subvol(const char *path, const u8 *uuid, 
u64 ctransid,
ret = -EINVAL;
goto out;
}
+   reset_progress(rctx, "Subvolume");
 
if (*rctx->dest_dir_path == 0) {
strncpy_null(rctx->cur_subvol_path, path);
@@ -249,6 +269,7 @@ static int process_snapshot(const char *path, const u8 
*uuid, u64 ctransid,
ret = -EINVAL;
goto out;
}
+   reset_progress(rctx, "Snapshot");
 
if (*rctx->dest_dir_path == 0) {
strncpy_null(rctx->cur_subvol_path, path);
@@ -388,6 +409,73 @@ out:
return ret;
 }
 
+static int process_total_data_size(u64 size, void *user)
+{
+   struct btrfs_receive *rctx = user;
+
+   rctx->total_data_size = size;
+   fprintf(stdout, "About to receive %llu bytes\n", size);
+
+   return 0;
+}
+
+static void update_progress(struct btrfs_receive *rctx, u64 bytes)
+{
+   float new_progress;
+   time_t now;
+   time_t tdiff;
+
+   if (rctx->total_data_size == 0)
+   return;
+
+   rctx->bytes_received += bytes;
+
+   now = time(NULL);
+   tdiff = now - rctx->last_progress_update;
+   if (tdiff < 1) {
+   if (rctx->bytes_received == rctx->total_data_size)
+   fprintf(stdout, "\n");
+   return;
+   }
+
+   new_progress = ((float)rctx->bytes_received / rctx->total_data_size) * 
100.0;
+
+   if ((int)(new_progress * 100) > (int)(rctx->progress * 100) ||
+   rctx->bytes_received == rctx->total_data_size) {
+   char line[5000];
+   float rate = rctx->bytes_received - 
rctx->bytes_received_last_update;
+   const char *rate_units;
+
+   rate /= tdiff;
+   if (rate > (1024 * 1024)) {
+   rate_units = "MB/s";
+   rate /= 1024 * 1024;
+   } else if (rate > 1024) {
+   rate_units = "KB/s";
+   rate /= 1024;
+   } else {
+ 

[RFC PATCH v2 5/6] btrfs: add send_stream_version attribute to sysfs

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

So that applications can find out what's the highest send stream
version supported/implemented by the running kernel:

$ cat /sys/fs/btrfs/send/stream_version
2

[Howard: rebased on 4.17-rc7]
Reviewed-by: Omar Sandoval <osan...@fb.com>
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
Reviewed-by: David Sterba <dste...@suse.cz>
---
 fs/btrfs/send.h  |  1 +
 fs/btrfs/sysfs.c | 27 +++
 2 files changed, 28 insertions(+)

diff --git a/fs/btrfs/send.h b/fs/btrfs/send.h
index b6e9281f171a..267674d3e954 100644
--- a/fs/btrfs/send.h
+++ b/fs/btrfs/send.h
@@ -12,6 +12,7 @@
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
 #define BTRFS_SEND_STREAM_VERSION_1 1
 #define BTRFS_SEND_STREAM_VERSION_2 2
+#define BTRFS_SEND_STREAM_VERSION_LATEST BTRFS_SEND_STREAM_VERSION_2
 
 #define BTRFS_SEND_BUF_SIZE SZ_64K
 #define BTRFS_SEND_READ_SIZE (48 * SZ_1K)
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 4848a4318fb5..718bc927ea13 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -18,6 +18,7 @@
 #include "transaction.h"
 #include "sysfs.h"
 #include "volumes.h"
+#include "send.h"
 
 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj);
 static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj);
@@ -884,6 +885,26 @@ static int btrfs_init_debugfs(void)
return 0;
 }
 
+static ssize_t send_stream_version_show(struct kobject *kobj,
+   struct kobj_attribute *a,
+   char *buf)
+{
+   return snprintf(buf, PAGE_SIZE, "%d\n",
+   BTRFS_SEND_STREAM_VERSION_LATEST);
+}
+
+BTRFS_ATTR(, stream_version, send_stream_version_show);
+
+static struct attribute *btrfs_send_attrs[] = {
+   BTRFS_ATTR_PTR(, stream_version),
+   NULL
+};
+
+static const struct attribute_group btrfs_send_attr_group = {
+   .name = "send",
+   .attrs = btrfs_send_attrs,
+};
+
 int __init btrfs_init_sysfs(void)
 {
int ret;
@@ -900,8 +921,13 @@ int __init btrfs_init_sysfs(void)
ret = sysfs_create_group(_kset->kobj, _feature_attr_group);
if (ret)
goto out2;
+   ret = sysfs_create_group(_kset->kobj, _send_attr_group);
+   if (ret)
+   goto out3;
 
return 0;
+out3:
+   sysfs_remove_group(_kset->kobj, _feature_attr_group);
 out2:
debugfs_remove_recursive(btrfs_debugfs_root_dentry);
 out1:
@@ -913,6 +939,7 @@ int __init btrfs_init_sysfs(void)
 void __cold btrfs_exit_sysfs(void)
 {
sysfs_remove_group(_kset->kobj, _feature_attr_group);
+   sysfs_remove_group(_kset->kobj, _send_attr_group);
kset_unregister(btrfs_kset);
debugfs_remove_recursive(btrfs_debugfs_root_dentry);
 }
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH v2 6/6] btrfs: add chattr support for send/receive

[Howard McLauchlan]

From: Howard McLauchlan <hmclauch...@fb.com>

Presently btrfs send/receive does not propagate inode attribute flags;
all chattr operations are effectively discarded upon transmission.

This patch adds kernel support for inode attribute flags. Userspace
support can be found under the commit:

btrfs-progs: add chattr support for send/receive

An associated xfstest can be found at:

btrfs: add verify chattr support for send/receive test

These changes are only enabled for send stream version 2

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 fs/btrfs/ctree.h |   2 +
 fs/btrfs/ioctl.c |   2 +-
 fs/btrfs/send.c  | 183 +++
 3 files changed, 158 insertions(+), 29 deletions(-)

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 0d422c9908b8..002fe3ad193a 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -1461,6 +1461,8 @@ struct btrfs_map_token {
unsigned long offset;
 };
 
+unsigned int btrfs_flags_to_ioctl(unsigned int flags);
+
 #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \
((bytes) >> (fs_info)->sb->s_blocksize_bits)
 
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index 632e26d6f7ce..36ce1e589f9e 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, unsigned 
int flags)
 /*
  * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl.
  */
-static unsigned int btrfs_flags_to_ioctl(unsigned int flags)
+unsigned int btrfs_flags_to_ioctl(unsigned int flags)
 {
unsigned int iflags = 0;
 
diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index 84dacb20d832..a36a2983b34a 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -108,6 +108,13 @@ struct send_ctx {
u64 cur_inode_last_extent;
u64 cur_inode_next_write_offset;
 
+   /*
+* state for chattr purposes
+*/
+   u64 cur_inode_flip_flags;
+   u64 cur_inode_receive_flags;
+   bool receive_flags_valid;
+
u64 send_progress;
u64 total_data_size;
 
@@ -815,7 +822,7 @@ static int send_rmdir(struct send_ctx *sctx, struct fs_path 
*path)
  */
 static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path,
  u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid,
- u64 *gid, u64 *rdev)
+ u64 *gid, u64 *rdev, u64 *flags)
 {
int ret;
struct btrfs_inode_item *ii;
@@ -845,6 +852,8 @@ static int __get_inode_info(struct btrfs_root *root, struct 
btrfs_path *path,
*gid = btrfs_inode_gid(path->nodes[0], ii);
if (rdev)
*rdev = btrfs_inode_rdev(path->nodes[0], ii);
+   if (flags)
+   *flags = btrfs_inode_flags(path->nodes[0], ii);
 
return ret;
 }
@@ -852,7 +861,7 @@ static int __get_inode_info(struct btrfs_root *root, struct 
btrfs_path *path,
 static int get_inode_info(struct btrfs_root *root,
  u64 ino, u64 *size, u64 *gen,
  u64 *mode, u64 *uid, u64 *gid,
- u64 *rdev)
+ u64 *rdev, u64 *flags)
 {
struct btrfs_path *path;
int ret;
@@ -861,7 +870,7 @@ static int get_inode_info(struct btrfs_root *root,
if (!path)
return -ENOMEM;
ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid,
-  rdev);
+  rdev, flags);
btrfs_free_path(path);
return ret;
 }
@@ -1250,7 +1259,7 @@ static int __iterate_backrefs(u64 ino, u64 offset, u64 
root, void *ctx_)
 * accept clones from these extents.
 */
ret = __get_inode_info(found->root, bctx->path, ino, _size, NULL, 
NULL,
-  NULL, NULL, NULL);
+  NULL, NULL, NULL, NULL);
btrfs_release_path(bctx->path);
if (ret < 0)
return ret;
@@ -1610,7 +1619,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 
ino, u64 gen)
u64 right_gen;
 
ret = get_inode_info(sctx->send_root, ino, NULL, _gen, NULL, NULL,
-   NULL, NULL);
+   NULL, NULL, NULL);
if (ret < 0 && ret != -ENOENT)
goto out;
left_ret = ret;
@@ -1619,7 +1628,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 
ino, u64 gen)
right_ret = -ENOENT;
} else {
ret = get_inode_info(sctx->parent_root, ino, NULL, _gen,
-   NULL, NULL, NULL, NULL);
+   NULL, NULL, NULL, NULL, NULL);
if (ret < 0 && ret != -ENOENT)
goto out;
right_ret = ret;
@@ -1788,7 +1797,7 @@ static int get_first_ref(struct btrfs_root *root, u64 ino,
 
i

[PATCH v2 3/6] Btrfs-progs: send, implement fallocate command callback

[Howard McLauchlan]

From: Filipe Manana 

The fallocate send stream command, added in stream version 2, is used to
pre-allocate space for files and punch file holes. This change implements
the callback for that new command, using the fallocate function from the
standard C library to carry out the specified action (allocate file space
or punch a file hole).

Signed-off-by: Filipe David Borba Manana 
---
 cmds-receive.c | 44 
 send-stream.c  | 13 +
 send-stream.h  |  2 ++
 3 files changed, 59 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index d8ff5194..510b6bc8 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -39,6 +39,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -1149,6 +1150,48 @@ static int process_update_extent(const char *path, u64 
offset, u64 len,
return 0;
 }
 
+static int process_fallocate(const char *path, u32 flags, u64 offset,
+u64 len, void *user)
+{
+   struct btrfs_receive *rctx = user;
+   int mode = 0;
+   int ret;
+   char full_path[PATH_MAX];
+
+   ret = path_cat_out(full_path, rctx->full_subvol_path, path);
+   if (ret < 0) {
+   error("fallocate: path invalid: %s", path);
+   goto out;
+   }
+
+   if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE)
+   mode |= FALLOC_FL_KEEP_SIZE;
+   if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+   mode |= FALLOC_FL_PUNCH_HOLE;
+
+   if (g_verbose >= 2)
+   fprintf(stderr,
+   "fallocate %s - flags %u, offset %llu, len %llu\n",
+   path, flags, offset, len);
+
+   ret = open_inode_for_write(rctx, full_path);
+   if (ret < 0)
+   goto out;
+
+   ret = fallocate(rctx->write_fd, mode, offset, len);
+   if (ret) {
+   ret = -errno;
+   fprintf(stderr,
+   "ERROR: fallocate against %s failed. %s\n",
+   path, strerror(-ret));
+   goto out;
+   }
+   update_progress(rctx, len);
+
+out:
+   return ret;
+}
+
 static struct btrfs_send_ops send_ops = {
.subvol = process_subvol,
.snapshot = process_snapshot,
@@ -1172,6 +1215,7 @@ static struct btrfs_send_ops send_ops = {
.utimes = process_utimes,
.update_extent = process_update_extent,
.total_data_size = process_total_data_size,
+   .fallocate = process_fallocate,
 };
 
 static int do_receive(struct btrfs_receive *rctx, const char *tomnt,
diff --git a/send-stream.c b/send-stream.c
index d30fd5a7..74ec37dd 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -457,6 +457,19 @@ static int read_and_process_cmd(struct btrfs_send_stream 
*sctx)
TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, );
ret = sctx->ops->total_data_size(tmp, sctx->user);
break;
+   case BTRFS_SEND_C_FALLOCATE:
+   {
+   u32 flags;
+   u64 len;
+
+   TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, );
+   TLV_GET_U32(sctx, BTRFS_SEND_A_FALLOCATE_FLAGS, );
+   TLV_GET_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, );
+   TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, );
+   ret = sctx->ops->fallocate(path, flags, offset, len,
+   sctx->user);
+   }
+   break;
case BTRFS_SEND_C_END:
ret = 1;
break;
diff --git a/send-stream.h b/send-stream.h
index 5b244ab6..89e64043 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -67,6 +67,8 @@ struct btrfs_send_ops {
  void *user);
int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
int (*total_data_size)(u64 size, void *user);
+   int (*fallocate)(const char *path, u32 flags, u64 offset,
+u64 len, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2 1/2] btrfs: add verify chattr support for send/receive test

[Howard McLauchlan]

From: Howard McLauchlan <hmclauch...@fb.com>

This test aims to verify correct behaviour with chattr operations and
btrfs send/receive. The intent is to check general correctness as well
as special interactions with troublesome flags(immutable, append only).

This test is motivated by a bug in btrfs which demonstrates a lack of
chattr support in btrfs send/receive.

A kernel patch to fix this can be found at:

btrfs: add chattr support for send/receive

The accompanying userspace patch can be found at:

btrfs-progs: add chattr support for send/receive

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 tests/btrfs/161 | 202 
 tests/btrfs/161.out |  38 +
 tests/btrfs/group   |   1 +
 3 files changed, 241 insertions(+)
 create mode 100755 tests/btrfs/161
 create mode 100644 tests/btrfs/161.out

diff --git a/tests/btrfs/161 b/tests/btrfs/161
new file mode 100755
index ..6c30a5e2
--- /dev/null
+++ b/tests/btrfs/161
@@ -0,0 +1,202 @@
+#! /bin/bash
+# FS QA Test 161
+#
+# This test verifies the correct behaviour of chattr support for btrfs
+# send/receive; 6 cases will be tested:
+# 1. New inode created with an inode flag set
+# 2. Existing inode with BTRFS_INODE_APPEND is written to with sequence:
+#  chattr -a
+#  pwrite something
+#  chattr +a
+# 3. Existing inode with BTRFS_INODE_APPEND is written to with sequence:
+#  chattr -a
+#  pwrite something
+#  chattr +d
+# 4. Existing inode is written to with sequence:
+#  setfattr something
+#  chattr +a
+# 5. Existing inode with BTRFS_INODE_APPEND is written to with sequence:
+#  chattr -a
+#  setfattr something
+#  setfattr something else
+#  chattr +a
+# 6. As above, but with pwrite instead of setfattr
+# The goal of 5 and 6 is not to test correctness, but to ensure we
+# don't send extra chattrs that are unnecessary.
+#
+# We verify the md5sum of the snapshots in the receive directory to ensure file
+# contents have changed appropriately. We also observe the flags changing (or
+# not changing) as appropriate.
+#
+#---
+# Copyright (c) 2018 Facebook.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#---
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1   # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+   cd /
+   rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+# Modify as appropriate.
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+
+send_files_dir=$TEST_DIR/btrfs-test-$seq
+
+rm -rf $send_files_dir
+mkdir $send_files_dir
+
+_scratch_mkfs >>$seqres.full 2>&1
+_scratch_mount
+
+# Create receive directory
+mkdir $SCRATCH_MNT/receive
+
+# Create test file and set chattr flag
+_run_btrfs_util_prog subvolume create $SCRATCH_MNT/parent
+$XFS_IO_PROG -f -c "pwrite -S 0xaa 0K 32K" $SCRATCH_MNT/parent/foo | 
_filter_xfs_io
+$CHATTR_PROG +a $SCRATCH_MNT/parent/foo
+
+# Send/Receive initial snapshot
+_run_btrfs_util_prog subvolume snapshot -r $SCRATCH_MNT/parent \
+   $SCRATCH_MNT/old_parent
+_run_btrfs_util_prog send --stream-version 2 -f $SCRATCH_MNT/out 
$SCRATCH_MNT/old_parent
+_run_btrfs_util_prog receive -f $SCRATCH_MNT/out $SCRATCH_MNT/receive
+
+# Verify post-send content and flags
+echo "post-send file digest for old_parent:"
+md5sum $SCRATCH_MNT/old_parent/foo | _filter_scratch
+echo "post-send file flag for old_parent:"
+lsattr $SCRATCH_MNT/receive/old_parent/foo | _filter_scratch
+
+# Make change
+$CHATTR_PROG -a $SCRATCH_MNT/parent/foo
+$XFS_IO_PROG -f -c "pwrite -S 0xab 0K 32K" $SCRATCH_MNT/parent/foo | 
_filter_xfs_io
+$CHATTR_PROG +a $SCRATCH_MNT/parent/foo
+
+# Send incremental change
+_run_btrfs_util_prog subvolume snapshot -r $SCRATCH_MNT/parent \
+   $SCRATCH_MNT/child_1
+_run_btrfs_util_prog send --stream-version 2 -p $SCRA

[RFC PATCH v2 2/6] btrfs: send, implement total data size command to allow for progress estimation

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

This new send flag makes send calculate first the amount of new file data (in 
bytes)
the send root has relatively to the parent root, or for the case of a 
non-incremental
send, the total amount of file data the stream will create (including holes and 
prealloc
extents). In other words, it computes the sum of the lengths of all write, 
clone and
fallocate operations that will be sent through the send stream.

This data size value is sent in a new command, named 
BTRFS_SEND_C_TOTAL_DATA_SIZE, that
immediately follows a BTRFS_SEND_C_SUBVOL or BTRFS_SEND_C_SNAPSHOT command, and 
precedes
any command that changes a file or the filesystem hierarchy. Upon receiving a 
write, clone
or fallocate command, the receiving end can increment a counter by the data 
length of that
command and therefore report progress by comparing the counter's value with the 
data size
value received in the BTRFS_SEND_C_TOTAL_DATA_SIZE command.

The approach is simple, before the normal operation of send, do a scan in the 
file system
tree for new inodes and new/changed file extent items, just like in send's 
normal operation,
and keep incrementing a counter with new inodes' size and the size of file 
extents (and file
holes)  that are going to be written, cloned or fallocated. This is actually a 
simpler and
more lightweight tree scan/processing than the one we do when sending the 
changes, as it
doesn't process inode references nor does any lookups in the extent tree for 
example.

After modifying btrfs-progs to understand this new command and report progress, 
here's an
example (the -o flag tells btrfs send to pass the new flag to the kernel's send 
ioctl):

$ btrfs send -s --stream-version 2 /mnt/sdd/snap_base | btrfs receive 
/mnt/sdc
At subvol /mnt/sdd/snap_base
At subvol snap_base
About to receive 9212392667 bytes
Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 
44.07%, 40.32MB/s

$ btrfs send -s --stream-version 2 -p /mnt/sdd/snap_base /mnt/sdd/snap_incr 
| btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_incr
At subvol snap_incr
About to receive 9571342213 bytes
Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 
68.51%, 51.04MB/s

[Howard: rebased on 4.17-rc7]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
---
 fs/btrfs/send.c | 189 
 1 file changed, 157 insertions(+), 32 deletions(-)

diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index eccd69387065..7b184831812b 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -66,7 +66,13 @@ struct clone_root {
 #define SEND_CTX_MAX_NAME_CACHE_SIZE 128
 #define SEND_CTX_NAME_CACHE_CLEAN_SIZE (SEND_CTX_MAX_NAME_CACHE_SIZE * 2)
 
+enum btrfs_send_phase {
+   SEND_PHASE_STREAM_CHANGES,
+   SEND_PHASE_COMPUTE_DATA_SIZE,
+};
+
 struct send_ctx {
+   enum btrfs_send_phase phase;
struct file *send_filp;
loff_t send_off;
char *send_buf;
@@ -102,6 +108,7 @@ struct send_ctx {
u64 cur_inode_next_write_offset;
 
u64 send_progress;
+   u64 total_data_size;
 
struct list_head new_refs;
struct list_head deleted_refs;
@@ -709,6 +716,8 @@ static int send_rename(struct send_ctx *sctx,
struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
int ret;
 
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
btrfs_debug(fs_info, "send_rename %s -> %s", from->start, to->start);
 
ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);
@@ -758,6 +767,8 @@ static int send_unlink(struct send_ctx *sctx, struct 
fs_path *path)
struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
int ret;
 
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
btrfs_debug(fs_info, "send_unlink %s", path->start);
 
ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);
@@ -780,6 +791,7 @@ static int send_rmdir(struct send_ctx *sctx, struct fs_path 
*path)
 {
struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
int ret;
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
 
btrfs_debug(fs_info, "send_rmdir %s", path->start);
 
@@ -2419,6 +2431,8 @@ static int send_truncate(struct send_ctx *sctx, u64 ino, 
u64 gen, u64 size)
int ret = 0;
struct fs_path *p;
 
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
btrfs_debug(fs_info, "send_truncate %llu size=%llu", ino, size);
 
p = fs_path_alloc();
@@ -2449,6 +2463,8 @@ static int send_chmod(struct send_ctx *sctx, u64 ino, u64 
gen, u64 mode)
int ret = 0;
struct fs_path *p;
 
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
btrfs_debug(fs_info, "send_chmod %llu mode=%llu", i

[RFC PATCH v2 0/6] btrfs send stream version 2

[Howard McLauchlan]

This is v2 of send stream version 2. The goal is to provide proper
versioning/compatibility as new features are implemented. v1 can be found here
[1].

v2 adds BTRFS_SEND_C_WRITE_COMPRESSED to patch 1 and style fixes/logic
simplifications to patches 5,6.

v2 also updates btrfs-progs to reflect BTRFS_SEND_C_WRITE_COMPRESSED and updates
the pertinent xfstests to also test the total size command.

As of 4.17-rc7, these changes pass all "send" group xfstests

Cheers,

1: https://patchwork.kernel.org/patch/10388003/

Filipe David Borba Manana (5):
  btrfs: send, bump stream version
  btrfs: send, implement total data size command to allow for progress
estimation
  btrfs: send, use fallocate command to punch holes
  btrfs: send, use fallocate command to allocate extents
  btrfs: add send_stream_version attribute to sysfs

Howard McLauchlan (1):
  btrfs: add chattr support for send/receive

 fs/btrfs/ctree.h   |   2 +
 fs/btrfs/ioctl.c   |   2 +-
 fs/btrfs/send.c| 495 +++--
 fs/btrfs/send.h|  27 +-
 fs/btrfs/sysfs.c   |  27 ++
 include/uapi/linux/btrfs.h |  21 +-
 6 files changed, 493 insertions(+), 81 deletions(-)

-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 2/2] xfstests: btrfs, test send's ability to punch holes and prealloc extents

[Howard McLauchlan]

From: Filipe Manana <fdman...@gmail.com>

This test verifies that after an incremental btrfs send the
replicated file has the same exact hole and data structure as in
the origin filesystem. This didn't use to be the case before the
send stream version 2 - holes were sent as write operations of 0
valued bytes instead of punching holes with the fallocate system
call, and pre-allocated extents were sent as well as write
operations of 0 valued bytes instead of intructions for the
receiver to use the fallocate system call.

It also checks that prealloc extents that lie beyond the file's
size are replicated by an incremental send.

[Howard: rebased on kernel v4.17-rc4, btrfs progs v4.16.1]
Signed-off-by: Howard McLauchlan <li...@hmclauchlan.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
---
 common/rc   |  10 
 tests/btrfs/160 | 121 
 tests/btrfs/160.out |  42 +++
 tests/btrfs/group   |   1 +
 4 files changed, 174 insertions(+)
 create mode 100755 tests/btrfs/160
 create mode 100644 tests/btrfs/160.out

diff --git a/common/rc b/common/rc
index 9ffab7fd..e2e9904a 100644
--- a/common/rc
+++ b/common/rc
@@ -3802,6 +3802,16 @@ _require_scratch_feature()
esac
 }
 
+_require_btrfs_stream_version_2()
+{
+   $BTRFS_UTIL_PROG send 2>&1 | \
+   grep '^[ \t]*\--stream-version[ \t]\+.*' > /dev/null 2>&1
+   if [ $? -ne 0 ]; then
+   _notrun "Missing btrfs-progs send -a command line option, 
skipped this test"
+   fi
+}
+
+
 init_rc
 
 

diff --git a/tests/btrfs/160 b/tests/btrfs/160
new file mode 100755
index ..e4b7264c
--- /dev/null
+++ b/tests/btrfs/160
@@ -0,0 +1,121 @@
+#! /bin/bash
+# FS QA Test No. btrfs/160
+#
+# Verify that after an incremental btrfs send the replicated file has
+# the same exact hole and data structure as in the origin filesystem.
+# This didn't use to be the case before the send stream version 2 -
+# holes were sent as write operations of 0 valued bytes instead of punching
+# holes with the fallocate system call, and pre-allocated extents were sent
+# as well as write operations of 0 valued bytes instead of intructions for
+# the receiver to use the fallocate system call. Also check that prealloc
+# extents that lie beyond the file's size are replicated by an incremental
+# send.
+#
+# More specifically, this structure preserving guarantee was added by the
+# following linux kernel commits:
+#
+#Btrfs: send, use fallocate command to punch holes
+#Btrfs: send, use fallocate command to allocate extents
+#
+#---
+# Copyright (c) 2014 Filipe Manana.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#---
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+tmp=/tmp/$$
+status=1   # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+rm -fr $send_files_dir
+rm -fr $tmp
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/punch
+
+# real QA test starts here
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+_require_fssum
+_require_xfs_io_command "fiemap"
+_require_btrfs_stream_version_2
+
+send_files_dir=$TEST_DIR/btrfs-test-$seq
+
+rm -f $seqres.full
+rm -fr $send_files_dir
+mkdir $send_files_dir
+
+_scratch_mkfs >/dev/null 2>&1
+_scratch_mount
+
+$XFS_IO_PROG -f -c "pwrite -S 0x01 -b 30 0 30" $SCRATCH_MNT/foo \
+   | _filter_xfs_io
+
+_run_btrfs_util_prog subvolume snapshot -r $SCRATCH_MNT $SCRATCH_MNT/mysnap1
+
+$XFS_IO_PROG -c "fpunch 10 5" $SCRATCH_MNT/foo
+$XFS_IO_PROG -c "falloc 10 5" $SCRATCH_MNT/foo
+$XFS_IO_PROG -c "pwrite -S 0xff -b 1000 12 1000" $SCRATCH_MNT/foo \
+   | _filter_xfs_io
+$XFS_IO_PROG -c "fpunch 25 2" $SCRATCH_MNT/foo
+
+$XFS_IO_PROG -c "falloc -k 30 100" $SCRATCH_MNT/foo
+$XFS_IO_PROG -c "falloc -k 900 200" $SCRATCH_MNT/foo
+
+_run_btrfs_util_prog subvolume snapshot -

[PATCH 1/2] btrfs: add verify chattr support for send/receive test

[Howard McLauchlan]

From: Howard McLauchlan <hmclauch...@fb.com>

This test aims to verify correct behaviour with chattr operations and
btrfs send/receive. The intent is to check general correctness as well
as special interactions with troublesome flags(immutable, append only).

This test is motivated by a bug in btrfs which demonstrates a lack of
chattr support in btrfs send/receive.

A kernel patch to fix this can be found at:

btrfs: add chattr support for send/receive

The accompanying userspace patch can be found at:

btrfs-progs: add chattr support for send/receive

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 tests/btrfs/161 | 202 
 tests/btrfs/161.out |  38 +
 tests/btrfs/group   |   1 +
 3 files changed, 241 insertions(+)
 create mode 100755 tests/btrfs/161
 create mode 100644 tests/btrfs/161.out

diff --git a/tests/btrfs/161 b/tests/btrfs/161
new file mode 100755
index ..6c30a5e2
--- /dev/null
+++ b/tests/btrfs/161
@@ -0,0 +1,202 @@
+#! /bin/bash
+# FS QA Test 161
+#
+# This test verifies the correct behaviour of chattr support for btrfs
+# send/receive; 6 cases will be tested:
+# 1. New inode created with an inode flag set
+# 2. Existing inode with BTRFS_INODE_APPEND is written to with sequence:
+#  chattr -a
+#  pwrite something
+#  chattr +a
+# 3. Existing inode with BTRFS_INODE_APPEND is written to with sequence:
+#  chattr -a
+#  pwrite something
+#  chattr +d
+# 4. Existing inode is written to with sequence:
+#  setfattr something
+#  chattr +a
+# 5. Existing inode with BTRFS_INODE_APPEND is written to with sequence:
+#  chattr -a
+#  setfattr something
+#  setfattr something else
+#  chattr +a
+# 6. As above, but with pwrite instead of setfattr
+# The goal of 5 and 6 is not to test correctness, but to ensure we
+# don't send extra chattrs that are unnecessary.
+#
+# We verify the md5sum of the snapshots in the receive directory to ensure file
+# contents have changed appropriately. We also observe the flags changing (or
+# not changing) as appropriate.
+#
+#---
+# Copyright (c) 2018 Facebook.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#---
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1   # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+   cd /
+   rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+# Modify as appropriate.
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+
+send_files_dir=$TEST_DIR/btrfs-test-$seq
+
+rm -rf $send_files_dir
+mkdir $send_files_dir
+
+_scratch_mkfs >>$seqres.full 2>&1
+_scratch_mount
+
+# Create receive directory
+mkdir $SCRATCH_MNT/receive
+
+# Create test file and set chattr flag
+_run_btrfs_util_prog subvolume create $SCRATCH_MNT/parent
+$XFS_IO_PROG -f -c "pwrite -S 0xaa 0K 32K" $SCRATCH_MNT/parent/foo | 
_filter_xfs_io
+$CHATTR_PROG +a $SCRATCH_MNT/parent/foo
+
+# Send/Receive initial snapshot
+_run_btrfs_util_prog subvolume snapshot -r $SCRATCH_MNT/parent \
+   $SCRATCH_MNT/old_parent
+_run_btrfs_util_prog send --stream-version 2 -f $SCRATCH_MNT/out 
$SCRATCH_MNT/old_parent
+_run_btrfs_util_prog receive -f $SCRATCH_MNT/out $SCRATCH_MNT/receive
+
+# Verify post-send content and flags
+echo "post-send file digest for old_parent:"
+md5sum $SCRATCH_MNT/old_parent/foo | _filter_scratch
+echo "post-send file flag for old_parent:"
+lsattr $SCRATCH_MNT/receive/old_parent/foo | _filter_scratch
+
+# Make change
+$CHATTR_PROG -a $SCRATCH_MNT/parent/foo
+$XFS_IO_PROG -f -c "pwrite -S 0xab 0K 32K" $SCRATCH_MNT/parent/foo | 
_filter_xfs_io
+$CHATTR_PROG +a $SCRATCH_MNT/parent/foo
+
+# Send incremental change
+_run_btrfs_util_prog subvolume snapshot -r $SCRATCH_MNT/parent \
+   $SCRATCH_MNT/child_1
+_run_btrfs_util_prog send --stream-version 2 -p $SCRA

[RFC PATCH 6/6] btrfs-progs: add chattr support for send/receive

[Howard McLauchlan]

From: Howard McLauchlan <hmclauch...@fb.com>

Presently, btrfs send/receive does not propagate inode attribute flags;
all chattr operations are effectively discarded upon transmission.

This patch adds userspace support for inode attribute flags. Kernel
support can be found under the commit:

btrfs: add chattr support for send/receive

An associated xfstest can also be found at:

btrfs: verify chattr support for send/receive test

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 cmds-receive.c | 37 +
 send-dump.c|  6 ++
 send-stream.c  |  5 +
 send-stream.h  |  1 +
 send.h |  4 ++--
 5 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/cmds-receive.c b/cmds-receive.c
index 20e593f7..2a841bfc 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -40,6 +40,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -1201,6 +1202,41 @@ out:
return ret;
 }
 
+static int process_chattr(const char *path, u64 flags, void *user)
+{
+   int ret = 0;
+   int fd = 0;
+   int _flags = flags;
+   struct btrfs_receive *rctx = user;
+   char full_path[PATH_MAX];
+
+   ret = path_cat_out(full_path, rctx->full_subvol_path, path);
+   if (ret < 0) {
+   error("chattr: path invalid: %s", path);
+   goto out;
+   }
+
+   if (g_verbose >= 2)
+   fprintf(stderr, "chattr %s - flags=0%o\n", path, (int)flags);
+
+   fd = open(full_path, O_RDONLY);
+   if (fd < 0) {
+   ret = -errno;
+   error("cannot open %s: %s", path, strerror(-ret));
+   goto out;
+   }
+
+   ret = ioctl(fd, FS_IOC_SETFLAGS, &_flags);
+   if (ret < 0) {
+   ret = -errno;
+   error("chattr %s failed: %s", path, strerror(-ret));
+   goto out;
+   }
+
+out:
+   return ret;
+}
+
 static struct btrfs_send_ops send_ops = {
.subvol = process_subvol,
.snapshot = process_snapshot,
@@ -1225,6 +1261,7 @@ static struct btrfs_send_ops send_ops = {
.update_extent = process_update_extent,
.total_data_size = process_total_data_size,
.fallocate = process_fallocate,
+   .chattr = process_chattr,
 };
 
 static int do_receive(struct btrfs_receive *rctx, const char *tomnt,
diff --git a/send-dump.c b/send-dump.c
index c5a695a2..15aea402 100644
--- a/send-dump.c
+++ b/send-dump.c
@@ -331,6 +331,11 @@ static int print_fallocate(const char *path, u32 flags, 
u64 offset, u64 len,
  len);
 }
 
+static int print_chattr(const char *path, u64 flags, void *user)
+{
+   return PRINT_DUMP(user, path, "chattr", "flags=%llu", flags);
+}
+
 struct btrfs_send_ops btrfs_print_send_ops = {
.subvol = print_subvol,
.snapshot = print_snapshot,
@@ -355,4 +360,5 @@ struct btrfs_send_ops btrfs_print_send_ops = {
.update_extent = print_update_extent,
.total_data_size = print_total_data_size,
.fallocate = print_fallocate,
+   .chattr = print_chattr,
 };
diff --git a/send-stream.c b/send-stream.c
index 74ec37dd..4f26fae3 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -470,6 +470,11 @@ static int read_and_process_cmd(struct btrfs_send_stream 
*sctx)
sctx->user);
}
break;
+   case BTRFS_SEND_C_CHATTR:
+   TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, );
+   TLV_GET_U64(sctx, BTRFS_SEND_A_CHATTR, );
+   ret = sctx->ops->chattr(path, tmp, sctx->user);
+   break;
case BTRFS_SEND_C_END:
ret = 1;
break;
diff --git a/send-stream.h b/send-stream.h
index 89e64043..a9f08d52 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -69,6 +69,7 @@ struct btrfs_send_ops {
int (*total_data_size)(u64 size, void *user);
int (*fallocate)(const char *path, u32 flags, u64 offset,
 u64 len, void *user);
+   int (*chattr)(const char *path, u64 flags, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
diff --git a/send.h b/send.h
index eb14fba3..5b5ada61 100644
--- a/send.h
+++ b/send.h
@@ -103,8 +103,8 @@ enum btrfs_send_cmd {
 */
BTRFS_SEND_C_TOTAL_DATA_SIZE,
BTRFS_SEND_C_FALLOCATE,
-   BTRFS_SEND_C_INODE_SET_FLAGS,
BTRFS_SEND_C_UTIMES2, /* Same as UTIMES, but it includes OTIME too. */
+   BTRFS_SEND_C_CHATTR,
 
__BTRFS_SEND_C_MAX,
 };
@@ -148,7 +148,7 @@ enum {
 * The following attributes were added in stream version 2.
 */
BTRFS_SEND_A_FALLOCATE_FLAGS, /* 32 bits */
-   BTRFS_SEND_A_INODE_FLAGS, /* 32 bits */
+   BTRFS_SEND_A_CHATTR,  /* 32 bits */
 
__BTRFS_SEND_A_MAX,
 

[RFC PATCH 5/6] btrfs-progs: add total data size, fallocate to dump

[Howard McLauchlan]

From: Howard McLauchlan <hmclauch...@fb.com>

Adding entries to dump for new commands (total data size, fallocate).

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 send-dump.c | 19 ++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/send-dump.c b/send-dump.c
index 1591e0cc..c5a695a2 100644
--- a/send-dump.c
+++ b/send-dump.c
@@ -316,6 +316,21 @@ static int print_update_extent(const char *path, u64 
offset, u64 len,
  offset, len);
 }
 
+static int print_total_data_size(u64 size, void *user)
+{
+   char path;
+
+   return PRINT_DUMP(user, , "total_data_size", "size=%llu", size);
+}
+
+static int print_fallocate(const char *path, u32 flags, u64 offset, u64 len,
+  void *user)
+{
+   return PRINT_DUMP(user, path, "fallocate",
+ "flags=%u offset=%llu len=%llu", flags, offset,
+ len);
+}
+
 struct btrfs_send_ops btrfs_print_send_ops = {
.subvol = print_subvol,
.snapshot = print_snapshot,
@@ -337,5 +352,7 @@ struct btrfs_send_ops btrfs_print_send_ops = {
.chmod = print_chmod,
.chown = print_chown,
.utimes = print_utimes,
-   .update_extent = print_update_extent
+   .update_extent = print_update_extent,
+   .total_data_size = print_total_data_size,
+   .fallocate = print_fallocate,
 };
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH 4/6] Btrfs-progs: add write and clone commands debug info to receive

[Howard McLauchlan]

From: Filipe Manana 

When specifying -vv print information about received write and clone commands 
too,
as we do this for other commands already and it's very useful for debugging and
troubleshooting.

Signed-off-by: Filipe David Borba Manana 
---
 cmds-receive.c | 9 +
 1 file changed, 9 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index 510b6bc8..20e593f7 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -790,6 +790,10 @@ static int process_write(const char *path, const void 
*data, u64 offset,
u64 pos = 0;
int w;
 
+   if (g_verbose >= 2)
+   fprintf(stderr, "write %s, offset %llu, len %llu\n",
+   path, offset, len);
+
ret = path_cat_out(full_path, rctx->full_subvol_path, path);
if (ret < 0) {
error("write: path invalid: %s", path);
@@ -831,6 +835,11 @@ static int process_clone(const char *path, u64 offset, u64 
len,
char full_clone_path[PATH_MAX];
int clone_fd = -1;
 
+   if (g_verbose >= 2)
+   fprintf(stderr,
+   "clone %s, offset %llu, len %llu, clone path %s, clone 
offset %llu\n",
+   path, offset, len, clone_path, clone_offset);
+
ret = path_cat_out(full_path, rctx->full_subvol_path, path);
if (ret < 0) {
error("clone: source path invalid: %s", path);
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH 3/6] Btrfs-progs: send, implement fallocate command callback

[Howard McLauchlan]

From: Filipe Manana 

The fallocate send stream command, added in stream version 2, is used to
pre-allocate space for files and punch file holes. This change implements
the callback for that new command, using the fallocate function from the
standard C library to carry out the specified action (allocate file space
or punch a file hole).

Signed-off-by: Filipe David Borba Manana 
---
 cmds-receive.c | 44 
 send-stream.c  | 13 +
 send-stream.h  |  2 ++
 3 files changed, 59 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index d8ff5194..510b6bc8 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -39,6 +39,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -1149,6 +1150,48 @@ static int process_update_extent(const char *path, u64 
offset, u64 len,
return 0;
 }
 
+static int process_fallocate(const char *path, u32 flags, u64 offset,
+u64 len, void *user)
+{
+   struct btrfs_receive *rctx = user;
+   int mode = 0;
+   int ret;
+   char full_path[PATH_MAX];
+
+   ret = path_cat_out(full_path, rctx->full_subvol_path, path);
+   if (ret < 0) {
+   error("fallocate: path invalid: %s", path);
+   goto out;
+   }
+
+   if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE)
+   mode |= FALLOC_FL_KEEP_SIZE;
+   if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+   mode |= FALLOC_FL_PUNCH_HOLE;
+
+   if (g_verbose >= 2)
+   fprintf(stderr,
+   "fallocate %s - flags %u, offset %llu, len %llu\n",
+   path, flags, offset, len);
+
+   ret = open_inode_for_write(rctx, full_path);
+   if (ret < 0)
+   goto out;
+
+   ret = fallocate(rctx->write_fd, mode, offset, len);
+   if (ret) {
+   ret = -errno;
+   fprintf(stderr,
+   "ERROR: fallocate against %s failed. %s\n",
+   path, strerror(-ret));
+   goto out;
+   }
+   update_progress(rctx, len);
+
+out:
+   return ret;
+}
+
 static struct btrfs_send_ops send_ops = {
.subvol = process_subvol,
.snapshot = process_snapshot,
@@ -1172,6 +1215,7 @@ static struct btrfs_send_ops send_ops = {
.utimes = process_utimes,
.update_extent = process_update_extent,
.total_data_size = process_total_data_size,
+   .fallocate = process_fallocate,
 };
 
 static int do_receive(struct btrfs_receive *rctx, const char *tomnt,
diff --git a/send-stream.c b/send-stream.c
index d30fd5a7..74ec37dd 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -457,6 +457,19 @@ static int read_and_process_cmd(struct btrfs_send_stream 
*sctx)
TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, );
ret = sctx->ops->total_data_size(tmp, sctx->user);
break;
+   case BTRFS_SEND_C_FALLOCATE:
+   {
+   u32 flags;
+   u64 len;
+
+   TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, );
+   TLV_GET_U32(sctx, BTRFS_SEND_A_FALLOCATE_FLAGS, );
+   TLV_GET_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, );
+   TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, );
+   ret = sctx->ops->fallocate(path, flags, offset, len,
+   sctx->user);
+   }
+   break;
case BTRFS_SEND_C_END:
ret = 1;
break;
diff --git a/send-stream.h b/send-stream.h
index 5b244ab6..89e64043 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -67,6 +67,8 @@ struct btrfs_send_ops {
  void *user);
int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
int (*total_data_size)(u64 size, void *user);
+   int (*fallocate)(const char *path, u32 flags, u64 offset,
+u64 len, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH 2/6] Btrfs-progs: send, implement total data size callback and progress report

[Howard McLauchlan]

From: Filipe Manana 

This is a followup to the kernel patch titled:

Btrfs: send, implement total data size command to allow for progress 
estimation

This makes the btrfs send and receive commands aware of the new send flag,
named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data
that is new between the parent and send snapshots/roots. As this command
immediately follows the commands to start a snapshot/subvolume, it can be
used to report and compute progress, by keeping a counter that is incremented
with the data length of each write, clone and fallocate command that is received
from the stream.

Example:

$ btrfs send -s --stream-version 2 /mnt/sdd/snap_base | btrfs receive 
/mnt/sdc
At subvol /mnt/sdd/snap_base
At subvol snap_base
About to receive 9212392667 bytes
Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 
44.07%, 40.32MB/s

$ btrfs send -s --stream-version 2 -p /mnt/sdd/snap_base /mnt/sdd/snap_incr 
| btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_incr
At subvol snap_incr
About to receive 9571342213 bytes
Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 
68.51%, 51.04MB/s

At the moment progress is only reported by btrfs-receive, but it is possible 
and simple
to do it for btrfs-send too, so that we can get progress report when not piping 
btrfs-send
output to btrfs-receive (directly to a file).

Signed-off-by: Filipe David Borba Manana 
---
 cmds-receive.c | 91 ++
 cmds-send.c| 23 +++--
 send-stream.c  |  4 +++
 send-stream.h  |  1 +
 4 files changed, 117 insertions(+), 2 deletions(-)

diff --git a/cmds-receive.c b/cmds-receive.c
index 68123a31..d8ff5194 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -30,6 +30,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -79,6 +80,14 @@ struct btrfs_receive
 
int honor_end_cmd;
 
+   /* For the subvolume/snapshot we're currently receiving. */
+   u64 total_data_size;
+   u64 bytes_received;
+   time_t last_progress_update;
+   u64 bytes_received_last_update;
+   float progress;
+   const char *target;
+
/*
 * Buffer to store capabilities from security.capabilities xattr,
 * usually 20 bytes, but make same room for potentially larger
@@ -156,6 +165,16 @@ out:
return ret;
 }
 
+static void reset_progress(struct btrfs_receive *rctx, const char *dest)
+{
+   rctx->total_data_size = 0;
+   rctx->bytes_received = 0;
+   rctx->progress = 0.0;
+   rctx->last_progress_update = 0;
+   rctx->bytes_received_last_update = 0;
+   rctx->target = dest;
+}
+
 static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
  void *user)
 {
@@ -180,6 +199,7 @@ static int process_subvol(const char *path, const u8 *uuid, 
u64 ctransid,
ret = -EINVAL;
goto out;
}
+   reset_progress(rctx, "Subvolume");
 
if (*rctx->dest_dir_path == 0) {
strncpy_null(rctx->cur_subvol_path, path);
@@ -249,6 +269,7 @@ static int process_snapshot(const char *path, const u8 
*uuid, u64 ctransid,
ret = -EINVAL;
goto out;
}
+   reset_progress(rctx, "Snapshot");
 
if (*rctx->dest_dir_path == 0) {
strncpy_null(rctx->cur_subvol_path, path);
@@ -388,6 +409,73 @@ out:
return ret;
 }
 
+static int process_total_data_size(u64 size, void *user)
+{
+   struct btrfs_receive *rctx = user;
+
+   rctx->total_data_size = size;
+   fprintf(stdout, "About to receive %llu bytes\n", size);
+
+   return 0;
+}
+
+static void update_progress(struct btrfs_receive *rctx, u64 bytes)
+{
+   float new_progress;
+   time_t now;
+   time_t tdiff;
+
+   if (rctx->total_data_size == 0)
+   return;
+
+   rctx->bytes_received += bytes;
+
+   now = time(NULL);
+   tdiff = now - rctx->last_progress_update;
+   if (tdiff < 1) {
+   if (rctx->bytes_received == rctx->total_data_size)
+   fprintf(stdout, "\n");
+   return;
+   }
+
+   new_progress = ((float)rctx->bytes_received / rctx->total_data_size) * 
100.0;
+
+   if ((int)(new_progress * 100) > (int)(rctx->progress * 100) ||
+   rctx->bytes_received == rctx->total_data_size) {
+   char line[5000];
+   float rate = rctx->bytes_received - 
rctx->bytes_received_last_update;
+   const char *rate_units;
+
+   rate /= tdiff;
+   if (rate > (1024 * 1024)) {
+   rate_units = "MB/s";
+   rate /= 1024 * 1024;
+   } else if (rate > 1024) {
+   rate_units = "KB/s";
+   rate /= 1024;
+   } else {
+ 

[RFC PATCH 1/6] Btrfs-progs: send, bump stream version

[Howard McLauchlan]

From: Filipe Manana 

This increases the send stream version from version 1 to version 2, adding
new commands:

1) total data size - used to tell the receiver how much file data the stream
   will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files;

3) inode set flags;

4) set inode otime.

This is preparation work for subsequent changes that implement the new features.

This doesn't break compatibility with older kernels or clients. In order to get
a version 2 send stream, new flags must be passed to the send ioctl.

Signed-off-by: Filipe David Borba Manana 
---
 cmds-send.c   | 61 ---
 ioctl.h   | 15 +
 send-stream.c |  2 +-
 send.h| 23 ++-
 4 files changed, 81 insertions(+), 20 deletions(-)

diff --git a/cmds-send.c b/cmds-send.c
index c5ecdaa1..0ec557c7 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -52,6 +52,7 @@
  * the 'At subvol' message.
  */
 static int g_verbose = 1;
+static int g_stream_version = BTRFS_SEND_STREAM_VERSION_1;
 
 struct btrfs_send {
int send_fd;
@@ -343,6 +344,8 @@ static int do_send(struct btrfs_send *send, u64 
parent_root_id,
io_send.flags |= BTRFS_SEND_FLAG_OMIT_STREAM_HEADER;
if (!is_last_subvol)
io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
+   if (g_stream_version == BTRFS_SEND_STREAM_VERSION_2)
+   io_send.flags |= BTRFS_SEND_FLAG_STREAM_V2;
ret = ioctl(subvol_fd, BTRFS_IOC_SEND, _send);
if (ret < 0) {
ret = -errno;
@@ -513,7 +516,8 @@ int cmd_send(int argc, char **argv)
static const struct option long_options[] = {
{ "verbose", no_argument, NULL, 'v' },
{ "quiet", no_argument, NULL, 'q' },
-   { "no-data", no_argument, NULL, GETOPT_VAL_SEND_NO_DATA 
}
+   { "no-data", no_argument, NULL, GETOPT_VAL_SEND_NO_DATA 
},
+   { "stream-version", 1, NULL, 'V' },
};
int c = getopt_long(argc, argv, "vqec:f:i:p:", long_options, 
NULL);
 
@@ -597,6 +601,24 @@ int cmd_send(int argc, char **argv)
error("option -i was removed, use -c instead");
ret = 1;
goto out;
+   case 'V':
+   if (sscanf(optarg, "%d", _stream_version) != 1) {
+   fprintf(stderr,
+   "ERROR: invalid value for stream 
version: %s\n",
+   optarg);
+   ret = 1;
+   goto out;
+   }
+   if (g_stream_version <= 0 ||
+   g_stream_version > BTRFS_SEND_STREAM_VERSION_MAX) {
+   fprintf(stderr,
+   "ERROR: unsupported stream version %d, 
minimum: 1, maximum: %d\n",
+   g_stream_version,
+   BTRFS_SEND_STREAM_VERSION_MAX);
+   ret = 1;
+   goto out;
+   }
+   break;
case GETOPT_VAL_SEND_NO_DATA:
send_flags |= BTRFS_SEND_FLAG_NO_FILE_DATA;
break;
@@ -776,7 +798,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-   "btrfs send [-ve] [-p ] [-c ] [-f ] 
 [...]",
+   "btrfs send [-ve] [--stream-version ] [-p ] [-c 
] [-f ]  [...]",
"Send the subvolume(s) to stdout.",
"Sends the subvolume(s) specified by  to stdout.",
" should be read-only here.",
@@ -790,21 +812,24 @@ const char * const cmd_send_usage[] = {
"which case 'btrfs send' will determine a suitable parent among the",
"clone sources itself.",
"\n",
-   "-e   If sending multiple subvols at once, use the new",
-   " format and omit the end-cmd between the subvols.",
-   "-p   Send an incremental stream from  to",
-   " .",
-   "-cUse this snapshot as a clone source for an ",
-   " incremental send (multiple allowed)",
-   "-f  Output is normally written to stdout. To write to",
-   " a file, use this option. An alternative would be to",
-   " use pipes.",
-   "--no-datasend in NO_FILE_DATA mode, Note: the output stream",
-   " does not contain any file data and thus cannot be 
used",
-   " to transfer changes. This mode is faster and useful 
to",
-   " show the differences in metadata.",
-   "-v|--verbose enable verbose output to stderr, each occurrence of",
-   "

[RFC PATCH 2/6] btrfs: send, implement total data size command to allow for progress estimation

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

This new send flag makes send calculate first the amount of new file data (in 
bytes)
the send root has relatively to the parent root, or for the case of a 
non-incremental
send, the total amount of file data the stream will create (including holes and 
prealloc
extents). In other words, it computes the sum of the lengths of all write, 
clone and
fallocate operations that will be sent through the send stream.

This data size value is sent in a new command, named 
BTRFS_SEND_C_TOTAL_DATA_SIZE, that
immediately follows a BTRFS_SEND_C_SUBVOL or BTRFS_SEND_C_SNAPSHOT command, and 
precedes
any command that changes a file or the filesystem hierarchy. Upon receiving a 
write, clone
or fallocate command, the receiving end can increment a counter by the data 
length of that
command and therefore report progress by comparing the counter's value with the 
data size
value received in the BTRFS_SEND_C_TOTAL_DATA_SIZE command.

The approach is simple, before the normal operation of send, do a scan in the 
file system
tree for new inodes and new/changed file extent items, just like in send's 
normal operation,
and keep incrementing a counter with new inodes' size and the size of file 
extents (and file
holes)  that are going to be written, cloned or fallocated. This is actually a 
simpler and
more lightweight tree scan/processing than the one we do when sending the 
changes, as it
doesn't process inode references nor does any lookups in the extent tree for 
example.

After modifying btrfs-progs to understand this new command and report progress, 
here's an
example (the -o flag tells btrfs send to pass the new flag to the kernel's send 
ioctl):

$ btrfs send -s --stream-version 2 /mnt/sdd/snap_base | btrfs receive 
/mnt/sdc
At subvol /mnt/sdd/snap_base
At subvol snap_base
About to receive 9212392667 bytes
Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 
44.07%, 40.32MB/s

$ btrfs send -s --stream-version 2 -p /mnt/sdd/snap_base /mnt/sdd/snap_incr 
| btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_incr
At subvol snap_incr
About to receive 9571342213 bytes
Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 
68.51%, 51.04MB/s

[Howard: rebased on 4.17-rc4]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
---
 fs/btrfs/send.c | 189 
 1 file changed, 157 insertions(+), 32 deletions(-)

diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index eccd69387065..7b184831812b 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -66,7 +66,13 @@ struct clone_root {
 #define SEND_CTX_MAX_NAME_CACHE_SIZE 128
 #define SEND_CTX_NAME_CACHE_CLEAN_SIZE (SEND_CTX_MAX_NAME_CACHE_SIZE * 2)
 
+enum btrfs_send_phase {
+   SEND_PHASE_STREAM_CHANGES,
+   SEND_PHASE_COMPUTE_DATA_SIZE,
+};
+
 struct send_ctx {
+   enum btrfs_send_phase phase;
struct file *send_filp;
loff_t send_off;
char *send_buf;
@@ -102,6 +108,7 @@ struct send_ctx {
u64 cur_inode_next_write_offset;
 
u64 send_progress;
+   u64 total_data_size;
 
struct list_head new_refs;
struct list_head deleted_refs;
@@ -709,6 +716,8 @@ static int send_rename(struct send_ctx *sctx,
struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
int ret;
 
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
btrfs_debug(fs_info, "send_rename %s -> %s", from->start, to->start);
 
ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);
@@ -758,6 +767,8 @@ static int send_unlink(struct send_ctx *sctx, struct 
fs_path *path)
struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
int ret;
 
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
btrfs_debug(fs_info, "send_unlink %s", path->start);
 
ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);
@@ -780,6 +791,7 @@ static int send_rmdir(struct send_ctx *sctx, struct fs_path 
*path)
 {
struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
int ret;
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
 
btrfs_debug(fs_info, "send_rmdir %s", path->start);
 
@@ -2419,6 +2431,8 @@ static int send_truncate(struct send_ctx *sctx, u64 ino, 
u64 gen, u64 size)
int ret = 0;
struct fs_path *p;
 
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
btrfs_debug(fs_info, "send_truncate %llu size=%llu", ino, size);
 
p = fs_path_alloc();
@@ -2449,6 +2463,8 @@ static int send_chmod(struct send_ctx *sctx, u64 ino, u64 
gen, u64 mode)
int ret = 0;
struct fs_path *p;
 
+   ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
btrfs_debug(fs_info, "send_chmod %llu mode=%llu", i

[RFC PATCH 3/6] btrfs: send, use fallocate command to punch holes

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

Instead of sending a write command with a data buffer filled with 0 value bytes,
use the fallocate command, introduced in the send stream version 2, to tell the
receiver to punch a file hole using the fallocate system call.

[Howard: rebased on 4.17-rc4]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
---
 fs/btrfs/send.c | 54 ++---
 fs/btrfs/send.h |  4 
 2 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index 7b184831812b..328c7a2857ae 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -581,6 +581,7 @@ static int tlv_put(struct send_ctx *sctx, u16 attr, const 
void *data, int len)
return tlv_put(sctx, attr, &__tmp, sizeof(__tmp));  \
}
 
+TLV_PUT_DEFINE_INT(32)
 TLV_PUT_DEFINE_INT(64)
 
 static int tlv_put_string(struct send_ctx *sctx, u16 attr,
@@ -5047,17 +5048,57 @@ static int send_update_extent(struct send_ctx *sctx,
return ret;
 }
 
+static int send_fallocate(struct send_ctx *sctx, u32 flags,
+ u64 offset, u64 len)
+{
+   struct fs_path *p = NULL;
+   int ret = 0;
+
+   ASSERT(sctx->flags & BTRFS_SEND_FLAG_STREAM_V2);
+
+   if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+   sctx->total_data_size += len;
+   return 0;
+   }
+
+   p = fs_path_alloc();
+   if (!p)
+   return -ENOMEM;
+   ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
+   if (ret < 0)
+   goto out;
+
+   ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE);
+   if (ret < 0)
+   goto out;
+   TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
+   TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_FLAGS, flags);
+   TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
+   TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
+   ret = send_cmd(sctx);
+
+tlv_put_failure:
+out:
+   fs_path_free(p);
+   return ret;
+}
+
+
 static int send_hole(struct send_ctx *sctx, u64 end)
 {
struct fs_path *p = NULL;
u64 offset = sctx->cur_inode_last_extent;
-   u64 len;
+   u64 len = end - offset;
int ret = 0;
 
if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-   sctx->total_data_size += end - offset;
+   sctx->total_data_size += len;
return 0;
}
+   if (sctx->flags & BTRFS_SEND_FLAG_STREAM_V2)
+   return send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+ offset, len);
+
if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
return send_update_extent(sctx, offset, end - offset);
 
@@ -5304,7 +5345,8 @@ static int send_write_or_clone(struct send_ctx *sctx,
ret = 0;
goto out;
}
-   if (offset + len > sctx->cur_inode_size)
+   if (offset < sctx->cur_inode_size &&
+   offset + len > sctx->cur_inode_size)
len = sctx->cur_inode_size - offset;
if (len == 0) {
ret = 0;
@@ -5325,6 +5367,12 @@ static int send_write_or_clone(struct send_ctx *sctx,
data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
ret = clone_range(sctx, clone_root, disk_byte, data_offset,
  offset, len);
+   } else if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0 &&
+type != BTRFS_FILE_EXTENT_INLINE &&
+(sctx->flags & BTRFS_SEND_FLAG_STREAM_V2) &&
+offset < sctx->cur_inode_size) {
+   ret = send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+offset, len);
} else {
ret = send_extent_data(sctx, offset, len);
}
diff --git a/fs/btrfs/send.h b/fs/btrfs/send.h
index a9b5489d690e..a5830d216ac1 100644
--- a/fs/btrfs/send.h
+++ b/fs/btrfs/send.h
@@ -138,6 +138,10 @@ enum {
 #define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
 #define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
 
+#define BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS\
+   (BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE |  \
+BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args 
*arg);
 #endif
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH 5/6] btrfs: add send_stream_version attribute to sysfs

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

So that applications can find out what's the highest send stream
version supported/implemented by the running kernel:

$ cat /sys/fs/btrfs/send/stream_version
2

[Howard: rebased on 4.17-rc4]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
Reviewed-by: David Sterba <dste...@suse.cz>
---
 fs/btrfs/send.h  |  1 +
 fs/btrfs/sysfs.c | 29 +
 2 files changed, 30 insertions(+)

diff --git a/fs/btrfs/send.h b/fs/btrfs/send.h
index a5830d216ac1..2f5e25e03def 100644
--- a/fs/btrfs/send.h
+++ b/fs/btrfs/send.h
@@ -12,6 +12,7 @@
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
 #define BTRFS_SEND_STREAM_VERSION_1 1
 #define BTRFS_SEND_STREAM_VERSION_2 2
+#define BTRFS_SEND_STREAM_VERSION_LATEST BTRFS_SEND_STREAM_VERSION_2
 
 #define BTRFS_SEND_BUF_SIZE SZ_64K
 #define BTRFS_SEND_READ_SIZE (48 * SZ_1K)
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 4848a4318fb5..3c82cba91ff6 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -18,6 +18,7 @@
 #include "transaction.h"
 #include "sysfs.h"
 #include "volumes.h"
+#include "send.h"
 
 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj);
 static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj);
@@ -884,6 +885,28 @@ static int btrfs_init_debugfs(void)
return 0;
 }
 
+static ssize_t send_stream_version_show(struct kobject *kobj,
+   struct kobj_attribute *a,
+   char *buf)
+{
+   return snprintf(buf, PAGE_SIZE, "%d\n",
+   BTRFS_SEND_STREAM_VERSION_LATEST);
+}
+
+BTRFS_ATTR(, stream_version, send_stream_version_show);
+
+static struct attribute *btrfs_send_attrs[] = {
+   BTRFS_ATTR_PTR(, stream_version),
+   NULL
+};
+
+static const struct attribute_group btrfs_send_attr_group = {
+   .name = "send",
+   .attrs = btrfs_send_attrs,
+};
+
+
+
 int __init btrfs_init_sysfs(void)
 {
int ret;
@@ -900,8 +923,13 @@ int __init btrfs_init_sysfs(void)
ret = sysfs_create_group(_kset->kobj, _feature_attr_group);
if (ret)
goto out2;
+   ret = sysfs_create_group(_kset->kobj, _send_attr_group);
+   if (ret)
+   goto out3;
 
return 0;
+out3:
+   sysfs_remove_group(_kset->kobj, _feature_attr_group);
 out2:
debugfs_remove_recursive(btrfs_debugfs_root_dentry);
 out1:
@@ -913,6 +941,7 @@ int __init btrfs_init_sysfs(void)
 void __cold btrfs_exit_sysfs(void)
 {
sysfs_remove_group(_kset->kobj, _feature_attr_group);
+   sysfs_remove_group(_kset->kobj, _send_attr_group);
kset_unregister(btrfs_kset);
debugfs_remove_recursive(btrfs_debugfs_root_dentry);
 }
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH 6/6] btrfs: add chattr support for send/receive

[Howard McLauchlan]

From: Howard McLauchlan <hmclauch...@fb.com>

Presently btrfs send/receive does not propagate inode attribute flags;
all chattr operations are effectively discarded upon transmission.

This patch adds kernel support for inode attribute flags. Userspace
support can be found under the commit:

btrfs-progs: add chattr support for send/receive

An associated xfstest can be found at:

btrfs: add verify chattr support for send/receive test

These changes are only enabled for send stream version 2

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 fs/btrfs/ctree.h |   2 +
 fs/btrfs/ioctl.c |   2 +-
 fs/btrfs/send.c  | 181 ---
 fs/btrfs/send.h  |   4 +-
 4 files changed, 159 insertions(+), 30 deletions(-)

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 2771cc56a622..0a2359144b18 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -1461,6 +1461,8 @@ struct btrfs_map_token {
unsigned long offset;
 };
 
+unsigned int btrfs_flags_to_ioctl(unsigned int flags);
+
 #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \
((bytes) >> (fs_info)->sb->s_blocksize_bits)
 
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index 632e26d6f7ce..36ce1e589f9e 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, unsigned 
int flags)
 /*
  * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl.
  */
-static unsigned int btrfs_flags_to_ioctl(unsigned int flags)
+unsigned int btrfs_flags_to_ioctl(unsigned int flags)
 {
unsigned int iflags = 0;
 
diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index c8ea1ccaa3d8..fa7db1474a7f 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -108,6 +108,13 @@ struct send_ctx {
u64 cur_inode_last_extent;
u64 cur_inode_next_write_offset;
 
+   /*
+* state for chattr purposes
+*/
+   u64 cur_inode_flip_flags;
+   u64 cur_inode_receive_flags;
+   int receive_flags_valid;
+
u64 send_progress;
u64 total_data_size;
 
@@ -815,7 +822,7 @@ static int send_rmdir(struct send_ctx *sctx, struct fs_path 
*path)
  */
 static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path,
  u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid,
- u64 *gid, u64 *rdev)
+ u64 *gid, u64 *rdev, u64 *flags)
 {
int ret;
struct btrfs_inode_item *ii;
@@ -845,6 +852,8 @@ static int __get_inode_info(struct btrfs_root *root, struct 
btrfs_path *path,
*gid = btrfs_inode_gid(path->nodes[0], ii);
if (rdev)
*rdev = btrfs_inode_rdev(path->nodes[0], ii);
+   if (flags)
+   *flags = btrfs_inode_flags(path->nodes[0], ii);
 
return ret;
 }
@@ -852,7 +861,7 @@ static int __get_inode_info(struct btrfs_root *root, struct 
btrfs_path *path,
 static int get_inode_info(struct btrfs_root *root,
  u64 ino, u64 *size, u64 *gen,
  u64 *mode, u64 *uid, u64 *gid,
- u64 *rdev)
+ u64 *rdev, u64 *flags)
 {
struct btrfs_path *path;
int ret;
@@ -861,7 +870,7 @@ static int get_inode_info(struct btrfs_root *root,
if (!path)
return -ENOMEM;
ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid,
-  rdev);
+  rdev, flags);
btrfs_free_path(path);
return ret;
 }
@@ -1250,7 +1259,7 @@ static int __iterate_backrefs(u64 ino, u64 offset, u64 
root, void *ctx_)
 * accept clones from these extents.
 */
ret = __get_inode_info(found->root, bctx->path, ino, _size, NULL, 
NULL,
-  NULL, NULL, NULL);
+  NULL, NULL, NULL, NULL);
btrfs_release_path(bctx->path);
if (ret < 0)
return ret;
@@ -1610,7 +1619,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 
ino, u64 gen)
u64 right_gen;
 
ret = get_inode_info(sctx->send_root, ino, NULL, _gen, NULL, NULL,
-   NULL, NULL);
+   NULL, NULL, NULL);
if (ret < 0 && ret != -ENOENT)
goto out;
left_ret = ret;
@@ -1619,7 +1628,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 
ino, u64 gen)
right_ret = -ENOENT;
} else {
ret = get_inode_info(sctx->parent_root, ino, NULL, _gen,
-   NULL, NULL, NULL, NULL);
+   NULL, NULL, NULL, NULL, NULL);
if (ret < 0 && ret != -ENOENT)
goto out;
right_ret = ret;
@@ -1788,7 +1797,7 @@ static int get_first_ref(struct btrfs_root *roo

[RFC PATCH 0/6] btrfs send stream version 2

[Howard McLauchlan]

In trying to implement support for inode flags in send/receive, the need for
proper versioning/compatibility came up. I found some of Filipe's old patches
[1] for send stream v2 and rebased them on 4.17-rc4. My chattr changes are
landed on top and also gated behind send stream v2. Similar was done for
btrfs-progs (v4.16.1) [2] and a relevant xfstest (master) [3].

Unfortunately, since Filipe's changes are about 4 years old, rebasing required
quite a few "I guess this is how it works" guesses. A critical eye would be
greatly appreciated.

As of 4.17-rc4, commit a6aa10c70bf7 ("Btrfs: send, fix missing truncate for
inode with prealloc extent past eof") is causing some strange behaviour with the
rebased changes (this can be best seen in the xfstest output for btrfs/160).
Specifically, the hole/data structure is consistent between sender/receiver, but
the receiver is missing some information in fiemap for prealloc extents past
eof. Filipe mentioned that his fix was considering the lack of fallocate command
in send, so that's something we can look at if this patch set gets anywhere.

A few things I was unsure about:

- I couldn't use open_inode_for_write() in process_chattr() in btrfs-progs.
  Ended up having to open() with O_RDONLY. This is probably because I was
  setting immutable on the inode for my test cases.
- Filipe's original patches had BTRFS_SEND_{A/C}_INODE_FLAGS as placeholders,
  but I'd already implemented everything as BTRFS_SEND_{A/C}_CHATTR, since
  send_chown(), send_chmod(), etc. seemed to set a precedent in naming. If this
  needs to be changed let me know:

As of v4.17-rc4, these changes pass all "send" group xfstests.

Cheers, Howard

1: https://www.spinics.net/lists/linux-btrfs/msg35169.html
2: https://patchwork.kernel.org/patch/4021491/
3: https://patchwork.kernel.org/patch/4004861/

Filipe David Borba Manana (5):
  btrfs: send, bump stream version
  btrfs: send, implement total data size command to allow for progress
estimation
  btrfs: send, use fallocate command to punch holes
  btrfs: send, use fallocate command to allocate extents
  btrfs: add send_stream_version attribute to sysfs

Howard McLauchlan (1):
  btrfs: add chattr support for send/receive

 fs/btrfs/ctree.h   |   2 +
 fs/btrfs/ioctl.c   |   2 +-
 fs/btrfs/send.c| 496 +++--
 fs/btrfs/send.h|  26 +-
 fs/btrfs/sysfs.c   |  29 +++
 include/uapi/linux/btrfs.h |  21 +-
 6 files changed, 495 insertions(+), 81 deletions(-)

-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH 4/6] btrfs: send, use fallocate command to allocate extents

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

The send stream version 2 adds the fallocate command, which can be used to
allocate extents for a file or punch holes in a file. Previously we were
ignoring file prealloc extents or treating them as extents filled with 0
bytes and sending a regular write command to the stream.

After this change, together with my previous change titled:

"Btrfs: send, use fallocate command to punch holes"

an incremental send preserves the hole and data structure of files, which can
be seen via calls to lseek with the whence parameter set to SEEK_DATA or 
SEEK_HOLE,
as the example below shows:

mkfs.btrfs -f /dev/sdc
mount /dev/sdc /mnt
xfs_io -f -c "pwrite -S 0x01 -b 30 0 30" /mnt/foo
btrfs subvolume snapshot -r /mnt /mnt/mysnap1

xfs_io -c "fpunch 10 5" /mnt/foo
xfs_io -c "falloc 10 5" /mnt/foo
xfs_io -c "pwrite -S 0xff -b 1000 12 1000" /mnt/foo
xfs_io -c "fpunch 25 2" /mnt/foo

# prealloc extents that start beyond the inode's size
xfs_io -c "falloc -k 30 100" /mnt/foo
xfs_io -c "falloc -k 900 200" /mnt/foo

btrfs subvolume snapshot -r /mnt /mnt/mysnap2

btrfs send /mnt/mysnap1 -f /tmp/1.snap
btrfs send -p /mnt/mysnap1 /mnt/mysnap2 -f /tmp/2.snap

mkfs.btrfs -f /dev/sdd
mount /dev/sdd /mnt2
btrfs receive /mnt2 -f /tmp/1.snap
btrfs receive /mnt2 -f /tmp/2.snap

Before this change the hole/data structure differed between both filesystems:

$ xfs_io -r -c 'seek -r -a 0' /mnt/mysnap2/foo
Whence  Result
DATA0
HOLE102400
DATA118784
HOLE122880
DATA147456
HOLE253952
DATA266240
HOLE30

$ xfs_io -r -c 'seek -r -a 0' /mnt2/mysnap2/foo
Whence  Result
DATA0
HOLE30

After this change the second filesystem (/dev/sdd) ends up with the same 
hole/data
structure as the first filesystem.

Also, after this change, prealloc extents that lie beyond the inode's size (were
allocated with fallocate + keep size flag) are also replicated by an incremental
send. For the above test, it can be observed via fiemap (or btrfs-debug-tree):

$ xfs_io -r -c 'fiemap -l' /mnt2/mysnap2/foo
0: [0..191]: 25096..25287 192 blocks
1: [192..199]: 24672..24679 8 blocks
2: [200..231]: 24584..24615 32 blocks
3: [232..239]: 24680..24687 8 blocks
4: [240..287]: 24616..24663 48 blocks
5: [288..295]: 24688..24695 8 blocks
6: [296..487]: 25392..25583 192 blocks
7: [488..495]: 24696..24703 8 blocks
8: [496..519]: hole 24 blocks
9: [520..527]: 24704..24711 8 blocks
10: [528..583]: 25624..25679 56 blocks
11: [584..591]: 24712..24719 8 blocks
12: [592..2543]: 26192..28143 1952 blocks
13: [2544..17575]: hole 15032 blocks
14: [17576..21487]: 28144..32055 3912 blocks

The proposed xfstest can be found at:

xfstests: btrfs, test send's ability to punch holes and prealloc extents

This test verifies that send-stream version 2 does space pre-allocation
and hole punching.

[Howard: rebased on 4.17-rc4]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
---
 fs/btrfs/send.c | 71 -
 1 file changed, 53 insertions(+), 18 deletions(-)

diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index 328c7a2857ae..c8ea1ccaa3d8 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -98,9 +98,10 @@ struct send_ctx {
 */
u64 cur_ino;
u64 cur_inode_gen;
-   int cur_inode_new;
-   int cur_inode_new_gen;
-   int cur_inode_deleted;
+   u8 cur_inode_new:1;
+   u8 cur_inode_new_gen:1;
+   u8 cur_inode_skip_truncate:1;
+   u8 cur_inode_deleted:1;
u64 cur_inode_size;
u64 cur_inode_mode;
u64 cur_inode_rdev;
@@ -5313,6 +5314,19 @@ static int clone_range(struct send_ctx *sctx,
return ret;
 }
 
+static int truncate_before_falloc(struct send_ctx *sctx)
+{
+   int ret = 0;
+
+   if (!sctx->cur_inode_skip_truncate) {
+   ret = send_truncate(sctx, sctx->cur_ino,
+   sctx->cur_inode_gen,
+   sctx->cur_inode_size);
+   sctx->cur_inode_skip_truncate = 1;
+   }
+   return ret;
+}
+
 static int send_write_or_clone(struct send_ctx *sctx,
   struct btrfs_path *path,
   struct btrfs_key *key,
@@ -5354,8 +5368,7 @@ static int send_write_or_clone(struct send_ctx *sctx,
}
 
if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-   if (offset < sctx->cur_inode_size)
-   sctx->total_data_size 

[RFC PATCH 1/6] btrfs: send, bump stream version

[Howard McLauchlan]

From: Filipe David Borba Manana <fdman...@gmail.com>

This increases the send stream version from version 1 to version 2, adding
new commands:

1) total data size - used to tell the receiver how much file data the stream
   will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files;

3) inode set flags;

4) set inode otime.

This is preparation work for subsequent changes that implement the new features.

A version 2 stream is only produced if the send ioctl caller passes in one of 
the
new flags (BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | BTRFS_SEND_FLAG_STREAM_V2), 
meaning
old clients are unaffected.

[Howard: rebased on 4.17-rc4]
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Signed-off-by: Filipe David Borba Manana <fdman...@gmail.com>
---
 fs/btrfs/send.c|  7 ++-
 fs/btrfs/send.h| 21 -
 include/uapi/linux/btrfs.h | 21 -
 3 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index c0074d2d7d6d..eccd69387065 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -649,7 +649,10 @@ static int send_header(struct send_ctx *sctx)
struct btrfs_stream_header hdr;
 
strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
-   hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION);
+   if (sctx->flags & BTRFS_SEND_FLAG_STREAM_V2)
+   hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_2);
+   else
+   hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_1);
 
return write_buf(sctx->send_filp, , sizeof(hdr),
>send_off);
@@ -6535,6 +6538,8 @@ long btrfs_ioctl_send(struct file *mnt_file, struct 
btrfs_ioctl_send_args *arg)
INIT_LIST_HEAD(>name_cache_list);
 
sctx->flags = arg->flags;
+   if (sctx->flags & BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE)
+   sctx->flags |= BTRFS_SEND_FLAG_STREAM_V2;
 
sctx->send_filp = fget(arg->send_fd);
if (!sctx->send_filp) {
diff --git a/fs/btrfs/send.h b/fs/btrfs/send.h
index ead397f7034f..a9b5489d690e 100644
--- a/fs/btrfs/send.h
+++ b/fs/btrfs/send.h
@@ -10,7 +10,8 @@
 #include "ctree.h"
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION_1 1
+#define BTRFS_SEND_STREAM_VERSION_2 2
 
 #define BTRFS_SEND_BUF_SIZE SZ_64K
 #define BTRFS_SEND_READ_SIZE (48 * SZ_1K)
@@ -77,6 +78,15 @@ enum btrfs_send_cmd {
 
BTRFS_SEND_C_END,
BTRFS_SEND_C_UPDATE_EXTENT,
+
+   /*
+* The following commands were added in stream version 2.
+*/
+   BTRFS_SEND_C_TOTAL_DATA_SIZE,
+   BTRFS_SEND_C_FALLOCATE,
+   BTRFS_SEND_C_INODE_SET_FLAGS,
+   BTRFS_SEND_C_UTIMES2, /* Same as UTIMES, but it includes OTIME too. */
+
__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -115,10 +125,19 @@ enum {
BTRFS_SEND_A_CLONE_OFFSET,
BTRFS_SEND_A_CLONE_LEN,
 
+   /*
+* The following attributes were added in stream version 2.
+*/
+   BTRFS_SEND_A_FALLOCATE_FLAGS,
+   BTRFS_SEND_A_INODE_FLAGS,
+
__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args 
*arg);
 #endif
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
index c8d99b9ca550..ed63176660d2 100644
--- a/include/uapi/linux/btrfs.h
+++ b/include/uapi/linux/btrfs.h
@@ -711,10 +711,29 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD   0x4
 
+/*
+ * Calculate the amount (in bytes) of new file data between the send and
+ * parent snapshots, or in case of a full send, the total amount of file data
+ * we will send.
+ * This corresponds to the sum of the data lengths of each write, clone and
+ * fallocate commands that are sent through the send stream. The receiving end
+ * can use this information to compute progress.
+ *
+ * Added in send stream version 2, and implies producing a version 2 stream.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE0x8
+
+/*
+ * Used by a client to request a version 2 of the send stream.
+ */
+#define BTRFS_SEND_FLAG_STREAM_V2  0x10
+
 #define BTRFS_SEND_FLAG_MASK \
(BTRFS_SEND_FLAG_NO_FILE_DATA | \
 BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \
-BTRFS_SEND_FLAG_OMIT_END_CMD)
+BTRFS_SEND_FLAG_OMIT_END_CMD | \
+BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | \
+BTRFS_SEND_FLAG_STREAM_V2)
 
 struct btrfs_ioctl_send_args {
__s64 send_fd;  /* in */
-- 
2.17.0

--
To unsubscribe from this list: send the line "un

Re: [PATCH] btrfs: add chattr support for send/receive

[Howard McLauchlan]

On 04/18/2018 01:15 AM, Filipe Manana wrote:
> On Wed, Apr 18, 2018 at 12:39 AM, Howard McLauchlan <hmclauch...@fb.com> 
> wrote:
>> Presently btrfs send/receive does not propagate inode attribute flags;
>> all chattr operations are effectively discarded upon transmission.
>>
>> This patch adds kernel support for inode attribute flags. Userspace
>> support can be found under the commit:
>>
>> btrfs-progs: add chattr support for send/receive
>>
>> An associated xfstest can be found at:
>>
>> btrfs: add verify chattr support for send/receive test
>>
>> A caveat is that a user with an updated kernel (aware of chattrs) and an
>> older version of btrfs-progs (unaware of chattrs) will fail to receive
>> if a chattr is included in the send stream.
> So we do have several things missing in send besides attribute flags,
> like hole punching for example (there's a list on the wiki).
> We can't just add a new command and introduce such caveat every time
> we implement one of the missing and desired features.
>
> In 2014, while wanting to implement some of those features, I
> introduced a way to bump the send stream version with room (commands)
> for all those missing features, so that all could be implemented later
> without adding further backward incompatibility between kernel
> versions btrfs-progs versions.
> Some of the threads for reference:
>
> https://patchwork.kernel.org/patch/4021491/
> https://urldefense.proofpoint.com/v2/url?u=https-3A__www.spinics.net_lists_linux-2Dbtrfs_msg35169.html=DwIFaQ=5VD0RTtNlTh3ycd41b3MUw=UA4c4GV4shA70-jKB4kwcF99U6K6bzKVYdicFvu-DtQ=pWSiTBXI54TJfXnctkAeLJUbyIs9VZqupmGKs8JsETE=rsgeLq1QeHGaRs6xIXtGcGV-UgjCuqnWMATCw_KaxY8=
>
> It never took off, and honestly I don't remember why as no one add
> more comments on the latest versions of the kernel and btrfs-progs
> patchsets.
Thanks for the heads up, I'll go dig up your patches and try to get them 
working on 4.17

Howard

>
>> Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
>> ---
>> Based on 4.17-rc1
>>
>>  fs/btrfs/ctree.h |   2 +
>>  fs/btrfs/ioctl.c |   2 +-
>>  fs/btrfs/send.c  | 176 +++
>>  fs/btrfs/send.h  |   2 +
>>  4 files changed, 154 insertions(+), 28 deletions(-)
>>
>> diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
>> index 5474ef14d6e6..a0dc6a8a37eb 100644
>> --- a/fs/btrfs/ctree.h
>> +++ b/fs/btrfs/ctree.h
>> @@ -1436,6 +1436,8 @@ struct btrfs_map_token {
>> unsigned long offset;
>>  };
>>
>> +unsigned int btrfs_flags_to_ioctl(unsigned int flags);
>> +
>>  #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \
>> ((bytes) >> (fs_info)->sb->s_blocksize_bits)
>>
>> diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
>> index 632e26d6f7ce..36ce1e589f9e 100644
>> --- a/fs/btrfs/ioctl.c
>> +++ b/fs/btrfs/ioctl.c
>> @@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, 
>> unsigned int flags)
>>  /*
>>   * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl.
>>   */
>> -static unsigned int btrfs_flags_to_ioctl(unsigned int flags)
>> +unsigned int btrfs_flags_to_ioctl(unsigned int flags)
>>  {
>> unsigned int iflags = 0;
>>
>> diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
>> index 221e5cdb060b..da521a5a1843 100644
>> --- a/fs/btrfs/send.c
>> +++ b/fs/btrfs/send.c
>> @@ -101,6 +101,13 @@ struct send_ctx {
>> u64 cur_inode_last_extent;
>> u64 cur_inode_next_write_offset;
>>
>> +   /*
>> +* state for chattr purposes
>> +*/
>> +   u64 cur_inode_flip_flags;
>> +   u64 cur_inode_receive_flags;
>> +   int receive_flags_valid;
>> +
>> u64 send_progress;
>>
>> struct list_head new_refs;
>> @@ -798,7 +805,7 @@ static int send_rmdir(struct send_ctx *sctx, struct 
>> fs_path *path)
>>   */
>>  static int __get_inode_info(struct btrfs_root *root, struct btrfs_path 
>> *path,
>>   u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid,
>> - u64 *gid, u64 *rdev)
>> + u64 *gid, u64 *rdev, u64 *flags)
>>  {
>> int ret;
>> struct btrfs_inode_item *ii;
>> @@ -828,6 +835,8 @@ static int __get_inode_info(struct btrfs_root *root, 
>> struct btrfs_path *path,
>> *gid = btrfs_inode_gid(path->nodes[0], ii);
>> if (rdev)
>> *rdev = btrfs

[PATCH v2 3/3] btrfs: remove le_test_bit()

[Howard McLauchlan]

With commit b18253ec57c0 ("btrfs: optimize free space tree bitmap
conversion"), there are no more callers to le_test_bit(). This patch
removes le_test_bit().

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 fs/btrfs/extent_io.h | 5 -
 1 file changed, 5 deletions(-)

diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index d34416c831bf..c5e80d60d71b 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -79,11 +79,6 @@
 #define BITMAP_LAST_BYTE_MASK(nbits) \
(BYTE_MASK >> (-(nbits) & (BITS_PER_BYTE - 1)))
 
-static inline int le_test_bit(int nr, const u8 *addr)
-{
-   return 1U & (addr[BIT_BYTE(nr)] >> (nr & (BITS_PER_BYTE-1)));
-}
-
 struct extent_state;
 struct btrfs_root;
 struct btrfs_inode;
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2 1/3] btrfs: clean up le_bitmap_{set, clear}()

[Howard McLauchlan]

le_bitmap_set() is only used by free-space-tree, so move it there and
make it static. le_bitmap_clear() is not used, so remove it.

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
V1->V2: Also move le_bitmap_set()
based on 4.17-rc1
 fs/btrfs/extent_io.c   | 40 --
 fs/btrfs/extent_io.h   |  3 ---
 fs/btrfs/free-space-tree.c | 20 +++
 3 files changed, 20 insertions(+), 43 deletions(-)

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index e99b329002cf..9a521e5e297d 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -5620,46 +5620,6 @@ void copy_extent_buffer(struct extent_buffer *dst, 
struct extent_buffer *src,
}
 }
 
-void le_bitmap_set(u8 *map, unsigned int start, int len)
-{
-   u8 *p = map + BIT_BYTE(start);
-   const unsigned int size = start + len;
-   int bits_to_set = BITS_PER_BYTE - (start % BITS_PER_BYTE);
-   u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
-
-   while (len - bits_to_set >= 0) {
-   *p |= mask_to_set;
-   len -= bits_to_set;
-   bits_to_set = BITS_PER_BYTE;
-   mask_to_set = ~0;
-   p++;
-   }
-   if (len) {
-   mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
-   *p |= mask_to_set;
-   }
-}
-
-void le_bitmap_clear(u8 *map, unsigned int start, int len)
-{
-   u8 *p = map + BIT_BYTE(start);
-   const unsigned int size = start + len;
-   int bits_to_clear = BITS_PER_BYTE - (start % BITS_PER_BYTE);
-   u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(start);
-
-   while (len - bits_to_clear >= 0) {
-   *p &= ~mask_to_clear;
-   len -= bits_to_clear;
-   bits_to_clear = BITS_PER_BYTE;
-   mask_to_clear = ~0;
-   p++;
-   }
-   if (len) {
-   mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
-   *p &= ~mask_to_clear;
-   }
-}
-
 /*
  * eb_bitmap_offset() - calculate the page and offset of the byte containing 
the
  * given bit number
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index a53009694b16..d34416c831bf 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -84,9 +84,6 @@ static inline int le_test_bit(int nr, const u8 *addr)
return 1U & (addr[BIT_BYTE(nr)] >> (nr & (BITS_PER_BYTE-1)));
 }
 
-void le_bitmap_set(u8 *map, unsigned int start, int len);
-void le_bitmap_clear(u8 *map, unsigned int start, int len);
-
 struct extent_state;
 struct btrfs_root;
 struct btrfs_inode;
diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
index 32a0f6cb5594..e03830d83311 100644
--- a/fs/btrfs/free-space-tree.c
+++ b/fs/btrfs/free-space-tree.c
@@ -157,6 +157,26 @@ static u8 *alloc_bitmap(u32 bitmap_size)
return ret;
 }
 
+static void le_bitmap_set(u8 *map, unsigned int start, int len)
+{
+   u8 *p = map + BIT_BYTE(start);
+   const unsigned int size = start + len;
+   int bits_to_set = BITS_PER_BYTE - (start % BITS_PER_BYTE);
+   u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
+
+   while (len - bits_to_set >= 0) {
+   *p |= mask_to_set;
+   len -= bits_to_set;
+   bits_to_set = BITS_PER_BYTE;
+   mask_to_set = ~0;
+   p++;
+   }
+   if (len) {
+   mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
+   *p |= mask_to_set;
+   }
+}
+
 int convert_free_space_to_bitmaps(struct btrfs_trans_handle *trans,
  struct btrfs_fs_info *fs_info,
  struct btrfs_block_group_cache *block_group,
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2 2/3] btrfs: optimize free space tree bitmap conversion

[Howard McLauchlan]

Presently, convert_free_space_to_extents() does a linear scan of the
bitmap. We can speed this up with find_next_{bit,zero_bit}_le().

This patch replaces the linear scan with find_next_{bit,zero_bit}_le().
Testing shows a 20-33% decrease in execution time for
convert_free_space_to_extents().

Suggested-by: Omar Sandoval <osan...@osandov.com>
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
V1->V2: Round the bitmap size accordingly when allocating bitmap.

 fs/btrfs/free-space-tree.c | 61 ++
 1 file changed, 23 insertions(+), 38 deletions(-)

diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
index e03830d83311..7019afe6e727 100644
--- a/fs/btrfs/free-space-tree.c
+++ b/fs/btrfs/free-space-tree.c
@@ -138,10 +138,11 @@ static inline u32 free_space_bitmap_size(u64 size, u32 
sectorsize)
return DIV_ROUND_UP((u32)div_u64(size, sectorsize), BITS_PER_BYTE);
 }
 
-static u8 *alloc_bitmap(u32 bitmap_size)
+static unsigned long *alloc_bitmap(u32 bitmap_size)
 {
-   u8 *ret;
+   unsigned long *ret;
unsigned int nofs_flag;
+   u32 bitmap_rounded_size = round_up(bitmap_size, sizeof(unsigned long));
 
/*
 * GFP_NOFS doesn't work with kvmalloc(), but we really can't recurse
@@ -152,14 +153,14 @@ static u8 *alloc_bitmap(u32 bitmap_size)
 * know that recursion is unsafe.
 */
nofs_flag = memalloc_nofs_save();
-   ret = kvzalloc(bitmap_size, GFP_KERNEL);
+   ret = kvzalloc(bitmap_rounded_size, GFP_KERNEL);
memalloc_nofs_restore(nofs_flag);
return ret;
 }
 
-static void le_bitmap_set(u8 *map, unsigned int start, int len)
+static void le_bitmap_set(unsigned long *map, unsigned int start, int len)
 {
-   u8 *p = map + BIT_BYTE(start);
+   u8 *p = ((u8 *)map) + BIT_BYTE(start);
const unsigned int size = start + len;
int bits_to_set = BITS_PER_BYTE - (start % BITS_PER_BYTE);
u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
@@ -186,7 +187,8 @@ int convert_free_space_to_bitmaps(struct btrfs_trans_handle 
*trans,
struct btrfs_free_space_info *info;
struct btrfs_key key, found_key;
struct extent_buffer *leaf;
-   u8 *bitmap, *bitmap_cursor;
+   unsigned long *bitmap;
+   char *bitmap_cursor;
u64 start, end;
u64 bitmap_range, i;
u32 bitmap_size, flags, expected_extent_count;
@@ -275,7 +277,7 @@ int convert_free_space_to_bitmaps(struct btrfs_trans_handle 
*trans,
goto out;
}
 
-   bitmap_cursor = bitmap;
+   bitmap_cursor = (char *)bitmap;
bitmap_range = fs_info->sectorsize * BTRFS_FREE_SPACE_BITMAP_BITS;
i = start;
while (i < end) {
@@ -324,13 +326,10 @@ int convert_free_space_to_extents(struct 
btrfs_trans_handle *trans,
struct btrfs_free_space_info *info;
struct btrfs_key key, found_key;
struct extent_buffer *leaf;
-   u8 *bitmap;
+   unsigned long *bitmap;
u64 start, end;
-   /* Initialize to silence GCC. */
-   u64 extent_start = 0;
-   u64 offset;
u32 bitmap_size, flags, expected_extent_count;
-   int prev_bit = 0, bit, bitnr;
+   unsigned long nrbits, start_bit, end_bit;
u32 extent_count = 0;
int done = 0, nr;
int ret;
@@ -368,7 +367,7 @@ int convert_free_space_to_extents(struct btrfs_trans_handle 
*trans,
break;
} else if (found_key.type == 
BTRFS_FREE_SPACE_BITMAP_KEY) {
unsigned long ptr;
-   u8 *bitmap_cursor;
+   char *bitmap_cursor;
u32 bitmap_pos, data_size;
 
ASSERT(found_key.objectid >= start);
@@ -378,7 +377,7 @@ int convert_free_space_to_extents(struct btrfs_trans_handle 
*trans,
bitmap_pos = div_u64(found_key.objectid - start,
 fs_info->sectorsize *
 BITS_PER_BYTE);
-   bitmap_cursor = bitmap + bitmap_pos;
+   bitmap_cursor = ((char *)bitmap) + bitmap_pos;
data_size = 
free_space_bitmap_size(found_key.offset,
   
fs_info->sectorsize);
 
@@ -412,32 +411,16 @@ int convert_free_space_to_extents(struct 
btrfs_trans_handle *trans,
btrfs_mark_buffer_dirty(leaf);
btrfs_release_path(path);
 
-   offset = start;
-   bitnr = 0;
-   while (offset < end) {
-   bit = !!le_test_bit(bitnr, bitmap);
-   if (prev_bit == 0 && bit == 1) {
-   extent_start = offset;
-   } else if (prev_bit == 1 && bit == 0) {
- 

Re: [PATCH 2/2] btrfs: optimize free space tree bitmap conversion

[Howard McLauchlan]

On 04/18/2018 03:26 PM, David Sterba wrote:
> On Wed, Apr 18, 2018 at 02:30:59PM -0700, Howard McLauchlan wrote:
>> Presently, convert_free_space_to_extents() does a linear scan of the
>> bitmap. We can speed this up with find_next_{bit,zero_bit}_le().
>>
>> This patch replaces the linear scan with find_next_{bit,zero_bit}_le().
>> Testing shows a 20-33% decrease in execution time for
>> convert_free_space_to_extents().
>>
>> Suggested-by: Omar Sandoval <osan...@osandov.com>
>> Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
>> ---
>>
>> Since we change bitmap to be unsigned long, we have to do some casting for 
>> the
>> bitmap cursor. In le_bitmap_set() it makes sense to use u8, as we are doing
>> bit operations. Everywhere else, we're just using it for pointer arithmetic 
>> and
>> not directly accessing it, so char seems more appropriate.
> Ok, makes sense for just passing the pointers around. I'll add the text
> to changelog and apply the patch to next.
>
>> -bit = !!le_test_bit(bitnr, bitmap);
> This is the last use of le_test_bit, so it can be removed (in another
> patch).
I just found an issue in this patch that should be fixed. I'll address
moving le_bitmap_set(), le_test_bit and send a V2 for both these patches.

Howard
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 2/2] btrfs: optimize free space tree bitmap conversion

[Howard McLauchlan]

Presently, convert_free_space_to_extents() does a linear scan of the
bitmap. We can speed this up with find_next_{bit,zero_bit}_le().

This patch replaces the linear scan with find_next_{bit,zero_bit}_le().
Testing shows a 20-33% decrease in execution time for
convert_free_space_to_extents().

Suggested-by: Omar Sandoval <osan...@osandov.com>
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---

Since we change bitmap to be unsigned long, we have to do some casting for the
bitmap cursor. In le_bitmap_set() it makes sense to use u8, as we are doing
bit operations. Everywhere else, we're just using it for pointer arithmetic and
not directly accessing it, so char seems more appropriate.

Howard

 fs/btrfs/extent_io.c   |  4 +--
 fs/btrfs/extent_io.h   |  2 +-
 fs/btrfs/free-space-tree.c | 54 ++
 3 files changed, 22 insertions(+), 38 deletions(-)

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index cf50278f1c59..1d984b0acd66 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -5620,9 +5620,9 @@ void copy_extent_buffer(struct extent_buffer *dst, struct 
extent_buffer *src,
}
 }
 
-void le_bitmap_set(u8 *map, unsigned int start, int len)
+void le_bitmap_set(unsigned long *map, unsigned int start, int len)
 {
-   u8 *p = map + BIT_BYTE(start);
+   u8 *p = ((u8 *)map) + BIT_BYTE(start);
const unsigned int size = start + len;
int bits_to_set = BITS_PER_BYTE - (start % BITS_PER_BYTE);
u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index 4b8b072d9594..01261306e5f9 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -84,7 +84,7 @@ static inline int le_test_bit(int nr, const u8 *addr)
return 1U & (addr[BIT_BYTE(nr)] >> (nr & (BITS_PER_BYTE-1)));
 }
 
-void le_bitmap_set(u8 *map, unsigned int start, int len);
+void le_bitmap_set(unsigned long *map, unsigned int start, int len);
 
 struct extent_state;
 struct btrfs_root;
diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
index 32a0f6cb5594..0ddf96b01a3a 100644
--- a/fs/btrfs/free-space-tree.c
+++ b/fs/btrfs/free-space-tree.c
@@ -138,9 +138,9 @@ static inline u32 free_space_bitmap_size(u64 size, u32 
sectorsize)
return DIV_ROUND_UP((u32)div_u64(size, sectorsize), BITS_PER_BYTE);
 }
 
-static u8 *alloc_bitmap(u32 bitmap_size)
+static unsigned long *alloc_bitmap(u32 bitmap_size)
 {
-   u8 *ret;
+   unsigned long *ret;
unsigned int nofs_flag;
 
/*
@@ -166,7 +166,8 @@ int convert_free_space_to_bitmaps(struct btrfs_trans_handle 
*trans,
struct btrfs_free_space_info *info;
struct btrfs_key key, found_key;
struct extent_buffer *leaf;
-   u8 *bitmap, *bitmap_cursor;
+   unsigned long *bitmap;
+   char *bitmap_cursor;
u64 start, end;
u64 bitmap_range, i;
u32 bitmap_size, flags, expected_extent_count;
@@ -255,7 +256,7 @@ int convert_free_space_to_bitmaps(struct btrfs_trans_handle 
*trans,
goto out;
}
 
-   bitmap_cursor = bitmap;
+   bitmap_cursor = (char *)bitmap;
bitmap_range = fs_info->sectorsize * BTRFS_FREE_SPACE_BITMAP_BITS;
i = start;
while (i < end) {
@@ -304,13 +305,10 @@ int convert_free_space_to_extents(struct 
btrfs_trans_handle *trans,
struct btrfs_free_space_info *info;
struct btrfs_key key, found_key;
struct extent_buffer *leaf;
-   u8 *bitmap;
+   unsigned long *bitmap;
u64 start, end;
-   /* Initialize to silence GCC. */
-   u64 extent_start = 0;
-   u64 offset;
u32 bitmap_size, flags, expected_extent_count;
-   int prev_bit = 0, bit, bitnr;
+   unsigned long nrbits, start_bit, end_bit;
u32 extent_count = 0;
int done = 0, nr;
int ret;
@@ -348,7 +346,7 @@ int convert_free_space_to_extents(struct btrfs_trans_handle 
*trans,
break;
} else if (found_key.type == 
BTRFS_FREE_SPACE_BITMAP_KEY) {
unsigned long ptr;
-   u8 *bitmap_cursor;
+   char *bitmap_cursor;
u32 bitmap_pos, data_size;
 
ASSERT(found_key.objectid >= start);
@@ -358,7 +356,7 @@ int convert_free_space_to_extents(struct btrfs_trans_handle 
*trans,
bitmap_pos = div_u64(found_key.objectid - start,
 fs_info->sectorsize *
 BITS_PER_BYTE);
-   bitmap_cursor = bitmap + bitmap_pos;
+   bitmap_cursor = ((char *)bitmap) + bitmap_pos;
data_size = 
f

[PATCH 1/2] btrfs: remove le_bitmap_clear()

[Howard McLauchlan]

le_bitmap_clear() was implemented as a parallel to le_bitmap_set() but
is actually not being used. This patch removes it.

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 fs/btrfs/extent_io.c | 20 
 fs/btrfs/extent_io.h |  1 -
 2 files changed, 21 deletions(-)

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index e99b329002cf..cf50278f1c59 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -5640,26 +5640,6 @@ void le_bitmap_set(u8 *map, unsigned int start, int len)
}
 }
 
-void le_bitmap_clear(u8 *map, unsigned int start, int len)
-{
-   u8 *p = map + BIT_BYTE(start);
-   const unsigned int size = start + len;
-   int bits_to_clear = BITS_PER_BYTE - (start % BITS_PER_BYTE);
-   u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(start);
-
-   while (len - bits_to_clear >= 0) {
-   *p &= ~mask_to_clear;
-   len -= bits_to_clear;
-   bits_to_clear = BITS_PER_BYTE;
-   mask_to_clear = ~0;
-   p++;
-   }
-   if (len) {
-   mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
-   *p &= ~mask_to_clear;
-   }
-}
-
 /*
  * eb_bitmap_offset() - calculate the page and offset of the byte containing 
the
  * given bit number
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index a53009694b16..4b8b072d9594 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -85,7 +85,6 @@ static inline int le_test_bit(int nr, const u8 *addr)
 }
 
 void le_bitmap_set(u8 *map, unsigned int start, int len);
-void le_bitmap_clear(u8 *map, unsigned int start, int len);
 
 struct extent_state;
 struct btrfs_root;
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] btrfs: optimize free space tree bitmap conversion

[Howard McLauchlan]

On 04/18/2018 08:50 AM, David Sterba wrote:
> On Tue, Apr 17, 2018 at 04:19:04PM -0700, Howard McLauchlan wrote:
>> Presently, convert_free_space_to_extents() does a linear scan of the
>> bitmap. We can speed this up with find_next_{bit,zero_bit}_le().
>>
>> This patch replaces the linear scan with find_next_{bit,zero_bit}_le().
>> Testing shows a 20-33% decrease in execution time for
>> convert_free_space_to_extents().
> Sounds good.
>
>> Suggested-by: Omar Sandoval <osan...@osandov.com>
>> Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
>> ---
>> Based on 4.17-rc1
>>
>>  fs/btrfs/extent_io.c   | 12 -
>>  fs/btrfs/extent_io.h   |  4 +--
>>  fs/btrfs/free-space-tree.c | 54 ++
>>  3 files changed, 27 insertions(+), 43 deletions(-)
>>
>> diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
>> index e99b329002cf..1c0e7ce49556 100644
>> --- a/fs/btrfs/extent_io.c
>> +++ b/fs/btrfs/extent_io.c
>> @@ -5620,12 +5620,12 @@ void copy_extent_buffer(struct extent_buffer *dst, 
>> struct extent_buffer *src,
>>  }
>>  }
>>  
>> -void le_bitmap_set(u8 *map, unsigned int start, int len)
>> +void le_bitmap_set(unsigned long *map, unsigned int start, int len)
>>  {
>> -u8 *p = map + BIT_BYTE(start);
>> +char *p = ((char *)map) + BIT_BYTE(start);
>>  const unsigned int size = start + len;
>>  int bits_to_set = BITS_PER_BYTE - (start % BITS_PER_BYTE);
>> -u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
>> +char mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
> Why do you switch to char? As there are bit operations done below (that
> you don't change), the unsigned type seems correct.
>
>>  
>>  while (len - bits_to_set >= 0) {
>>  *p |= mask_to_set;
>> @@ -5640,12 +5640,12 @@ void le_bitmap_set(u8 *map, unsigned int start, int 
>> len)
>>  }
>>  }
>>  
>> -void le_bitmap_clear(u8 *map, unsigned int start, int len)
>> +void le_bitmap_clear(unsigned long *map, unsigned int start, int len)
>>  {
>> -u8 *p = map + BIT_BYTE(start);
>> +char *p = ((char *)map) + BIT_BYTE(start);
>>  const unsigned int size = start + len;
>>  int bits_to_clear = BITS_PER_BYTE - (start % BITS_PER_BYTE);
>> -u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(start);
>> +char mask_to_clear = BITMAP_FIRST_BYTE_MASK(start);
> Same here and in below. Otherwise it looks ok.
>
>>  
>>  while (len - bits_to_clear >= 0) {
>>  *p &= ~mask_to_clear;
>> diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
>> index a53009694b16..a6db233f4a40 100644
>> --- a/fs/btrfs/extent_io.h
>> +++ b/fs/btrfs/extent_io.h
>> @@ -84,8 +84,8 @@ static inline int le_test_bit(int nr, const u8 *addr)
>>  return 1U & (addr[BIT_BYTE(nr)] >> (nr & (BITS_PER_BYTE-1)));
>>  }
>>  
>> -void le_bitmap_set(u8 *map, unsigned int start, int len);
>> -void le_bitmap_clear(u8 *map, unsigned int start, int len);
>> +void le_bitmap_set(unsigned long *map, unsigned int start, int len);
>> +void le_bitmap_clear(unsigned long *map, unsigned int start, int len);
>>  
>>  struct extent_state;
>>  struct btrfs_root;
>> diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
>> index 32a0f6cb5594..0ddf96b01a3a 100644
>> --- a/fs/btrfs/free-space-tree.c
>> +++ b/fs/btrfs/free-space-tree.c
>> @@ -138,9 +138,9 @@ static inline u32 free_space_bitmap_size(u64 size, u32 
>> sectorsize)
>>  return DIV_ROUND_UP((u32)div_u64(size, sectorsize), BITS_PER_BYTE);
>>  }
>>  
>> -static u8 *alloc_bitmap(u32 bitmap_size)
>> +static unsigned long *alloc_bitmap(u32 bitmap_size)
>>  {
>> -u8 *ret;
>> +unsigned long *ret;
>>  unsigned int nofs_flag;
>>  
>>  /*
>> @@ -166,7 +166,8 @@ int convert_free_space_to_bitmaps(struct 
>> btrfs_trans_handle *trans,
>>  struct btrfs_free_space_info *info;
>>  struct btrfs_key key, found_key;
>>  struct extent_buffer *leaf;
>> -u8 *bitmap, *bitmap_cursor;
>> +unsigned long *bitmap;
>> +char *bitmap_cursor;
>>  u64 start, end;
>>  u64 bitmap_range, i;
>>  u32 bitmap_size, flags, expected_extent_count;
>> @@ -255,7 +256,7 @@ int convert_free_space_to_bitmaps(struct 
>> btrfs_trans_handle *trans,
>>  goto out;
>>  }
>>  
>> -bitmap_cursor = bitmap;
>> +bitmap_cursor = (char *

Re: [PATCH] btrfs: add chattr support for send/receive

[Howard McLauchlan]

On 04/17/2018 04:39 PM, Howard McLauchlan wrote:
> Presently btrfs send/receive does not propagate inode attribute flags;
> all chattr operations are effectively discarded upon transmission.
>
> This patch adds kernel support for inode attribute flags. Userspace
> support can be found under the commit:
>
> btrfs-progs: add chattr support for send/receive
>
> An associated xfstest can be found at:
>
> btrfs: add verify chattr support for send/receive test
>
> A caveat is that a user with an updated kernel (aware of chattrs) and an
> older version of btrfs-progs (unaware of chattrs) will fail to receive
> if a chattr is included in the send stream.
>
> Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
> ---
> Based on 4.17-rc1
>
>  fs/btrfs/ctree.h |   2 +
>  fs/btrfs/ioctl.c |   2 +-
>  fs/btrfs/send.c  | 176 +++
>  fs/btrfs/send.h  |   2 +
>  4 files changed, 154 insertions(+), 28 deletions(-)
>
> diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
> index 5474ef14d6e6..a0dc6a8a37eb 100644
> --- a/fs/btrfs/ctree.h
> +++ b/fs/btrfs/ctree.h
> @@ -1436,6 +1436,8 @@ struct btrfs_map_token {
>   unsigned long offset;
>  };
>  
> +unsigned int btrfs_flags_to_ioctl(unsigned int flags);
> +
>  #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \
>   ((bytes) >> (fs_info)->sb->s_blocksize_bits)
>  
> diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
> index 632e26d6f7ce..36ce1e589f9e 100644
> --- a/fs/btrfs/ioctl.c
> +++ b/fs/btrfs/ioctl.c
> @@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, 
> unsigned int flags)
>  /*
>   * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl.
>   */
> -static unsigned int btrfs_flags_to_ioctl(unsigned int flags)
> +unsigned int btrfs_flags_to_ioctl(unsigned int flags)
>  {
>   unsigned int iflags = 0;
>  
> diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
> index 221e5cdb060b..da521a5a1843 100644
> --- a/fs/btrfs/send.c
> +++ b/fs/btrfs/send.c
> @@ -101,6 +101,13 @@ struct send_ctx {
>   u64 cur_inode_last_extent;
>   u64 cur_inode_next_write_offset;
>  
> + /*
> +  * state for chattr purposes
> +  */
> + u64 cur_inode_flip_flags;
> + u64 cur_inode_receive_flags;
> + int receive_flags_valid;
> +
>   u64 send_progress;
>  
>   struct list_head new_refs;
> @@ -798,7 +805,7 @@ static int send_rmdir(struct send_ctx *sctx, struct 
> fs_path *path)
>   */
>  static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path,
> u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid,
> -   u64 *gid, u64 *rdev)
> +   u64 *gid, u64 *rdev, u64 *flags)
>  {
>   int ret;
>   struct btrfs_inode_item *ii;
> @@ -828,6 +835,8 @@ static int __get_inode_info(struct btrfs_root *root, 
> struct btrfs_path *path,
>   *gid = btrfs_inode_gid(path->nodes[0], ii);
>   if (rdev)
>   *rdev = btrfs_inode_rdev(path->nodes[0], ii);
> + if (flags)
> + *flags = btrfs_inode_flags(path->nodes[0], ii);
>  
>   return ret;
>  }
> @@ -835,7 +844,7 @@ static int __get_inode_info(struct btrfs_root *root, 
> struct btrfs_path *path,
>  static int get_inode_info(struct btrfs_root *root,
> u64 ino, u64 *size, u64 *gen,
> u64 *mode, u64 *uid, u64 *gid,
> -   u64 *rdev)
> +   u64 *rdev, u64 *flags)
>  {
>   struct btrfs_path *path;
>   int ret;
> @@ -844,7 +853,7 @@ static int get_inode_info(struct btrfs_root *root,
>   if (!path)
>   return -ENOMEM;
>   ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid,
> -rdev);
> +rdev, flags);
>   btrfs_free_path(path);
>   return ret;
>  }
> @@ -1233,7 +1242,7 @@ static int __iterate_backrefs(u64 ino, u64 offset, u64 
> root, void *ctx_)
>* accept clones from these extents.
>*/
>   ret = __get_inode_info(found->root, bctx->path, ino, _size, NULL, 
> NULL,
> -NULL, NULL, NULL);
> +NULL, NULL, NULL, NULL);
>   btrfs_release_path(bctx->path);
>   if (ret < 0)
>   return ret;
> @@ -1593,7 +1602,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, 
> u64 ino, u64 gen)
>   u64 right_gen;
>  
>   ret = get_inode_info(sctx->send_root, ino, NULL, _gen, NULL, NULL,
> - NU

[PATCH] btrfs-progs: add chattr support for send/receive

[Howard McLauchlan]

Presently, btrfs send/receive does not propagate inode attribute flags;
all chattr operations are effectively discarded upon transmission.

This patch adds userspace support for inode attribute flags. Kernel
support can be found under the commit:

btrfs: add chattr support for send/receive

An associated xfstest can also be found at:

btrfs: verify chattr support for send/receive test

A caveat is that a user with an updated kernel (aware of chattrs) and an
older version of btrfs-progs (unaware of chattrs) will fail to receive
if a chattr is included in the send stream.

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 cmds-receive.c | 31 +++
 send-dump.c|  8 +++-
 send-stream.c  |  5 +
 send-stream.h  |  1 +
 send.h |  2 ++
 5 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/cmds-receive.c b/cmds-receive.c
index 68123a31..883aee4e 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -38,6 +38,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -1059,6 +1060,35 @@ static int process_update_extent(const char *path, u64 
offset, u64 len,
return 0;
 }
 
+static int process_chattr(const char *path, u64 flags, void *user)
+{
+   int ret = 0;
+   int fd = 0;
+   int _flags = flags;
+   struct btrfs_receive *rctx = user;
+   char full_path[PATH_MAX];
+
+   ret = path_cat_out(full_path, rctx->full_subvol_path, path);
+   if (ret < 0) {
+   error("chattr: path invalid: %s", path);
+   goto out;
+   }
+
+   if (g_verbose >= 2)
+   fprintf(stderr, "chattr %s - flags=0%o\n", path, (int)flags);
+
+   fd = open(full_path, O_RDONLY);
+   ret = ioctl(fd, FS_IOC_SETFLAGS, &_flags);
+
+   if (ret < 0) {
+   ret = -errno;
+   error("chattr %s failed: %s", path, strerror(-ret));
+   goto out;
+   }
+
+out:
+   return ret;
+}
 static struct btrfs_send_ops send_ops = {
.subvol = process_subvol,
.snapshot = process_snapshot,
@@ -1081,6 +,7 @@ static struct btrfs_send_ops send_ops = {
.chown = process_chown,
.utimes = process_utimes,
.update_extent = process_update_extent,
+   .chattr = process_chattr,
 };
 
 static int do_receive(struct btrfs_receive *rctx, const char *tomnt,
diff --git a/send-dump.c b/send-dump.c
index 1591e0cc..757ed5ec 100644
--- a/send-dump.c
+++ b/send-dump.c
@@ -316,6 +316,11 @@ static int print_update_extent(const char *path, u64 
offset, u64 len,
  offset, len);
 }
 
+static int print_chattr(const char *path, u64 flags, void *user)
+{
+   return PRINT_DUMP(user, path, "chattr", "flags=%llu", flags);
+}
+
 struct btrfs_send_ops btrfs_print_send_ops = {
.subvol = print_subvol,
.snapshot = print_snapshot,
@@ -337,5 +342,6 @@ struct btrfs_send_ops btrfs_print_send_ops = {
.chmod = print_chmod,
.chown = print_chown,
.utimes = print_utimes,
-   .update_extent = print_update_extent
+   .update_extent = print_update_extent,
+   .chattr = print_chattr,
 };
diff --git a/send-stream.c b/send-stream.c
index 78f2571a..2c927af2 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -453,6 +453,11 @@ static int read_and_process_cmd(struct btrfs_send_stream 
*sctx)
TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, );
ret = sctx->ops->update_extent(path, offset, tmp, sctx->user);
break;
+   case BTRFS_SEND_C_CHATTR:
+   TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, );
+   TLV_GET_U64(sctx, BTRFS_SEND_A_CHATTR, );
+   ret = sctx->ops->chattr(path, tmp, sctx->user);
+   break;
case BTRFS_SEND_C_END:
ret = 1;
break;
diff --git a/send-stream.h b/send-stream.h
index 39901f86..2446f22d 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -66,6 +66,7 @@ struct btrfs_send_ops {
  struct timespec *mt, struct timespec *ct,
  void *user);
int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
+   int (*chattr)(const char *path, u64 flags, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
diff --git a/send.h b/send.h
index fe613cbb..fd07495f 100644
--- a/send.h
+++ b/send.h
@@ -94,6 +94,7 @@ enum btrfs_send_cmd {
 
BTRFS_SEND_C_END,
BTRFS_SEND_C_UPDATE_EXTENT,
+   BTRFS_SEND_C_CHATTR,
__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -131,6 +132,7 @@ enum {
BTRFS_SEND_A_CLONE_PATH,
BTRFS_SEND_A_CLONE_OFFSET,
BTRFS_SEND_A_CLONE_LEN,
+   BTRFS_SEND_A_CHATTR,
 
__BTRFS_SEND_A_MAX,
 };
-- 
2.17.0

--
To unsubscribe from this list: send the line "unsubscrib

[PATCH] btrfs: add chattr support for send/receive

[Howard McLauchlan]

Presently btrfs send/receive does not propagate inode attribute flags;
all chattr operations are effectively discarded upon transmission.

This patch adds kernel support for inode attribute flags. Userspace
support can be found under the commit:

btrfs-progs: add chattr support for send/receive

An associated xfstest can be found at:

btrfs: add verify chattr support for send/receive test

A caveat is that a user with an updated kernel (aware of chattrs) and an
older version of btrfs-progs (unaware of chattrs) will fail to receive
if a chattr is included in the send stream.

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
Based on 4.17-rc1

 fs/btrfs/ctree.h |   2 +
 fs/btrfs/ioctl.c |   2 +-
 fs/btrfs/send.c  | 176 +++
 fs/btrfs/send.h  |   2 +
 4 files changed, 154 insertions(+), 28 deletions(-)

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 5474ef14d6e6..a0dc6a8a37eb 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -1436,6 +1436,8 @@ struct btrfs_map_token {
unsigned long offset;
 };
 
+unsigned int btrfs_flags_to_ioctl(unsigned int flags);
+
 #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \
((bytes) >> (fs_info)->sb->s_blocksize_bits)
 
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index 632e26d6f7ce..36ce1e589f9e 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, unsigned 
int flags)
 /*
  * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl.
  */
-static unsigned int btrfs_flags_to_ioctl(unsigned int flags)
+unsigned int btrfs_flags_to_ioctl(unsigned int flags)
 {
unsigned int iflags = 0;
 
diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index 221e5cdb060b..da521a5a1843 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -101,6 +101,13 @@ struct send_ctx {
u64 cur_inode_last_extent;
u64 cur_inode_next_write_offset;
 
+   /*
+* state for chattr purposes
+*/
+   u64 cur_inode_flip_flags;
+   u64 cur_inode_receive_flags;
+   int receive_flags_valid;
+
u64 send_progress;
 
struct list_head new_refs;
@@ -798,7 +805,7 @@ static int send_rmdir(struct send_ctx *sctx, struct fs_path 
*path)
  */
 static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path,
  u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid,
- u64 *gid, u64 *rdev)
+ u64 *gid, u64 *rdev, u64 *flags)
 {
int ret;
struct btrfs_inode_item *ii;
@@ -828,6 +835,8 @@ static int __get_inode_info(struct btrfs_root *root, struct 
btrfs_path *path,
*gid = btrfs_inode_gid(path->nodes[0], ii);
if (rdev)
*rdev = btrfs_inode_rdev(path->nodes[0], ii);
+   if (flags)
+   *flags = btrfs_inode_flags(path->nodes[0], ii);
 
return ret;
 }
@@ -835,7 +844,7 @@ static int __get_inode_info(struct btrfs_root *root, struct 
btrfs_path *path,
 static int get_inode_info(struct btrfs_root *root,
  u64 ino, u64 *size, u64 *gen,
  u64 *mode, u64 *uid, u64 *gid,
- u64 *rdev)
+ u64 *rdev, u64 *flags)
 {
struct btrfs_path *path;
int ret;
@@ -844,7 +853,7 @@ static int get_inode_info(struct btrfs_root *root,
if (!path)
return -ENOMEM;
ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid,
-  rdev);
+  rdev, flags);
btrfs_free_path(path);
return ret;
 }
@@ -1233,7 +1242,7 @@ static int __iterate_backrefs(u64 ino, u64 offset, u64 
root, void *ctx_)
 * accept clones from these extents.
 */
ret = __get_inode_info(found->root, bctx->path, ino, _size, NULL, 
NULL,
-  NULL, NULL, NULL);
+  NULL, NULL, NULL, NULL);
btrfs_release_path(bctx->path);
if (ret < 0)
return ret;
@@ -1593,7 +1602,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 
ino, u64 gen)
u64 right_gen;
 
ret = get_inode_info(sctx->send_root, ino, NULL, _gen, NULL, NULL,
-   NULL, NULL);
+   NULL, NULL, NULL);
if (ret < 0 && ret != -ENOENT)
goto out;
left_ret = ret;
@@ -1602,7 +1611,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 
ino, u64 gen)
right_ret = -ENOENT;
} else {
ret = get_inode_info(sctx->parent_root, ino, NULL, _gen,
-   NULL, NULL, NULL, NULL);
+   NULL, NULL, NULL, NULL, NULL);
if (ret < 0 && ret != -ENOENT)
goto out

[PATCH] btrfs: optimize free space tree bitmap conversion

[Howard McLauchlan]

Presently, convert_free_space_to_extents() does a linear scan of the
bitmap. We can speed this up with find_next_{bit,zero_bit}_le().

This patch replaces the linear scan with find_next_{bit,zero_bit}_le().
Testing shows a 20-33% decrease in execution time for
convert_free_space_to_extents().

Suggested-by: Omar Sandoval <osan...@osandov.com>
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
Based on 4.17-rc1

 fs/btrfs/extent_io.c   | 12 -
 fs/btrfs/extent_io.h   |  4 +--
 fs/btrfs/free-space-tree.c | 54 ++
 3 files changed, 27 insertions(+), 43 deletions(-)

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index e99b329002cf..1c0e7ce49556 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -5620,12 +5620,12 @@ void copy_extent_buffer(struct extent_buffer *dst, 
struct extent_buffer *src,
}
 }
 
-void le_bitmap_set(u8 *map, unsigned int start, int len)
+void le_bitmap_set(unsigned long *map, unsigned int start, int len)
 {
-   u8 *p = map + BIT_BYTE(start);
+   char *p = ((char *)map) + BIT_BYTE(start);
const unsigned int size = start + len;
int bits_to_set = BITS_PER_BYTE - (start % BITS_PER_BYTE);
-   u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
+   char mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
 
while (len - bits_to_set >= 0) {
*p |= mask_to_set;
@@ -5640,12 +5640,12 @@ void le_bitmap_set(u8 *map, unsigned int start, int len)
}
 }
 
-void le_bitmap_clear(u8 *map, unsigned int start, int len)
+void le_bitmap_clear(unsigned long *map, unsigned int start, int len)
 {
-   u8 *p = map + BIT_BYTE(start);
+   char *p = ((char *)map) + BIT_BYTE(start);
const unsigned int size = start + len;
int bits_to_clear = BITS_PER_BYTE - (start % BITS_PER_BYTE);
-   u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(start);
+   char mask_to_clear = BITMAP_FIRST_BYTE_MASK(start);
 
while (len - bits_to_clear >= 0) {
*p &= ~mask_to_clear;
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index a53009694b16..a6db233f4a40 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -84,8 +84,8 @@ static inline int le_test_bit(int nr, const u8 *addr)
return 1U & (addr[BIT_BYTE(nr)] >> (nr & (BITS_PER_BYTE-1)));
 }
 
-void le_bitmap_set(u8 *map, unsigned int start, int len);
-void le_bitmap_clear(u8 *map, unsigned int start, int len);
+void le_bitmap_set(unsigned long *map, unsigned int start, int len);
+void le_bitmap_clear(unsigned long *map, unsigned int start, int len);
 
 struct extent_state;
 struct btrfs_root;
diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
index 32a0f6cb5594..0ddf96b01a3a 100644
--- a/fs/btrfs/free-space-tree.c
+++ b/fs/btrfs/free-space-tree.c
@@ -138,9 +138,9 @@ static inline u32 free_space_bitmap_size(u64 size, u32 
sectorsize)
return DIV_ROUND_UP((u32)div_u64(size, sectorsize), BITS_PER_BYTE);
 }
 
-static u8 *alloc_bitmap(u32 bitmap_size)
+static unsigned long *alloc_bitmap(u32 bitmap_size)
 {
-   u8 *ret;
+   unsigned long *ret;
unsigned int nofs_flag;
 
/*
@@ -166,7 +166,8 @@ int convert_free_space_to_bitmaps(struct btrfs_trans_handle 
*trans,
struct btrfs_free_space_info *info;
struct btrfs_key key, found_key;
struct extent_buffer *leaf;
-   u8 *bitmap, *bitmap_cursor;
+   unsigned long *bitmap;
+   char *bitmap_cursor;
u64 start, end;
u64 bitmap_range, i;
u32 bitmap_size, flags, expected_extent_count;
@@ -255,7 +256,7 @@ int convert_free_space_to_bitmaps(struct btrfs_trans_handle 
*trans,
goto out;
}
 
-   bitmap_cursor = bitmap;
+   bitmap_cursor = (char *)bitmap;
bitmap_range = fs_info->sectorsize * BTRFS_FREE_SPACE_BITMAP_BITS;
i = start;
while (i < end) {
@@ -304,13 +305,10 @@ int convert_free_space_to_extents(struct 
btrfs_trans_handle *trans,
struct btrfs_free_space_info *info;
struct btrfs_key key, found_key;
struct extent_buffer *leaf;
-   u8 *bitmap;
+   unsigned long *bitmap;
u64 start, end;
-   /* Initialize to silence GCC. */
-   u64 extent_start = 0;
-   u64 offset;
u32 bitmap_size, flags, expected_extent_count;
-   int prev_bit = 0, bit, bitnr;
+   unsigned long nrbits, start_bit, end_bit;
u32 extent_count = 0;
int done = 0, nr;
int ret;
@@ -348,7 +346,7 @@ int convert_free_space_to_extents(struct btrfs_trans_handle 
*trans,
break;
} else if (found_key.type == 
BTRFS_FREE_SPACE_BITMAP_KEY) {
unsigned long ptr;
-   u8 *bitmap_cursor;
+   char *bitmap_cursor;
u32 bitmap_pos, data_size

[PATCH v2] btrfs: Add nossd_spread mount option

[Howard McLauchlan]

Btrfs has two mount options for SSD optimizations: ssd and ssd_spread.
Presently there is an option to disable all SSD optimizations, but there
isn't an option to disable just ssd_spread.

This patch adds a mount option nossd_spread that disables ssd_spread
only.

Reviewed-by: Josef Bacik <jba...@fb.com>
Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
V2:remove duplicate code with fallthrough
Based on 4.16-rc4

 fs/btrfs/super.c | 11 +++
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index 4b817947e00f..5bfcbf6f7eea 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -310,10 +310,10 @@ static void btrfs_put_super(struct super_block *sb)
 enum {
Opt_degraded, Opt_subvol, Opt_subvolid, Opt_device, Opt_nodatasum,
Opt_nodatacow, Opt_max_inline, Opt_alloc_start, Opt_nobarrier, Opt_ssd,
-   Opt_nossd, Opt_ssd_spread, Opt_thread_pool, Opt_noacl, Opt_compress,
-   Opt_compress_type, Opt_compress_force, Opt_compress_force_type,
-   Opt_notreelog, Opt_ratio, Opt_flushoncommit, Opt_discard,
-   Opt_space_cache, Opt_space_cache_version, Opt_clear_cache,
+   Opt_nossd, Opt_ssd_spread, Opt_nossd_spread, Opt_thread_pool, Opt_noacl,
+   Opt_compress, Opt_compress_type, Opt_compress_force,
+   Opt_compress_force_type, Opt_notreelog, Opt_ratio, Opt_flushoncommit,
+   Opt_discard, Opt_space_cache, Opt_space_cache_version, Opt_clear_cache,
Opt_user_subvol_rm_allowed, Opt_enospc_debug, Opt_subvolrootid,
Opt_defrag, Opt_inode_cache, Opt_no_space_cache, Opt_recovery,
Opt_skip_balance, Opt_check_integrity,
@@ -353,6 +353,7 @@ static const match_table_t tokens = {
{Opt_ssd, "ssd"},
{Opt_ssd_spread, "ssd_spread"},
{Opt_nossd, "nossd"},
+   {Opt_nossd_spread, "nossd_spread"},
{Opt_acl, "acl"},
{Opt_noacl, "noacl"},
{Opt_notreelog, "notreelog"},
@@ -579,6 +580,8 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char 
*options,
btrfs_set_opt(info->mount_opt, NOSSD);
btrfs_clear_and_info(info, SSD,
 "not using ssd optimizations");
+   /* Fallthrough */
+   case Opt_nossd_spread:
btrfs_clear_and_info(info, SSD_SPREAD,
 "not using spread ssd allocation 
scheme");
break;
-- 
2.14.1

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] btrfs: Add nossd_spread mount option

[Howard McLauchlan]

On 02/21/2018 05:20 PM, Hans van Kranenburg wrote:
> On 02/22/2018 12:31 AM, Howard McLauchlan wrote:
>> Btrfs has two mount options for SSD optimizations: ssd and ssd_spread.
>> Presently there is an option to disable all SSD optimizations, but there
>> isn't an option to disable just ssd_spread.
>>
>> This patch adds a mount option nossd_spread that disables ssd_spread
>> only.
>>
>> Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
>> ---
>>  fs/btrfs/super.c | 13 +
>>  1 file changed, 9 insertions(+), 4 deletions(-)
>>
>> diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
>> index 6e71a2a78363..4c0fcf5b3e7e 100644
>> --- a/fs/btrfs/super.c
>> +++ b/fs/btrfs/super.c
>> @@ -310,10 +310,10 @@ static void btrfs_put_super(struct super_block *sb)
>>  enum {
>>  Opt_degraded, Opt_subvol, Opt_subvolid, Opt_device, Opt_nodatasum,
>>  Opt_nodatacow, Opt_max_inline, Opt_alloc_start, Opt_nobarrier, Opt_ssd,
>> -Opt_nossd, Opt_ssd_spread, Opt_thread_pool, Opt_noacl, Opt_compress,
>> -Opt_compress_type, Opt_compress_force, Opt_compress_force_type,
>> -Opt_notreelog, Opt_ratio, Opt_flushoncommit, Opt_discard,
>> -Opt_space_cache, Opt_space_cache_version, Opt_clear_cache,
>> +Opt_nossd, Opt_ssd_spread, Opt_nossd_spread, Opt_thread_pool, Opt_noacl,
>> +Opt_compress, Opt_compress_type, Opt_compress_force,
>> +Opt_compress_force_type, Opt_notreelog, Opt_ratio, Opt_flushoncommit,
>> +Opt_discard, Opt_space_cache, Opt_space_cache_version, Opt_clear_cache,
>>  Opt_user_subvol_rm_allowed, Opt_enospc_debug, Opt_subvolrootid,
>>  Opt_defrag, Opt_inode_cache, Opt_no_space_cache, Opt_recovery,
>>  Opt_skip_balance, Opt_check_integrity,
>> @@ -353,6 +353,7 @@ static const match_table_t tokens = {
>>  {Opt_ssd, "ssd"},
>>  {Opt_ssd_spread, "ssd_spread"},
>>  {Opt_nossd, "nossd"},
>> +{Opt_nossd_spread, "nossd_spread"},
>>  {Opt_acl, "acl"},
>>  {Opt_noacl, "noacl"},
>>  {Opt_notreelog, "notreelog"},
> 
> .oO(Why doesn't the enum just have one option per line, so the changelog
> is less invasive?)
> 
>> @@ -582,6 +583,10 @@ int btrfs_parse_options(struct btrfs_fs_info *info, 
>> char *options,
>>  btrfs_clear_and_info(info, SSD_SPREAD,
>>   "not using spread ssd allocation 
>> scheme");
>>  break;
>> +case Opt_nossd_spread:
>> +btrfs_clear_and_info(info, SSD_SPREAD,
>> + "not using spread ssd allocation 
>> scheme");
>> +break;
>>  case Opt_barrier:
>>  btrfs_clear_and_info(info, NOBARRIER,
>>   "turning on barriers");
>>
> 
> Related:
> * 
> https://urldefense.proofpoint.com/v2/url?u=https-3A__www.spinics.net_lists_linux-2Dbtrfs_msg64247.html=DwIDaQ=5VD0RTtNlTh3ycd41b3MUw=UA4c4GV4shA70-jKB4kwcF99U6K6bzKVYdicFvu-DtQ=oYU5SWdFSoawOXAWRYLqZvvDYpROHFkzMQLlAZEehaU=0LRSD37KWUOyo_i9ypRAaZrv7_8R_kJ0yHv9vNoqDB4=
> * 
> https://urldefense.proofpoint.com/v2/url?u=https-3A__www.spinics.net_lists_linux-2Dbtrfs_msg64277.html=DwIDaQ=5VD0RTtNlTh3ycd41b3MUw=UA4c4GV4shA70-jKB4kwcF99U6K6bzKVYdicFvu-DtQ=oYU5SWdFSoawOXAWRYLqZvvDYpROHFkzMQLlAZEehaU=WB_Fyhm3SDx3XdaE71X6aYRxonuk0Q8Jr4f1ai6K2dI=
> * 
> https://urldefense.proofpoint.com/v2/url?u=https-3A__www.spinics.net_lists_linux-2Dbtrfs_msg64499.html=DwIDaQ=5VD0RTtNlTh3ycd41b3MUw=UA4c4GV4shA70-jKB4kwcF99U6K6bzKVYdicFvu-DtQ=oYU5SWdFSoawOXAWRYLqZvvDYpROHFkzMQLlAZEehaU=FcKlny0UlOCTWEfvCgEz_QtGnYI20EiDF3weQVKPyDs=
> 
> Apparently that discussion never resulted in actual changes, so thanks
> for continuing it now.
> 
> The mount options are a bit weird, because ssd_spread also includes ssd,
> but doesn't show it.
> 
> I personally don't like all of them at all, and I should really finish
> and send my proposal to get them replaced by options that can choose
> extent allocator for data and metadata individually (instead of some
> setting that changes them both at the same time) because there are
> proper real life situations that e.g. benefit from nossd style 'tetris'
> data allocator with ssd_spread style 'contiguous' metadata extent allocator.
> 
I agree; this change is just something very specific that came up in prod for 
us, so I decided to push a patch for it. 
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH] btrfs: Add nossd_spread mount option

[Howard McLauchlan]

Btrfs has two mount options for SSD optimizations: ssd and ssd_spread.
Presently there is an option to disable all SSD optimizations, but there
isn't an option to disable just ssd_spread.

This patch adds a mount option nossd_spread that disables ssd_spread
only.

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 fs/btrfs/super.c | 13 +
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index 6e71a2a78363..4c0fcf5b3e7e 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -310,10 +310,10 @@ static void btrfs_put_super(struct super_block *sb)
 enum {
Opt_degraded, Opt_subvol, Opt_subvolid, Opt_device, Opt_nodatasum,
Opt_nodatacow, Opt_max_inline, Opt_alloc_start, Opt_nobarrier, Opt_ssd,
-   Opt_nossd, Opt_ssd_spread, Opt_thread_pool, Opt_noacl, Opt_compress,
-   Opt_compress_type, Opt_compress_force, Opt_compress_force_type,
-   Opt_notreelog, Opt_ratio, Opt_flushoncommit, Opt_discard,
-   Opt_space_cache, Opt_space_cache_version, Opt_clear_cache,
+   Opt_nossd, Opt_ssd_spread, Opt_nossd_spread, Opt_thread_pool, Opt_noacl,
+   Opt_compress, Opt_compress_type, Opt_compress_force,
+   Opt_compress_force_type, Opt_notreelog, Opt_ratio, Opt_flushoncommit,
+   Opt_discard, Opt_space_cache, Opt_space_cache_version, Opt_clear_cache,
Opt_user_subvol_rm_allowed, Opt_enospc_debug, Opt_subvolrootid,
Opt_defrag, Opt_inode_cache, Opt_no_space_cache, Opt_recovery,
Opt_skip_balance, Opt_check_integrity,
@@ -353,6 +353,7 @@ static const match_table_t tokens = {
{Opt_ssd, "ssd"},
{Opt_ssd_spread, "ssd_spread"},
{Opt_nossd, "nossd"},
+   {Opt_nossd_spread, "nossd_spread"},
{Opt_acl, "acl"},
{Opt_noacl, "noacl"},
{Opt_notreelog, "notreelog"},
@@ -582,6 +583,10 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char 
*options,
btrfs_clear_and_info(info, SSD_SPREAD,
 "not using spread ssd allocation 
scheme");
break;
+   case Opt_nossd_spread:
+   btrfs_clear_and_info(info, SSD_SPREAD,
+"not using spread ssd allocation 
scheme");
+   break;
case Opt_barrier:
btrfs_clear_and_info(info, NOBARRIER,
 "turning on barriers");
-- 
2.14.1

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3] btrfs: print error if primary super block write fails

[Howard McLauchlan]

On 02/02/2018 11:09 AM, Howard McLauchlan wrote:
> Presently, failing a primary super block write but succeeding in at
> least one super block write in general will appear to users as if
> nothing important went wrong. However, upon unmounting and re-mounting,
> the file system will be in a rolled back state. This was discovered
> with a BCC program that uses bpf_override_return() to fail super block
> writes.
> 
> This patch outputs an error clarifying that the primary super block
> write has failed, so users can expect potentially erroneous behaviour.
> It also forces wait_dev_supers() to return an error to its caller if
> the primary super block write fails.
> 
> Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
Reviewed-by: Qu Wenruo <w...@suse.com>
> ---
> V3: Rewrote boolean setting logic for clarity
> V2: Added devid to output, removed unnecessary fs_info parameter
> 
>  fs/btrfs/disk-io.c | 15 ++-
>  1 file changed, 14 insertions(+), 1 deletion(-)
> 
> diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
> index 5da18ebc9222..6ef32eb94669 100644
> --- a/fs/btrfs/disk-io.c
> +++ b/fs/btrfs/disk-io.c
> @@ -3298,6 +3298,7 @@ static int wait_dev_supers(struct btrfs_device *device, 
> int max_mirrors)
>   struct buffer_head *bh;
>   int i;
>   int errors = 0;
> + bool primary_failed = false;
>   u64 bytenr;
>  
>   if (max_mirrors == 0)
> @@ -3314,11 +3315,16 @@ static int wait_dev_supers(struct btrfs_device 
> *device, int max_mirrors)
> BTRFS_SUPER_INFO_SIZE);
>   if (!bh) {
>   errors++;
> + if (i == 0)
> + primary_failed = true;
>   continue;
>   }
>   wait_on_buffer(bh);
> - if (!buffer_uptodate(bh))
> + if (!buffer_uptodate(bh)) {
>   errors++;
> + if (i == 0)
> + primary_failed = true;
> + }
>  
>   /* drop our reference */
>   brelse(bh);
> @@ -3327,6 +,13 @@ static int wait_dev_supers(struct btrfs_device 
> *device, int max_mirrors)
>   brelse(bh);
>   }
>  
> + /* log error, force error return */
> + if (primary_failed) {
> + btrfs_err(device->fs_info, "error writing primary super block 
> to device %llu",
> +   device->devid);
> + return -1;
> + }
> +
>   return errors < i ? 0 : -1;
>  }
>  
> 
Forgot to add Qu's reviewed by
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v3] btrfs: print error if primary super block write fails

[Howard McLauchlan]

Presently, failing a primary super block write but succeeding in at
least one super block write in general will appear to users as if
nothing important went wrong. However, upon unmounting and re-mounting,
the file system will be in a rolled back state. This was discovered
with a BCC program that uses bpf_override_return() to fail super block
writes.

This patch outputs an error clarifying that the primary super block
write has failed, so users can expect potentially erroneous behaviour.
It also forces wait_dev_supers() to return an error to its caller if
the primary super block write fails.

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
V3: Rewrote boolean setting logic for clarity
V2: Added devid to output, removed unnecessary fs_info parameter

 fs/btrfs/disk-io.c | 15 ++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 5da18ebc9222..6ef32eb94669 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -3298,6 +3298,7 @@ static int wait_dev_supers(struct btrfs_device *device, 
int max_mirrors)
struct buffer_head *bh;
int i;
int errors = 0;
+   bool primary_failed = false;
u64 bytenr;
 
if (max_mirrors == 0)
@@ -3314,11 +3315,16 @@ static int wait_dev_supers(struct btrfs_device *device, 
int max_mirrors)
  BTRFS_SUPER_INFO_SIZE);
if (!bh) {
errors++;
+   if (i == 0)
+   primary_failed = true;
continue;
}
wait_on_buffer(bh);
-   if (!buffer_uptodate(bh))
+   if (!buffer_uptodate(bh)) {
errors++;
+   if (i == 0)
+   primary_failed = true;
+   }
 
/* drop our reference */
brelse(bh);
@@ -3327,6 +,13 @@ static int wait_dev_supers(struct btrfs_device *device, 
int max_mirrors)
brelse(bh);
}
 
+   /* log error, force error return */
+   if (primary_failed) {
+   btrfs_err(device->fs_info, "error writing primary super block 
to device %llu",
+ device->devid);
+   return -1;
+   }
+
return errors < i ? 0 : -1;
 }
 
-- 
2.14.1

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v2] btrfs: print error if primary super block write fails

[Howard McLauchlan]

Presently, failing a primary super block write but succeeding in at
least one super block write in general will appear to users as if
nothing important went wrong. However, upon unmounting and re-mounting,
the file system will be in a rolled back state. This was discovered
with a BCC program that uses bpf_override_return() to fail super block
writes.

This patch outputs an error clarifying that the primary super block
write has failed, so users can expect potentially erroneous behaviour.
It also forces wait_dev_supers() to return an error to its caller if
the primary super block write fails.

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
V2: Added devid to output, removed unnecessary fs_info parameter

 fs/btrfs/disk-io.c | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 5da18ebc9222..6d98f2f21d5f 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -3298,6 +3298,7 @@ static int wait_dev_supers(struct btrfs_device *device, 
int max_mirrors)
struct buffer_head *bh;
int i;
int errors = 0;
+   bool primary_failed = false;
u64 bytenr;
 
if (max_mirrors == 0)
@@ -3314,11 +3315,14 @@ static int wait_dev_supers(struct btrfs_device *device, 
int max_mirrors)
  BTRFS_SUPER_INFO_SIZE);
if (!bh) {
errors++;
+   primary_failed = (i == 0) || primary_failed;
continue;
}
wait_on_buffer(bh);
-   if (!buffer_uptodate(bh))
+   if (!buffer_uptodate(bh)) {
errors++;
+   primary_failed = (i == 0) || primary_failed;
+   }
 
/* drop our reference */
brelse(bh);
@@ -3327,6 +3331,13 @@ static int wait_dev_supers(struct btrfs_device *device, 
int max_mirrors)
brelse(bh);
}
 
+   /* log error, force error return */
+   if (primary_failed) {
+   btrfs_err(device->fs_info, "error writing primary super block 
to device %llu",
+ device->devid);
+   return -1;
+   }
+
return errors < i ? 0 : -1;
 }
 
-- 
2.14.1

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH] btrfs: print error if primary super block write fails

[Howard McLauchlan]

Presently, failing a primary super block write but succeeding in at
least one super block write in general will appear to users as if
nothing important went wrong. However, upon unmounting and re-mounting,
the file system will be in a rolled back state. This was discovered
with a BCC program that uses bpf_override_return() to fail super block
writes.

This patch outputs an error clarifying that the primary super block
write has failed, so users can expect potentially erroneous behaviour.
It also forces wait_dev_supers() to return an error to its caller if
the primary super block write fails.

Signed-off-by: Howard McLauchlan <hmclauch...@fb.com>
---
 fs/btrfs/disk-io.c | 17 ++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 5da18ebc9222..8f96e1e4c613 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -3293,11 +3293,13 @@ static int write_dev_supers(struct btrfs_device *device,
  * Return number of errors when buffer head is not found or not marked up to
  * date.
  */
-static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
+static int wait_dev_supers(struct btrfs_fs_info *fs_info,
+  struct btrfs_device *device, int max_mirrors)
 {
struct buffer_head *bh;
int i;
int errors = 0;
+   bool primary_failed = false;
u64 bytenr;
 
if (max_mirrors == 0)
@@ -3314,11 +3316,14 @@ static int wait_dev_supers(struct btrfs_device *device, 
int max_mirrors)
  BTRFS_SUPER_INFO_SIZE);
if (!bh) {
errors++;
+   primary_failed = (i == 0) || primary_failed;
continue;
}
wait_on_buffer(bh);
-   if (!buffer_uptodate(bh))
+   if (!buffer_uptodate(bh)) {
errors++;
+   primary_failed = (i == 0) || primary_failed;
+   }
 
/* drop our reference */
brelse(bh);
@@ -3327,6 +3332,12 @@ static int wait_dev_supers(struct btrfs_device *device, 
int max_mirrors)
brelse(bh);
}
 
+   /* log error, force error return */
+   if (primary_failed) {
+   btrfs_err(fs_info, "error encountered writing primary super 
block");
+   return -1;
+   }
+
return errors < i ? 0 : -1;
 }
 
@@ -3557,7 +3568,7 @@ int write_all_supers(struct btrfs_fs_info *fs_info, int 
max_mirrors)
if (!dev->in_fs_metadata || !dev->writeable)
continue;
 
-   ret = wait_dev_supers(dev, max_mirrors);
+   ret = wait_dev_supers(fs_info, dev, max_mirrors);
if (ret)
total_errors++;
}
-- 
2.14.1

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html