Re: [PATCH] btrfs: add chattr support for send/receive
On 04/18/2018 01:15 AM, Filipe Manana wrote: > On Wed, Apr 18, 2018 at 12:39 AM, Howard McLauchlan > wrote: >> Presently btrfs send/receive does not propagate inode attribute flags; >> all chattr operations are effectively discarded upon transmission. >> >> This patch adds kernel support for inode attribute flags. Userspace >> support can be found under the commit: >> >> btrfs-progs: add chattr support for send/receive >> >> An associated xfstest can be found at: >> >> btrfs: add verify chattr support for send/receive test >> >> A caveat is that a user with an updated kernel (aware of chattrs) and an >> older version of btrfs-progs (unaware of chattrs) will fail to receive >> if a chattr is included in the send stream. > So we do have several things missing in send besides attribute flags, > like hole punching for example (there's a list on the wiki). > We can't just add a new command and introduce such caveat every time > we implement one of the missing and desired features. > > In 2014, while wanting to implement some of those features, I > introduced a way to bump the send stream version with room (commands) > for all those missing features, so that all could be implemented later > without adding further backward incompatibility between kernel > versions btrfs-progs versions. > Some of the threads for reference: > > https://patchwork.kernel.org/patch/4021491/ > https://urldefense.proofpoint.com/v2/url?u=https-3A__www.spinics.net_lists_linux-2Dbtrfs_msg35169.html&d=DwIFaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=UA4c4GV4shA70-jKB4kwcF99U6K6bzKVYdicFvu-DtQ&m=pWSiTBXI54TJfXnctkAeLJUbyIs9VZqupmGKs8JsETE&s=rsgeLq1QeHGaRs6xIXtGcGV-UgjCuqnWMATCw_KaxY8&e= > > It never took off, and honestly I don't remember why as no one add > more comments on the latest versions of the kernel and btrfs-progs > patchsets. Thanks for the heads up, I'll go dig up your patches and try to get them working on 4.17 Howard > >> Signed-off-by: Howard McLauchlan >> --- >> Based on 4.17-rc1 >> >> fs/btrfs/ctree.h | 2 + >> fs/btrfs/ioctl.c | 2 +- >> fs/btrfs/send.c | 176 +++ >> fs/btrfs/send.h | 2 + >> 4 files changed, 154 insertions(+), 28 deletions(-) >> >> diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h >> index 5474ef14d6e6..a0dc6a8a37eb 100644 >> --- a/fs/btrfs/ctree.h >> +++ b/fs/btrfs/ctree.h >> @@ -1436,6 +1436,8 @@ struct btrfs_map_token { >> unsigned long offset; >> }; >> >> +unsigned int btrfs_flags_to_ioctl(unsigned int flags); >> + >> #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \ >> ((bytes) >> (fs_info)->sb->s_blocksize_bits) >> >> diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c >> index 632e26d6f7ce..36ce1e589f9e 100644 >> --- a/fs/btrfs/ioctl.c >> +++ b/fs/btrfs/ioctl.c >> @@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, >> unsigned int flags) >> /* >> * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl. >> */ >> -static unsigned int btrfs_flags_to_ioctl(unsigned int flags) >> +unsigned int btrfs_flags_to_ioctl(unsigned int flags) >> { >> unsigned int iflags = 0; >> >> diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c >> index 221e5cdb060b..da521a5a1843 100644 >> --- a/fs/btrfs/send.c >> +++ b/fs/btrfs/send.c >> @@ -101,6 +101,13 @@ struct send_ctx { >> u64 cur_inode_last_extent; >> u64 cur_inode_next_write_offset; >> >> + /* >> +* state for chattr purposes >> +*/ >> + u64 cur_inode_flip_flags; >> + u64 cur_inode_receive_flags; >> + int receive_flags_valid; >> + >> u64 send_progress; >> >> struct list_head new_refs; >> @@ -798,7 +805,7 @@ static int send_rmdir(struct send_ctx *sctx, struct >> fs_path *path) >> */ >> static int __get_inode_info(struct btrfs_root *root, struct btrfs_path >> *path, >> u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid, >> - u64 *gid, u64 *rdev) >> + u64 *gid, u64 *rdev, u64 *flags) >> { >> int ret; >> struct btrfs_inode_item *ii; >> @@ -828,6 +835,8 @@ static int __get_inode_info(struct btrfs_root *root, >> struct btrfs_path *path, >> *gid = btrfs_inode_gid(path->nodes[0], ii); >> if (rdev) >> *rdev = btrfs_inode_rdev(path->nodes[0], ii); >> + if (flags) >> + *flags = btrfs_inode_flags(path->nodes[0], ii); >> >> return ret; >> } >> @@ -835,7 +844,7 @@ static int __get_inode_info(struct btrfs_root *root, >> struct btrfs_path *path, >> static int get_inode_info(struct btrfs_root *root, >> u64 ino, u64 *size, u64 *gen, >> u64 *mode, u64 *uid, u64 *gid, >> - u64 *rdev) >> + u64 *rdev, u64 *flags) >> { >> struct btrfs_path *path; >> int ret; >> @@ -844,7 +853,7 @@ static int get_inode_info(struc
Re: [PATCH] btrfs: add chattr support for send/receive
On Tue, Apr 17, 2018 at 04:39:35PM -0700, Howard McLauchlan wrote: > Presently btrfs send/receive does not propagate inode attribute flags; > all chattr operations are effectively discarded upon transmission. > > This patch adds kernel support for inode attribute flags. Userspace > support can be found under the commit: > > btrfs-progs: add chattr support for send/receive > > An associated xfstest can be found at: > > btrfs: add verify chattr support for send/receive test > > A caveat is that a user with an updated kernel (aware of chattrs) and an > older version of btrfs-progs (unaware of chattrs) will fail to receive > if a chattr is included in the send stream. > > Signed-off-by: Howard McLauchlan This is a send protocol change and must be properly versioned. There are more known defficiencies from v1, see https://btrfs.wiki.kernel.org/index.php/Design_notes_on_Send/Receive#Send_stream_v2_draft > --- a/fs/btrfs/send.h > +++ b/fs/btrfs/send.h > @@ -77,6 +77,7 @@ enum btrfs_send_cmd { > > BTRFS_SEND_C_END, > BTRFS_SEND_C_UPDATE_EXTENT, > + BTRFS_SEND_C_CHATTR, > __BTRFS_SEND_C_MAX, This change > }; > #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1) > @@ -114,6 +115,7 @@ enum { > BTRFS_SEND_A_CLONE_PATH, > BTRFS_SEND_A_CLONE_OFFSET, > BTRFS_SEND_A_CLONE_LEN, > + BTRFS_SEND_A_CHATTR, > > __BTRFS_SEND_A_MAX, and this will change numbers of __BTRFS_SEND_*_MAX and defines derived from them, that must stay fixed for v1, because they're part of the public API exported from progs as send.h. Unfortunatelly for anybody who wants to implement new additions to the send stream, the full versioning, backward compatibility, commandline options, ioctl extensions need to happen first. A concrete example how this can be done wrong is the Synology version of btrfs shipped with their NAS. There are some extensions to the protocol that work on their kernel, but the send stream cannot be used on upstram kernels. -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] btrfs: add chattr support for send/receive
On Wed, Apr 18, 2018 at 12:39 AM, Howard McLauchlan wrote: > Presently btrfs send/receive does not propagate inode attribute flags; > all chattr operations are effectively discarded upon transmission. > > This patch adds kernel support for inode attribute flags. Userspace > support can be found under the commit: > > btrfs-progs: add chattr support for send/receive > > An associated xfstest can be found at: > > btrfs: add verify chattr support for send/receive test > > A caveat is that a user with an updated kernel (aware of chattrs) and an > older version of btrfs-progs (unaware of chattrs) will fail to receive > if a chattr is included in the send stream. So we do have several things missing in send besides attribute flags, like hole punching for example (there's a list on the wiki). We can't just add a new command and introduce such caveat every time we implement one of the missing and desired features. In 2014, while wanting to implement some of those features, I introduced a way to bump the send stream version with room (commands) for all those missing features, so that all could be implemented later without adding further backward incompatibility between kernel versions btrfs-progs versions. Some of the threads for reference: https://patchwork.kernel.org/patch/4021491/ https://www.spinics.net/lists/linux-btrfs/msg35169.html It never took off, and honestly I don't remember why as no one add more comments on the latest versions of the kernel and btrfs-progs patchsets. > > Signed-off-by: Howard McLauchlan > --- > Based on 4.17-rc1 > > fs/btrfs/ctree.h | 2 + > fs/btrfs/ioctl.c | 2 +- > fs/btrfs/send.c | 176 +++ > fs/btrfs/send.h | 2 + > 4 files changed, 154 insertions(+), 28 deletions(-) > > diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h > index 5474ef14d6e6..a0dc6a8a37eb 100644 > --- a/fs/btrfs/ctree.h > +++ b/fs/btrfs/ctree.h > @@ -1436,6 +1436,8 @@ struct btrfs_map_token { > unsigned long offset; > }; > > +unsigned int btrfs_flags_to_ioctl(unsigned int flags); > + > #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \ > ((bytes) >> (fs_info)->sb->s_blocksize_bits) > > diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c > index 632e26d6f7ce..36ce1e589f9e 100644 > --- a/fs/btrfs/ioctl.c > +++ b/fs/btrfs/ioctl.c > @@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, > unsigned int flags) > /* > * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl. > */ > -static unsigned int btrfs_flags_to_ioctl(unsigned int flags) > +unsigned int btrfs_flags_to_ioctl(unsigned int flags) > { > unsigned int iflags = 0; > > diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c > index 221e5cdb060b..da521a5a1843 100644 > --- a/fs/btrfs/send.c > +++ b/fs/btrfs/send.c > @@ -101,6 +101,13 @@ struct send_ctx { > u64 cur_inode_last_extent; > u64 cur_inode_next_write_offset; > > + /* > +* state for chattr purposes > +*/ > + u64 cur_inode_flip_flags; > + u64 cur_inode_receive_flags; > + int receive_flags_valid; > + > u64 send_progress; > > struct list_head new_refs; > @@ -798,7 +805,7 @@ static int send_rmdir(struct send_ctx *sctx, struct > fs_path *path) > */ > static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path, > u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid, > - u64 *gid, u64 *rdev) > + u64 *gid, u64 *rdev, u64 *flags) > { > int ret; > struct btrfs_inode_item *ii; > @@ -828,6 +835,8 @@ static int __get_inode_info(struct btrfs_root *root, > struct btrfs_path *path, > *gid = btrfs_inode_gid(path->nodes[0], ii); > if (rdev) > *rdev = btrfs_inode_rdev(path->nodes[0], ii); > + if (flags) > + *flags = btrfs_inode_flags(path->nodes[0], ii); > > return ret; > } > @@ -835,7 +844,7 @@ static int __get_inode_info(struct btrfs_root *root, > struct btrfs_path *path, > static int get_inode_info(struct btrfs_root *root, > u64 ino, u64 *size, u64 *gen, > u64 *mode, u64 *uid, u64 *gid, > - u64 *rdev) > + u64 *rdev, u64 *flags) > { > struct btrfs_path *path; > int ret; > @@ -844,7 +853,7 @@ static int get_inode_info(struct btrfs_root *root, > if (!path) > return -ENOMEM; > ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid, > - rdev); > + rdev, flags); > btrfs_free_path(path); > return ret; > } > @@ -1233,7 +1242,7 @@ static int __iterate_backrefs(u64 ino, u64 offset, u64 > root, void *ctx_) > * accept clones from these extents. > */ > ret = __get_inode_info(found->root,
Re: [PATCH] btrfs: add chattr support for send/receive
On 04/17/2018 04:39 PM, Howard McLauchlan wrote: > Presently btrfs send/receive does not propagate inode attribute flags; > all chattr operations are effectively discarded upon transmission. > > This patch adds kernel support for inode attribute flags. Userspace > support can be found under the commit: > > btrfs-progs: add chattr support for send/receive > > An associated xfstest can be found at: > > btrfs: add verify chattr support for send/receive test > > A caveat is that a user with an updated kernel (aware of chattrs) and an > older version of btrfs-progs (unaware of chattrs) will fail to receive > if a chattr is included in the send stream. > > Signed-off-by: Howard McLauchlan > --- > Based on 4.17-rc1 > > fs/btrfs/ctree.h | 2 + > fs/btrfs/ioctl.c | 2 +- > fs/btrfs/send.c | 176 +++ > fs/btrfs/send.h | 2 + > 4 files changed, 154 insertions(+), 28 deletions(-) > > diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h > index 5474ef14d6e6..a0dc6a8a37eb 100644 > --- a/fs/btrfs/ctree.h > +++ b/fs/btrfs/ctree.h > @@ -1436,6 +1436,8 @@ struct btrfs_map_token { > unsigned long offset; > }; > > +unsigned int btrfs_flags_to_ioctl(unsigned int flags); > + > #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \ > ((bytes) >> (fs_info)->sb->s_blocksize_bits) > > diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c > index 632e26d6f7ce..36ce1e589f9e 100644 > --- a/fs/btrfs/ioctl.c > +++ b/fs/btrfs/ioctl.c > @@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, > unsigned int flags) > /* > * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl. > */ > -static unsigned int btrfs_flags_to_ioctl(unsigned int flags) > +unsigned int btrfs_flags_to_ioctl(unsigned int flags) > { > unsigned int iflags = 0; > > diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c > index 221e5cdb060b..da521a5a1843 100644 > --- a/fs/btrfs/send.c > +++ b/fs/btrfs/send.c > @@ -101,6 +101,13 @@ struct send_ctx { > u64 cur_inode_last_extent; > u64 cur_inode_next_write_offset; > > + /* > + * state for chattr purposes > + */ > + u64 cur_inode_flip_flags; > + u64 cur_inode_receive_flags; > + int receive_flags_valid; > + > u64 send_progress; > > struct list_head new_refs; > @@ -798,7 +805,7 @@ static int send_rmdir(struct send_ctx *sctx, struct > fs_path *path) > */ > static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path, > u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid, > - u64 *gid, u64 *rdev) > + u64 *gid, u64 *rdev, u64 *flags) > { > int ret; > struct btrfs_inode_item *ii; > @@ -828,6 +835,8 @@ static int __get_inode_info(struct btrfs_root *root, > struct btrfs_path *path, > *gid = btrfs_inode_gid(path->nodes[0], ii); > if (rdev) > *rdev = btrfs_inode_rdev(path->nodes[0], ii); > + if (flags) > + *flags = btrfs_inode_flags(path->nodes[0], ii); > > return ret; > } > @@ -835,7 +844,7 @@ static int __get_inode_info(struct btrfs_root *root, > struct btrfs_path *path, > static int get_inode_info(struct btrfs_root *root, > u64 ino, u64 *size, u64 *gen, > u64 *mode, u64 *uid, u64 *gid, > - u64 *rdev) > + u64 *rdev, u64 *flags) > { > struct btrfs_path *path; > int ret; > @@ -844,7 +853,7 @@ static int get_inode_info(struct btrfs_root *root, > if (!path) > return -ENOMEM; > ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid, > -rdev); > +rdev, flags); > btrfs_free_path(path); > return ret; > } > @@ -1233,7 +1242,7 @@ static int __iterate_backrefs(u64 ino, u64 offset, u64 > root, void *ctx_) >* accept clones from these extents. >*/ > ret = __get_inode_info(found->root, bctx->path, ino, &i_size, NULL, > NULL, > -NULL, NULL, NULL); > +NULL, NULL, NULL, NULL); > btrfs_release_path(bctx->path); > if (ret < 0) > return ret; > @@ -1593,7 +1602,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, > u64 ino, u64 gen) > u64 right_gen; > > ret = get_inode_info(sctx->send_root, ino, NULL, &left_gen, NULL, NULL, > - NULL, NULL); > + NULL, NULL, NULL); > if (ret < 0 && ret != -ENOENT) > goto out; > left_ret = ret; > @@ -1602,7 +1611,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, > u64 ino, u64 gen) > right_ret = -ENOENT; > } else { > ret = get_inode_info(sctx->parent_root, ino, NULL, &right_gen, > - NULL, NULL, NULL, NULL); > +
[PATCH] btrfs: add chattr support for send/receive
Presently btrfs send/receive does not propagate inode attribute flags; all chattr operations are effectively discarded upon transmission. This patch adds kernel support for inode attribute flags. Userspace support can be found under the commit: btrfs-progs: add chattr support for send/receive An associated xfstest can be found at: btrfs: add verify chattr support for send/receive test A caveat is that a user with an updated kernel (aware of chattrs) and an older version of btrfs-progs (unaware of chattrs) will fail to receive if a chattr is included in the send stream. Signed-off-by: Howard McLauchlan --- Based on 4.17-rc1 fs/btrfs/ctree.h | 2 + fs/btrfs/ioctl.c | 2 +- fs/btrfs/send.c | 176 +++ fs/btrfs/send.h | 2 + 4 files changed, 154 insertions(+), 28 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 5474ef14d6e6..a0dc6a8a37eb 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1436,6 +1436,8 @@ struct btrfs_map_token { unsigned long offset; }; +unsigned int btrfs_flags_to_ioctl(unsigned int flags); + #define BTRFS_BYTES_TO_BLKS(fs_info, bytes) \ ((bytes) >> (fs_info)->sb->s_blocksize_bits) diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 632e26d6f7ce..36ce1e589f9e 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -106,7 +106,7 @@ static unsigned int btrfs_mask_flags(umode_t mode, unsigned int flags) /* * Export inode flags to the format expected by the FS_IOC_GETFLAGS ioctl. */ -static unsigned int btrfs_flags_to_ioctl(unsigned int flags) +unsigned int btrfs_flags_to_ioctl(unsigned int flags) { unsigned int iflags = 0; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 221e5cdb060b..da521a5a1843 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -101,6 +101,13 @@ struct send_ctx { u64 cur_inode_last_extent; u64 cur_inode_next_write_offset; + /* +* state for chattr purposes +*/ + u64 cur_inode_flip_flags; + u64 cur_inode_receive_flags; + int receive_flags_valid; + u64 send_progress; struct list_head new_refs; @@ -798,7 +805,7 @@ static int send_rmdir(struct send_ctx *sctx, struct fs_path *path) */ static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path, u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid, - u64 *gid, u64 *rdev) + u64 *gid, u64 *rdev, u64 *flags) { int ret; struct btrfs_inode_item *ii; @@ -828,6 +835,8 @@ static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path, *gid = btrfs_inode_gid(path->nodes[0], ii); if (rdev) *rdev = btrfs_inode_rdev(path->nodes[0], ii); + if (flags) + *flags = btrfs_inode_flags(path->nodes[0], ii); return ret; } @@ -835,7 +844,7 @@ static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path, static int get_inode_info(struct btrfs_root *root, u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid, u64 *gid, - u64 *rdev) + u64 *rdev, u64 *flags) { struct btrfs_path *path; int ret; @@ -844,7 +853,7 @@ static int get_inode_info(struct btrfs_root *root, if (!path) return -ENOMEM; ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid, - rdev); + rdev, flags); btrfs_free_path(path); return ret; } @@ -1233,7 +1242,7 @@ static int __iterate_backrefs(u64 ino, u64 offset, u64 root, void *ctx_) * accept clones from these extents. */ ret = __get_inode_info(found->root, bctx->path, ino, &i_size, NULL, NULL, - NULL, NULL, NULL); + NULL, NULL, NULL, NULL); btrfs_release_path(bctx->path); if (ret < 0) return ret; @@ -1593,7 +1602,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen) u64 right_gen; ret = get_inode_info(sctx->send_root, ino, NULL, &left_gen, NULL, NULL, - NULL, NULL); + NULL, NULL, NULL); if (ret < 0 && ret != -ENOENT) goto out; left_ret = ret; @@ -1602,7 +1611,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen) right_ret = -ENOENT; } else { ret = get_inode_info(sctx->parent_root, ino, NULL, &right_gen, - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); if (ret < 0 && ret != -ENOENT) goto out; right_ret = ret; @@ -1771,7 +1780,7 @@ static int get_first