Re: [PATCH v2 01/11] btrfs-progs: don't blindly assume crc32c in csum_tree_block_size()
On 27/08/2019 18:36, David Sterba wrote: > On Tue, Aug 27, 2019 at 06:33:02PM +0200, David Sterba wrote: >> On Mon, Aug 26, 2019 at 01:48:43PM +0200, Johannes Thumshirn wrote: >>> The callers of csum_tree_block_size() blindly assume we're only having >>> crc32c as a possible checksum and thus pass in >>> btrfs_csum_sizes[BTRFS_CSUM_TYPE_CRC32] for the size argument of >>> csum_tree_block_size(). >>> >>> Signed-off-by: Johannes Thumshirn >>> Reviewed-by: Nikolay Borisov >>> --- >>> mkfs/common.c | 14 +++--- >>> 1 file changed, 7 insertions(+), 7 deletions(-) >>> >>> diff --git a/mkfs/common.c b/mkfs/common.c >>> index caca5e707233..b6e549b19272 100644 >>> --- a/mkfs/common.c >>> +++ b/mkfs/common.c >>> @@ -101,7 +101,7 @@ static int btrfs_create_tree_root(int fd, struct >>> btrfs_mkfs_config *cfg, >>> } >>> >>> /* generate checksum */ >>> - csum_tree_block_size(buf, btrfs_csum_sizes[BTRFS_CSUM_TYPE_CRC32], 0); >>> + csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0); >> >> Where is btrfs_mkfs_config::csum_type defined? > > Aha it's in 8/11. Grr, more artifacts of rebasing/reorganizing the series. Sorry for that. -- Johannes ThumshirnSUSE Labs Filesystems jthumsh...@suse.de+49 911 74053 689 SUSE Software Solutions Germany GmbH Maxfeldstr. 5 90409 Nürnberg Germany (HRB 247165, AG München) Key fingerprint = EC38 9CAB C2C4 F25D 8600 D0D0 0393 969D 2D76 0850
Re: [PATCH v2] btrfs-progs: add verbose option to btrfs device scan
Ping?
Thanks, Anand
On 16/7/19 11:05 AM, Anand Jain wrote:
To help debug device scan issues, add verbose option to btrfs device scan.
Signed-off-by: Anand Jain
---
v2: Use bool instead of int as a btrfs_scan_device() argument.
cmds/device.c| 8 ++--
cmds/filesystem.c| 2 +-
common/device-scan.c | 4 +++-
common/device-scan.h | 3 ++-
common/utils.c | 2 +-
disk-io.c| 2 +-
6 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/cmds/device.c b/cmds/device.c
index 24158308a41b..9b715ffc42a3 100644
--- a/cmds/device.c
+++ b/cmds/device.c
@@ -313,6 +313,7 @@ static int cmd_device_scan(const struct cmd_struct *cmd,
int argc, char **argv)
int all = 0;
int ret = 0;
int forget = 0;
+ bool verbose = false;
optind = 0;
while (1) {
@@ -323,7 +324,7 @@ static int cmd_device_scan(const struct cmd_struct *cmd,
int argc, char **argv)
{ NULL, 0, NULL, 0}
};
- c = getopt_long(argc, argv, "du", long_options, NULL);
+ c = getopt_long(argc, argv, "duv", long_options, NULL);
if (c < 0)
break;
switch (c) {
@@ -333,6 +334,9 @@ static int cmd_device_scan(const struct cmd_struct *cmd,
int argc, char **argv)
case 'u':
forget = 1;
break;
+ case 'v':
+ verbose = true;
+ break;
default:
usage_unknown_option(cmd, argv);
}
@@ -354,7 +358,7 @@ static int cmd_device_scan(const struct cmd_struct *cmd,
int argc, char **argv)
}
} else {
printf("Scanning for Btrfs filesystems\n");
- ret = btrfs_scan_devices();
+ ret = btrfs_scan_devices(verbose);
error_on(ret, "error %d while scanning", ret);
ret = btrfs_register_all_devices();
error_on(ret,
diff --git a/cmds/filesystem.c b/cmds/filesystem.c
index 4f22089abeaa..02d47a43a792 100644
--- a/cmds/filesystem.c
+++ b/cmds/filesystem.c
@@ -746,7 +746,7 @@ devs_only:
else
ret = 1;
} else {
- ret = btrfs_scan_devices();
+ ret = btrfs_scan_devices(false);
}
if (ret) {
diff --git a/common/device-scan.c b/common/device-scan.c
index 2c5ae225f710..71c91dee6a86 100644
--- a/common/device-scan.c
+++ b/common/device-scan.c
@@ -351,7 +351,7 @@ void free_seen_fsid(struct seen_fsid *seen_fsid_hash[])
}
}
-int btrfs_scan_devices(void)
+int btrfs_scan_devices(bool verbose)
{
int fd = -1;
int ret;
@@ -380,6 +380,8 @@ int btrfs_scan_devices(void)
continue;
/* if we are here its definitely a btrfs disk*/
strncpy_null(path, blkid_dev_devname(dev));
+ if (verbose)
+ printf("blkid: btrfs device: %s\n", path);
fd = open(path, O_RDONLY);
if (fd < 0) {
diff --git a/common/device-scan.h b/common/device-scan.h
index eda2bae5c6c4..3e473c48d1af 100644
--- a/common/device-scan.h
+++ b/common/device-scan.h
@@ -1,6 +1,7 @@
#ifndef __DEVICE_SCAN_H__
#define __DEVICE_SCAN_H__
+#include
#include "kerncompat.h"
#include "ioctl.h"
@@ -29,7 +30,7 @@ struct seen_fsid {
int fd;
};
-int btrfs_scan_devices(void);
+int btrfs_scan_devices(bool verbose);
int btrfs_register_one_device(const char *fname);
int btrfs_register_all_devices(void);
int btrfs_add_to_fsid(struct btrfs_trans_handle *trans,
diff --git a/common/utils.c b/common/utils.c
index ad938409a94f..07648f07fbd4 100644
--- a/common/utils.c
+++ b/common/utils.c
@@ -277,7 +277,7 @@ int check_mounted_where(int fd, const char *file, char
*where, int size,
/* scan other devices */
if (is_btrfs && total_devs > 1) {
- ret = btrfs_scan_devices();
+ ret = btrfs_scan_devices(false);
if (ret)
return ret;
}
diff --git a/disk-io.c b/disk-io.c
index be44eead5cef..71ed78b671da 100644
--- a/disk-io.c
+++ b/disk-io.c
@@ -1085,7 +1085,7 @@ int btrfs_scan_fs_devices(int fd, const char *path,
}
if (!skip_devices && total_devs != 1) {
- ret = btrfs_scan_devices();
+ ret = btrfs_scan_devices(false);
if (ret)
return ret;
}
[PATCH 1/2] btrfs-progs: add BTRFS_DEV_ITEMS_OBJECTID in comment in print-tree
So when searching for BTRFS_DEV_ITEMS_OBJECTID it hits. Albeit it is
defined same as BTRFS_ROOT_TREE_OBJECTID.
Signed-off-by: Anand Jain
---
print-tree.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/print-tree.c b/print-tree.c
index b31e515f8989..5832f3089e3d 100644
--- a/print-tree.c
+++ b/print-tree.c
@@ -704,6 +704,7 @@ void print_objectid(FILE *stream, u64 objectid, u8 type)
}
switch (objectid) {
+ /* BTRFS_DEV_ITEMS_OBJECTID */
case BTRFS_ROOT_TREE_OBJECTID:
if (type == BTRFS_DEV_ITEM_KEY)
fprintf(stream, "DEV_ITEMS");
--
2.21.0 (Apple Git-120)
[PATCH 2/2] btrfs-progs: print-tree add missing DEV_STATS
Add missing BTRFS_DEV_STATS_OBJECTID into the print-tree.
before:
item 0 key (0 PERSISTENT_ITEM 1) itemoff 16243 itemsize 40
persistent item objectid 0 offset 1
after:
item 0 key (DEV_STATS PERSISTENT_ITEM 1) itemoff 16243 itemsize 40
persistent item objectid DEV_STATS offset 1
Signed-off-by: Anand Jain
---
print-tree.c | 6 ++
1 file changed, 6 insertions(+)
diff --git a/print-tree.c b/print-tree.c
index 5832f3089e3d..0f9b3a8f0f83 100644
--- a/print-tree.c
+++ b/print-tree.c
@@ -689,6 +689,12 @@ void print_key_type(FILE *stream, u64 objectid, u8 type)
void print_objectid(FILE *stream, u64 objectid, u8 type)
{
switch (type) {
+ case BTRFS_PERSISTENT_ITEM_KEY:
+ if (objectid == BTRFS_DEV_STATS_OBJECTID)
+ fprintf(stream, "DEV_STATS");
+ else
+ fprintf(stream, "%llu", (unsigned long long)objectid);
+ return;
case BTRFS_DEV_EXTENT_KEY:
/* device id */
fprintf(stream, "%llu", (unsigned long long)objectid);
--
2.21.0 (Apple Git-120)
Re: [RFC PATCH 5/5] Btrfs: add ioctl for directly writing compressed data
On Thu, Aug 15, 2019 at 02:04:06PM -0700, Omar Sandoval wrote:
> #define BTRFS_IOC_SEND_32 _IOW(BTRFS_IOCTL_MAGIC, 38, \
> struct btrfs_ioctl_send_args_32)
> +
> +struct btrfs_ioctl_compressed_pwrite_args_32 {
> + __u64 offset; /* in */
> + __u32 compressed_len; /* in */
> + __u32 orig_len; /* in */
> + __u32 compress_type;/* in */
> + __u32 reserved[9];
> + compat_uptr_t buf; /* in */
> +} __attribute__ ((__packed__));
> +
> +#define BTRFS_IOC_COMPRESSED_PWRITE_32 _IOW(BTRFS_IOCTL_MAGIC, 63, \
> + struct btrfs_ioctl_compressed_pwrite_args_32)
Note that the _32 is a workaround for a mistake in the send ioctl
definitions that slipped trhough. Any pointer in the structure changes
the ioctl number on 32bit and 64bit.
But as the raw data ioctl is new there's point to copy the mistake. The
alignment and width can be forced eg. like
> + void __user *buf; /* in */
union {
void __user *buf;
__u64 __buf_alignment;
};
This allows to user buf as a buffer without casts to a intermediate
type.
Re: [PATCH 1/2] btrfs-progs: add BTRFS_DEV_ITEMS_OBJECTID in comment in print-tree
On 28.08.19 г. 12:56 ч., Anand Jain wrote:
> So when searching for BTRFS_DEV_ITEMS_OBJECTID it hits. Albeit it is
> defined same as BTRFS_ROOT_TREE_OBJECTID.
>
> Signed-off-by: Anand Jain
> ---
> print-tree.c | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/print-tree.c b/print-tree.c
> index b31e515f8989..5832f3089e3d 100644
> --- a/print-tree.c
> +++ b/print-tree.c
> @@ -704,6 +704,7 @@ void print_objectid(FILE *stream, u64 objectid, u8 type)
> }
>
> switch (objectid) {
> + /* BTRFS_DEV_ITEMS_OBJECTID */
That comment looks really cryptic to someone just looking at the code.
Adding case BTRFS_DEV_ITEMS_OBJECTID: is better.
> case BTRFS_ROOT_TREE_OBJECTID:
> if (type == BTRFS_DEV_ITEM_KEY)
> fprintf(stream, "DEV_ITEMS");
>
Re: [PATCH] btrfs: use percentage for fractions, replace helpers
On Tue, Aug 27, 2019 at 08:24:53PM +0200, David Sterba wrote: > The div_factor* helpers calculate fraction or percentual fraction. > There's a nice helper mult_frac that's for general fractions, we'll add > a local wrapper suited for our purposes and replace all instances of > div_factor and update naming in fuctions that pass the fractions. > > * div_factor calculates tenths and the numbers need to be adjusted > * div_factor_fine is direct replacement > > Signed-off-by: David Sterba Dropped for now. Does not compile on 32bit and due to 64bit division and adding 64bit version of mult_frac does not make things significantly easier. I'll probably do a reduced version that only renames div_factor.
Re: [PATCH 1/2] btrfs-progs: add BTRFS_DEV_ITEMS_OBJECTID in comment in print-tree
On Wed, Aug 28, 2019 at 04:01:04PM +0300, Nikolay Borisov wrote:
>
>
> On 28.08.19 г. 12:56 ч., Anand Jain wrote:
> > So when searching for BTRFS_DEV_ITEMS_OBJECTID it hits. Albeit it is
> > defined same as BTRFS_ROOT_TREE_OBJECTID.
> >
> > Signed-off-by: Anand Jain
> > ---
> > print-tree.c | 1 +
> > 1 file changed, 1 insertion(+)
> >
> > diff --git a/print-tree.c b/print-tree.c
> > index b31e515f8989..5832f3089e3d 100644
> > --- a/print-tree.c
> > +++ b/print-tree.c
> > @@ -704,6 +704,7 @@ void print_objectid(FILE *stream, u64 objectid, u8 type)
> > }
> >
> > switch (objectid) {
> > + /* BTRFS_DEV_ITEMS_OBJECTID */
>
> That comment looks really cryptic to someone just looking at the code.
> Adding case BTRFS_DEV_ITEMS_OBJECTID: is better.
> > case BTRFS_ROOT_TREE_OBJECTID:
Both constants have the same value so they can't be in one switch, but
yeah the comment should be a bit more verbose, just the constant name is
bringing more questions than answers.
Re: [PATCH 1/2] btrfs-progs: add BTRFS_DEV_ITEMS_OBJECTID in comment in print-tree
> On 28 Aug 2019, at 9:01 PM, Nikolay Borisov wrote:
>
>
>
> On 28.08.19 г. 12:56 ч., Anand Jain wrote:
>> So when searching for BTRFS_DEV_ITEMS_OBJECTID it hits. Albeit it is
>> defined same as BTRFS_ROOT_TREE_OBJECTID.
>>
>> Signed-off-by: Anand Jain
>> ---
>> print-tree.c | 1 +
>> 1 file changed, 1 insertion(+)
>>
>> diff --git a/print-tree.c b/print-tree.c
>> index b31e515f8989..5832f3089e3d 100644
>> --- a/print-tree.c
>> +++ b/print-tree.c
>> @@ -704,6 +704,7 @@ void print_objectid(FILE *stream, u64 objectid, u8 type)
>> }
>>
>> switch (objectid) {
>> +/* BTRFS_DEV_ITEMS_OBJECTID */
>
> That comment looks really cryptic to someone just looking at the code.
> Adding case BTRFS_DEV_ITEMS_OBJECTID: is better.
>
Both of them defined with the same value (we find which object by type).
So only one of it can be in the case.
#define BTRFS_DEV_ITEMS_OBJECTID 1ULL
::
#define BTRFS_ROOT_TREE_OBJECTID 1ULL
Thanks, Anand
>> case BTRFS_ROOT_TREE_OBJECTID:
>> if (type == BTRFS_DEV_ITEM_KEY)
>> fprintf(stream, "DEV_ITEMS");
>>
Re: [PATCH 1/2] btrfs-progs: add BTRFS_DEV_ITEMS_OBJECTID in comment in print-tree
> On 28 Aug 2019, at 9:24 PM, David Sterba wrote:
>
> On Wed, Aug 28, 2019 at 04:01:04PM +0300, Nikolay Borisov wrote:
>>
>>
>> On 28.08.19 г. 12:56 ч., Anand Jain wrote:
>>> So when searching for BTRFS_DEV_ITEMS_OBJECTID it hits. Albeit it is
>>> defined same as BTRFS_ROOT_TREE_OBJECTID.
>>>
>>> Signed-off-by: Anand Jain
>>> ---
>>> print-tree.c | 1 +
>>> 1 file changed, 1 insertion(+)
>>>
>>> diff --git a/print-tree.c b/print-tree.c
>>> index b31e515f8989..5832f3089e3d 100644
>>> --- a/print-tree.c
>>> +++ b/print-tree.c
>>> @@ -704,6 +704,7 @@ void print_objectid(FILE *stream, u64 objectid, u8 type)
>>> }
>>>
>>> switch (objectid) {
>>> + /* BTRFS_DEV_ITEMS_OBJECTID */
>>
>> That comment looks really cryptic to someone just looking at the code.
>> Adding case BTRFS_DEV_ITEMS_OBJECTID: is better.
>
>>> case BTRFS_ROOT_TREE_OBJECTID:
>
> Both constants have the same value so they can't be in one switch, but
> yeah the comment should be a bit more verbose, just the constant name is
> bringing more questions than answers.
Oh. Hmm. Let me try.
Thanks, Anand
[PATCH v2 1/2] btrfs-progs: add BTRFS_DEV_ITEMS_OBJECTID in comment in print-tree
So when searching for BTRFS_DEV_ITEMS_OBJECTID, it hits, albeit it is
defined same as BTRFS_ROOT_TREE_OBJECTID.
Signed-off-by: Anand Jain
---
v1->v2: Improve comment.
print-tree.c | 5 +
1 file changed, 5 insertions(+)
diff --git a/print-tree.c b/print-tree.c
index b31e515f8989..b1c59d776547 100644
--- a/print-tree.c
+++ b/print-tree.c
@@ -705,6 +705,11 @@ void print_objectid(FILE *stream, u64 objectid, u8 type)
switch (objectid) {
case BTRFS_ROOT_TREE_OBJECTID:
+ /*
+* BTRFS_ROOT_TREE_OBJECTID and BTRFS_DEV_ITEMS_OBJECTID are
+* defined with the same value of 1ULL, distinguish them by
+* checking the type.
+*/
if (type == BTRFS_DEV_ITEM_KEY)
fprintf(stream, "DEV_ITEMS");
else
--
2.21.0 (Apple Git-120)
Re: WARNING: CPU: 3 PID: 439 at fs/btrfs/ctree.h:1559 btrfs_update_device+0x1c5/0x1d0 [btrfs]
On 2017/9/19 下午11:56, Rich Rauenzahn wrote: > I've filed a bug on this kernel trace -- I get 100's of these a day. > I'd like to make them go away If you want to solve the problem sooner, then mail list is faster than kernel bugzilla. For your case, it's just a warning for unaligned device size, normally caused by older mkfs or older kernel. You can fix it by using "btrfs rescue fix-device-size " on a unmounted fs, or use "btrfs resize :-4K " to resize *each* device of your mounted fs. I'd prefer "btrfs rescue fix-device-size " if you have the latest version of btrfs-progs, as it does not only fix the unaligned device size, but also fix your super block total bytes. Thanks, Qu > > https://bugzilla.kernel.org/show_bug.cgi?id=196949 > > [4.747356] [ cut here ] > [4.747377] WARNING: CPU: 3 PID: 439 at fs/btrfs/ctree.h:1559 > btrfs_update_device+0x1c5/0x1d0 [btrfs] > [4.747377] Modules linked in: nfs_acl lockd grace sunrpc ip_tables > btrfs xor raid6_pq sd_mod crc32c_intel firewire_ohci igb ahci > firewire_core crc_itu_t dca libahci i915 libata i2c_algo_bit e1000e > drm_kms_helper ptp syscopyarea sysfillrect pps_core sysimgblt f > b_sys_fops drm video > [4.747385] CPU: 3 PID: 439 Comm: btrfs-cleaner Not tainted > 4.13.2-1.el7.elrepo.x86_64 #1 > [4.747385] Hardware name: Supermicro X10SAE/X10SAE, BIOS 2.0a 05/09/2014 > [4.747386] task: 88040cdcae80 task.stack: c900021f4000 > [4.747396] RIP: 0010:btrfs_update_device+0x1c5/0x1d0 [btrfs] > [4.747396] RSP: 0018:c900021f7d00 EFLAGS: 00010206 > [4.747397] RAX: 0fff RBX: 880407b7aa80 RCX: > 001bc6c71e00 > [4.747397] RDX: 8800 RSI: 880404cd3f3c RDI: > 880409417b58 > [4.747398] RBP: c900021f7d48 R08: 3f60 R09: > c900021f7cb8 > [4.747398] R10: 1000 R11: 0003 R12: > 88040559f800 > [4.747398] R13: R14: 880409417b58 R15: > 3f3c > [4.747399] FS: () GS:88041fac() > knlGS: > [4.747399] CS: 0010 DS: ES: CR0: 80050033 > [4.747400] CR2: 7f29c3000248 CR3: 0004056a4000 CR4: > 001406e0 > [4.747400] Call Trace: > [4.747410] btrfs_remove_chunk+0x2fb/0x8b0 [btrfs] > [4.747418] btrfs_delete_unused_bgs+0x363/0x440 [btrfs] > [4.747426] cleaner_kthread+0x150/0x180 [btrfs] > [4.747429] kthread+0x109/0x140 > [4.747436] ? btree_invalidatepage+0xa0/0xa0 [btrfs] > [4.747437] ? kthread_park+0x60/0x60 > [4.747439] ret_from_fork+0x25/0x30 > [4.747439] Code: 10 00 00 00 4c 89 fe e8 8a 30 ff ff 4c 89 f7 e8 > 32 f6 fc ff e9 d3 fe ff ff b8 f4 ff ff ff e9 d4 fe ff ff 0f 1f 00 e8 > bb 2e 9e e0 <0f> ff eb af 0f 1f 80 00 00 00 00 0f 1f 44 00 00 55 31 d2 > be 02 > [4.747450] ---[ end trace 1ef80a625983d73b ]--- > -- > To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in > the body of a message to majord...@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html > signature.asc Description: OpenPGP digital signature
Re: [PATCH v2 1/2] btrfs-progs: add BTRFS_DEV_ITEMS_OBJECTID in comment in print-tree
On 2019/8/28 下午10:06, Anand Jain wrote:
> So when searching for BTRFS_DEV_ITEMS_OBJECTID, it hits, albeit it is
> defined same as BTRFS_ROOT_TREE_OBJECTID.
>
> Signed-off-by: Anand Jain
Reviewed-by: Qu Wenruo
> ---
> v1->v2: Improve comment.
>
> print-tree.c | 5 +
> 1 file changed, 5 insertions(+)
>
> diff --git a/print-tree.c b/print-tree.c
> index b31e515f8989..b1c59d776547 100644
> --- a/print-tree.c
> +++ b/print-tree.c
> @@ -705,6 +705,11 @@ void print_objectid(FILE *stream, u64 objectid, u8 type)
>
> switch (objectid) {
> case BTRFS_ROOT_TREE_OBJECTID:
> + /*
> + * BTRFS_ROOT_TREE_OBJECTID and BTRFS_DEV_ITEMS_OBJECTID are
> + * defined with the same value of 1ULL, distinguish them by
> + * checking the type.
> + */
Oh, some bad design from the very beginning of btrfs.
Any other duplicated objectid?
Thanks,
Qu
> if (type == BTRFS_DEV_ITEM_KEY)
> fprintf(stream, "DEV_ITEMS");
> else
>
signature.asc
Description: OpenPGP digital signature
Re: No files in snapshot
Hi, I was thinking of this, too. But it does not apply. root@ld5507:~# btrfs su list -to /var/lib ID gen top level path -- --- - root@ld5507:~# btrfs su list -to /var ID gen top level path -- --- - And there are files in other directories: root@ld5507:~# ls -l /.snapshots/158/snapshot/var/lib/ceph/mgr/ceph-ld5507/ insgesamt 4 -rw-r--r-- 1 ceph ceph 61 Mai 28 14:33 keyring root@ld5507:~# ls -l /.snapshots/158/snapshot/var/lib/ceph/mon/ceph-ld5507/ insgesamt 12 -rw--- 1 ceph ceph 77 Mai 28 14:33 keyring -rw-r--r-- 1 ceph ceph 8 Mai 28 14:33 kv_backend -rw-r--r-- 1 ceph ceph 3 Aug 23 09:41 min_mon_release drwxr-xr-x 1 ceph ceph 244 Aug 26 18:37 store.db Only this directories /.snapshots/158/snapshot/var/lib/ceph/osd/ceph-/ are empty: root@ld5507:~# ls -l /.snapshots/158/snapshot/var/lib/ceph/osd/ceph-219/ insgesamt 0 To create a snapshot I run this command: snapper create --type single --description "validate /var/lib/ceph/osd/ceph-" Am 28.08.2019 um 00:24 schrieb Chris Murphy: On Tue, Aug 27, 2019 at 3:33 AM Thomas Schneider <74cmo...@gmail.com> wrote: However, I run into an issue and need to restore various files. I thought that I could simply take the files from a snapshot created before. However, the files required don't exist in any snapshot! Therefore I have created a new snapshot manually to verify if the files will be included, but there's nothing. Snapshots are not recursive on Btrfs. The snapshot will not extend into nested subvolumes. Check to see if you are snapshotting the proper subvolume. # btrfs sub list -to /var/lib # btrfs sub list -to /var/ In some sense these are redundant, I'm not sure if your /var/lib is a subvolume or not. Also please include the exact snapshot command you're making.
[PATCH][v2] btrfs: rework wake_all_tickets
Now that we no longer partially fill tickets we need to rework
wake_all_tickets to call btrfs_try_to_wakeup_tickets() in order to see
if any subsequent tickets are able to be satisfied. If our tickets_id
changes we know something happened and we can keep flushing.
Also if we find a ticket that is smaller than the first ticket in our
queue then we want to retry the flushing loop again in case
may_commit_transaction() decides we could satisfy the ticket by
committing the transaction.
Rename this to maybe_fail_all_tickets() while we're at it, to better
reflect what the function is actually doing.
Signed-off-by: Josef Bacik
---
v1->v2:
- added a comment for maybe_fail_all_tickets
fs/btrfs/space-info.c | 56 +--
1 file changed, 49 insertions(+), 7 deletions(-)
diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c
index c2143ddb7f4a..b2bb9d0bd44e 100644
--- a/fs/btrfs/space-info.c
+++ b/fs/btrfs/space-info.c
@@ -679,19 +679,61 @@ static inline int need_do_async_reclaim(struct
btrfs_fs_info *fs_info,
!test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state));
}
-static bool wake_all_tickets(struct list_head *head)
+/*
+ * maybe_fail_all_tickets - we've exhausted our flushing, start failing tickets
+ * @fs_info - fs_info for this fs
+ * @space_info - the space info we were flushing
+ *
+ * We call this when we've exhausted our flushing ability and haven't made
+ * progress in satisfying tickets. The reservation code handles tickets in
+ * order, so if there is a large ticket first and then smaller ones we could
+ * very well satisfy the smaller tickets. This will attempt to wake up any
+ * tickets in the list to catch this case.
+ *
+ * This function returns true if it was able to make progress by clearing out
+ * other tickets, or if it stumbles across a ticket that was smaller than the
+ * first ticket.
+ */
+static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info,
+ struct btrfs_space_info *space_info)
{
struct reserve_ticket *ticket;
+ u64 tickets_id = space_info->tickets_id;
+ u64 first_ticket_bytes = 0;
+
+ while (!list_empty(&space_info->tickets) &&
+ tickets_id == space_info->tickets_id) {
+ ticket = list_first_entry(&space_info->tickets,
+ struct reserve_ticket, list);
+
+ /*
+* may_commit_transaction will avoid committing the transaction
+* if it doesn't feel like the space reclaimed by the commit
+* would result in the ticket succeeding. However if we have a
+* smaller ticket in the queue it may be small enough to be
+* satisified by committing the transaction, so if any
+* subsequent ticket is smaller than the first ticket go ahead
+* and send us back for another loop through the enospc flushing
+* code.
+*/
+ if (first_ticket_bytes == 0)
+ first_ticket_bytes = ticket->bytes;
+ else if (first_ticket_bytes > ticket->bytes)
+ return true;
- while (!list_empty(head)) {
- ticket = list_first_entry(head, struct reserve_ticket, list);
list_del_init(&ticket->list);
ticket->error = -ENOSPC;
wake_up(&ticket->wait);
- if (ticket->bytes != ticket->orig_bytes)
- return true;
+
+ /*
+* We're just throwing tickets away, so more flushing may not
+* trip over btrfs_try_granting_tickets, so we need to call it
+* here to see if we can make progress with the next ticket in
+* the list.
+*/
+ btrfs_try_granting_tickets(fs_info, space_info);
}
- return false;
+ return (tickets_id != space_info->tickets_id);
}
/*
@@ -759,7 +801,7 @@ static void btrfs_async_reclaim_metadata_space(struct
work_struct *work)
if (flush_state > COMMIT_TRANS) {
commit_cycles++;
if (commit_cycles > 2) {
- if (wake_all_tickets(&space_info->tickets)) {
+ if (maybe_fail_all_tickets(fs_info,
space_info)) {
flush_state = FLUSH_DELAYED_ITEMS_NR;
commit_cycles--;
} else {
--
2.21.0
[PATCH][v2] btrfs: stop partially refilling tickets when releasing space
btrfs_space_info_add_old_bytes is used when adding the extra space from
an existing reservation back into the space_info to be used by any
waiting tickets. In order to keep us from overcommitting we check to
make sure that we can still use this space for our reserve ticket, and
if we cannot we'll simply subtract it from space_info->bytes_may_use.
However this is problematic, because it assumes that only changes to
bytes_may_use would affect our ability to make reservations. Any
changes to bytes_reserved would be missed. If we were unable to make a
reservation prior because of reserved space, but that reserved space was
free'd due to unlink or truncate and we were allowed to immediately
reclaim that metadata space we would still ENOSPC.
Consider the example where we create a file with a bunch of extents,
using up 2mib of actual space for the new tree blocks. Then we try to
make a reservation of 2mib but we do not have enough space to make this
reservation. The iput() occurs in another thread and we remove this
space, and since we did not write the blocks we simply do
space_info->bytes_reserved -= 2mib. We would never see this because we
do not check our space info used, we just try to re-use the freed
reservations.
To fix this problem, and to greatly simplify the wakeup code, do away
with this partial refilling nonsense. Use
btrfs_space_info_add_old_bytes to subtract the reservation from
space_info->bytes_may_use, and then check the ticket against the total
used of the space_info the same way we do with the initial reservation
attempt.
This keeps the reservation logic consistent and solves the problem of
early ENOSPC in the case that we free up space in places other than
bytes_may_use and bytes_pinned. Thanks,
Signed-off-by: Josef Bacik
---
v1->v2:
- Simply updated the changelog, what I was describing couldn't actually happen.
I went back and re-ran tests and added in tracing and realized it was
bytes_reserved that was changing without telling anybody, not that we were
removing more of bytes_may_use than expected. Updated the changelog to
reflect this as well as hopefully make it clearer the motivation for the
change.
fs/btrfs/space-info.c | 43 ---
1 file changed, 16 insertions(+), 27 deletions(-)
diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c
index a0a36d5768e1..357fe7548e07 100644
--- a/fs/btrfs/space-info.c
+++ b/fs/btrfs/space-info.c
@@ -233,52 +233,41 @@ void btrfs_space_info_add_old_bytes(struct btrfs_fs_info
*fs_info,
struct btrfs_space_info *space_info,
u64 num_bytes)
{
- struct reserve_ticket *ticket;
struct list_head *head;
- u64 used;
enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH;
- bool check_overcommit = false;
spin_lock(&space_info->lock);
head = &space_info->priority_tickets;
+ btrfs_space_info_update_bytes_may_use(fs_info, space_info, -num_bytes);
- /*
-* If we are over our limit then we need to check and see if we can
-* overcommit, and if we can't then we just need to free up our space
-* and not satisfy any requests.
-*/
- used = btrfs_space_info_used(space_info, true);
- if (used - num_bytes >= space_info->total_bytes)
- check_overcommit = true;
again:
- while (!list_empty(head) && num_bytes) {
- ticket = list_first_entry(head, struct reserve_ticket,
- list);
- /*
-* We use 0 bytes because this space is already reserved, so
-* adding the ticket space would be a double count.
-*/
- if (check_overcommit &&
- !can_overcommit(fs_info, space_info, 0, flush, false))
- break;
- if (num_bytes >= ticket->bytes) {
+ while (!list_empty(head)) {
+ struct reserve_ticket *ticket;
+ u64 used = btrfs_space_info_used(space_info, true);
+
+ ticket = list_first_entry(head, struct reserve_ticket, list);
+
+ /* Check and see if our ticket can be satisified now. */
+ if ((used + ticket->bytes <= space_info->total_bytes) ||
+ can_overcommit(fs_info, space_info, ticket->bytes, flush,
+ false)) {
+ btrfs_space_info_update_bytes_may_use(fs_info,
+ space_info,
+ ticket->bytes);
list_del_init(&ticket->list);
- num_bytes -= ticket->bytes;
ticket->bytes = 0;
space_info->tickets_id++;
wake_up(&ticket->wait);
} else {
- ticket
Re: [PATCH v2 1/2] btrfs-progs: add BTRFS_DEV_ITEMS_OBJECTID in comment in print-tree
On Wed, Aug 28, 2019 at 10:27:41PM +0800, Qu Wenruo wrote:
>
>
> On 2019/8/28 下午10:06, Anand Jain wrote:
> > So when searching for BTRFS_DEV_ITEMS_OBJECTID, it hits, albeit it is
> > defined same as BTRFS_ROOT_TREE_OBJECTID.
> >
> > Signed-off-by: Anand Jain
>
> Reviewed-by: Qu Wenruo
>
> > ---
> > v1->v2: Improve comment.
> >
> > print-tree.c | 5 +
> > 1 file changed, 5 insertions(+)
> >
> > diff --git a/print-tree.c b/print-tree.c
> > index b31e515f8989..b1c59d776547 100644
> > --- a/print-tree.c
> > +++ b/print-tree.c
> > @@ -705,6 +705,11 @@ void print_objectid(FILE *stream, u64 objectid, u8
> > type)
> >
> > switch (objectid) {
> > case BTRFS_ROOT_TREE_OBJECTID:
> > + /*
> > +* BTRFS_ROOT_TREE_OBJECTID and BTRFS_DEV_ITEMS_OBJECTID are
> > +* defined with the same value of 1ULL, distinguish them by
> > +* checking the type.
> > +*/
>
> Oh, some bad design from the very beginning of btrfs.
No, why bad design? Objectids need to be unique inside one group, like
tree ids, but are otherwise independent and must be interpreted together
with the key type. Which is exactly what print_objectid does.
Re: [PATCH 0/9][v3] Rework reserve ticket handling
On Fri, Aug 23, 2019 at 02:55:33PM +0200, David Sterba wrote: > On Thu, Aug 22, 2019 at 03:10:53PM -0400, Josef Bacik wrote: > > This is the next round of my reserve ticket refinements. Most of the > > changes > > are just fixing issues brought up by review. The updated diffstat is as > > follows > > > > fs/btrfs/block-group.c| 5 +- > > fs/btrfs/block-rsv.c | 10 +-- > > fs/btrfs/delalloc-space.c | 4 -- > > fs/btrfs/delayed-ref.c| 2 +- > > fs/btrfs/extent-tree.c| 13 +--- > > fs/btrfs/space-info.c | 171 > > +++--- > > fs/btrfs/space-info.h | 30 +--- > > 7 files changed, 98 insertions(+), 137 deletions(-) > > I'll add the series to for-next as topic branch, the comments seem to be > more in the changelog or function names, that I'll updated and fold to > the patches (no need to resend the whole patchset). Updated and pushed to misc-next.
Re: [PATCH 0/5][v2] Fix global reserve size and can overcommit
On Fri, Aug 23, 2019 at 03:17:26PM +0200, David Sterba wrote: > On Thu, Aug 22, 2019 at 03:18:59PM -0400, Josef Bacik wrote: > > This didn't series didn't change much, but I did move things around a > > little bit > > in previous series so these needed to be updated. > > > > fs/btrfs/block-rsv.c | 36 ++-- > > fs/btrfs/space-info.c | 51 > > ++- > > 2 files changed, 52 insertions(+), 35 deletions(-) > > Added as topic branch to for-next (based on the ticket rework patchset). Moved to misc-next.
Re: [PATCH v3 2/4] btrfs: create structure to encode checksum type and length
On Mon, Aug 26, 2019 at 01:48:32PM +0200, Johannes Thumshirn wrote:
> Create a structure to encode the type and length for the known on-disk
> checksums.
>
> This makes it easier to add new checksums later.
>
> Signed-off-by: Johannes Thumshirn
>
> ---
> Changes to v2:
> - Really remove initializer macro *doh*
>
> Changes to v1:
> - Remove initializer macro (David)
> ---
> fs/btrfs/ctree.h | 13 -
> 1 file changed, 8 insertions(+), 5 deletions(-)
>
> diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
> index b161224b5a0b..139354d02dfa 100644
> --- a/fs/btrfs/ctree.h
> +++ b/fs/btrfs/ctree.h
> @@ -82,9 +82,12 @@ struct btrfs_ref;
> */
> #define BTRFS_LINK_MAX 65535U
>
> -/* four bytes for CRC32 */
> -static const int btrfs_csum_sizes[] = { 4 };
> -static const char *btrfs_csum_names[] = { "crc32c" };
> +static const struct btrfs_csums {
> + u16 size;
> + const char *name;
> +} btrfs_csums[] = {
> + [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" },
> +};
In one of the previous iterations, I pointed out that the definition is
in a header, thus each file that includes "ctree.h" (many of them)
has a private copy of the table. With just crc32c it's just a few bytes
that gets lost in the noise but now the table is going to be larger the
impact will be noticeable.
Re: [PATCH v3 2/4] btrfs: create structure to encode checksum type and length
On Wed, Aug 28, 2019 at 09:19:52PM +0200, David Sterba wrote:
> On Mon, Aug 26, 2019 at 01:48:32PM +0200, Johannes Thumshirn wrote:
> > Create a structure to encode the type and length for the known on-disk
> > checksums.
> >
> > This makes it easier to add new checksums later.
> >
> > Signed-off-by: Johannes Thumshirn
> >
> > ---
> > Changes to v2:
> > - Really remove initializer macro *doh*
> >
> > Changes to v1:
> > - Remove initializer macro (David)
> > ---
> > fs/btrfs/ctree.h | 13 -
> > 1 file changed, 8 insertions(+), 5 deletions(-)
> >
> > diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
> > index b161224b5a0b..139354d02dfa 100644
> > --- a/fs/btrfs/ctree.h
> > +++ b/fs/btrfs/ctree.h
> > @@ -82,9 +82,12 @@ struct btrfs_ref;
> > */
> > #define BTRFS_LINK_MAX 65535U
> >
> > -/* four bytes for CRC32 */
> > -static const int btrfs_csum_sizes[] = { 4 };
> > -static const char *btrfs_csum_names[] = { "crc32c" };
> > +static const struct btrfs_csums {
> > + u16 size;
> > + const char *name;
> > +} btrfs_csums[] = {
> > + [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" },
> > +};
>
> In one of the previous iterations, I pointed out that the definition is
> in a header, thus each file that includes "ctree.h" (many of them)
> has a private copy of the table. With just crc32c it's just a few bytes
> that gets lost in the noise but now the table is going to be larger the
> impact will be noticeable.
With definitions moved to ctree.c and only exported the two helpers that
need btrfs_csumss
textdata bss dec hex filename
1080108 17316 14912 1112336 10f910 btrfs.ko.before
1079655 17316 14912 883 10f74b btrfs.ko.after
The difference is 453.
Spare Volume Features
Hi All, I am currently buidling a small data server for an experiment. I was wondering if the features of the spare volume introduced a couple of years ago (ttps://patchwork.kernel.org/patch/8687721/) would be release soon. I think this would be awesome to have a drive installed, that can be used as a spare if one drive of an array died to avoid downtime. Does anyone have news about it, and when it will be officially in the kernel/btrfs-progs ? Marc P.S. It took me a long time to switch to btrfs. I did it less than a year ago, and I love it. Keep the great job going, y'all
Re: Spare Volume Features
On August 28, 2019 5:51:02 PM PDT, Marc Oggier wrote: >Hi All, > >I am currently buidling a small data server for an experiment. > >I was wondering if the features of the spare volume introduced a couple > >of years ago (ttps://patchwork.kernel.org/patch/8687721/) would be >release soon. I think this would be awesome to have a drive installed, >that can be used as a spare if one drive of an array died to avoid >downtime. > >Does anyone have news about it, and when it will be officially in the >kernel/btrfs-progs ? > >Marc > >P.S. It took me a long time to switch to btrfs. I did it less than a >year ago, and I love it. Keep the great job going, y'all I've been thinking about this issue myself, and I have an (untested) idea for how to accomplish something similar. My file server has three disks in a btrfs raid1. I added a fourth disk to the array as just a normal, participating disk. I keep an eye on the usage to make sure that I never exceed 3 disk's worth of usage. That way, if one disk dies, there are still enough disks to mount RW (though I may still need to do an explicit degraded mount, not sure). In that scenario, I can just trigger an online full balance to rebuild the missing raid copies on the remaining disks. In theory, minimal to no downtime. I'm curious if anyone can see any problems with this idea. I've never tested it, and my offsite backups are thorough enough to survive downtime anyway. --Sean
[PATCH RFC v2.1] btrfs-progs: add readmirror property and ioctl to set get
This patch adds readmirror property to be applied at the filesystem object.
And uses ioctl BTRFS_IOC_GET_READMIRROR and BTRFS_IOC_SET_READMIRROR to get
and set the property respectively.
Signed-off-by: Anand Jain
---
RFC v2->RFC v2.1:
add dump-tree support, depends on the patch
[PATCH] btrfs-progs: print-tree add missing DEV_STATS
v1->RFC v2:
Changed format specifier from devid1,2,3.. to devid:1,2,3..
ctree.h | 14 ++
ioctl.h | 14 ++
libbtrfsutil/btrfs_tree.h | 11 +
print-tree.c | 15 ++
props.c | 98 +++
5 files changed, 152 insertions(+)
diff --git a/ctree.h b/ctree.h
index 0d12563b7261..f9523ea4cea6 100644
--- a/ctree.h
+++ b/ctree.h
@@ -92,6 +92,9 @@ struct btrfs_free_space_ctl;
/* device stats in the device tree */
#define BTRFS_DEV_STATS_OBJECTID 0ULL
+/* store readmirror policy inforamtion in the device tree */
+#define BTRFS_READMIRROR_OBJECTID -3ULL
+
/* for storing balance parameters in the root tree */
#define BTRFS_BALANCE_OBJECTID -4ULL
@@ -879,6 +882,14 @@ struct btrfs_balance_item {
__le64 unused[4];
} __attribute__ ((__packed__));
+/*
+ * readmirror's persistent storage format
+ */
+struct btrfs_readmirror_item {
+ __le64 type;
+ __le64 unused[3];
+} __attribute__ ((__packed__));
+
#define BTRFS_FILE_EXTENT_INLINE 0
#define BTRFS_FILE_EXTENT_REG 1
#define BTRFS_FILE_EXTENT_PREALLOC 2
@@ -2389,6 +2400,9 @@ BTRFS_SETGET_STACK_FUNCS(stack_qgroup_limit_rsv_exclusive,
/* btrfs_balance_item */
BTRFS_SETGET_FUNCS(balance_item_flags, struct btrfs_balance_item, flags, 64);
+/* btrfs_readmirror_item */
+BTRFS_SETGET_FUNCS(readmirror_type, struct btrfs_readmirror_item, type, 64);
+
static inline struct btrfs_disk_balance_args* btrfs_balance_item_data(
struct extent_buffer *eb, struct btrfs_balance_item *bi)
{
diff --git a/ioctl.h b/ioctl.h
index 66ee599f7a82..ccdf600bae77 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -765,6 +765,16 @@ struct btrfs_ioctl_get_subvol_rootref_args {
};
BUILD_ASSERT(sizeof(struct btrfs_ioctl_get_subvol_rootref_args) == 4096);
+enum btrfs_readmirror_policy {
+ BTRFS_READMIRROR_DEFAULT = 0,
+ BTRFS_READMIRROR_DEVID,
+};
+
+struct btrfs_ioctl_readmirror_args {
+ __u64 type; /* RW */
+ __u64 device_bitmap; /* RW */
+};
+
/* Error codes as returned by the kernel */
enum btrfs_err_code {
notused,
@@ -929,6 +939,10 @@ static inline char *btrfs_err_str(enum btrfs_err_code
err_code)
struct btrfs_ioctl_get_subvol_rootref_args)
#define BTRFS_IOC_INO_LOOKUP_USER _IOWR(BTRFS_IOCTL_MAGIC, 62, \
struct btrfs_ioctl_ino_lookup_user_args)
+#define BTRFS_IOC_GET_READMIRROR _IOWR(BTRFS_IOCTL_MAGIC, 63, \
+ struct btrfs_ioctl_readmirror_args)
+#define BTRFS_IOC_SET_READMIRROR _IOWR(BTRFS_IOCTL_MAGIC, 64, \
+ struct btrfs_ioctl_readmirror_args)
#ifdef __cplusplus
}
#endif
diff --git a/libbtrfsutil/btrfs_tree.h b/libbtrfsutil/btrfs_tree.h
index 8ea3e31d9b96..b6785fef4a8d 100644
--- a/libbtrfsutil/btrfs_tree.h
+++ b/libbtrfsutil/btrfs_tree.h
@@ -51,6 +51,9 @@
/* device stats in the device tree */
#define BTRFS_DEV_STATS_OBJECTID 0ULL
+/* store readmirror policy inforamtion in the device tree */
+#define BTRFS_READMIRROR_OBJECTID -3ULL
+
/* for storing balance parameters in the root tree */
#define BTRFS_BALANCE_OBJECTID -4ULL
@@ -962,4 +965,12 @@ struct btrfs_qgroup_limit_item {
__le64 rsv_excl;
} __attribute__ ((__packed__));
+/*
+ * readmirror's persistent storage format
+ */
+struct btrfs_readmirror_item {
+ __le64 type;
+ __le64 unused[3];
+} __attribute__ ((__packed__));
+
#endif /* _BTRFS_CTREE_H_ */
diff --git a/print-tree.c b/print-tree.c
index 15d8806b6f28..4ed698ee3a62 100644
--- a/print-tree.c
+++ b/print-tree.c
@@ -692,6 +692,8 @@ void print_objectid(FILE *stream, u64 objectid, u8 type)
case BTRFS_PERSISTENT_ITEM_KEY:
if (objectid == BTRFS_DEV_STATS_OBJECTID)
fprintf(stream, "DEV_STATS");
+ else if (objectid == BTRFS_READMIRROR_OBJECTID)
+ fprintf(stream, "READMIRROR");
else
fprintf(stream, "%llu", (unsigned long long)objectid);
return;
@@ -960,6 +962,16 @@ static void print_balance_item(struct extent_buffer *eb,
print_disk_balance_args(btrfs_balance_item_sys(eb, bi));
}
+static void print_readmirror(struct extent_buffer *eb,
+struct btrfs_readmirror_item *rm, u32 size)
+{
+ if (btrfs_readmirror_type(eb, rm) == 1)
+ printf("\t\treadmirror.type devid\n");
+ else
+ printf("\t\treadmirror.type 0x%llx\n",
+ btrfs_readmirror_type(eb, rm));
+}
+
static void print_dev_stats(struc
