[PATCH] btrfs-progs: remove unused function path_cat and path_cat3

[Gu Jinxiang]

Since function path_cat and path_cat3 are not used anymore, remove them.

Signed-off-by: Gu Jinxiang 
---
 send-utils.c | 25 -
 send-utils.h |  9 -
 2 files changed, 34 deletions(-)

diff --git a/send-utils.c b/send-utils.c
index b5289e76..3ecbdea6 100644
--- a/send-utils.c
+++ b/send-utils.c
@@ -757,18 +757,6 @@ int path_cat_out(char *out, const char *p1, const char *p2)
return 0;
 }
 
-__attribute__((deprecated))
-char *path_cat(const char *p1, const char *p2)
-{
-   int p1_len = strlen(p1);
-   int p2_len = strlen(p2);
-   char *new = malloc(p1_len + p2_len + 2);
-
-   path_cat_out(new, p1, p2);
-
-   return new;
-}
-
 int path_cat3_out(char *out, const char *p1, const char *p2, const char *p3)
 {
int p1_len = strlen(p1);
@@ -788,16 +776,3 @@ int path_cat3_out(char *out, const char *p1, const char 
*p2, const char *p3)
 
return 0;
 }
-
-__attribute__((deprecated))
-char *path_cat3(const char *p1, const char *p2, const char *p3)
-{
-   int p1_len = strlen(p1);
-   int p2_len = strlen(p2);
-   int p3_len = strlen(p3);
-   char *new = malloc(p1_len + p2_len + p3_len + 3);
-
-   path_cat3_out(new, p1, p2, p3);
-
-   return new;
-}
diff --git a/send-utils.h b/send-utils.h
index e8f86912..5f5e096f 100644
--- a/send-utils.h
+++ b/send-utils.h
@@ -104,15 +104,6 @@ void subvol_uuid_search_add(struct subvol_uuid_search *s,
 
 int btrfs_subvolid_resolve(int fd, char *path, size_t path_len, u64 subvol_id);
 
-/*
- * DEPRECATED: the functions path_cat and path_cat3 are unsafe and should not
- * be used, use the _out variants and always check the return code.
- */
-__attribute__((deprecated))
-char *path_cat(const char *p1, const char *p2);
-__attribute__((deprecated))
-char *path_cat3(const char *p1, const char *p2, const char *p3);
-
 int path_cat_out(char *out, const char *p1, const char *p2);
 int path_cat3_out(char *out, const char *p1, const char *p2, const char *p3);
 
-- 
2.14.3



--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v3] fstests: btrfs/159 superblock corruption test case

[Anand Jain]

Verify if the superblock corruption is handled correctly.

Signed-off-by: Anand Jain 
---
v2->v3:
 Provide the disk to be corrupted as an arg, instead of swapping the devices,
  so drop mount_opt_minus_args().
 159.out slightly changed.
v1->v2:
 $subject slightly changed
 Added more info about the test-case
 Keep the stuff from the ./new btrfs
 Add mount_opt_minus_args() to get the options (if) set at the config file
 Move DEV_GOOD & DEV_BAD to where it starts to use
 To help debugging added run_check where possible
 Remove {} in the out file
 Use _filter_error_mount for mount fail cases other than -EINVAL

 tests/btrfs/159 | 149 
 tests/btrfs/159.out |  21 
 tests/btrfs/group   |   1 +
 3 files changed, 171 insertions(+)
 create mode 100755 tests/btrfs/159
 create mode 100644 tests/btrfs/159.out

diff --git a/tests/btrfs/159 b/tests/btrfs/159
new file mode 100755
index ..c3a50b58b0b9
--- /dev/null
+++ b/tests/btrfs/159
@@ -0,0 +1,149 @@
+#! /bin/bash
+# FS QA Test 159
+#
+# Test if the superblock corruption is handled correctly:
+#  - Test fsid miss-match (csum ok) between primary and copy superblock
+#  Fixed by the ML patch:
+#  btrfs: check if the fsid in the primary sb and copy sb are same
+#  - Test if the mount fails if the primary superblock csum is
+#  corrupted on any disk
+#  - Test if the mount does not fail if the copy1 sb csum is corrupted
+#  Fixed by the ML patches:
+#  btrfs: verify superblock checksum during scan
+#  btrfs: verify checksum for all devices in mount context
+#
+#-
+# Copyright (c) 2018 Oracle.  All Rights Reserved.
+# Author: Anand Jain 
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#---
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1   # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+   cd /
+   rm -f $tmp.*
+   _scratch_dev_pool_put
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/module
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch_dev_pool 2
+_require_loadable_fs_module "btrfs"
+_require_command "$WIPEFS_PROG" wipefs
+
+_scratch_dev_pool_get 2
+
+wipe()
+{
+   $WIPEFS_PROG -a $SCRATCH_DEV_POOL > /dev/null 2>&1
+}
+
+# Test for fsid miss-match (csum ok) with primary and copy superblock.
+check_copy1_fsid()
+{
+   local bytenr=67108864
+   echo -e "\\ncheck_copy1_fsid\\n" | tee -a $seqres.full
+
+   wipe
+   $MKFS_BTRFS_PROG -fq $EXTRA_DEV
+   _scratch_mkfs >> $seqres.full 2>&1
+   _reload_fs_module "btrfs"
+
+   run_check dd status=none of=$SCRATCH_DEV if=$EXTRA_DEV ibs=1 obs=1\
+   skip=$bytenr seek=$bytenr count=4K
+   $WIPEFS_PROG -a $EXTRA_DEV > /dev/null 2>&1
+
+   #must fail
+   _try_scratch_mount 2>&1 | _filter_scratch_pool
+   #If mount didn't fail
+   _scratch_unmount > /dev/null 2>&1
+}
+
+# Test if the mount fails if the primary superblock csum is corrupted.
+# arg 1 is dev to corrupt
+check_primary()
+{
+   local bytenr=65536
+   echo -e "\\ncheck_primary\\n" | tee -a $seqres.full
+
+   wipe
+   _scratch_pool_mkfs "-mraid1 -draid1"
+   _reload_fs_module "btrfs"
+
+   #corrupt primary superblock bytenr
+   od -j$bytenr -N1 -An -x $1 |\
+   dd status=none of=$1 ibs=1 obs=1 seek=$bytenr count=1 conv=fsync
+
+   #must fail
+   _try_scratch_mount "-o device=$EXTRA_DEV" 2>&1 | _filter_error_mount
+
+   #If mount didn't fail
+   _scratch_unmount > /dev/null 2>&1
+}
+
+# If copy1 is corrupted we still should be able to mount, check that.
+# arg 1 is dev to corrupt
+check_copy1()
+{
+   local bytenr=67108864
+   echo -e "\\ncheck_copy1" | tee -a $seqres.full
+
+   wipe
+   _scratch_pool_mkfs "-mraid1 -draid1"
+   _reload_fs_module "btrfs"
+
+   #corrupt copy1 superrblock bytenr
+   od -j$bytenr -N1 -An -x $1 |\
+   dd status=none of=$1 

Re: [PATCH v2] fstests: btrfs/159 superblock corruption test case

[Anand Jain]

+mount_opt_minus_args()
+{
+   local nr
+   local mnt_opt=""
+
+   nr=`_scratch_mount_options | awk '{print NF}'`
+   if [ $nr -eq 4 ]; then


Seems this only works with "scratch_mount_options" returns something
like:

"-o opt dev mnt" or "-o opt1,opt2 dev mnt"

but not

"-oopt dev mnt" nor
"-o opt1 -o opt2 dev mnt" nor if $SELINUX_MOUNT_OPTIONS not empty.

Also if MOUNT_OPTIONS is "-oopt1 -oopt2", mount_opt_minus_args would
return something like "-o -oopt2,device=", which are not valid
mount options.


 Oh. Yes. This function is getting complicated. I have dropped this
 function in v3 which is kind of much better.


+# Test if the mount fails if the primary superblock csum is corrupted.
+check_primary()
+{
+   local bytenr=65536
+   echo -e "\\ncheck_primary\\n" | tee -a $seqres.full
+
+   wipe
+   _scratch_pool_mkfs "-mraid1 -draid1"
+   _reload_fs_module "btrfs"
+
+   #corrupt bytenr DEV_BAD
+   od -j$bytenr -N1 -An -x $DEV_BAD |\
+   dd status=none of=$DEV_BAD ibs=1 obs=1 seek=$bytenr count=1 
conv=fsync
+
+   #must fail
+   _mount `mount_opt_minus_args device=$DEV_GOOD` $DEV_BAD $SCRATCH_MNT 
2>&1 |\
+ _filter_error_mount


If I read the code correctly, the purpose of mount_opt_minus_args is to
remove the device and mount point mount arguments and keep other options
if there's any, then append "-o device=" as mount options.

If so, I think you could use "_common_dev_mount_options", e.g.

_mount `_common_dev_mount_options -o device=$DEV_GOOD` $DEV_BAD 
$SCRATCH_MNT ...

But do mount options really matter in this test? Could we just mount
with "-o device="?


 In fact for this test it doesn't matter, yes we could just use
 "-o device=". But I thought its xfstests rule that it must be
 used. Anyway in V3, I am changing the device itself on which
 corrupted-SB would reside, so now we can let the default _scratch_mount
 do its job as usual. Which means it shall use the config mount options.
 V3 is in the ML for the review.

Thanks, Anand


Thanks,
Eryu



--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH] btrfs-progs: mark -f option of btrfs scrub start deprecated

[Gu Jinxiang]

Since in the commit below detect the scrub information automatically,
-f option is no more need.
Let's make it deprecated.
commit d5fd05a773e2
("btrfs-progs: scrub, detect stale information in the status file")

Signed-off-by: Gu Jinxiang 
---
 Documentation/btrfs-scrub.asciidoc | 9 ++---
 cmds-scrub.c   | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/Documentation/btrfs-scrub.asciidoc 
b/Documentation/btrfs-scrub.asciidoc
index d2d20627..89bca18e 100644
--- a/Documentation/btrfs-scrub.asciidoc
+++ b/Documentation/btrfs-scrub.asciidoc
@@ -75,9 +75,12 @@ set IO priority class (see `ionice`(1) manpage)
 -n 
 set IO priority classdata (see `ionice`(1) manpage)
 -f
-force starting new scrub even if a scrub is already running,
-this can useful when scrub status file is damaged and reports a running
-scrub although it is not, but should not normally be necessary
+*deprecated, will be removed.*
++
+An option to force starting new scrub when scrub status file is damaged and
+reports a running scrub although it is not, since whether a scrub is really
+running can be judged automatically now, this option should not be necessary
+anymore.
 
 *status* [-d] |::
 Show status of a running scrub for the filesystem identified by 'path' or
diff --git a/cmds-scrub.c b/cmds-scrub.c
index 6b909f20..8cf1e5df 100644
--- a/cmds-scrub.c
+++ b/cmds-scrub.c
@@ -1569,7 +1569,7 @@ static const char * const cmd_scrub_start_usage[] = {
"-R raw print mode, print full data instead of summary",
"-c set ioprio class (see ionice(1) manpage)",
"-n set ioprio classdata (see ionice(1) manpage)",
-   "-f force starting new scrub even if a scrub is already running",
+   "-f (deprecated) force starting new scrub even if a scrub is 
already running",
"   this is useful when scrub stats record file is damaged",
NULL
 };
-- 
2.14.3



--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] btrfs: Fix possible softlock on single core machines

[Sasha Levin]

Hi,

[This is an automated email]

This commit has been processed because it contains a "Fixes:" tag,
fixing commit: 6d74119f1a3e Btrfs: avoid taking the chunk_mutex in 
do_chunk_alloc.

The bot has also determined it's probably a bug fixing patch. (score: 55.2868)

The bot has tested the following trees: v4.16, v4.15.15, v4.14.32, v4.9.92, 
v4.4.126.

v4.16: Build OK!
v4.15.15: Build OK!
v4.14.32: Build OK!
v4.9.92: Build OK!
v4.4.126: Build OK!

--
Thanks,
Sasha--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

btrfs check lowmem, referencer count mismatch

[Chris Murphy]

Hi,

btrfs-progs 4.16, kernel 4.16

--mode original comes up with no errors, --mode lowmem comes up with a
bunch of "referencer count mismatch" messages and also ERROR: errors
found in extent allocation tree or chunk allocation, and most of the
byte totals differ as well.

lowmem

found 807241723904 bytes used, error(s) found
total csum bytes: 783568540
total tree bytes: 8539635712
total fs tree bytes: 7533740032
total extent tree bytes: 171425792
btree space waste bytes: 1366949310
file data blocks allocated: 35905640476672
< referenced 3122882674688
> referenced 6583743541248

A btrfs-image for this same file system was submitted for other lowmem
complaints including this one, but seems there are more of this
message now with 4.16, and no other messages. But I can make another
image if that's helpful or whatever other info is needed.

File system is working fine near as I can tell, and passed an online
scrub in the past week with zero errors.

compat_flags0x0
compat_ro_flags0x0
incompat_flags0x161
( MIXED_BACKREF |
  BIG_METADATA |
  EXTENDED_IREF |
  SKINNY_METADATA )

[chris@f28s ~]$ sudo btrfs fi df -b /mnt/first
Data, single: total=803158884352, used=802571710464
System, DUP: total=33554432, used=98304
Metadata, DUP: total=5905580032, used=4669915136
GlobalReserve, single: total=536870912, used=0

[chris@f28s ~]$ sudo btrfs fi show --raw /mnt/first
Label: 'first'  uuid: 0f43f49d-6e63-4b1b-bc8c-c54da409872d
Total devices 1 FS bytes used 807241723904
devid2 size 1099509530624 used 815037153280 path /dev/mapper/first


128 subvolumes, 122 of which are ro snapshots


Attached file contains both btrfs checks and dump super


-- 
Chris Murphy
[chris@f28s ~]$ sudo btrfs --version
btrfs-progs v4.16
[chris@f28s ~]$ sudo btrfs check /dev/mapper/first
Checking filesystem on /dev/mapper/first
UUID: 0f43f49d-6e63-4b1b-bc8c-c54da409872d
checking extents
checking free space cache
checking fs roots
checking csums
checking root refs
found 807219568640 bytes used, no error found
total csum bytes: 783568540
total tree bytes: 4670013440
total fs tree bytes: 3664117760
total extent tree bytes: 171425792
btree space waste bytes: 714424438
file data blocks allocated: 18927213621248
 referenced 3122882674688

[chris@f28s ~]$ sudo btrfs check --mode lowmem /dev/mapper/first
Checking filesystem on /dev/mapper/first
UUID: 0f43f49d-6e63-4b1b-bc8c-c54da409872d
checking extents
ERROR: extent[1258793181184, 81903616] referencer count mismatch (root: 2192, 
owner: 323075, offset: 0) wanted: 551, have: 625
ERROR: extent[1258899460096, 82878464] referencer count mismatch (root: 2192, 
owner: 323115, offset: 0) wanted: 521, have: 633
ERROR: extent[1259036663808, 64319488] referencer count mismatch (root: 2192, 
owner: 323125, offset: 0) wanted: 376, have: 491
ERROR: extent[1259102384128, 64552960] referencer count mismatch (root: 2192, 
owner: 323137, offset: 0) wanted: 439, have: 493
ERROR: extent[1259171381248, 113295360] referencer count mismatch (root: 2192, 
owner: 323147, offset: 0) wanted: 763, have: 865
ERROR: extent[1259307835392, 2576384] referencer count mismatch (root: 2192, 
owner: 323191, offset: 0) wanted: 9, have: 20
ERROR: extent[1259327455232, 30089216] referencer count mismatch (root: 2192, 
owner: 323203, offset: 0) wanted: 204, have: 230
ERROR: extent[1259372060672, 1974272] referencer count mismatch (root: 2192, 
owner: 323218, offset: 0) wanted: 13, have: 16
ERROR: extent[1259924090880, 22102016] referencer count mismatch (root: 2192, 
owner: 323199, offset: 427819008) wanted: 52, have: 169
ERROR: extent[1259960025088, 2211840] referencer count mismatch (root: 2192, 
owner: 323244, offset: 0) wanted: 11, have: 17
ERROR: extent[1260158959616, 53563392] referencer count mismatch (root: 2192, 
owner: 323328, offset: 0) wanted: 328, have: 409
ERROR: extent[1260258131968, 134217728] referencer count mismatch (root: 2192, 
owner: 323609, offset: 0) wanted: 959, have: 1024
ERROR: extent[1260410531840, 1716224] referencer count mismatch (root: 2192, 
owner: 323625, offset: 0) wanted: 4, have: 14
ERROR: extent[1261505388544, 1839104] referencer count mismatch (root: 2192, 
owner: 323654, offset: 0) wanted: 10, have: 15
ERROR: extent[1261846781952, 56676352] referencer count mismatch (root: 2192, 
owner: 323731, offset: 0) wanted: 354, have: 433
ERROR: extent[1262246526976, 2256896] referencer count mismatch (root: 2192, 
owner: 323750, offset: 0) wanted: 10, have: 18
ERROR: extent[1262765875200, 765952] referencer count mismatch (root: 2192, 
owner: 319045, offset: 0) wanted: 5, have: 6
ERROR: extent[1262984384512, 1642496] referencer count mismatch (root: 2192, 
owner: 322213, offset: 0) wanted: 6, have: 13
ERROR: extent[1263573057536, 1159168] referencer count mismatch (root: 2192, 
owner: 323009, offset: 0) wanted: 2, have: 9
ERROR: extent[1263615098880, 1220608] referencer count mismatch (root: 2192, 
owner: 323062, offset: 0) 

Re: Unable to compile btrfs progs 4.16 on ubuntu Xenial

[Menion]

Ok, that was missing, also python3—setuptools is required.
I think it is worth to add in the wiki the packages dependencies
Bye

2018-04-08 10:51 GMT+02:00 Nikolay Borisov :
>
>
> On  7.04.2018 20:16, Menion wrote:
>> Hi all
>> Apparently it is not possible to compile with python bindings the
>> btrfs progs on ubuntu xenial
>>
>> checking for a Python interpreter with version >= 3.4... python3
>> checking for python3... /usr/bin/python3
>> checking for python3 version... 3.5
>> checking for python3 platform... linux
>> checking for python3 script directory... 
>> ${prefix}/lib/python3.5/site-packages
>> checking for python3 extension module directory...
>> ${exec_prefix}/lib/python3.5/site-packages
>> checking for python-3.5... no
>> configure: error: Package requirements (python-3.5) were not met:
>>
>> No package 'python-3.5' found
>>
>> Consider adjusting the PKG_CONFIG_PATH environment variable if you
>> installed software in a non-standard prefix.
>>
>> Alternatively, you may set the environment variables PYTHON_CFLAGS
>> and PYTHON_LIBS to avoid the need to call pkg-config.
>> See the pkg-config man page for more details.
>>
>> /usr/lib/python3.5/site-packages exists, but on Ubuntu the package
>> name is python3.5 and not python-3.5
>
>
> Do you have libpython3-dev installed? When I installed it on my ubuntu i
> could compile the progs irrespective of whether --prefix was passed or not
>>
>> Bye
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
>> the body of a message to majord...@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] fstests: generic test for fsync after fallocate

[Filipe Manana]

On Sun, Apr 8, 2018 at 3:07 AM, Eryu Guan  wrote:
> On Thu, Apr 05, 2018 at 10:56:14PM +0100, fdman...@kernel.org wrote:
>> From: Filipe Manana 
>>
>> Test that fsync operations preserve extents allocated with fallocate(2)
>> that are placed beyond a file's size.
>>
>> This test is motivated by a bug found in btrfs where unwritten extents
>> beyond the inode's i_size were not preserved after a fsync and power
>> failure. The btrfs bug is fixed by the following patch for the linux
>> kernel:
>>
>>  "Btrfs: fix loss of prealloc extents past i_size after fsync log replay"
>>
>> Signed-off-by: Filipe Manana 
>
> Hmm, xfs fails this test, while ext4 passes.
>
> # diff -u tests/generic/483.out 
> /root/workspace/xfstests/results//xfs_4k_crc/generic/483.out.bad
> --- tests/generic/483.out   2018-04-07 23:35:00.55511 +0800
> +++ /root/workspace/xfstests/results//xfs_4k_crc/generic/483.out.bad
> 2018-04-07 23:39:48.780659707 +0800
> @@ -6,5 +6,5 @@
>  0: [0..511]: data
>  1: [512..2559]: unwritten
>  File baz fiemap:
> -0: [0..511]: data
> -1: [512..6143]: unwritten
> +0: [0..895]: data
> +1: [896..6143]: unwritten
>
> I'm not sure what the problem is yet, but IMHO controlling on-disk
> layout of a file from userspace is hard and should be avoided if
> possible.

Well, both ext4 and f2fs give the same result, which makes sense, if
userspace has only written to the first 256K of the file, only blocks
0..511 should be reported as written.
However xfs decides to report more than that was written (448K), seems
like a bug to me.
The goal of fallocate(), specifically here, is precisely to create
unwritten extents beyond eof.

>
> Why not dumping md5sum to .out file like other dmflakey tests? I've
> checked the md5sum of all the three test files, and they're the same on
> xfs as on ext4, so the files are not corrupted on xfs.

Nop, that won't work. Because userspace can't read extents beyond eof.
That would make unpatched btrfs succeed the test (or any fs with
similar bug persisting prealloc extents beyond eof).
Without fiemap, how can one check that unwritten extents beyond eof exist?

>
> Thanks,
> Eryu
>
>> ---
>>  tests/generic/482 | 118 
>> ++
>>  tests/generic/482.out |  10 +
>>  tests/generic/group   |   1 +
>>  3 files changed, 129 insertions(+)
>>  create mode 100755 tests/generic/482
>>  create mode 100644 tests/generic/482.out
>>
>> diff --git a/tests/generic/482 b/tests/generic/482
>> new file mode 100755
>> index ..43bbc913
>> --- /dev/null
>> +++ b/tests/generic/482
>> @@ -0,0 +1,118 @@
>> +#! /bin/bash
>> +# FSQA Test No. 482
>> +#
>> +# Test that fsync operations preserve extents allocated with fallocate(2) 
>> that
>> +# are placed beyond a file's size.
>> +#
>> +#---
>> +#
>> +# Copyright (C) 2018 SUSE Linux Products GmbH. All Rights Reserved.
>> +# Author: Filipe Manana 
>> +#
>> +# This program is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU General Public License as
>> +# published by the Free Software Foundation.
>> +#
>> +# This program is distributed in the hope that it would be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> +# GNU General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program; if not, write the Free Software Foundation,
>> +# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
>> +#---
>> +#
>> +
>> +seq=`basename $0`
>> +seqres=$RESULT_DIR/$seq
>> +echo "QA output created by $seq"
>> +tmp=/tmp/$$
>> +status=1 # failure is the default!
>> +trap "_cleanup; exit \$status" 0 1 2 3 15
>> +
>> +_cleanup()
>> +{
>> + _cleanup_flakey
>> + cd /
>> + rm -f $tmp.*
>> +}
>> +
>> +# get standard environment, filters and checks
>> +. ./common/rc
>> +. ./common/filter
>> +. ./common/dmflakey
>> +. ./common/punch
>> +
>> +# real QA test starts here
>> +_supported_fs generic
>> +_supported_os Linux
>> +_require_scratch
>> +_require_dm_target flakey
>> +_require_xfs_io_command "falloc" "-k"
>> +_require_xfs_io_command "fiemap"
>> +
>> +rm -f $seqres.full
>> +
>> +_scratch_mkfs >>$seqres.full 2>&1
>> +_require_metadata_journaling $SCRATCH_DEV
>> +_init_flakey
>> +_mount_flakey
>> +
>> +# Create our test files.
>> +$XFS_IO_PROG -f -c "pwrite -S 0xea 0 256K" $SCRATCH_MNT/foo >/dev/null
>> +
>> +# Create a file with many extents. We later want to shrink truncate it and
>> +# add a prealloc extent beyond its new size.
>> +for ((i = 1; i <= 500; i++)); do
>> + offset=$(((i - 1) * 4 * 1024))
>> + $XFS_IO_PROG -f -s -c "pwrite -S 0xcf $offset 4K" \
>> +   

Re: Unable to compile btrfs progs 4.16 on ubuntu Xenial

[Nikolay Borisov]

On  7.04.2018 20:16, Menion wrote:
> Hi all
> Apparently it is not possible to compile with python bindings the
> btrfs progs on ubuntu xenial
> 
> checking for a Python interpreter with version >= 3.4... python3
> checking for python3... /usr/bin/python3
> checking for python3 version... 3.5
> checking for python3 platform... linux
> checking for python3 script directory... ${prefix}/lib/python3.5/site-packages
> checking for python3 extension module directory...
> ${exec_prefix}/lib/python3.5/site-packages
> checking for python-3.5... no
> configure: error: Package requirements (python-3.5) were not met:
> 
> No package 'python-3.5' found
> 
> Consider adjusting the PKG_CONFIG_PATH environment variable if you
> installed software in a non-standard prefix.
> 
> Alternatively, you may set the environment variables PYTHON_CFLAGS
> and PYTHON_LIBS to avoid the need to call pkg-config.
> See the pkg-config man page for more details.
> 
> /usr/lib/python3.5/site-packages exists, but on Ubuntu the package
> name is python3.5 and not python-3.5


Do you have libpython3-dev installed? When I installed it on my ubuntu i
could compile the progs irrespective of whether --prefix was passed or not
> 
> Bye
> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2] fstests: test btrfs fsync after hole punching with no-holes mode

[Filipe Manana]

On Sun, Apr 8, 2018 at 8:46 AM, Eryu Guan  wrote:
> On Wed, Mar 28, 2018 at 12:55:30PM +0100, fdman...@kernel.org wrote:
>> From: Filipe Manana 
>>
>> Test that when we have the no-holes mode enabled and a specific metadata
>> layout, if we punch a hole and fsync the file, at replay time the whole
>> hole was preserved.
>>
>> This issue is fixed by the following btrfs patch for the linux kernel:
>>
>>   "Btrfs: fix fsync after hole punching when using no-holes feature"
>>
>> Signed-off-by: Filipe Manana 
>> ---
>>
>> V2: Made the test work when selinux is enabled, and made it use direct IO
>> writes to ensure 256K extents.
>
> Test fails with selinux enabled now on unpatched kernel. But I found
> that, in my release testing, test still fails when testing with current
> Linus tree (HEAD is 642e7fd23353, without selinux this time), which
> should contain the mentioned fix. Does that mean the bug is not fully
> fixed?

The bug is fully fixed. But HEAD 642e7fd23353 does not contain the
fix. The current  linus' master has it.

>
> --- tests/btrfs/159.out 2018-04-03 18:25:41.566105514 -0700
> +++ /root/xfstests/results//btrfs/159.out.bad   2018-04-08 00:28:26.921968949 
> -0700
> @@ -6,4 +6,4 @@
>  File digest before power failure:
>  c5c0a13588a639529c979c57c336f441  SCRATCH_MNT/foobar
>  File digest after power failure and log replay:
> -c5c0a13588a639529c979c57c336f441  SCRATCH_MNT/foobar
> +c84746bbd1b97420f076d417a6a7d81c  SCRATCH_MNT/foobar
>
> I'll drop this patch for now.
>
> Thanks,
> Eryu
>
>>
>>  tests/btrfs/159 | 115 
>> 
>>  tests/btrfs/159.out |   9 
>>  tests/btrfs/group   |   1 +
>>  3 files changed, 125 insertions(+)
>>  create mode 100755 tests/btrfs/159
>>  create mode 100644 tests/btrfs/159.out
>>
>> diff --git a/tests/btrfs/159 b/tests/btrfs/159
>> new file mode 100755
>> index ..eb667692
>> --- /dev/null
>> +++ b/tests/btrfs/159
>> @@ -0,0 +1,115 @@
>> +#! /bin/bash
>> +# FSQA Test No. 159
>> +#
>> +# Test that when we have the no-holes mode enabled and a specific metadata
>> +# layout, if we punch a hole and fsync the file, at replay time the whole
>> +# hole was preserved.
>> +#
>> +#---
>> +#
>> +# Copyright (C) 2018 SUSE Linux Products GmbH. All Rights Reserved.
>> +# Author: Filipe Manana 
>> +#
>> +# This program is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU General Public License as
>> +# published by the Free Software Foundation.
>> +#
>> +# This program is distributed in the hope that it would be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> +# GNU General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program; if not, write the Free Software Foundation,
>> +# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
>> +#---
>> +#
>> +
>> +seq=`basename $0`
>> +seqres=$RESULT_DIR/$seq
>> +echo "QA output created by $seq"
>> +tmp=/tmp/$$
>> +status=1 # failure is the default!
>> +trap "_cleanup; exit \$status" 0 1 2 3 15
>> +
>> +_cleanup()
>> +{
>> + _cleanup_flakey
>> + cd /
>> + rm -f $tmp.*
>> +}
>> +
>> +# get standard environment, filters and checks
>> +. ./common/rc
>> +. ./common/filter
>> +. ./common/dmflakey
>> +
>> +# real QA test starts here
>> +_supported_fs btrfs
>> +_supported_os Linux
>> +_require_scratch
>> +_require_dm_target flakey
>> +_require_xfs_io_command "fpunch"
>> +_require_odirect
>> +
>> +rm -f $seqres.full
>> +
>> +run_test()
>> +{
>> + local punch_offset=$1
>> +
>> + # We create the filesystem with a node size of 64Kb because we need to
>> + # create a specific metadata layout in order to trigger the bug we are
>> + # testing. At the moment the node size can not be smaller then the
>> + # system's page size, so given that the largest possible page size is
>> + # 64Kb and by default the node size is set to the system's page size
>> + # value, we explicitly create a filesystem with a 64Kb node size.
>> + _scratch_mkfs -O no-holes -n $((64 * 1024)) >>$seqres.full 2>&1
>> + _require_metadata_journaling $SCRATCH_DEV
>> + _init_flakey
>> + _mount_flakey
>> +
>> + # Create our test file with 832 extents of 256Kb each. Before each
>> + # extent, there is a 256Kb hole (except for the first extent, which
>> + # starts at offset 0). This creates two leafs in the filesystem tree.
>> + # We use direct IO to ensure we get exactly 256K extents (with buffered
>> + # IO we can get writeback triggered at any time and therefore get
>> + # extents smaller than 256K).
>> +   

Re: Unable to compile btrfs progs 4.16 on ubuntu Xenial

[Nikolay Borisov]

On  8.04.2018 11:11, Menion wrote:
> ./configure --prefix=/usr --disable-documentation --enable-zstd

I was able to reproduce with --prefix added, thanks.
> 
> 2018-04-08 9:17 GMT+02:00 Nikolay Borisov :
>>
>>
>> On  7.04.2018 23:40, Menion wrote:
>>> I am adding - - prefix=/usr that seems you are not using
>>>
>>
>> Clearly you haven't shared all the necessary information, post your
>> entire configure line
>>
>>> 2018-04-07 21:55 GMT+02:00 Nikolay Borisov :


 On  7.04.2018 20:16, Menion wrote:
> Hi all
> Apparently it is not possible to compile with python bindings the
> btrfs progs on ubuntu xenial
>
> checking for a Python interpreter with version >= 3.4... python3
> checking for python3... /usr/bin/python3
> checking for python3 version... 3.5
> checking for python3 platform... linux
> checking for python3 script directory... 
> ${prefix}/lib/python3.5/site-packages
> checking for python3 extension module directory...
> ${exec_prefix}/lib/python3.5/site-packages
> checking for python-3.5... no
> configure: error: Package requirements (python-3.5) were not met:
>
> No package 'python-3.5' found
>
> Consider adjusting the PKG_CONFIG_PATH environment variable if you
> installed software in a non-standard prefix.
>
> Alternatively, you may set the environment variables PYTHON_CFLAGS
> and PYTHON_LIBS to avoid the need to call pkg-config.
> See the pkg-config man page for more details.
>
> /usr/lib/python3.5/site-packages exists, but on Ubuntu the package
> name is python3.5 and not python-3.5


 Works for me, I'm also on xenial:

 checking for python3... /usr/bin/python3
 checking for python3 version... 3.5
 checking for python3 platform... linux
 checking for python3 script directory... 
 ${prefix}/lib/python3.5/site-packages
 checking for python3 extension module directory... 
 ${exec_prefix}/lib/python3.5/site-packages
 checking for PYTHON... yes
 checking for lzo_version in -llzo2... yes
 configure: creating ./config.status
 config.status: creating Makefile.inc
 config.status: creating Documentation/Makefile
 config.status: creating version.h
 config.status: creating config.h

 btrfs-progs v4.16

 prefix: /usr/local
 exec prefix:${prefix}

 bindir: ${exec_prefix}/bin
 libdir: ${exec_prefix}/lib
 includedir: ${prefix}/include

 compiler:   gcc
 cflags: -g -O1 -Wall -D_FORTIFY_SOURCE=2
 ldflags:

 documentation:  no
 doc generator:  none
 backtrace support:  yes
 btrfs-convert:  no
 btrfs-restore zstd: no
 Python bindings:yes
 Python interpreter: /usr/bin/python3




>
> Bye
> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majord...@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>
>>>
> 
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: Unable to compile btrfs progs 4.16 on ubuntu Xenial

[Menion]

./configure --prefix=/usr --disable-documentation --enable-zstd

2018-04-08 9:17 GMT+02:00 Nikolay Borisov :
>
>
> On  7.04.2018 23:40, Menion wrote:
>> I am adding - - prefix=/usr that seems you are not using
>>
>
> Clearly you haven't shared all the necessary information, post your
> entire configure line
>
>> 2018-04-07 21:55 GMT+02:00 Nikolay Borisov :
>>>
>>>
>>> On  7.04.2018 20:16, Menion wrote:
 Hi all
 Apparently it is not possible to compile with python bindings the
 btrfs progs on ubuntu xenial

 checking for a Python interpreter with version >= 3.4... python3
 checking for python3... /usr/bin/python3
 checking for python3 version... 3.5
 checking for python3 platform... linux
 checking for python3 script directory... 
 ${prefix}/lib/python3.5/site-packages
 checking for python3 extension module directory...
 ${exec_prefix}/lib/python3.5/site-packages
 checking for python-3.5... no
 configure: error: Package requirements (python-3.5) were not met:

 No package 'python-3.5' found

 Consider adjusting the PKG_CONFIG_PATH environment variable if you
 installed software in a non-standard prefix.

 Alternatively, you may set the environment variables PYTHON_CFLAGS
 and PYTHON_LIBS to avoid the need to call pkg-config.
 See the pkg-config man page for more details.

 /usr/lib/python3.5/site-packages exists, but on Ubuntu the package
 name is python3.5 and not python-3.5
>>>
>>>
>>> Works for me, I'm also on xenial:
>>>
>>> checking for python3... /usr/bin/python3
>>> checking for python3 version... 3.5
>>> checking for python3 platform... linux
>>> checking for python3 script directory... 
>>> ${prefix}/lib/python3.5/site-packages
>>> checking for python3 extension module directory... 
>>> ${exec_prefix}/lib/python3.5/site-packages
>>> checking for PYTHON... yes
>>> checking for lzo_version in -llzo2... yes
>>> configure: creating ./config.status
>>> config.status: creating Makefile.inc
>>> config.status: creating Documentation/Makefile
>>> config.status: creating version.h
>>> config.status: creating config.h
>>>
>>> btrfs-progs v4.16
>>>
>>> prefix: /usr/local
>>> exec prefix:${prefix}
>>>
>>> bindir: ${exec_prefix}/bin
>>> libdir: ${exec_prefix}/lib
>>> includedir: ${prefix}/include
>>>
>>> compiler:   gcc
>>> cflags: -g -O1 -Wall -D_FORTIFY_SOURCE=2
>>> ldflags:
>>>
>>> documentation:  no
>>> doc generator:  none
>>> backtrace support:  yes
>>> btrfs-convert:  no
>>> btrfs-restore zstd: no
>>> Python bindings:yes
>>> Python interpreter: /usr/bin/python3
>>>
>>>
>>>
>>>

 Bye
 --
 To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
 the body of a message to majord...@vger.kernel.org
 More majordomo info at  http://vger.kernel.org/majordomo-info.html

>>
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v2] fstests: test btrfs fsync after hole punching with no-holes mode

[Eryu Guan]

On Wed, Mar 28, 2018 at 12:55:30PM +0100, fdman...@kernel.org wrote:
> From: Filipe Manana 
> 
> Test that when we have the no-holes mode enabled and a specific metadata
> layout, if we punch a hole and fsync the file, at replay time the whole
> hole was preserved.
> 
> This issue is fixed by the following btrfs patch for the linux kernel:
> 
>   "Btrfs: fix fsync after hole punching when using no-holes feature"
> 
> Signed-off-by: Filipe Manana 
> ---
> 
> V2: Made the test work when selinux is enabled, and made it use direct IO
> writes to ensure 256K extents.

Test fails with selinux enabled now on unpatched kernel. But I found
that, in my release testing, test still fails when testing with current
Linus tree (HEAD is 642e7fd23353, without selinux this time), which
should contain the mentioned fix. Does that mean the bug is not fully
fixed?

--- tests/btrfs/159.out 2018-04-03 18:25:41.566105514 -0700
+++ /root/xfstests/results//btrfs/159.out.bad   2018-04-08 00:28:26.921968949 
-0700
@@ -6,4 +6,4 @@
 File digest before power failure:
 c5c0a13588a639529c979c57c336f441  SCRATCH_MNT/foobar
 File digest after power failure and log replay:
-c5c0a13588a639529c979c57c336f441  SCRATCH_MNT/foobar
+c84746bbd1b97420f076d417a6a7d81c  SCRATCH_MNT/foobar

I'll drop this patch for now.

Thanks,
Eryu

> 
>  tests/btrfs/159 | 115 
> 
>  tests/btrfs/159.out |   9 
>  tests/btrfs/group   |   1 +
>  3 files changed, 125 insertions(+)
>  create mode 100755 tests/btrfs/159
>  create mode 100644 tests/btrfs/159.out
> 
> diff --git a/tests/btrfs/159 b/tests/btrfs/159
> new file mode 100755
> index ..eb667692
> --- /dev/null
> +++ b/tests/btrfs/159
> @@ -0,0 +1,115 @@
> +#! /bin/bash
> +# FSQA Test No. 159
> +#
> +# Test that when we have the no-holes mode enabled and a specific metadata
> +# layout, if we punch a hole and fsync the file, at replay time the whole
> +# hole was preserved.
> +#
> +#---
> +#
> +# Copyright (C) 2018 SUSE Linux Products GmbH. All Rights Reserved.
> +# Author: Filipe Manana 
> +#
> +# This program is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU General Public License as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it would be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write the Free Software Foundation,
> +# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
> +#---
> +#
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +tmp=/tmp/$$
> +status=1 # failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> + _cleanup_flakey
> + cd /
> + rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/filter
> +. ./common/dmflakey
> +
> +# real QA test starts here
> +_supported_fs btrfs
> +_supported_os Linux
> +_require_scratch
> +_require_dm_target flakey
> +_require_xfs_io_command "fpunch"
> +_require_odirect
> +
> +rm -f $seqres.full
> +
> +run_test()
> +{
> + local punch_offset=$1
> +
> + # We create the filesystem with a node size of 64Kb because we need to
> + # create a specific metadata layout in order to trigger the bug we are
> + # testing. At the moment the node size can not be smaller then the
> + # system's page size, so given that the largest possible page size is
> + # 64Kb and by default the node size is set to the system's page size
> + # value, we explicitly create a filesystem with a 64Kb node size.
> + _scratch_mkfs -O no-holes -n $((64 * 1024)) >>$seqres.full 2>&1
> + _require_metadata_journaling $SCRATCH_DEV
> + _init_flakey
> + _mount_flakey
> +
> + # Create our test file with 832 extents of 256Kb each. Before each
> + # extent, there is a 256Kb hole (except for the first extent, which
> + # starts at offset 0). This creates two leafs in the filesystem tree.
> + # We use direct IO to ensure we get exactly 256K extents (with buffered
> + # IO we can get writeback triggered at any time and therefore get
> + # extents smaller than 256K).
> + for ((i = 0; i <= 831; i++)); do
> + local offset=$((i * 2 * 256 * 1024))
> + $XFS_IO_PROG -f -d -c "pwrite -S 0xab -b 256K $offset 256K" \
> + $SCRATCH_MNT/foobar >/dev/null
> + done
> +
> + # Make sure everything done so far is durably persisted.
> + sync
> +

Re: Unable to compile btrfs progs 4.16 on ubuntu Xenial

[Nikolay Borisov]

On  7.04.2018 23:40, Menion wrote:
> I am adding - - prefix=/usr that seems you are not using
> 

Clearly you haven't shared all the necessary information, post your
entire configure line

> 2018-04-07 21:55 GMT+02:00 Nikolay Borisov :
>>
>>
>> On  7.04.2018 20:16, Menion wrote:
>>> Hi all
>>> Apparently it is not possible to compile with python bindings the
>>> btrfs progs on ubuntu xenial
>>>
>>> checking for a Python interpreter with version >= 3.4... python3
>>> checking for python3... /usr/bin/python3
>>> checking for python3 version... 3.5
>>> checking for python3 platform... linux
>>> checking for python3 script directory... 
>>> ${prefix}/lib/python3.5/site-packages
>>> checking for python3 extension module directory...
>>> ${exec_prefix}/lib/python3.5/site-packages
>>> checking for python-3.5... no
>>> configure: error: Package requirements (python-3.5) were not met:
>>>
>>> No package 'python-3.5' found
>>>
>>> Consider adjusting the PKG_CONFIG_PATH environment variable if you
>>> installed software in a non-standard prefix.
>>>
>>> Alternatively, you may set the environment variables PYTHON_CFLAGS
>>> and PYTHON_LIBS to avoid the need to call pkg-config.
>>> See the pkg-config man page for more details.
>>>
>>> /usr/lib/python3.5/site-packages exists, but on Ubuntu the package
>>> name is python3.5 and not python-3.5
>>
>>
>> Works for me, I'm also on xenial:
>>
>> checking for python3... /usr/bin/python3
>> checking for python3 version... 3.5
>> checking for python3 platform... linux
>> checking for python3 script directory... 
>> ${prefix}/lib/python3.5/site-packages
>> checking for python3 extension module directory... 
>> ${exec_prefix}/lib/python3.5/site-packages
>> checking for PYTHON... yes
>> checking for lzo_version in -llzo2... yes
>> configure: creating ./config.status
>> config.status: creating Makefile.inc
>> config.status: creating Documentation/Makefile
>> config.status: creating version.h
>> config.status: creating config.h
>>
>> btrfs-progs v4.16
>>
>> prefix: /usr/local
>> exec prefix:${prefix}
>>
>> bindir: ${exec_prefix}/bin
>> libdir: ${exec_prefix}/lib
>> includedir: ${prefix}/include
>>
>> compiler:   gcc
>> cflags: -g -O1 -Wall -D_FORTIFY_SOURCE=2
>> ldflags:
>>
>> documentation:  no
>> doc generator:  none
>> backtrace support:  yes
>> btrfs-convert:  no
>> btrfs-restore zstd: no
>> Python bindings:yes
>> Python interpreter: /usr/bin/python3
>>
>>
>>
>>
>>>
>>> Bye
>>> --
>>> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
>>> the body of a message to majord...@vger.kernel.org
>>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>>
> 
--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html