Re: [PATCH] dd: add a flag to discard cached data
On 24/02/11 07:52, Jim Meyering wrote:
One quick question: does the test need
something to make it skip (not fail)
on systems that lack kernel support for the feature?
Oops right.
Attached is latest version.
thanks for the review,
Pádraig.
From fe067f8f52defc54636ae862df03a2115ac6266f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A1draig=20Brady?= p...@draigbrady.com
Date: Tue, 22 Feb 2011 21:14:00 +
Subject: [PATCH] dd: add a flag to discard cached data
* src/dd.c (FFS_MASK): A new macro (Find First Set) refactored
from the following enum as it's now used twice.
(usage): Mention the new 'nocache' flag.
(cache_round): A new function to help ignore cache
drop requests less than page_size.
(invalidate_cache): A new function to call posix_fadvise()
with the appropriate offset and length. Note we don't
use fdadvise() so we can detect errors when count=0.
(dd_copy): Call invalidate_cache() for the read portions.
(iwrite): Likewise for the written portions.
(main): Call invalidate_cache for page_size slop or
for full file when count=0.
* cfg.mk (sc_dd_O_FLAGS): Adjust to pass.
* doc/coreutils.texi (dd invocation): Describe the 'nocache' flag,
and give some examples of how it can be used.
* tests/dd/nocache: A new test.
* tests/Makefile.am: Reference the new test.
* NEWS: Mention the new feature.
---
NEWS |6 ++
cfg.mk |4 +-
doc/coreutils.texi | 25
src/dd.c | 160 +---
tests/Makefile.am |1 +
tests/dd/nocache | 58 +++
6 files changed, 244 insertions(+), 10 deletions(-)
create mode 100755 tests/dd/nocache
diff --git a/NEWS b/NEWS
index a367d8d..9ec24a6 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,12 @@ GNU coreutils NEWS-*- outline -*-
delimiter and an unbounded range like -f1234567890-.
[bug introduced in coreutils-5.3.0]
+** New features
+
+ dd now accepts the 'nocache' flag to the iflag and oflag options,
+ which will discard any cache associated with the files, or
+ processed portion thereof.
+
* Noteworthy changes in release 8.10 (2011-02-04) [stable]
diff --git a/cfg.mk b/cfg.mk
index e4f3faa..c897dc4 100644
--- a/cfg.mk
+++ b/cfg.mk
@@ -39,8 +39,8 @@ _hv_file ?= $(srcdir)/tests/misc/help-version
dd = $(srcdir)/src/dd.c
sc_dd_O_FLAGS:
@rm -f $@.1 $@.2
- @{ echo O_FULLBLOCK; perl -nle '/^ +\| (O_\w*)$$/ and print $$1' \
- $(dd); } | sort $@.1
+ @{ echo O_FULLBLOCK; echo O_NOCACHE;\
+ perl -nle '/^ +\| (O_\w*)$$/ and print $$1' $(dd); } | sort $@.1
@{ echo O_NOFOLLOW; perl -nle '/{[a-z]+,\s*(O_\w+)},/ and print $$1' \
$(dd); } | sort $@.2
@diff -u $@.1 $@.2 || diff=1 || diff=;\
diff --git a/doc/coreutils.texi b/doc/coreutils.texi
index ea35afe..529adab 100644
--- a/doc/coreutils.texi
+++ b/doc/coreutils.texi
@@ -8168,6 +8168,31 @@ last-access and last-modified time) is not necessarily synchronized.
@cindex synchronized data and metadata I/O
Use synchronized I/O for both data and metadata.
+@item nocache
+@opindex nocache
+@cindex discarding file cache
+Discard the data cache for a file.
+When count=0 all cache is discarded,
+otherwise the cache is dropped for the processed
+portion of the file. Also when count=0
+failure to discard the cache is diagnosed
+and reflected in the exit status.
+Here as some usage examples:
+
+@example
+# Advise to drop cache for whole file
+dd if=ifile iflag=nocache count=0
+
+# Ensure drop cache for the whole file
+dd of=ofile oflag=nocache conv=notrunc,fdatasync count=0
+
+# Drop cache for part of file
+dd if=ifile iflag=nocache skip=10 count=10 of=/dev/null
+
+# Stream data just using read-ahead cache
+dd if=ifile of=ofile iflag=nocache oflag=nocache
+@end example
+
@item nonblock
@opindex nonblock
@cindex nonblocking I/O
diff --git a/src/dd.c b/src/dd.c
index daddc1e..f62aaed 100644
--- a/src/dd.c
+++ b/src/dd.c
@@ -225,6 +225,9 @@ static sig_atomic_t volatile interrupt_signal;
/* A count of the number of pending info signals that have been received. */
static sig_atomic_t volatile info_signal_count;
+/* Whether to discard cache for input or output. */
+static bool i_nocache, o_nocache;
+
/* Function used for read (to handle iflag=fullblock parameter). */
static ssize_t (*iread_fnc) (int fd, char *buf, size_t size);
@@ -259,6 +262,7 @@ static struct symbol_value const conversions[] =
{, 0}
};
+#define FFS_MASK(x) ((x) ^ ((x) ((x) - 1)))
enum
{
/* Compute a value that's bitwise disjoint from the union
@@ -278,17 +282,23 @@ enum
| O_SYNC
| O_TEXT
),
-/* Use its lowest bit. */
-O_FULLBLOCK = v ^ (v (v - 1))
+
+/* Use its lowest bits for private flags. */
+O_FULLBLOCK = FFS_MASK (v),
+v2 = v ^ O_FULLBLOCK,
+
+O_NOCACHE = FFS_MASK (v2)
};
/* Ensure that we got something. */
verify (O_FULLBLOCK != 0);
+verify (O_NOCACHE != 0);
#define MULTIPLE_BITS_SET(i) (((i
Re: [PATCH] dd: add a flag to discard cached data
Pádraig Brady wrote: On 24/02/11 07:52, Jim Meyering wrote: One quick question: does the test need something to make it skip (not fail) on systems that lack kernel support for the feature? Oops right. Attached is latest version. ... +# Advise to drop cache for whole file +if ! dd if=ifile iflag=nocache count=0 2err; then + if grep -F 'Operation not supported' err /dev/null; then + warn_ 'skipping part; this file system lacks support for posix_fadvise()' + skip=1 + else +fail=1 + fi +fi Thanks for the quick fix. That looks fine, but I'd drop the use of -F, since it's just an optimization in this case, and probably not portable to some crufty targets. Hmm.. but I see uses of grep -F in other tests and even in bootstrap, so maybe it's not a problem, these days: $ git grep -l grep.-F bootstrap tests/cp/cp-mv-enotsup-xattr tests/cp/fiemap-perf tests/misc/xattr
[PATCH] dd: add a flag to discard cached data
I noticed fadvise(DONTNEED) getting some love in the kernel recently
http://lkml.org/lkml/2011/2/17/169
Which prompted me to implement this. With the attached one can now:
# Advise to drop cache for whole file
dd if=ifile iflag=nocache count=0
# Ensure drop cache for whole file
dd of=ofile oflag=nocache conv=notrunc,fdatasync count=0
# Drop cache for part of file
dd if=ifile iflag=nocache skip=10 count=10 of=/dev/null
# Stream data just using readahead cache
dd if=ifile of=ofile iflag=nocache oflag=nocache
When count=0, i.e. when only manipulating the cache,
we propagate the posix_fadvise() return to the exit status.
Note this will invalidate the cache even if another
process has the file open. That could be avoided with mincor:
http://insights.oetiker.ch/linux/fadvise.html
However, I don't think that's needed for a tool like dd.
cheers,
Pádraig.
From 5ebe7618f8d62a81e053a57390132e4013218416 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A1draig=20Brady?= p...@draigbrady.com
Date: Tue, 22 Feb 2011 21:14:00 +
Subject: [PATCH] dd: add a flag to discard cached data
* src/dd.c (usage): Add the 'nocache' flag.
(cache_round): A new function to help ignore cache
drop requests less than page_size.
(invalidate_cache): A new function to call posix_fadvise()
with the appropriate offset and length. Note we don't
use fdadvise() so we can detect errors when count=0.
(dd_copy): Call invalidate_cache() for the processed portions.
(main): Call invalidate_cache for page_size slop or
for full file when count=0.
* cfg.mk (sc_dd_O_FLAGS): Adjust to pass.
* doc/coreutils.texi (dd invocation): Describe the 'nocache' flag,
and give some examples of how it can be used.
* tests/dd/nocache: A new test.
* tests/Makefile.am: Reference the new test.
* NEWS: Mention the new feature.
---
NEWS |6 ++
cfg.mk |4 +-
doc/coreutils.texi | 25
src/dd.c | 160 +---
tests/Makefile.am |1 +
tests/dd/nocache | 48
6 files changed, 234 insertions(+), 10 deletions(-)
create mode 100755 tests/dd/nocache
diff --git a/NEWS b/NEWS
index a367d8d..bbb8bd3 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,12 @@ GNU coreutils NEWS-*- outline -*-
delimiter and an unbounded range like -f1234567890-.
[bug introduced in coreutils-5.3.0]
+** New features
+
+ dd now accepts the nocache option to iflag and oflag,
+ which will discard any cache associated with the files,
+ or processed portion thereof.
+
* Noteworthy changes in release 8.10 (2011-02-04) [stable]
diff --git a/cfg.mk b/cfg.mk
index e4f3faa..c897dc4 100644
--- a/cfg.mk
+++ b/cfg.mk
@@ -39,8 +39,8 @@ _hv_file ?= $(srcdir)/tests/misc/help-version
dd = $(srcdir)/src/dd.c
sc_dd_O_FLAGS:
@rm -f $@.1 $@.2
- @{ echo O_FULLBLOCK; perl -nle '/^ +\| (O_\w*)$$/ and print $$1' \
- $(dd); } | sort $@.1
+ @{ echo O_FULLBLOCK; echo O_NOCACHE;\
+ perl -nle '/^ +\| (O_\w*)$$/ and print $$1' $(dd); } | sort $@.1
@{ echo O_NOFOLLOW; perl -nle '/{[a-z]+,\s*(O_\w+)},/ and print $$1' \
$(dd); } | sort $@.2
@diff -u $@.1 $@.2 || diff=1 || diff=;\
diff --git a/doc/coreutils.texi b/doc/coreutils.texi
index ea35afe..529adab 100644
--- a/doc/coreutils.texi
+++ b/doc/coreutils.texi
@@ -8168,6 +8168,31 @@ last-access and last-modified time) is not necessarily synchronized.
@cindex synchronized data and metadata I/O
Use synchronized I/O for both data and metadata.
+@item nocache
+@opindex nocache
+@cindex discarding file cache
+Discard the data cache for a file.
+When count=0 all cache is discarded,
+otherwise the cache is dropped for the processed
+portion of the file. Also when count=0
+failure to discard the cache is diagnosed
+and reflected in the exit status.
+Here as some usage examples:
+
+@example
+# Advise to drop cache for whole file
+dd if=ifile iflag=nocache count=0
+
+# Ensure drop cache for the whole file
+dd of=ofile oflag=nocache conv=notrunc,fdatasync count=0
+
+# Drop cache for part of file
+dd if=ifile iflag=nocache skip=10 count=10 of=/dev/null
+
+# Stream data just using read-ahead cache
+dd if=ifile of=ofile iflag=nocache oflag=nocache
+@end example
+
@item nonblock
@opindex nonblock
@cindex nonblocking I/O
diff --git a/src/dd.c b/src/dd.c
index daddc1e..f62aaed 100644
--- a/src/dd.c
+++ b/src/dd.c
@@ -225,6 +225,9 @@ static sig_atomic_t volatile interrupt_signal;
/* A count of the number of pending info signals that have been received. */
static sig_atomic_t volatile info_signal_count;
+/* Whether to discard cache for input or output. */
+static bool i_nocache, o_nocache;
+
/* Function used for read (to handle iflag=fullblock parameter). */
static ssize_t (*iread_fnc) (int fd, char *buf, size_t size);
@@ -259,6 +262,7 @@ static struct symbol_value const conversions[] =
{, 0}
};
+#define FFS_MASK(x) ((x) ^ ((x) ((x) - 1)))
enum
{
/* Compute
