Hi, I've been using "cat" to feed large files into some data cruncher application using something like this: cat my_data | data_cruncher
However, cat was reading/writing the file in sub-optimal speeds (not even half as fast as the disk & os can provide it). I traced this to the buffer size selection algorithm in "cat", while generally provides good balance with low memory footprint, it constraints cat from reaching the disk's (or OS caches) peak performance. While it is usually not crucial for most applications to have "cat" operating at peak performance, I thought it would be useful to let the user determine that. I have made the following changes to allow the user provide and override the buffer sizes in "cat", effectively improving performance. Here's a quick benchmark (it's a typical result after multiple runs on Linux x86 with kernel 2.6.18): $ time ./cat test_sample_150mb_file.txt > /dev/null 0.00user 0.54system 0:00.59elapsed 90%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+186minor)pagefaults 0swaps $ time ./cat -r 1048576 test_sample_150mb_file.txt > /dev/null 0.00user 0.09system 0:00.12elapsed 73%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+444minor)pagefaults 0swaps The ability to specify an explicit (and larger) buffer size has improved the performance by a factor of x5 on my test system, which is quite a noticeable gain, especially when dealing with files at least 50GB in size. Let me know what do you think of it. The patch I used is available below. -- Tzvi *** coreutils-7.1/src/cat.c 2008-12-21 08:13:31.000000000 -0600 --- coreutils-7.1-new/src/cat.c 2009-03-05 22:52:47.000000000 -0600 *************** *** 40,45 **** --- 40,46 ---- #include "quote.h" #include "safe-read.h" #include "xfreopen.h" + #include "xstrtol.h" /* The official name of this program (e.g., no `g' prefix). */ #define PROGRAM_NAME "cat" *************** *** 102,107 **** --- 103,109 ---- -e equivalent to -vE\n\ -E, --show-ends display $ at end of each line\n\ -n, --number number all output lines\n\ + -r, --read-buffer-size enforce a read buffer size (in bytes)\n\ -s, --squeeze-blank suppress repeated empty output lines\n\ "), stdout); fputs (_("\ *************** *** 109,114 **** --- 111,117 ---- -T, --show-tabs display TAB characters as ^I\n\ -u (ignored)\n\ -v, --show-nonprinting use ^ and M- notation, except for LFD and TAB\n\ + -w, --write-buffer-size enforce a write buffer size (in bytes)\n\ "), stdout); fputs (HELP_OPTION_DESCRIPTION, stdout); fputs (VERSION_OPTION_DESCRIPTION, stdout); *************** *** 546,551 **** --- 549,556 ---- bool show_ends = false; bool show_nonprinting = false; bool show_tabs = false; + size_t read_buffer_size = 0; + size_t write_buffer_size = 0; int file_open_mode = O_RDONLY; static struct option const long_options[] = *************** *** 557,562 **** --- 562,569 ---- {"show-ends", no_argument, NULL, 'E'}, {"show-tabs", no_argument, NULL, 'T'}, {"show-all", no_argument, NULL, 'A'}, + {"read-buffer-size", required_argument, NULL, 'r'}, + {"write-buffer-size", required_argument, NULL, 'w'}, {GETOPT_HELP_OPTION_DECL}, {GETOPT_VERSION_OPTION_DECL}, {NULL, 0, NULL, 0} *************** *** 576,582 **** /* Parse command line options. */ ! while ((c = getopt_long (argc, argv, "benstuvAET", long_options, NULL)) != -1) { switch (c) --- 583,589 ---- /* Parse command line options. */ ! while ((c = getopt_long (argc, argv, "benr:stuvw:AET", long_options, NULL)) != -1) { switch (c) *************** *** 595,600 **** --- 602,616 ---- number = true; break; + case 'r': + { + long int buffer_size; + if ( xstrtol (optarg, NULL, 10, &buffer_size, "") != LONGINT_OK ) + error (EXIT_FAILURE, 0, _("%s: invalid read buffer size"), optarg); + read_buffer_size = buffer_size; + } + break; + case 's': squeeze_blank = true; break; *************** *** 612,617 **** --- 628,642 ---- show_nonprinting = true; break; + case 'w': + { + long int buffer_size; + if ( xstrtol (optarg, NULL, 10, &buffer_size, "") != LONGINT_OK ) + error (EXIT_FAILURE, 0, _("%s: invalid write buffer size"), optarg); + write_buffer_size = buffer_size; + } + break; + case 'A': show_nonprinting = true; show_ends = true; *************** *** 641,646 **** --- 666,674 ---- error (EXIT_FAILURE, errno, _("standard output")); outsize = ST_BLKSIZE (stat_buf); + if ( write_buffer_size > 0 ) + outsize = write_buffer_size; + /* Input file can be output file for non-regular files. fstat on pipes returns S_IFSOCK on some systems, S_IFIFO on others, so the checking should not be done for those types, *************** *** 705,710 **** --- 733,740 ---- goto contin; } insize = ST_BLKSIZE (stat_buf); + if ( read_buffer_size > 0 ) + insize = read_buffer_size; /* Compare the device and i-node numbers of this input file with the corresponding values of the (output file associated with) _______________________________________________ Bug-coreutils mailing list Bug-coreutils@gnu.org http://lists.gnu.org/mailman/listinfo/bug-coreutils