> -----Original Message-----
> From: Hans Reiser [mailto:[EMAIL PROTECTED]
> Sent: Monday, August 09, 2004 1:18 AM
> To: David Dabbs
> Cc: 'ReiserFS List'; 'Nikita Danilov'; 'Alexander Zarochentcev'; 'E.
> Gryaznova'
> Subject: Re: Was able to reproduce "cp: cannot stat file.x: Input/output
> error"
>
>
...snip...
> Hans:
> yes, it [hashing] is only a good idea for filenames larger than 15
> characters.
>
Actually, larger than 7 + 8 + 8 characters, if LARGE_KEY is used. This means
that no keys were hashed in my tests, or the original mongo for that matter,
since no file name would have been considered 'large' in kassign.c. I should
up the file name limit slightly to exercise it with at least some hashed
keys.
> >
> it is extremely unrealistic to create the files in one phase and write
> to them in the next phase. the filesystem will behave completely
> differently.
>
Of course. The extra files I added are created once, and should never be
touched again since mongo only operates on the objects it creates. I am
simply loading the test filesystem with a reasonable facsimile of a 'normal'
Linux file set (with the exception that the files are all zero length). To
make these 'static' dirs/files really useful, mongo would need to be written
to use these directories for its randomly created files. As for my files all
being zero length, it would be easy to give them all some small file
content. If these additional dirs/files as currently created add no value to
the test, then do not enable these phases.
On top of this Mongo randomly creates (in PHASE CREATE) its directories and
files, with each file filled at creation with a random number of 'a'
characters. Each subsequent phase, if specified, does one of the following
operations in the order you see below. Since you said you hadn't looked at
mongo for some time, I have included the commands mongo executes. These
examples are from a run with three processes and three iterations, so each
command below was run three times in succession. Note that mongo drives each
phase from a file list, so the files are always manipulated in the same
order - the order in which they were created.
a) COPY command: Copy all files from some base iteration directory.
(update-flist.pl /mnt/testfs/testdir0-0-0 /mnt/testfs/testdir1-0-0
/var/tmp/mongo.flist0-0-0 /var/tmp/mongo.flist1-0-0 || touch ERR.file ) &
(update-flist.pl /mnt/testfs/testdir0-0-1 /mnt/testfs/testdir1-0-1
/var/tmp/mongo.flist0-0-1 /var/tmp/mongo.flist1-0-1 || touch ERR.file ) &
(update-flist.pl /mnt/testfs/testdir0-0-2 /mnt/testfs/testdir1-0-2
/var/tmp/mongo.flist0-0-2 /var/tmp/mongo.flist1-0-2 || touch ERR.file ) &
wait; sync
/*******************************************************************/
update-flist.pl
($DIR0, $DIR1, $FLIST0, $FLIST1) = @ARGV;
system("cp -r $DIR0 $DIR1");
open(FL0, $FLIST0) || DIE("Cannot open $FLIST0");
open(FL1, ">$FLIST1") || DIE("Cannot open $FLIST1");
while (<FL0>) {
s|^$DIR0|$DIR1|;
print FL1 $_;
}
close(FL1);
close(FL0);
/*******************************************************************/
b) APPEND command:
(cat /var/tmp/mongo.flist[01]-0-0 | grep -v -e /\$ | mongo_append 0.5 4096
off || touch ERR.file ) &
(cat /var/tmp/mongo.flist[01]-0-1 | grep -v -e /\$ | mongo_append 0.5 4096
off || touch ERR.file ) &
(cat /var/tmp/mongo.flist[01]-0-2 | grep -v -e /\$ | mongo_append 0.5 4096
off || touch ERR.file ) &
wait; sync
/*******************************************************************/
mongo_append.c
while (getline(&line_buffer, &line_buffer_size, stdin) != -1) {
int fd, written = 0;
char * lf_pos;
if ((lf_pos = index(line_buffer, '\n')) != NULL) *lf_pos = '\0';
if ( (fd = open(line_buffer,O_RDWR)) == -1) {
fprintf(stderr, "%s :", line_buffer);
perror("cannot open file");
continue;
}
append_size = append_factor * lseek(fd, 0, SEEK_END);
while (written < append_size) {
int error;
written += (error = write(fd, buffer,
( append_size - written < writesize ) ? append_size -
written:writesize));
if (error == -1) {
close(fd);
break;
}
}
if (use_fsync) {
if (fsync(fd) < 0) {
fprintf(stderr, "%s : failed to fsync file\n",
line_buffer);
exit(1);
}
}
close(fd);
}
free (buffer);
free (line_buffer);
return 0;
/*******************************************************************/
c) MODIFY command:
(cat /var/tmp/mongo.flist[01]-0-0 | grep -v -e /\$ | mongo_modify 0.02 4096
off || touch ERR.file ) &
(cat /var/tmp/mongo.flist[01]-0-1 | grep -v -e /\$ | mongo_modify 0.02 4096
off || touch ERR.file ) &
(cat /var/tmp/mongo.flist[01]-0-2 | grep -v -e /\$ | mongo_modify 0.02 4096
off || touch ERR.file ) &
wait; sync
/*******************************************************************/
mongo_modify.c
while (getline(&line_buffer, &line_buffer_size, stdin) != -1) {
int fd;
off_t fileend;
char * lf_pos;
if ((lf_pos = index(line_buffer, '\n')) != NULL) *lf_pos = '\0';
if ( (fd = open(line_buffer,O_RDWR)) == -1) {
perror("cannot open file");
continue;
}
fileend = lseek(fd,0,SEEK_END);
if (fileend == (off_t) - 1) {
perror("lseek failed");
close (fd);
continue;
}
if (!fileend) {
/* nothing to modify */
close (fd);
continue;
}
{
off_t region_size = (double)fileend * modify_factor;
off_t write_pos = (double)(fileend - region_size) * rand() /
(RAND_MAX + 1.0);
size_t bytes = 0;
if (lseek(fd, write_pos, SEEK_SET) == (off_t) - 1) {
perror("lseek failed");
close (fd);
continue;
}
while (bytes < region_size) {
size_t write_count = region_size - bytes;
if (write_count > write_buffer_size)
write_count = write_buffer_size;
write_count = write(fd, buffer, write_count);
if (write_count == 0) break;
if (write_count == -1) {
perror("write failed");
break;
}
bytes += write_count;
}
}
if (use_fsync) {
if (fsync(fd) < 0) {
fprintf(stderr, "%s : failed to fsync file\n",
line_buffer);
exit(1);
}
}
close(fd);
}
free (buffer);
free (line_buffer);
return 0;
/*******************************************************************/
d) OVERWRITE command:
( cat /var/tmp/mongo.flist[01]-0-0 | grep -v -e /\$ | mongo_modify 1 4096
off || touch ERR.file ) &
( cat /var/tmp/mongo.flist[01]-0-1 | grep -v -e /\$ | mongo_modify 1 4096
off || touch ERR.file ) &
( cat /var/tmp/mongo.flist[01]-0-2 | grep -v -e /\$ | mongo_modify 1 4096
off || touch ERR.file ) &
wait; sync
See mongo_modify.c above.
/*******************************************************************/
e) READ command:
(find /mnt/testfs/testdir[01]-1-0 -type f | mongo_read || touch ERR.file ) &
(find /mnt/testfs/testdir[01]-1-1 -type f | mongo_read || touch ERR.file ) &
(find /mnt/testfs/testdir[01]-1-2 -type f | mongo_read || touch ERR.file ) &
wait; sync
/*******************************************************************/
mongo_read.c
/* Read all file names from the standard input */
while ((rd = getline(&file_name_buf, &file_name_buf_size, stdin)) !=
-1) {
/* remove the new line character. */
if (rd > 0 && file_name_buf[rd - 1] == '\n')
file_name_buf[rd - 1] = 0;
/* open the file */
fd = open (file_name_buf, O_RDONLY);
if (fd == -1) {
fprintf (stderr, "Open failed (%s)\n", strerror
(errno));
return errno;
}
/* read the file */
while (1) {
rd = read (fd, read_buf, bufsize);
if (rd == -1) {
fprintf (stderr, "Read failed (%s)\n",
strerror (errno));
return errno;
}
if (rd == 0)
break; /* EOF */
/* file consists of 'a'. Check that */
if (char_to_check) {
int j;
for (j = 0; j < rd; j ++)
if (read_buf[j] != char_to_check) {
fprintf (stderr, "Incorrect
data were read\n");
return EIO;
}
}
}
close (fd);
}
if (file_name_buf)
free(file_name_buf);
free (read_buf);
snapstats();
return 0;
/*******************************************************************/
f) STATS command:
((cat /var/tmp/mongo.flist[01]-0-0 | xargs ls -d) > /dev/null || touch
ERR.file ) &
((cat /var/tmp/mongo.flist[01]-0-1 | xargs ls -d) > /dev/null || touch
ERR.file ) &
((cat /var/tmp/mongo.flist[01]-0-2 | xargs ls -d) > /dev/null || touch
ERR.file ) &
wait; sync
g) DELETE command:
( rm -r /mnt/testfs/testdir[01]-0-0 || touch ERR.file ) &
( rm -r /mnt/testfs/testdir[01]-0-1 || touch ERR.file ) &
( rm -r /mnt/testfs/testdir[01]-0-2 || touch ERR.file ) &
wait; sync
Finally, if specified, it executes the large file phases
h) dd_writing_largefile command:
( dd if=/dev/zero of=/mnt/testfs/largefile bs=1M count=768 ) &
( dd if=/dev/zero of=/mnt/testfs/largefile bs=1M count=768 ) &
( dd if=/dev/zero of=/mnt/testfs/largefile bs=1M count=768 ) &
wait; sync
i) dd_reading_largefile command:
( dd of=/dev/null if=/mnt/testfs/largefile bs=1M count=768 ) &
( dd of=/dev/null if=/mnt/testfs/largefile bs=1M count=768 ) &
( dd of=/dev/null if=/mnt/testfs/largefile bs=1M count=768 ) &
wait; sync
