After several weeks of inactivity due to pressure of other commitments,
I've finally found some time to get some mairix work done.

The current 'wavefront' version can be pulled for testing from the
integ_20aug branch of the git repository at

  http://www.rpcurnow.force9.co.uk/git/mairix.git

I've attached the patch from v0.19 to this version also.

Here's the git-shortlog output:

Chris Mason:
      cache uncompressed mbox data

Oliver Braun:
      Fix missing ; in dfasyn/parse.y

Richard P. Curnow:
      Fix gaps in date range.
      Merge fix for date ranges
      Fix tabulation
      Fix conditional compilation issues
      Remove restriction on filename length in compressed mbox cache
      Preparatory work for fast-index option
      Implement -F/--fast-index flag
      Update documentation to include -F option
      Merge topics

Currently, the fast-index work and the compressed mbox cache work are on
separate topic branches.  I've used the fast-index changes myself
already on maildir and they seem to be OK.  I don't use compressed mbox
folders so I cannot easily test the other one.  Any 'field testing' +
bug reports (+ patches) much appreciated.

-- 
Richard P. Curnow | Free software developer
Weston-super-Mare | 
United Kingdom    | http://www.rc0.org.uk/
diff --git a/dates.c b/dates.c
index 6982445..c02d4e3 100644
--- a/dates.c
+++ b/dates.c
@@ -333,6 +333,12 @@ int scan_date_string(char *in, time_t *s
 
   time(&now);
   start_tm = end_tm = *localtime(&now);
+  start_tm.tm_hour = 0;
+  start_tm.tm_min = 0;
+  start_tm.tm_sec = 0;
+  end_tm.tm_hour = 23;
+  end_tm.tm_min = 59;
+  end_tm.tm_sec = 59;
 
   hyphen = strchr(in, '-');
   if (!hyphen) {
diff --git a/db.c b/db.c
index 1ad3a47..be13bee 100755
--- a/db.c
+++ b/db.c
@@ -838,7 +838,23 @@ static void add_msg_path(struct database
   ++db->n_msgs;
 }
 /*}}}*/
-int update_database(struct database *db, struct msgpath *sorted_paths, int 
n_msgs)/*{{{*/
+
+static int do_stat(struct msgpath *mp)/*{{{*/
+{
+  struct stat sb;
+  int status;
+  status = stat(mp->src.mpf.path, &sb);
+  if ((status < 0) ||
+      !S_ISREG(sb.st_mode)) {
+    return 0;
+  } else {
+    mp->src.mpf.mtime = sb.st_mtime;
+    mp->src.mpf.size = sb.st_size;
+    return 1;
+  }
+}
+/*}}}*/
+int update_database(struct database *db, struct msgpath *sorted_paths, int 
n_msgs, int do_fast_index)/*{{{*/
 {
   /* The incoming list must be sorted into order, to make binary searching
    * possible.  We search for each existing path in the incoming sorted array.
@@ -853,6 +869,7 @@ int update_database(struct database *db,
   int matched_index;
   int i, new_entries_start_at;
   int any_new, n_newly_pruned, n_already_dead;
+  int status;
 
   file_in_db = new_array(char, n_msgs);
   file_in_new_list = new_array(char, db->n_msgs);
@@ -866,11 +883,27 @@ int update_database(struct database *db,
     switch (db->type[i]) {
       case MTY_FILE:
         matched_index = lookup_msgpath(sorted_paths, n_msgs, 
db->msgs[i].src.mpf.path);
-        if ((matched_index >= 0) &&
-            (sorted_paths[matched_index].src.mpf.mtime == 
db->msgs[i].src.mpf.mtime)) {
-        /* Treat stale files as though the path has changed. */
-          file_in_db[matched_index] = 1;
-          file_in_new_list[i] = 1;
+        if (matched_index >= 0) {
+          if (do_fast_index) {
+            /* Assume the presence of a matching path is good enough without
+             * even bothering to stat the file that's there now. */
+            file_in_db[matched_index] = 1;
+            file_in_new_list[i] = 1;
+          } else {
+            status = do_stat(sorted_paths + matched_index);
+            if (status) {
+              if (sorted_paths[matched_index].src.mpf.mtime == 
db->msgs[i].src.mpf.mtime) {
+                /* Treat stale files as though the path has changed. */
+                file_in_db[matched_index] = 1;
+                file_in_new_list[i] = 1;
+              } else {
+                fprintf(stderr, "mtime failed for '%s'\n", 
sorted_paths[matched_index].src.mpf.path);
+              }
+            } else {
+              /* This path will get treated as dead, and be re-stated below.
+               * When that stat fails, the path won't get added to the db. */
+            }
+          }
         }
         break;
       case MTY_MBOX:
@@ -924,9 +957,17 @@ int update_database(struct database *db,
   any_new = 0;
   for (i=0; i<n_msgs; i++) {
     if (!file_in_db[i]) {
+      int status;
       any_new = 1;
       /* The 'sorted_paths' array is only used for file-per-message folders. */
-      add_msg_path(db, sorted_paths[i].src.mpf.path, 
sorted_paths[i].src.mpf.mtime, sorted_paths[i].src.mpf.size);
+      status = do_stat(sorted_paths + i);
+      if (status) {
+        /* We only add files that could be successfully stat()'d as regular
+         * files. */
+        add_msg_path(db, sorted_paths[i].src.mpf.path, 
sorted_paths[i].src.mpf.mtime, sorted_paths[i].src.mpf.size);
+      } else {
+        fprintf(stderr, "Cannot add '%s' to database; stat() failed\n", 
sorted_paths[i].src.mpf.path);
+      }
     }
   }
 
diff --git a/dfasyn/parse.y b/dfasyn/parse.y
index ec04769..1e5d573 100644
--- a/dfasyn/parse.y
+++ b/dfasyn/parse.y
@@ -242,6 +242,7 @@ attr_decl_seq : /* empty */
                 ;
 
 tag_decl : TAG STRING EQUAL expr { define_tag($2, $4); }
+         ;
 
 entrystruct_decl :
               ENTRYSTRUCT STRING STRING   { define_entrystruct($2, $3); }
diff --git a/dirscan.c b/dirscan.c
index 49b30fd..33e36a8 100755
--- a/dirscan.c
+++ b/dirscan.c
@@ -62,7 +62,7 @@ void free_msgpath_array(struct msgpath_a
   free(x);
 }
 /*}}}*/
-static void add_file_to_list(char *x, unsigned long mtime, size_t 
message_size, struct msgpath_array *arr) {/*{{{*/
+static void add_file_to_list(char *x, struct msgpath_array *arr) {/*{{{*/
   char *y = new_string(x);
   if (arr->n == arr->max) {
     arr->max += 1024;
@@ -71,8 +71,6 @@ static void add_file_to_list(char *x, un
   }
   arr->type[arr->n] = MTY_FILE;
   arr->paths[arr->n].src.mpf.path = y;
-  arr->paths[arr->n].src.mpf.mtime = mtime;
-  arr->paths[arr->n].src.mpf.size = message_size;
   ++arr->n;
   return;
 }
@@ -84,7 +82,6 @@ static void get_maildir_message_paths(ch
   static char *subdirs[] = {"new", "cur"};
   DIR *d;
   struct dirent *de;
-  struct stat sb;
   int folder_len = strlen(folder);
 
   /* FIXME : just store mdir-rooted paths in array and have common prefix 
elsewhere. */
@@ -100,14 +97,14 @@ static void get_maildir_message_paths(ch
       while ((de = readdir(d))) {
         /* TODO : Perhaps we ought to do some validation on the path here?
            i.e. check that the filename looks valid for a maildir message. */
+        if (!strcmp(de->d_name, ".") ||
+            !strcmp(de->d_name, "..")) {
+          continue;
+        }
         strcpy(fname, subdir);
         strcat(fname, "/");
         strcat(fname, de->d_name);
-        if (stat(fname, &sb) >= 0) {
-          if (S_ISREG(sb.st_mode)) {
-            add_file_to_list(fname, sb.st_mtime, sb.st_size, arr);
-          }
-        }
+        add_file_to_list(fname, arr);
       }
       closedir(d);
     }
@@ -140,7 +137,6 @@ static void get_mh_message_paths(char *f
   char *fname;
   DIR *d;
   struct dirent *de;
-  struct stat sb;
   int folder_len = strlen(folder);
 
   fname = new_array(char, folder_len + 8 + NAME_MAX);
@@ -150,12 +146,8 @@ static void get_mh_message_paths(char *f
       strcpy(fname, folder);
       strcat(fname, "/");
       strcat(fname, de->d_name);
-      if (stat(fname, &sb) >= 0) {
-        if (S_ISREG(sb.st_mode)) {
-          if (valid_mh_filename_p(de->d_name)) {
-            add_file_to_list(fname, sb.st_mtime, sb.st_size, arr);
-          }
-        }
+      if (valid_mh_filename_p(de->d_name)) {
+        add_file_to_list(fname, arr);
       }
     }
     closedir(d);
diff --git a/mairix.1 b/mairix.1
index d6a7525..cca8bee 100644
--- a/mairix.1
+++ b/mairix.1
@@ -37,6 +37,8 @@ mairix \- index and search mail folders
 ] [
 .BR \-f | \-\-rcfile
 .I mairixrc
+] [
+.BR \-F | \-\-fast-index
 ]
 
 .SS Searching
@@ -175,6 +177,24 @@ Cause stale (dead) messages to be purged
 run.  (Normally, stale messages are left in the database because of the
 additional const of compacting away the storage that they take up.)
 
+.TP
+.B -F, --fast-index
+.br
+When processing maildir and MH folders,
+.I mairix
+normally compares the mtime and size of each message against the values stored
+in the database.  If they have changed, the message will be rescanned.  This
+check requires each message file to be stat'ed.  For large numbers of messages
+in these folder types, this can be a sizeable overhead.
+
+This option tells
+.I mairix
+to assume that when a message currently on-disc has a name matching one already
+in the database, it should assume the message is unchanged.
+
+A later indexing run without using this option will fix up any rescans that
+were missed due to its use.
+
 .SS Search options
 .TP
 .B -a, --augment
diff --git a/mairix.c b/mairix.c
index b588ab0..e86a097 100755
--- a/mairix.c
+++ b/mairix.c
@@ -391,7 +391,7 @@ static void usage(void)/*{{{*/
   print_copyright();
 
   printf("mairix [-h]                                    : Show help\n"
-         "mairix [-f <rcfile>] [-v] [-p]                 : Build index\n"
+         "mairix [-f <rcfile>] [-v] [-p] [-F]            : Build index\n"
          "mairix [-f <rcfile>] [-a] [-t] expr1 ... exprN : Run search\n"
          "mairix [-f <rcfile>] -d                        : Dump database to 
stdout\n"
          "-h           : show this help\n"
@@ -399,6 +399,7 @@ static void usage(void)/*{{{*/
          "-V           : show version\n"
          "-v           : be verbose\n"
          "-p           : purge messages that no longer exist\n"
+         "-F           : fast scan for maildir and MH folders (no mtime or 
size checks)\n"
          "-a           : add new matches to match folder (default : clear it 
first)\n"
          "-t           : include all messages in same threads as matching 
messages\n"
          "-o <mfolder> : override setting of mfolder from mairixrc file\n"
@@ -473,6 +474,7 @@ int main (int argc, char **argv)/*{{{*/
   int do_dump = 0;
   int do_integrity_checks = 1;
   int do_forced_unlock = 0;
+  int do_fast_index = 0;
 
   struct globber_array *omit_globs;
 
@@ -507,6 +509,9 @@ int main (int argc, char **argv)/*{{{*/
       do_integrity_checks = 0;
     } else if (!strcmp(*argv, "--unlock")) {
       do_forced_unlock = 1;
+    } else if (!strcmp(*argv, "-F") ||
+               !strcmp(*argv, "--fast-index")) {
+      do_fast_index = 1;
     } else if (!strcmp(*argv, "-v") || !strcmp(*argv, "--verbose")) {
       verbose = 1;
     } else if (!strcmp(*argv, "-V") || !strcmp(*argv, "--version")) {
@@ -701,7 +706,7 @@ int main (int argc, char **argv)/*{{{*/
 
     build_mbox_lists(db, folder_base, mboxen, omit_globs);
 
-    any_updates = update_database(db, msgs->paths, msgs->n);
+    any_updates = update_database(db, msgs->paths, msgs->n, do_fast_index);
     if (do_purge) {
       any_purges = cull_dead_messages(db, do_integrity_checks);
     }
diff --git a/mairix.h b/mairix.h
index 4ecc75b..1311b66 100755
--- a/mairix.h
+++ b/mairix.h
@@ -337,7 +337,7 @@ struct database *new_database_from_file(
 void free_database(struct database *db);
 void maybe_grow_message_arrays(struct database *db);
 void tokenise_message(int file_index, struct database *db, struct rfc822 *msg);
-int update_database(struct database *db, struct msgpath *sorted_paths, int 
n_paths);
+int update_database(struct database *db, struct msgpath *sorted_paths, int 
n_paths, int do_fast_index);
 void check_database_integrity(struct database *db);
 int cull_dead_messages(struct database *db, int do_integrity_checks);
 
diff --git a/rfc822.c b/rfc822.c
index 680ec2b..74b26e0 100755
--- a/rfc822.c
+++ b/rfc822.c
@@ -338,7 +338,7 @@ static void decode_header_value(char *te
         }
       }
     } else {
-      continue;        /* unknown encoding */
+      continue; /* unknown encoding */
     }
     q = e + 2;
   }
@@ -1004,9 +1004,9 @@ struct rfc822 *data_to_rfc822(struct msg
 }
 /*}}}*/
 
-#define ALLOC_NONE     1
-#define ALLOC_MMAP     2
-#define ALLOC_MALLOC   3
+#define ALLOC_NONE   1
+#define ALLOC_MMAP   2
+#define ALLOC_MALLOC 3
 
 int data_alloc_type;
 
@@ -1129,6 +1129,78 @@ #endif
 /*}}}*/
 #endif
 
+#if USE_GZIP_MBOX || USE_BZIP_MBOX
+/* do we need ROCACHE_SIZE > 1? the code supports any number here */
+#define ROCACHE_SIZE 1
+struct ro_mapping {
+  char *filename;
+  unsigned char *map;
+  size_t len;
+};
+static int ro_cache_init = 0;
+static struct ro_mapping ro_mapping_cache[ROCACHE_SIZE];
+
+/* find a temp file in the mapping cache.  If nothing is found lasti is
+ * set to the next slot to use for insertion.  You have to check that slot
+ * to see if it is currently in use
+ */
+static struct ro_mapping *find_ro_cache(const char *filename, int *lasti)
+{
+  int i = 0;
+  struct ro_mapping *ro = NULL;
+  if (lasti)
+    *lasti = 0;
+  if (!ro_cache_init)
+    return NULL;
+  for (i = 0 ; i < ROCACHE_SIZE ; i++) {
+    ro = ro_mapping_cache + i;
+    if (!ro->map) {
+      if (lasti)
+        *lasti = i;
+      return NULL;
+    }
+    if (strcmp(filename, ro->filename) == 0)
+      return ro;
+  }
+  /* if we're here, the map is full.  They will reuse slot 0 */
+  return NULL;
+}
+
+/*
+ * put a new tempfile into the cache.  It is mmaped as part of this function
+ * so you can safely close the file handle after calling this.
+ */
+static struct ro_mapping *add_ro_cache(const char *filename, int fd, size_t 
len)
+{
+  int i = 0;
+  struct ro_mapping *ro = NULL;
+  if (!ro_cache_init) {
+    memset(&ro_mapping_cache, 0, sizeof(ro_mapping_cache));
+    ro_cache_init = 1;
+  }
+  ro = find_ro_cache(filename, &i);
+  if (ro) {
+    fprintf(stderr, "%s already in ro cache\n", filename);
+    return NULL;
+  }
+  ro = ro_mapping_cache + i;
+  if (ro->map) {
+    munmap(ro->map, ro->len);
+    ro->map = NULL;
+    free(ro->filename);
+  }
+  ro->map = (unsigned char *)mmap(0, len, PROT_READ, MAP_SHARED, fd, 0);
+  if (ro->map == MAP_FAILED) {
+    ro->map = NULL;
+    perror("rfc822:mmap");
+    return NULL;
+  }
+  ro->len = len;
+  ro->filename = new_string(filename);
+  return ro;
+}
+#endif /* USE_GZIP_MBOX || USE_BZIP_MBOX */
+
 void create_ro_mapping(const char *filename, unsigned char **data, int 
*len)/*{{{*/
 {
   struct stat sb;
@@ -1147,37 +1219,75 @@ #endif
 
 #if USE_GZIP_MBOX || USE_BZIP_MBOX
   if(is_compressed(filename)) {
+    unsigned char *p;
+    size_t cur_read;
+    struct ro_mapping *ro;
+    FILE *tmpf;
+
+    /* this branch never returns things that are freeable */
+    data_alloc_type = ALLOC_NONE;
+    ro = find_ro_cache(filename, NULL);
+    if (ro) {
+      *data = ro->map;
+      *len = ro->len;
+      return;
+    }
+
     if(verbose) {
-       fprintf(stderr, "Decompressing %s...\n", filename);
+      fprintf(stderr, "Decompressing %s...\n", filename);
     }
 
+    tmpf = tmpfile();
+    if (!tmpf) {
+      perror("tmpfile");
+      goto comp_error;
+    }
     zf = xx_zopen(filename, "rb");
     if (!zf) {
       fprintf(stderr, "Could not open %s\n", filename);
-      *data = NULL;
-      *len = 0;
-      return;
+      goto comp_error;
     }
-    *data = new_array(unsigned char, SIZE_STEP);
-    *len = xx_zread(zf, *data, SIZE_STEP);
-    if (*len >= SIZE_STEP) {
-      int extra_bytes_read;
-      do {
-        *data = grow_array(unsigned char, *len + SIZE_STEP, *data);
-        extra_bytes_read = xx_zread(zf, *data + *len, SIZE_STEP);
-        *len += extra_bytes_read;
-      } while (extra_bytes_read > 0);
+    p = new_array(unsigned char, SIZE_STEP);
+    cur_read = xx_zread(zf, p, SIZE_STEP);
+    if (fwrite(p, cur_read, 1, tmpf) != 1) {
+      fprintf(stderr, "failed writing to temp file for %s\n", filename);
+      goto comp_error;
     }
+    *len = cur_read;
+    if (cur_read >= SIZE_STEP) {
+      while(1) {
+        int ret;
+        cur_read = xx_zread(zf, p, SIZE_STEP);
+        if (cur_read <= 0)
+          break;
+        *len += cur_read;
+        ret = fwrite(p, cur_read, 1, tmpf);
+        if (ret != 1) {
+          fprintf(stderr, "failed writing to temp file for %s\n", filename);
+          goto comp_error;
+        }
+      }
+    }
+    free(p);
     xx_zclose(zf);
 
     if(*len > 0) {
-      *data = grow_array(unsigned char, *len, *data);
-       data_alloc_type = ALLOC_MALLOC;
+      ro = add_ro_cache(filename, fileno(tmpf), *len);
+      if (!ro)
+        goto comp_error;
+      *data = ro->map;
+      *len = ro->len;
     } else {
-      free(*data);
-      data_alloc_type = ALLOC_NONE;
+      *data = NULL;
     }
+    fclose(tmpf);
+    return;
 
+comp_error:
+    *data = NULL;
+    *len = 0;
+    if (tmpf)
+      fclose(tmpf);
     return;
   }
 #endif /* USE_GZIP_MBOX || USE_BZIP_MBOX */
-------------------------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security?
Get stuff done quickly with pre-integrated technology to make your job easier
Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642
_______________________________________________
Mairix-users mailing list
[EMAIL PROTECTED]
https://lists.sourceforge.net/lists/listinfo/mairix-users

Reply via email to