On Sun, Dec 04, 2005 at 10:33:31PM +1030, Alan Modra wrote:
> On Sun, Dec 04, 2005 at 11:45:21AM +0000, Andrew Haley wrote:
> > Alan Modra writes:
> > > On Sun, Dec 04, 2005 at 12:35:31AM +0100, Gerald Pfeifer wrote:
> > > > spawns a recursive make (GNU make 3.80) that consumes some 450MB of
> > memory
> > > > and triggers a system load of 12+, basically rendering the machine
> > dead
> > > > for about a minute.
> > > >
> > > > On a different machine with only 512MB + 1GB swap, this time running
> > > > FreeBSD 5.3, I cannot install GCC any longer.
> > >
> > > I noticed something similar on a Linux machine with 512M + 1G swap when
> > > remaking libjava after editing some files. Thrashing for around 15
> > > minutes before finally proceeding.
> >
> > This might be make or the disastrously slow split-for-gcj shell
>
> It was "make". I forget the exact figures, but I saw approx the same
> mem usage as Gerald.
>
GNU make 3.80 is a HUGE memory hog. It calls xstrdup to build
dependency list. gnu-src-gcc.deps in libjava has 3000+ targets depend
the same 3000+ files, whose filenames are more than 260K. For this
dependency alone, make takes 3000*260K == 761MB.
This patch is a quick hack. It reduces the memory from 1.6GB to around
600MB for "make -f gnu-src-gcc.deps". I think make should use a better
memory management for strings. If my approach is OK, I can try to
use it through out make.
H.J.
----
--- make-3.80/file.c.memory 2002-10-03 19:13:42.000000000 -0700
+++ make-3.80/file.c 2005-12-08 15:37:27.000000000 -0800
@@ -434,7 +434,7 @@ snap_deps ()
if (d->file == 0)
d->file = enter_file (d->name);
else
- free (d->name);
+ hash_strfree (d->name);
d->name = 0;
}
free (file_slot_0);
--- make-3.80/implicit.c.memory 2002-09-04 00:26:19.000000000 -0700
+++ make-3.80/implicit.c 2005-12-08 15:37:27.000000000 -0800
@@ -539,7 +539,7 @@ pattern_search (file, archive, depth, re
dep->file = enter_file (dep->name);
/* enter_file uses dep->name _if_ we created a new file. */
if (dep->name != dep->file->name)
- free (dep->name);
+ hash_strfree (dep->name);
dep->name = 0;
dep->file->tried_implicit |= dep->changed;
}
--- make-3.80/main.c.memory 2002-08-09 18:27:17.000000000 -0700
+++ make-3.80/main.c 2005-12-08 15:37:27.000000000 -0800
@@ -501,6 +501,7 @@ initialize_global_hash_tables ()
init_hash_files ();
hash_init_directories ();
hash_init_function_table ();
+ init_hash_strings ();
}
static struct file *
--- make-3.80/make.h.memory 2002-09-11 09:55:44.000000000 -0700
+++ make-3.80/make.h 2005-12-08 15:37:27.000000000 -0800
@@ -427,6 +427,11 @@ extern char *find_char_unquote PARAMS ((
extern char *find_percent PARAMS ((char *));
extern FILE *open_tmpfile PARAMS ((char **, const char *));
+extern void init_hash_strings PARAMS ((void));
+extern char *hash_strdup PARAMS ((const char *));
+extern char *hash_savestring PARAMS ((const char *, unsigned int));
+extern void hash_strfree PARAMS ((char *));
+
#ifndef NO_ARCHIVES
extern int ar_name PARAMS ((char *));
extern void ar_parse_name PARAMS ((char *, char **, char **));
--- make-3.80/misc.c.memory 2002-09-12 15:15:58.000000000 -0700
+++ make-3.80/misc.c 2005-12-08 15:59:37.000000000 -0800
@@ -18,8 +18,10 @@ along with GNU Make; see the file COPYIN
the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA. */
+#include <assert.h>
#include "make.h"
#include "dep.h"
+#include "hash.h"
#include "debug.h"
/* Variadic functions. We go through contortions to allow proper function
@@ -564,7 +566,7 @@ copy_dep_chain (d)
c = (struct dep *) xmalloc (sizeof (struct dep));
bcopy ((char *) d, (char *) c, sizeof (struct dep));
if (c->name != 0)
- c->name = xstrdup (c->name);
+ c->name = hash_strdup (c->name);
c->next = 0;
if (firstnew == 0)
firstnew = lastnew = c;
@@ -891,3 +893,154 @@ atomic_readdir(dir)
}
#endif /* HAVE_BROKEN_RESTART */
+
+/* Hash table of duplicated strings. */
+
+struct hash_string
+{
+ char *string;
+ unsigned int count;
+};
+
+static unsigned long
+string_hash_1 (key)
+ const void *key;
+{
+ return_ISTRING_HASH_1 (((const struct hash_string *) key)->string);
+}
+
+static unsigned long
+string_hash_2 (key)
+ const void *key;
+{
+ return_ISTRING_HASH_2 (((const struct hash_string *) key)->string);
+}
+
+static int
+string_hash_cmp (x, y)
+ const void *x;
+ const void *y;
+{
+ return_ISTRING_COMPARE (((const struct hash_string *) x)->string,
+ ((const struct hash_string *) y)->string);
+}
+
+static struct hash_table strings;
+
+void
+init_hash_strings ()
+{
+ hash_init (&strings, 1000, string_hash_1, string_hash_2,
+ string_hash_cmp);
+}
+
+/* Keep track duplicated string and return the old one if exists. */
+
+char *
+hash_strdup (ptr)
+ const char *ptr;
+{
+ struct hash_string *h, key;
+
+ if (*ptr == '\0')
+ return "";
+
+ key.string = (char *) ptr;
+ key.count = 0;
+ h = (struct hash_string *) hash_find_item (&strings, &key);
+ if (h == NULL)
+ {
+ char *result = (char *) malloc (strlen (ptr) + 1);
+
+ if (result == NULL)
+ fatal (NILF, _("virtual memory exhausted"));
+
+ strcpy (result, ptr);
+
+ h = (struct hash_string *) malloc (sizeof (struct hash_string *));
+ if (h == NULL)
+ fatal (NILF, _("virtual memory exhausted"));
+
+ h->string = result;
+ h->count = 1;
+ hash_insert (&strings, h);
+ }
+ else
+ {
+ h->count++;
+ assert (h->count != 0);
+ }
+
+ return h->string;
+}
+
+char *
+hash_savestring (str, length)
+ const char *str;
+ unsigned int length;
+{
+ struct hash_string *h, key;
+
+ if (length == 0 || *str == '\0')
+ return "";
+
+ key.string = alloca (length + 1);
+ key.count = 0;
+ bcopy (str, key.string, length);
+ key.string [length] = '\0';
+
+ h = (struct hash_string *) hash_find_item (&strings, &key);
+ if (h == NULL)
+ {
+ char *out = (char *) xmalloc (length + 1);
+ bcopy (str, out, length);
+ out[length] = '\0';
+
+ h = (struct hash_string *) malloc (sizeof (struct hash_string *));
+ if (h == NULL)
+ fatal (NILF, _("virtual memory exhausted"));
+
+ h->string = out;
+ h->count = 1;
+ hash_insert (&strings, h);
+ }
+ else
+ {
+ h->count++;
+ assert (h->count != 0);
+ }
+
+ return h->string;
+}
+
+void
+hash_strfree (ptr)
+ char *ptr;
+{
+ struct hash_string *h, key;
+
+ if (*ptr == '\0')
+ return;
+
+ key.string = ptr;
+ key.count = 0;
+ h = (struct hash_string *) hash_find_item (&strings, &key);
+
+ /* Check if string comes from hash_strdup or hash_savestring. */
+ if (h == NULL || h->string != ptr)
+ {
+ free (ptr);
+ return;
+ }
+
+ h->count--;
+ if (h->count == 0)
+ {
+ struct hash_string *d;
+
+ d = hash_delete (&strings, h);
+ assert (d == h);
+ free (h->string);
+ free (h);
+ }
+}
--- make-3.80/read.c.memory 2005-12-07 09:50:43.000000000 -0800
+++ make-3.80/read.c 2005-12-08 15:37:27.000000000 -0800
@@ -1871,8 +1871,8 @@ record_files (filenames, pattern, patter
fatal (flocp,
_("target `%s' leaves prerequisite pattern empty"),
name);
- free (d->name);
- d->name = savestring (buffer, o - buffer);
+ hash_strfree (d->name);
+ d->name = hash_savestring (buffer, o - buffer);
}
}
}
@@ -2017,7 +2017,7 @@ record_files (filenames, pattern, patter
while (d != 0)
{
struct dep *nextd = d->next;
- free (d->name);
+ hash_strfree (d->name);
free ((char *)d);
d = nextd;
}
