The attached patch helps with low-memory performance by implementing
segregated memory storage. A separate obstack is used for allocating
information related to the Installed-Size, Origin, Maintainer, Bugs,
Architecture, Source, Filename, Size, MD5sum, MSDOS-Filename and
Description fields, as well as any arbitrary field contents
(which these days seems to include SHA1, SHA256, Tag and Task fields).
Using a separate obstack does not reduce memory usage, but it helps cope
with memory pressure. Operations such as dependency computations or
package sorting do not depend on anything that's stored in the secondary
obstack, so the segregation greatly reduces the working set size.
As a practical result, starting 'dselect select' with a non-trimmed
available package set (my sources.list lists just etch main) used to take
5 minutes and 25 seconds before it got to the dselect screen, much of
it being spent in the package quicksort, it now takes about 42 seconds
(most of which seems to be spent parsing the available file).
There is still some swapping while reading the available file, my next
step will be to try to reduce this by avoiding mmap'ing the whole file.
--
Michel "Walken" Lespinasse
"Bill Gates is a monocle and a Persian cat away from being the villain
in a James Bond movie." -- Dennis Miller
diff -ru dpkg-1.13.24.orig/lib/dpkg-db.h dpkg-1.13.24/lib/dpkg-db.h
--- dpkg-1.13.24.orig/lib/dpkg-db.h 2006-06-28 17:15:10.000000000 -0700
+++ dpkg-1.13.24/lib/dpkg-db.h 2006-10-27 05:16:21.000000000 -0700
@@ -303,8 +303,11 @@
/*** from nfmalloc.c ***/
extern void *nfmalloc(size_t);
+extern void *nfmalloc_secondary(size_t);
char *nfstrsave(const char*);
+char *nfstrsave_secondary(const char*);
char *nfstrnsave(const char*, int);
+char *nfstrnsave_secondary(const char*, int);
void nffreeall(void);
/*** from showpkg.c ***/
diff -ru dpkg-1.13.24.orig/lib/fields.c dpkg-1.13.24/lib/fields.c
--- dpkg-1.13.24.orig/lib/fields.c 2006-06-28 17:13:45.000000000 -0700
+++ dpkg-1.13.24/lib/fields.c 2006-10-27 05:14:47.000000000 -0700
@@ -94,7 +94,7 @@
_("file details field `%s' not allowed in status
file"),fip->name);
allowextend= !pigp->files;
fdpp= &pigp->files;
- cpos= nfstrsave(value);
+ cpos= nfstrsave_secondary(value);
while (*cpos) {
space= cpos; while (*space && !isspace(*space)) space++;
if (*space) *space++= 0;
@@ -103,7 +103,7 @@
if (!allowextend)
parseerr(NULL,filename,lno, warnto,warncount,pigp,0, _("too many
values "
"in file details field `%s' (compared to others)"),fip->name);
- fdp= nfmalloc(sizeof(struct filedetails));
+ fdp= nfmalloc_secondary(sizeof(struct filedetails));
fdp->next= NULL;
fdp->name= fdp->msdosname= fdp->size= fdp->md5sum= NULL;
*fdpp= fdp;
@@ -122,7 +122,7 @@
enum parsedbflags flags,
const char *filename, int lno, FILE *warnto, int *warncount,
const char *value, const struct fieldinfo *fip) {
- if (*value) PKGPFIELD(pifp,fip->integer,char*)= nfstrsave(value);
+ if (*value) PKGPFIELD(pifp,fip->integer,char*)= nfstrsave_secondary(value);
}
void f_boolean(struct pkginfo *pigp, struct pkginfoperfile *pifp,
diff -ru dpkg-1.13.24.orig/lib/nfmalloc.c dpkg-1.13.24/lib/nfmalloc.c
--- dpkg-1.13.24.orig/lib/nfmalloc.c 2006-06-21 06:41:12.000000000 -0700
+++ dpkg-1.13.24/lib/nfmalloc.c 2006-10-27 05:15:34.000000000 -0700
@@ -31,18 +31,20 @@
#define obstack_chunk_alloc m_malloc
#define obstack_chunk_free free
-static struct obstack db_obs;
+static struct obstack db_obs, db_obs_secondary;
static int dbobs_init = 0;
/* We use lots of mem, so use a large chunk */
-#define CHUNK_SIZE 8192
+#define CHUNK_SIZE 65536
#define OBSTACK_INIT if (!dbobs_init) { nfobstack_init(); }
static void nfobstack_init(void) {
obstack_init(&db_obs);
+ obstack_init(&db_obs_secondary);
dbobs_init = 1;
obstack_chunk_size(&db_obs) = CHUNK_SIZE;
+ obstack_chunk_size(&db_obs_secondary) = CHUNK_SIZE;
}
inline void *nfmalloc(size_t size)
@@ -51,19 +53,36 @@
return obstack_alloc(&db_obs, size);
}
+inline void *nfmalloc_secondary(size_t size)
+{
+ OBSTACK_INIT;
+ return obstack_alloc(&db_obs_secondary, size);
+}
+
char *nfstrsave(const char *string) {
OBSTACK_INIT;
return obstack_copy0 (&db_obs, string, strlen(string));
}
+char *nfstrsave_secondary(const char *string) {
+ OBSTACK_INIT;
+ return obstack_copy0 (&db_obs_secondary, string, strlen(string));
+}
+
char *nfstrnsave(const char *string, int l) {
OBSTACK_INIT;
return obstack_copy0 (&db_obs, string, l);
}
+char *nfstrnsave_secondary(const char *string, int l) {
+ OBSTACK_INIT;
+ return obstack_copy0 (&db_obs_secondary, string, l);
+}
+
void nffreeall(void) {
if (dbobs_init) {
obstack_free(&db_obs, NULL);
+ obstack_free(&db_obs_secondary, NULL);
dbobs_init = 0;
}
}
diff -ru dpkg-1.13.24.orig/lib/parse.c dpkg-1.13.24/lib/parse.c
--- dpkg-1.13.24.orig/lib/parse.c 2006-06-28 17:20:48.000000000 -0700
+++ dpkg-1.13.24/lib/parse.c 2006-10-27 04:52:09.000000000 -0700
@@ -96,6 +96,7 @@
char *data, *dataptr, *endptr;
const char *fieldstart, *valuestart;
char *value= NULL;
+ int valuealloc= 0;
int fieldlen= 0, valuelen= 0;
int *ip, c;
struct stat stat;
@@ -203,7 +204,10 @@
fip->name && strncasecmp(fieldstart,fip->name, fieldlen);
fip++, ip++);
if (fip->name) {
- value= realloc(value,valuelen+1);
+ if (valuelen >= valuealloc) {
+ valuealloc=valuelen+1;
+ value= realloc(value,valuealloc);
+ }
memcpy(value,valuestart,valuelen);
*(value+valuelen)= 0;
if (*ip++)
@@ -223,7 +227,7 @@
}
arp= nfmalloc(sizeof(struct arbitraryfield));
arp->name= nfstrnsave(fieldstart,fieldlen);
- arp->value= nfstrnsave(valuestart,valuelen);
+ arp->value= nfstrnsave_secondary(valuestart,valuelen);
arp->next= NULL;
*larpp= arp;
}
