On Tue, Jun 26, 2007 at 12:59:28AM -0400, Bill Nottingham wrote:
> Testing used for the following patch:
>
> - P4, 2Ghz
> - 512MB memory
> - local repos
> - upgrade a F7 GA chroot (866 packages, desktop install)
> to updates + updates-testing, using yum
> - upgrades done on a fresh reboot (clean cache)
>
> The attached patch shaves the run time for the full update from
> 22m37s to 15m19s. Even accounting for minor differences in disk
> location, that's a *significant* savings.
If you just want to speed up ldconfig, here's a glibc patch for you.
We've got it in SUSE Linux since SL10.0, no problems so far.
Cheers,
Michael.
--
Michael Schroeder [EMAIL PROTECTED]
SUSE LINUX Products GmbH, GF Markus Rex, HRB 16746 AG Nuernberg
main(_){while(_=~getchar())putchar(~_-1/(~(_|32)/13*2-11)*13);}
--- ./elf/cache.c.orig 2006-04-07 03:36:30.000000000 +0000
+++ ./elf/cache.c 2006-10-18 15:42:27.000000000 +0000
@@ -32,6 +32,16 @@
#include <ldconfig.h>
#include <dl-cache.h>
+#define CACHEMAGIC_ID "glibc-ld.so.id.1.0"
+
+struct cache_entry_id
+{
+ uint32_t ino;
+ uint32_t mtime;
+ uint32_t size;
+ uint32_t dev;
+};
+
struct cache_entry
{
char *lib; /* Library name. */
@@ -40,12 +50,16 @@ struct cache_entry
unsigned int osversion; /* Required OS version. */
uint64_t hwcap; /* Important hardware capabilities. */
int bits_hwcap; /* Number of bits set in hwcap. */
+ struct cache_entry_id id; /* unique id of entry */
struct cache_entry *next; /* Next entry in list. */
};
/* List of all cache entries. */
static struct cache_entry *entries;
+/* List of all old entries for incremental mode */
+static struct cache_entry *old_entries;
+
static const char *flag_descr[] =
{ "libc4", "ELF", "libc5", "libc6"};
@@ -298,6 +312,10 @@ save_cache (const char *cache_name)
++cache_entry_old_count;
}
+ if (opt_format != 0)
+ total_strlen += sizeof(CACHEMAGIC_ID) - 1 +
+ cache_entry_count * sizeof(struct cache_entry_id);
+
/* Create the on disk cache structure. */
/* First an array for all strings. */
strings = (char *)xmalloc (total_strlen);
@@ -399,6 +417,19 @@ save_cache (const char *cache_name)
&& idx_old < cache_entry_old_count)
file_entries->libs[idx_old] = file_entries->libs[idx_old - 1];
+ /* Add cache_id block to string space */
+ if (opt_format != 0)
+ {
+ file_entries_new->idblock = str_offset;
+ memcpy(str, CACHEMAGIC_ID, sizeof(CACHEMAGIC_ID) - 1);
+ str += sizeof(CACHEMAGIC_ID) - 1;
+ for (entry = entries; entry != NULL; entry = entry->next)
+ {
+ memcpy(str, &entry->id, sizeof entry->id);
+ str += sizeof entry->id;
+ }
+ }
+
/* Write out the cache. */
/* Write cache first to a temporary file and rename it later. */
@@ -473,7 +504,8 @@ save_cache (const char *cache_name)
/* Add one library to the cache. */
void
add_to_cache (const char *path, const char *lib, int flags,
- unsigned int osversion, uint64_t hwcap)
+ unsigned int osversion, uint64_t hwcap,
+ struct stat64 *stat_buf)
{
struct cache_entry *new_entry, *ptr, *prev;
char *full_path;
@@ -492,6 +524,10 @@ add_to_cache (const char *path, const ch
new_entry->osversion = osversion;
new_entry->hwcap = hwcap;
new_entry->bits_hwcap = 0;
+ new_entry->id.ino = (uint32_t)stat_buf->st_ino;
+ new_entry->id.mtime = (uint32_t)stat_buf->st_mtime;
+ new_entry->id.size = (uint32_t)stat_buf->st_size;
+ new_entry->id.dev = (uint32_t)stat_buf->st_dev;
/* Count the number of bits set in the masked value. */
for (i = 0; (~((1ULL << i) - 1) & hwcap) != 0 && i < 8 * sizeof (hwcap); ++i)
@@ -521,3 +557,163 @@ add_to_cache (const char *path, const ch
prev->next = new_entry;
}
}
+
+/* Load old cache to search for unchanged entries. Mostly copied
+ from print_cache. Be paraniod about data checking, as we don't
+ know if the cache is from the right architecture or currupted. */
+
+void
+load_old_cache (const char *cache_name)
+{
+ size_t cache_size;
+ const char *cache_data, *cache_data_id;
+ struct stat64 st;
+ int fd;
+ unsigned int i;
+ size_t j;
+ struct cache_file *cache;
+ struct cache_file_new *cache_new = NULL;
+ struct cache_entry *new_entry;
+ uint64_t hwcap;
+
+ old_entries = NULL;
+ if (!cache_name)
+ return;
+ fd = open (cache_name, O_RDONLY);
+ if (fd < 0)
+ return;
+ if (fstat64 (fd, &st) < 0 || st.st_size == 0)
+ {
+ close (fd);
+ return;
+ }
+ cache = mmap (0, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+ if (cache == MAP_FAILED)
+ {
+ close (fd);
+ return;
+ }
+ cache_size = st.st_size;
+ if (cache_size < sizeof (struct cache_file))
+ {
+ close (fd);
+ return;
+ }
+ cache_new = (struct cache_file_new *)cache;
+ if (!memcmp (cache->magic, CACHEMAGIC, sizeof CACHEMAGIC - 1))
+ {
+ if (cache->nlibs < 0 || cache->nlibs >= cache_size)
+ {
+ close (fd);
+ return;
+ }
+ size_t offset = ALIGN_CACHE (sizeof (struct cache_file)
+ + (cache->nlibs
+ * sizeof (struct file_entry)));
+ if (cache_size >
+ (offset + sizeof (struct cache_file_new)))
+ cache_new = (struct cache_file_new *) ((void *)cache + offset);
+ }
+ if (memcmp (cache_new->magic, CACHEMAGIC_NEW, sizeof CACHEMAGIC_NEW - 1)
+ || memcmp (cache_new->version, CACHE_VERSION,
+ sizeof CACHE_VERSION - 1))
+ {
+ close (fd);
+ return;
+ }
+ cache_data = (const char *) cache_new;
+ if (cache_new->idblock <= 0)
+ {
+ close (fd);
+ return;
+ }
+ if (cache_new->idblock + (cache_new->nlibs + 1) * sizeof(struct cache_entry_id) > cache_size)
+ {
+ close (fd);
+ return;
+ }
+ cache_data_id = cache_data + cache_new->idblock;
+ if (memcmp(cache_data_id, CACHEMAGIC_ID, sizeof(CACHEMAGIC_ID) - 1))
+ {
+ close (fd);
+ return;
+ }
+ cache_data_id += sizeof(CACHEMAGIC_ID) - 1;
+ memset(&st, 0, sizeof st);
+ for (i = 0; i < cache_new->nlibs; i++)
+ {
+ new_entry = (struct cache_entry *) xmalloc (sizeof (struct cache_entry));
+ new_entry->lib = xstrdup (cache_data + cache_new->libs[i].key);
+ new_entry->path = xstrdup (cache_data + cache_new->libs[i].value);
+ new_entry->flags = cache_new->libs[i].flags;
+ new_entry->osversion = cache_new->libs[i].osversion;
+ hwcap = cache_new->libs[i].hwcap;
+ new_entry->hwcap = hwcap;
+ new_entry->bits_hwcap = 0;
+ memcpy(&new_entry->id, cache_data_id + sizeof(struct cache_entry_id) * i, sizeof(struct cache_entry_id));
+ /* Count the number of bits set in the masked value. */
+ for (j = 0; (~((1ULL << j) - 1) & hwcap) != 0 && j < 8 * sizeof (hwcap); ++j)
+ if ((hwcap & (1ULL << j)) != 0)
+ ++new_entry->bits_hwcap;
+ new_entry->next = old_entries;
+ old_entries = new_entry;
+ }
+ close (fd);
+ return;
+}
+
+int
+search_old_cache(const char *file, struct stat64 *stat_buf, int *flags, unsigned int *osversion, char **soname)
+{
+ struct cache_entry *entry;
+ struct cache_entry_id id;
+
+ id.ino = (uint32_t)stat_buf->st_ino;
+ id.mtime = (uint32_t)stat_buf->st_mtime;
+ id.size = (uint32_t)stat_buf->st_size;
+ id.dev = (uint32_t)stat_buf->st_dev;
+ for (entry = old_entries; entry; entry = entry->next)
+ {
+ if (id.ino == entry->id.ino &&
+ id.mtime == entry->id.mtime &&
+ id.size == entry->id.size &&
+ id.dev == entry->id.dev)
+ {
+#if 0
+ const char *p1, *p2;
+ /* basename check for extra safety */
+ p1 = strrchr(entry->path, '/');
+ if (!p1)
+ p1 = entry->path;
+ else
+ p1++;
+ for (p2 = p1; *p2; p2++)
+ if (*p2 == '.' || *p2 == '-')
+ break;
+ if (p2 != p1 && strncmp(file, p1, p2 - p1))
+ return 0;
+#endif
+ *flags = entry->flags;
+ *osversion = entry->osversion;
+ *soname = xstrdup(entry->lib);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+void
+free_old_cache(void)
+{
+ struct cache_entry *entry;
+
+ while (old_entries)
+ {
+ entry = old_entries;
+ free (entry->path);
+ free (entry->lib);
+ old_entries = entry->next;
+ free (entry);
+ }
+}
+
--- ./elf/ldconfig.c.orig 2006-04-07 06:57:49.000000000 +0000
+++ ./elf/ldconfig.c 2006-10-18 15:43:23.000000000 +0000
@@ -122,6 +122,9 @@ static unsigned long int hwcap_mask = HW
/* Configuration-defined capabilities defined in kernel vDSOs. */
static const char *hwcap_extra[64 - _DL_FIRST_EXTRA];
+/* Should we ignore an old cache file? */
+static int opt_ignore_old_cache;
+
/* Name and version of program. */
static void print_version (FILE *stream, struct argp_state *state);
void (*argp_program_version_hook) (FILE *, struct argp_state *)
@@ -140,6 +143,7 @@ static const struct argp_option options[
{ NULL, 'n', NULL, 0, N_("Only process directories specified on the command line. Don't build cache."), 0},
{ NULL, 'l', NULL, 0, N_("Manually link individual libraries."), 0},
{ "format", 'c', N_("FORMAT"), 0, N_("Format to use: new, old or compat (default)"), 0},
+ { "ignore-old-cache", 'i', NULL, 0, N_("Ignore old cache file"), 0},
{ NULL, 0, NULL, 0, NULL, 0 }
};
@@ -270,6 +274,9 @@ parse_opt (int key, char *arg, struct ar
else if (strcmp (arg, "new") == 0)
opt_format = 2;
break;
+ case 'i':
+ opt_ignore_old_cache = 1;
+ break;
default:
return ARGP_ERR_UNKNOWN;
}
@@ -571,7 +578,7 @@ manual_link (char *library)
return;
}
if (process_file (real_library, library, libname, &flag, &osversion,
- &soname, 0))
+ &soname, 0, NULL))
{
error (0, 0, _("No link created since soname could not be found for %s"),
library);
@@ -616,6 +623,7 @@ struct dlib_entry
int flag;
int is_link;
unsigned int osversion;
+ struct stat64 stat_buf;
struct dlib_entry *next;
};
@@ -631,7 +639,7 @@ search_dir (const struct dir_entry *entr
struct dlib_entry *dlibs;
struct dlib_entry *dlib_ptr;
struct stat64 lstat_buf, stat_buf;
- int is_link, is_dir;
+ int is_link, is_dir, has_soname;
uint64_t hwcap = path_hwcap (entry->path);
unsigned int osversion;
@@ -725,16 +733,11 @@ search_dir (const struct dir_entry *entr
}
sprintf (real_file_name, "%s/%s", dir_name, direntry->d_name);
}
-#ifdef _DIRENT_HAVE_D_TYPE
- if (direntry->d_type != DT_UNKNOWN)
- lstat_buf.st_mode = DTTOIF (direntry->d_type);
- else
-#endif
- if (__builtin_expect (lstat64 (real_file_name, &lstat_buf), 0))
- {
- error (0, errno, _("Cannot lstat %s"), file_name);
- continue;
- }
+ if (__builtin_expect (lstat64 (real_file_name, &lstat_buf), 0))
+ {
+ error (0, errno, _("Cannot lstat %s"), file_name);
+ continue;
+ }
is_link = S_ISLNK (lstat_buf.st_mode);
if (is_link)
@@ -752,6 +755,10 @@ search_dir (const struct dir_entry *entr
continue;
}
is_dir = S_ISDIR (stat_buf.st_mode);
+ lstat_buf.st_ino = stat_buf.st_ino;
+ lstat_buf.st_mtime = stat_buf.st_mtime;
+ lstat_buf.st_size = stat_buf.st_size;
+ lstat_buf.st_dev = stat_buf.st_dev;
}
else
is_dir = S_ISDIR (lstat_buf.st_mode);
@@ -772,20 +779,6 @@ search_dir (const struct dir_entry *entr
}
else
{
-#ifdef _DIRENT_HAVE_D_TYPE
- /* We have filled in lstat only #ifndef
- _DIRENT_HAVE_D_TYPE. Fill it in if needed. */
- if (direntry->d_type != DT_UNKNOWN
- && __builtin_expect (lstat64 (real_file_name, &lstat_buf),
- 0))
- {
- error (0, errno, _("Cannot lstat %s"), file_name);
- free (new_entry->path);
- free (new_entry);
- continue;
- }
-#endif
-
new_entry->ino = lstat_buf.st_ino;
new_entry->dev = lstat_buf.st_dev;
}
@@ -808,15 +801,18 @@ search_dir (const struct dir_entry *entr
else
real_name = real_file_name;
- if (process_file (real_name, file_name, direntry->d_name, &flag,
- &osversion, &soname, is_link))
+ has_soname = 1;
+ if (!search_old_cache(direntry->d_name, &lstat_buf, &flag, &osversion, &soname))
{
- if (real_name != real_file_name)
- free (real_name);
- continue;
+ if (process_file (real_name, file_name, direntry->d_name, &flag,
+ &osversion, &soname, is_link, &has_soname))
+ {
+ if (real_name != real_file_name)
+ free (real_name);
+ continue;
+ }
}
-
/* A link may just point to itself. */
if (is_link)
{
@@ -839,6 +835,8 @@ search_dir (const struct dir_entry *entr
if (is_link)
{
+ if (strcmp(soname, direntry->d_name) != 0)
+ has_soname = 0;
free (soname);
soname = xstrdup (direntry->d_name);
}
@@ -846,7 +844,21 @@ search_dir (const struct dir_entry *entr
if (flag == FLAG_ELF
&& (entry->flag == FLAG_ELF_LIBC5
|| entry->flag == FLAG_ELF_LIBC6))
- flag = entry->flag;
+ {
+ flag = entry->flag;
+ /* invalidate id data as we changed flag */
+ has_soname = 0;
+ }
+
+ if (!has_soname)
+ {
+ /* soname or flags depend on file name of dir properties,
+ so invalidate id entry */
+ lstat_buf.st_ino = 0;
+ lstat_buf.st_mtime = 0;
+ lstat_buf.st_size = 0;
+ lstat_buf.st_dev = 0;
+ }
/* Some sanity checks to print warnings. */
if (opt_verbose)
{
@@ -885,13 +897,20 @@ search_dir (const struct dir_entry *entr
&& flag == FLAG_ELF)
dlib_ptr->flag = flag;
else
- error (0, 0, _("libraries %s and %s in directory %s have same soname but different type."),
- dlib_ptr->name, direntry->d_name, entry->path);
+ {
+ error (0, 0, _("libraries %s and %s in directory %s have same soname but different type."),
+ dlib_ptr->name, direntry->d_name, entry->path);
+ lstat_buf.st_ino = 0;
+ lstat_buf.st_mtime = 0;
+ lstat_buf.st_size = 0;
+ lstat_buf.st_dev = 0;
+ }
}
free (dlib_ptr->name);
dlib_ptr->osversion = osversion;
dlib_ptr->name = xstrdup (direntry->d_name);
dlib_ptr->is_link = is_link;
+ dlib_ptr->stat_buf = lstat_buf;
}
/* Don't add this library, abort loop. */
/* Also free soname, since it's dynamically allocated. */
@@ -908,6 +927,7 @@ search_dir (const struct dir_entry *entr
dlib_ptr->osversion = osversion;
dlib_ptr->soname = soname;
dlib_ptr->is_link = is_link;
+ dlib_ptr->stat_buf = lstat_buf;
/* Add at head of list. */
dlib_ptr->next = dlibs;
dlibs = dlib_ptr;
@@ -926,7 +946,7 @@ search_dir (const struct dir_entry *entr
dlib_ptr->soname);
if (opt_build_cache)
add_to_cache (entry->path, dlib_ptr->soname, dlib_ptr->flag,
- dlib_ptr->osversion, hwcap);
+ dlib_ptr->osversion, hwcap, &dlib_ptr->stat_buf);
}
/* Free all resources. */
@@ -1288,8 +1308,12 @@ main (int argc, char **argv)
add_system_dir (LIBDIR);
}
+ load_old_cache(opt_ignore_old_cache ? NULL : cache_file);
+
search_dirs ();
+ free_old_cache();
+
if (opt_build_cache)
save_cache (cache_file);
--- ./elf/readelflib.c.orig 2005-12-14 10:05:56.000000000 +0000
+++ ./elf/readelflib.c 2006-10-18 09:44:04.000000000 +0000
@@ -213,7 +213,7 @@ process_elf_file (const char *file_name,
/* We reach this point only if the file doesn't contain a DT_SONAME
or if we can't classify the library. If it doesn't have a
soname, return the name of the library. */
- if (*soname == NULL)
+ if (*soname == NULL && lib != NULL)
*soname = xstrdup (lib);
return 0;
--- ./elf/readlib.c.orig 2005-12-21 22:16:20.000000000 +0000
+++ ./elf/readlib.c 2006-10-18 09:48:36.000000000 +0000
@@ -68,7 +68,7 @@ static struct known_names known_libs[] =
int
process_file (const char *real_file_name, const char *file_name,
const char *lib, int *flag, unsigned int *osversion,
- char **soname, int is_link)
+ char **soname, int is_link, int *has_soname)
{
FILE *file;
struct stat64 statbuf;
@@ -80,6 +80,8 @@ process_file (const char *real_file_name
ret = 0;
*flag = FLAG_ANY;
*soname = NULL;
+ if (has_soname)
+ *has_soname = 0;
file = fopen (real_file_name, "rb");
if (file == NULL)
@@ -165,9 +167,16 @@ process_file (const char *real_file_name
/* Libraries have to be shared object files. */
else if (elf_header->e_type != ET_DYN)
ret = 1;
- else if (process_elf_file (file_name, lib, flag, osversion, soname,
+ else
+ {
+ if (process_elf_file (file_name, NULL, flag, osversion, soname,
file_contents, statbuf.st_size))
- ret = 1;
+ ret = 1;
+ else if (!*soname)
+ *soname = xstrdup(lib);
+ else if (has_soname)
+ *has_soname = 1;
+ }
done:
/* Clean up allocated memory and resources. */
--- ./sysdeps/generic/dl-cache.h.orig 2003-06-25 08:01:22.000000000 +0000
+++ ./sysdeps/generic/dl-cache.h 2006-10-17 18:58:08.000000000 +0000
@@ -90,7 +90,8 @@ struct cache_file_new
char version[sizeof CACHE_VERSION - 1];
uint32_t nlibs; /* Number of entries. */
uint32_t len_strings; /* Size of string table. */
- uint32_t unused[5]; /* Leave space for future extensions
+ uint32_t idblock; /* Id block index in string table */
+ uint32_t unused[4]; /* Leave space for future extensions
and align to 8 byte boundary. */
struct file_entry_new libs[0]; /* Entries describing libraries. */
/* After this the string table of size len_strings is found. */
--- ./sysdeps/generic/ldconfig.h.orig 2003-03-14 05:32:49.000000000 +0000
+++ ./sysdeps/generic/ldconfig.h 2006-10-18 10:28:30.000000000 +0000
@@ -43,12 +43,21 @@ extern void init_cache (void);
extern void save_cache (const char *cache_name);
extern void add_to_cache (const char *path, const char *lib, int flags,
- unsigned int osversion, uint64_t hwcap);
+ unsigned int osversion, uint64_t hwcap,
+ struct stat64 *stat_buf);
+
+extern void load_old_cache (const char *cache_name);
+
+extern int search_old_cache (const char *file, struct stat64 *stat_buf,
+ int *flags, unsigned int *osversion,
+ char **soname);
+
+extern void free_old_cache (void);
/* Declared in readlib.c. */
extern int process_file (const char *real_file_name, const char *file_name,
const char *lib, int *flag, unsigned int *osversion,
- char **soname, int is_link);
+ char **soname, int is_link, int *has_soname);
/* Declared in readelflib.c. */
extern int process_elf_file (const char *file_name, const char *lib, int *flag,
_______________________________________________
Rpm-maint mailing list
Rpm-maint@lists.rpm.org
https://lists.rpm.org/mailman/listinfo/rpm-maint
