OpenPKG CVS Repository
http://cvs.openpkg.org/
____________________________________________________________________________
Server: cvs.openpkg.org Name: Ralf S. Engelschall
Root: /v/openpkg/cvs Email: [EMAIL PROTECTED]
Module: openpkg-tools Date: 20-Jul-2006 09:46:44
Branch: HEAD Handle: 2006072008464100
Modified files:
openpkg-tools/cmd build.pl
Log:
"openpkg build" run-time BOOOOOOOOSTING by reducing the run-time by
95%(!!). This achieved by performing a larger "all-by-one" query for
"openpkg rpm -qa --provides" and "openpkg rpm -qa -i" and caching the
results instead of running the two commands individually for each(!)
installed package. On rm0.openpkg.net:/openpkg-dev with 295 installed
packages, a "openpkg build -Ua" call now takes just about 5s(!) instead
of 81s.
Summary:
Revision Changes Path
1.31 +57 -13 openpkg-tools/cmd/build.pl
____________________________________________________________________________
patch -p0 <<'@@ .'
Index: openpkg-tools/cmd/build.pl
============================================================================
$ cvs diff -u -r1.30 -r1.31 build.pl
--- openpkg-tools/cmd/build.pl 19 Jul 2006 18:12:57 -0000 1.30
+++ openpkg-tools/cmd/build.pl 20 Jul 2006 07:46:41 -0000 1.31
@@ -38,8 +38,6 @@
$opt_L $opt_W $opt_K $opt_e $opt_b $opt_B $opt_g
};
-$| = 1; # autoflush STDOUT
-
# global context variables
my $prg = "openpkg build";
my %env = ('' => {});
@@ -91,7 +89,7 @@
" -q ignore all reverse dependencies\n" .
" -s generate status map instead of shell script\n" .
" -S generate status map instead of shell script
(including new)\n" .
- " -X ignore installed Perl XML parser module and use
internal one\n" .
+ " -X use external XML/RDF parser instead of internal
one\n" .
" -M generate short dependency map instead of shell
script\n" .
" -L generate list of packages in repository
depending on target\n" .
" -W include dependencies as if all build options are
enabled\n" .
@@ -829,7 +827,7 @@
# fetch XML/RDF index from file or URL
# (recursively fetches sub-indexes, too)
sub get_index ($$$$$) {
- my ($url, $fn, $noxml, $pfmatch, $installed) = @_;
+ my ($url, $fn, $xml, $pfmatch, $installed) = @_;
my (%map, $include);
my ($fetch, $bzip2, $path);
my ($parser);
@@ -858,14 +856,14 @@
die "openpkg:build:FATAL: cannot open '$fetch' ($!)\n";
# if XML parser can be used, try to lazy-load it
- if (not $noxml) {
+ if ($xml) {
eval { require XML::Simple; };
- $noxml = 1 if $@;
+ $xml = 0 if ($@);
}
# determine and run XML parser
# (returns contained index includes)
- $parser = ($noxml ? \&simple_text_parser : \&xml_parser);
+ $parser = ($xml ? \&xml_parser : \&simple_text_parser);
$include = $parser->(\*RFH, $url, \%map, $pfmatch, $installed);
# close index
@@ -878,7 +876,7 @@
foreach (@$include) {
my ($submap);
my ($suburl, $subfn) = relurl($url, $fn, $_);
- $submap = get_index($suburl, $subfn, $noxml, $pfmatch, $installed);
# RECURSION
+ $submap = get_index($suburl, $subfn, $xml, $pfmatch, $installed); #
RECURSION
while (my ($name, $vmap) = each(%$submap)) {
while (my ($vs, $recs) = each(%$vmap)) {
push(@{$map{$name}->{$vs}}, @$recs);
@@ -1378,6 +1376,7 @@
}
# pull in OPTIONS for a package or an RPM file
+my $get_with_cache = {};
sub get_with ($;$) {
my ($t, $fn) = @_;
my (@l, %with);
@@ -1389,14 +1388,47 @@
if (defined($fn)) {
@l = run($config->{"rpm"} . " -q --provides -p $fn");
} else {
- @l = run($config->{"rpm"} . " -q --provides $t->{name}");
+ if (not exists($get_with_cache->{-provides})) {
+ # pre-cache the "provides" query for all(!) packages at
once for speedup
+ my @c = run($config->{"rpm"} . " -q -a --qf " .
+ '\'[%{NAME} %{PROVIDENAME}
%|PROVIDEFLAGS?{%{PROVIDEFLAGS:depflags} %{PROVIDEVERSION}}:{}|\n]\'');
+ $get_with_cache->{-provides} = {};
+ foreach my $c (@c) {
+ if (m/^(\S+)\s+(.+)$/s) {
+ $get_with_cache->{-provides}->{$1} = [] if (not
exists($get_with_cache->{-provides}->{$1}));
+ push(@{$get_with_cache->{-provides}->{$1}}, $2);
+ }
+ }
+ }
+ @l = $get_with_cache->{-provides}->{$t->{name}};
+ if (not @l) {
+ # (should not happen in practice, but anyway)
+ @l = run($config->{"rpm"} . " -q --provides $t->{name}");
+ $get_with_cache->{-provides}->{$t->{name}} = [ @l ];
+ }
}
$opt = parse_provideslist([EMAIL PROTECTED]);
if (scalar(keys(%$opt)) == 0) {
if (defined($fn)) {
@l = run($config->{"rpm"} . " -qi -p $fn");
} else {
- @l = run($config->{"rpm"} . " -qi $t->{name}");
+ if (not exists($get_with_cache->{-infos})) {
+ # pre-cache the "infos" query for all(!) packages at
once for speedup
+ my @c = run($config->{"rpm"} . " -qi -a");
+ my $p = "";
+ $get_with_cache->{-infos} = {};
+ foreach my $c (@c) {
+ $p = $1 if ($c =~ m/^Name:\s+(\S+)/s);
+ $get_with_cache->{-infos}->{$p} = [] if (not
exists($get_with_cache->{-infos}->{$p}));
+ push(@{$get_with_cache->{-infos}->{$p}}, $c);
+ }
+ }
+ @l = $get_with_cache->{-infos}->{$t->{name}};
+ if (not @l) {
+ # (should not happen in practice, but anyway)
+ @l = run($config->{"rpm"} . " -qi $t->{name}");
+ $get_with_cache->{-infos}->{$t->{name}} = [ @l ];
+ }
}
$opt = parse_options([EMAIL PROTECTED]);
}
@@ -2265,7 +2297,19 @@
}
sub priv ($) { cmd($opt_P, $_[0]); }
sub npriv ($) { cmd($opt_N, $_[0]); }
-sub run ($) { my ($c) = cmd($opt_N, $_[0]); `$c` }
+
+# execute a command
+my $run_cache = {};
+sub run ($) {
+ my $cmd = cmd($opt_N, $_[0]);
+ my $out = $run_cache->{$cmd};
+ if (not defined($out)) {
+ my @out = `$cmd`;
+ $out = [ @out ];
+ $run_cache->{$cmd} = $out;
+ }
+ return (wantarray ? @{$out} : join(//, @{$out}));
+}
# print dependency list
sub print_deps ($) {
@@ -2727,8 +2771,8 @@
=item B<-X>
-Ignore an installed Perl XML parser module and use the internal
-simple XML/RDF parser instead.
+Use the slower but more robust external Perl XML parser module
+XML::Simple instead of the simple internal XML/RDF parser.
=item B<-K>
@@ .
______________________________________________________________________
The OpenPKG Project www.openpkg.org
CVS Repository Commit List [email protected]
