This looks very great! The recent work on pkg_add proved very useful,
especially updating dependencies proved very useful! Great work!
Cheers,
Jasper
On Wed, 17 Aug 2005 18:44:59 +0200
Marc Espie <[EMAIL PROTECTED]> wrote:
> I'm currently running with the following patch.
>
> - this makes find-all-conflicts much more useful (and faster)
> when dealing with huge sets of 5000~10000 packages.
>
> - if PLIST_DB is defined to an existing directory, all packaging
> activity automatically registers a `cleaned' plist in that directory.
> By cleaned, I mean that it removes local contingencies (such as the md5
> of files, or the exact value of dependencies and cvs tags), and also
> upgrades some old behavior of the pkg tools to what the current tools
> would do.
>
> This serves two purposes: people running bulk-builds can start creating
> a database of packing-lists for all exiting versions of all ports (which
> is useful when looking at update issues), and they can also detect bugs
> where a maintainer forgot to commit a package bump along with a plist
> change.
>
>
> I don't intend to commit this before 3.8, but it proved to be very useful
> for me, so I'm sharing.
>
> Index: mk/bsd.port.mk
> ===================================================================
> RCS file: /cvs/ports/infrastructure/mk/bsd.port.mk,v
> retrieving revision 1.704
> diff -u -p -r1.704 bsd.port.mk
> --- mk/bsd.port.mk 4 Jul 2005 12:32:51 -0000 1.704
> +++ mk/bsd.port.mk 17 Aug 2005 16:39:25 -0000
> @@ -101,6 +101,7 @@ BULK_COOKIES_DIR?= ${PORTSDIR}/bulk/${MA
> UPDATE_COOKIES_DIR?= ${PORTSDIR}/update/${MACHINE_ARCH}
> TEMPLATES?= ${PORTSDIR}/infrastructure/templates
> TMPDIR?= /tmp
> +PLIST_DB?=
>
> PKGREPOSITORYBASE?= ${PORTSDIR}/packages/${MACHINE_ARCH}
> PKGREPOSITORY?= ${PKGREPOSITORYBASE}/all
> @@ -1968,6 +1969,9 @@ _package: ${_PKG_PREREQ}
> ${SUDO} ${MAKE} _internal-clean=package; \
> exit 1; \
> fi
> +.if !empty(PLIST_DB)
> + @perl /usr/ports/infrastructure/package/register-plist ${PLIST_DB}
> ${PKGFILE${SUBPACKAGE}}
> +.endif
> # End of PACKAGE.
> .endif
> .if target(post-package)
> Index: package/find-all-conflicts
> ===================================================================
> RCS file: /cvs/ports/infrastructure/package/find-all-conflicts,v
> retrieving revision 1.9
> diff -u -p -r1.9 find-all-conflicts
> --- package/find-all-conflicts 14 Aug 2005 11:57:17 -0000 1.9
> +++ package/find-all-conflicts 17 Aug 2005 16:39:25 -0000
> @@ -1,7 +1,7 @@
> #!/usr/bin/perl
>
> -# $OpenBSD: find-all-conflicts,v 1.9 2005/08/14 11:57:17 espie Exp $
> -# Copyright (c) 2000
> +# $OpenBSD: find-all-conflicts,v 1.8 2004/11/11 15:38:02 espie Exp $
> +# Copyright (c) 2000-2005
> # Marc Espie. All rights reserved.
> # Redistribution and use in source and binary forms, with or without
> # modification, are permitted provided that the following conditions
> @@ -36,39 +36,102 @@ use OpenBSD::PackageInfo;
> use OpenBSD::PackingList;
> use OpenBSD::PkgCfl;
>
> -sub analyze
> +package OpenBSD::PackingElement;
> +sub register
> {
> - my ($plist, $all) = @_;
> - my $pkgname = $plist->pkgname();
> +}
> +
> +package OpenBSD::PackingElement::FileBase;
> +
> +sub register
> +{
> + my ($self, $all_conflict, $all_deps, $pkgname) = @_;
> +
> + my $file= File::Spec->canonpath($self->fullname());
> + unless (defined $all_conflict->{$file}) {
> + $all_conflict->{$file} = [];
> + }
> + push @{$all_conflict->{$file}}, $pkgname;
> +}
>
> - for my $item (@{$plist->{items}}) {
> - next unless $item->IsFile();
> - my $file= File::Spec->canonpath($item->fullname());
> - unless (defined $all->{$file}) {
> - $all->{$file} = [];
> +package OpenBSD::PackingElement::Depend;
> +sub register
> +{
> + my ($self, $all_conflict, $all_deps, $pkgname) = @_;
> + if (defined $self->{def}) {
> + unless (defined $all_deps->{$pkgname}) {
> + $all_deps->{$pkgname} = [];
> }
> - push @{$all->{$file}}, $pkgname;
> + push @{$all_deps->{$pkgname}}, $self->{def};
> }
> }
>
> -sub show_problems
> +
> +package main;
> +
> +my $cache = {};
> +my $cache2 = {};
> +
> +sub find_a_conflict
> {
> - my ($h, $conflicts) = @_;
> + my ($conflicts, $deps, $pkg, $pkg2) = @_;
> +
> + if (defined $conflicts->{$pkg} &&
> $conflicts->{$pkg}->conflicts_with($pkg2)) {
> + return 1;
> + }
> + if (defined $deps->{$pkg}) {
> + for my $dep (@{$deps->{$pkg}}) {
> + if (find_a_conflict($conflicts, $deps, $dep, $pkg2)) {
> + return 1;
> + }
> + }
> + }
> + if (defined $deps->{$pkg2}) {
> + for my $dep (@{$deps->{$pkg2}}) {
> + if (find_a_conflict($conflicts, $deps, $pkg, $dep)) {
> + return 1;
> + }
> + }
> + }
> + return 0;
> +}
> +
> +sub compute_problems
> +{
> + my ($h, $conflicts, $deps) = @_;
>
> while (my ($key, $l) = each %$h) {
> my %s = map {($_, 1)} @$l;
> - @$l = keys %s;
> + @$l = sort keys %s;
> if (@$l > 1) {
> - my $notfound = 0;
> - # create a list of unconflicting packages.
> - my @l2 = ();
> - for my $pkg (@$l) {
> - next if $conflicts->{$pkg}->conflicts_with(
> - grep { $_ ne $pkg } (@$l));
> - push(@l2, $pkg);
> + my $hv = join(',', @$l);
> + if (!defined $cache->{$hv}) {
> + # create a list of unconflicting packages.
> + my $l2 = [];
> + for my $pkg (@$l) {
> + my $keepit = 0;
> + for my $pkg2 (@$l) {
> + next if $pkg2 eq $pkg;
> + if (!(find_a_conflict($conflicts, $deps, $pkg,
> $pkg2) ||
> + find_a_conflict($conflicts, $deps,
> $pkg2, $pkg))) {
> + $keepit = 1;
> + last;
> + }
> + }
> + if ($keepit) {
> + push(@$l2, $pkg);
> + }
> + }
> + $cache->{$hv} = $l2;
> }
> - if (@l2 != 0) {
> - print "$key: ", join(',', @l2), "\n";
> + my $result = $cache->{$hv};
> + if (@$result != 0) {
> + my $newkey = join(',', @$result);
> + if (@$result == 1) {
> + $newkey.="-> was ".join(',', @$l);
> + }
> + $cache2->{$newkey} = [] unless defined($cache2->{$newkey});
> + push(@{$cache2->{$newkey}}, $key);
> }
> }
> }
> @@ -77,6 +140,7 @@ sub show_problems
> my $filehash={};
> my %dirhash=();
> my $conflicts={};
> +my $dephash={};
>
> print "Scanning packages\n";
> print "-----------------\n";
> @@ -94,9 +158,15 @@ for my $pkgname (@ARGV) {
> $plist->forget();
> $conflicts->{$plist->pkgname()} =
> OpenBSD::PkgCfl->make_conflict_list($plist);
> - analyze($plist, $filehash);
> + $plist->visit('register', $filehash, $dephash, $plist->pkgname());
> }
>
> print "File problems:\n";
> print "-------------\n";
> -show_problems($filehash, $conflicts);
> +compute_problems($filehash, $conflicts, $dephash);
> +for my $cfl (sort keys %$cache2) {
> + print "$cfl\n";
> + for my $f (sort @{$cache2->{$cfl}}) {
> + print "\t$f\n";
> + }
> +}
> Index: package/register-plist
> ===================================================================
> RCS file: package/register-plist
> diff -N package/register-plist
> --- /dev/null 1 Jan 1970 00:00:00 -0000
> +++ package/register-plist 17 Aug 2005 16:39:25 -0000
> @@ -0,0 +1,125 @@
> +#! /usr/bin/perl
> +
> +# $OpenBSD$
> +# Copyright (c) 2005
> +# Marc Espie. All rights reserved.
> +# Redistribution and use in source and binary forms, with or without
> +# modification, are permitted provided that the following conditions
> +# are met:
> +# 1. Redistributions of code must retain the above copyright
> +# notice, this list of conditions and the following disclaimer.
> +# 2. Neither the name of OpenBSD nor the names of its contributors
> +# may be used to endorse or promote products derived from this software
> +# without specific prior written permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY ITS AUTHOR AND THE OpenBSD project ``AS IS''
> AND
> +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
> +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
> +# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
> +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
> +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
> +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
> +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
> +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
> +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
> +# SUCH DAMAGE.
> +
> +use strict;
> +
> +use OpenBSD::PackageLocator;
> +use OpenBSD::PackageInfo;
> +use OpenBSD::PackingList;
> +use File::Path;
> +use File::Compare;
> +
> +package OpenBSD::PackingElement;
> +
> +sub forget_details
> +{
> +}
> +
> +package OpenBSD::PackingElement::FileBase;
> +
> +sub forget_details
> +{
> + my $self = shift;
> + undef $self->{md5};
> + undef $self->{size};
> +}
> +
> +package OpenBSD::PackingElement::SpecialFile;
> +sub forget_details
> +{
> + my $self = shift;
> + undef $self->{md5};
> + undef $self->{size};
> +}
> +
> +package OpenBSD::PackingElement::Dependency;
> +sub forget_details
> +{
> + my $self = shift;
> + $self->{def} = 'def';
> +}
> +
> +package OpenBSD::PackingElement::Wantlib;
> +
> +sub forget_details
> +{
> + my $self = shift;
> + $self->{name} =~ s/\d+\.\d+$/0.0/;
> +}
> +
> +package OpenBSD::PackingElement::CVSTag;
> +sub forget_details
> +{
> + my $self = shift;
> + $self->{name} =~ s/^(\$OpenBSD:\s\S+,v)\s+.*\$$/$1\$/;
> +}
> +
> +package main;
> +
> +if (@ARGV < 2) {
> + die "usage issue";
> +}
> +my $dir = shift;
> +if (!-d $dir) {
> + die "not a directory: $dir";
> +}
> +
> +my $error =0;
> +
> +for my $pkgfile (@ARGV) {
> + my $pkg = OpenBSD::PackageLocator->find($pkgfile);
> + if (!$pkg) {
> + die "Bad package $pkgfile";
> + }
> +
> + my $infodir = $pkg->info();
> + my $plist = OpenBSD::PackingList->fromfile($infodir.CONTENTS);
> + undef $plist->{ OpenBSD::PackageInfo::COMMENT };
> +
> + my $l = $plist->{items};
> + if ($l->[EMAIL PROTECTED]>isa('OpenBSD::PackingElement::Cwd') &&
> + $l->[EMAIL PROTECTED]>{name} eq '.') {
> + pop @$l;
> + }
> +
> + $plist->visit('forget_details');
> + $pkg->close();
> + rmtree($infodir);
> + my $result = $dir.'/'.$plist->pkgname();
> + if (-f $result) {
> + my $t = "$result-new";
> + $plist->tofile($t);
> + if (compare($t, $result) == 0) {
> + unlink($t);
> + } else {
> + print STDERR "Error: $t and $result are different\n";
> + $error++;
> + }
> + } else {
> + $plist->tofile($result);
> + }
> +}
> +exit($error != 0);
>
--
"Security is decided by quality" -- Theo de Raadt
