Package: www.debian.org Followup-For: Bug #818761 I've prepared a patch for this. Please find it attached.
I don't just want to push it though, since it would be good to clean up the archive/ directory on packages.debian.org. So it would be preferable that someone with access to the deployment merges it and then takes care of the deployment immediately. Regards, Frank -- System Information: Debian Release: 8.3 APT prefers stable APT policy: (500, 'stable') Architecture: amd64 (x86_64) Kernel: Linux 3.16.0-4-amd64 (SMP w/2 CPU cores) Locale: LANG=de_DE.UTF-8, LC_CTYPE=de_DE.UTF-8 (charmap=UTF-8) Shell: /bin/sh linked to /bin/dash Init: systemd (via /run/systemd/system)
>From 63322b969c931107c3d26d3e9c8b1b24a41b7da5 Mon Sep 17 00:00:00 2001 From: Frank Lichtenheld <fr...@lichtenheld.de> Date: Mon, 28 Mar 2016 20:21:12 +0200 Subject: [PATCH] Teach packages.debian.org to handle .xz archive files Currently only for main archive, not backports or debports, yet. Crude mix of autodetection and configuration but since at least experimental is completely broken right now my priority was to get it working. Closes: #818761 --- bin/parse-packages | 38 ++++++++++++++++++++++++++++++++++---- bin/parse-sources | 35 ++++++++++++++++++++++++++++++++--- config.sh.sed.in | 4 ++++ cron.d/100syncarchive | 33 +++++++++++++++++++++------------ 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/bin/parse-packages b/bin/parse-packages index a7403a9..280adaf 100755 --- a/bin/parse-packages +++ b/bin/parse-packages @@ -63,6 +63,37 @@ mkpath( "$DBDIR/xapian.new" ); my %descriptions_english_db; tie %descriptions_english_db, "DB_File", "files/db/descriptions_translated_english_only.db", O_RDONLY, 0666, $DB_BTREE; +my %ext_to_prog = ( + xz => 'xzcat', + gz => 'zcat', + ); +sub open_packages_files { + my( $suite_dir, $component) = @_; + + my (@files, $prog); + for my $ext (qw(xz gz)){ + my $packages_match = "binary-*/Packages.$ext"; + @files = (); + push @files, <"$suite_dir/$component/$packages_match">; + push @files, <"$suite_dir/updates/$component/$packages_match">; + push @files, <"$suite_dir/$component/debian-installer/$packages_match">; + + if( @files ){ + $prog = $ext_to_prog{$ext}; + last; + } + } + + if( @files && $prog ){ + print "\tprog=$prog\n"; + print "\tfiles=@files\n"; + open my $fh, '-|', $prog, @files; + return $fh; + } + print "\tno files found, skipping...\n"; + return; +} + for my $suite (@SUITES) { my %package_names_suite = (); my %packages_all_db; @@ -76,10 +107,9 @@ for my $suite (@SUITES) { print "\tseems not to exist, skipping...\n"; next; } - open PKG, "zcat $TOPDIR/archive/$archive/$suite/$what/binary-*/Packages.gz" - . " $TOPDIR/archive/$archive/$suite/updates/$what/binary-*/Packages.gz" - . " $TOPDIR/archive/$archive/$suite/$what/debian-installer/binary-*/Packages.gz|"; - while (<PKG>) { + + my $fh = open_packages_files("$TOPDIR/archive/$archive/$suite/", $what) || next; + while (<$fh>) { next if /^\s*$/; my $data = ""; my %data = (); diff --git a/bin/parse-sources b/bin/parse-sources index 1f73bcc..a4e6d06 100755 --- a/bin/parse-sources +++ b/bin/parse-sources @@ -38,6 +38,36 @@ $/ = ""; -d $DBDIR || mkpath( $DBDIR ); +my %ext_to_prog = ( + xz => 'xzcat', + gz => 'zcat', + ); +sub open_sources_files { + my( $suite_dir, $component) = @_; + + my (@files, $prog); + for my $ext (qw(xz gz)){ + my $sources_match = "source/Sources.$ext"; + @files = (); + push @files, <"$suite_dir/$component/$sources_match">; + push @files, <"$suite_dir/updates/$component/$sources_match">; + + if( @files ){ + $prog = $ext_to_prog{$ext}; + last; + } + } + + if( @files && $prog ){ + print "\tprog=$prog\n"; + print "\tfiles=@files\n"; + open my $fh, '-|', $prog, @files; + return $fh; + } + print "\tno files found, skipping...\n"; + return; +} + for my $archive (@ARCHIVES) { for my $suite (@SUITES) { @@ -51,9 +81,8 @@ for my $archive (@ARCHIVES) { print "\tseems not to exist, skipping...\n"; next; } - open PKG, "zcat $TOPDIR/archive/$archive/$suite/$what/source/Sources.gz" - . " $TOPDIR/archive/$archive/$suite/updates/$what/source/Sources.gz|"; - while (<PKG>) { + my $fh = open_sources_files("$TOPDIR/archive/$archive/$suite/", $what) || next; + while (<$fh>) { next if /^\s*$/; my $data = ""; my %data = (); diff --git a/config.sh.sed.in b/config.sh.sed.in index 10970d2..03ffc31 100644 --- a/config.sh.sed.in +++ b/config.sh.sed.in @@ -62,6 +62,10 @@ arch_sid="${arch_stretch} alpha avr32 hppa hurd-i386 m68k powerpcspe ppc64el sh4 arch_experimental="${arch_sid}" arch_wheezy_proposed_updates="${arch_wheezy}" +ext_wheezy=gz +#for security Sources +ext_jessie=gz + # from http://ftp.de.debian.org/debian-ports/dists/{sid,experimental}/Release arch_debports="alpha hppa m68k powerpcspe ppc64 sh4 sparc64 x32" diff --git a/cron.d/100syncarchive b/cron.d/100syncarchive index d6459e0..b7f7a47 100755 --- a/cron.d/100syncarchive +++ b/cron.d/100syncarchive @@ -17,8 +17,11 @@ then (cd us/${dist} && $wget_cmd -N ${ftpsite}/dists/${dist}/Release && $wget_cmd -N ${ftpsite}/dists/${dist}/Release.gpg ) - - foo=\$arch_${dist//-/_} + + ext=$(eval echo \$ext_${dist%-*}) + if [ -z "$ext" ]; then ext=xz; fi + echo "using $ext as file extension" + foo=\$arch_${dist%-*} for arch in `eval echo $foo` do for part in ${parts} @@ -26,7 +29,7 @@ then echo retrieve Packages $dist/$part/$arch test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch} (cd us/${dist}/${part}/binary-${arch} && - $wget_cmd -N ${ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz) + $wget_cmd -N ${ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.$ext) #FIXME: no Contents files for p-u echo retrieve Contents $dist/${part}/$arch (cd us/${dist}/${part} && @@ -38,7 +41,7 @@ then echo retrieve Sources $dist/$part test -d us/${dist}/${part}/source || mkdir -p us/${dist}/${part}/source (cd us/${dist}/${part}/source && - $wget_cmd -N ${ftpsite}/dists/${dist}/${part}/source/Sources.gz) + $wget_cmd -N ${ftpsite}/dists/${dist}/${part}/source/Sources.$ext) done $check_release_cmd $archivedir/us/${dist}/Release @@ -52,15 +55,18 @@ else rsync -Lt ${localdir}/dists/${dist}/Release us/${dist}/Release rsync -Lt ${localdir}/dists/${dist}/Release.gpg us/${dist}/Release.gpg - foo=\$arch_${dist//-/_} + ext=$(eval echo \$ext_${dist%-*}) + if [ -z "$ext" ]; then ext=xz; fi + echo "using $ext as file extension" + foo=\$arch_${dist%-*} for arch in `eval echo $foo` do for part in ${parts} do echo retrieve Packages $dist/$part/$arch test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch} - rsync -Lt ${localdir}/dists/${dist}/${part}/binary-${arch}/Packages.gz \ - us/${dist}/${part}/binary-${arch}/Packages.gz + rsync -Lt ${localdir}/dists/${dist}/${part}/binary-${arch}/Packages.$ext \ + us/${dist}/${part}/binary-${arch}/Packages.$ext #FIXME: no Contents files for p-u #FIXME: the symlinks for wheezy/sid/experimental will go away @@ -74,8 +80,8 @@ else do echo retrieve Sources $dist/$part test -d us/${dist}/${part}/source || mkdir -p us/${dist}/${part}/source - rsync -Lt ${localdir}/dists/${dist}/${part}/source/Sources.gz \ - us/${dist}/${part}/source/Sources.gz + rsync -Lt ${localdir}/dists/${dist}/${part}/source/Sources.$ext \ + us/${dist}/${part}/source/Sources.$ext done $check_release_cmd $archivedir/us/${dist}/Release @@ -92,7 +98,10 @@ do $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/Release && $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/Release.gpg ) - foo=\$arch_${dist//-/_} + ext=$(eval echo \$ext_${dist%-*}) + if [ -z "$ext" ]; then ext=xz; fi + echo "using $ext as file extension" + foo=\$arch_${dist%-*} for arch in `eval echo $foo` do for part in ${parts} @@ -100,7 +109,7 @@ do echo retrieve Packages $dist/$part/security/$arch test -d security/${dist}/${part}/binary-${arch} || mkdir -p security/${dist}/${part}/binary-${arch} (cd security/${dist}/${part}/binary-${arch} && - $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/${part}/binary-${arch}/Packages.gz) + $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/${part}/binary-${arch}/Packages.$ext) done done for part in ${parts} @@ -108,7 +117,7 @@ do echo retrieve Sources $dist/$part/security test -d security/${dist}/${part}/source || mkdir -p security/${dist}/${part}/source (cd security/${dist}/${part}/source && - $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/${part}/source/Sources.gz) + $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/${part}/source/Sources.$ext) done $check_release_cmd $archivedir/security/${dist}/Release -- 2.1.4