Hi! On Sun, 2009-11-22 at 19:53:29 -0200, Otavio Salvador wrote: > debootstrap don't _need_ to support all features of all valid .debs > but the features accepted in base packages. This doesn't mean we > shouldn't add features when possible but it is not a requirement for > debootstrap POV.
One of my initial arguments was that debootstrap can be easily used with stuff other than Debian, were those policies/restrictions might not apply. > On Sun, Nov 22, 2009 at 4:51 AM, Guillem Jover <guil...@debian.org> wrote: > > Right. Additionally an option could be added to explicitly choose the > > extractor, to ease testing. > > This indeed is a possible solution to be easier to test it. As Frans > pointed out d-i is one "tester" of "basic *nix tools backend" already. I went ahead and implemented it as a separate patch, take it if you want, or leave it out if you wish. > > The extractor choosed could be output to avoid that kind of situation. > > I can amend the patch adding that if desired. > > Ok, you convinced me. Could you amend the patch and send it for review? I'm attaching the three patches, the first one has not changed, but adding it for convenience. thanks, guillem
>From 0f78dd071235cbcbc9d2d27a74a76c673e06b4a8 Mon Sep 17 00:00:00 2001 From: Guillem Jover <guil...@debian.org> Date: Fri, 20 Nov 2009 19:51:44 +0100 Subject: [PATCH 1/3] Refactor deb extractors into two new functions --- functions | 43 ++++++++++++++++++++++++++------------ scripts/debian/potato | 6 +---- scripts/debian/sarge | 6 +---- scripts/debian/sarge.buildd | 6 +---- scripts/debian/sarge.fakechroot | 6 +---- scripts/debian/sid | 6 +---- scripts/debian/woody | 6 +---- scripts/debian/woody.buildd | 6 +---- scripts/ubuntu/breezy | 6 +---- scripts/ubuntu/dapper | 6 +---- scripts/ubuntu/edgy | 6 +---- scripts/ubuntu/feisty | 6 +---- scripts/ubuntu/gutsy | 6 +---- scripts/ubuntu/hoary | 6 +---- scripts/ubuntu/hoary.buildd | 6 +---- scripts/ubuntu/warty | 6 +---- scripts/ubuntu/warty.buildd | 6 +---- 17 files changed, 45 insertions(+), 94 deletions(-) diff --git a/functions b/functions index e832d70..66021e8 100644 --- a/functions +++ b/functions @@ -717,27 +717,42 @@ get_debs () { ################################################################ extraction +extract_deb_field () { + local pkg="$1" + local field="$2" + + ar -p "$pkg" control.tar.gz | zcat | + tar -O -xf - control ./control 2>/dev/null | + grep -i "^$field:" | sed -e 's/[^:]*: *//' | head -n 1 +} + +extract_deb_data () { + local pkg="$1" + local tarball=$(ar -t "$pkg" | grep "^data.tar.[bgx]z") + + case "$tarball" in + data.tar.gz) cat_cmd=zcat ;; + data.tar.bz2) cat_cmd=bzcat ;; + data.tar.xz) cat_cmd=xzcat ;; + *) error 1 UNKNOWNDATACOMP "Unknown compression type for %s in %s" "$tarball" "$pkg" ;; + esac + + if type $cat_cmd >/dev/null 2>&1; then + ar -p "$pkg" data.tar.gz | $cat_cmd | tar -xf - + else + error 1 UNPACKCMDUNVL "The $cat_cmd is not available on the system" + fi +} + extract () { ( cd "$TARGET" - local p=0 tarball cat_cmd + local p=0 cat_cmd for pkg in $(debfor "$@"); do p="$(($p + 1))" progress "$p" "$#" EXTRACTPKGS "Extracting packages" packagename="$(echo "$pkg" | sed 's,^.*/,,;s,_.*$,,')" info EXTRACTING "Extracting %s..." "$packagename" - tarball=$(ar -t "./$pkg" | grep "^data.tar.[bgx]z") - case "$tarball" in - data.tar.gz) cat_cmd=zcat ;; - data.tar.bz2) cat_cmd=bzcat ;; - data.tar.xz) cat_cmd=xzcat ;; - *) error 1 UNKNOWNDATACOMP "Unknown compression type for %s in %s" "$tarball" "$pkg" ;; - esac - - if type $cat_cmd >/dev/null 2>&1; then - ar -p "./$pkg" data.tar.gz | $cat_cmd | tar -xf - - else - error 1 UNPACKCMDUNVL "The $cat_cmd is not available on the system" - fi + extract_deb_data "./$pkg" done ); } diff --git a/scripts/debian/potato b/scripts/debian/potato index 3204c7d..304cbe0 100644 --- a/scripts/debian/potato +++ b/scripts/debian/potato @@ -43,11 +43,7 @@ first_stage_install () { x_feign_install () { local pkg=$1 local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/debian/sarge b/scripts/debian/sarge index e49a490..252e180 100644 --- a/scripts/debian/sarge +++ b/scripts/debian/sarge @@ -111,11 +111,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/debian/sarge.buildd b/scripts/debian/sarge.buildd index 0b9ad9a..249a035 100644 --- a/scripts/debian/sarge.buildd +++ b/scripts/debian/sarge.buildd @@ -72,11 +72,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/debian/sarge.fakechroot b/scripts/debian/sarge.fakechroot index 36234fa..2fe4a38 100644 --- a/scripts/debian/sarge.fakechroot +++ b/scripts/debian/sarge.fakechroot @@ -77,11 +77,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/debian/sid b/scripts/debian/sid index 59e70fd..50a9f50 100644 --- a/scripts/debian/sid +++ b/scripts/debian/sid @@ -59,11 +59,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/debian/woody b/scripts/debian/woody index 66a1261..d174be8 100644 --- a/scripts/debian/woody +++ b/scripts/debian/woody @@ -97,11 +97,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/debian/woody.buildd b/scripts/debian/woody.buildd index 8485135..06cb8e3 100644 --- a/scripts/debian/woody.buildd +++ b/scripts/debian/woody.buildd @@ -72,11 +72,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/breezy b/scripts/ubuntu/breezy index f445598..2646ca2 100644 --- a/scripts/ubuntu/breezy +++ b/scripts/ubuntu/breezy @@ -49,11 +49,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/dapper b/scripts/ubuntu/dapper index 7f88d55..e57680a 100644 --- a/scripts/ubuntu/dapper +++ b/scripts/ubuntu/dapper @@ -56,11 +56,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/edgy b/scripts/ubuntu/edgy index 0dbc5dc..483e992 100644 --- a/scripts/ubuntu/edgy +++ b/scripts/ubuntu/edgy @@ -66,11 +66,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/feisty b/scripts/ubuntu/feisty index 4036828..0b98170 100644 --- a/scripts/ubuntu/feisty +++ b/scripts/ubuntu/feisty @@ -66,11 +66,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/gutsy b/scripts/ubuntu/gutsy index 37d4b53..ba2a3f8 100644 --- a/scripts/ubuntu/gutsy +++ b/scripts/ubuntu/gutsy @@ -66,11 +66,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/hoary b/scripts/ubuntu/hoary index 7944999..e5fe9fc 100644 --- a/scripts/ubuntu/hoary +++ b/scripts/ubuntu/hoary @@ -70,11 +70,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/hoary.buildd b/scripts/ubuntu/hoary.buildd index 2596086..afb0904 100644 --- a/scripts/ubuntu/hoary.buildd +++ b/scripts/ubuntu/hoary.buildd @@ -66,11 +66,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/warty b/scripts/ubuntu/warty index 0e16a62..e21a6d8 100644 --- a/scripts/ubuntu/warty +++ b/scripts/ubuntu/warty @@ -58,11 +58,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" diff --git a/scripts/ubuntu/warty.buildd b/scripts/ubuntu/warty.buildd index b980e45..165377e 100644 --- a/scripts/ubuntu/warty.buildd +++ b/scripts/ubuntu/warty.buildd @@ -66,11 +66,7 @@ first_stage_install () { x_feign_install () { local pkg="$1" local deb="$(debfor $pkg)" - local ver="$( - ar -p "$TARGET/$deb" control.tar.gz | zcat | - tar -O -xf - control ./control 2>/dev/null | - grep -i ^Version: | sed -e 's/[^:]*: *//' | head -n 1 - )" + local ver="$(extract_deb_field "$TARGET/$deb" Version)" mkdir -p "$TARGET/var/lib/dpkg/info" -- 1.6.5.3
>From 863fb59476947ef125b24ad89afc040301938b96 Mon Sep 17 00:00:00 2001 From: Guillem Jover <guil...@debian.org> Date: Fri, 20 Nov 2009 21:25:32 +0100 Subject: [PATCH 2/3] Use dpkg-deb if available instead of ar --- debian/control | 2 +- debootstrap | 2 ++ functions | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/debian/control b/debian/control index 577b759..01e7d2d 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,7 @@ Vcs-Svn: svn://svn.debian.org/d-i/trunk/packages/debootstrap Package: debootstrap Architecture: all -Depends: wget, binutils +Depends: wget Recommends: gnupg Description: Bootstrap a basic Debian system debootstrap is used to create a Debian base system from scratch, diff --git a/debootstrap b/debootstrap index ac821b8..54ae85d 100755 --- a/debootstrap +++ b/debootstrap @@ -517,6 +517,8 @@ if am_doing_phase maketarball; then fi if am_doing_phase first_stage; then + choose_extractor + # first stage sets up the chroot -- no calls should be made to # "chroot $TARGET" here; but they should be possible by the time it's # finished diff --git a/functions b/functions index 66021e8..e2a9f73 100644 --- a/functions +++ b/functions @@ -717,7 +717,22 @@ get_debs () { ################################################################ extraction -extract_deb_field () { +# Native dpkg-deb based extractors +extract_dpkg_deb_field () { + local pkg="$1" + local field="$2" + + dpkg-deb -f "$pkg" "$field" +} + +extract_dpkg_deb_data () { + local pkg="$1" + + dpkg-deb --fsys-tarfile "$pkg" | tar -xf - +} + +# Raw .deb extractors +extract_ar_deb_field () { local pkg="$1" local field="$2" @@ -726,7 +741,7 @@ extract_deb_field () { grep -i "^$field:" | sed -e 's/[^:]*: *//' | head -n 1 } -extract_deb_data () { +extract_ar_deb_data () { local pkg="$1" local tarball=$(ar -t "$pkg" | grep "^data.tar.[bgx]z") @@ -744,6 +759,28 @@ extract_deb_data () { fi } +choose_extractor () { + local extractor + + if type dpkg-deb >/dev/null 2>&1; then + extractor="dpkg-deb" + else + extractor="ar" + fi + + info CHOSENEXTRACTOR "Chosen extractor for .deb packages: %s" "$extractor" + case "$extractor" in + dpkg-deb) + extract_deb_field () { extract_dpkg_deb_field "$@"; } + extract_deb_data () { extract_dpkg_deb_data "$@"; } + ;; + ar) + extract_deb_field () { extract_ar_deb_field "$@"; } + extract_deb_data () { extract_ar_deb_data "$@"; } + ;; + esac +} + extract () { ( cd "$TARGET" local p=0 cat_cmd -- 1.6.5.3
>From 6eca98330e91eda672c5fa89610a423deda64798 Mon Sep 17 00:00:00 2001 From: Guillem Jover <guil...@hadrons.org> Date: Fri, 27 Nov 2009 19:00:39 +0100 Subject: [PATCH 3/3] Add an --extractor option to override the automatic extractor selection --- debootstrap | 20 ++++++++++++++++++++ functions | 18 +++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletions(-) diff --git a/debootstrap b/debootstrap index 54ae85d..171a5ef 100755 --- a/debootstrap +++ b/debootstrap @@ -90,6 +90,8 @@ usage() Run second stage in a subdirectory instead of root (can be used to create a foreign chroot) (requires --second-stage) + --extractor=TYPE override automatic .deb extractor selection + (supported: $EXTRACTORS_SUPPORTED) --boot-floppies used for internal purposes by boot-floppies --debian-installer used for internal purposes by debian-installer EOF @@ -201,6 +203,24 @@ if [ $# != 0 ] ; then error 1 NEEDARG "option requires an argument %s" "$1" fi ;; + --extractor|--extractor=?*) + if [ "$1" = "--extractor" -a -n "$2" ] ; then + EXTRACTOR_OVERRIDE="$2" + shift 2 + elif [ "$1" != "${1#--extractor=}" ]; then + EXTRACTOR_OVERRIDE="${1#--extractor=}" + shift + else + error 1 NEEDARG "option requires an argument %s" "$1" + fi + if valid_extractor "$EXTRACTOR_OVERRIDE"; then + if ! type "$EXTRACTOR_OVERRIDE" >/dev/null 2>&1; then + error 1 MISSINGEXTRACTOR "The selected extractor cannot be found: %s" "$EXTRACTOR_OVERRIDE" + fi + else + error 1 BADEXTRACTOR "%s: unknown extractor" "$EXTRACTOR_OVERRIDE" + fi + ;; --unpack-tarball|--unpack-tarball=?*) if [ "$1" = "--unpack-tarball" -a -n "$2" ] ; then UNPACK_TARBALL="$2" diff --git a/functions b/functions index e2a9f73..33b5a46 100644 --- a/functions +++ b/functions @@ -717,6 +717,8 @@ get_debs () { ################################################################ extraction +EXTRACTORS_SUPPORTED="dpkg-deb ar" + # Native dpkg-deb based extractors extract_dpkg_deb_field () { local pkg="$1" @@ -759,10 +761,24 @@ extract_ar_deb_data () { fi } +valid_extractor () { + local extractor="$1" + + for E in $EXTRACTORS_SUPPORTED; do + if [ "$extractor" = "$E" ]; then + return 0 + fi + done + + return 1 +} + choose_extractor () { local extractor - if type dpkg-deb >/dev/null 2>&1; then + if [ -n "$EXTRACTOR_OVERRIDE" ]; then + extractor="$EXTRACTOR_OVERRIDE" + elif type dpkg-deb >/dev/null 2>&1; then extractor="dpkg-deb" else extractor="ar" -- 1.6.5.3