On Sunday 20 September 2009 12:16:48 Roger Leigh wrote: > commit fdd9dd75b460aad58ab3916110e562830c0cda32 > Author: Andres Mejia <[email protected]> > Date: Sun Sep 13 23:28:38 2009 -0400 > > Implement support to detect and build from remote source files. This > also changed dsc_files to support reading dsc files downloaded from the > web. > > commit 02f033d4d85254c7c76a4793ea0e24f03b954159 > Author: Andres Mejia <[email protected]> > Date: Sun Sep 13 23:27:00 2009 -0400 > > Add utility subroutines that are used to check and download from a URL > and parse dsc files. > > -----------------------------------------------------------------------
Here are the two other patches that I submitted earlier, updated to go right into the master branch. The first one cuts down the size of the download() subroutine. It also makes using LWP::UserAgent optional. The second one will allow stuff printed to STDOUT and STDERR from the subroutines in Sbuild::Utility to appear in the build logs. -- Regards, Andres
From cbf45b3a7bcb40f4923f63609b30e9d570cbd487 Mon Sep 17 00:00:00 2001 From: Andres Mejia <[email protected]> Date: Sun, 20 Sep 2009 13:02:42 -0400 Subject: [PATCH 1/2] Drastically cut down the size of the download() subroutine. This makes the use of LWP::UserAgent optional. This also takes advantage of the LWP::UserAgent module's own implementation of displaying progress. --- lib/Sbuild/Utility.pm | 92 +++++++++---------------------------------------- 1 files changed, 17 insertions(+), 75 deletions(-) diff --git a/lib/Sbuild/Utility.pm b/lib/Sbuild/Utility.pm index 2ae95a7..308882f 100644 --- a/lib/Sbuild/Utility.pm +++ b/lib/Sbuild/Utility.pm @@ -40,8 +40,7 @@ use warnings; use Sbuild::Conf; use Sbuild::Chroot; use File::Temp qw(tempfile); -use LWP::UserAgent; # Needed to grab content from the WWW. -use Time::HiRes qw ( time ); # Needed for high resolution timers +use Module::Load::Conditional qw(can_load); # Used to check for LWP::UserAgent sub get_dist ($); sub setup ($$); @@ -142,6 +141,11 @@ sub check_url { # If $url is a readable plain file on the local system, just return true. return 1 if (-f $url && -r $url); + # Load LWP::UserAgent if possible, else return 0. + if (! can_load( modules => { 'LWP::UserAgent' => undef, } )) { + return 0; + } + # Setup the user agent. my $ua = LWP::UserAgent->new; @@ -172,14 +176,16 @@ sub download { # $url. return $url if (-f $url && -r $url); - # Filehandle we'll be writing to. - my $fh; + # Load LWP::UserAgent if possible, else return 0. + if (! can_load( modules => { 'LWP::UserAgent' => undef, } )) { + return 0; + } # If $file isn't defined, a temporary file will be used instead. - ($fh, $file) = tempfile( UNLINK => 0 ) if (! $file); + (undef, $file) = tempfile( UNLINK => 0 ) if (! $file); # Setup the user agent. - my $ua = LWP::UserAgent->new; + my $ua = LWP::UserAgent->new( show_progress => 1 ); # Determine if we need to specify any proxy settings. $ua->env_proxy; @@ -189,79 +195,15 @@ sub download { } # Download the file. - my $expected_length; # Total size we expect of content - my $bytes_received = 0; # Size of content as it is received - my $percent; # The percentage downloaded - my $tick; # Used for counting. - my $start_time = time; # Record of the start time - open($fh, '>', $file); # Destination file to download content to - my $response = $ua->get($url, ":content_cb" => - sub { - my ($chunk, $response) = @_; - - $bytes_received += length($chunk); - unless (defined $expected_length) { - $expected_length = $response->content_length or undef; - } - if ($expected_length) { - # Here we calculate the speed of the download to print out later - my $speed; - my $duration = time - $start_time; - if ($bytes_received/$duration >= 1024 * 1024) { - $speed = sprintf("%.3g MB", - ($bytes_received/$duration) / (1024.0 * 1024)) . "/s"; - } elsif ($bytes_received/$duration >= 1024) { - $speed = sprintf("%.3g KB", - ($bytes_received/$duration) / 1024.0) . "/s"; - } else { - $speed = $bytes_received/$duration . " bytes/s"; - } - # Calculate the percentage downloaded - $percent = sprintf("%d", - 100 * $bytes_received / $expected_length); - $tick++; # Keep count - # Here we print out a progress of the download. We start by - # printing out the amount of data retrieved so far, and then - # show a progress bar. After 50 ticks, the percentage is printed - # and the speed of the download is printed. A new line is - # started and the process repeats until the download is - # complete. - if (($tick == 250) or ($percent == 100)) { - print STDERR ".]"; - printf STDERR "%5s", "$percent%"; - printf STDERR "%12s", "$speed\n"; - $tick = 0; - } elsif ($tick == 1) { - printf STDERR "%8s", sprintf("%d", - $bytes_received / 1024) . "KB"; - print STDERR " [."; - } elsif ($tick % 5 == 0) { - print STDERR "."; - } - } - # Write the contents of the download to our specified file - if ($response->is_success) { - print $fh $chunk; # Print content to file - } else { - # Print message upon failure during download - print STDERR "\n" . $response->status_line . "\n"; - return 0; - } - } - ); # Our GET request - close $fh; # Close the destination file + my $request = HTTP::Request->new(GET => $url); + my $response = $ua->request($request, $file); - # Print error message in case we couldn't get a response at all. + # Check if we succeeded, return 0 if we didn't succeed, else continue to + # return the path of the file. if (!$response->is_success) { - print $response->status_line . "\n"; + print $response->status_line, "\n"; return 0; } - - # At this point, the download should have been successful. Print a success - # message and return the location of the file. - print STDERR "Download of $url successful.\n"; - print STDERR "Total size of content downloaded: " . - sprintf("%d", $bytes_received/1024) . "KB\n"; return $file; } -- 1.6.4.3
From 5d13961b5833b4418d92413423cff5ea5ee7e2ff Mon Sep 17 00:00:00 2001 From: Andres Mejia <[email protected]> Date: Sun, 20 Sep 2009 13:06:09 -0400 Subject: [PATCH 2/2] Allow stuff printed to STDOUT and STDERR from the utility subroutines to show up in the logs. --- lib/Sbuild/Build.pm | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/lib/Sbuild/Build.pm b/lib/Sbuild/Build.pm index 4469156..63c210f 100644 --- a/lib/Sbuild/Build.pm +++ b/lib/Sbuild/Build.pm @@ -2598,6 +2598,10 @@ sub open_build_log { $self->set('Log File', $filename); $self->set('Log Stream', $PLOG); + # Duplicate STDOUT and STDERR to the log stream. + open(STDOUT, '>&', $PLOG) or warn "Couldn't duplicate STDOUT.\n"; + open(STDERR, '>&', $PLOG) or warn "Couldn't duplicate STDERR.\n"; + my $hostname = $self->get_conf('HOSTNAME'); $self->log("sbuild (Debian sbuild) $version ($release_date) on $hostname\n"); @@ -2664,6 +2668,8 @@ sub close_build_log { $self->set('Log File', undef); if (defined($self->get('Log Stream'))) { + close STDOUT or warn "Couldn't close STDOUT.\n"; + close STDERR or warn "Couldn't close STDERR.\n"; $self->get('Log Stream')->close(); # Close child logger process $self->set('Log Stream', undef); } -- 1.6.4.3
