bootstrap.1 | 8 +
solenv/bin/download_external_dependencies.pl | 98 +++-------------
solenv/bin/modules/ExtensionsLst.pm | 77 ++-----------
solenv/javadownloader/AOOJavaDownloader.java | 156 +++++++++++++++++++++++++++
4 files changed, 200 insertions(+), 139 deletions(-)
New commits:
commit 9813bbc278e11149aa0519750f2496f1b3d5ab89
Author: Damjan Jovanovic <dam...@apache.org>
Date: Sun Jul 24 15:53:36 2016 +0000
Give up on using Perl's LWP::UserAgent and LWP::Protocol::https to download
files during ./bootstrap. It's a nightmare to get it working on the
buildbots - Infra has been trying on INFRA-11296 for 5 months to get it
installed. Installing through CPAN doesn't always work - tests fail on
CentOS 5 and on Cygwin. Even when installed, it's not always found. The
gain just doesn't justify the effort. Worst of all, it's holding back
development.
What else is there? A CLI tool like wget could work, but it's not listed as
a dependency, and it breaks on CentOS 5 for https://, the thing it's needed
for most.
I instead re-implemented it in Java. Java is freely available, highly
portable, and rock solid. We already use it in the build, and it's
described as being a mandatory build requirement even though
./configure.ac treats it as optional. Best of all it supports https://
out of the box in java.net.URLConnection and uses its own root CA
certificates. Tests show my AOOJavaDownloader class works on FreeBSD and
Windows, supports HTTP redirection, and generally works like a charm.
Patch by: me
diff --git a/bootstrap.1 b/bootstrap.1
index 0981e5f..045b547 100644
--- a/bootstrap.1
+++ b/bootstrap.1
@@ -41,6 +41,14 @@ chmod +x "$SRC_ROOT/solenv/bin/gccinstlib.pl"
if [ "$DO_FETCH_TARBALLS" = "yes" ]; then
# check perl include locations
"$PERL" -e 'print "\nInclude locations: @INC\n\n"';
+
+ mkdir -p "$SOLARENV/$INPATH/class"
+ "$JAVACOMPILER" "$SOLARENV/javadownloader/AOOJavaDownloader.java" -d
"$SOLARENV/$INPATH/class"
+ if [ "$?" != "0" ]; then
+ echo "*** Failed to build AOOJavaDownloader, aborting! ***"
+ exit 1
+ fi
+
"$PERL" "$SOLARENV/bin/download_external_dependencies.pl"
$SRC_ROOT/external_deps.lst
if [ "$?" != "0" ]; then
echo "*** Error downloading external dependencies, please fix the
previous problems and try again ***"
diff --git a/solenv/bin/download_external_dependencies.pl
b/solenv/bin/download_external_dependencies.pl
index 9db822b..ec68036 100755
--- a/solenv/bin/download_external_dependencies.pl
+++ b/solenv/bin/download_external_dependencies.pl
@@ -507,91 +507,39 @@ sub DownloadFile ($$$)
my $URL = shift;
my $checksum = shift;
- my $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $name);
-
- my $temporary_filename = $filename . ".part";
-
- print "downloading to $temporary_filename\n";
- my $out;
- open $out, ">$temporary_filename";
- binmode($out);
-
- # Prepare checksum
- my $digest;
- if (defined $checksum && $checksum->{'type'} eq "SHA1")
- {
- # Use SHA1 only when explicitly requested (by the presence of a
"SHA1=..." line.)
- $digest = Digest::SHA->new("1");
- }
- elsif ( ! defined $checksum || $checksum->{'type'} eq "MD5")
- {
- # Use MD5 when explicitly requested or when no checksum type is given.
- $digest = Digest::MD5->new();
+ if (defined $checksum)
+ {
+ system(
+ $ENV{'JAVAINTERPRETER'},
+ "-cp",
+ File::Spec->catfile(
+ File::Spec->catfile($ENV{'SOLARENV'}, $ENV{'INPATH'}),
+ "class"),
+ "AOOJavaDownloader",
+ $name,
+ $URL,
+ $checksum->{'type'},
+ $checksum->{'value'});
}
else
{
- die "checksum type ".$checksum->{'type'}." is not supported";
+ system(
+ $ENV{'JAVAINTERPRETER'},
+ "-cp",
+ File::Spec->catfile(
+ File::Spec->catfile($ENV{'SOLARENV'}, $ENV{'INPATH'}),
+ "class"),
+ "AOOJavaDownloader",
+ $name,
+ $URL);
}
- # Download the extension.
- my $success = 0;
-
- my $agent = LWP::UserAgent->new();
- $agent->env_proxy;
- my $response = $agent->get($URL);
-
- $success = $response->is_success;
- if ($success)
- {
- my $content = $response->content;
- open $out, ">$temporary_filename";
- binmode($out);
- print $out $content;
- $digest->add($content);
- }
- else
+ if ($? == 0)
{
- print "download from $URL failed (" . $response->status_line . ")\n";
- }
- close($out);
-
- # When download was successful then check the checksum and rename the
.part file
- # into the actual extension name.
- if ($success)
- {
- my $file_checksum = $digest->hexdigest();
- if (defined $checksum)
- {
- if ($checksum->{'value'} eq $file_checksum)
- {
- printf("%s checksum is OK\n", $checksum->{'type'});
- }
- else
- {
- unlink($temporary_filename);
- printf(" %s checksum does not match (%s instead of %s)\n",
- $checksum->{'type'},
- $file_checksum,
- $checksum->{'value'});
- return 0;
- }
- }
- else
- {
- # The datafile does not contain a checksum to match against.
- # Display the one that was calculated for the downloaded file so
that
- # it can be integrated manually into the data file.
- printf("checksum not given, md5 of file is %s\n", $file_checksum);
- $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'},
$file_checksum . "-" . $name);
- }
-
- rename($temporary_filename, $filename) || die "can not rename
$temporary_filename to $filename";
return 1;
}
else
{
- unlink($temporary_filename);
- print " download failed\n";
return 0;
}
}
diff --git a/solenv/bin/modules/ExtensionsLst.pm
b/solenv/bin/modules/ExtensionsLst.pm
index 39bb6d0..eaed270 100644
--- a/solenv/bin/modules/ExtensionsLst.pm
+++ b/solenv/bin/modules/ExtensionsLst.pm
@@ -468,70 +468,19 @@ sub Download (@)
{
my ($protocol, $name, $URL, $md5sum) = @{$entry};
- # Open a .part file for writing.
- my $filename = File::Spec->catfile($download_path, $name);
- my $temporary_filename = $filename . ".part";
- print "downloading to $temporary_filename\n";
- open my $out, ">$temporary_filename";
- binmode($out);
-
- # Prepare md5
- my $md5 = Digest::MD5->new();
-
- # Download the extension.
- my $agent = LWP::UserAgent->new();
- $agent->timeout(120);
- $agent->show_progress(1);
- my $last_was_redirect = 0;
- $agent->add_handler('response_redirect'
- => sub{
- $last_was_redirect = 1;
- return;
- });
- $agent->add_handler('response_data'
- => sub{
- if ($last_was_redirect)
- {
- $last_was_redirect = 0;
- # Throw away the data we got so far.
- $md5->reset();
- close $out;
- open $out, ">$temporary_filename";
- binmode($out);
- }
- my($response,$agent,$h,$data)=@_;
- print $out $data;
- $md5->add($data);
- });
- my $response = $agent->get($URL);
- close $out;
-
- # When download was successful then check the md5 checksum and rename
the .part file
- # into the actual extension name.
- if ($response->is_success())
- {
- if (defined $md5sum && length($md5sum)==32)
- {
- my $file_md5 = $md5->hexdigest();
- if ($md5sum eq $file_md5)
- {
- print "md5 is OK\n";
- }
- else
- {
- unlink($temporary_filename) if ! $Debug;
- die "downloaded file has the wrong md5 checksum: $file_md5
instead of $md5sum";
- }
- }
- else
- {
- print "md5 is not present\n";
- printf " is %s, length is %d\n", $md5sum, length(md5sum);
- }
-
- rename($temporary_filename, $filename) || die "can not rename
$temporary_filename to $filename";
- }
- else
+ system(
+ $ENV{'JAVAINTERPRETER'},
+ "-cp",
+ File::Spec->catfile(
+ File::Spec->catfile($ENV{'SOLARENV'}, $ENV{'INPATH'}),
+ "class"),
+ "AOOJavaDownloader",
+ $name,
+ $URL,
+ 'MD5',
+ $md5sum);
+
+ if ($? != 0)
{
die "failed to download $URL";
}
diff --git a/solenv/javadownloader/AOOJavaDownloader.java
b/solenv/javadownloader/AOOJavaDownloader.java
new file mode 100644
index 0000000..612402a
--- /dev/null
+++ b/solenv/javadownloader/AOOJavaDownloader.java
@@ -0,0 +1,156 @@
+/**************************************************************
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *************************************************************/
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class AOOJavaDownloader {
+ private static final int MAX_REDIRECTS = 10;
+ private static final String hexChars = "0123456789abcdef";
+
+ public static void main(String[] args) throws MalformedURLException,
NoSuchAlgorithmException {
+ if (args.length != 2 && args.length != 4) {
+ System.err.println("AOOJavaDownloader - required args: filename
URL [checksumType] [checksum]");
+ System.exit(1);
+ }
+ String name = args[0];
+ URL url = new URL(args[1]);
+ String checksumType = args.length == 4 ? args[2] : null;
+ String checksum = args.length == 4 ? args[3] : null;
+
+ File filename = new File(System.getenv("TARFILE_LOCATION"), name);
+
+ MessageDigest digest = null;
+ if (checksumType != null && checksumType.equals("SHA1")) {
+ digest = MessageDigest.getInstance(checksumType);
+ } else {
+ digest = MessageDigest.getInstance("MD5");
+ }
+
+ File temporaryFile = new File(filename.getAbsolutePath() + ".part");
+ System.out.println("downloading " + url + " to " +
temporaryFile.getAbsolutePath());
+
+ boolean succeeded = false;
+ InputStream inputStream = null;
+ try {
+ inputStream = openURL(url);
+
+ OutputStream outputStream = new BufferedOutputStream(new
FileOutputStream(temporaryFile));
+ try {
+ copyStreamAndHash(inputStream, outputStream, digest);
+ } finally {
+ try {
+ outputStream.close();
+ } catch (IOException ignored) {
+ }
+ }
+
+ String fileChecksum = bytesToHex(digest.digest());
+ if (checksumType != null) {
+ if (fileChecksum.equals(checksum)) {
+ System.out.println(checksumType + " checksum is OK");
+ } else {
+ temporaryFile.delete();
+ throw new IOException(String.format("%s checksum does not
match (%s instead of %s)",
+ checksumType, fileChecksum, checksum));
+ }
+ } else {
+ System.out.println("checksum not given, md5 of file is " +
fileChecksum);
+ filename = new File(System.getenv("TARFILE_LOCATION"),
fileChecksum + "-" + name);
+ }
+ if (!temporaryFile.renameTo(filename)) {
+ throw new IOException("Renaming " + temporaryFile + " to " +
filename + " failed");
+ }
+ succeeded = true;
+ } catch (Exception exception) {
+ System.out.println("download failed:");
+ exception.printStackTrace(System.out);
+ } finally {
+ if (inputStream != null) {
+ try {
+ inputStream.close();
+ } catch (IOException ignored) {
+ }
+ }
+ if (!succeeded) {
+ temporaryFile.delete();
+ }
+ }
+ System.exit(succeeded ? 0 : 1);
+ }
+
+ private static InputStream openURL(URL url) throws IOException {
+ URLConnection connection = url.openConnection();
+ for (int redir = 0; redir < MAX_REDIRECTS; redir++) {
+ connection.setAllowUserInteraction(false);
+ connection.setConnectTimeout(60000);
+ connection.setReadTimeout(60000);
+ if (connection instanceof HttpURLConnection) {
+ HttpURLConnection httpURLConnection = (HttpURLConnection)
connection;
+ // Only works when protocol (http:// or https://) remains the
same:
+ httpURLConnection.setInstanceFollowRedirects(true);
+ }
+ connection.connect();
+ if (connection instanceof HttpURLConnection) {
+ HttpURLConnection httpURLConnection = (HttpURLConnection)
connection;
+ // handle protocol change:
+ int status = httpURLConnection.getResponseCode();
+ if (status == HttpURLConnection.HTTP_MOVED_TEMP ||
+ status == HttpURLConnection.HTTP_MOVED_PERM ||
+ status == HttpURLConnection.HTTP_SEE_OTHER) {
+ connection = new
URL(connection.getHeaderField("Location")).openConnection();
+ continue;
+ }
+ }
+ return new BufferedInputStream(connection.getInputStream());
+ }
+ throw new IOException("Too many http redirects");
+ }
+
+ private static void copyStreamAndHash(InputStream inputStream,
OutputStream outputStream, MessageDigest digest) throws IOException {
+ byte[] buffer = new byte[4 * 4096];
+ int bytesRead;
+ while ((bytesRead = inputStream.read(buffer)) >= 0) {
+ outputStream.write(buffer, 0, bytesRead);
+ digest.update(buffer, 0, bytesRead);
+ }
+ }
+
+ private static String bytesToHex(byte[] bytes) {
+ char[] output = new char[bytes.length * 2];
+ for (int i = 0; i < bytes.length; i++) {
+ output[2*i] = hexChars.charAt(0xf & (bytes[i] >> 4));
+ output[2*i + 1] = hexChars.charAt(0xf & bytes[i]);
+ }
+ return new String(output);
+ }
+}
_______________________________________________
Libreoffice-commits mailing list
libreoffice-comm...@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
