Author: maartenc Date: Sat Jun 5 21:25:26 2010 New Revision: 951772 URL: http://svn.apache.org/viewvc?rev=951772&view=rev Log: IMPROVEMENT: added support for 'deflate' HTTP Content-Encoding (IVY-1194)
Modified: ant/ivy/core/trunk/CHANGES.txt ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java Modified: ant/ivy/core/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/CHANGES.txt?rev=951772&r1=951771&r2=951772&view=diff ============================================================================== --- ant/ivy/core/trunk/CHANGES.txt (original) +++ ant/ivy/core/trunk/CHANGES.txt Sat Jun 5 21:25:26 2010 @@ -115,7 +115,7 @@ for detailed view of each issue, please - NEW: Support ivy.xml parent mechanism (IVY-742) (thanks to Jason Trump and Jean-Louis Boudart) - NEW: Make ivy.xml <conf description> available (IVY-1158) -- IMPROVEMENT: added support for 'gzip' HTTP Content-Encoding (IVY-1194) +- IMPROVEMENT: added support for 'gzip' and 'deflate' HTTP Content-Encoding (IVY-1194) - IMPROVEMENT: retrieve doesn't retrive files if the current one is more recent (IVY-1044) - IMPROVEMENT: better diagnostics when reporting bad ivy file in cache (IVY-1190) - IMPROVEMENT: Support changing="true" for inline retrieve (IVY-1180) Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java?rev=951772&r1=951771&r2=951772&view=diff ============================================================================== --- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java (original) +++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java Sat Jun 5 21:25:26 2010 @@ -17,13 +17,19 @@ */ package org.apache.ivy.util.url; +import java.io.BufferedInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Pattern; +import java.util.zip.DataFormatException; +import java.util.zip.GZIPInputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; public abstract class AbstractURLHandler implements URLHandler { @@ -118,4 +124,44 @@ public abstract class AbstractURLHandler return new URL(normalizeToString(url)); } + + protected InputStream getDecodingInputStream(String encoding, InputStream in) + throws IOException { + InputStream result = null; + + if ("gzip".equals(encoding)) { + result = new GZIPInputStream(in); + } else if ("deflate".equals(encoding)) { + // There seems to be 2 variants of the "deflate"-encoding. + // I couldn't find a way to auto-detect which variant was + // used, so as (a not really good) work-around we try do + // decompress the first 100 bytes using the "zlib"-variant. + BufferedInputStream bStream = new BufferedInputStream(in); + bStream.mark(100); + byte[] bytes = new byte[100]; + int nbBytes = bStream.read(bytes); + bStream.reset(); + + Inflater inflater = new Inflater(); + inflater.setInput(bytes, 0, nbBytes); + try { + inflater.inflate(new byte[1000]); + + // no error decompressing the first 100 bytes, so we + // assume the "zlib"-variant was used. + result = new InflaterInputStream(bStream); + } catch (DataFormatException e) { + // there was an error decompressing the first 100 bytes, + // so we assume the "gzip/raw"-variant was used. + result = new InflaterInputStream(bStream, new Inflater(true)); + } finally { + inflater.end(); + } + } else { + result = in; + } + + return result; + } + } Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java?rev=951772&r1=951771&r2=951772&view=diff ============================================================================== --- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java (original) +++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java Sat Jun 5 21:25:26 2010 @@ -28,7 +28,6 @@ import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.net.UnknownHostException; -import java.util.zip.GZIPInputStream; import org.apache.ivy.Ivy; import org.apache.ivy.util.CopyProgressListener; @@ -117,7 +116,7 @@ public class BasicURLHandler extends Abs url = normalizeToURL(url); conn = url.openConnection(); conn.setRequestProperty("User-Agent", "Apache Ivy/" + Ivy.getIvyVersion()); - conn.setRequestProperty("Accept-Encoding", "gzip"); + conn.setRequestProperty("Accept-Encoding", "gzip,deflate"); if (conn instanceof HttpURLConnection) { HttpURLConnection httpCon = (HttpURLConnection) conn; if (!checkStatusCode(url, httpCon)) { @@ -126,12 +125,8 @@ public class BasicURLHandler extends Abs + " See log for more detail."); } } - InputStream inStream; - if ("gzip".equals(conn.getContentEncoding())) { - inStream = new GZIPInputStream(conn.getInputStream()); - } else { - inStream = conn.getInputStream(); - } + InputStream inStream = getDecodingInputStream(conn.getContentEncoding(), + conn.getInputStream()); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); byte[] buffer = new byte[BUFFER_SIZE]; @@ -154,7 +149,7 @@ public class BasicURLHandler extends Abs src = normalizeToURL(src); srcConn = src.openConnection(); srcConn.setRequestProperty("User-Agent", "Apache Ivy/" + Ivy.getIvyVersion()); - srcConn.setRequestProperty("Accept-Encoding", "gzip"); + srcConn.setRequestProperty("Accept-Encoding", "gzip,deflate"); if (srcConn instanceof HttpURLConnection) { HttpURLConnection httpCon = (HttpURLConnection) srcConn; if (!checkStatusCode(src, httpCon)) { @@ -163,23 +158,24 @@ public class BasicURLHandler extends Abs + " See log for more detail."); } } - int contentLength = srcConn.getContentLength(); - - InputStream inStream; - if ("gzip".equals(srcConn.getContentEncoding())) { - inStream = new GZIPInputStream(srcConn.getInputStream()); - contentLength = -1; - } else { - inStream = srcConn.getInputStream(); - } + // do the download + InputStream inStream = getDecodingInputStream(srcConn.getContentEncoding(), + srcConn.getInputStream()); FileUtil.copy(inStream, dest, l); - if (dest.length() != contentLength && contentLength != -1) { - dest.delete(); - throw new IOException( - "Downloaded file size doesn't match expected Content Length for " + src - + ". Please retry."); + + // check content length only if content was not encoded + if (srcConn.getContentEncoding() == null) { + int contentLength = srcConn.getContentLength(); + if (contentLength != -1 && dest.length() != contentLength) { + dest.delete(); + throw new IOException( + "Downloaded file size doesn't match expected Content Length for " + src + + ". Please retry."); + } } + + // update modification date long lastModified = srcConn.getLastModified(); if (lastModified > 0) { dest.setLastModified(lastModified); Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java?rev=951772&r1=951771&r2=951772&view=diff ============================================================================== --- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java (original) +++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java Sat Jun 5 21:25:26 2010 @@ -28,7 +28,6 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.zip.GZIPInputStream; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Header; @@ -101,7 +100,10 @@ public class HttpClientHandler extends A throw new IOException("The HTTP response code for " + url + " did not indicate a success." + " See log for more detail."); } - return new GETInputStream(get); + + Header encoding = get.getResponseHeader("Content-Encoding"); + return getDecodingInputStream(encoding == null ? null : encoding.getValue(), + get.getResponseBodyAsStream()); } public void download(URL src, File dest, CopyProgressListener l) throws IOException { @@ -113,14 +115,9 @@ public class HttpClientHandler extends A + " did not indicate a success." + " See log for more detail."); } - InputStream is; - Header[] contentEncodings = get.getResponseHeaders("Content-Encoding"); - if ((contentEncodings.length > 0) && "gzip".equals(contentEncodings[0].getValue())) { - is = new GZIPInputStream(get.getResponseBodyAsStream()); - } else { - is = get.getResponseBodyAsStream(); - } - + Header encoding = get.getResponseHeader("Content-Encoding"); + InputStream is = getDecodingInputStream(encoding == null ? null : encoding.getValue(), + get.getResponseBodyAsStream()); FileUtil.copy(is, dest, l); dest.setLastModified(getLastModified(get)); } finally { @@ -255,7 +252,7 @@ public class HttpClientHandler extends A GetMethod get = new GetMethod(normalizeToString(url)); get.setDoAuthentication(useAuthentication(url) || useProxyAuthentication()); - get.setRequestHeader("Accept-Encoding", "gzip"); + get.setRequestHeader("Accept-Encoding", "gzip,deflate"); client.executeMethod(get); return get; } @@ -321,72 +318,6 @@ public class HttpClientHandler extends A return (proxyUserName != null && proxyUserName.trim().length() > 0); } - private static final class GETInputStream extends InputStream { - private InputStream is; - - private GetMethod get; - - private GETInputStream(GetMethod get) throws IOException { - this.get = get; - - Header[] contentEncodings = get.getResponseHeaders("Content-Encoding"); - if ((contentEncodings.length > 0) && "gzip".equals(contentEncodings[0].getValue())) { - is = new GZIPInputStream(get.getResponseBodyAsStream()); - } else { - is = get.getResponseBodyAsStream(); - } - } - - public int available() throws IOException { - return is.available(); - } - - public void close() throws IOException { - is.close(); - get.releaseConnection(); - } - - public boolean equals(Object obj) { - return is.equals(obj); - } - - public int hashCode() { - return is.hashCode(); - } - - public void mark(int readlimit) { - is.mark(readlimit); - } - - public boolean markSupported() { - return is.markSupported(); - } - - public int read() throws IOException { - return is.read(); - } - - public int read(byte[] b, int off, int len) throws IOException { - return is.read(b, off, len); - } - - public int read(byte[] b) throws IOException { - return is.read(b); - } - - public void reset() throws IOException { - is.reset(); - } - - public long skip(long n) throws IOException { - return is.skip(n); - } - - public String toString() { - return is.toString(); - } - } - private static final class HttpClientHelper3x implements HttpClientHelper { private static final int VERSION = 3; Modified: ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java?rev=951772&r1=951771&r2=951772&view=diff ============================================================================== --- ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java (original) +++ ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java Sat Jun 5 21:25:26 2010 @@ -20,6 +20,8 @@ package org.apache.ivy.util.url; import java.io.File; import java.net.URL; +import org.apache.ivy.util.FileUtil; + import junit.framework.TestCase; /** @@ -27,8 +29,21 @@ import junit.framework.TestCase; */ public class BasicURLHandlerTest extends TestCase { // remote.test + private File testDir; + private BasicURLHandler handler; + + protected void setUp() throws Exception { + testDir = new File("build/BasicURLHandlerTest"); + testDir.mkdirs(); + + handler = new BasicURLHandler(); + } + + protected void tearDown() throws Exception { + FileUtil.forceDelete(testDir); + } + public void testIsReachable() throws Exception { - URLHandler handler = new BasicURLHandler(); assertTrue(handler.isReachable(new URL("http://www.google.fr/"))); assertFalse(handler.isReachable(new URL("http://www.google.fr/unknownpage.html"))); @@ -39,4 +54,19 @@ public class BasicURLHandlerTest extends // assertTrue(handler.isReachable(new URL("ftp://ftp.mozilla.org/pub/dir.sizes"))); assertFalse(handler.isReachable(new URL("ftp://ftp.mozilla.org/unknown.file"))); } + + public void testContentEncoding() throws Exception { + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html"), new File(testDir, "gzip.txt")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on&zlib=on"), new File(testDir, "deflate-zlib.txt")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on"), new File(testDir, "deflate.txt")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps"), new File(testDir, "a5-gzip.ps")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps?deflate=on"), new File(testDir, "a5-deflate.ps")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf"), new File(testDir, "nh80-gzip.pdf")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf?deflate=on"), new File(testDir, "nh80-deflate.pdf")); + } + + private void assertDownloadOK(URL url, File file) throws Exception { + handler.download(url, file, null); + assertTrue(file.exists()); + } } Modified: ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java?rev=951772&r1=951771&r2=951772&view=diff ============================================================================== --- ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java (original) +++ ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java Sat Jun 5 21:25:26 2010 @@ -17,8 +17,10 @@ */ package org.apache.ivy.util.url; +import java.io.File; import java.net.URL; +import org.apache.ivy.util.FileUtil; import org.apache.ivy.util.url.URLHandler.URLInfo; import junit.framework.TestCase; @@ -28,8 +30,21 @@ import junit.framework.TestCase; */ public class HttpclientURLHandlerTest extends TestCase { // remote.test + private File testDir; + private HttpClientHandler handler; + + protected void setUp() throws Exception { + testDir = new File("build/HttpclientURLHandlerTest"); + testDir.mkdirs(); + + handler = new HttpClientHandler(); + } + + protected void tearDown() throws Exception { + FileUtil.forceDelete(testDir); + } + public void testIsReachable() throws Exception { - URLHandler handler = new HttpClientHandler(); assertTrue(handler.isReachable(new URL("http://www.google.fr/"))); assertFalse(handler.isReachable(new URL("http://www.google.fr/unknownpage.html"))); } @@ -43,4 +58,19 @@ public class HttpclientURLHandlerTest ex assertEquals(URLHandler.UNAVAILABLE, info); } + + public void testContentEncoding() throws Exception { + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html"), new File(testDir, "gzip.txt")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on&zlib=on"), new File(testDir, "deflate-zlib.txt")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on"), new File(testDir, "deflate.txt")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps"), new File(testDir, "a5-gzip.ps")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps?deflate=on"), new File(testDir, "a5-deflate.ps")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf"), new File(testDir, "nh80-gzip.pdf")); + assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf?deflate=on"), new File(testDir, "nh80-deflate.pdf")); + } + + private void assertDownloadOK(URL url, File file) throws Exception { + handler.download(url, file, null); + assertTrue(file.exists()); + } }