This is 4+megs of jars. Is an addition of this size really necessary for this task?
davep On Wed, Jun 18, 2008 at 1:20 AM, <[EMAIL PROTECTED]> wrote: > Author: etnu > Date: Wed Jun 18 01:19:58 2008 > New Revision: 669108 > > URL: http://svn.apache.org/viewvc?rev=669108&view=rev > Log: > Applied SHINDIG-391, which adds more robust character encoding detection for > HttpResponse using ICU, from Patrick Fairbank. > > > Modified: > incubator/shindig/trunk/java/gadgets/pom.xml > > incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java > > incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/HttpResponseTest.java > incubator/shindig/trunk/pom.xml > > Modified: incubator/shindig/trunk/java/gadgets/pom.xml > URL: > http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/pom.xml?rev=669108&r1=669107&r2=669108&view=diff > ============================================================================== > --- incubator/shindig/trunk/java/gadgets/pom.xml (original) > +++ incubator/shindig/trunk/java/gadgets/pom.xml Wed Jun 18 01:19:58 2008 > @@ -135,7 +135,7 @@ > <artifactId>core</artifactId> > <scope>compile</scope> > </dependency> > - <dependency> > + <dependency> > <groupId>com.google.code.google-collections</groupId> > <artifactId>google-collect</artifactId> > </dependency> > @@ -153,9 +153,14 @@ > <artifactId>jetty</artifactId> > <scope>test</scope> > </dependency> > - <dependency> > + <dependency> > <groupId>rome</groupId> > <artifactId>rome</artifactId> > </dependency> > + <dependency> > + <groupId>com.ibm.icu</groupId> > + <artifactId>icu4j</artifactId> > + <scope>compile</scope> > + </dependency> > </dependencies> > </project> > > Modified: > incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java > URL: > http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java?rev=669108&r1=669107&r2=669108&view=diff > ============================================================================== > --- > incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java > (original) > +++ > incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java > Wed Jun 18 01:19:58 2008 > @@ -19,6 +19,9 @@ > > import org.apache.shindig.common.util.DateUtil; > > +import com.ibm.icu.text.CharsetDetector; > +import com.ibm.icu.text.CharsetMatch; > + > import java.io.ByteArrayInputStream; > import java.io.InputStream; > import java.nio.ByteBuffer; > @@ -28,8 +31,10 @@ > import java.util.Collections; > import java.util.Date; > import java.util.HashMap; > +import java.util.HashSet; > import java.util.List; > import java.util.Map; > +import java.util.Set; > import java.util.TreeMap; > import java.util.concurrent.ConcurrentHashMap; > > @@ -42,6 +47,9 @@ > public final static int SC_NOT_FOUND = 404; > public final static int SC_INTERNAL_SERVER_ERROR = 500; > public final static int SC_TIMEOUT = 504; > + private final static Set<String> BINARY_CONTENT_TYPES = new > HashSet<String>(Arrays.asList( > + "image/jpeg", "image/png", "image/gif", "image/jpg", > "application/x-shockwave-flash" > + )); > > private final int httpStatusCode; > private static final String DEFAULT_ENCODING = "UTF-8"; > @@ -142,6 +150,9 @@ > String contentType = getHeader("Content-Type"); > if (contentType != null) { > String[] parts = contentType.split(";"); > + if (BINARY_CONTENT_TYPES.contains(parts[0])) { > + return DEFAULT_ENCODING; > + } > if (parts.length == 2) { > int offset = parts[1].indexOf("charset="); > if (offset != -1) { > @@ -149,7 +160,12 @@ > } > } > } > - return DEFAULT_ENCODING; > + > + // If the header doesn't specify the charset, try to determine it by > examining the content. > + CharsetDetector detector = new CharsetDetector(); > + detector.setText(responseBytes); > + CharsetMatch match = detector.detect(); > + return match.getName().toUpperCase(); > } > > public int getHttpStatusCode() { > > Modified: > incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/HttpResponseTest.java > URL: > http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/HttpResponseTest.java?rev=669108&r1=669107&r2=669108&view=diff > ============================================================================== > --- > incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/HttpResponseTest.java > (original) > +++ > incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/HttpResponseTest.java > Wed Jun 18 01:19:58 2008 > @@ -25,12 +25,30 @@ > > import java.util.Arrays; > import java.util.HashMap; > -import java.util.Iterator; > import java.util.LinkedList; > import java.util.List; > import java.util.Map; > > public class HttpResponseTest extends TestCase { > + private final static byte[] UTF8_DATA = new byte[] { > + (byte)0xEF, (byte)0xBB, (byte)0xBF, 'h', 'e', 'l', 'l', 'o' > + }; > + private final static String UTF8_STRING = "hello"; > + > + // A large string is needed for accurate charset detection. > + private final static byte[] LATIN1_DATA = new byte[] { > + 'G', 'a', 'm', 'e', 's', ',', ' ', 'H', 'Q', ',', ' ', 'M', 'a', 'n', > 'g', (byte)0xE1, ',', ' ', > + 'A', 'n', 'i', 'm', 'e', ' ', 'e', ' ', 't', 'u', 'd', 'o', ' ', 'q', > 'u', 'e', ' ', 'u', 'm', > + ' ', 'b', 'o', 'm', ' ', 'n', 'e', 'r', 'd', ' ', 'a', 'm', 'a' > + }; > + private final static String LATIN1_STRING > + = "Games, HQ, Mang\u00E1, Anime e tudo que um bom nerd ama"; > + > + private final static byte[] BIG5_DATA = new byte[] { > + (byte)0xa7, (byte)0x41, (byte)0xa6, (byte)0x6e > + }; > + private final static String BIG5_STRING = "\u4F60\u597D"; > + > private Map<String, List<String>> headers; > > @Override > @@ -56,32 +74,60 @@ > > public void testEncodingDetectionUtf8WithBom() throws Exception { > // Input is UTF-8 with BOM. > - byte[] data = new byte[] { > - (byte)0xEF, (byte)0xBB, (byte)0xBF, 'h', 'e', 'l', 'l', 'o' > - }; > addHeader("Content-Type", "text/plain; charset=UTF-8"); > - HttpResponse response = new HttpResponse(200, data, headers); > - assertEquals("hello", response.getResponseAsString()); > + HttpResponse response = new HttpResponse(200, UTF8_DATA, headers); > + assertEquals(UTF8_STRING, response.getResponseAsString()); > } > > public void testEncodingDetectionLatin1() throws Exception { > // Input is a basic latin-1 string with 1 non-UTF8 compatible char. > - byte[] data = new byte[] { > - 'h', (byte)0xE9, 'l', 'l', 'o' > - }; > addHeader("Content-Type", "text/plain; charset=iso-8859-1"); > - HttpResponse response = new HttpResponse(200, data, headers); > - assertEquals("h\u00E9llo", response.getResponseAsString()); > + HttpResponse response = new HttpResponse(200, LATIN1_DATA, headers); > + assertEquals(LATIN1_STRING, response.getResponseAsString()); > } > > public void testEncodingDetectionBig5() throws Exception { > - byte[] data = new byte[] { > - (byte)0xa7, (byte)0x41, (byte)0xa6, (byte)0x6e > - }; > addHeader("Content-Type", "text/plain; charset=BIG5"); > - HttpResponse response = new HttpResponse(200, data, headers); > - String resp = response.getResponseAsString(); > - assertEquals("\u4F60\u597D", response.getResponseAsString()); > + HttpResponse response = new HttpResponse(200, BIG5_DATA, headers); > + assertEquals(BIG5_STRING, response.getResponseAsString()); > + } > + > + public void testEncodingDetectionUtf8WithBomNoCharsetSpecified() throws > Exception { > + addHeader("Content-Type", "text/plain"); > + HttpResponse response = new HttpResponse(200, UTF8_DATA, headers); > + assertEquals("UTF-8", response.getEncoding().toUpperCase()); > + assertEquals(UTF8_STRING, response.getResponseAsString()); > + } > + > + public void testEncodingDetectionLatin1NoCharsetSpecified() throws > Exception { > + addHeader("Content-Type", "text/plain;"); > + HttpResponse response = new HttpResponse(200, LATIN1_DATA, headers); > + assertEquals("ISO-8859-1", response.getEncoding().toUpperCase()); > + assertEquals(LATIN1_STRING, response.getResponseAsString()); > + } > + > + public void testEncodingDetectionUtf8WithBomNoContentHeader() throws > Exception { > + HttpResponse response = new HttpResponse(200, UTF8_DATA, headers); > + assertEquals("UTF-8", response.getEncoding().toUpperCase()); > + assertEquals(UTF8_STRING, response.getResponseAsString()); > + } > + > + public void testEncodingDetectionLatin1NoContentHeader() throws Exception { > + HttpResponse response = new HttpResponse(200, LATIN1_DATA, headers); > + assertEquals("ISO-8859-1", response.getEncoding().toUpperCase()); > + assertEquals(LATIN1_STRING, response.getResponseAsString()); > + } > + > + public void testGetEncodingForImageContentType() throws Exception { > + addHeader("Content-Type", "image/png; charset=iso-8859-1"); > + HttpResponse response = new HttpResponse(200, LATIN1_DATA, headers); > + assertEquals("UTF-8", response.getEncoding().toUpperCase()); > + } > + > + public void testGetEncodingForFlashContentType() throws Exception { > + addHeader("Content-Type", "application/x-shockwave-flash; > charset=iso-8859-1"); > + HttpResponse response = new HttpResponse(200, LATIN1_DATA, headers); > + assertEquals("UTF-8", response.getEncoding().toUpperCase()); > } > > public void testPreserveBinaryData() throws Exception { > @@ -98,12 +144,14 @@ > addHeader("Cache-Control", "no-cache"); > HttpResponse response = new HttpResponse(200, new byte[0], headers); > assertTrue(response.isStrictNoCache()); > + assertEquals(-1, response.getCacheExpiration()); > } > > public void testStrictPragmaNoCache() throws Exception { > addHeader("Pragma", "no-cache"); > HttpResponse response = new HttpResponse(200, new byte[0], headers); > assertTrue(response.isStrictNoCache()); > + assertEquals(-1, response.getCacheExpiration()); > } > > public void testStrictPragmaJunk() throws Exception { > @@ -118,7 +166,17 @@ > long time = ((System.currentTimeMillis() / 1000) * 1000) + 10000L; > addHeader("Expires", DateUtil.formatDate(time)); > HttpResponse response = new HttpResponse(200, new byte[0], headers); > - assertEquals(time, response.getExpiration()); > + assertEquals(time, response.getCacheExpiration()); > + } > + > + public void testMaxAge() throws Exception { > + int maxAge = 10; > + long expected = ((System.currentTimeMillis() / 1000) * 1000) + (maxAge * > 1000); > + addHeader("Cache-Control", "public, max-age=" + maxAge); > + HttpResponse response = new HttpResponse(200, new byte[0], headers); > + long expiration = response.getCacheExpiration(); > + assertTrue("getExpiration is less than start time + maxAge", expiration > >= expected); > + assertTrue("getExpiration is too high.", expiration <= expected + 1000); > } > > public void testNegativeCaching() { > > Modified: incubator/shindig/trunk/pom.xml > URL: > http://svn.apache.org/viewvc/incubator/shindig/trunk/pom.xml?rev=669108&r1=669107&r2=669108&view=diff > ============================================================================== > --- incubator/shindig/trunk/pom.xml (original) > +++ incubator/shindig/trunk/pom.xml Wed Jun 18 01:19:58 2008 > @@ -832,11 +832,16 @@ > <artifactId>joda-time</artifactId> > <version>1.5.2</version> > </dependency> > - <dependency> > + <dependency> > <groupId>rome</groupId> > <artifactId>rome</artifactId> > <version>0.9</version> > - </dependency> > + </dependency> > + <dependency> > + <groupId>com.ibm.icu</groupId> > + <artifactId>icu4j</artifactId> > + <version>3.8</version> > + </dependency> > </dependencies> > </dependencyManagement> > </project> > > >
