Author: scottcarey
Date: Wed Feb 16 01:12:35 2011
New Revision: 1071117
URL: http://svn.apache.org/viewvc?rev=1071117&view=rev
Log:
AVRO-762. Java: Bug in BinaryData.compare() with offset comparison.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/BinaryData.java
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroKeyComparator.java
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherKeyComparator.java
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1071117&r1=1071116&r2=1071117&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Feb 16 01:12:35 2011
@@ -162,6 +162,8 @@ Avro 1.5.0 (unreleased)
BUG FIXES
+ AVRO-762. Java: Bug in BinaryData.compare() with offset comparison.
+
AVRO-743. Java: Performance Regression and memory pressure with
GenericDatumReader. (scottcarey)
Modified:
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/BinaryData.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/BinaryData.java?rev=1071117&r1=1071116&r2=1071117&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/BinaryData.java
(original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/BinaryData.java
Wed Feb 16 01:12:35 2011
@@ -57,8 +57,17 @@ public class BinaryData {
public static int compare(byte[] b1, int s1,
byte[] b2, int s2,
Schema schema) {
+ return compare(b1, s1, b1.length - s1, b2, s2, b2.length - s2, schema);
+ }
+
+ /** Compare binary encoded data. If equal, return zero. If greater-than,
+ * return 1, if less than return -1. Order is consistent with that of {@link
+ * org.apache.avro.generic.GenericData#compare(Object, Object, Schema)}.*/
+ public static int compare(byte[] b1, int s1, int l1,
+ byte[] b2, int s2, int l2,
+ Schema schema) {
Decoders decoders = DECODERS.get();
- decoders.set(b1, s1, b1.length, b2, s2, b2.length);
+ decoders.set(b1, s1, l1, b2, s2, l2);
try {
return compare(decoders, schema);
} catch (IOException e) {
Modified:
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java?rev=1071117&r1=1071116&r2=1071117&view=diff
==============================================================================
---
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java
(original)
+++
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java
Wed Feb 16 01:12:35 2011
@@ -17,6 +17,7 @@
*/
package org.apache.avro.generic;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -31,6 +32,9 @@ import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema.Type;
+import org.apache.avro.io.BinaryData;
+import org.apache.avro.io.BinaryEncoder;
+import org.apache.avro.generic.GenericData.Record;
import org.apache.avro.util.Utf8;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonParseException;
@@ -166,4 +170,46 @@ public class TestGenericData {
// will throw exception if string is not parsable json
mapper.readTree(parser);
}
+
+ @Test
+ public void testCompare() {
+ // Prepare a schema for testing.
+ Field integerField = new Field("test", Schema.create(Type.INT), null,
null);
+ List<Field> fields = new ArrayList<Field>();
+ fields.add(integerField);
+ Schema record = Schema.createRecord("test", null, null, false);
+ record.setFields(fields);
+
+ ByteArrayOutputStream b1 = new ByteArrayOutputStream(5);
+ ByteArrayOutputStream b2 = new ByteArrayOutputStream(5);
+ BinaryEncoder b1Enc = new BinaryEncoder(b1);
+ BinaryEncoder b2Enc = new BinaryEncoder(b2);
+ // Prepare two different datums
+ Record testDatum1 = new Record(record);
+ testDatum1.put(0, 1);
+ Record testDatum2 = new Record(record);
+ testDatum2.put(0, 2);
+ GenericDatumWriter<Record> gWriter = new
GenericDatumWriter<Record>(record);
+ Integer start1 = 0, start2 = 0;
+ try {
+ // Write two datums in each stream
+ // and get the offset length after the first write in each.
+ gWriter.write(testDatum1, b1Enc);
+ b1Enc.flush();
+ start1 = b1.size();
+ gWriter.write(testDatum1, b1Enc);
+ b1Enc.flush();
+ b1.close();
+ gWriter.write(testDatum2, b2Enc);
+ b2Enc.flush();
+ start2 = b2.size();
+ gWriter.write(testDatum2, b2Enc);
+ b2Enc.flush();
+ b2.close();
+ // Compare to check if offset-based compare works right.
+ assertEquals(-1, BinaryData.compare(b1.toByteArray(), start1,
b2.toByteArray(), start2, record));
+ } catch (IOException e) {
+ fail("IOException while writing records to output stream.");
+ }
+ }
}
Modified:
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroKeyComparator.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroKeyComparator.java?rev=1071117&r1=1071116&r2=1071117&view=diff
==============================================================================
---
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroKeyComparator.java
(original)
+++
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroKeyComparator.java
Wed Feb 16 01:12:35 2011
@@ -40,7 +40,7 @@ public class AvroKeyComparator<T>
}
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- return BinaryData.compare(b1, s1, b2, s2, schema);
+ return BinaryData.compare(b1, s1, l1, b2, s2, l2, schema);
}
public int compare(AvroWrapper<T> x, AvroWrapper<T> y) {
Modified:
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherKeyComparator.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherKeyComparator.java?rev=1071117&r1=1071116&r2=1071117&view=diff
==============================================================================
---
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherKeyComparator.java
(original)
+++
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherKeyComparator.java
Wed Feb 16 01:12:35 2011
@@ -43,8 +43,8 @@ class TetherKeyComparator
@Override
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- int diff = BinaryData.compare(b1, BinaryData.skipLong(b1, s1),
- b2, BinaryData.skipLong(b2, s2),
+ int diff = BinaryData.compare(b1, BinaryData.skipLong(b1, s1), l1,
+ b2, BinaryData.skipLong(b2, s2), l2,
schema);
return diff == 0 ? -1 : diff;
}
