This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2af4bc2 [Bug] Java Version BitmapValue deserialized failed when only
has 32-bit bitmap (#4884)
2af4bc2 is described below
commit 2af4bc294f06a00fc04374e61b294d0ab3f8a147
Author: wangbo <[email protected]>
AuthorDate: Mon Nov 16 21:54:07 2020 +0800
[Bug] Java Version BitmapValue deserialized failed when only has 32-bit
bitmap (#4884)
---
.../apache/doris/load/loadv2/dpp/Roaring64Map.java | 15 +++++++++++----
.../doris/load/loadv2/dpp/BitmapValueTest.java | 21 ++++++++++++++++++---
2 files changed, 29 insertions(+), 7 deletions(-)
diff --git
a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java
b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java
index 56ad1a7..a606766 100644
---
a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java
+++
b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java
@@ -24,6 +24,7 @@ import org.roaringbitmap.BitmapDataProvider;
import org.roaringbitmap.BitmapDataProviderSupplier;
import org.roaringbitmap.IntConsumer;
import org.roaringbitmap.IntIterator;
+import org.roaringbitmap.InvalidRoaringFormat;
import org.roaringbitmap.RoaringBitmap;
import org.roaringbitmap.RoaringBitmapSupplier;
import org.roaringbitmap.Util;
@@ -1345,16 +1346,22 @@ public class Roaring64Map {
this.clear();
highToBitmap = new TreeMap<>();
- long nbHighs = 1;
- if (bitmapType == BitmapValue.BITMAP64) {
- nbHighs = Codec.decodeVarint64(in);
+ if (bitmapType == BitmapValue.BITMAP32) {
+ RoaringBitmap provider = new RoaringBitmap();
+ provider.deserialize(in);
+ highToBitmap.put(0, provider);
+ return;
}
+ if (bitmapType != BitmapValue.BITMAP64) {
+ throw new InvalidRoaringFormat("invalid bitmap type");
+ }
+
+ long nbHighs = Codec.decodeVarint64(in);
for (int i = 0; i < nbHighs; i++) {
int high = in.readInt();
RoaringBitmap provider = new RoaringBitmap();
provider.deserialize(in);
-
highToBitmap.put(high, provider);
}
diff --git
a/fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java
b/fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java
index d9c71e8..e90e885 100644
---
a/fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java
+++
b/fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java
@@ -324,13 +324,11 @@ public class BitmapValueTest {
Assert.assertTrue(serializeSingleValueBitmapValue.equals(deserializeSingleValueBitmapValue));
// bitmap
+ // case 1 : 32-bit bitmap
BitmapValue serializeBitmapBitmapValue = new BitmapValue();
for (int i = 0; i < 10; i++) {
serializeBitmapBitmapValue.add(i);
}
- for (long i = Long.MAX_VALUE; i > Long.MAX_VALUE - 10; i--) {
- serializeBitmapBitmapValue.add(i);
- }
ByteArrayOutputStream bitmapOutputStream = new ByteArrayOutputStream();
DataOutput bitmapOutput = new DataOutputStream(bitmapOutputStream);
serializeBitmapBitmapValue.serialize(bitmapOutput);
@@ -340,6 +338,23 @@ public class BitmapValueTest {
deserializeBitmapBitmapValue.deserialize(bitmapInputStream);
Assert.assertTrue(serializeBitmapBitmapValue.equals(deserializeBitmapBitmapValue));
+
+
+ // bitmap
+ // case 2 : 64-bit bitmap
+ BitmapValue serializeBitmapBitmapValue64 = new BitmapValue();
+ for (long i = Long.MAX_VALUE; i > Long.MAX_VALUE - 10; i--) {
+ serializeBitmapBitmapValue64.add(i);
+ }
+ ByteArrayOutputStream bitmapOutputStream64 = new
ByteArrayOutputStream();
+ DataOutput bitmapOutput64 = new DataOutputStream(bitmapOutputStream64);
+ serializeBitmapBitmapValue64.serialize(bitmapOutput64);
+
+ DataInputStream bitmapInputStream64 = new DataInputStream(new
ByteArrayInputStream(bitmapOutputStream64.toByteArray()));
+ BitmapValue deserializeBitmapBitmapValue64 = new BitmapValue();
+ deserializeBitmapBitmapValue64.deserialize(bitmapInputStream64);
+
+
Assert.assertTrue(serializeBitmapBitmapValue64.equals(deserializeBitmapBitmapValue64));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
