This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 1a72c15 [SPARK-27216][CORE][BACKPORT-2.4] Upgrade RoaringBitmap to
0.7.45 to fix Kryo unsafe ser/dser issue
1a72c15 is described below
commit 1a72c15a65426f15b7575263831d500c2766cc01
Author: LantaoJin <jinlan...@gmail.com>
AuthorDate: Thu Apr 4 18:22:57 2019 -0500
[SPARK-27216][CORE][BACKPORT-2.4] Upgrade RoaringBitmap to 0.7.45 to fix
Kryo unsafe ser/dser issue
## What changes were proposed in this pull request?
Back-port of #24264 to branch-2.4.
HighlyCompressedMapStatus uses RoaringBitmap to record the empty blocks.
But RoaringBitmap couldn't be ser/deser with unsafe KryoSerializer.
It's a bug of RoaringBitmap-0.5.11 and fixed in latest version.
## How was this patch tested?
Add a UT
Closes #24290 from LantaoJin/SPARK-27216_BACKPORT-2.4.
Authored-by: LantaoJin <jinlan...@gmail.com>
Signed-off-by: Sean Owen <sean.o...@databricks.com>
---
.../spark/serializer/KryoSerializerSuite.scala | 35 ++++++----------------
dev/deps/spark-deps-hadoop-2.6 | 3 +-
dev/deps/spark-deps-hadoop-2.7 | 3 +-
dev/deps/spark-deps-hadoop-3.1 | 3 +-
pom.xml | 2 +-
5 files changed, 16 insertions(+), 30 deletions(-)
diff --git
a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index fcb1315..bf5ff10 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.serializer
-import java.io.{ByteArrayInputStream, ByteArrayOutputStream, FileInputStream,
FileOutputStream}
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
import java.nio.ByteBuffer
import scala.collection.JavaConverters._
@@ -32,7 +32,6 @@ import org.apache.spark.{SharedSparkContext, SparkConf,
SparkFunSuite}
import org.apache.spark.scheduler.HighlyCompressedMapStatus
import org.apache.spark.serializer.KryoTest._
import org.apache.spark.storage.BlockManagerId
-import org.apache.spark.util.Utils
class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
@@ -365,30 +364,6 @@ class KryoSerializerSuite extends SparkFunSuite with
SharedSparkContext {
assert(thrown.getCause.isInstanceOf[KryoException])
}
- test("SPARK-12222: deserialize RoaringBitmap throw Buffer underflow
exception") {
- val dir = Utils.createTempDir()
- val tmpfile = dir.toString + "/RoaringBitmap"
- val outStream = new FileOutputStream(tmpfile)
- val output = new KryoOutput(outStream)
- val bitmap = new RoaringBitmap
- bitmap.add(1)
- bitmap.add(3)
- bitmap.add(5)
- // Ignore Kryo because it doesn't use writeObject
- bitmap.serialize(new KryoOutputObjectOutputBridge(null, output))
- output.flush()
- output.close()
-
- val inStream = new FileInputStream(tmpfile)
- val input = new KryoInput(inStream)
- val ret = new RoaringBitmap
- // Ignore Kryo because it doesn't use readObject
- ret.deserialize(new KryoInputObjectInputBridge(null, input))
- input.close()
- assert(ret == bitmap)
- Utils.deleteRecursively(dir)
- }
-
test("KryoOutputObjectOutputBridge.writeObject and
KryoInputObjectInputBridge.readObject") {
val kryo = new KryoSerializer(conf).newKryo()
@@ -462,6 +437,14 @@ class KryoSerializerSuite extends SparkFunSuite with
SharedSparkContext {
testSerializerInstanceReuse(autoReset = autoReset, referenceTracking =
referenceTracking)
}
}
+
+ test("SPARK-27216: test RoaringBitmap ser/dser with Kryo") {
+ val expected = new RoaringBitmap()
+ expected.add(1787)
+ val ser = new KryoSerializer(conf).newInstance()
+ val actual: RoaringBitmap = ser.deserialize(ser.serialize(expected))
+ assert(actual === expected)
+ }
}
class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with
SharedSparkContext {
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index ade5be5..b2bc8e2 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -1,5 +1,5 @@
JavaEWAH-0.3.2.jar
-RoaringBitmap-0.5.11.jar
+RoaringBitmap-0.7.45.jar
ST4-4.0.4.jar
activation-1.1.1.jar
aircompressor-0.10.jar
@@ -176,6 +176,7 @@ scala-parser-combinators_2.11-1.1.0.jar
scala-reflect-2.11.12.jar
scala-xml_2.11-1.0.5.jar
shapeless_2.11-2.3.2.jar
+shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
snakeyaml-1.15.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 2ce82e8..b2767d3 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -1,5 +1,5 @@
JavaEWAH-0.3.2.jar
-RoaringBitmap-0.5.11.jar
+RoaringBitmap-0.7.45.jar
ST4-4.0.4.jar
activation-1.1.1.jar
aircompressor-0.10.jar
@@ -177,6 +177,7 @@ scala-parser-combinators_2.11-1.1.0.jar
scala-reflect-2.11.12.jar
scala-xml_2.11-1.0.5.jar
shapeless_2.11-2.3.2.jar
+shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
snakeyaml-1.15.jar
diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1
index eabc163..9b2cb8d 100644
--- a/dev/deps/spark-deps-hadoop-3.1
+++ b/dev/deps/spark-deps-hadoop-3.1
@@ -1,6 +1,6 @@
HikariCP-java7-2.4.12.jar
JavaEWAH-0.3.2.jar
-RoaringBitmap-0.5.11.jar
+RoaringBitmap-0.7.45.jar
ST4-4.0.4.jar
accessors-smart-1.2.jar
activation-1.1.1.jar
@@ -196,6 +196,7 @@ scala-parser-combinators_2.11-1.1.0.jar
scala-reflect-2.11.12.jar
scala-xml_2.11-1.0.5.jar
shapeless_2.11-2.3.2.jar
+shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
snakeyaml-1.15.jar
diff --git a/pom.xml b/pom.xml
index 1065b90..e5e8497 100644
--- a/pom.xml
+++ b/pom.xml
@@ -574,7 +574,7 @@
<dependency>
<groupId>org.roaringbitmap</groupId>
<artifactId>RoaringBitmap</artifactId>
- <version>0.5.11</version>
+ <version>0.7.45</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
