This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f985e3e84a2 [SPARK-44255][SQL] Relocate StorageLevel to common/utils
f985e3e84a2 is described below
commit f985e3e84a23ab5a83842047408e3fd92887447a
Author: Rui Wang <rui.w...@databricks.com>
AuthorDate: Sat Jul 1 12:10:22 2023 +0300
[SPARK-44255][SQL] Relocate StorageLevel to common/utils
### What changes were proposed in this pull request?
Relocate `StorageLevel` to `common/utils`.
### Why are the changes needed?
Scala client needs `StorageLevel` so this can be shared in the
`common/utils`.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests
Closes #41797 from amaliujia/move_storage_level_to_common_utils.
Authored-by: Rui Wang <rui.w...@databricks.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../java/org/apache/spark/memory/MemoryMode.java | 0
.../org/apache/spark/storage/StorageLevel.scala | 6 ++---
.../org/apache/spark/util/SparkErrorUtils.scala | 30 +++++++++++++++++-----
.../main/scala/org/apache/spark/util/Utils.scala | 11 +-------
project/MimaExcludes.scala | 4 +++
5 files changed, 32 insertions(+), 19 deletions(-)
diff --git a/core/src/main/java/org/apache/spark/memory/MemoryMode.java
b/common/utils/src/main/java/org/apache/spark/memory/MemoryMode.java
similarity index 100%
copy from core/src/main/java/org/apache/spark/memory/MemoryMode.java
copy to common/utils/src/main/java/org/apache/spark/memory/MemoryMode.java
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
b/common/utils/src/main/scala/org/apache/spark/storage/StorageLevel.scala
similarity index 97%
rename from core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
rename to
common/utils/src/main/scala/org/apache/spark/storage/StorageLevel.scala
index 4a2b705e069..73bc53dab89 100644
--- a/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
+++ b/common/utils/src/main/scala/org/apache/spark/storage/StorageLevel.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.memory.MemoryMode
-import org.apache.spark.util.Utils
+import org.apache.spark.util.SparkErrorUtils
/**
* :: DeveloperApi ::
@@ -98,12 +98,12 @@ class StorageLevel private(
ret
}
- override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException
{
+ override def writeExternal(out: ObjectOutput): Unit =
SparkErrorUtils.tryOrIOException {
out.writeByte(toInt)
out.writeByte(_replication)
}
- override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {
+ override def readExternal(in: ObjectInput): Unit =
SparkErrorUtils.tryOrIOException {
val flags = in.readByte()
_useDisk = (flags & 8) != 0
_useMemory = (flags & 4) != 0
diff --git a/core/src/main/java/org/apache/spark/memory/MemoryMode.java
b/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
similarity index 50%
rename from core/src/main/java/org/apache/spark/memory/MemoryMode.java
rename to
common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
index 3a5e72d8aae..8e4de01885e 100644
--- a/core/src/main/java/org/apache/spark/memory/MemoryMode.java
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
@@ -14,13 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.spark.util
-package org.apache.spark.memory;
+import java.io.IOException
-import org.apache.spark.annotation.Private;
+import scala.util.control.NonFatal
-@Private
-public enum MemoryMode {
- ON_HEAP,
- OFF_HEAP
+import org.apache.spark.internal.Logging
+
+object SparkErrorUtils extends Logging {
+ /**
+ * Execute a block of code that returns a value, re-throwing any non-fatal
uncaught
+ * exceptions as IOException. This is used when implementing Externalizable
and Serializable's
+ * read and write methods, since Java's serializer will not report
non-IOExceptions properly;
+ * see SPARK-4080 for more context.
+ */
+ def tryOrIOException[T](block: => T): T = {
+ try {
+ block
+ } catch {
+ case e: IOException =>
+ logError("Exception encountered", e)
+ throw e
+ case NonFatal(e) =>
+ logError("Exception encountered", e)
+ throw new IOException(e)
+ }
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index ada0cffd2b0..60895c791b5 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1449,16 +1449,7 @@ private[spark] object Utils extends Logging with
SparkClassUtils {
* see SPARK-4080 for more context.
*/
def tryOrIOException[T](block: => T): T = {
- try {
- block
- } catch {
- case e: IOException =>
- logError("Exception encountered", e)
- throw e
- case NonFatal(e) =>
- logError("Exception encountered", e)
- throw new IOException(e)
- }
+ SparkErrorUtils.tryOrIOException(block)
}
/** Executes the given block. Log non-fatal errors if any, and only throw
fatal errors */
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 6ca32e9d9e8..2e70fd9225c 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -100,6 +100,10 @@ object MimaExcludes {
// SPARK-44104: shaded protobuf code and Apis with parameters relocated
ProblemFilters.exclude[Problem]("org.sparkproject.spark_protobuf.protobuf.*"),
ProblemFilters.exclude[Problem]("org.apache.spark.sql.protobuf.utils.SchemaConverters.*"),
+
+ // SPARK-44255: Relocate StorageLevel to common/utils
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.storage.StorageLevel"),
+
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.storage.StorageLevel$"),
(problem: Problem) => problem match {
case MissingClassProblem(cls) =>
!cls.fullName.startsWith("org.sparkproject.jpmml") &&
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
