This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 29ed337d52c [SPARK-35579][SQL] Bump janino to 3.1.7
29ed337d52c is described below
commit 29ed337d52c5e821f628ff38d4ae01ebba95751a
Author: Prashant Singh <psing...@amazon.com>
AuthorDate: Mon Jul 18 08:11:27 2022 -0500
[SPARK-35579][SQL] Bump janino to 3.1.7
### What changes were proposed in this pull request?
upgrade janino to 3.1.7 from 3.0.16
### Why are the changes needed?
- The proposed version contains bug fix in janino by maropu.
- https://github.com/janino-compiler/janino/pull/148
- contains `getBytecodes` method which can be used to simplify the way to
get bytecodes from ClassBodyEvaluator in
CodeGenerator#updateAndGetCompilationStats method. (by LuciferYang)
- https://github.com/apache/spark/pull/32536
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing UTs
Closes #37202 from singhpk234/upgrade/bump-janino.
Authored-by: Prashant Singh <psing...@amazon.com>
Signed-off-by: Sean Owen <sro...@gmail.com>
---
dev/deps/spark-deps-hadoop-2-hive-2.3 | 4 ++--
dev/deps/spark-deps-hadoop-3-hive-2.3 | 4 ++--
pom.xml | 2 +-
.../sql/catalyst/expressions/codegen/CodeGenerator.scala | 14 +++-----------
.../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 3 +--
.../sql/catalyst/expressions/CodeGenerationSuite.scala | 1 -
6 files changed, 9 insertions(+), 19 deletions(-)
diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3
b/dev/deps/spark-deps-hadoop-2-hive-2.3
index ccc0d607010..af1db05ff4b 100644
--- a/dev/deps/spark-deps-hadoop-2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2-hive-2.3
@@ -39,7 +39,7 @@ commons-cli/1.5.0//commons-cli-1.5.0.jar
commons-codec/1.15//commons-codec-1.15.jar
commons-collections/3.2.2//commons-collections-3.2.2.jar
commons-collections4/4.4//commons-collections4-4.4.jar
-commons-compiler/3.0.16//commons-compiler-3.0.16.jar
+commons-compiler/3.1.7//commons-compiler-3.1.7.jar
commons-compress/1.21//commons-compress-1.21.jar
commons-configuration/1.6//commons-configuration-1.6.jar
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
@@ -128,7 +128,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.16//janino-3.0.16.jar
+janino/3.1.7//janino-3.1.7.jar
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
javax.inject/1//javax.inject-1.jar
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 31ada151e26..28644989282 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -40,7 +40,7 @@ commons-cli/1.5.0//commons-cli-1.5.0.jar
commons-codec/1.15//commons-codec-1.15.jar
commons-collections/3.2.2//commons-collections-3.2.2.jar
commons-collections4/4.4//commons-collections4-4.4.jar
-commons-compiler/3.0.16//commons-compiler-3.0.16.jar
+commons-compiler/3.1.7//commons-compiler-3.1.7.jar
commons-compress/1.21//commons-compress-1.21.jar
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
commons-dbcp/1.4//commons-dbcp-1.4.jar
@@ -116,7 +116,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.16//janino-3.0.16.jar
+janino/3.1.7//janino-3.1.7.jar
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
javolution/5.5.1//javolution-5.5.1.jar
diff --git a/pom.xml b/pom.xml
index 1976ea9db92..ffd2baae8aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -189,7 +189,7 @@
<commons-pool2.version>2.11.1</commons-pool2.version>
<datanucleus-core.version>4.1.17</datanucleus-core.version>
<guava.version>14.0.1</guava.version>
- <janino.version>3.0.16</janino.version>
+ <janino.version>3.1.7</janino.version>
<jersey.version>2.35</jersey.version>
<joda.version>2.10.14</joda.version>
<jodd.version>3.5.2</jodd.version>
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 592783a54e5..3f3e9f75cfa 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.catalyst.expressions.codegen
import java.io.ByteArrayInputStream
-import java.util.{Map => JavaMap}
import scala.annotation.tailrec
import scala.collection.JavaConverters._
@@ -28,8 +27,8 @@ import scala.util.control.NonFatal
import com.google.common.cache.{CacheBuilder, CacheLoader}
import com.google.common.util.concurrent.{ExecutionError,
UncheckedExecutionException}
-import org.codehaus.commons.compiler.CompileException
-import org.codehaus.janino.{ByteArrayClassLoader, ClassBodyEvaluator,
InternalCompilerException, SimpleCompiler}
+import org.codehaus.commons.compiler.{CompileException,
InternalCompilerException}
+import org.codehaus.janino.ClassBodyEvaluator
import org.codehaus.janino.util.ClassFile
import org.apache.spark.{TaskContext, TaskKilledException}
@@ -1524,14 +1523,7 @@ object CodeGenerator extends Logging {
*/
private def updateAndGetCompilationStats(evaluator: ClassBodyEvaluator):
ByteCodeStats = {
// First retrieve the generated classes.
- val classes = {
- val resultField = classOf[SimpleCompiler].getDeclaredField("result")
- resultField.setAccessible(true)
- val loader =
resultField.get(evaluator).asInstanceOf[ByteArrayClassLoader]
- val classesField = loader.getClass.getDeclaredField("classes")
- classesField.setAccessible(true)
- classesField.get(loader).asInstanceOf[JavaMap[String,
Array[Byte]]].asScala
- }
+ val classes = evaluator.getBytecodes.asScala
// Then walk the classes to get at the method bytecode.
val codeAttr =
Utils.classForName("org.codehaus.janino.util.ClassFile$CodeAttribute")
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index 4c030a8ac0c..64e6283c0e3 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -30,8 +30,7 @@ import java.util.concurrent.TimeoutException
import com.fasterxml.jackson.core.{JsonParser, JsonToken}
import org.apache.hadoop.fs.{FileAlreadyExistsException, FileStatus, Path}
import org.apache.hadoop.fs.permission.FsPermission
-import org.codehaus.commons.compiler.CompileException
-import org.codehaus.janino.InternalCompilerException
+import org.codehaus.commons.compiler.{CompileException,
InternalCompilerException}
import org.apache.spark.{Partition, SparkArithmeticException,
SparkArrayIndexOutOfBoundsException, SparkClassNotFoundException,
SparkConcurrentModificationException, SparkDateTimeException, SparkException,
SparkFileAlreadyExistsException, SparkFileNotFoundException,
SparkIllegalArgumentException, SparkIndexOutOfBoundsException,
SparkNoSuchElementException, SparkNoSuchMethodException,
SparkNumberFormatException, SparkRuntimeException, SparkSecurityException,
SparkSQLException, SparkSQLFea [...]
import org.apache.spark.executor.CommitDeniedException
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
index 1e4499a0ee3..737fcb1bada 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -568,7 +568,6 @@ class CodeGenerationSuite extends SparkFunSuite with
ExpressionEvalHelper {
assert(refTerm.contains("scala.math.LowPriorityOrderingImplicits$$anon$"))
}
- // TODO (SPARK-35579): Fix this bug in janino and upgrade janino in Spark.
test("SPARK-35578: final local variable bug in janino") {
val code =
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
