This is an automated email from the ASF dual-hosted git repository. ulyssesyou pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push: new 42b78780c [GLUTEN-4424][CORE] Follow up upgrading spark version to 3.5.1 (#4845) 42b78780c is described below commit 42b78780c71fe81f47be9be01cdedddb84388f9a Author: JiaKe <ke.a....@intel.com> AuthorDate: Fri Mar 8 12:17:33 2024 +0800 [GLUTEN-4424][CORE] Follow up upgrading spark version to 3.5.1 (#4845) --- dev/buildbundle-veloxbe.sh | 1 + dev/package.sh | 1 + .../main/scala/io/glutenproject/execution/WholeStageTransformer.scala | 2 -- .../scala/io/glutenproject/execution/GenerateTreeStringShim.scala | 4 ++-- .../scala/io/glutenproject/execution/GenerateTreeStringShim.scala | 4 ++-- .../scala/io/glutenproject/execution/GenerateTreeStringShim.scala | 4 ++-- .../scala/io/glutenproject/execution/GenerateTreeStringShim.scala | 4 ++-- tools/gluten-te/ubuntu/dockerfile-buildenv | 4 ++++ 8 files changed, 14 insertions(+), 10 deletions(-) diff --git a/dev/buildbundle-veloxbe.sh b/dev/buildbundle-veloxbe.sh index 3bfd6994a..1c77b52a6 100755 --- a/dev/buildbundle-veloxbe.sh +++ b/dev/buildbundle-veloxbe.sh @@ -7,3 +7,4 @@ cd $GLUTEN_DIR mvn clean package -Pbackends-velox -Prss -Pspark-3.2 -DskipTests mvn clean package -Pbackends-velox -Prss -Pspark-3.3 -DskipTests mvn clean package -Pbackends-velox -Prss -Pspark-3.4 -DskipTests +mvn clean package -Pbackends-velox -Prss -Pspark-3.5 -DskipTests diff --git a/dev/package.sh b/dev/package.sh index 0e1f362bc..2f6602d00 100755 --- a/dev/package.sh +++ b/dev/package.sh @@ -14,6 +14,7 @@ $GLUTEN_DIR/dev/builddeps-veloxbe.sh --build_tests=ON --build_benchmarks=ON --en mvn clean package -Pbackends-velox -Prss -Pspark-3.2 -DskipTests mvn clean package -Pbackends-velox -Prss -Pspark-3.3 -DskipTests mvn clean package -Pbackends-velox -Prss -Pspark-3.4 -DskipTests +mvn clean package -Pbackends-velox -Prss -Pspark-3.5 -DskipTests mkdir -p $THIRDPARTY_LIB function process_setup_ubuntu_2004 { diff --git a/gluten-core/src/main/scala/io/glutenproject/execution/WholeStageTransformer.scala b/gluten-core/src/main/scala/io/glutenproject/execution/WholeStageTransformer.scala index 2bbc80afc..24957d1c2 100644 --- a/gluten-core/src/main/scala/io/glutenproject/execution/WholeStageTransformer.scala +++ b/gluten-core/src/main/scala/io/glutenproject/execution/WholeStageTransformer.scala @@ -102,8 +102,6 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f def stageId: Int = transformStageId - def substraitPlanJsonValue: String = substraitPlanJson - def wholeStageTransformerContextDefined: Boolean = wholeStageTransformerContext.isDefined // For WholeStageCodegen-like operator, only pipeline time will be handled in graph plotting. diff --git a/shims/spark32/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala b/shims/spark32/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala index d99fb18d7..d06728bc7 100644 --- a/shims/spark32/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala +++ b/shims/spark32/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala @@ -29,7 +29,7 @@ trait GenerateTreeStringShim extends UnaryExecNode { def stageId: Int - def substraitPlanJsonValue: String + def substraitPlanJson: String def wholeStageTransformerContextDefined: Boolean @@ -57,7 +57,7 @@ trait GenerateTreeStringShim extends UnaryExecNode { if (verbose && wholeStageTransformerContextDefined) { append(prefix + "Substrait plan:\n") - append(substraitPlanJsonValue) + append(substraitPlanJson) append("\n") } } diff --git a/shims/spark33/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala b/shims/spark33/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala index d99fb18d7..d06728bc7 100644 --- a/shims/spark33/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala +++ b/shims/spark33/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala @@ -29,7 +29,7 @@ trait GenerateTreeStringShim extends UnaryExecNode { def stageId: Int - def substraitPlanJsonValue: String + def substraitPlanJson: String def wholeStageTransformerContextDefined: Boolean @@ -57,7 +57,7 @@ trait GenerateTreeStringShim extends UnaryExecNode { if (verbose && wholeStageTransformerContextDefined) { append(prefix + "Substrait plan:\n") - append(substraitPlanJsonValue) + append(substraitPlanJson) append("\n") } } diff --git a/shims/spark34/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala b/shims/spark34/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala index d99fb18d7..d06728bc7 100644 --- a/shims/spark34/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala +++ b/shims/spark34/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala @@ -29,7 +29,7 @@ trait GenerateTreeStringShim extends UnaryExecNode { def stageId: Int - def substraitPlanJsonValue: String + def substraitPlanJson: String def wholeStageTransformerContextDefined: Boolean @@ -57,7 +57,7 @@ trait GenerateTreeStringShim extends UnaryExecNode { if (verbose && wholeStageTransformerContextDefined) { append(prefix + "Substrait plan:\n") - append(substraitPlanJsonValue) + append(substraitPlanJson) append("\n") } } diff --git a/shims/spark35/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala b/shims/spark35/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala index 3753d1df4..9d0419967 100644 --- a/shims/spark35/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala +++ b/shims/spark35/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala @@ -29,7 +29,7 @@ trait GenerateTreeStringShim extends UnaryExecNode { def stageId: Int - def substraitPlanJsonValue: String + def substraitPlanJson: String def wholeStageTransformerContextDefined: Boolean @@ -57,7 +57,7 @@ trait GenerateTreeStringShim extends UnaryExecNode { if (verbose && wholeStageTransformerContextDefined) { append(prefix + "Substrait plan:\n") - append(substraitPlanJsonValue) + append(substraitPlanJson) append("\n") } } diff --git a/tools/gluten-te/ubuntu/dockerfile-buildenv b/tools/gluten-te/ubuntu/dockerfile-buildenv index 280837bb3..dbb9fc65b 100644 --- a/tools/gluten-te/ubuntu/dockerfile-buildenv +++ b/tools/gluten-te/ubuntu/dockerfile-buildenv @@ -115,6 +115,10 @@ RUN cd /opt/spark331 && ./build/mvn -Pyarn -DskipTests clean install RUN cd /opt && git clone --depth 1 --branch v3.4.2 https://github.com/apache/spark.git spark342 RUN cd /opt/spark342 && ./build/mvn -Pyarn -DskipTests clean install +# Build & install Spark 3.5.1 +# RUN cd /opt && git clone --depth 1 --branch v3.5.1 https://github.com/apache/spark.git spark351 +# RUN cd /opt/spark351 && ./build/mvn -Pyarn -DskipTests clean install + # Prepare entry command COPY scripts/cmd.sh /root/.cmd.sh CMD ["/root/.cmd.sh"] --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org For additional commands, e-mail: commits-h...@gluten.apache.org