This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin-on-parquet-v2 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 621868cb04d03df9683bb4c2508afc92f23058ad Author: Zhichao Zhang <441586...@qq.com> AuthorDate: Wed Aug 26 00:23:19 2020 +0800 KYLIN-4660 remove deprecated config 'spark.yarn.executor.memoryOverhead' --- core-common/src/main/resources/kylin-defaults.properties | 2 +- examples/test_case_data/sandbox/kylin.properties | 4 ++-- .../main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java | 4 ---- .../java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java | 3 --- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties index 00f433d..8903487 100644 --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@ -331,7 +331,7 @@ kylin.engine.spark-conf.spark.yarn.queue=default kylin.engine.spark-conf.spark.executor.memory=4G kylin.engine.spark-conf.spark.executor.instances=40 kylin.engine.spark-conf.spark.driver.cores=1 -kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=1024 +kylin.engine.spark-conf.spark.executor.memoryOverhead=1024 kylin.engine.spark-conf.spark.shuffle.service.enabled=true kylin.engine.spark-conf.spark.eventLog.enabled=true kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 28b61cf..40ae6e1 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -196,8 +196,8 @@ kylin.engine.spark.rdd-partition-cut-mb=100 kylin.engine.spark-conf.spark.yarn.submit.file.replication=1 kylin.engine.spark-conf.spark.master=yarn #kylin.engine.spark-conf.spark.submit.deployMode=cluster -kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=384 -kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=256 +kylin.engine.spark-conf.spark.executor.memoryOverhead=384 +kylin.engine.spark-conf.spark.driver.memoryOverhead=256 kylin.engine.spark-conf.spark.executor.memory=768M kylin.engine.spark-conf.spark.executor.cores=1 kylin.engine.spark-conf.spark.executor.instances=1 diff --git a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java index 96de267..4e0be04 100644 --- a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java +++ b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepFactory.java @@ -20,11 +20,7 @@ package org.apache.kylin.engine.spark.job; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; import org.apache.kylin.job.execution.DefaultChainedExecutable; -import org.apache.kylin.metadata.MetadataConstants; -import org.apache.kylin.metadata.model.Segments; public class JobStepFactory { diff --git a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java index 14a424f..f788d15 100644 --- a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java +++ b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/job/JobStepFactoryTest.java @@ -35,10 +35,7 @@ import org.junit.Test; import org.spark_project.guava.collect.Sets; import java.io.IOException; -import java.util.Arrays; -import java.util.List; import java.util.Set; -import java.util.stream.Collectors; public class JobStepFactoryTest extends LocalWithSparkSessionTest { private KylinConfig config;