This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit ee2bfd18a2d679a0f485bd0db9bac84f95604363 Author: Guoliang Sun <guoliang....@kyligence.io> AuthorDate: Tue Nov 15 11:24:39 2022 +0800 KYLIN-5398 Fix the problem of code override caused by revert --- dev-support/sandbox/conf/kylin.properties | 4 ++++ src/examples/test_case_data/sandbox/kylin.properties | 4 ++++ .../kylin/query/asyncprofiler/AsyncProfilingTest.scala | 2 ++ .../QueryAsyncProfilerDriverPluginTest.scala | 8 ++++---- .../kylin/common/asyncprofiler/AsyncArchUtil.java | 2 +- .../kylin/common/asyncprofiler/AsyncProfiler.java | 9 ++++----- .../async-profiler-lib/linux64/libasyncProfiler.so | Bin 314098 -> 0 bytes .../async-profiler-lib/macOS/libasyncProfiler.so | Bin 239460 -> 0 bytes .../kylin/common/asyncprofiler/AsyncArchUtilTest.java | 2 +- .../kylin/common/asyncprofiler/AsyncProfilerTest.java | 3 +-- .../BuildAsyncProfilerDriverPluginTest.scala | 4 ++-- 11 files changed, 23 insertions(+), 15 deletions(-) diff --git a/dev-support/sandbox/conf/kylin.properties b/dev-support/sandbox/conf/kylin.properties index 56d8d99730..7ca8e1ee4f 100644 --- a/dev-support/sandbox/conf/kylin.properties +++ b/dev-support/sandbox/conf/kylin.properties @@ -139,6 +139,10 @@ kylin.security.saml.context-path=/kylin kylin.influxdb.address=influxdb:8086 kylin.swagger.enabled=true +### Flame Graph +kylin.query.async-profiler-enabled=false +kylin.engine.async-profiler-enabled=false + # for tests kylin.test.bcc.new-key=some-value kylin.engine.mr.config-override.test1=test1 diff --git a/src/examples/test_case_data/sandbox/kylin.properties b/src/examples/test_case_data/sandbox/kylin.properties index 73426da7ae..5fe04b2fe6 100644 --- a/src/examples/test_case_data/sandbox/kylin.properties +++ b/src/examples/test_case_data/sandbox/kylin.properties @@ -127,6 +127,10 @@ kylin.security.saml.context-path=/kylin kylin.influxdb.address=sandbox:8086 kylin.swagger.enabled=true +### Flame Graph +kylin.query.async-profiler-enabled=false +kylin.engine.async-profiler-enabled=false + # for tests kylin.test.bcc.new-key=some-value kylin.engine.mr.config-override.test1=test1 diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala index 21c08c4ee7..33f497ee33 100644 --- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala +++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala @@ -40,7 +40,9 @@ class AsyncProfilingTest extends AsyncPluginWithMeta { .setAppName(getClass.getName) .set(SparkLauncher.SPARK_MASTER, "local[1]") .set("spark.plugins", sparkPluginName) + .set("spark.submit.deployMode", "client") + System.setProperty("spark.profiler.flagsDir", flagFileDir) sc = new SparkContext(conf) } diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala index 8d82c3c684..44c564b3ba 100644 --- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala +++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala @@ -37,12 +37,12 @@ class QueryAsyncProfilerDriverPluginTest extends AsyncPluginWithMeta { } test("plugin initialization") { - Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins")) + Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString) new QueryAsyncProfilerDriverPlugin().receive("NEX-1:start,event=cpu") } test("plugin initialization receive result") { - Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins")) + Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString) try { new QueryAsyncProfilerDriverPlugin().receive("RES-1:flamegraph") } catch { @@ -51,11 +51,11 @@ class QueryAsyncProfilerDriverPluginTest extends AsyncPluginWithMeta { } test("plugin initialization receive others") { - Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins")) + Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString) try { new QueryAsyncProfilerDriverPlugin().receive("OTH-1:start,event=cpu") } catch { case _: Throwable => } } -} +} \ No newline at end of file diff --git a/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java index a1b77ab622..17b9904ff1 100644 --- a/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java +++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java @@ -74,4 +74,4 @@ public class AsyncArchUtil { private static void addProcessors(ArchType archType, final String... keys) { Stream.of(keys).forEach(key -> ARCH_TO_PROCESSOR.put(key, archType)); } -} \ No newline at end of file +} diff --git a/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java index 1c0b3a795e..4f7ab73de6 100644 --- a/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java +++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java @@ -18,6 +18,10 @@ package org.apache.kylin.common.asyncprofiler; +import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_ARM64; +import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_X64; +import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_MAC; + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -31,12 +35,7 @@ public class AsyncProfiler { private static final Logger logger = LoggerFactory.getLogger(AsyncProfiler.class); - // async profiler native files - public static final String ASYNC_PROFILER_LIB_MAC = "libasyncProfiler-mac.so"; - public static final String ASYNC_PROFILER_LIB_LINUX_X64 = "libasyncProfiler-linux-x64.so"; - public static final String ASYNC_PROFILER_LIB_LINUX_ARM64 = "libasyncProfiler-linux-arm64.so"; private static final String LIB_PARENT = "/async-profiler-lib/"; - private static AsyncProfiler profiler; private boolean loaded = false; diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so deleted file mode 100755 index 4153f52868..0000000000 Binary files a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so and /dev/null differ diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so deleted file mode 100755 index c7298c6b82..0000000000 Binary files a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so and /dev/null differ diff --git a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java index 0ba16c4a75..b2a3a26044 100644 --- a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java +++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java @@ -37,4 +37,4 @@ public class AsyncArchUtilTest { archType = AsyncArchUtil.getProcessor("aarch64"); Assert.assertEquals(AsyncArchUtil.ArchType.LINUX_ARM64, archType); } -} \ No newline at end of file +} diff --git a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java index c819afc4f5..42cebf7eef 100644 --- a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java +++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java @@ -66,5 +66,4 @@ public class AsyncProfilerTest { Assert.assertThrows("Profiler is not active", IllegalStateException.class, AsyncProfiler.getInstance(true)::stop); } - -} +} \ No newline at end of file diff --git a/src/spark-project/spark-common/src/test/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPluginTest.scala b/src/spark-project/spark-common/src/test/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPluginTest.scala index 6a23c9e154..63fd2ffbfe 100644 --- a/src/spark-project/spark-common/src/test/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPluginTest.scala +++ b/src/spark-project/spark-common/src/test/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPluginTest.scala @@ -75,7 +75,7 @@ class BuildAsyncProfilerDriverPluginTest extends SparkFunSuite with BeforeAndAft .set("spark.plugins", sparkPluginName) sc = new SparkContext(conf) - Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins")) + Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString) sc.stop() sc = null @@ -89,7 +89,7 @@ class BuildAsyncProfilerDriverPluginTest extends SparkFunSuite with BeforeAndAft System.setProperty("spark.profiler.flagsDir", flagFileDir) sc = new SparkContext(conf) - Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins")) + Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString) sc.stop() sc = null