This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 28f9cfd364 [GLUTEN-8655][CH] Refactor: remove clickhouse.lib.path
(#8656)
28f9cfd364 is described below
commit 28f9cfd364856eb6b76d9b4be753baaf402bfbef
Author: Chang chen <[email protected]>
AuthorDate: Wed Feb 5 16:54:29 2025 +0800
[GLUTEN-8655][CH] Refactor: remove clickhouse.lib.path (#8656)
* remove SystemParameters.getClickHouseLibPath
* remove SystemParameters.getClickHouseLibPath - 3.2
* remove SystemParameters.getClickHouseLibPath - 3.3
* remove UTSystemParameters.clickHouseLibPath
* remove SystemParameters
* remvoe clickhouse.lib.path
* remove tpcds.data.path
* Fix
* Revert "Fix"
This reverts commit cd006817e40f47497833f99f1de96f80d56861f9.
* fix spark34
* Simplify the testing directory
* fix doc
* remove unnecessary test data
---
backends-clickhouse/pom.xml | 6 ----
.../execution/GlutenClickHouseJoinSuite.scala | 3 --
.../GlutenClickHouseNativeExceptionSuite.scala | 10 +-----
.../execution/GlutenClickHouseNativeLibSuite.scala | 2 --
.../GlutenClickHouseSyntheticDataSuite.scala | 2 --
.../GlutenClickHouseTPCDSAbstractSuite.scala | 1 -
.../GlutenClickHouseTPCHAbstractSuite.scala | 2 --
...lutenClickHouseWholeStageTransformerSuite.scala | 2 --
.../execution/GlutenFunctionValidateSuite.scala | 4 ---
.../GlutenClickhouseFunctionSuite.scala | 2 --
.../hive/GlutenClickHouseHiveTableSuite.scala | 3 --
.../GlutenClickHouseNativeWriteTableSuite.scala | 2 --
.../apache/gluten/utils/UTSystemParameters.scala | 30 ++--------------
.../execution/benchmarks/CHSqlBasedBenchmark.scala | 6 +---
.../tests/data/all_meta/data/f0/._SUCCESS.crc | Bin 8 -> 0 bytes
...-4f9b-8bdd-a6911d0d465a-c000.snappy.parquet.crc | Bin 20 -> 0 bytes
.../tests/data/all_meta/data/f0/_SUCCESS | 0
...7446-4f9b-8bdd-a6911d0d465a-c000.snappy.parquet | Bin 1282 -> 0 bytes
.../tests/data/all_meta/data/f1/._SUCCESS.crc | Bin 8 -> 0 bytes
...-495b-9cf1-aa914dc7cd57-c000.snappy.parquet.crc | Bin 20 -> 0 bytes
.../tests/data/all_meta/data/f1/_SUCCESS | 0
...a1e2-495b-9cf1-aa914dc7cd57-c000.snappy.parquet | Bin 1282 -> 0 bytes
docs/developers/clickhouse-backend-debug.md | 2 +-
gluten-ut/common/pom.xml | 6 ----
.../org/apache/gluten/utils/SystemParameters.scala | 38 ---------------------
.../spark/sql/DummyFilterColmnarHelper.scala | 4 +--
.../apache/spark/sql/GlutenSQLTestsBaseTrait.scala | 4 +--
.../org/apache/spark/sql/GlutenTestsTrait.scala | 3 +-
gluten-ut/spark32/pom.xml | 9 -----
.../apache/spark/sql/GlutenSQLQueryTestSuite.scala | 4 +--
.../benchmarks/ParquetReadBenchmark.scala | 7 ++--
.../execution/joins/GlutenBroadcastJoinSuite.scala | 4 +--
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 4 ---
.../sql/statistics/SparkFunctionStatistics.scala | 4 +--
gluten-ut/spark33/pom.xml | 9 -----
.../apache/spark/sql/GlutenSQLQueryTestSuite.scala | 4 +--
.../benchmarks/ParquetReadBenchmark.scala | 7 ++--
.../execution/joins/GlutenBroadcastJoinSuite.scala | 4 +--
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 4 ---
.../sql/statistics/SparkFunctionStatistics.scala | 4 +--
gluten-ut/spark34/pom.xml | 7 ----
.../apache/spark/sql/GlutenSQLQueryTestSuite.scala | 4 +--
.../benchmarks/ParquetReadBenchmark.scala | 7 ++--
.../execution/joins/GlutenBroadcastJoinSuite.scala | 4 +--
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 4 ---
.../sql/statistics/SparkFunctionStatistics.scala | 4 +--
gluten-ut/spark35/pom.xml | 7 ----
.../apache/spark/sql/GlutenSQLQueryTestSuite.scala | 4 +--
.../benchmarks/ParquetReadBenchmark.scala | 7 ++--
.../execution/joins/GlutenBroadcastJoinSuite.scala | 4 +--
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 4 ---
.../sql/statistics/SparkFunctionStatistics.scala | 4 +--
gluten-ut/test/pom.xml | 9 -----
.../MergeTwoPhasesHashBaseAggregateSuite.scala | 4 +--
.../expressions/GlutenExpressionMappingSuite.scala | 3 +-
.../GlutenExtensionRewriteRuleSuite.scala | 4 +--
.../org/apache/gluten/sql/SQLQuerySuite.scala | 2 --
.../sql/GlutenExpressionDataTypesValidation.scala | 4 +--
.../datasources/GlutenNoopWriterRuleSuite.scala | 9 +----
pom.xml | 2 --
60 files changed, 33 insertions(+), 260 deletions(-)
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
index 52c88354a1..9b954c2ccf 100644
--- a/backends-clickhouse/pom.xml
+++ b/backends-clickhouse/pom.xml
@@ -490,12 +490,6 @@
<goals>
<goal>test</goal>
</goals>
- <configuration>
- <systemProperties>
-
<clickhouse.lib.path>${clickhouse.lib.path}</clickhouse.lib.path>
- <tpcds.data.path>${tpcds.data.path}</tpcds.data.path>
- </systemProperties>
- </configuration>
</execution>
</executions>
</plugin>
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala
index 28aad55379..efde239f43 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala
@@ -17,8 +17,6 @@
package org.apache.gluten.execution
import org.apache.gluten.backendsapi.clickhouse.CHConf
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
@@ -43,7 +41,6 @@ class GlutenClickHouseJoinSuite extends
GlutenClickHouseWholeStageTransformerSui
.set("spark.sql.adaptive.enabled", "false")
.set("spark.sql.files.minPartitionNum", "1")
.set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set("spark.gluten.sql.enable.native.validation", "false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala
index 9b617b998c..394895fdc8 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala
@@ -16,18 +16,10 @@
*/
package org.apache.gluten.execution
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{TestExceptionUtils, UTSystemParameters}
-
-import org.apache.spark.SparkConf
+import org.apache.gluten.utils.TestExceptionUtils
class GlutenClickHouseNativeExceptionSuite extends
GlutenClickHouseWholeStageTransformerSuite {
- override protected def sparkConf: SparkConf = {
- super.sparkConf
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
- }
-
test("native exception caught by jvm") {
try {
TestExceptionUtils.generateNativeException()
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala
index 98d8fb9ee0..f91e841942 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala
@@ -18,7 +18,6 @@ package org.apache.gluten.execution
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
@@ -65,7 +64,6 @@ class GlutenClickHouseNativeLibSuite extends PlanTest {
.builder()
.master("local[1]")
.config(baseSparkConf)
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.config(GlutenConfig.GLUTEN_EXECUTOR_LIB_PATH.key,
"/path/not/exist/libch.so")
.getOrCreate()
spark.sql("select 1").show()
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala
index 6fffe30f05..25244c5861 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala
@@ -17,7 +17,6 @@
package org.apache.gluten.execution
import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
@@ -52,7 +51,6 @@ class GlutenClickHouseSyntheticDataSuite
.set("spark.databricks.delta.properties.defaults.checkpointInterval",
"5")
.set("spark.databricks.delta.stalenessLimit", "3600000")
.set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set("spark.gluten.sql.enable.native.validation", "false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala
index f1f93692ec..d15ac1334c 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala
@@ -132,7 +132,6 @@ abstract class GlutenClickHouseTPCDSAbstractSuite
.set("spark.databricks.delta.properties.defaults.checkpointInterval",
"5")
.set("spark.databricks.delta.stalenessLimit", "3600000")
.set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set("spark.gluten.sql.enable.native.validation", "false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala
index 4c748fcb1d..6a7b35337d 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala
@@ -17,7 +17,6 @@
package org.apache.gluten.execution
import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
@@ -568,7 +567,6 @@ abstract class GlutenClickHouseTPCHAbstractSuite
.set("spark.databricks.delta.stalenessLimit", "3600000")
.set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
.set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set("spark.gluten.sql.enable.native.validation", "false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala
index db5fede0d5..24efab5343 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala
@@ -17,7 +17,6 @@
package org.apache.gluten.execution
import org.apache.gluten.backendsapi.clickhouse.RuntimeConfig
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf}
@@ -79,7 +78,6 @@ class GlutenClickHouseWholeStageTransformerSuite extends
WholeStageTransformerSu
import org.apache.gluten.backendsapi.clickhouse.CHConf._
val conf = super.sparkConf
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.sql.warehouse.dir", warehouse)
.setCHConfig("user_defined_path", "/tmp/user_defined")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
index 5923f1484a..e14666976e 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
@@ -16,9 +16,6 @@
*/
package org.apache.gluten.execution
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.UTSystemParameters
-
import org.apache.spark.SparkConf
import org.apache.spark.sql.{DataFrame, Row, TestUtils}
import org.apache.spark.sql.catalyst.expressions.{Expression, GetJsonObject,
Literal}
@@ -57,7 +54,6 @@ class GlutenFunctionValidateSuite extends
GlutenClickHouseWholeStageTransformerS
.set("spark.databricks.delta.properties.defaults.checkpointInterval",
"5")
.set("spark.databricks.delta.stalenessLimit", "3600000")
.set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set("spark.gluten.sql.enable.native.validation", "false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
index a8ffc90dc3..faccfa105c 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
@@ -18,7 +18,6 @@ package org.apache.gluten.execution.compatibility
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{GlutenClickHouseTPCHAbstractSuite,
ProjectExecTransformer}
-import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import
org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig
@@ -51,7 +50,6 @@ class GlutenClickhouseFunctionSuite extends
GlutenClickHouseTPCHAbstractSuite {
.set("spark.databricks.delta.properties.defaults.checkpointInterval",
"5")
.set("spark.databricks.delta.stalenessLimit", "3600000")
.set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set("spark.gluten.sql.enable.native.validation", "false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
index 917317bcc6..dddeb43d6a 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
@@ -16,10 +16,8 @@
*/
package org.apache.gluten.execution.hive
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
GlutenClickHouseWholeStageTransformerSuite, ProjectExecTransformer,
TransformSupport}
import org.apache.gluten.test.AllDataTypesWithComplexType
-import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.sql.{DataFrame, SaveMode}
@@ -56,7 +54,6 @@ class GlutenClickHouseHiveTableSuite
.set("spark.sql.adaptive.enabled", "false")
.set("spark.sql.files.minPartitionNum", "1")
.set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set("spark.gluten.sql.enable.native.validation", "false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala
index b6067f138d..1ee0b18b11 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala
@@ -20,7 +20,6 @@ import org.apache.gluten.backendsapi.clickhouse.RuntimeConfig
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.GlutenClickHouseWholeStageTransformerSuite
import org.apache.gluten.test.AllDataTypesWithComplexType.genTestData
-import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.gluten.NativeWriteChecker
@@ -57,7 +56,6 @@ class GlutenClickHouseNativeWriteTableSuite
.set("spark.databricks.delta.properties.defaults.checkpointInterval",
"5")
.set("spark.databricks.delta.stalenessLimit", "3600000")
.set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.set("spark.gluten.sql.columnar.iterator", "true")
.set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
.set("spark.gluten.sql.enable.native.validation", "false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/utils/UTSystemParameters.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/utils/UTSystemParameters.scala
index 334f9fb4be..31b5514cce 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/utils/UTSystemParameters.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/utils/UTSystemParameters.scala
@@ -18,15 +18,6 @@ package org.apache.gluten.utils
object UTSystemParameters {
- private val CLICKHOUSE_LIB_PATH_KEY = "clickhouse.lib.path"
- private val CLICKHOUSE_LIB_PATH_DEFAULT_VALUE =
"/usr/local/clickhouse/lib/libch.so"
-
- def clickHouseLibPath: String = {
- System.getProperty(
- UTSystemParameters.CLICKHOUSE_LIB_PATH_KEY,
- UTSystemParameters.CLICKHOUSE_LIB_PATH_DEFAULT_VALUE)
- }
-
private val TEST_DATA_PATH_KEY = "gluten.test.data.path"
private val TEST_DATA_PATH_DEFAULT_VALUE = "/data"
@@ -43,29 +34,12 @@ object UTSystemParameters {
UTSystemParameters.TEST_DATA_DISK_OUTPUT_KEY,
UTSystemParameters.TEST_DATA_DISK_OUTPUT_DEFAULT_VALUE)
}
-
- private val TPCDS_DATA_PATH_KEY = "tpcds.data.path"
- private val TPCDS_DATA_DECIMAL_PATH_KEY = "tpcds.data.decimal.path"
private val TPCDS_RELATIVE_DATA_PATH = "tpcds-data-sf1"
private val TPCDS_DECIMAL_RELATIVE_DATA_PATH = "tpcds-data-sf1-decimal"
- def tpcdsDataPath: String = {
- val result = System.getProperty(UTSystemParameters.TPCDS_DATA_PATH_KEY,
null)
- if (result == null) {
- s"$testDataPath/$TPCDS_RELATIVE_DATA_PATH"
- } else {
- result
- }
- }
+ def tpcdsDataPath: String = s"$testDataPath/$TPCDS_RELATIVE_DATA_PATH"
- def tpcdsDecimalDataPath: String = {
- val result =
System.getProperty(UTSystemParameters.TPCDS_DATA_DECIMAL_PATH_KEY, null)
- if (result == null) {
- s"$testDataPath/$TPCDS_DECIMAL_RELATIVE_DATA_PATH"
- } else {
- result
- }
- }
+ def tpcdsDecimalDataPath: String =
s"$testDataPath/$TPCDS_DECIMAL_RELATIVE_DATA_PATH"
private val TEST_MERGETREE_ON_OBJECT_STORAGE =
"gluten.ch.test.mergetree.object.storage"
private val TEST_MERGETREE_ON_OBJECT_STORAGE_DEFAULT_VALUE = "true"
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
index 7b067643f6..2a9a2b233a 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
@@ -18,7 +18,6 @@ package org.apache.spark.sql.execution.benchmarks
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.jni.JniLibLoader
-import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.sql.delta.DeltaLog
@@ -32,7 +31,6 @@ trait CHSqlBasedBenchmark extends SqlBasedBenchmark {
def getSparkConf: SparkConf = {
val conf = new SparkConf()
.setAppName(appName)
- .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
.setIfMissing("spark.master", s"local[$thrdNum]")
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
.set(
@@ -57,9 +55,7 @@ trait CHSqlBasedBenchmark extends SqlBasedBenchmark {
override def afterAll(): Unit = {
DeltaLog.clearCache()
- val libPath =
- spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key,
UTSystemParameters.clickHouseLibPath)
- JniLibLoader.unloadFromPath(libPath)
+
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
// Wait for Ctrl+C, convenient for seeing Spark UI
// Thread.sleep(600000)
super.afterAll()
diff --git a/cpp-ch/local-engine/tests/data/all_meta/data/f0/._SUCCESS.crc
b/cpp-ch/local-engine/tests/data/all_meta/data/f0/._SUCCESS.crc
deleted file mode 100644
index 3b7b044936..0000000000
Binary files a/cpp-ch/local-engine/tests/data/all_meta/data/f0/._SUCCESS.crc
and /dev/null differ
diff --git
a/cpp-ch/local-engine/tests/data/all_meta/data/f0/.part-00000-92bb25d0-7446-4f9b-8bdd-a6911d0d465a-c000.snappy.parquet.crc
b/cpp-ch/local-engine/tests/data/all_meta/data/f0/.part-00000-92bb25d0-7446-4f9b-8bdd-a6911d0d465a-c000.snappy.parquet.crc
deleted file mode 100644
index a4db2dd16f..0000000000
Binary files
a/cpp-ch/local-engine/tests/data/all_meta/data/f0/.part-00000-92bb25d0-7446-4f9b-8bdd-a6911d0d465a-c000.snappy.parquet.crc
and /dev/null differ
diff --git a/cpp-ch/local-engine/tests/data/all_meta/data/f0/_SUCCESS
b/cpp-ch/local-engine/tests/data/all_meta/data/f0/_SUCCESS
deleted file mode 100644
index e69de29bb2..0000000000
diff --git
a/cpp-ch/local-engine/tests/data/all_meta/data/f0/part-00000-92bb25d0-7446-4f9b-8bdd-a6911d0d465a-c000.snappy.parquet
b/cpp-ch/local-engine/tests/data/all_meta/data/f0/part-00000-92bb25d0-7446-4f9b-8bdd-a6911d0d465a-c000.snappy.parquet
deleted file mode 100644
index 1e702dce83..0000000000
Binary files
a/cpp-ch/local-engine/tests/data/all_meta/data/f0/part-00000-92bb25d0-7446-4f9b-8bdd-a6911d0d465a-c000.snappy.parquet
and /dev/null differ
diff --git a/cpp-ch/local-engine/tests/data/all_meta/data/f1/._SUCCESS.crc
b/cpp-ch/local-engine/tests/data/all_meta/data/f1/._SUCCESS.crc
deleted file mode 100644
index 3b7b044936..0000000000
Binary files a/cpp-ch/local-engine/tests/data/all_meta/data/f1/._SUCCESS.crc
and /dev/null differ
diff --git
a/cpp-ch/local-engine/tests/data/all_meta/data/f1/.part-00000-963c0854-a1e2-495b-9cf1-aa914dc7cd57-c000.snappy.parquet.crc
b/cpp-ch/local-engine/tests/data/all_meta/data/f1/.part-00000-963c0854-a1e2-495b-9cf1-aa914dc7cd57-c000.snappy.parquet.crc
deleted file mode 100644
index 98fb620441..0000000000
Binary files
a/cpp-ch/local-engine/tests/data/all_meta/data/f1/.part-00000-963c0854-a1e2-495b-9cf1-aa914dc7cd57-c000.snappy.parquet.crc
and /dev/null differ
diff --git a/cpp-ch/local-engine/tests/data/all_meta/data/f1/_SUCCESS
b/cpp-ch/local-engine/tests/data/all_meta/data/f1/_SUCCESS
deleted file mode 100644
index e69de29bb2..0000000000
diff --git
a/cpp-ch/local-engine/tests/data/all_meta/data/f1/part-00000-963c0854-a1e2-495b-9cf1-aa914dc7cd57-c000.snappy.parquet
b/cpp-ch/local-engine/tests/data/all_meta/data/f1/part-00000-963c0854-a1e2-495b-9cf1-aa914dc7cd57-c000.snappy.parquet
deleted file mode 100644
index 4cdaff7168..0000000000
Binary files
a/cpp-ch/local-engine/tests/data/all_meta/data/f1/part-00000-963c0854-a1e2-495b-9cf1-aa914dc7cd57-c000.snappy.parquet
and /dev/null differ
diff --git a/docs/developers/clickhouse-backend-debug.md
b/docs/developers/clickhouse-backend-debug.md
index 6f27b25a09..8fee0eab5d 100644
--- a/docs/developers/clickhouse-backend-debug.md
+++ b/docs/developers/clickhouse-backend-debug.md
@@ -31,7 +31,7 @@ parent: /developer-overview/
VM Options:
- `-Dtpcds.data.path=/data/tpcds-data-sf1
-Dclickhouse.lib.path=/path/to/gluten/cpp-ch/build/utils/extern-local-engine/libch.so
-Dspark.test.home=/path/to/spark33`
+ `-Dgluten.test.data.path=/data
-Dspark.gluten.sql.columnar.libpath=/path/to/gluten/cpp-ch/build/utils/extern-local-engine/libch.so
-Dspark.test.home=/path/to/spark33`
> Download tpcds-data in https://gluten-nginx.kyligence.com/dataset/
> Download spark33 using `git clone --depth 1 --branch v3.3.1
https://github.com/apache/spark.git /tmp/spark33`
diff --git a/gluten-ut/common/pom.xml b/gluten-ut/common/pom.xml
index 11caaa4dd4..5d6dd94b43 100644
--- a/gluten-ut/common/pom.xml
+++ b/gluten-ut/common/pom.xml
@@ -69,12 +69,6 @@
<goals>
<goal>test</goal>
</goals>
- <configuration>
- <systemProperties>
-
<clickhouse.lib.path>${clickhouse.lib.path}</clickhouse.lib.path>
- <tpcds.data.path>${tpcds.data.path}</tpcds.data.path>
- </systemProperties>
- </configuration>
</execution>
</executions>
</plugin>
diff --git
a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/SystemParameters.scala
b/gluten-ut/common/src/test/scala/org/apache/gluten/utils/SystemParameters.scala
deleted file mode 100644
index 71fc26227b..0000000000
---
a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/SystemParameters.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.utils
-
-object SystemParameters {
-
- val CLICKHOUSE_LIB_PATH_KEY = "clickhouse.lib.path"
- val CLICKHOUSE_LIB_PATH_DEFAULT_VALUE = "/usr/local/clickhouse/lib/libch.so"
-
- val TPCDS_DATA_PATH_KEY = "tpcds.data.path"
- val TPCDS_DATA_PATH_DEFAULT_VALUE = "/data/tpcds-data-sf1"
-
- def getClickHouseLibPath: String = {
- System.getProperty(
- SystemParameters.CLICKHOUSE_LIB_PATH_KEY,
- SystemParameters.CLICKHOUSE_LIB_PATH_DEFAULT_VALUE)
- }
-
- def getTpcdsDataPath: String = {
- System.getProperty(
- SystemParameters.TPCDS_DATA_PATH_KEY,
- SystemParameters.TPCDS_DATA_PATH_DEFAULT_VALUE)
- }
-}
diff --git
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala
index ab4c6de441..41c652fc60 100644
---
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala
+++
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala
@@ -16,8 +16,7 @@
*/
package org.apache.spark.sql
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
@@ -86,7 +85,6 @@ object DummyFilterColmnarHelper {
.config("spark.memory.offHeap.size", "1024MB")
.config("spark.plugins", "org.apache.gluten.GlutenPlugin")
.config("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.io.compression.codec", "LZ4")
.config("spark.gluten.sql.enable.native.validation", "false")
} else {
diff --git
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala
index 128ce144f2..b6ece98e4a 100644
---
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala
+++
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala
@@ -16,8 +16,7 @@
*/
package org.apache.spark.sql
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.sql.execution.SparkPlan
@@ -105,7 +104,6 @@ object GlutenSQLTestsBaseTrait {
.set("spark.io.compression.codec", "LZ4")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.sql.files.openCostInBytes", "134217728")
.set("spark.unsafe.exceptionOnMemoryLeak", "true")
} else {
diff --git
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala
index 330bf3b7ac..5e0686095c 100644
---
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala
+++
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.ProjectExecTransformer
import org.apache.gluten.test.TestStats
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.GlutenQueryTestUtil.isNaNOrInf
import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
@@ -113,7 +113,6 @@ trait GlutenTestsTrait extends GlutenTestsCommonTrait {
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.config(GlutenConfig.UT_STATISTIC.key, "true")
.getOrCreate()
diff --git a/gluten-ut/spark32/pom.xml b/gluten-ut/spark32/pom.xml
index 8d0d0f4fca..15a56e009f 100644
--- a/gluten-ut/spark32/pom.xml
+++ b/gluten-ut/spark32/pom.xml
@@ -58,9 +58,6 @@
<scope>test</scope>
</dependency>
</dependencies>
- <properties>
- <clickhouse.lib.path></clickhouse.lib.path>
- </properties>
</profile>
</profiles>
@@ -105,12 +102,6 @@
<goals>
<goal>test</goal>
</goals>
- <configuration>
- <systemProperties>
-
<clickhouse.lib.path>${clickhouse.lib.path}</clickhouse.lib.path>
- <tpcds.data.path>${tpcds.data.path}</tpcds.data.path>
- </systemProperties>
- </configuration>
</execution>
</executions>
</plugin>
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
index 739446111b..7a4b1c7816 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils,
SystemParameters}
+import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils}
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
@@ -195,7 +194,6 @@ class GlutenSQLQueryTestSuite
.set("spark.io.compression.codec", "LZ4")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.sql.files.openCostInBytes", "134217728")
.set("spark.unsafe.exceptionOnMemoryLeak", "true")
} else {
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index c55f4bf7f0..aa561f17f0 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformer}
import org.apache.gluten.extension.columnar.transition.Transitions
import org.apache.gluten.jni.JniLibLoader
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.benchmark.Benchmark
@@ -82,7 +82,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false")
- .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set(
"spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog")
@@ -225,9 +224,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
override def afterAll(): Unit = {
if (BackendTestUtils.isCHBackendLoaded()) {
- val libPath =
- spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
- JniLibLoader.unloadFromPath(libPath)
+
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
}
super.afterAll()
}
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
index 838d30e7ca..7d83405eba 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql.execution.joins
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{BroadcastHashJoinExecTransformerBase,
BroadcastNestedLoopJoinExecTransformer, ColumnarToRowExecBase,
WholeStageTransformer}
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession}
import org.apache.spark.sql.catalyst.optimizer._
@@ -89,7 +88,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite
with GlutenTestsCommon
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.getOrCreate()
} else {
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 0b14467aba..859e6df484 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,9 +16,6 @@
*/
package org.apache.spark.sql.hive.execution
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.SystemParameters
-
import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.gluten.sql.native.writer.enabled", "true")
.set("spark.sql.storeAssignmentPolicy", "legacy")
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
index be9063c839..925d60df8e 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql.statistics
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.GlutenPlan
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession}
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
@@ -64,7 +63,6 @@ class SparkFunctionStatistics extends QueryTest {
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.getOrCreate()
} else {
diff --git a/gluten-ut/spark33/pom.xml b/gluten-ut/spark33/pom.xml
index f8377195ee..93683220d3 100644
--- a/gluten-ut/spark33/pom.xml
+++ b/gluten-ut/spark33/pom.xml
@@ -87,9 +87,6 @@
<scope>test</scope>
</dependency>
</dependencies>
- <properties>
- <clickhouse.lib.path></clickhouse.lib.path>
- </properties>
</profile>
</profiles>
@@ -133,12 +130,6 @@
<goals>
<goal>test</goal>
</goals>
- <configuration>
- <systemProperties>
-
<clickhouse.lib.path>${clickhouse.lib.path}</clickhouse.lib.path>
- <tpcds.data.path>${tpcds.data.path}</tpcds.data.path>
- </systemProperties>
- </configuration>
</execution>
</executions>
</plugin>
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
index 84d677ef59..e336ca88a7 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils,
SystemParameters}
+import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils}
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
@@ -195,7 +194,6 @@ class GlutenSQLQueryTestSuite
.set("spark.io.compression.codec", "LZ4")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.sql.files.openCostInBytes", "134217728")
.set("spark.unsafe.exceptionOnMemoryLeak", "true")
} else {
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index aa5ee0a881..517f17bba9 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformer}
import org.apache.gluten.extension.columnar.transition.Transitions
import org.apache.gluten.jni.JniLibLoader
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.benchmark.Benchmark
@@ -82,7 +82,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false")
- .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set(
"spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog")
@@ -225,9 +224,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
override def afterAll(): Unit = {
if (BackendTestUtils.isCHBackendLoaded()) {
- val libPath =
- spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
- JniLibLoader.unloadFromPath(libPath)
+
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
}
super.afterAll()
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
index f5fd730796..c8b371f48f 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
@@ -16,8 +16,7 @@
*/
package org.apache.spark.sql.execution.joins
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession}
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding,
ConvertToLocalRelation, NullPropagation}
@@ -66,7 +65,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite
with GlutenTestsCommon
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.getOrCreate()
} else {
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 0b14467aba..859e6df484 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,9 +16,6 @@
*/
package org.apache.spark.sql.hive.execution
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.SystemParameters
-
import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.gluten.sql.native.writer.enabled", "true")
.set("spark.sql.storeAssignmentPolicy", "legacy")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
index 69969e948e..c349d9ca91 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql.statistics
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.GlutenPlan
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession}
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
@@ -64,7 +63,6 @@ class SparkFunctionStatistics extends QueryTest {
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.getOrCreate()
} else {
diff --git a/gluten-ut/spark34/pom.xml b/gluten-ut/spark34/pom.xml
index e61142b79d..e0e55c23e8 100644
--- a/gluten-ut/spark34/pom.xml
+++ b/gluten-ut/spark34/pom.xml
@@ -87,7 +87,6 @@
<properties>
<slf4j.version>2.0.6</slf4j.version>
<log4j.version>2.19.0</log4j.version>
- <clickhouse.lib.path></clickhouse.lib.path>
</properties>
</profile>
</profiles>
@@ -132,12 +131,6 @@
<goals>
<goal>test</goal>
</goals>
- <configuration>
- <systemProperties>
-
<clickhouse.lib.path>${clickhouse.lib.path}</clickhouse.lib.path>
- <tpcds.data.path>${tpcds.data.path}</tpcds.data.path>
- </systemProperties>
- </configuration>
</execution>
</executions>
</plugin>
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
index 7c0e9801c2..39a36898af 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils,
SystemParameters}
+import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils}
import org.apache.spark.{SparkConf, SparkException, SparkThrowable}
import org.apache.spark.ErrorMessageFormat.MINIMAL
@@ -199,7 +198,6 @@ class GlutenSQLQueryTestSuite
.set("spark.io.compression.codec", "LZ4")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.sql.files.openCostInBytes", "134217728")
.set("spark.unsafe.exceptionOnMemoryLeak", "true")
} else {
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index d065495b35..62eb3ec8a3 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -21,7 +21,7 @@ import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformer}
import org.apache.gluten.extension.columnar.transition.Transitions
import org.apache.gluten.jni.JniLibLoader
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.benchmark.Benchmark
@@ -83,7 +83,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false")
- .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set(
"spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog")
@@ -227,9 +226,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
override def afterAll(): Unit = {
if (BackendTestUtils.isCHBackendLoaded()) {
- val libPath =
- spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
- JniLibLoader.unloadFromPath(libPath)
+
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
}
super.afterAll()
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
index e9d5f9bbdc..6543160f1a 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
@@ -16,8 +16,7 @@
*/
package org.apache.spark.sql.execution.joins
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession}
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding,
ConvertToLocalRelation, NullPropagation}
@@ -64,7 +63,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite
with GlutenTestsCommon
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.getOrCreate()
} else {
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 0b14467aba..859e6df484 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,9 +16,6 @@
*/
package org.apache.spark.sql.hive.execution
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.SystemParameters
-
import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.gluten.sql.native.writer.enabled", "true")
.set("spark.sql.storeAssignmentPolicy", "legacy")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
index f483d714f1..f45b5c659c 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql.statistics
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.GlutenPlan
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession}
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding,
ConvertToLocalRelation, NullPropagation}
@@ -63,7 +62,6 @@ class SparkFunctionStatistics extends QueryTest {
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.getOrCreate()
} else {
diff --git a/gluten-ut/spark35/pom.xml b/gluten-ut/spark35/pom.xml
index d95e4aeeee..1ee63b95dc 100644
--- a/gluten-ut/spark35/pom.xml
+++ b/gluten-ut/spark35/pom.xml
@@ -148,7 +148,6 @@
<properties>
<slf4j.version>2.0.6</slf4j.version>
<log4j.version>2.19.0</log4j.version>
- <clickhouse.lib.path></clickhouse.lib.path>
</properties>
<build>
<plugins>
@@ -215,12 +214,6 @@
<goals>
<goal>test</goal>
</goals>
- <configuration>
- <systemProperties>
-
<clickhouse.lib.path>${clickhouse.lib.path}</clickhouse.lib.path>
- <tpcds.data.path>${tpcds.data.path}</tpcds.data.path>
- </systemProperties>
- </configuration>
</execution>
</executions>
</plugin>
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
index 3eebda83a0..01d36635f3 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils,
SystemParameters}
+import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils}
import org.apache.spark.{SparkConf, SparkException, SparkThrowable}
import org.apache.spark.ErrorMessageFormat.MINIMAL
@@ -197,7 +196,6 @@ class GlutenSQLQueryTestSuite
.set("spark.io.compression.codec", "LZ4")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.sql.files.openCostInBytes", "134217728")
.set("spark.unsafe.exceptionOnMemoryLeak", "true")
} else {
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index aa5ee0a881..517f17bba9 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformer}
import org.apache.gluten.extension.columnar.transition.Transitions
import org.apache.gluten.jni.JniLibLoader
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.benchmark.Benchmark
@@ -82,7 +82,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false")
- .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set(
"spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog")
@@ -225,9 +224,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
override def afterAll(): Unit = {
if (BackendTestUtils.isCHBackendLoaded()) {
- val libPath =
- spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
- JniLibLoader.unloadFromPath(libPath)
+
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
}
super.afterAll()
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
index f5fd730796..c8b371f48f 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
@@ -16,8 +16,7 @@
*/
package org.apache.spark.sql.execution.joins
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession}
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding,
ConvertToLocalRelation, NullPropagation}
@@ -66,7 +65,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite
with GlutenTestsCommon
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.getOrCreate()
} else {
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 0b14467aba..859e6df484 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,9 +16,6 @@
*/
package org.apache.spark.sql.hive.execution
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.SystemParameters
-
import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.set("spark.gluten.sql.enable.native.validation", "false")
.set("spark.gluten.sql.native.writer.enabled", "true")
.set("spark.sql.storeAssignmentPolicy", "legacy")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
index f483d714f1..f45b5c659c 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql.statistics
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.GlutenPlan
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession}
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding,
ConvertToLocalRelation, NullPropagation}
@@ -63,7 +62,6 @@ class SparkFunctionStatistics extends QueryTest {
.config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
.config("spark.gluten.sql.enable.native.validation", "false")
.config("spark.sql.files.openCostInBytes", "134217728")
- .config(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
.config("spark.unsafe.exceptionOnMemoryLeak", "true")
.getOrCreate()
} else {
diff --git a/gluten-ut/test/pom.xml b/gluten-ut/test/pom.xml
index a641a19b2f..d7d04ed5ed 100644
--- a/gluten-ut/test/pom.xml
+++ b/gluten-ut/test/pom.xml
@@ -97,9 +97,6 @@
<scope>test</scope>
</dependency>
</dependencies>
- <properties>
- <clickhouse.lib.path></clickhouse.lib.path>
- </properties>
</profile>
</profiles>
@@ -143,12 +140,6 @@
<goals>
<goal>test</goal>
</goals>
- <configuration>
- <systemProperties>
-
<clickhouse.lib.path>${clickhouse.lib.path}</clickhouse.lib.path>
- <tpcds.data.path>${tpcds.data.path}</tpcds.data.path>
- </systemProperties>
- </configuration>
</execution>
</executions>
</plugin>
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala
index 32eb10a32f..061f5660e3 100644
---
a/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala
@@ -16,8 +16,7 @@
*/
package org.apache.gluten.execution
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.sql.DataFrame
@@ -57,7 +56,6 @@ abstract class BaseMergeTwoPhasesHashBaseAggregateSuite
extends WholeStageTransf
if (BackendTestUtils.isCHBackendLoaded()) {
conf
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
}
conf
}
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
index 14a07aea5b..b2fbd11c0a 100644
---
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
@@ -19,7 +19,7 @@ package org.apache.gluten.expressions
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.ProjectExecTransformer
import org.apache.gluten.expression.ExpressionMappings
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.sql.{GlutenQueryTest, Row}
@@ -43,7 +43,6 @@ class GlutenExpressionMappingSuite
if (BackendTestUtils.isCHBackendLoaded()) {
conf
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
}
conf
}
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/extension/GlutenExtensionRewriteRuleSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/extension/GlutenExtensionRewriteRuleSuite.scala
index 0068e83a6d..a295508150 100644
---
a/gluten-ut/test/src/test/scala/org/apache/gluten/extension/GlutenExtensionRewriteRuleSuite.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/gluten/extension/GlutenExtensionRewriteRuleSuite.scala
@@ -16,9 +16,8 @@
*/
package org.apache.gluten.extension
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{ProjectExecTransformer,
WholeStageTransformerSuite}
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
@@ -33,7 +32,6 @@ class GlutenExtensionRewriteRuleSuite extends
WholeStageTransformerSuite {
if (BackendTestUtils.isCHBackendLoaded()) {
conf
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
}
conf
}
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala
index 74d0e44db7..c584852b91 100644
--- a/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala
+++ b/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala
@@ -19,7 +19,6 @@ package org.apache.gluten.sql
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.WholeStageTransformerSuite
import org.apache.gluten.utils.BackendTestUtils
-import org.apache.gluten.utils.SystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.sql.Row
@@ -41,7 +40,6 @@ class SQLQuerySuite extends WholeStageTransformerSuite {
if (BackendTestUtils.isCHBackendLoaded()) {
conf
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
}
conf
}
diff --git
a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
index adf72a3f63..9bea137037 100644
---
a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
@@ -16,9 +16,8 @@
*/
package org.apache.spark.sql
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{ProjectExecTransformer, TransformSupport,
WholeStageTransformerSuite}
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
@@ -46,7 +45,6 @@ class GlutenExpressionDataTypesValidation extends
WholeStageTransformerSuite {
if (BackendTestUtils.isCHBackendLoaded()) {
conf
.set("spark.gluten.sql.enable.native.validation", "false")
- .set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
}
conf
}
diff --git
a/gluten-ut/test/src/test/scala/org/apache/spark/sql/datasources/GlutenNoopWriterRuleSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/spark/sql/datasources/GlutenNoopWriterRuleSuite.scala
index b01b543006..b2a3a2ba8a 100644
---
a/gluten-ut/test/src/test/scala/org/apache/spark/sql/datasources/GlutenNoopWriterRuleSuite.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/spark/sql/datasources/GlutenNoopWriterRuleSuite.scala
@@ -16,9 +16,6 @@
*/
package org.apache.spark.sql.datasources
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
-
import org.apache.spark.SparkConf
import org.apache.spark.sql.{GlutenQueryTest, SaveMode}
import org.apache.spark.sql.execution.QueryExecution
@@ -29,17 +26,13 @@ import org.apache.spark.sql.util.QueryExecutionListener
class GlutenNoopWriterRuleSuite extends GlutenQueryTest with
SharedSparkSession {
override def sparkConf: SparkConf = {
- val conf = super.sparkConf
+ super.sparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
.set("spark.default.parallelism", "1")
.set("spark.memory.offHeap.enabled", "true")
.set("spark.memory.offHeap.size", "1024MB")
.set("spark.ui.enabled", "false")
.set("spark.gluten.ui.enabled", "false")
- if (BackendTestUtils.isCHBackendLoaded()) {
- conf.set(GlutenConfig.GLUTEN_LIB_PATH.key,
SystemParameters.getClickHouseLibPath)
- }
- conf
}
class WriterColumnarListener extends QueryExecutionListener {
diff --git a/pom.xml b/pom.xml
index 41dfc80381..3dd3e70897 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,8 +82,6 @@
<!-- To build built-in backend c++ codes -->
<scala.recompile.mode>all</scala.recompile.mode>
<!-- For unit tests -->
-
<clickhouse.lib.path>/usr/local/clickhouse/lib/libch.so</clickhouse.lib.path>
- <tpcds.data.path>/data/tpcds-data-sf1</tpcds.data.path>
<fasterxml.version>2.13.5</fasterxml.version>
<junit.version>4.13.1</junit.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
