This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new 284a77f [Improve] Enable gzip compression by default for StreamLoad
writes (#356)
284a77f is described below
commit 284a77fbdf574750d358dc7716537a37e63d9a22
Author: wudi <[email protected]>
AuthorDate: Tue Apr 7 10:28:51 2026 +0800
[Improve] Enable gzip compression by default for StreamLoad writes (#356)
Enable gzip compression by default for StreamLoad writes to reduce
network transfer and improve write performance. Arrow format is
excluded as binary columnar data has low compression benefit.
Users can disable compression by setting:
doris.sink.properties.compress_type=
Also fixes compress_type check from "gzip" to "gz" to match Doris
server-side expected value.
---
.../client/write/AbstractStreamLoadProcessor.java | 7 +-
.../doris/spark/sql/DorisAnySchemaITCase.scala | 4 ++
.../apache/doris/spark/sql/DorisWriterITCase.scala | 76 ++++++++++++++++++++++
3 files changed, 85 insertions(+), 2 deletions(-)
diff --git
a/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/write/AbstractStreamLoadProcessor.java
b/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/write/AbstractStreamLoadProcessor.java
index 361100b..1ee3711 100644
---
a/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/write/AbstractStreamLoadProcessor.java
+++
b/spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/write/AbstractStreamLoadProcessor.java
@@ -111,8 +111,11 @@ public abstract class AbstractStreamLoadProcessor<R>
extends DorisWriter<R> impl
// init stream load props
this.isTwoPhaseCommitEnabled =
config.getValue(DorisOptions.DORIS_SINK_ENABLE_2PC);
this.format = DataFormat.valueOf(properties.getOrDefault("format",
"csv").toUpperCase());
- this.isGzipCompressionEnabled =
- properties.containsKey("compress_type") &&
"gzip".equals(properties.get("compress_type"));
+ // enable gzip compression by default for non-arrow formats
+ if (!DataFormat.ARROW.equals(format)) {
+ properties.putIfAbsent("compress_type", "gz");
+ }
+ this.isGzipCompressionEnabled =
"gz".equals(properties.get("compress_type"));
if (properties.containsKey(GROUP_COMMIT)) {
String message = "";
if (isTwoPhaseCommitEnabled) {
diff --git
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisAnySchemaITCase.scala
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisAnySchemaITCase.scala
index 2d86f9e..084d408 100644
---
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisAnySchemaITCase.scala
+++
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisAnySchemaITCase.scala
@@ -96,6 +96,7 @@ class DorisAnySchemaITCase extends AbstractContainerTestBase {
.option("user", getDorisUsername)
.option("password", getDorisPassword)
.option("doris.sink.properties.format", "json")
+ .option("doris.sink.properties.compress_type", "")
.option("sink.batch.size", 2)
.option("sink.max-retries", 2)
.mode(SaveMode.Append)
@@ -131,6 +132,7 @@ class DorisAnySchemaITCase extends
AbstractContainerTestBase {
.option("user", getDorisUsername)
.option("password", getDorisPassword)
.option("doris.sink.properties.format", "json")
+ .option("doris.sink.properties.compress_type", "")
.option("doris.sink.properties.partial_columns", "true")
.option("doris.write.fields", "siteid,citycode,username,pv")
.option("sink.batch.size", 2)
@@ -280,6 +282,7 @@ class DorisAnySchemaITCase extends
AbstractContainerTestBase {
| "user"="${getDorisUsername}",
| "password"="${getDorisPassword}",
| "doris.sink.properties.format" = "json",
+ | "doris.sink.properties.compress_type" = "",
| "doris.sink.properties.partial_columns" = "true"
|)
|""".stripMargin)
@@ -336,6 +339,7 @@ class DorisAnySchemaITCase extends
AbstractContainerTestBase {
.option("user", getDorisUsername)
.option("password", getDorisPassword)
.option("doris.sink.properties.format", "json")
+ .option("doris.sink.properties.compress_type", "")
.option("doris.write.schemaless", "true")
.option("sink.batch.size", 2)
.option("sink.max-retries", 2)
diff --git
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisWriterITCase.scala
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisWriterITCase.scala
index e591c47..3ebf670 100644
---
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisWriterITCase.scala
+++
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisWriterITCase.scala
@@ -46,6 +46,8 @@ class DorisWriterITCase extends AbstractContainerTestBase {
val TABLE_JSON_TBL_ARROW: String = "tbl_json_tbl_arrow"
val TABLE_BITMAP_TBL: String = "tbl_write_tbl_bitmap"
val TABLE_UNICODE_COL: String = "tbl_unicode_col"
+ val TABLE_CSV_NO_COMPRESS: String = "tbl_csv_no_compress"
+ val TABLE_CSV_GZ_COMPRESS: String = "tbl_csv_gz_compress"
@Test
@throws[Exception]
@@ -209,6 +211,7 @@ class DorisWriterITCase extends AbstractContainerTestBase {
.option("password", getDorisPassword)
.option("sink.properties.read_json_by_line", "true")
.option("sink.properties.format", "json")
+ .option("sink.properties.compress_type", "")
.option("doris.sink.auto-redirect", "false")
.option("doris.sink.enable-2pc", "true")
.mode(SaveMode.Append)
@@ -280,6 +283,7 @@ class DorisWriterITCase extends AbstractContainerTestBase {
.option("password", getDorisPassword)
.option("sink.properties.read_json_by_line", "true")
.option("sink.properties.format", "json")
+ .option("sink.properties.compress_type", "")
.option("doris.sink.auto-redirect", "false")
.mode(SaveMode.Append)
.save()
@@ -431,6 +435,78 @@ class DorisWriterITCase extends AbstractContainerTestBase {
}
+ @Test
+ @throws[Exception]
+ def testSinkCsvGzCompression(): Unit = {
+ initializeTable(TABLE_CSV_GZ_COMPRESS, DataModel.DUPLICATE)
+ val session = SparkSession.builder().master("local[1]").getOrCreate()
+ try {
+ val df = session.createDataFrame(Seq(
+ ("doris_gz", 1),
+ ("spark_gz", 2)
+ )).toDF("name", "age")
+ df.write
+ .format("doris")
+ .option("doris.fenodes", getFenodes)
+ .option("doris.table.identifier", DATABASE + "." +
TABLE_CSV_GZ_COMPRESS)
+ .option("user", getDorisUsername)
+ .option("password", getDorisPassword)
+ .option("sink.properties.column_separator", ",")
+ .option("sink.properties.line_delimiter", "\n")
+ .option("sink.properties.format", "csv")
+ .option("sink.properties.compress_type", "gz")
+ .mode(SaveMode.Append)
+ .save()
+
+ Thread.sleep(15000)
+ val actual = ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection,
+ LOG,
+ String.format("select * from %s.%s", DATABASE, TABLE_CSV_GZ_COMPRESS),
+ 2)
+ val expected = util.Arrays.asList("doris_gz,1", "spark_gz,2")
+ checkResultInAnyOrder("testSinkCsvGzCompression", expected.toArray(),
actual.toArray)
+ } finally {
+ session.stop()
+ }
+ }
+
+ @Test
+ @throws[Exception]
+ def testSinkCsvNoCompression(): Unit = {
+ initializeTable(TABLE_CSV_NO_COMPRESS, DataModel.DUPLICATE)
+ val session = SparkSession.builder().master("local[1]").getOrCreate()
+ try {
+ val df = session.createDataFrame(Seq(
+ ("doris_no_compress", 1),
+ ("spark_no_compress", 2)
+ )).toDF("name", "age")
+ df.write
+ .format("doris")
+ .option("doris.fenodes", getFenodes)
+ .option("doris.table.identifier", DATABASE + "." +
TABLE_CSV_NO_COMPRESS)
+ .option("user", getDorisUsername)
+ .option("password", getDorisPassword)
+ .option("sink.properties.column_separator", ",")
+ .option("sink.properties.line_delimiter", "\n")
+ .option("sink.properties.format", "csv")
+ .option("sink.properties.compress_type", "")
+ .mode(SaveMode.Append)
+ .save()
+
+ Thread.sleep(15000)
+ val actual = ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection,
+ LOG,
+ String.format("select * from %s.%s", DATABASE, TABLE_CSV_NO_COMPRESS),
+ 2)
+ val expected = util.Arrays.asList("doris_no_compress,1",
"spark_no_compress,2")
+ checkResultInAnyOrder("testSinkCsvNoCompression", expected.toArray(),
actual.toArray)
+ } finally {
+ session.stop()
+ }
+ }
+
private def initializeTable(table: String, dataModel: DataModel): Unit = {
val max = if (DataModel.AGGREGATE == dataModel) "MAX" else ""
val morProps = if (!(DataModel.UNIQUE_MOR == dataModel)) "" else
",\"enable_unique_key_merge_on_write\" = \"false\""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]