This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 30499f977f [Feature][Connector-V2] Support opt-in ORC table
compression in Hive sink (#13085)
30499f977f is described below
commit 30499f977f99aed80853a72f5925b9e8d3b1132d
Author: taowenjun <[email protected]>
AuthorDate: Wed Nov 18 23:32:54 2026 +0800
[Feature][Connector-V2] Support opt-in ORC table compression in Hive sink
(#13085)
Co-authored-by: 陶文俊 <[email protected]>
---
docs/en/connectors/sink/Hive.md | 24 +++-
docs/zh/connectors/sink/Hive.md | 28 ++++-
.../seatunnel/hive/config/HiveSinkOptions.java | 8 ++
.../connectors/seatunnel/hive/sink/HiveSink.java | 46 ++++++++
.../seatunnel/hive/sink/HiveSinkFactory.java | 1 +
.../seatunnel/hive/sink/HiveSinkConfigTest.java | 128 ++++++++++++++++++++-
.../seatunnel/hive/sink/HiveSinkFactoryTest.java | 8 ++
.../seatunnel/hive/sink/HiveSinkOptionsTest.java | 10 ++
.../seatunnel/e2e/connector/hive/HiveIT.java | 78 +++++++++++++
.../test/resources/fake_to_hive_orc_inherited.conf | 59 ++++++++++
.../test/resources/fake_to_hive_orc_override.conf | 60 ++++++++++
11 files changed, 447 insertions(+), 3 deletions(-)
diff --git a/docs/en/connectors/sink/Hive.md b/docs/en/connectors/sink/Hive.md
index 3f5b5c5bae..70404eb98c 100644
--- a/docs/en/connectors/sink/Hive.md
+++ b/docs/en/connectors/sink/Hive.md
@@ -41,7 +41,8 @@ By default, we use 2PC commit to ensure `exactly-once`
|---------------------------------------|---------|----------|----------------|-------------|
| table_name | string | yes | -
| Target Hive table name, e.g. `db1.table1`. For multi-table mode, you can use
`${database_name}.${table_name}` and SeaTunnel will substitute the upstream
values. |
| metastore_uri | string | no | -
| Hive metastore URI. Required unless `hive.metastore.client.factory.class` is
configured through `hive_site_path`, `hive.hadoop.conf-path`, or
`hive.hadoop.conf`. Comma-separated values enable HA failover. |
-| compress_codec | string | no | none
| Output compression codec. `lzo` and `none` are supported. Parquet / ORC
auto-detect compression. |
+| compress_codec | string | no | none
| Explicit output compression codec. ORC supports `NONE`, `SNAPPY`, `ZLIB`,
`LZ4` and `LZO`. Defaults to `none`; see `use_table_compression` to inherit
compression from an existing ORC table. |
+| use_table_compression | boolean | no | false
| Inherit `orc.compress` from an existing ORC table when no explicit codec is
configured. |
| hdfs_site_path | string | no | -
| Local path of `hdfs-site.xml`. Deprecated for new jobs — prefer
`hive.hadoop.conf` or `hive.hadoop.conf-path`. |
| hive_site_path | string | no | -
| Local path of `hive-site.xml`. |
| hive.hadoop.conf | Map | no | -
| Inline Hadoop configuration properties. |
@@ -58,6 +59,27 @@ By default, we use 2PC commit to ensure `exactly-once`
| save_mode_create_template | string | no | -
| Custom DDL template used when auto-creating the target Hive table. Variables:
`${database}`, `${table}`, `${rowtype_fields}`, `${rowtype_partition_fields}`,
`${table_location}`, `${current_timestamp}`. |
| common-options | | no | -
| Sink plugin common parameters. See [Sink Common
Options](../common-options/sink-common-options.md). |
+### use_table_compression [boolean]
+
+Default: `false`. Enable this option to inherit `orc.compress` from the
metastore when preparing an **existing ORC table**. It does not change
automatic table creation, non-ORC tables, or table metadata.
+
+For non-ORC tables, `use_table_compression` is ignored: no table compression
property is inherited or validated. If the table does not exist when the sink
is initialized and follows the automatic table-creation path, inheritance is
also skipped, including `orc.compress` declared in `save_mode_create_template`.
In both cases, enabling this option alone does not cause the job to fail;
explicit `compress_codec` handling, existing defaults, and normal configuration
validation and table-creat [...]
+
+Priority: explicit `compress_codec` (including `NONE`) > table `orc.compress`
when enabled > the existing default `NONE`. Table values are trimmed and
matched without case sensitivity. Missing, null or blank properties retain the
default. Unknown values or codecs without a supported ORC mapping fail during
sink initialization instead of silently writing uncompressed files. In versions
where ORC ZSTD maps to NONE, inherited ZSTD is rejected; verify your SeaTunnel
version supports ORC ZSTD [...]
+
+For an existing table created with `TBLPROPERTIES ('orc.compress'='SNAPPY')`:
+
+```hocon
+sink {
+ Hive {
+ table_name = "default.existing_orc_table"
+ metastore_uri = "thrift://metastore:9083"
+ use_table_compression = true
+ # Optional: compress_codec = "NONE" overrides the table property.
+ }
+}
+```
+
### table_name [string]
Target Hive table name eg: db1.table1, and if the source is multiple mode, you
can use `${database_name}.${table_name}` to generate the table name, it will
replace the `${database_name}` and `${table_name}` with the value of the
CatalogTable generate from the source.
diff --git a/docs/zh/connectors/sink/Hive.md b/docs/zh/connectors/sink/Hive.md
index 4430753fa6..4efcc9db42 100644
--- a/docs/zh/connectors/sink/Hive.md
+++ b/docs/zh/connectors/sink/Hive.md
@@ -41,7 +41,8 @@ import ChangeLog from '../changelog/connector-hive.md';
|---------------------------------------|---------|----|----------------|---------------------------------------------------------------------------------|
| table_name | string | 是 | - | 目标
Hive 表名,例如 `db1.table1`。多表模式下可使用 `${database_name}.${table_name}`,SeaTunnel
会用上游值进行替换。 |
| metastore_uri | string | 否 | - |
Hive Metastore URI。当未通过 `hive_site_path`、`hive.hadoop.conf-path` 或
`hive.hadoop.conf` 配置 `hive.metastore.client.factory.class` 时必填。多个值用逗号分隔可启用 HA
故障转移。 |
-| compress_codec | string | 否 | none |
输出压缩编解码器。支持 `lzo` 和 `none`。Parquet / ORC 会自动检测压缩。 |
+| compress_codec | string | 否 | none |
显式指定输出压缩格式,优先于表属性。ORC 支持 `NONE`、`SNAPPY`、`ZLIB`、`LZ4` 和 `LZO`;继承表属性需开启
`use_table_compression`。 |
+| use_table_compression | boolean | 否 | false |
开启后继承已有 ORC 表的 `orc.compress` 属性;显式 `compress_codec` 优先。非 ORC 表及自动建表流程跳过继承。 |
| hdfs_site_path | string | 否 | - |
`hdfs-site.xml` 的本地路径。新作业不建议使用——优先使用 `hive.hadoop.conf` 或
`hive.hadoop.conf-path`。 |
| hive_site_path | string | 否 | - |
`hive-site.xml` 的本地路径。 |
| hive.hadoop.conf | Map | 否 | - |
内联的 Hadoop 配置属性。 |
@@ -58,6 +59,31 @@ import ChangeLog from '../changelog/connector-hive.md';
| save_mode_create_template | string | 否 | - |
自动创建目标 Hive 表时使用的自定义 DDL
模板。变量:`${database}`、`${table}`、`${rowtype_fields}`、`${rowtype_partition_fields}`、`${table_location}`、`${current_timestamp}`。
|
| common-options | | 否 | - |
Sink 插件通用参数,详见 [Sink 常用选项](../common-options/sink-common-options.md)。 |
+### compress_codec [string]
+
+显式指定输出压缩格式,默认 `none`。ORC 支持 `NONE`、`SNAPPY`、`ZLIB`、`LZ4` 和
`LZO`。配置后优先于表属性;继承表属性请参阅 `use_table_compression`。
+
+### use_table_compression [boolean]
+
+默认值:`false`。开启后,初始化**已有 ORC 表**时从 metastore 继承 `orc.compress`。此选项不改变自动建表流程、非
ORC 表或表元数据。
+
+对于非 ORC 表,`use_table_compression` 会被忽略,不会继承或校验表的压缩属性。如果 sink
初始化时表尚不存在并走自动建表流程,也会跳过压缩继承,包括 `save_mode_create_template` 中声明的
`orc.compress`。这两种情况均不会仅因开启此选项而导致任务失败;显式 `compress_codec`
的处理方式、原有默认值以及正常的配置校验和建表错误仍然适用。
+
+优先级:显式 `compress_codec`(包括 `NONE`)> 开启继承后的表属性 `orc.compress` > 原默认值
`NONE`。表属性去除首尾空白后按大小写不敏感匹配;属性缺失、为 null 或为空白时保留默认值。未知值或没有有效 ORC 映射的压缩格式在 sink
初始化时明确报错,避免静默写为不压缩。在 ORC ZSTD 仍映射为 NONE 的版本中,继承 ZSTD 会报错;使用该表属性前请确认 SeaTunnel
版本支持 ORC ZSTD。下游读取器也必须支持所选压缩格式。
+
+对于通过 `TBLPROPERTIES ('orc.compress'='SNAPPY')` 创建的已有表:
+
+```hocon
+sink {
+ Hive {
+ table_name = "default.existing_orc_table"
+ metastore_uri = "thrift://metastore:9083"
+ use_table_compression = true
+ # 可选:compress_codec = "NONE" 覆盖表属性。
+ }
+}
+```
+
### table_name [string]
目标 Hive 表名,例如:`db1.table1`。如果源是多模式,您可以使用 `${database_name}.${table_name}`
来生成表名,它将用源生成的 CatalogTable 的值替换 `${database_name}` 和 `${table_name}`。
diff --git
a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/config/HiveSinkOptions.java
b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/config/HiveSinkOptions.java
index 66a0207e5c..61933ddc6c 100644
---
a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/config/HiveSinkOptions.java
+++
b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/config/HiveSinkOptions.java
@@ -24,6 +24,14 @@ import org.apache.seatunnel.api.sink.SchemaSaveMode;
public class HiveSinkOptions extends HiveBaseOptions {
+ public static final Option<Boolean> USE_TABLE_COMPRESSION =
+ Options.key("use_table_compression")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Use the orc.compress property of an existing ORC
table when compress_codec is not explicitly configured. "
+ + "Missing or blank table properties
retain the default compression codec.");
+
public static final Option<Boolean> ABORT_DROP_PARTITION_METADATA =
Options.key("abort_drop_partition_metadata")
.booleanType()
diff --git
a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
index aabd55f9e5..31e6d38211 100644
---
a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
+++
b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
@@ -34,6 +34,7 @@ import org.apache.seatunnel.api.sink.SupportSaveMode;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated;
+import org.apache.seatunnel.connectors.seatunnel.file.config.CompressFormat;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
import
org.apache.seatunnel.connectors.seatunnel.file.sink.commit.FileAggregatedCommitInfo;
@@ -52,15 +53,18 @@ import
org.apache.seatunnel.connectors.seatunnel.hive.utils.HiveTableUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.orc.CompressionKind;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
+import static
org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSinkOptions.COMPRESS_CODEC;
import static
org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSinkOptions.FIELD_DELIMITER;
import static
org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSinkOptions.FILE_FORMAT_TYPE;
import static
org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSinkOptions.FILE_NAME_EXPRESSION;
@@ -165,6 +169,7 @@ public class HiveSink
sinkFields.addAll(partitionKeys);
FileFormat fileFormat =
HiveTableUtils.parseFileFormat(tableInformation);
+ pluginConfig = inheritTableCompression(pluginConfig, fileFormat,
tableInformation);
switch (fileFormat) {
case TEXT:
Map<String, String> parameters =
@@ -224,6 +229,47 @@ public class HiveSink
ReadonlyConfig.fromConfig(pluginConfig),
catalogTable.getSeaTunnelRowType());
}
+ /**
+ * Inherits compression only while preparing an existing ORC table.
Explicit codecs, including
+ * NONE, take precedence. Rejects enum values mapped to ORC NONE to
prevent silent compression
+ * loss. Returns the original config or an immutable replacement without
changing table
+ * metadata.
+ */
+ private Config inheritTableCompression(
+ Config pluginConfig, FileFormat fileFormat, Table table) {
+ if (!readonlyConfig.get(HiveSinkOptions.USE_TABLE_COMPRESSION)
+ || fileFormat != FileFormat.ORC
+ || pluginConfig.hasPath(COMPRESS_CODEC.key())) {
+ return pluginConfig;
+ }
+ Map<String, String> parameters = table.getParameters();
+ String codec = parameters == null ? null :
parameters.get("orc.compress");
+ if (codec == null || codec.trim().isEmpty()) {
+ return pluginConfig;
+ }
+ CompressFormat compression;
+ try {
+ compression =
CompressFormat.valueOf(codec.trim().toUpperCase(Locale.ROOT));
+ } catch (IllegalArgumentException e) {
+ throw new HiveConnectorException(
+ CommonErrorCodeDeprecated.ILLEGAL_ARGUMENT,
+ String.format(
+ "Unsupported orc.compress '%s' for Hive table
%s.%s",
+ codec, table.getDbName(), table.getTableName()),
+ e);
+ }
+ if (compression != CompressFormat.NONE
+ && compression.getOrcCompression() == CompressionKind.NONE) {
+ throw new HiveConnectorException(
+ CommonErrorCodeDeprecated.ILLEGAL_ARGUMENT,
+ String.format(
+ "Unsupported orc.compress '%s' for Hive table
%s.%s",
+ codec, table.getDbName(), table.getTableName()));
+ }
+ return pluginConfig.withValue(
+ COMPRESS_CODEC.key(),
ConfigValueFactory.fromAnyRef(compression.name()));
+ }
+
@Override
public String getPluginName() {
return HiveConstants.CONNECTOR_NAME;
diff --git
a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkFactory.java
b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkFactory.java
index 6365ca040e..a160a0cf3a 100644
---
a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkFactory.java
+++
b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkFactory.java
@@ -47,6 +47,7 @@ public class HiveSinkFactory
.optional(HiveSinkOptions.METASTORE_URI)
.optional(HiveSinkOptions.HIVE_SITE_PATH)
.optional(HiveSinkOptions.ABORT_DROP_PARTITION_METADATA)
+ .optional(HiveSinkOptions.USE_TABLE_COMPRESSION)
.optional(HiveSinkOptions.KERBEROS_PRINCIPAL)
.optional(HiveSinkOptions.KERBEROS_KEYTAB_PATH)
.optional(HiveSinkOptions.REMOTE_USER)
diff --git
a/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkConfigTest.java
b/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkConfigTest.java
index f2bbff6a86..799214f095 100644
---
a/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkConfigTest.java
+++
b/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkConfigTest.java
@@ -23,15 +23,19 @@ import
org.apache.seatunnel.api.table.catalog.PhysicalColumn;
import org.apache.seatunnel.api.table.catalog.TableIdentifier;
import org.apache.seatunnel.api.table.catalog.TableSchema;
import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.connectors.seatunnel.file.config.CompressFormat;
import
org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSinkOptions;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
import
org.apache.seatunnel.connectors.seatunnel.file.sink.config.FileSinkConfig;
import org.apache.seatunnel.connectors.seatunnel.hive.config.HiveSinkOptions;
+import
org.apache.seatunnel.connectors.seatunnel.hive.exception.HiveConnectorException;
import org.apache.seatunnel.connectors.seatunnel.hive.utils.HiveTableUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.orc.CompressionKind;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -44,9 +48,131 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-/** Unit tests for HiveSink config generation focusing on file_name_expression
handling. */
+/** Unit tests for HiveSink config generation. */
public class HiveSinkConfigTest {
+ @Test
+ void testTableCompressionDisabledByDefault() throws Exception {
+ Assertions.assertEquals(
+ CompressFormat.NONE,
+ compressionConfig(null, null, "SNAPPY",
true).getCompressFormat());
+ }
+
+ @Test
+ void testTableCompressionExplicitlyDisabled() throws Exception {
+ Assertions.assertEquals(
+ CompressFormat.NONE,
+ compressionConfig(false, null, "SNAPPY",
true).getCompressFormat());
+ }
+
+ @Test
+ void testInheritOrcCompression() throws Exception {
+ Assertions.assertEquals(
+ CompressFormat.SNAPPY,
+ compressionConfig(true, null, " snappy ",
true).getCompressFormat());
+ for (CompressFormat codec : CompressFormat.values()) {
+ if (codec == CompressFormat.NONE || codec.getOrcCompression() !=
CompressionKind.NONE) {
+ Assertions.assertEquals(
+ codec,
+ compressionConfig(true, null, codec.name(),
true).getCompressFormat());
+ }
+ }
+ }
+
+ @Test
+ void testExplicitCompressionOverridesTable() throws Exception {
+ Assertions.assertEquals(
+ CompressFormat.NONE,
+ compressionConfig(true, "NONE", "SNAPPY",
true).getCompressFormat());
+ Assertions.assertEquals(
+ CompressFormat.ZLIB,
+ compressionConfig(true, "ZLIB", "invalid",
true).getCompressFormat());
+ }
+
+ @Test
+ void testMissingOrBlankTableCompressionKeepsDefault() throws Exception {
+ for (String codec : new String[] {null, "", " "}) {
+ Assertions.assertEquals(
+ CompressFormat.NONE,
+ compressionConfig(true, null, codec,
true).getCompressFormat());
+ }
+ }
+
+ @Test
+ void testTableCompressionDoesNotApplyToOtherFormats() throws Exception {
+ Assertions.assertEquals(
+ CompressFormat.NONE,
+ compressionConfig(true, null, "SNAPPY",
false).getCompressFormat());
+ Assertions.assertEquals(
+ CompressFormat.NONE,
+ compressionConfig(true, null, "invalid",
FileFormat.PARQUET).getCompressFormat());
+ }
+
+ @Test
+ void testInvalidOrcCompressionRejected() {
+ for (String codec : new String[] {"invalid", "GZIP", "BROTLI"}) {
+ HiveConnectorException error =
+ Assertions.assertThrows(
+ HiveConnectorException.class,
+ () -> compressionConfig(true, null, codec, true));
+ Assertions.assertTrue(error.getMessage().contains("orc.compress"));
+
Assertions.assertTrue(error.getMessage().contains("default.test_table"));
+ }
+ for (CompressFormat codec : CompressFormat.values()) {
+ if (codec != CompressFormat.NONE && codec.getOrcCompression() ==
CompressionKind.NONE) {
+ Assertions.assertThrows(
+ HiveConnectorException.class,
+ () -> compressionConfig(true, null, codec.name(),
true));
+ }
+ }
+ }
+
+ private static FileSinkConfig compressionConfig(
+ Boolean enabled, String explicitCodec, String tableCodec, boolean
orc)
+ throws Exception {
+ return compressionConfig(
+ enabled, explicitCodec, tableCodec, orc ? FileFormat.ORC :
FileFormat.TEXT);
+ }
+
+ private static FileSinkConfig compressionConfig(
+ Boolean enabled, String explicitCodec, String tableCodec,
FileFormat format)
+ throws Exception {
+ Map<String, Object> options = new HashMap<>();
+ options.put(HiveSinkOptions.TABLE_NAME.key(), "default.test_table");
+ if (enabled != null) {
+ options.put(HiveSinkOptions.USE_TABLE_COMPRESSION.key(), enabled);
+ }
+ if (explicitCodec != null) {
+ options.put(FileBaseSinkOptions.COMPRESS_CODEC.key(),
explicitCodec);
+ }
+ Table table =
+ mockTextTable(
+ "default",
+ "test_table",
+ "file:/tmp/hive/test_table",
+ listOf(
+ new FieldSchema("id", "string", null),
+ new FieldSchema("name", "string", null)),
+ new ArrayList<>());
+ if (format == FileFormat.ORC) {
+
table.getSd().setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
+ } else if (format == FileFormat.PARQUET) {
+ table.getSd()
+ .setInputFormat(
+
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat");
+ }
+ if (tableCodec != null) {
+ table.putToParameters("orc.compress", tableCodec);
+ }
+ try (MockedStatic<HiveTableUtils> mocked =
Mockito.mockStatic(HiveTableUtils.class)) {
+ mocked.when(() ->
HiveTableUtils.getTableInfo(Mockito.any())).thenReturn(table);
+ mocked.when(() ->
HiveTableUtils.parseFileFormat(Mockito.any(Table.class)))
+ .thenCallRealMethod();
+ return extractFileSinkConfig(
+ new HiveSink(ReadonlyConfig.fromMap(options),
buildCatalogTable()));
+ }
+ }
+
@Test
void testDefaultFileNameExpressionAppliedWhenAbsent() throws Exception {
// Build minimal input config without file_name_expression
diff --git
a/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkFactoryTest.java
b/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkFactoryTest.java
index e760a175e9..9b653221e3 100644
---
a/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkFactoryTest.java
+++
b/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkFactoryTest.java
@@ -79,6 +79,14 @@ public class HiveSinkFactoryTest {
assertEquals("Hive", factory.factoryIdentifier());
}
+ @Test
+ void testTableCompressionOptionRegistered() {
+ assertTrue(
+ factory.optionRule()
+ .getOptionalOptions()
+ .contains(HiveSinkOptions.USE_TABLE_COMPRESSION));
+ }
+
@Test
void testCreateSinkWithValidSaveMode() {
Map<String, Object> configMap = new HashMap<>();
diff --git
a/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkOptionsTest.java
b/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkOptionsTest.java
index 8a44cdd244..20d0607b4f 100644
---
a/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkOptionsTest.java
+++
b/seatunnel-connectors-v2/connector-hive/src/test/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkOptionsTest.java
@@ -34,6 +34,16 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/** Test for HiveSinkOptions configuration */
public class HiveSinkOptionsTest {
+ @Test
+ void testUseTableCompressionOption() {
+ assertEquals("use_table_compression",
HiveSinkOptions.USE_TABLE_COMPRESSION.key());
+ assertFalse(
+ ReadonlyConfig.fromMap(new
HashMap<>()).get(HiveSinkOptions.USE_TABLE_COMPRESSION));
+ Map<String, Object> options = new HashMap<>();
+ options.put(HiveSinkOptions.USE_TABLE_COMPRESSION.key(), true);
+
assertTrue(ReadonlyConfig.fromMap(options).get(HiveSinkOptions.USE_TABLE_COMPRESSION));
+ }
+
@Test
void testSchemaSaveModeOption() {
assertNotNull(HiveSinkOptions.SCHEMA_SAVE_MODE);
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java
index 839931e1e4..c12dee50a6 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java
@@ -26,6 +26,12 @@ import
org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcFile;
+import org.apache.orc.Reader;
+
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
@@ -37,7 +43,9 @@ import org.testcontainers.lifecycle.Startables;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
+import java.nio.file.Files;
import java.sql.Connection;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
@@ -352,6 +360,76 @@ public class HiveIT extends TestSuiteBase implements
TestResource {
"/auto_table_creation/hive_auto_orc_format_to_assert.conf");
}
+ @TestTemplate
+ public void testExistingOrcTableCompression(TestContainer container)
throws Exception {
+ verifyExistingOrcTableCompression(container, "inherited",
CompressionKind.SNAPPY);
+ verifyExistingOrcTableCompression(container, "override",
CompressionKind.NONE);
+ }
+
+ private void verifyExistingOrcTableCompression(
+ TestContainer container, String scenario, CompressionKind
expectedCodec)
+ throws Exception {
+ String table = "test_orc_compression_" + scenario;
+ String location = "/opt/hive/data/warehouse/" + table;
+ try (Statement statement = hiveConnection.createStatement()) {
+ statement.execute("DROP TABLE IF EXISTS default." + table);
+ statement.execute(
+ "CREATE TABLE default."
+ + table
+ + " (pk_id BIGINT, name STRING, score INT) STORED
AS ORC"
+ + " LOCATION '"
+ + location
+ + "'"
+ + " TBLPROPERTIES ('orc.compress'='SNAPPY')");
+ try {
+ Container.ExecResult result =
+ container.executeJob("/fake_to_hive_orc_" + scenario +
".conf");
+ Assertions.assertEquals(0, result.getExitCode(),
result.getStderr());
+ Container.ExecResult files =
+ hiveServerContainer.execInContainer(
+ "find", location, "-type", "f", "!", "-name",
".*", "!", "-name",
+ "_*");
+ Assertions.assertEquals(0, files.getExitCode(),
files.getStderr());
+ Assertions.assertFalse(
+ files.getStdout().trim().isEmpty(), "No published ORC
files");
+ long rows = 0;
+ for (String file : files.getStdout().trim().split("\\R")) {
+ java.nio.file.Path copy =
Files.createTempFile("hive-orc-compression-", ".orc");
+ try {
+ hiveServerContainer.copyFileFromContainer(file,
copy.toString());
+ try (Reader reader =
+ OrcFile.createReader(
+ new Path(copy.toUri()),
+ OrcFile.readerOptions(new
Configuration()))) {
+ Assertions.assertEquals(
+ expectedCodec,
reader.getCompressionKind(), file);
+ rows += reader.getNumberOfRows();
+ }
+ } finally {
+ Files.deleteIfExists(copy);
+ }
+ }
+ Assertions.assertEquals(3, rows);
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "SELECT pk_id, name, score FROM default."
+ + table
+ + " ORDER BY pk_id")) {
+ for (int id = 1; id <= 3; id++) {
+ Assertions.assertTrue(resultSet.next());
+ Assertions.assertEquals(id, resultSet.getLong(1));
+ Assertions.assertEquals(
+ String.valueOf((char) ('A' + id - 1)),
resultSet.getString(2));
+ Assertions.assertEquals(100, resultSet.getInt(3));
+ }
+ Assertions.assertFalse(resultSet.next());
+ }
+ } finally {
+ statement.execute("DROP TABLE IF EXISTS default." + table);
+ }
+ }
+ }
+
@TestTemplate
public void testAutoTableCreationDefaultTemplate(TestContainer container)
throws Exception {
executeJob(
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_orc_inherited.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_orc_inherited.conf
new file mode 100644
index 0000000000..17947b4d7d
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_orc_inherited.conf
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ FakeSource {
+ schema = {
+ fields {
+ pk_id = bigint
+ name = string
+ score = int
+ }
+ primaryKey {
+ name = "pk_id"
+ columnNames = [pk_id]
+ }
+ }
+ rows = [
+ {
+ kind = INSERT
+ fields = [1, "A", 100]
+ },
+ {
+ kind = INSERT
+ fields = [2, "B", 100]
+ },
+ {
+ kind = INSERT
+ fields = [3, "C", 100]
+ }
+ ]
+ }
+}
+
+sink {
+ Hive {
+ table_name = "default.test_orc_compression_inherited"
+ metastore_uri = "thrift://metastore:9083"
+ use_table_compression = true
+ }
+}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_orc_override.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_orc_override.conf
new file mode 100644
index 0000000000..124b818b47
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_orc_override.conf
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ FakeSource {
+ schema = {
+ fields {
+ pk_id = bigint
+ name = string
+ score = int
+ }
+ primaryKey {
+ name = "pk_id"
+ columnNames = [pk_id]
+ }
+ }
+ rows = [
+ {
+ kind = INSERT
+ fields = [1, "A", 100]
+ },
+ {
+ kind = INSERT
+ fields = [2, "B", 100]
+ },
+ {
+ kind = INSERT
+ fields = [3, "C", 100]
+ }
+ ]
+ }
+}
+
+sink {
+ Hive {
+ table_name = "default.test_orc_compression_override"
+ metastore_uri = "thrift://metastore:9083"
+ use_table_compression = true
+ compress_codec = "NONE"
+ }
+}