This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new b290301a35d1 refactor(config): deprecate
hoodie.parquet.outputtimestamptype; no-op since 1.1.0 (#19815)
b290301a35d1 is described below
commit b290301a35d177c065156486fcbd275fd33e39ba
Author: Ranga Reddy <[email protected]>
AuthorDate: Fri Sep 4 16:45:25 2026 +0530
refactor(config): deprecate hoodie.parquet.outputtimestamptype; no-op since
1.1.0 (#19815)
The config has had no effect on any write path since 1.1.0, while its
description still promised it set spark.sql.parquet.outputTimestampType.
Since #13882 (4d95b2c2d165, first released in 1.1.0)
HoodieRowParquetWriteSupport extends parquet's WriteSupport instead of
Spark's ParquetWriteSupport and builds the Parquet schema itself, taking
the timestamp unit from the writer schema's logical type. The Avro
Parquet writer has always derived it from the writer schema too. The
constructor kept copying the value into the Hadoop conf, but nothing has
read that key since, which is why the config still read as live.
Changes:
- HoodieStorageConfig.PARQUET_OUTPUT_TIMESTAMP_TYPE: description
rewritten to say it has no effect since 1.1.0 and to point at the
writer schema; adds @Deprecated and deprecatedAfter("1.1.0"). The key
and its default are untouched.
- HoodieRowParquetWriteSupport: drops the dead
hadoopConf.set("spark.sql.parquet.outputTimestampType", ...). The only
other reference in the tree, SparkReaderContextFactory, reads Spark's
own SQLConf on the read path and is unaffected.
- HoodieWriteConfig.parquetOutputTimestampType() and
HoodieStorageConfig.Builder.parquetOutputTimestampType(): marked
@Deprecated with javadoc rather than removed, so downstream
integrations keep compiling. Matches how the other deprecated members
of those classes are handled.
- TestHoodieInternalRowParquetWriter: new test pinning both directions,
so the description cannot drift back.
No behaviour change: the same Parquet footers are written before and
after.
This does not make the config work, so it is not a fix for #17302.
Offering a supported way to set the Parquet timestamp unit without
hand-writing a full writer schema is a separate storage-format
discussion, and that issue stays open for it.
---------
Co-authored-by: voon <[email protected]>
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 7 ++++
.../storage/row/HoodieRowParquetWriteSupport.java | 1 -
.../hudi/common/config/HoodieStorageConfig.java | 12 ++++++-
.../row/TestHoodieInternalRowParquetWriter.java | 41 ++++++++++++++++++++++
4 files changed, 59 insertions(+), 2 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 7bf46d9e9c33..dd93b0bd7faf 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -2535,6 +2535,13 @@ public class HoodieWriteConfig extends HoodieConfig {
return getBoolean(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED);
}
+ /**
+ * @deprecated the underlying config has had no effect since 1.1.0. Both the
Spark row writer and
+ * the Avro Parquet writer take the Parquet timestamp unit from the
writer schema's logical
+ * type, so the value returned here does not describe what gets written.
Declare the precision
+ * in the writer schema instead. Scheduled for removal.
+ */
+ @Deprecated
public String parquetOutputTimestampType() {
return getString(HoodieStorageConfig.PARQUET_OUTPUT_TIMESTAMP_TYPE);
}
diff --git
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java
index 6c55bbd5d487..caef37fcc0c5 100644
---
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java
+++
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java
@@ -151,7 +151,6 @@ public class HoodieRowParquetWriteSupport extends
WriteSupport<InternalRow> {
Configuration hadoopConf = new Configuration(conf);
String writeLegacyFormatEnabled =
config.getStringOrDefault(HoodieStorageConfig.PARQUET_WRITE_LEGACY_FORMAT_ENABLED,
"false");
hadoopConf.set("spark.sql.parquet.writeLegacyFormat",
writeLegacyFormatEnabled);
- hadoopConf.set("spark.sql.parquet.outputTimestampType",
config.getStringOrDefault(HoodieStorageConfig.PARQUET_OUTPUT_TIMESTAMP_TYPE));
hadoopConf.set("spark.sql.parquet.fieldId.write.enabled",
config.getStringOrDefault(PARQUET_FIELD_ID_WRITE_ENABLED));
// Variant shredding configs
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
index b92d984412c2..c5b14c6ffe50 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
@@ -244,11 +244,15 @@ public class HoodieStorageConfig extends HoodieConfig {
+ "For example, decimal values will be written in Parquet's
fixed-length byte array format which other systems such as Apache Hive and
Apache Impala use. "
+ "If false, the newer format in Parquet will be used. For example,
decimals will be written in int-based format.");
+ @Deprecated
public static final ConfigProperty<String> PARQUET_OUTPUT_TIMESTAMP_TYPE =
ConfigProperty
.key("hoodie.parquet.outputtimestamptype")
.defaultValue("TIMESTAMP_MICROS")
.markAdvanced()
- .withDocumentation("Sets spark.sql.parquet.outputTimestampType. Parquet
timestamp type to use when Spark writes data to Parquet files.");
+ .deprecatedAfter("1.1.0")
+ .withDocumentation("No effect since 1.1.0. Both the Spark row writer and
the Avro Parquet writer derive the "
+ + "Parquet timestamp unit from the writer schema's logical type
(timestamp-micros or timestamp-millis), "
+ + "so declare the precision in the writer schema (for example via
hoodie.write.schema) instead.");
// SPARK-38094 Spark 3.3 checks if this field is enabled. Hudi has to
provide this or there would be NPE thrown
// Would ONLY be effective with Spark 3.3+
@@ -671,6 +675,12 @@ public class HoodieStorageConfig extends HoodieConfig {
return this;
}
+ /**
+ * @deprecated since 1.1.0; the config has no effect. Both the Spark row
writer and the Avro
+ * Parquet writer take the Parquet timestamp unit from the writer
schema's logical type, so
+ * declare the precision in the writer schema instead.
+ */
+ @Deprecated
public Builder parquetOutputTimestampType(String
parquetOutputTimestampType) {
storageConfig.setValue(PARQUET_OUTPUT_TIMESTAMP_TYPE,
parquetOutputTimestampType);
return this;
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
index 04fd0e98324c..4092ccbd9694 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
+++
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
@@ -36,6 +36,7 @@ import org.apache.hudi.testutils.SparkDatasetTestUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.hadoop.metadata.FileMetaData;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.spark.sql.Dataset;
@@ -143,6 +144,46 @@ public class TestHoodieInternalRowParquetWriter extends
HoodieSparkClientTestHar
"bytes decimal keeps the precision-minimal FIXED_LEN width (9)");
}
+ /**
+ * hoodie.parquet.outputtimestamptype has had no effect since #13882: the
Parquet timestamp
+ * unit comes from the writer schema's logical type, not from the config.
Pin both directions
+ * so the config description cannot drift back to claiming otherwise.
+ */
+ @Test
+ public void testOutputTimestampTypeConfigDoesNotOverrideWriterSchema() {
+ // Config asks for MILLIS, writer schema says micros: the schema wins.
+ assertEquals(LogicalTypeAnnotation.TimeUnit.MICROS,
+ timestampUnitOf(timestampRecordSchema("timestamp-micros"),
"TIMESTAMP_MILLIS"));
+ // Config left at its default of MICROS, writer schema says millis: the
schema wins again.
+ assertEquals(LogicalTypeAnnotation.TimeUnit.MILLIS,
+ timestampUnitOf(timestampRecordSchema("timestamp-millis"), null));
+ }
+
+ private static String timestampRecordSchema(String logicalType) {
+ return
"{\"type\":\"record\",\"name\":\"rec\",\"fields\":[{\"name\":\"ts\","
+ + "\"type\":{\"type\":\"long\",\"logicalType\":\"" + logicalType +
"\"}}]}";
+ }
+
+ /** Builds the write support for the given writer schema and returns the
Parquet timestamp unit it emits. */
+ @SuppressWarnings("deprecation")
+ private LogicalTypeAnnotation.TimeUnit timestampUnitOf(String
avroSchemaJson, String outputTimestampType) {
+ StructType structType = new StructType().add("ts",
DataTypes.TimestampType, false);
+ HoodieWriteConfig.Builder builder = HoodieWriteConfig.newBuilder()
+ .withPath(basePath)
+ .withSchema(avroSchemaJson);
+ if (outputTimestampType != null) {
+ builder.withStorageConfig(HoodieStorageConfig.newBuilder()
+ .parquetOutputTimestampType(outputTimestampType).build());
+ }
+ HoodieRowParquetWriteSupport writeSupport =
HoodieRowParquetWriteSupport.getHoodieRowParquetWriteSupport(
+ storageConf.unwrap(), structType, Option.empty(), builder.build());
+ MessageType parquetSchema =
writeSupport.init(writeSupport.getHadoopConf()).getSchema();
+ LogicalTypeAnnotation annotation =
parquetSchema.getType("ts").asPrimitiveType().getLogicalTypeAnnotation();
+ assertTrue(annotation instanceof
LogicalTypeAnnotation.TimestampLogicalTypeAnnotation,
+ "ts must carry a timestamp logical type, got: " + annotation);
+ return ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation)
annotation).getUnit();
+ }
+
private static String decimalRecordSchema(String decType) {
return
"{\"type\":\"record\",\"name\":\"rec\",\"fields\":[{\"name\":\"dec\",\"type\":"
+ decType + "}]}";
}
