slachiewicz commented on code in PR #724:
URL: https://github.com/apache/hudi-rs/pull/724#discussion_r4504723991
##########
crates/core/src/config/util.rs:
##########
@@ -77,12 +77,96 @@ pub fn parse_data_for_options(data: &Bytes, split_chars:
&str) -> Result<HashMap
.trim()
.to_owned();
let value = parts.next().unwrap_or("").trim().to_owned();
- options.insert(key, value);
+ options.insert(
+ unescape_java_properties(&key),
Review Comment:
Dropped the unescaping from `parse_data_for_options`: since #851,
`imbue_table_properties` already reads `hoodie.properties` through
`parse_properties`, so after the rebase it only touched `hudi-defaults.conf`.
`test_parse_data_keeps_backslashes` and `get_global_table_props` now pin that a
backslash in a `hudi-defaults.conf` value survives (1d77367).
##########
crates/core/src/config/table.rs:
##########
@@ -261,10 +261,15 @@ impl ConfigParser for HudiTableConfig {
// A table written without partitioning records the key with an
// empty value rather than omitting it; splitting that yields one
// field named "", which no schema contains.
+ //
+ // `CustomKeyGenerator` annotates each field with its key type
+ // (`region:SIMPLE,ts_str:TIMESTAMP`); the field name is what
+ // precedes the annotation.
Self::PartitionFields => get_result.map(|v| {
HudiConfigValue::List(
v.split(',')
.map(str::trim)
+ .map(|field|
field.split(':').next().unwrap_or_default().trim())
Review Comment:
Moved the annotation stripping, the key-generator inference change that goes
with it, the two custom-keygen fixtures and `custom_keygen_partition_paths` to
#977. Nothing in this PR's own tests needed the stripping, so none of it stayed
here.
##########
crates/core/src/keygen/timestamp_based.rs:
##########
@@ -597,18 +576,207 @@ mod tests {
TimestampBasedKeyGenerator::java_to_chrono_format("yyyy-MM-dd'T'HH:mm:ss.SSSZ"),
"%Y-%m-%dT%H:%M:%S.%3f%#z"
);
+ }
- // Hive-style: format segments → semantic names
+ #[test]
+ fn test_scalar_time_unit_parsing_and_conversion() {
Review Comment:
Moved the scalar-unit, timezone, epoch-overflow, guard,
`is_table_partitioned` and `should_include` fail-open tests to #976. The two
`_hoodie_partition_path` whole-path tests stayed, since they pin the hive-style
path this PR builds.
##########
crates/core/src/table/partition.rs:
##########
@@ -401,6 +401,136 @@ mod tests {
assert!(!pruner.should_include("date=2023-02-01/category=B/count=10"));
}
+ #[test]
+ fn test_hoodie_partition_path_whole_path_pruning() {
+ // A timestamp key generator rewrites a source-column predicate into a
predicate on
+ // the opaque whole-path meta field, so the pruner holds a
single-field schema and
+ // must compare the path as one string rather than split it into
segments.
+ let configs = HudiConfigs::new([
+ ("hoodie.table.partition.fields", "ts_str"),
+ (
+ "hoodie.table.keygenerator.class",
+ "org.apache.hudi.keygen.TimestampBasedKeyGenerator",
+ ),
+ ("hoodie.keygen.timebased.timestamp.type", "DATE_STRING"),
+ (
+ "hoodie.keygen.timebased.input.dateformat",
+ "yyyy-MM-dd'T'HH:mm:ss.SSSZ",
+ ),
+ ("hoodie.keygen.timebased.output.dateformat", "yyyy/MM/dd/HH"),
+ ("hoodie.datasource.write.hive_style_partitioning", "true"),
+ ("hoodie.datasource.write.partitionpath.urlencode", "false"),
+ ]);
+ let partition_schema = Schema::new(vec![Field::new(
+ MetaField::PartitionPath.as_ref(),
+ DataType::Utf8,
+ false,
+ )]);
+ let filter = Filter::try_from(("ts_str", "=",
"2023-04-01T12:01:00.123Z")).unwrap();
+
+ let pruner = PartitionPruner::new(&[filter], &partition_schema,
&configs).unwrap();
+ assert!(!pruner.is_empty());
+ assert!(pruner.should_include("ts_str=2023/04/01/12"));
+ assert!(!pruner.should_include("ts_str=2023/05/01/08"));
+ }
+
+ #[test]
+ fn test_hoodie_partition_path_whole_path_pruning_url_encoded() {
+ let configs = HudiConfigs::new([
+ ("hoodie.table.partition.fields", "ts_str"),
+ (
+ "hoodie.table.keygenerator.class",
+ "org.apache.hudi.keygen.TimestampBasedKeyGenerator",
+ ),
+ ("hoodie.keygen.timebased.timestamp.type", "DATE_STRING"),
+ (
+ "hoodie.keygen.timebased.input.dateformat",
+ "yyyy-MM-dd'T'HH:mm:ss.SSSZ",
+ ),
+ ("hoodie.keygen.timebased.output.dateformat", "yyyy/MM/dd/HH"),
+ ("hoodie.datasource.write.hive_style_partitioning", "true"),
+ ("hoodie.datasource.write.partitionpath.urlencode", "true"),
+ ]);
+ let partition_schema = Schema::new(vec![Field::new(
+ MetaField::PartitionPath.as_ref(),
+ DataType::Utf8,
+ false,
+ )]);
+ let filter = Filter::try_from(("ts_str", "=",
"2023-04-01T12:01:00.123Z")).unwrap();
+
+ let pruner = PartitionPruner::new(&[filter], &partition_schema,
&configs).unwrap();
+ // The encoded path decodes to `ts_str=2023/04/01/12` before
comparison.
+ assert!(pruner.should_include("ts_str%3D2023%2F04%2F01%2F12"));
Review Comment:
The url-encoded test now uses the writer's spelling,
`ts_str=2023%2F04%2F01%2F12`, with the prefix left unencoded (c60106e). The
excluded partition uses the same shape.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]