debabhishek53 commented on code in PR #4171: URL: https://github.com/apache/gobblin/pull/4171#discussion_r2936964211
########## gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergSource.java: ########## @@ -80,23 +82,39 @@ * iceberg.table.name=table1 * iceberg.catalog.uri=ICEBERG_CATALOG_URI * - * # Partition filtering with lookback - Static date (hourly partitions by default) + * # Partition filtering with lookback - Static date * iceberg.filter.enabled=true - * iceberg.partition.column=datepartition # Optional, defaults to "datepartition" - * iceberg.filter.date=2025-04-01 # Date value (yyyy-MM-dd format) + * iceberg.partition.column=datepartition # Optional, defaults to "datepartition" + * iceberg.filter.date=2025-04-01 # Input date always in yyyy-MM-dd format * iceberg.lookback.days=3 - * # iceberg.hourly.partition.enabled=true # Optional, defaults to true; generates yyyy-MM-dd-00 format * - * # Partition filtering with lookback - Scheduled flows (dynamic date) + * # Partition filtering - Scheduled flows (dynamic date) * iceberg.filter.enabled=true - * iceberg.filter.date=CURRENT_DATE # Uses current date at runtime + * iceberg.filter.date=CURRENT_DATE # Resolved to current date at runtime * iceberg.lookback.days=3 * - * # Standard daily partitions (disable hourly suffix if table uses yyyy-MM-dd format) - * iceberg.filter.enabled=true - * iceberg.filter.date=2025-04-01 # Date in yyyy-MM-dd format - * iceberg.hourly.partition.enabled=false # Set false for non-hourly partitions - * iceberg.lookback.days=3 + * # --- Recommended: configurable partition value format --- + * # iceberg.partition.value.format is a DateTimeFormatter pattern applied to the output + * # partition value used in the filter expression. The input date is always yyyy-MM-dd. + * # When set, it supersedes iceberg.hourly.partition.enabled. + * + * # Standard hourly partitions (yyyy-MM-dd-HH) + * iceberg.partition.value.format=yyyy-MM-dd-HH # → "2025-04-01-05" + * iceberg.partition.hour=5 # Hour 0-23, default 0 Review Comment: Resolved with new commit -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
