Zouxxyy commented on code in PR #4657:
URL: https://github.com/apache/paimon/pull/4657#discussion_r1889491063
##########
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java:
##########
@@ -75,21 +79,33 @@ public String identifier() {
String timestampFormatter,
String timestampPattern,
String expireStrategy,
- Integer maxExpires)
+ Integer maxExpires,
+ String options)
throws Catalog.TableNotExistException {
- FileStoreTable fileStoreTable = (FileStoreTable) table(tableId);
+ Map<String, String> dynamicOptions =
+ ProcedureUtils.fillInPartitionOptions(
+ expireStrategy,
+ timestampFormatter,
+ timestampPattern,
+ expirationTime,
+ maxExpires,
+ options);
+
+ Table table = table(tableId).copy(dynamicOptions);
+ FileStoreTable fileStoreTable = (FileStoreTable) table;
FileStore fileStore = fileStoreTable.store();
- Map<String, String> map = new HashMap<>();
- map.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(),
expireStrategy);
- map.put(CoreOptions.PARTITION_TIMESTAMP_FORMATTER.key(),
timestampFormatter);
- map.put(CoreOptions.PARTITION_TIMESTAMP_PATTERN.key(),
timestampPattern);
+
+ // check expiration time not null
+ Preconditions.checkNotNull(
+ fileStore.options().partitionExpireTime(),
+ "The partition expiration time is must been required, you can
set it by configuring the property 'partition.expiration-time' or adding the
'expiration_time' parameter in procedure. ");
PartitionExpire partitionExpire =
new PartitionExpire(
- TimeUtils.parseDuration(expirationTime),
+ fileStore.options().partitionExpireTime(),
Review Comment:
use `newPartitionExpire` in `FileStore`
##########
paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ExpireSnapshotsProcedure.java:
##########
@@ -76,29 +80,22 @@ public InternalRow[] call(InternalRow args) {
Integer retainMin = args.isNullAt(2) ? null : args.getInt(2);
String olderThanStr = args.isNullAt(3) ? null : args.getString(3);
Integer maxDeletes = args.isNullAt(4) ? null : args.getInt(4);
+ String options = args.isNullAt(5) ? null : args.getString(5);
return modifyPaimonTable(
tableIdent,
table -> {
- ExpireSnapshots expireSnapshots =
table.newExpireSnapshots();
- ExpireConfig.Builder builder = ExpireConfig.builder();
- if (retainMax != null) {
- builder.snapshotRetainMax(retainMax);
- }
- if (retainMin != null) {
- builder.snapshotRetainMin(retainMin);
- }
- if (!StringUtils.isNullOrWhitespaceOnly(olderThanStr)) {
- long olderThanMills =
- DateTimeUtils.parseTimestampData(
- olderThanStr, 3,
TimeZone.getDefault())
- .getMillisecond();
- builder.snapshotTimeRetain(
- Duration.ofMillis(System.currentTimeMillis() -
olderThanMills));
- }
- if (maxDeletes != null) {
- builder.snapshotMaxDeletes(maxDeletes);
+ Map<String, String> dynamicOptions = new HashMap<>();
+ if (!StringUtils.isNullOrWhitespaceOnly(options)) {
+
dynamicOptions.putAll(ParameterUtils.parseCommaSeparatedKeyValues(options));
}
+ table = table.copy(dynamicOptions);
+ ExpireSnapshots expireSnapshots =
table.newExpireSnapshots();
+
+ CoreOptions tableOptions = ((FileStoreTable)
table).store().options();
+ ExpireConfig.Builder builder =
+ ProcedureUtils.fillInSnapshotOptions(
+ tableOptions, retainMax, retainMin,
olderThanStr, maxDeletes);
int deleted =
expireSnapshots.config(builder.build()).expire();
Review Comment:
`table.newExpireSnapshots()` should include the dynamicOptions instead of
inserting it through the builder, if not maybe we should fix it
##########
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java:
##########
@@ -75,21 +79,33 @@ public String identifier() {
String timestampFormatter,
String timestampPattern,
String expireStrategy,
- Integer maxExpires)
+ Integer maxExpires,
+ String options)
throws Catalog.TableNotExistException {
- FileStoreTable fileStoreTable = (FileStoreTable) table(tableId);
+ Map<String, String> dynamicOptions =
+ ProcedureUtils.fillInPartitionOptions(
+ expireStrategy,
+ timestampFormatter,
+ timestampPattern,
+ expirationTime,
+ maxExpires,
+ options);
+
+ Table table = table(tableId).copy(dynamicOptions);
+ FileStoreTable fileStoreTable = (FileStoreTable) table;
FileStore fileStore = fileStoreTable.store();
- Map<String, String> map = new HashMap<>();
- map.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(),
expireStrategy);
- map.put(CoreOptions.PARTITION_TIMESTAMP_FORMATTER.key(),
timestampFormatter);
- map.put(CoreOptions.PARTITION_TIMESTAMP_PATTERN.key(),
timestampPattern);
+
+ // check expiration time not null
+ Preconditions.checkNotNull(
Review Comment:
this maybe can be checked in FileStore internal
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
