This is an automated email from the ASF dual-hosted git repository. xuba pushed a commit to branch v0.7.x-test-front in repository https://gitbox.apache.org/repos/asf/amoro.git
commit 4620af8fed0866985e0d66402d9dd3f67cfd32da Author: Xavier Bai <[email protected]> AuthorDate: Wed Mar 20 10:22:10 2024 +0800 [WAP] Enhance TimeUtils and fix config parsing error --- .../arctic/api/config/DataExpirationConfig.java | 166 ++++++++-- .../maintainer/IcebergTableMaintainer.java | 4 +- .../maintainer/MixedTableMaintainer.java | 4 +- .../arctic/server/table/DataExpirationConfig.java | 101 +++--- .../arctic/server/utils/ConfigurationUtil.java | 222 +------------ .../java/com/netease/arctic/utils/TimeUtils.java | 342 +++++++++++++++++++++ 6 files changed, 543 insertions(+), 296 deletions(-) diff --git a/ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java b/ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java index aff7ba623..f4536084b 100644 --- a/ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java @@ -19,17 +19,23 @@ package com.netease.arctic.api.config; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import com.netease.arctic.table.ArcticTable; +import com.netease.arctic.table.TableProperties; +import com.netease.arctic.utils.CompatiblePropertyUtil; +import com.netease.arctic.utils.TimeUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; -import org.apache.iceberg.relocated.com.google.common.base.Objects; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Locale; +import java.util.Map; import java.util.Set; /** Data expiration configuration. */ @@ -40,23 +46,29 @@ public class DataExpirationConfig { // data-expire.field private String expirationField; // data-expire.level + @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT) private ExpireLevel expirationLevel; // data-expire.retention-time private long retentionTime; // data-expire.datetime-string-pattern + @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT) private String dateTimePattern; // data-expire.datetime-number-format + @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT) private String numberDateFormat; - // data-expire.since - private Since since; + // data-expire.base-on-rule + @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT) + private BaseOnRule baseOnRule; + // Retention time must be positive + public static final long INVALID_RETENTION_TIME = 0L; - @VisibleForTesting + @com.google.common.annotations.VisibleForTesting public enum ExpireLevel { PARTITION, FILE; public static ExpireLevel fromString(String level) { - Preconditions.checkArgument(null != level, "Invalid level type: null"); + com.google.common.base.Preconditions.checkArgument(null != level, "Invalid level type: null"); try { return ExpireLevel.valueOf(level.toUpperCase(Locale.ENGLISH)); } catch (IllegalArgumentException e) { @@ -66,14 +78,15 @@ public class DataExpirationConfig { } @VisibleForTesting - public enum Since { - LATEST_SNAPSHOT, - CURRENT_TIMESTAMP; + public enum BaseOnRule { + LAST_COMMIT_TIME, + CURRENT_TIME; - public static Since fromString(String since) { - Preconditions.checkArgument(null != since, "data-expire.since is invalid: null"); + public static BaseOnRule fromString(String since) { + com.google.common.base.Preconditions.checkArgument( + null != since, TableProperties.DATA_EXPIRATION_BASE_ON_RULE + " is invalid: null"); try { - return Since.valueOf(since.toUpperCase(Locale.ENGLISH)); + return BaseOnRule.valueOf(since.toUpperCase(Locale.ENGLISH)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( String.format("Unable to expire data since: %s", since), e); @@ -95,14 +108,115 @@ public class DataExpirationConfig { long retentionTime, String dateTimePattern, String numberDateFormat, - Since since) { + BaseOnRule baseOnRule) { this.enabled = enabled; this.expirationField = expirationField; this.expirationLevel = expirationLevel; this.retentionTime = retentionTime; this.dateTimePattern = dateTimePattern; this.numberDateFormat = numberDateFormat; - this.since = since; + this.baseOnRule = baseOnRule; + } + + public DataExpirationConfig(ArcticTable table) { + Map<String, String> properties = table.properties(); + expirationField = + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_FIELD, null); + Types.NestedField field = table.schema().findField(expirationField); + com.google.common.base.Preconditions.checkArgument( + StringUtils.isNoneBlank(expirationField) && null != field, + String.format( + "Field(%s) used to determine data expiration is illegal for table(%s)", + expirationField, table.name())); + Type.TypeID typeID = field.type().typeId(); + Preconditions.checkArgument( + FIELD_TYPES.contains(typeID), + String.format( + "The type(%s) of filed(%s) is incompatible for table(%s)", + typeID.name(), expirationField, table.name())); + + expirationLevel = + ExpireLevel.fromString( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_LEVEL, + TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT)); + retentionTime = + parseRetentionToMillis( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null)); + dateTimePattern = + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT); + numberDateFormat = + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT); + baseOnRule = + BaseOnRule.fromString( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT)); + } + + public static DataExpirationConfig parse(Map<String, String> properties) { + boolean gcEnabled = + CompatiblePropertyUtil.propertyAsBoolean( + properties, org.apache.iceberg.TableProperties.GC_ENABLED, true); + + return new DataExpirationConfig() + .setEnabled( + gcEnabled + && CompatiblePropertyUtil.propertyAsBoolean( + properties, + TableProperties.ENABLE_DATA_EXPIRATION, + TableProperties.ENABLE_DATA_EXPIRATION_DEFAULT)) + .setExpirationLevel( + ExpireLevel.fromString( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_LEVEL, + TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT))) + .setExpirationField( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_FIELD, null)) + .setDateTimePattern( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT)) + .setNumberDateFormat( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT)) + .setBaseOnRule( + BaseOnRule.fromString( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT))) + .setRetentionTime( + parseRetentionToMillis( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null))); + } + + private static long parseRetentionToMillis(String retention) { + try { + return TimeUtils.parseTime(retention).toMillis(); + } catch (Exception e) { + return INVALID_RETENTION_TIME; + } + } + + public boolean isPositive() { + return retentionTime > INVALID_RETENTION_TIME; } public boolean isEnabled() { @@ -159,12 +273,12 @@ public class DataExpirationConfig { return this; } - public Since getSince() { - return since; + public BaseOnRule getBaseOnRule() { + return baseOnRule; } - public DataExpirationConfig setSince(Since since) { - this.since = since; + public DataExpirationConfig setBaseOnRule(BaseOnRule baseOnRule) { + this.baseOnRule = baseOnRule; return this; } @@ -179,11 +293,11 @@ public class DataExpirationConfig { DataExpirationConfig config = (DataExpirationConfig) o; return enabled == config.enabled && retentionTime == config.retentionTime - && Objects.equal(expirationField, config.expirationField) + && com.google.common.base.Objects.equal(expirationField, config.expirationField) && expirationLevel == config.expirationLevel - && Objects.equal(dateTimePattern, config.dateTimePattern) - && Objects.equal(numberDateFormat, config.numberDateFormat) - && since == config.since; + && com.google.common.base.Objects.equal(dateTimePattern, config.dateTimePattern) + && com.google.common.base.Objects.equal(numberDateFormat, config.numberDateFormat) + && baseOnRule == config.baseOnRule; } @Override @@ -195,12 +309,12 @@ public class DataExpirationConfig { retentionTime, dateTimePattern, numberDateFormat, - since); + baseOnRule); } public boolean isValid(Types.NestedField field, String name) { return isEnabled() - && getRetentionTime() > 0 + && isPositive() && validateExpirationField(field, name, getExpirationField()); } diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java index a16e6a5b0..d4c04f917 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java @@ -265,8 +265,8 @@ public class IcebergTableMaintainer implements TableMaintainer { * zone */ @VisibleForTesting - public void expireDataFrom(DataExpirationConfig expirationConfig, Instant instant) { - if (instant.equals(Instant.MIN)) { + protected void expireDataFrom(DataExpirationConfig expirationConfig, Instant instant) { + if (instant.equals(Instant.MIN) || !expirationConfig.isPositive()) { return; } diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java index e97e1a422..0251d53b2 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java @@ -174,8 +174,8 @@ public class MixedTableMaintainer implements TableMaintainer { } @VisibleForTesting - public void expireDataFrom(DataExpirationConfig expirationConfig, Instant instant) { - if (instant.equals(Instant.MIN)) { + protected void expireDataFrom(DataExpirationConfig expirationConfig, Instant instant) { + if (instant.equals(Instant.MIN) || !expirationConfig.isPositive()) { return; } diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/DataExpirationConfig.java b/ams/server/src/main/java/com/netease/arctic/server/table/DataExpirationConfig.java index e53b2d45e..f750368d8 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/DataExpirationConfig.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/DataExpirationConfig.java @@ -24,10 +24,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; -import com.netease.arctic.server.utils.ConfigurationUtil; import com.netease.arctic.table.ArcticTable; import com.netease.arctic.table.TableProperties; import com.netease.arctic.utils.CompatiblePropertyUtil; +import com.netease.arctic.utils.TimeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -38,7 +38,10 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -/** TODO Use DataExpirationConfig class in API module, this class shall be removed after 0.7.0 */ +/** + * TODO Use {@link com.netease.arctic.api.config.DataExpirationConfig} class in API module, this + * class shall be removed after 0.7.0 + */ @Deprecated @JsonIgnoreProperties(ignoreUnknown = true) public class DataExpirationConfig { @@ -60,6 +63,8 @@ public class DataExpirationConfig { // data-expire.base-on-rule @JsonProperty(defaultValue = TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT) private BaseOnRule baseOnRule; + // Retention time must be positive + public static final long INVALID_RETENTION_TIME = 0L; @VisibleForTesting public enum ExpireLevel { @@ -141,14 +146,10 @@ public class DataExpirationConfig { properties, TableProperties.DATA_EXPIRATION_LEVEL, TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT)); - - String retention = - CompatiblePropertyUtil.propertyAsString( - properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null); - if (StringUtils.isNotBlank(retention)) { - retentionTime = ConfigurationUtil.TimeUtils.parseDuration(retention).toMillis(); - } - + retentionTime = + parseRetentionToMillis( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null)); dateTimePattern = CompatiblePropertyUtil.propertyAsString( properties, @@ -171,47 +172,55 @@ public class DataExpirationConfig { boolean gcEnabled = CompatiblePropertyUtil.propertyAsBoolean( properties, org.apache.iceberg.TableProperties.GC_ENABLED, true); - DataExpirationConfig config = - new DataExpirationConfig() - .setEnabled( - gcEnabled - && CompatiblePropertyUtil.propertyAsBoolean( - properties, - TableProperties.ENABLE_DATA_EXPIRATION, - TableProperties.ENABLE_DATA_EXPIRATION_DEFAULT)) - .setExpirationLevel( - ExpireLevel.fromString( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_LEVEL, - TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT))) - .setExpirationField( - CompatiblePropertyUtil.propertyAsString( - properties, TableProperties.DATA_EXPIRATION_FIELD, null)) - .setDateTimePattern( + + return new DataExpirationConfig() + .setEnabled( + gcEnabled + && CompatiblePropertyUtil.propertyAsBoolean( + properties, + TableProperties.ENABLE_DATA_EXPIRATION, + TableProperties.ENABLE_DATA_EXPIRATION_DEFAULT)) + .setExpirationLevel( + ExpireLevel.fromString( CompatiblePropertyUtil.propertyAsString( properties, - TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN, - TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT)) - .setNumberDateFormat( + TableProperties.DATA_EXPIRATION_LEVEL, + TableProperties.DATA_EXPIRATION_LEVEL_DEFAULT))) + .setExpirationField( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_FIELD, null)) + .setDateTimePattern( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN, + TableProperties.DATA_EXPIRATION_DATE_STRING_PATTERN_DEFAULT)) + .setNumberDateFormat( + CompatiblePropertyUtil.propertyAsString( + properties, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT, + TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT)) + .setBaseOnRule( + BaseOnRule.fromString( CompatiblePropertyUtil.propertyAsString( properties, - TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT, - TableProperties.DATA_EXPIRATION_DATE_NUMBER_FORMAT_DEFAULT)) - .setBaseOnRule( - BaseOnRule.fromString( - CompatiblePropertyUtil.propertyAsString( - properties, - TableProperties.DATA_EXPIRATION_BASE_ON_RULE, - TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT))); - String retention = - CompatiblePropertyUtil.propertyAsString( - properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null); - if (StringUtils.isNotBlank(retention)) { - config.setRetentionTime(ConfigurationUtil.TimeUtils.parseDuration(retention).toMillis()); + TableProperties.DATA_EXPIRATION_BASE_ON_RULE, + TableProperties.DATA_EXPIRATION_BASE_ON_RULE_DEFAULT))) + .setRetentionTime( + parseRetentionToMillis( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.DATA_EXPIRATION_RETENTION_TIME, null))); + } + + private static long parseRetentionToMillis(String retention) { + try { + return TimeUtils.parseTime(retention).toMillis(); + } catch (Exception e) { + return INVALID_RETENTION_TIME; } + } - return config; + public boolean isPositive() { + return retentionTime > INVALID_RETENTION_TIME; } public boolean isEnabled() { @@ -309,7 +318,7 @@ public class DataExpirationConfig { public boolean isValid(Types.NestedField field, String name) { return isEnabled() - && getRetentionTime() > 0 + && isPositive() && validateExpirationField(field, name, getExpirationField()); } diff --git a/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigurationUtil.java b/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigurationUtil.java index 86f22f05d..def707f4d 100755 --- a/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigurationUtil.java +++ b/ams/server/src/main/java/com/netease/arctic/server/utils/ConfigurationUtil.java @@ -19,23 +19,20 @@ package com.netease.arctic.server.utils; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; + +import com.netease.arctic.utils.TimeUtils; import javax.annotation.Nonnull; import java.io.File; import java.time.Duration; -import java.time.temporal.ChronoUnit; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.IntStream; /** Utility class for {@link Configurations} related helper functions. */ public class ConfigurationUtil { @@ -292,219 +289,4 @@ public class ConfigurationUtil { return Double.parseDouble(o.toString()); } - - /** Collection of utilities about time intervals. */ - public static class TimeUtils { - - private static final Map<String, ChronoUnit> LABEL_TO_UNIT_MAP = - Collections.unmodifiableMap(initMap()); - - /** - * Parse the given string to a java {@link Duration}. The string is in format "{length - * value}{time unit label}", e.g. "123ms", "321 s". If no time unit label is specified, it will - * be considered as milliseconds. - * - * <p>Supported time unit labels are: - * - * <ul> - * <li>DAYS： "d", "day" - * <li>HOURS： "h", "hour" - * <li>MINUTES： "min", "minute" - * <li>SECONDS： "s", "sec", "second" - * <li>MILLISECONDS： "ms", "milli", "millisecond" - * <li>MICROSECONDS： "µs", "micro", "microsecond" - * <li>NANOSECONDS： "ns", "nano", "nanosecond" - * </ul> - * - * @param text string to parse. - */ - public static Duration parseDuration(String text) { - checkNotNull(text); - - final String trimmed = text.trim(); - checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string"); - - final int len = trimmed.length(); - int pos = 0; - - char current; - while (pos < len && (current = trimmed.charAt(pos)) >= '0' && current <= '9') { - pos++; - } - - final String number = trimmed.substring(0, pos); - final String unitLabel = trimmed.substring(pos).trim().toLowerCase(Locale.US); - - if (number.isEmpty()) { - throw new NumberFormatException("text does not start with a number"); - } - - final long value; - try { - value = Long.parseLong(number); // this throws a NumberFormatException on overflow - } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "The value '" - + number - + "' cannot be re represented as 64bit number (numeric overflow)."); - } - - if (unitLabel.isEmpty()) { - return Duration.of(value, ChronoUnit.MILLIS); - } - - ChronoUnit unit = LABEL_TO_UNIT_MAP.get(unitLabel); - if (unit != null) { - return Duration.of(value, unit); - } else { - throw new IllegalArgumentException( - "Time interval unit label '" - + unitLabel - + "' does not match any of the recognized units: " - + TimeUnit.getAllUnits()); - } - } - - private static Map<String, ChronoUnit> initMap() { - Map<String, ChronoUnit> labelToUnit = new HashMap<>(); - for (TimeUnit timeUnit : TimeUnit.values()) { - for (String label : timeUnit.getLabels()) { - labelToUnit.put(label, timeUnit.getUnit()); - } - } - return labelToUnit; - } - - /** - * @param duration to convert to string - * @return duration string in millis - */ - public static String getStringInMillis(final Duration duration) { - return duration.toMillis() + TimeUnit.MILLISECONDS.labels.get(0); - } - - /** - * Pretty prints the duration as a lowest granularity unit that does not lose precision. - * - * <p>Examples: - * - * <pre>{@code - * Duration.ofMilliseconds(60000) will be printed as 1 min - * Duration.ofHours(1).plusSeconds(1) will be printed as 3601 s - * }</pre> - * - * <b>NOTE:</b> It supports only durations that fit into long. - */ - public static String formatWithHighestUnit(Duration duration) { - long nanos = duration.toNanos(); - - List<TimeUnit> orderedUnits = - Arrays.asList( - TimeUnit.NANOSECONDS, - TimeUnit.MICROSECONDS, - TimeUnit.MILLISECONDS, - TimeUnit.SECONDS, - TimeUnit.MINUTES, - TimeUnit.HOURS, - TimeUnit.DAYS); - - TimeUnit highestIntegerUnit = - IntStream.range(0, orderedUnits.size()) - .sequential() - .filter(idx -> nanos % orderedUnits.get(idx).unit.getDuration().toNanos() != 0) - .boxed() - .findFirst() - .map( - idx -> { - if (idx == 0) { - return orderedUnits.get(0); - } else { - return orderedUnits.get(idx - 1); - } - }) - .orElse(TimeUnit.MILLISECONDS); - - return String.format( - "%d %s", - nanos / highestIntegerUnit.unit.getDuration().toNanos(), - highestIntegerUnit.getLabels().get(0)); - } - - private static ChronoUnit toChronoUnit(java.util.concurrent.TimeUnit timeUnit) { - switch (timeUnit) { - case NANOSECONDS: - return ChronoUnit.NANOS; - case MICROSECONDS: - return ChronoUnit.MICROS; - case MILLISECONDS: - return ChronoUnit.MILLIS; - case SECONDS: - return ChronoUnit.SECONDS; - case MINUTES: - return ChronoUnit.MINUTES; - case HOURS: - return ChronoUnit.HOURS; - case DAYS: - return ChronoUnit.DAYS; - default: - throw new IllegalArgumentException(String.format("Unsupported time unit %s.", timeUnit)); - } - } - - /** Enum which defines time unit, mostly used to parse value from configuration file. */ - private enum TimeUnit { - DAYS(ChronoUnit.DAYS, singular("d"), plural("day")), - HOURS(ChronoUnit.HOURS, singular("h"), plural("hour")), - MINUTES(ChronoUnit.MINUTES, singular("min"), plural("minute")), - SECONDS(ChronoUnit.SECONDS, singular("s"), plural("sec"), plural("second")), - MILLISECONDS(ChronoUnit.MILLIS, singular("ms"), plural("milli"), plural("millisecond")), - MICROSECONDS(ChronoUnit.MICROS, singular("µs"), plural("micro"), plural("microsecond")), - NANOSECONDS(ChronoUnit.NANOS, singular("ns"), plural("nano"), plural("nanosecond")); - - private static final String PLURAL_SUFFIX = "s"; - - private final List<String> labels; - - private final ChronoUnit unit; - - TimeUnit(ChronoUnit unit, String[]... labels) { - this.unit = unit; - this.labels = Arrays.stream(labels).flatMap(Arrays::stream).collect(Collectors.toList()); - } - - /** - * @param label the original label - * @return the singular format of the original label - */ - private static String[] singular(String label) { - return new String[] {label}; - } - - /** - * @param label the original label - * @return both the singular format and plural format of the original label - */ - private static String[] plural(String label) { - return new String[] {label, label + PLURAL_SUFFIX}; - } - - public static String getAllUnits() { - return Arrays.stream(TimeUnit.values()) - .map(TimeUnit::createTimeUnitString) - .collect(Collectors.joining(", ")); - } - - private static String createTimeUnitString(TimeUnit timeUnit) { - return timeUnit.name() + ": (" + String.join(" | ", timeUnit.getLabels()) + ")"; - } - - public List<String> getLabels() { - return labels; - } - - public ChronoUnit getUnit() { - return unit; - } - } - } } diff --git a/core/src/main/java/com/netease/arctic/utils/TimeUtils.java b/core/src/main/java/com/netease/arctic/utils/TimeUtils.java new file mode 100644 index 000000000..089288b3f --- /dev/null +++ b/core/src/main/java/com/netease/arctic/utils/TimeUtils.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netease.arctic.utils; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.Objects; +import org.apache.commons.lang3.StringUtils; + +import java.time.Duration; +import java.time.Period; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** Collection of utilities about time intervals. */ +public class TimeUtils { + + private static Map<String, ChronoUnit> initMap() { + Map<String, ChronoUnit> labelToUnit = new HashMap<>(); + for (TimeUnit timeUnit : TimeUnit.values()) { + for (String label : timeUnit.getLabels()) { + labelToUnit.put(label, timeUnit.getUnit()); + } + } + return labelToUnit; + } + + private static final Map<String, ChronoUnit> LABEL_TO_UNIT_MAP = + Collections.unmodifiableMap(initMap()); + + public static class Time { + String number; + ChronoUnit unit; + + public Time(String number, String unitLabel) { + if (StringUtils.isBlank(number)) { + throw new IllegalArgumentException("Number text cannot be empty"); + } + + if (!LABEL_TO_UNIT_MAP.containsKey(unitLabel)) { + throw new IllegalArgumentException( + "Time interval unit label '" + + unitLabel + + "' does not match any of the recognized units: " + + TimeUnit.getAllUnits()); + } + + this.number = number; + this.unit = LABEL_TO_UNIT_MAP.get(unitLabel); + } + + public String getNumber() { + return number; + } + + public ChronoUnit getUnit() { + return unit; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Time)) { + return false; + } + Time time = (Time) o; + return Objects.equal(number, time.number) && Objects.equal(unit, time.unit); + } + + @Override + public int hashCode() { + return Objects.hashCode(number, unit); + } + + /** @return duration in millis */ + public long toMillis() { + if (isPeriod()) { + // Estimated 30 days per month + return parsePeriod(this).toTotalMonths() * 30 * 24 * 60 * 60 * 1000; + } else { + return parseDuration(this).toMillis(); + } + } + + public boolean isPeriod() { + return unit.isDateBased() && !ChronoUnit.DAYS.equals(unit); + } + } + + public static Time parseTime(String text) { + checkNotNull(text); + + final String trimmed = text.trim(); + checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string"); + + final int len = trimmed.length(); + int pos = 0; + + char current; + while (pos < len && (current = trimmed.charAt(pos)) >= '0' && current <= '9') { + pos++; + } + + String number = trimmed.substring(0, pos); + String unitLabel = trimmed.substring(pos).trim().toLowerCase(Locale.US); + + return new Time(number, unitLabel); + } + + /** + * Parse the given string to a java {@link Duration}. The string is in format "{length value}{time + * unit label}", e.g. "123ms", "321 s". If no time unit label is specified, it will be considered + * as milliseconds. + * + * <p>Supported time unit labels are: + * + * <ul> + * <li>DAYS： "d", "day" + * <li>HOURS： "h", "hour" + * <li>MINUTES： "min", "minute" + * <li>SECONDS： "s", "sec", "second" + * <li>MILLISECONDS： "ms", "milli", "millisecond" + * <li>MICROSECONDS： "µs", "micro", "microsecond" + * <li>NANOSECONDS： "ns", "nano", "nanosecond" + * </ul> + * + * @param text string to parse. + */ + public static Duration parseDuration(String text) { + return parseDuration(parseTime(text)); + } + + public static Duration parseDuration(Time time) { + if (time.isPeriod()) { + throw new IllegalArgumentException( + "The time unit '" + time.getUnit() + "' cannot parse to Duration"); + } + + try { + long value = Long.parseLong(time.getNumber()); + return Duration.of(value, time.getUnit()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "The value '" + + time.getNumber() + + "' cannot be re represented as 64bit number (numeric overflow)."); + } + } + + /** + * Parse the given string to a java {@link Period}. The string is in format "{length value}{time + * unit label}", e.g. "13m", "1 y". + * + * <p>Supported time unit labels are: + * + * <ul> + * <li>MONTHS: "m", "month" + * <li>YEARS: "y", "year" + * </ul> + * + * @param text string to parse. + */ + public static Period parsePeriod(String text) { + return parsePeriod(parseTime(text)); + } + + public static Period parsePeriod(Time time) { + if (!time.isPeriod()) { + throw new IllegalArgumentException( + "The time unit '" + time.getUnit() + "' cannot parse to Period"); + } + + try { + int value = Integer.parseInt(time.getNumber()); + switch (time.getUnit()) { + case MONTHS: + return Period.ofMonths(value); + case YEARS: + return Period.ofYears(value); + default: + throw new IllegalArgumentException("Unsupported time unit: " + time.getUnit()); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "The value '" + + time.getNumber() + + "' cannot be re represented as 32bit number (numeric overflow)."); + } + } + + /** + * Pretty prints the duration as a lowest granularity unit that does not lose precision. + * + * <p>Examples: + * + * <pre>{@code + * Duration.ofMilliseconds(60000) will be printed as 1 min + * Duration.ofHours(1).plusSeconds(1) will be printed as 3601 s + * }</pre> + * + * <b>NOTE:</b> It supports only durations that fit into long. + */ + public static String formatWithHighestUnit(Duration duration) { + long nanos = duration.toNanos(); + + List<TimeUnit> orderedUnits = + Arrays.asList( + TimeUnit.NANOSECONDS, + TimeUnit.MICROSECONDS, + TimeUnit.MILLISECONDS, + TimeUnit.SECONDS, + TimeUnit.MINUTES, + TimeUnit.HOURS, + TimeUnit.DAYS); + + TimeUnit highestIntegerUnit = + IntStream.range(0, orderedUnits.size()) + .sequential() + .filter(idx -> nanos % orderedUnits.get(idx).unit.getDuration().toNanos() != 0) + .boxed() + .findFirst() + .map( + idx -> { + if (idx == 0) { + return orderedUnits.get(0); + } else { + return orderedUnits.get(idx - 1); + } + }) + .orElse(TimeUnit.MILLISECONDS); + + return String.format( + "%d %s", + nanos / highestIntegerUnit.unit.getDuration().toNanos(), + highestIntegerUnit.getLabels().get(0)); + } + + private static ChronoUnit toChronoUnit(java.util.concurrent.TimeUnit timeUnit) { + switch (timeUnit) { + case NANOSECONDS: + return ChronoUnit.NANOS; + case MICROSECONDS: + return ChronoUnit.MICROS; + case MILLISECONDS: + return ChronoUnit.MILLIS; + case SECONDS: + return ChronoUnit.SECONDS; + case MINUTES: + return ChronoUnit.MINUTES; + case HOURS: + return ChronoUnit.HOURS; + case DAYS: + return ChronoUnit.DAYS; + default: + throw new IllegalArgumentException(String.format("Unsupported time unit %s.", timeUnit)); + } + } + + /** Enum which defines time unit, mostly used to parse value from configuration file. */ + private enum TimeUnit { + YEARS(ChronoUnit.YEARS, singular("y"), plural("year")), + MONTHS(ChronoUnit.MONTHS, singular("m"), plural("month")), + DAYS(ChronoUnit.DAYS, singular("d"), plural("day")), + HOURS(ChronoUnit.HOURS, singular("h"), plural("hour")), + MINUTES(ChronoUnit.MINUTES, singular("min"), plural("minute")), + SECONDS(ChronoUnit.SECONDS, singular("s"), plural("sec"), plural("second")), + MILLISECONDS(ChronoUnit.MILLIS, singular("ms"), plural("milli"), plural("millisecond")), + MICROSECONDS(ChronoUnit.MICROS, singular("µs"), plural("micro"), plural("microsecond")), + NANOSECONDS(ChronoUnit.NANOS, singular("ns"), plural("nano"), plural("nanosecond")); + + private static final String PLURAL_SUFFIX = "s"; + + private final List<String> labels; + + private final ChronoUnit unit; + + TimeUnit(ChronoUnit unit, String[]... labels) { + this.unit = unit; + this.labels = Arrays.stream(labels).flatMap(Arrays::stream).collect(Collectors.toList()); + } + + /** + * @param label the original label + * @return the singular format of the original label + */ + private static String[] singular(String label) { + return new String[] {label}; + } + + /** + * @param label the original label + * @return both the singular format and plural format of the original label + */ + private static String[] plural(String label) { + return new String[] {label, label + PLURAL_SUFFIX}; + } + + public static String getAllUnits() { + return Arrays.stream(TimeUnit.values()) + .map(TimeUnit::createTimeUnitString) + .collect(Collectors.joining(", ")); + } + + private static String createTimeUnitString(TimeUnit timeUnit) { + return timeUnit.name() + ": (" + String.join(" | ", timeUnit.getLabels()) + ")"; + } + + public List<String> getLabels() { + return labels; + } + + public ChronoUnit getUnit() { + return unit; + } + } +}
