This is an automated email from the ASF dual-hosted git repository. xuba pushed a commit to branch v0.7.x-test-front in repository https://gitbox.apache.org/repos/asf/amoro.git
commit 46ca9d31086efb405266de3a693dc828b5cd4e89 Author: Congxian Qiu <[email protected]> AuthorDate: Wed Apr 3 16:35:11 2024 +0800 [AMORO-1747] Ban guava direct import when building (#2679) * [AMORO-1747] Ban guava direct import when building * spotless:apply for trino mixed format * fix can find class * fix all invalid guava import --------- resolve conflicts use iceberg-relocated Co-authored-by: Qishang Zhong <[email protected]> (cherry picked from commit 0c5a17368cce5a39292f52d316a20821d64a237e) --- .../netease/arctic/api/config/ConfigHelpers.java | 230 ++++++++++++++++++++- .../maintainer/IcebergTableMaintainer.java | 2 +- .../maintainer/MixedTableMaintainer.java | 2 +- .../java/com/netease/arctic/utils/TimeUtils.java | 6 +- tools/maven/suppressions.xml | 58 +++--- 5 files changed, 257 insertions(+), 41 deletions(-) diff --git a/ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java old mode 100755 new mode 100644 index def707f4d..693c55084 --- a/ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java @@ -16,31 +16,34 @@ * limitations under the License. */ -package com.netease.arctic.server.utils; +package com.netease.arctic.api.config; -import static com.google.common.base.Preconditions.checkArgument; - -import com.netease.arctic.utils.TimeUtils; +import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkArgument; +import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull; import javax.annotation.Nonnull; import java.io.File; import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; /** Utility class for {@link Configurations} related helper functions. */ -public class ConfigurationUtil { +public class ConfigHelpers { private static final String[] EMPTY = new String[0]; // Make sure that we cannot instantiate this class - private ConfigurationUtil() {} + private ConfigHelpers() {} /** * Creates a new {@link Configurations} from the given {@link Properties}. @@ -289,4 +292,219 @@ public class ConfigurationUtil { return Double.parseDouble(o.toString()); } + + /** Collection of utilities about time intervals. */ + public static class TimeUtils { + + private static final Map<String, ChronoUnit> LABEL_TO_UNIT_MAP = + Collections.unmodifiableMap(initMap()); + + /** + * Parse the given string to a java {@link Duration}. The string is in format "{length + * value}{time unit label}", e.g. "123ms", "321 s". If no time unit label is specified, it will + * be considered as milliseconds. + * + * <p>Supported time unit labels are: + * + * <ul> + * <li>DAYS： "d", "day" + * <li>HOURS： "h", "hour" + * <li>MINUTES： "min", "minute" + * <li>SECONDS： "s", "sec", "second" + * <li>MILLISECONDS： "ms", "milli", "millisecond" + * <li>MICROSECONDS： "µs", "micro", "microsecond" + * <li>NANOSECONDS： "ns", "nano", "nanosecond" + * </ul> + * + * @param text string to parse. + */ + public static Duration parseDuration(String text) { + checkNotNull(text); + + final String trimmed = text.trim(); + checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string"); + + final int len = trimmed.length(); + int pos = 0; + + char current; + while (pos < len && (current = trimmed.charAt(pos)) >= '0' && current <= '9') { + pos++; + } + + final String number = trimmed.substring(0, pos); + final String unitLabel = trimmed.substring(pos).trim().toLowerCase(Locale.US); + + if (number.isEmpty()) { + throw new NumberFormatException("text does not start with a number"); + } + + final long value; + try { + value = Long.parseLong(number); // this throws a NumberFormatException on overflow + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "The value '" + + number + + "' cannot be re represented as 64bit number (numeric overflow)."); + } + + if (unitLabel.isEmpty()) { + return Duration.of(value, ChronoUnit.MILLIS); + } + + ChronoUnit unit = LABEL_TO_UNIT_MAP.get(unitLabel); + if (unit != null) { + return Duration.of(value, unit); + } else { + throw new IllegalArgumentException( + "Time interval unit label '" + + unitLabel + + "' does not match any of the recognized units: " + + TimeUnit.getAllUnits()); + } + } + + private static Map<String, ChronoUnit> initMap() { + Map<String, ChronoUnit> labelToUnit = new HashMap<>(); + for (TimeUnit timeUnit : TimeUnit.values()) { + for (String label : timeUnit.getLabels()) { + labelToUnit.put(label, timeUnit.getUnit()); + } + } + return labelToUnit; + } + + /** + * @param duration to convert to string + * @return duration string in millis + */ + public static String getStringInMillis(final Duration duration) { + return duration.toMillis() + TimeUnit.MILLISECONDS.labels.get(0); + } + + /** + * Pretty prints the duration as a lowest granularity unit that does not lose precision. + * + * <p>Examples: + * + * <pre>{@code + * Duration.ofMilliseconds(60000) will be printed as 1 min + * Duration.ofHours(1).plusSeconds(1) will be printed as 3601 s + * }</pre> + * + * <b>NOTE:</b> It supports only durations that fit into long. + */ + public static String formatWithHighestUnit(Duration duration) { + long nanos = duration.toNanos(); + + List<TimeUnit> orderedUnits = + Arrays.asList( + TimeUnit.NANOSECONDS, + TimeUnit.MICROSECONDS, + TimeUnit.MILLISECONDS, + TimeUnit.SECONDS, + TimeUnit.MINUTES, + TimeUnit.HOURS, + TimeUnit.DAYS); + + TimeUnit highestIntegerUnit = + IntStream.range(0, orderedUnits.size()) + .sequential() + .filter(idx -> nanos % orderedUnits.get(idx).unit.getDuration().toNanos() != 0) + .boxed() + .findFirst() + .map( + idx -> { + if (idx == 0) { + return orderedUnits.get(0); + } else { + return orderedUnits.get(idx - 1); + } + }) + .orElse(TimeUnit.MILLISECONDS); + + return String.format( + "%d %s", + nanos / highestIntegerUnit.unit.getDuration().toNanos(), + highestIntegerUnit.getLabels().get(0)); + } + + private static ChronoUnit toChronoUnit(java.util.concurrent.TimeUnit timeUnit) { + switch (timeUnit) { + case NANOSECONDS: + return ChronoUnit.NANOS; + case MICROSECONDS: + return ChronoUnit.MICROS; + case MILLISECONDS: + return ChronoUnit.MILLIS; + case SECONDS: + return ChronoUnit.SECONDS; + case MINUTES: + return ChronoUnit.MINUTES; + case HOURS: + return ChronoUnit.HOURS; + case DAYS: + return ChronoUnit.DAYS; + default: + throw new IllegalArgumentException(String.format("Unsupported time unit %s.", timeUnit)); + } + } + + /** Enum which defines time unit, mostly used to parse value from configuration file. */ + private enum TimeUnit { + DAYS(ChronoUnit.DAYS, singular("d"), plural("day")), + HOURS(ChronoUnit.HOURS, singular("h"), plural("hour")), + MINUTES(ChronoUnit.MINUTES, singular("min"), plural("minute")), + SECONDS(ChronoUnit.SECONDS, singular("s"), plural("sec"), plural("second")), + MILLISECONDS(ChronoUnit.MILLIS, singular("ms"), plural("milli"), plural("millisecond")), + MICROSECONDS(ChronoUnit.MICROS, singular("µs"), plural("micro"), plural("microsecond")), + NANOSECONDS(ChronoUnit.NANOS, singular("ns"), plural("nano"), plural("nanosecond")); + + private static final String PLURAL_SUFFIX = "s"; + + private final List<String> labels; + + private final ChronoUnit unit; + + TimeUnit(ChronoUnit unit, String[]... labels) { + this.unit = unit; + this.labels = Arrays.stream(labels).flatMap(Arrays::stream).collect(Collectors.toList()); + } + + /** + * @param label the original label + * @return the singular format of the original label + */ + private static String[] singular(String label) { + return new String[] {label}; + } + + /** + * @param label the original label + * @return both the singular format and plural format of the original label + */ + private static String[] plural(String label) { + return new String[] {label, label + PLURAL_SUFFIX}; + } + + public static String getAllUnits() { + return Arrays.stream(TimeUnit.values()) + .map(TimeUnit::createTimeUnitString) + .collect(Collectors.joining(", ")); + } + + private static String createTimeUnitString(TimeUnit timeUnit) { + return timeUnit.name() + ": (" + String.join(" | ", timeUnit.getLabels()) + ")"; + } + + public List<String> getLabels() { + return labels; + } + + public ChronoUnit getUnit() { + return unit; + } + } + } } diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java index 4ffae9ae7..13dbb8b8f 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/IcebergTableMaintainer.java @@ -270,7 +270,7 @@ public class IcebergTableMaintainer implements TableMaintainer { */ @VisibleForTesting protected void expireDataFrom(DataExpirationConfig expirationConfig, Instant instant) { - if (instant.equals(Instant.MIN) || !expirationConfig.isPositive()) { + if (instant.equals(Instant.MIN) || expirationConfig.getRetentionTime() == 0) { return; } diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java index 86954f37c..bb3f9083d 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/MixedTableMaintainer.java @@ -176,7 +176,7 @@ public class MixedTableMaintainer implements TableMaintainer { @VisibleForTesting protected void expireDataFrom(DataExpirationConfig expirationConfig, Instant instant) { - if (instant.equals(Instant.MIN) || !expirationConfig.isPositive()) { + if (instant.equals(Instant.MIN) || expirationConfig.getRetentionTime() == 0) { return; } diff --git a/core/src/main/java/com/netease/arctic/utils/TimeUtils.java b/core/src/main/java/com/netease/arctic/utils/TimeUtils.java index 089288b3f..94d6fefd6 100644 --- a/core/src/main/java/com/netease/arctic/utils/TimeUtils.java +++ b/core/src/main/java/com/netease/arctic/utils/TimeUtils.java @@ -18,11 +18,11 @@ package com.netease.arctic.utils; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkArgument; +import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull; -import com.google.common.base.Objects; import org.apache.commons.lang3.StringUtils; +import org.apache.iceberg.relocated.com.google.common.base.Objects; import java.time.Duration; import java.time.Period; diff --git a/tools/maven/suppressions.xml b/tools/maven/suppressions.xml index 5eb514294..e2722bca2 100644 --- a/tools/maven/suppressions.xml +++ b/tools/maven/suppressions.xml @@ -23,93 +23,91 @@ under the License. "http://www.puppycrawl.com/dtds/suppressions_1_1.dtd";> <suppressions> - <!-- Using ([/\\]) as the file path separator instead of (/) - enables it to be compatible with both Linux and Windows systems. --> <suppress - files="com[/\\]netease[/\\]arctic[/\\]utils[/\\]StatisticsFileUtil\.java" + files="com/netease/arctic/utils/StatisticsFileUtil\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]log[/\\]LogDataJsonSerialization\.java" + files="com/netease/arctic/log/LogDataJsonSerialization\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]log[/\\]LogDataToJsonConverters\.java" + files="com/netease/arctic/log/LogDataToJsonConverters\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]log[/\\]LogDataJsonDeserialization\.java" + files="com/netease/arctic/log/LogDataJsonDeserialization\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]log[/\\]JsonToLogDataConverters\.java" + files="com/netease/arctic/log/JsonToLogDataConverters\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]utils[/\\]TablePropertyUtil\.java" + files="com/netease/arctic/utils/TablePropertyUtil\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]api[/\\]config[/\\]OptimizingConfig\.java" + files="com/netease/arctic/api/config/OptimizingConfig\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]api[/\\]config[/\\]DataExpirationConfig\.java" + files="com/netease/arctic/api/config/DataExpirationConfig\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]api[/\\]config[/\\]TagConfiguration\.java" + files="com/netease/arctic/api/config/TagConfiguration\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]api[/\\]config[/\\]TableConfiguration\.java" + files="com/netease/arctic/api/config/TableConfiguration\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]server[/\\]RestCatalogService\.java" + files="com/netease/arctic/server/RestCatalogService\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]server[/\\]persistence[/\\]converter[/\\]JsonObjectConverter\.java" + files="com/netease/arctic/server/persistence/converter/JsonObjectConverter\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]server[/\\]optimizing[/\\]OptimizingConfig\.java" + files="com/netease/arctic/server/optimizing/OptimizingConfig\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]server[/\\]optimizing[/\\]plan[/\\]OptimizingEvaluator\.java" + files="com/netease/arctic/server/optimizing/plan/OptimizingEvaluator\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]server[/\\]optimizing[/\\]MetricsSummary\.java" + files="com/netease/arctic/server/optimizing/MetricsSummary\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]server[/\\]table[/\\]TagConfiguration\.java" + files="com/netease/arctic/server/table/TagConfiguration\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]server[/\\]table[/\\]TableConfiguration\.java" + files="com/netease/arctic/server/table/TableConfiguration\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]flink[/\\]interceptor[/\\]ProxyFactory\.java" + files="com/netease/arctic/flink/interceptor/ProxyFactory\.java" checks="TodoComment"/> <suppress - files="org[/\\]apache[/\\]hadoop[/\\]util[/\\]VersionInfo\.java" + files="org/apache/hadoop/util/VersionInfo\.java" checks="MethodName"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]iceberg[/\\]TestBaseArcticConnectorTest\.java" + files="com/netease/arctic/trino/iceberg/TestBaseArcticConnectorTest\.java" checks="FileLength"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]iceberg[/\\]BaseConnectorTest\.java" + files="com/netease/arctic/trino/iceberg/BaseConnectorTest\.java" checks="FileLength"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]keyed[/\\]KeyedConnectorSplit\.java" + files="com/netease/arctic/trino/keyed/KeyedConnectorSplit\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]keyed[/\\]KeyedTableHandle\.java" + files="com/netease/arctic/trino/keyed/KeyedTableHandle\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]delete[/\\]TrinoDeleteFile\.java" + files="com/netease/arctic/trino/delete/TrinoDeleteFile\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]unkeyed[/\\]IcebergSplit\.java" + files="com/netease/arctic/trino/unkeyed/IcebergSplit\.java" checks="Regexp"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]unkeyed[/\\]AdaptHiveIcebergTableHandle\.java" + files="com/netease/arctic/trino/unkeyed/AdaptHiveIcebergTableHandle\.java" checks="Regexp"/> <!-- BaseConnectorTest and TestBaseArcticConnectorTest can be replace by shaded guava because the shaded guava version is too low --> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]iceberg[/\\]BaseConnectorTest.java" + files="com/netease/arctic/trino/iceberg/BaseConnectorTest.java" checks="IllegalImport"/> <suppress - files="com[/\\]netease[/\\]arctic[/\\]trino[/\\]iceberg[/\\]TestBaseArcticConnectorTest.java" + files="com/netease/arctic/trino/iceberg/TestBaseArcticConnectorTest.java" checks="IllegalImport"/> </suppressions>
