This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 62d09d7 Avoid write failures if metrics mode is invalid (#301)
62d09d7 is described below
commit 62d09d7fcd9bc50f2a691fb4ad3dc73bc3491807
Author: Ryan Blue <[email protected]>
AuthorDate: Thu Aug 1 13:48:02 2019 -0700
Avoid write failures if metrics mode is invalid (#301)
---
.../java/org/apache/iceberg/MetricsConfig.java | 29 +++++++++++++++++-----
.../java/org/apache/iceberg/TableProperties.java | 1 +
.../java/org/apache/iceberg/TestMetricsModes.java | 24 ++++++++++++++++++
3 files changed, 48 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java
b/core/src/main/java/org/apache/iceberg/MetricsConfig.java
index 864ec06..816f23e 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java
@@ -22,13 +22,15 @@ package org.apache.iceberg;
import com.google.common.collect.Maps;
import java.util.Map;
import org.apache.iceberg.MetricsModes.MetricsMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.iceberg.TableProperties.DEFAULT_WRITE_METRICS_MODE;
import static
org.apache.iceberg.TableProperties.DEFAULT_WRITE_METRICS_MODE_DEFAULT;
public class MetricsConfig {
- private static final String COLUMN_CONF_PREFIX =
"write.metadata.metrics.column.";
+ private static final Logger LOG =
LoggerFactory.getLogger(MetricsConfig.class);
private Map<String, MetricsMode> columnModes = Maps.newHashMap();
private MetricsMode defaultMode;
@@ -43,15 +45,30 @@ public class MetricsConfig {
public static MetricsConfig fromProperties(Map<String, String> props) {
MetricsConfig spec = new MetricsConfig();
+ String defaultModeAsString =
props.getOrDefault(DEFAULT_WRITE_METRICS_MODE,
DEFAULT_WRITE_METRICS_MODE_DEFAULT);
+ try {
+ spec.defaultMode = MetricsModes.fromString(defaultModeAsString);
+ } catch (IllegalArgumentException err) {
+ // Mode was invalid, log the error and use the default
+ LOG.warn("Ignoring invalid default metrics mode: {}",
defaultModeAsString, err);
+ spec.defaultMode =
MetricsModes.fromString(DEFAULT_WRITE_METRICS_MODE_DEFAULT);
+ }
+
props.keySet().stream()
- .filter(key -> key.startsWith(COLUMN_CONF_PREFIX))
+ .filter(key ->
key.startsWith(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX))
.forEach(key -> {
- MetricsMode mode = MetricsModes.fromString(props.get(key));
- String columnAlias = key.replaceFirst(COLUMN_CONF_PREFIX, "");
+ String columnAlias =
key.replaceFirst(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX, "");
+ MetricsMode mode;
+ try {
+ mode = MetricsModes.fromString(props.get(key));
+ } catch (IllegalArgumentException err) {
+ // Mode was invalid, log the error and use the default
+ LOG.warn("Ignoring invalid metrics mode for column {}: {}",
columnAlias, props.get(key), err);
+ mode = spec.defaultMode;
+ }
spec.columnModes.put(columnAlias, mode);
});
- String defaultModeAsString =
props.getOrDefault(DEFAULT_WRITE_METRICS_MODE,
DEFAULT_WRITE_METRICS_MODE_DEFAULT);
- spec.defaultMode = MetricsModes.fromString(defaultModeAsString);
+
return spec;
}
diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java
b/core/src/main/java/org/apache/iceberg/TableProperties.java
index 2ca8a25..b1e4ab0 100644
--- a/core/src/main/java/org/apache/iceberg/TableProperties.java
+++ b/core/src/main/java/org/apache/iceberg/TableProperties.java
@@ -89,6 +89,7 @@ public class TableProperties {
public static final String METADATA_COMPRESSION =
"write.metadata.compression-codec";
public static final String METADATA_COMPRESSION_DEFAULT = "none";
+ public static final String METRICS_MODE_COLUMN_CONF_PREFIX =
"write.metadata.metrics.column.";
public static final String DEFAULT_WRITE_METRICS_MODE =
"write.metadata.metrics.default";
public static final String DEFAULT_WRITE_METRICS_MODE_DEFAULT =
"truncate(16)";
diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
b/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
index 03b6bb7..698f5c3 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
@@ -19,6 +19,8 @@
package org.apache.iceberg;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
import org.apache.iceberg.MetricsModes.Counts;
import org.apache.iceberg.MetricsModes.Full;
import org.apache.iceberg.MetricsModes.None;
@@ -51,4 +53,26 @@ public class TestMetricsModes {
exceptionRule.expectMessage("length should be positive");
MetricsModes.fromString("truncate(0)");
}
+
+ @Test
+ public void testInvalidColumnModeValue() {
+ Map<String, String> properties = ImmutableMap.of(
+ TableProperties.DEFAULT_WRITE_METRICS_MODE, "full",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col",
"troncate(5)");
+
+ MetricsConfig config = MetricsConfig.fromProperties(properties);
+ Assert.assertEquals("Invalid mode should be defaulted to table default
(full)",
+ MetricsModes.Full.get(), config.columnMode("col"));
+ }
+
+ @Test
+ public void testInvalidDefaultColumnModeValue() {
+ Map<String, String> properties = ImmutableMap.of(
+ TableProperties.DEFAULT_WRITE_METRICS_MODE, "fuull",
+ TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col",
"troncate(5)");
+
+ MetricsConfig config = MetricsConfig.fromProperties(properties);
+ Assert.assertEquals("Invalid mode should be defaulted to library default
(truncate(16))",
+ MetricsModes.Truncate.withLength(16), config.columnMode("col"));
+ }
}
