This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 690384a19 [core] Rename metadata stats-mode option key
690384a19 is described below
commit 690384a19f091f3ff9945354f286d2da97933cef
Author: JingsongLi <[email protected]>
AuthorDate: Tue Jul 4 11:00:41 2023 +0800
[core] Rename metadata stats-mode option key
---
docs/content/how-to/creating-tables.md | 8 ++++----
docs/layouts/shortcodes/generated/core_configuration.html | 4 ++--
paimon-core/src/main/java/org/apache/paimon/CoreOptions.java | 11 ++++++-----
.../paimon/mergetree/compact/PartialUpdateMergeFunction.java | 7 ++++---
.../mergetree/compact/aggregate/AggregateMergeFunction.java | 6 +++---
.../src/main/java/org/apache/paimon/utils/StatsUtils.java | 8 +++-----
.../org/apache/paimon/utils/FieldStatsCollectorUtilsTest.java | 7 ++-----
7 files changed, 24 insertions(+), 27 deletions(-)
diff --git a/docs/content/how-to/creating-tables.md
b/docs/content/how-to/creating-tables.md
index 6121991a5..04795702c 100644
--- a/docs/content/how-to/creating-tables.md
+++ b/docs/content/how-to/creating-tables.md
@@ -235,18 +235,18 @@ The following three types of fields may be defined as
partition fields in the wa
if you declare the primary key containing partition field, you can achieve
the unique effect.
- CDC op_ts: It cannot be defined as a partition field, unable to know
previous record timestamp.
-### Specify the statistics collector mode
+### Specify Statistics Mode
Paimon will automatically collect the statistics of the data file for speeding
up the query process. There are four modes supported:
-- `full`: collect the full metrics: `null_count, min, max`
+- `full`: collect the full metrics: `null_count, min, max` .
- `truncate(length)`: length can be any positive number, the default mode is
`truncate(16)`, which means collect the null count, min/max value with
truncated length of 16.
This is mainly to avoid too big column which will enlarge the manifest file.
- `counts`: only collect the null count.
- `none`: disable the metadata stats collection.
-The statistics collector mode can also be configured at the field level by
setting [field.{field_name}.stats.mode]({{< ref
"maintenance/configurations#coreoptions" >}}).
-
+The statistics collector mode can be configured by `'metadata.stats-mode'`, by
default is `'truncate(16)'`.
+You can configure the field level by setting
`'fields.{field_name}.stats-mode'`.
## Create Table As
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 5417737df..0c15e6c5e 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -243,10 +243,10 @@ under the License.
<td>Specify the merge engine for table with primary key.<br /><br
/>Possible values:<ul><li>"deduplicate": De-duplicate and keep the last
row.</li><li>"partial-update": Partial update non-null
fields.</li><li>"aggregation": Aggregate fields with same primary
key.</li></ul></td>
</tr>
<tr>
- <td><h5>metadata.stats.mode</h5></td>
+ <td><h5>metadata.stats-mode</h5></td>
<td style="word-wrap: break-word;">"truncate(16)"</td>
<td>String</td>
- <td>The mode of metadata stats collection. none, counts,
truncate(16), full is available.<br /><ul><li>"none": means disable the
metadata stats collection.</li></ul><ul><li>"counts" means only collect the
null count.</li></ul><ul><li>"full": means collect the null count, min/max
value.</li></ul><ul><li>"truncate(16)": means collect the null count, min/max
value with truncated length of 16.</li></ul><ul><li>Field level stats mode can
be specified by field.{field_name}.stats.mo [...]
+ <td>The mode of metadata stats collection. none, counts,
truncate(16), full is available.<br /><ul><li>"none": means disable the
metadata stats collection.</li></ul><ul><li>"counts" means only collect the
null count.</li></ul><ul><li>"full": means collect the null count, min/max
value.</li></ul><ul><li>"truncate(16)": means collect the null count, min/max
value with truncated length of 16.</li></ul><ul><li>Field level stats mode can
be specified by fields.{field_name}.stats-m [...]
</tr>
<tr>
<td><h5>num-levels</h5></td>
diff --git a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
index d0baf4935..eb127d872 100644
--- a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
@@ -52,6 +52,8 @@ import static
org.apache.paimon.options.description.TextElement.text;
/** Core options for paimon. */
public class CoreOptions implements Serializable {
+ public static final String FIELDS_PREFIX = "fields";
+
public static final ConfigOption<Integer> BUCKET =
key("bucket")
.intType()
@@ -694,11 +696,10 @@ public class CoreOptions implements Serializable {
"Read incremental changes between start snapshot
(exclusive) and end snapshot, "
+ "for example, '5,10' means changes
between snapshot 5 and snapshot 10.");
- public static final String FIELD_STATS_MODE_PREFIX = "field";
- public static final String FIELD_STATS_MODE_SUFFIX = "stats.mode";
+ public static final String STATS_MODE_SUFFIX = "stats-mode";
public static final ConfigOption<String> STATS_MODE =
- key("metadata.stats.mode")
+ key("metadata." + STATS_MODE_SUFFIX)
.stringType()
.defaultValue("truncate(16)")
.withDescription(
@@ -719,10 +720,10 @@ public class CoreOptions implements Serializable {
.list(
text(
"Field level stats mode
can be specified by "
- +
FIELD_STATS_MODE_PREFIX
+ + FIELDS_PREFIX
+ "."
+ "{field_name}."
- +
FIELD_STATS_MODE_SUFFIX))
+ +
STATS_MODE_SUFFIX))
.build());
private final Options options;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java
index cb114970d..3bbcafd00 100644
---
a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java
+++
b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java
@@ -41,6 +41,7 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
+import static org.apache.paimon.CoreOptions.FIELDS_PREFIX;
import static org.apache.paimon.utils.InternalRowUtils.createFieldGetters;
/**
@@ -49,7 +50,6 @@ import static
org.apache.paimon.utils.InternalRowUtils.createFieldGetters;
*/
public class PartialUpdateMergeFunction implements MergeFunction<KeyValue> {
- public static final String FIELDS = "fields";
public static final String SEQUENCE_GROUP = "sequence-group";
private final InternalRow.FieldGetter[] getters;
@@ -167,10 +167,11 @@ public class PartialUpdateMergeFunction implements
MergeFunction<KeyValue> {
for (Map.Entry<String, String> entry : options.toMap().entrySet())
{
String k = entry.getKey();
String v = entry.getValue();
- if (k.startsWith(FIELDS) && k.endsWith(SEQUENCE_GROUP)) {
+ if (k.startsWith(FIELDS_PREFIX) && k.endsWith(SEQUENCE_GROUP))
{
String sequenceFieldName =
k.substring(
- FIELDS.length() + 1, k.length() -
SEQUENCE_GROUP.length() - 1);
+ FIELDS_PREFIX.length() + 1,
+ k.length() - SEQUENCE_GROUP.length() - 1);
SequenceGenerator sequenceGen =
new SequenceGenerator(sequenceFieldName, rowType);
Arrays.stream(v.split(","))
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/AggregateMergeFunction.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/AggregateMergeFunction.java
index a37751739..4dafc9004 100644
---
a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/AggregateMergeFunction.java
+++
b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/AggregateMergeFunction.java
@@ -32,6 +32,7 @@ import javax.annotation.Nullable;
import java.util.List;
+import static org.apache.paimon.CoreOptions.FIELDS_PREFIX;
import static org.apache.paimon.options.ConfigOptions.key;
import static org.apache.paimon.utils.InternalRowUtils.createFieldGetters;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
@@ -42,7 +43,6 @@ import static
org.apache.paimon.utils.Preconditions.checkNotNull;
*/
public class AggregateMergeFunction implements MergeFunction<KeyValue> {
- public static final String FIELDS = "fields";
public static final String AGG_FUNCTION = "aggregate-function";
public static final String IGNORE_RETRACT = "ignore-retract";
@@ -141,12 +141,12 @@ public class AggregateMergeFunction implements
MergeFunction<KeyValue> {
boolean isPrimaryKey = primaryKeys.contains(fieldName);
String strAggFunc =
conf.get(
- key(FIELDS + "." + fieldName + "." +
AGG_FUNCTION)
+ key(FIELDS_PREFIX + "." + fieldName + "." +
AGG_FUNCTION)
.stringType()
.noDefaultValue());
boolean ignoreRetract =
conf.get(
- key(FIELDS + "." + fieldName + "." +
IGNORE_RETRACT)
+ key(FIELDS_PREFIX + "." + fieldName + "." +
IGNORE_RETRACT)
.booleanType()
.defaultValue(false));
fieldAggregators[i] =
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/StatsUtils.java
b/paimon-core/src/main/java/org/apache/paimon/utils/StatsUtils.java
index 35141c6b7..ab3d5e1d7 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/StatsUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/StatsUtils.java
@@ -26,8 +26,8 @@ import org.apache.paimon.statistics.FieldStatsCollector;
import java.util.List;
-import static org.apache.paimon.CoreOptions.FIELD_STATS_MODE_PREFIX;
-import static org.apache.paimon.CoreOptions.FIELD_STATS_MODE_SUFFIX;
+import static org.apache.paimon.CoreOptions.FIELDS_PREFIX;
+import static org.apache.paimon.CoreOptions.STATS_MODE_SUFFIX;
import static org.apache.paimon.options.ConfigOptions.key;
/** The stats utils. */
@@ -44,9 +44,7 @@ public class StatsUtils {
cfg.get(
key(String.format(
"%s.%s.%s",
- FIELD_STATS_MODE_PREFIX,
- fields.get(i),
- FIELD_STATS_MODE_SUFFIX))
+ FIELDS_PREFIX, fields.get(i),
STATS_MODE_SUFFIX))
.stringType()
.noDefaultValue());
if (fieldMode != null) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/utils/FieldStatsCollectorUtilsTest.java
b/paimon-core/src/test/java/org/apache/paimon/utils/FieldStatsCollectorUtilsTest.java
index 00b4ec0c3..74ad41da2 100644
---
a/paimon-core/src/test/java/org/apache/paimon/utils/FieldStatsCollectorUtilsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/utils/FieldStatsCollectorUtilsTest.java
@@ -49,11 +49,8 @@ public class FieldStatsCollectorUtilsTest {
Options options = new Options();
options.set(
- CoreOptions.FIELD_STATS_MODE_PREFIX + ".b." +
CoreOptions.FIELD_STATS_MODE_SUFFIX,
- "truncate(12)");
- options.set(
- CoreOptions.FIELD_STATS_MODE_PREFIX + ".c." +
CoreOptions.FIELD_STATS_MODE_SUFFIX,
- "full");
+ CoreOptions.FIELDS_PREFIX + ".b." +
CoreOptions.STATS_MODE_SUFFIX, "truncate(12)");
+ options.set(CoreOptions.FIELDS_PREFIX + ".c." +
CoreOptions.STATS_MODE_SUFFIX, "full");
FieldStatsCollector[] stats =
StatsUtils.getFieldsStatsMode(new CoreOptions(options),
type.getFieldNames());
