This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6ce7ad1c749 HIVE-28886: Addendum: Column names in the DESCRIBE
FORMATTED stats output should be in lowercase (Denys Kuzmenko, reviewed by
Krisztian Kasa)
6ce7ad1c749 is described below
commit 6ce7ad1c74902ca01da5150fb1737d08db82681f
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Wed Apr 16 10:23:32 2025 +0300
HIVE-28886: Addendum: Column names in the DESCRIBE FORMATTED stats output
should be in lowercase (Denys Kuzmenko, reviewed by Krisztian Kasa)
Closes #5773
---
.../src/test/results/positive/col_stats.q.out | 2 +-
.../apache/hadoop/hive/common/StatsSetupConst.java | 37 ++++++----------------
.../hadoop/hive/common/TestStatsSetupConst.java | 4 +--
3 files changed, 12 insertions(+), 31 deletions(-)
diff --git a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
index 3ef9a6b9fe9..2ea758790f8 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
@@ -505,7 +505,7 @@ num_trues
num_falses
bit_vector HL
comment
-COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"A\":\"true\",\"C\":\"true\"}}
+COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"c\":\"true\"}}
PREHOOK: query: create table t1 (a int) stored by iceberg tblproperties
('format-version'='2')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
index aec1a3894f8..f2c6ef27f57 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
@@ -118,11 +118,6 @@ public String getAggregator(Configuration conf) {
*/
public static final String NUM_ERASURE_CODED_FILES = "numFilesErasureCoded";
- /**
- * Temp dir for writing stats from tasks.
- */
- public static final String STATS_TMP_LOC = "hive.stats.tmp.loc";
-
public static final String STATS_FILE_PREFIX = "tmpstats-";
/**
* List of all supported statistics
@@ -298,7 +293,7 @@ public static boolean areColumnStatsUptoDate(Map<String,
String> params, String
// note that set basic stats false will wipe out column stats too.
public static void setBasicStatsState(Map<String, String> params, String
setting) {
if (setting.equals(FALSE)) {
- if (params!=null && params.containsKey(COLUMN_STATS_ACCURATE)) {
+ if (params != null) {
params.remove(COLUMN_STATS_ACCURATE);
}
return;
@@ -323,12 +318,8 @@ public static void setColumnStatsState(Map<String, String>
params, List<String>
return;
}
ColumnStatsAccurate stats =
parseStatsAcc(params.get(COLUMN_STATS_ACCURATE));
-
- for (String colName : colNames) {
- if (!stats.columnStats.containsKey(colName)) {
- stats.columnStats.put(colName, true);
- }
- }
+ colNames.forEach(colName ->
+ stats.columnStats.putIfAbsent(colName.toLowerCase(), true));
try {
params.put(COLUMN_STATS_ACCURATE,
ColumnStatsAccurate.objectWriter.writeValueAsString(stats));
@@ -346,22 +337,13 @@ public static List<String>
getColumnsHavingStats(Map<String, String> params) {
// No table/partition params, no statistics available
return null;
}
-
ColumnStatsAccurate stats =
parseStatsAcc(params.get(COLUMN_STATS_ACCURATE));
// No stats available.
if (stats == null) {
return null;
}
-
- List<String> colNames = new ArrayList<String>();
- for (Map.Entry<String, Boolean> entry : stats.columnStats.entrySet()) {
- if (entry.getValue()) {
- colNames.add(entry.getKey());
- }
- }
-
- return colNames;
+ return ImmutableList.copyOf(stats.columnStats.keySet());
}
public static boolean canColumnStatsMerge(Map<String, String> params, String
colName) {
@@ -377,7 +359,6 @@ public static void clearColumnStatsState(Map<String,
String> params) {
if (params == null || params.get(COLUMN_STATS_ACCURATE) == null) {
return;
}
-
ColumnStatsAccurate stats =
parseStatsAcc(params.get(COLUMN_STATS_ACCURATE));
stats.columnStats.clear();
@@ -392,11 +373,11 @@ public static void removeColumnStatsState(Map<String,
String> params, List<Strin
if (params == null) {
return;
}
- try {
- ColumnStatsAccurate stats =
parseStatsAcc(params.get(COLUMN_STATS_ACCURATE));
- for (String string : colNames) {
- stats.columnStats.remove(string);
- }
+ ColumnStatsAccurate stats =
parseStatsAcc(params.get(COLUMN_STATS_ACCURATE));
+ colNames.forEach(colName ->
+ stats.columnStats.remove(colName.toLowerCase()));
+
+ try {
params.put(COLUMN_STATS_ACCURATE,
ColumnStatsAccurate.objectWriter.writeValueAsString(stats));
} catch (JsonProcessingException e) {
LOG.trace(e.getMessage());
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/common/TestStatsSetupConst.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/common/TestStatsSetupConst.java
index 24689215c2f..caa5ca82886 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/common/TestStatsSetupConst.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/common/TestStatsSetupConst.java
@@ -73,7 +73,7 @@ public void testStatColumnEntriesCompat() {
StatsSetupConst.setBasicStatsState(params0, String.valueOf(true));
StatsSetupConst.setColumnStatsState(params0, Lists.newArrayList("Foo"));
-
assertEquals("{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"Foo\":\"true\"}}",params0.get(StatsSetupConst.COLUMN_STATS_ACCURATE));
+
assertEquals("{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"foo\":\"true\"}}",params0.get(StatsSetupConst.COLUMN_STATS_ACCURATE));
}
@Test
@@ -109,6 +109,6 @@ public void testColumnEntries_areKept_whenBasicIsAbsent() {
Map<String, String> params=new HashMap<>();
StatsSetupConst.setBasicStatsState(params, String.valueOf(false));
StatsSetupConst.setColumnStatsState(params, Lists.newArrayList("Foo"));
-
assertEquals("{\"COLUMN_STATS\":{\"Foo\":\"true\"}}",params.get(StatsSetupConst.COLUMN_STATS_ACCURATE));
+
assertEquals("{\"COLUMN_STATS\":{\"foo\":\"true\"}}",params.get(StatsSetupConst.COLUMN_STATS_ACCURATE));
}
}
