DanielZhu58 commented on code in PR #6198:
URL: https://github.com/apache/hive/pull/6198#discussion_r2921703913
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java:
##########
@@ -2760,6 +2763,165 @@ public List<Void> run(List<String> input) throws
Exception {
return true;
}
+ /**
+ a helper function which will get the current COLUMN_STATS_ACCURATE
parameter on table level
+ and update the COLUMN_STATS_ACCURATE parameter with the new value on
table level using directSql
+ */
+ public long updateColumnStatsAccurateForTable(Table table, List<String>
droppedCols) throws MetaException {
+ Map<String, String> params = table.getParameters();
+ // get the current COLUMN_STATS_ACCURATE
+ String currentValue = params.get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+ if (currentValue == null) {
+ return 0;
+ }
+ try {
+ // if the dropping columns is empty, that means we delete all the columns
+ if (droppedCols == null || droppedCols.isEmpty()) {
+ StatsSetupConst.clearColumnStatsState(params);
+ } else {
+ StatsSetupConst.removeColumnStatsState(params, droppedCols);
+ }
+
+ String updatedValue = params.get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+ // if the COL_STATS_ACCURATE has changed, then update it using directSql
+ if (currentValue.equals(updatedValue)) {
+ return 0;
+ }
+ return updateTableParam(table, StatsSetupConst.COLUMN_STATS_ACCURATE,
currentValue, updatedValue);
+ } catch (Exception e) {
+ throw new MetaException("Failed to parse/update COLUMN_STATS_ACCURATE: "
+ e.getMessage());
+ }
+ }
+
+
+
+ public boolean updateColumnStatsAccurateForPartitions(String catName, String
dbName, Table table,
+ List<String> partNames,
List<String> colNames) throws MetaException {
+ if (partNames == null || partNames.isEmpty()) {
+ return true;
+ }
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ // If colNames is empty, then all the column stats of all columns should
be deleted fetch all table column names
+ List<String> effectiveColNames;
+ if (colNames == null || colNames.isEmpty()) {
+ if (table.getSd().getCols() == null) {
+ effectiveColNames = new ArrayList<>();
+ } else {
+ effectiveColNames = table.getSd().getCols().stream()
+ .map(f -> f.getName().toLowerCase())
+ .collect(Collectors.toList());
+ }
+ } else {
+ effectiveColNames =
colNames.stream().map(String::toLowerCase).collect(Collectors.toList());
+ }
+
+ try {
+ Batchable.runBatched(batchSize, partNames, new Batchable<String, Void>()
{
+ @Override
+ public List<Void> run(List<String> input) throws Exception {
+ // 1. Construct SQL filter for partition names
+ String sqlFilter = PARTITIONS + ".\"PART_NAME\" in (" +
makeParams(input.size()) + ")";
+
+ // 2. Fetch PART_IDs of the partitions which are need to be changed
+ List<Long> partitionIds = getPartitionIdsViaSqlFilter(
+ catName, dbName, table.getTableName(), sqlFilter, input,
Collections.emptyList(), -1);
+
+ if (partitionIds.isEmpty()) return null;
+
+ // 3. Get current COLUMN_STATS_ACCURATE values
+ Map<Long, String> partStatsAccurateMap =
getColumnStatsAccurateByPartitionIds(partitionIds);
+
+ // 4. Iterate each partition to update COLUMN_STATS_ACCURATE
+ for (Long partId : partitionIds) {
+ String currentValue = partStatsAccurateMap.get(partId);
+ if (currentValue == null) continue;
+
+ try {
+ Map<String, Object> statsMap = mapper.readValue(
+ currentValue, new TypeReference<Map<String, Object>>()
{});
+ Object columnStatsObj = statsMap.get("COLUMN_STATS");
+
+ boolean changed = false;
+ if (columnStatsObj instanceof Map) {
+ Map<String, String> columnStats = (Map<String, String>)
columnStatsObj;
+ for (String col : effectiveColNames) {
+ if (columnStats.remove(col) != null) {
+ changed = true;
+ }
+ }
+
+ if (columnStats.isEmpty()) {
+ statsMap.remove("COLUMN_STATS");
+ changed = true;
+ }
+ }
+
+ if (!statsMap.containsKey("COLUMN_STATS")) {
+ if (statsMap.remove("BASIC_STATS") != null) {
Review Comment:
Acknowledged.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
