This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 81c02a72e88 HIVE-28972: HMS performace degradation post HIVE-28909 for
alter query (#5835)
81c02a72e88 is described below
commit 81c02a72e88dee8bf28a88b23401e9c3f1fad4a7
Author: dengzh <[email protected]>
AuthorDate: Tue Jun 3 11:05:46 2025 +0800
HIVE-28972: HMS performace degradation post HIVE-28909 for alter query
(#5835)
---
.../apache/hadoop/hive/metastore/HiveAlterHandler.java | 5 +++--
.../org/apache/hadoop/hive/metastore/ObjectStore.java | 15 +++++----------
.../org/apache/hadoop/hive/metastore/model/MColumn.java | 3 ++-
.../hadoop/hive/metastore/model/MColumnDescriptor.java | 5 +----
4 files changed, 11 insertions(+), 17 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index c9586dfe6b9..d814f82116a 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -407,6 +407,9 @@ public List<Void> run(List<Partition> input) throws
Exception {
MetastoreConf.getBoolVar(handler.getConf(),
MetastoreConf.ConfVars.COLSTATS_RETAIN_ON_COLUMN_REMOVAL);
if (runPartitionMetadataUpdate) {
+ // Don't validate table-level stats for a partitoned table.
+ msdb.alterTable(catName, dbname, name, newt, null);
+
if (cascade || retainOnColRemoval) {
parts = msdb.getPartitions(catName, dbname, name, -1);
for (Partition part : parts) {
@@ -431,8 +434,6 @@ public List<Void> run(List<Partition> input) throws
Exception {
TableName tableName = new TableName(catName, dbname, name);
msdb.deleteAllPartitionColumnStatistics(tableName, writeIdList);
}
- // Don't validate table-level stats for a partitoned table.
- msdb.alterTable(catName, dbname, name, newt, null);
} else {
LOG.warn("Alter table not cascaded to partitions.");
msdb.alterTable(catName, dbname, name, newt, writeIdList);
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index ab638722277..5e5b91d642a 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2715,7 +2715,7 @@ private void addPartitionsInternal(String catName, String
dbName,
throw new MetaException("Partition does not belong to target table "
+ dbName + "." + tblName + ": " + part);
}
- MPartition mpart = convertToMPart(part, table, true);
+ MPartition mpart = convertToMPart(part, table);
mParts.add(mpart);
int now = (int) (System.currentTimeMillis() / 1000);
List<MPartitionPrivilege> mPartPrivileges = new ArrayList<>();
@@ -2817,7 +2817,7 @@ public boolean addPartitions(String catName, String
dbName, String tblName,
Partition part = iterator.next();
if (isValidPartition(part, partitionKeys, ifNotExists)) {
- MPartition mpart = convertToMPart(part, table, true);
+ MPartition mpart = convertToMPart(part, table);
pm.makePersistent(mpart);
if (tabGrants != null) {
for (MTablePrivilege tab : tabGrants) {
@@ -3013,10 +3013,9 @@ private MPartition getMPartition(String catName, String
dbName, String tableName
* to the same one as the table's storage descriptor.
* @param part the partition to convert
* @param mt the parent table object
- * @param useTableCD whether to try to use the parent table's column
descriptor.
* @return the model partition object, and null if the input partition is
null.
*/
- private MPartition convertToMPart(Partition part, MTable mt, boolean
useTableCD)
+ private MPartition convertToMPart(Partition part, MTable mt)
throws InvalidObjectException, MetaException {
// NOTE: we don't set writeId in this method. Write ID is only set after
validating the
// existing write ID against the caller's valid list.
@@ -3032,8 +3031,7 @@ private MPartition convertToMPart(Partition part, MTable
mt, boolean useTableCD)
// use the parent table's, so we do not create a duplicate column
descriptor,
// thereby saving space
MStorageDescriptor msd;
- if (useTableCD &&
- mt.getSd() != null && mt.getSd().getCD() != null &&
+ if (mt.getSd() != null && mt.getSd().getCD() != null &&
mt.getSd().getCD().getCols() != null &&
part.getSd() != null &&
convertToFieldSchemas(mt.getSd().getCD().getCols()).
@@ -5188,7 +5186,7 @@ private Partition alterPartitionNoTxn(String catName,
String dbname,
catName = normalizeIdentifier(catName);
name = normalizeIdentifier(name);
dbname = normalizeIdentifier(dbname);
- MPartition newp = convertToMPart(newPart, table, false);
+ MPartition newp = convertToMPart(newPart, table);
MColumnDescriptor oldCD = null;
MStorageDescriptor oldSD = oldp.getSd();
if (oldSD != null) {
@@ -5248,9 +5246,6 @@ public Partition alterPartition(String catName, String
dbname, String name, List
Partition result = null;
try {
openTransaction();
- if (newPart.isSetWriteId()) {
- LOG.warn("Alter partitions with write ID called without transaction
information");
- }
Ref<MColumnDescriptor> oldCd = new Ref<>();
result = alterPartitionNoTxn(catName, dbname, name, part_vals, newPart,
validWriteIds, oldCd);
removeUnusedColumnDescriptor(oldCd.t);
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumn.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumn.java
index 8bdb81e9891..8fc4c0ff7af 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumn.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumn.java
@@ -62,7 +62,8 @@ public boolean equals(Object other) {
public MColumn() {
}
- public MColumn(String name, String type, String comment) {
+ public MColumn(MColumnDescriptor cd, String name, String type, String
comment) {
+ this.cd = cd;
this.name = name;
this.type = type;
this.comment = comment;
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumnDescriptor.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumnDescriptor.java
index d25eb272109..af5c19a0c5d 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumnDescriptor.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumnDescriptor.java
@@ -16,9 +16,6 @@
* limitations under the License.
*/
-/**
- *
- */
package org.apache.hadoop.hive.metastore.model;
import javax.jdo.annotations.NotPersistent;
@@ -42,7 +39,7 @@ public MColumnDescriptor() {}
public MColumnDescriptor(List<MFieldSchema> cols) {
fields = cols.stream().map(schema ->
- new MColumn(schema.getName(), schema.getType(), schema.getComment()))
+ new MColumn(this, schema.getName(), schema.getType(),
schema.getComment()))
.collect(Collectors.toList());
}
