This is an automated email from the ASF dual-hosted git repository. lpinter pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 8863468898c HIVE-26421: HMSClient atler_table_req() is called twice when running an alter operation on iceberg table (#3469) (Laszlo Pinter, reviewed by Adam Szita) 8863468898c is described below commit 8863468898c9242fc334b9c93bf79e249e53842d Author: László Pintér <47777102+lcspin...@users.noreply.github.com> AuthorDate: Fri Jul 29 10:57:57 2022 +0200 HIVE-26421: HMSClient atler_table_req() is called twice when running an alter operation on iceberg table (#3469) (Laszlo Pinter, reviewed by Adam Szita) --- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 6 ++- .../apache/hadoop/hive/metastore/HiveMetaHook.java | 2 + .../hadoop/hive/metastore/HiveMetaStoreClient.java | 24 ++++++----- .../hive/metastore/utils/MetaStoreUtils.java | 46 +++++++++++----------- 4 files changed, 46 insertions(+), 32 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 1191ca8e431..f3572673742 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -268,8 +268,8 @@ public class HiveIcebergMetaHook implements HiveMetaHook { @Override public void preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, EnvironmentContext context) throws MetaException { - setupAlterOperationType(hmsTable, context); catalogProperties = getCatalogProperties(hmsTable); + setupAlterOperationType(hmsTable, context); try { icebergTable = IcebergTableUtil.getTable(conf, catalogProperties); } catch (NoSuchTableException nte) { @@ -500,6 +500,10 @@ public class HiveIcebergMetaHook implements HiveMetaHook { "Unsupported ALTER TABLE operation type on Iceberg table " + tableName + ", must be one of: " + SUPPORTED_ALTER_OPS); } + + if (currentAlterTableOp != AlterTableType.ADDPROPS && Catalogs.hiveCatalog(conf, catalogProperties)) { + context.getProperties().put(SKIP_METASTORE_ALTER, "true"); + } } } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java index b2e15ee9a66..3accd3cd9ea 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java @@ -52,6 +52,8 @@ public interface HiveMetaHook { String PROPERTIES_SEPARATOR = "'"; String MIGRATE_HIVE_TO_ICEBERG = "migrate_hive_to_iceberg"; String INITIALIZE_ROLLBACK_MIGRATION = "initialize_rollback_migration"; + // if this flag is set to true, the HMS call from HiveMetaStoreClient#alter_table() will be skipped + String SKIP_METASTORE_ALTER = "skip_metastore_alter"; /** * Called before a new table definition is added to the metastore diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index ecf85fb817a..ab6a1aa9607 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -562,17 +562,23 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { if (hook != null) { hook.preAlterTable(new_tbl, envContext); } - AlterTableRequest req = new AlterTableRequest(dbName, tbl_name, new_tbl); - req.setCatName(catName); - req.setValidWriteIdList(validWriteIds); - req.setEnvironmentContext(envContext); - if (processorCapabilities != null) { - req.setProcessorCapabilities(new ArrayList<String>(Arrays.asList(processorCapabilities))); - req.setProcessorIdentifier(processorIdentifier); - } boolean success = false; try { - client.alter_table_req(req); + boolean skipAlter = envContext != null && envContext.getProperties() != null && + Boolean.valueOf(envContext.getProperties().getOrDefault(HiveMetaHook.SKIP_METASTORE_ALTER, "false")); + if (!skipAlter) { + AlterTableRequest req = new AlterTableRequest(dbName, tbl_name, new_tbl); + req.setCatName(catName); + req.setValidWriteIdList(validWriteIds); + req.setEnvironmentContext(envContext); + if (processorCapabilities != null) { + req.setProcessorCapabilities(new ArrayList<String>(Arrays.asList(processorCapabilities))); + req.setProcessorIdentifier(processorIdentifier); + } + + client.alter_table_req(req); + } + if (hook != null) { hook.commitAlterTable(new_tbl, envContext); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 6e1a8ef370e..d41310906a7 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -799,29 +799,31 @@ public class MetaStoreUtils { } } - String partString = StringUtils.EMPTY; - String partStringSep = StringUtils.EMPTY; - String partTypesString = StringUtils.EMPTY; - String partTypesStringSep = StringUtils.EMPTY; - for (FieldSchema partKey : partitionKeys) { - partString = partString.concat(partStringSep); - partString = partString.concat(partKey.getName()); - partTypesString = partTypesString.concat(partTypesStringSep); - partTypesString = partTypesString.concat(partKey.getType()); - if (partStringSep.length() == 0) { - partStringSep = "/"; - partTypesStringSep = ":"; + if (partitionKeys != null) { + String partString = StringUtils.EMPTY; + String partStringSep = StringUtils.EMPTY; + String partTypesString = StringUtils.EMPTY; + String partTypesStringSep = StringUtils.EMPTY; + for (FieldSchema partKey : partitionKeys) { + partString = partString.concat(partStringSep); + partString = partString.concat(partKey.getName()); + partTypesString = partTypesString.concat(partTypesStringSep); + partTypesString = partTypesString.concat(partKey.getType()); + if (partStringSep.length() == 0) { + partStringSep = "/"; + partTypesStringSep = ":"; + } + } + if (partString.length() > 0) { + schema + .setProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, + partString); + schema + .setProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES, + partTypesString); } - } - if (partString.length() > 0) { - schema - .setProperty( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, - partString); - schema - .setProperty( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES, - partTypesString); } if (parameters != null) {