This is an automated email from the ASF dual-hosted git repository. pvary pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 4641f6e HIVE-26101: Port Iceberg Hive fix - Hive: Avoid recursive listing in HiveCatalog#renameTable (Peter Vary reviewed by Marton Bod) (#3163) 4641f6e is described below commit 4641f6e484915cb8dfa5abedfc4732b68c88a758 Author: pvary <pv...@cloudera.com> AuthorDate: Fri Apr 1 13:51:27 2022 +0200 HIVE-26101: Port Iceberg Hive fix - Hive: Avoid recursive listing in HiveCatalog#renameTable (Peter Vary reviewed by Marton Bod) (#3163) --- .../java/org/apache/iceberg/hive/HiveCatalog.java | 2 +- .../apache/iceberg/hive/HiveTableOperations.java | 16 ++----------- .../org/apache/iceberg/hive/MetastoreUtil.java | 27 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index 880d60d..4737dd6 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -214,7 +214,7 @@ public class HiveCatalog extends BaseMetastoreCatalog implements SupportsNamespa table.setTableName(to.name()); clients.run(client -> { - client.alter_table(fromDatabase, fromName, table); + MetastoreUtil.alterTable(client, fromDatabase, fromName, table); return null; }); diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java index ccde59c..3ad450c 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockLevel; import org.apache.hadoop.hive.metastore.api.LockRequest; @@ -57,7 +56,6 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotSummary; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.common.DynMethods; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.CommitStateUnknownException; @@ -93,14 +91,7 @@ public class HiveTableOperations extends BaseMetastoreTableOperations { private static final long HIVE_LOCK_CHECK_MIN_WAIT_MS_DEFAULT = 50; // 50 milliseconds private static final long HIVE_LOCK_CHECK_MAX_WAIT_MS_DEFAULT = 5 * 1000; // 5 seconds private static final long HIVE_TABLE_LEVEL_LOCK_EVICT_MS_DEFAULT = TimeUnit.MINUTES.toMillis(10); - private static final DynMethods.UnboundMethod ALTER_TABLE = DynMethods.builder("alter_table") - .impl(IMetaStoreClient.class, "alter_table_with_environmentContext", - String.class, String.class, Table.class, EnvironmentContext.class) - .impl(IMetaStoreClient.class, "alter_table", - String.class, String.class, Table.class, EnvironmentContext.class) - .impl(IMetaStoreClient.class, "alter_table", - String.class, String.class, Table.class) - .build(); + private static final BiMap<String, String> ICEBERG_TO_HMS_TRANSLATION = ImmutableBiMap.of( // gc.enabled in Iceberg and external.table.purge in Hive are meant to do the same things but with different names GC_ENABLED, "external.table.purge" @@ -310,10 +301,7 @@ public class HiveTableOperations extends BaseMetastoreTableOperations { void persistTable(Table hmsTable, boolean updateHiveTable) throws TException, InterruptedException { if (updateHiveTable) { metaClients.run(client -> { - EnvironmentContext envContext = new EnvironmentContext( - ImmutableMap.of(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE) - ); - ALTER_TABLE.invoke(client, database, tableName, hmsTable, envContext); + MetastoreUtil.alterTable(client, database, tableName, hmsTable); return null; }); } else { diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index ad0ec80..76363f1 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -19,12 +19,28 @@ package org.apache.iceberg.hive; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.iceberg.common.DynMethods; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; + public class MetastoreUtil { // this class is unique to Hive3 and cannot be found in Hive2, therefore a good proxy to see if // we are working against Hive3 dependencies private static final String HIVE3_UNIQUE_CLASS = "org.apache.hadoop.hive.serde2.io.DateWritableV2"; + private static final DynMethods.UnboundMethod ALTER_TABLE = DynMethods.builder("alter_table") + .impl(IMetaStoreClient.class, "alter_table_with_environmentContext", + String.class, String.class, Table.class, EnvironmentContext.class) + .impl(IMetaStoreClient.class, "alter_table", + String.class, String.class, Table.class, EnvironmentContext.class) + .impl(IMetaStoreClient.class, "alter_table", + String.class, String.class, Table.class) + .build(); + private static final boolean HIVE3_PRESENT_ON_CLASSPATH = detectHive3(); private MetastoreUtil() { @@ -37,6 +53,17 @@ public class MetastoreUtil { return HIVE3_PRESENT_ON_CLASSPATH; } + /** + * Calls alter_table method using the metastore client. If possible, an environmental context will be used that + * turns off stats updates to avoid recursive listing. + */ + public static void alterTable(IMetaStoreClient client, String databaseName, String tblName, Table table) { + EnvironmentContext envContext = new EnvironmentContext( + ImmutableMap.of(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE) + ); + ALTER_TABLE.invoke(client, databaseName, tblName, table, envContext); + } + private static boolean detectHive3() { try { Class.forName(HIVE3_UNIQUE_CLASS);