[
https://issues.apache.org/jira/browse/HIVE-27183?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Rajesh Balamohan updated HIVE-27183:
------------------------------------
Description:
HMS::getTable invokes "HiveIcebergMetaHook::postGetTable" which internally
loads iceberg table again.
If this isn't needed or needed only for show-create-table, do not load the
table again.
Note: It looks like it invokes loadTable around 6 times during entire planning
(semAnalyzer, stats etc). Attached the snapshot for reference.
{noformat}
at jdk.internal.misc.Unsafe.park(java.base@11.0.18/Native Method)
- parking to wait for <0x000000066f84eef0> (a
java.util.concurrent.CompletableFuture$Signaller)
at
java.util.concurrent.locks.LockSupport.park(java.base@11.0.18/LockSupport.java:194)
at
java.util.concurrent.CompletableFuture$Signaller.block(java.base@11.0.18/CompletableFuture.java:1796)
at
java.util.concurrent.ForkJoinPool.managedBlock(java.base@11.0.18/ForkJoinPool.java:3128)
at
java.util.concurrent.CompletableFuture.waitingGet(java.base@11.0.18/CompletableFuture.java:1823)
at
java.util.concurrent.CompletableFuture.get(java.base@11.0.18/CompletableFuture.java:1998)
at org.apache.hadoop.util.functional.FutureIO.awaitFuture(FutureIO.java:77)
at
org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:196)
at org.apache.iceberg.TableMetadataParser.read(TableMetadataParser.java:263)
at org.apache.iceberg.TableMetadataParser.read(TableMetadataParser.java:258)
at
org.apache.iceberg.BaseMetastoreTableOperations.lambda$refreshFromMetadataLocation$0(BaseMetastoreTableOperations.java:177)
at
org.apache.iceberg.BaseMetastoreTableOperations$$Lambda$609/0x0000000840e18040.apply(Unknown
Source)
at
org.apache.iceberg.BaseMetastoreTableOperations.lambda$refreshFromMetadataLocation$1(BaseMetastoreTableOperations.java:191)
at
org.apache.iceberg.BaseMetastoreTableOperations$$Lambda$610/0x0000000840e18440.run(Unknown
Source)
at org.apache.iceberg.util.Tasks$Builder.runTaskWithRetry(Tasks.java:404)
at org.apache.iceberg.util.Tasks$Builder.runSingleThreaded(Tasks.java:214)
at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:198)
at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:190)
at
org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:191)
at
org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:176)
at
org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:171)
at
org.apache.iceberg.hive.HiveTableOperations.doRefresh(HiveTableOperations.java:153)
at
org.apache.iceberg.BaseMetastoreTableOperations.refresh(BaseMetastoreTableOperations.java:96)
at
org.apache.iceberg.BaseMetastoreTableOperations.current(BaseMetastoreTableOperations.java:79)
at
org.apache.iceberg.BaseMetastoreCatalog.loadTable(BaseMetastoreCatalog.java:44)
at org.apache.iceberg.mr.Catalogs.loadTable(Catalogs.java:115)
at org.apache.iceberg.mr.Catalogs.loadTable(Catalogs.java:105)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.lambda$getTable$1(IcebergTableUtil.java:99)
at
org.apache.iceberg.mr.hive.IcebergTableUtil$$Lambda$552/0x0000000840d59840.apply(Unknown
Source)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.lambda$getTable$4(IcebergTableUtil.java:111)
at
org.apache.iceberg.mr.hive.IcebergTableUtil$$Lambda$557/0x0000000840d58c40.get(Unknown
Source)
at java.util.Optional.orElseGet(java.base@11.0.18/Optional.java:369)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:108)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:69)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:73)
at
org.apache.iceberg.mr.hive.HiveIcebergMetaHook.postGetTable(HiveIcebergMetaHook.java:931)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.executePostGetTableHook(HiveMetaStoreClient.java:2638)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getTable(HiveMetaStoreClient.java:2624)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.getTable(SessionHiveMetaStoreClient.java:267)
at jdk.internal.reflect.GeneratedMethodAccessor137.invoke(Unknown Source)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.18/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@11.0.18/Method.java:566)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:216)
at com.sun.proxy.$Proxy56.getTable(Unknown Source)
at jdk.internal.reflect.GeneratedMethodAccessor137.invoke(Unknown Source)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.18/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@11.0.18/Method.java:566)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:4347)
- locked <0x00000006068632d0> (a
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler)
{noformat}
was:
HMS::getTable invokes "HiveIcebergMetaHook::postGetTable" which internally
loads iceberg table again.
If this isn't needed or needed only for show-create-table, do not load the
table again.
{noformat}
at jdk.internal.misc.Unsafe.park(java.base@11.0.18/Native Method)
- parking to wait for <0x000000066f84eef0> (a
java.util.concurrent.CompletableFuture$Signaller)
at
java.util.concurrent.locks.LockSupport.park(java.base@11.0.18/LockSupport.java:194)
at
java.util.concurrent.CompletableFuture$Signaller.block(java.base@11.0.18/CompletableFuture.java:1796)
at
java.util.concurrent.ForkJoinPool.managedBlock(java.base@11.0.18/ForkJoinPool.java:3128)
at
java.util.concurrent.CompletableFuture.waitingGet(java.base@11.0.18/CompletableFuture.java:1823)
at
java.util.concurrent.CompletableFuture.get(java.base@11.0.18/CompletableFuture.java:1998)
at org.apache.hadoop.util.functional.FutureIO.awaitFuture(FutureIO.java:77)
at
org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:196)
at org.apache.iceberg.TableMetadataParser.read(TableMetadataParser.java:263)
at org.apache.iceberg.TableMetadataParser.read(TableMetadataParser.java:258)
at
org.apache.iceberg.BaseMetastoreTableOperations.lambda$refreshFromMetadataLocation$0(BaseMetastoreTableOperations.java:177)
at
org.apache.iceberg.BaseMetastoreTableOperations$$Lambda$609/0x0000000840e18040.apply(Unknown
Source)
at
org.apache.iceberg.BaseMetastoreTableOperations.lambda$refreshFromMetadataLocation$1(BaseMetastoreTableOperations.java:191)
at
org.apache.iceberg.BaseMetastoreTableOperations$$Lambda$610/0x0000000840e18440.run(Unknown
Source)
at org.apache.iceberg.util.Tasks$Builder.runTaskWithRetry(Tasks.java:404)
at org.apache.iceberg.util.Tasks$Builder.runSingleThreaded(Tasks.java:214)
at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:198)
at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:190)
at
org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:191)
at
org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:176)
at
org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:171)
at
org.apache.iceberg.hive.HiveTableOperations.doRefresh(HiveTableOperations.java:153)
at
org.apache.iceberg.BaseMetastoreTableOperations.refresh(BaseMetastoreTableOperations.java:96)
at
org.apache.iceberg.BaseMetastoreTableOperations.current(BaseMetastoreTableOperations.java:79)
at
org.apache.iceberg.BaseMetastoreCatalog.loadTable(BaseMetastoreCatalog.java:44)
at org.apache.iceberg.mr.Catalogs.loadTable(Catalogs.java:115)
at org.apache.iceberg.mr.Catalogs.loadTable(Catalogs.java:105)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.lambda$getTable$1(IcebergTableUtil.java:99)
at
org.apache.iceberg.mr.hive.IcebergTableUtil$$Lambda$552/0x0000000840d59840.apply(Unknown
Source)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.lambda$getTable$4(IcebergTableUtil.java:111)
at
org.apache.iceberg.mr.hive.IcebergTableUtil$$Lambda$557/0x0000000840d58c40.get(Unknown
Source)
at java.util.Optional.orElseGet(java.base@11.0.18/Optional.java:369)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:108)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:69)
at
org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:73)
at
org.apache.iceberg.mr.hive.HiveIcebergMetaHook.postGetTable(HiveIcebergMetaHook.java:931)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.executePostGetTableHook(HiveMetaStoreClient.java:2638)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getTable(HiveMetaStoreClient.java:2624)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.getTable(SessionHiveMetaStoreClient.java:267)
at jdk.internal.reflect.GeneratedMethodAccessor137.invoke(Unknown Source)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.18/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@11.0.18/Method.java:566)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:216)
at com.sun.proxy.$Proxy56.getTable(Unknown Source)
at jdk.internal.reflect.GeneratedMethodAccessor137.invoke(Unknown Source)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.18/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@11.0.18/Method.java:566)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:4347)
- locked <0x00000006068632d0> (a
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler)
{noformat}
> Iceberg: Table information is loaded multiple times
> ---------------------------------------------------
>
> Key: HIVE-27183
> URL: https://issues.apache.org/jira/browse/HIVE-27183
> Project: Hive
> Issue Type: Improvement
> Reporter: Rajesh Balamohan
> Priority: Major
> Labels: performance
> Attachments: Screenshot 2023-03-28 at 8.13.52 AM.png,
> hs2_iceberg_load.html
>
>
> HMS::getTable invokes "HiveIcebergMetaHook::postGetTable" which internally
> loads iceberg table again.
> If this isn't needed or needed only for show-create-table, do not load the
> table again.
>
> Note: It looks like it invokes loadTable around 6 times during entire
> planning (semAnalyzer, stats etc). Attached the snapshot for reference.
>
> {noformat}
> at jdk.internal.misc.Unsafe.park(java.base@11.0.18/Native Method)
> - parking to wait for <0x000000066f84eef0> (a
> java.util.concurrent.CompletableFuture$Signaller)
> at
> java.util.concurrent.locks.LockSupport.park(java.base@11.0.18/LockSupport.java:194)
> at
> java.util.concurrent.CompletableFuture$Signaller.block(java.base@11.0.18/CompletableFuture.java:1796)
> at
> java.util.concurrent.ForkJoinPool.managedBlock(java.base@11.0.18/ForkJoinPool.java:3128)
> at
> java.util.concurrent.CompletableFuture.waitingGet(java.base@11.0.18/CompletableFuture.java:1823)
> at
> java.util.concurrent.CompletableFuture.get(java.base@11.0.18/CompletableFuture.java:1998)
> at
> org.apache.hadoop.util.functional.FutureIO.awaitFuture(FutureIO.java:77)
> at
> org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:196)
> at
> org.apache.iceberg.TableMetadataParser.read(TableMetadataParser.java:263)
> at
> org.apache.iceberg.TableMetadataParser.read(TableMetadataParser.java:258)
> at
> org.apache.iceberg.BaseMetastoreTableOperations.lambda$refreshFromMetadataLocation$0(BaseMetastoreTableOperations.java:177)
> at
> org.apache.iceberg.BaseMetastoreTableOperations$$Lambda$609/0x0000000840e18040.apply(Unknown
> Source)
> at
> org.apache.iceberg.BaseMetastoreTableOperations.lambda$refreshFromMetadataLocation$1(BaseMetastoreTableOperations.java:191)
> at
> org.apache.iceberg.BaseMetastoreTableOperations$$Lambda$610/0x0000000840e18440.run(Unknown
> Source)
> at org.apache.iceberg.util.Tasks$Builder.runTaskWithRetry(Tasks.java:404)
> at org.apache.iceberg.util.Tasks$Builder.runSingleThreaded(Tasks.java:214)
> at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:198)
> at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:190)
> at
> org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:191)
> at
> org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:176)
> at
> org.apache.iceberg.BaseMetastoreTableOperations.refreshFromMetadataLocation(BaseMetastoreTableOperations.java:171)
> at
> org.apache.iceberg.hive.HiveTableOperations.doRefresh(HiveTableOperations.java:153)
> at
> org.apache.iceberg.BaseMetastoreTableOperations.refresh(BaseMetastoreTableOperations.java:96)
> at
> org.apache.iceberg.BaseMetastoreTableOperations.current(BaseMetastoreTableOperations.java:79)
> at
> org.apache.iceberg.BaseMetastoreCatalog.loadTable(BaseMetastoreCatalog.java:44)
> at org.apache.iceberg.mr.Catalogs.loadTable(Catalogs.java:115)
> at org.apache.iceberg.mr.Catalogs.loadTable(Catalogs.java:105)
> at
> org.apache.iceberg.mr.hive.IcebergTableUtil.lambda$getTable$1(IcebergTableUtil.java:99)
> at
> org.apache.iceberg.mr.hive.IcebergTableUtil$$Lambda$552/0x0000000840d59840.apply(Unknown
> Source)
> at
> org.apache.iceberg.mr.hive.IcebergTableUtil.lambda$getTable$4(IcebergTableUtil.java:111)
> at
> org.apache.iceberg.mr.hive.IcebergTableUtil$$Lambda$557/0x0000000840d58c40.get(Unknown
> Source)
> at java.util.Optional.orElseGet(java.base@11.0.18/Optional.java:369)
> at
> org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:108)
> at
> org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:69)
> at
> org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:73)
> at
> org.apache.iceberg.mr.hive.HiveIcebergMetaHook.postGetTable(HiveIcebergMetaHook.java:931)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.executePostGetTableHook(HiveMetaStoreClient.java:2638)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getTable(HiveMetaStoreClient.java:2624)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.getTable(SessionHiveMetaStoreClient.java:267)
> at jdk.internal.reflect.GeneratedMethodAccessor137.invoke(Unknown Source)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.18/DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(java.base@11.0.18/Method.java:566)
> at
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:216)
> at com.sun.proxy.$Proxy56.getTable(Unknown Source)
> at jdk.internal.reflect.GeneratedMethodAccessor137.invoke(Unknown Source)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.18/DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(java.base@11.0.18/Method.java:566)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:4347)
> - locked <0x00000006068632d0> (a
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
