[
https://issues.apache.org/jira/browse/HIVE-29268?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Work on HIVE-29268 started by Dmitriy Fingerman.
------------------------------------------------
> Iceberg: Close catalog in Catalogs.loadTable() to fix resource leak from open
> ResolvingFileIO
> ---------------------------------------------------------------------------------------------
>
> Key: HIVE-29268
> URL: https://issues.apache.org/jira/browse/HIVE-29268
> Project: Hive
> Issue Type: Improvement
> Components: Iceberg integration
> Reporter: Dmitriy Fingerman
> Assignee: Dmitriy Fingerman
> Priority: Major
>
> `Catalogs.loadTable()` loads an Iceberg catalog and then loads a table from
> that catalog by calling the`loadTable()` method on the loaded catalog.
> It doesn't close the catalog which causes exception like following in the
> hive.log:
> {code:java}
> 2025-10-14 16:52:12 2025-10-14T20:52:12,131 INFO
> [HiveServer2-Background-Pool: Thread-150] tez.TezSessionState: Opening new
> Tez Session (id: c8064268-e392-40ee-a062-008ade6a3b8e, scratch dir:
> file:/tmp/hive/hive/_tez_session_dir/c8064268-e392-40ee-a062-008ade6a3b8e)
> 2025-10-14 16:52:12 2025-10-14T20:52:12,142 INFO
> [HiveServer2-Background-Pool: Thread-150] client.TezClient: Session mode.
> Starting session.
> 2025-10-14 16:52:12 2025-10-14T20:52:12,171 INFO
> [HiveServer2-Background-Pool: Thread-150] client.TezClientUtils: Ignoring
> 'tez.lib.uris' since 'tez.ignore.lib.uris' is set to true
> 2025-10-14 16:52:12 2025-10-14T20:52:12,184 INFO
> [HiveServer2-Background-Pool: Thread-150] client.TezClient: Tez system stage
> directory
> file:/tmp/hive/hive/_tez_session_dir/c8064268-e392-40ee-a062-008ade6a3b8e/.tez/application_1760475132141_0001
> doesn't exist and is created
> 2025-10-14 16:52:12 2025-10-14T20:52:12,222 INFO
> [HiveServer2-Background-Pool: Thread-150] conf.Configuration:
> resource-types.xml not found
> 2025-10-14 16:52:12 2025-10-14T20:52:12,222 INFO
> [HiveServer2-Background-Pool: Thread-150] resource.ResourceUtils: Unable to
> find 'resource-types.xml'.
> 2025-10-14 16:52:12 2025-10-14T20:52:12,236 INFO
> [HiveServer2-Background-Pool: Thread-150] common.TezYARNUtils: Ignoring
> 'tez.lib.uris' since 'tez.ignore.lib.uris' is set to true
> 2025-10-14 16:52:12 2025-10-14T20:52:12,243 INFO
> [HiveServer2-Background-Pool: Thread-150] Configuration.deprecation:
> yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead,
> use yarn.system-metrics-publisher.enabled
> 2025-10-14 16:52:12 2025-10-14T20:52:12,323 INFO
> [HiveServer2-Background-Pool: Thread-150] client.LocalClient: DAGAppMaster
> thread has been created
> 2025-10-14 16:52:12 2025-10-14T20:52:12,323 INFO
> [HiveServer2-Background-Pool: Thread-150] client.LocalClient: DAGAppMaster is
> not created wait for 100ms...
> 2025-10-14 16:52:12 2025-10-14T20:52:12,323 INFO [DAGAppMaster Thread]
> client.LocalClient: Using working directory:
> /tmp/hive/hive/_tez_session_dir/c8064268-e392-40ee-a062-008ade6a3b8e/.tez/application_1760475132141_0001_wd
> 2025-10-14 16:52:12 2025-10-14T20:52:12,364 WARN [Finalizer]
> io.ResolvingFileIO: Unclosed ResolvingFileIO instance created by:
> 2025-10-14 16:52:12
> org.apache.iceberg.io.ResolvingFileIO.<init>(ResolvingFileIO.java:84)
> 2025-10-14 16:52:12
> java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(Unknown
> Source)
> 2025-10-14 16:52:12
> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Unknown Source)
> 2025-10-14 16:52:12
> java.base/java.lang.reflect.Constructor.newInstance(Unknown Source)
> 2025-10-14 16:52:12
> org.apache.iceberg.common.DynConstructors$Ctor.newInstanceChecked(DynConstructors.java:51)
> 2025-10-14 16:52:12
> org.apache.iceberg.common.DynConstructors$Ctor.newInstance(DynConstructors.java:64)
> 2025-10-14 16:52:12
> org.apache.iceberg.CatalogUtil.loadFileIO(CatalogUtil.java:391)
> 2025-10-14 16:52:12
> org.apache.iceberg.rest.RESTSessionCatalog.newFileIO(RESTSessionCatalog.java:983)
> 2025-10-14 16:52:12
> org.apache.iceberg.rest.RESTSessionCatalog.newFileIO(RESTSessionCatalog.java:974)
> 2025-10-14 16:52:12
> org.apache.iceberg.rest.RESTSessionCatalog.initialize(RESTSessionCatalog.java:233)
> 2025-10-14 16:52:12
> org.apache.iceberg.rest.RESTCatalog.initialize(RESTCatalog.java:82)
> 2025-10-14 16:52:12
> org.apache.iceberg.CatalogUtil.loadCatalog(CatalogUtil.java:277)
> 2025-10-14 16:52:12
> org.apache.iceberg.CatalogUtil.buildIcebergCatalog(CatalogUtil.java:331)
> 2025-10-14 16:52:12
> org.apache.iceberg.mr.Catalogs.loadCatalog(Catalogs.java:237)
> 2025-10-14 16:52:12
> org.apache.iceberg.mr.Catalogs.loadTable(Catalogs.java:110)
> 2025-10-14 16:52:12
> org.apache.iceberg.mr.Catalogs.loadTable(Catalogs.java:104)
> 2025-10-14 16:52:12
> org.apache.iceberg.mr.hive.IcebergTableUtil.lambda$getTable$1(IcebergTableUtil.java:178)
> 2025-10-14 16:52:12
> org.apache.iceberg.mr.hive.IcebergTableUtil.getTable(IcebergTableUtil.java:184)
> 2025-10-14 16:52:12
> org.apache.iceberg.mr.hive.BaseHiveIcebergMetaHook.preCreateTable(BaseHiveIcebergMetaHook.java:140)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.metastore.client.HookEnabledMetaStoreClient.createTable(HookEnabledMetaStoreClient.java:113)
> 2025-10-14 16:52:12
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Unknown
> Source)
> 2025-10-14 16:52:12 java.base/java.lang.reflect.Method.invoke(Unknown
> Source)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.metastore.client.SynchronizedMetaStoreClient$SynchronizedHandler.invoke(SynchronizedMetaStoreClient.java:69)
> 2025-10-14 16:52:12 jdk.proxy2/jdk.proxy2.$Proxy24.createTable(Unknown
> Source)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.metastore.client.MetaStoreClientWrapper.createTable(MetaStoreClientWrapper.java:445)
> 2025-10-14 16:52:12
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Unknown
> Source)
> 2025-10-14 16:52:12 java.base/java.lang.reflect.Method.invoke(Unknown
> Source)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:232)
> 2025-10-14 16:52:12 jdk.proxy2/jdk.proxy2.$Proxy24.createTable(Unknown
> Source)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1419)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1431)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.createTableNonReplaceMode(CreateTableOperation.java:158)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.execute(CreateTableOperation.java:116)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.ddl.DDLTask.execute(DDLTask.java:84)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:214)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:354)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:327)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:244)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.Executor.execute(Executor.java:105)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.Driver.execute(Driver.java:345)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:189)
> 2025-10-14 16:52:12 org.apache.hadoop.hive.ql.Driver.run(Driver.java:142)
> 2025-10-14 16:52:12 org.apache.hadoop.hive.ql.Driver.run(Driver.java:137)
> 2025-10-14 16:52:12
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:190)
> 2025-10-14 16:52:12
> org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:234)
> 2025-10-14 16:52:12
> org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:334)
> 2025-10-14 16:52:12
> java.base/java.security.AccessController.doPrivileged(Unknown Source)
> 2025-10-14 16:52:12 java.base/javax.security.auth.Subject.doAs(Unknown
> Source)
> 2025-10-14 16:52:12
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
> 2025-10-14 16:52:12
> org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:354)
> 2025-10-14 16:52:12
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
> 2025-10-14 16:52:12 java.base/java.util.concurrent.FutureTask.run(Unknown
> Source)
> 2025-10-14 16:52:12
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
> 2025-10-14 16:52:12
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
> 2025-10-14 16:52:12 java.base/java.lang.Thread.run(Unknown Source) {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
