[
https://issues.apache.org/jira/browse/HIVE-29566?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
László Bodor updated HIVE-29566:
--------------------------------
Summary: Tez client creation fails with docker LLAP mode on client
reconnect - JvmMetrics is not idempotent in LlapTaskSchedulerService (was: Tez
client creation fails with docker LLAP mode on client reconnect - metrics
idempotency problem inside LlapTaskSchedulerService)
> Tez client creation fails with docker LLAP mode on client reconnect -
> JvmMetrics is not idempotent in LlapTaskSchedulerService
> ------------------------------------------------------------------------------------------------------------------------------
>
> Key: HIVE-29566
> URL: https://issues.apache.org/jira/browse/HIVE-29566
> Project: Hive
> Issue Type: Bug
> Reporter: László Bodor
> Assignee: László Bodor
> Priority: Major
>
> This is a follow-up after HIVE-29411 for the problem discovered as:
> https://github.com/apache/hive/pull/6390#issuecomment-4148486357
> repro is most probably as below:
> {code}
> docker-compose down --rmi local # cleanup previous containers and images
> export POSTGRES_LOCAL_PATH=... # set the path to the postgres driver jar on
> the host machine
> ./build.sh -hive 4.2.0 -hadoop 3.4.1 -tez 0.10.5 # build image from the
> common Dockerfile
> ./start-hive.sh --llap
> beeline -u 'jdbc:hive2://localhost:10000/'
> DROP table IF EXISTS iceberg_table; CREATE TABLE iceberg_table (id BIGINT)
> STORED BY iceberg; INSERT INTO iceberg_table VALUES(1);
> # quit from beeline and reconnect
> beeline -u 'jdbc:hive2://localhost:10000/'
> DROP table IF EXISTS iceberg_table; CREATE TABLE iceberg_table (id BIGINT)
> STORED BY iceberg; INSERT INTO iceberg_table VALUES(1);
> {code}
> {code}
> ERROR : Failed to execute tez graph.
> org.apache.tez.dag.api.TezException: java.io.IOException:
> org.apache.hadoop.service.ServiceStateException:
> org.apache.tez.dag.api.TezReflectionException: Unable to instantiate class
> with 1 arguments:
> org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService
> at org.apache.tez.client.TezClient.start(TezClient.java:411)
> ~[tez-api-0.10.5.jar:0.10.5]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:465)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:383)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.openInternal(TezSessionPoolSession.java:122)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:249)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.ql.exec.tez.TezTask.ensureSessionHasResources(TezTask.java:481)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:232)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:214)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:354)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:327)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:244)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:105)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:345)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:189)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:142)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:137)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:190)
> ~[hive-exec-4.2.0.jar:4.2.0]
> at
> org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:234)
> ~[hive-service-4.2.0.jar:4.2.0]
> at
> org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:334)
> ~[hive-service-4.2.0.jar:4.2.0]
> at
> java.base/java.security.AccessController.doPrivileged(AccessController.java:714)
> ~[?:?]
> at java.base/javax.security.auth.Subject.doAs(Subject.java:525) ~[?:?]
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
> ~[hadoop-common-3.4.1.jar:?]
> at
> org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:354)
> ~[hive-service-4.2.0.jar:4.2.0]
> at
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
> ~[?:?]
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
> ~[?:?]
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
> ~[?:?]
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
> ~[?:?]
> at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
> Caused by: java.io.IOException:
> org.apache.hadoop.service.ServiceStateException:
> org.apache.tez.dag.api.TezReflectionException: Unable to instantiate class
> with 1 arguments:
> org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService
> at
> org.apache.tez.client.LocalClient.startDAGAppMaster(LocalClient.java:300)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.client.LocalClient.submitApplication(LocalClient.java:156)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at org.apache.tez.client.TezClient.start(TezClient.java:402)
> ~[tez-api-0.10.5.jar:0.10.5]
> ... 28 more
> Caused by: org.apache.hadoop.service.ServiceStateException:
> org.apache.tez.dag.api.TezReflectionException: Unable to instantiate class
> with 1 arguments:
> org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService
> at
> org.apache.hadoop.service.ServiceStateException.convert(ServiceStateException.java:105)
> ~[hadoop-common-3.4.1.jar:?]
> at
> org.apache.hadoop.service.AbstractService.start(AbstractService.java:204)
> ~[hadoop-common-3.4.1.jar:?]
> at
> org.apache.tez.dag.app.DAGAppMaster$ServiceWithDependency.start(DAGAppMaster.java:1835)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.DAGAppMaster$ServiceThread.run(DAGAppMaster.java:1856)
> ~[tez-dag-0.10.5.jar:0.10.5]
> Caused by: org.apache.tez.dag.api.TezReflectionException: Unable to
> instantiate class with 1 arguments:
> org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService
> at
> org.apache.tez.common.ReflectionUtils.getNewInstance(ReflectionUtils.java:70)
> ~[tez-api-0.10.5.jar:0.10.5]
> at
> org.apache.tez.common.ReflectionUtils.createClazzInstance(ReflectionUtils.java:88)
> ~[tez-api-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.createCustomTaskScheduler(TaskSchedulerManager.java:628)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.createTaskScheduler(TaskSchedulerManager.java:596)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.instantiateSchedulers(TaskSchedulerManager.java:649)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.serviceStart(TaskSchedulerManager.java:668)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.hadoop.service.AbstractService.start(AbstractService.java:195)
> ~[hadoop-common-3.4.1.jar:?]
> at
> org.apache.tez.dag.app.DAGAppMaster$ServiceWithDependency.start(DAGAppMaster.java:1835)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.DAGAppMaster$ServiceThread.run(DAGAppMaster.java:1856)
> ~[tez-dag-0.10.5.jar:0.10.5]
> Caused by: java.lang.reflect.InvocationTargetException
> at
> java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:74)
> ~[?:?]
> at
> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
> ~[?:?]
> at
> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
> ~[?:?]
> at
> org.apache.tez.common.ReflectionUtils.getNewInstance(ReflectionUtils.java:67)
> ~[tez-api-0.10.5.jar:0.10.5]
> at
> org.apache.tez.common.ReflectionUtils.createClazzInstance(ReflectionUtils.java:88)
> ~[tez-api-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.createCustomTaskScheduler(TaskSchedulerManager.java:628)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.createTaskScheduler(TaskSchedulerManager.java:596)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.instantiateSchedulers(TaskSchedulerManager.java:649)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.serviceStart(TaskSchedulerManager.java:668)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.hadoop.service.AbstractService.start(AbstractService.java:195)
> ~[hadoop-common-3.4.1.jar:?]
> at
> org.apache.tez.dag.app.DAGAppMaster$ServiceWithDependency.start(DAGAppMaster.java:1835)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.DAGAppMaster$ServiceThread.run(DAGAppMaster.java:1856)
> ~[tez-dag-0.10.5.jar:0.10.5]
> Caused by: org.apache.hadoop.metrics2.MetricsException: Metrics source
> JvmMetrics already exists!
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newSourceName(DefaultMetricsSystem.java:152)
> ~[hadoop-common-3.4.1.jar:?]
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.sourceName(DefaultMetricsSystem.java:125)
> ~[hadoop-common-3.4.1.jar:?]
> at
> org.apache.hadoop.metrics2.impl.MetricsSystemImpl.register(MetricsSystemImpl.java:229)
> ~[hadoop-common-3.4.1.jar:?]
> at org.apache.hadoop.hive.common.JvmMetrics.create(JvmMetrics.java:82)
> ~[hive-common-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.llap.tezplugins.metrics.LlapTaskSchedulerMetrics.create(LlapTaskSchedulerMetrics.java:106)
> ~[hive-llap-tez-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.<init>(LlapTaskSchedulerService.java:459)
> ~[hive-llap-tez-4.2.0.jar:4.2.0]
> at
> org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.<init>(LlapTaskSchedulerService.java:334)
> ~[hive-llap-tez-4.2.0.jar:4.2.0]
> at
> java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
> ~[?:?]
> at
> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
> ~[?:?]
> at
> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
> ~[?:?]
> at
> org.apache.tez.common.ReflectionUtils.getNewInstance(ReflectionUtils.java:67)
> ~[tez-api-0.10.5.jar:0.10.5]
> at
> org.apache.tez.common.ReflectionUtils.createClazzInstance(ReflectionUtils.java:88)
> ~[tez-api-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.createCustomTaskScheduler(TaskSchedulerManager.java:628)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.createTaskScheduler(TaskSchedulerManager.java:596)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.instantiateSchedulers(TaskSchedulerManager.java:649)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.rm.TaskSchedulerManager.serviceStart(TaskSchedulerManager.java:668)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.hadoop.service.AbstractService.start(AbstractService.java:195)
> ~[hadoop-common-3.4.1.jar:?]
> at
> org.apache.tez.dag.app.DAGAppMaster$ServiceWithDependency.start(DAGAppMaster.java:1835)
> ~[tez-dag-0.10.5.jar:0.10.5]
> at
> org.apache.tez.dag.app.DAGAppMaster$ServiceThread.run(DAGAppMaster.java:1856)
> ~[tez-dag-0.10.5.jar:0.10.5]
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
