[ https://issues.apache.org/jira/browse/HIVE-19874?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16510430#comment-16510430 ]
Alexander Kolbasov commented on HIVE-19874: ------------------------------------------- Looks like this was introduced with HIVE-15522 - TDserializer isn't thread-safe. > JSONMessageFactory::getTObj is not thread safe > ---------------------------------------------- > > Key: HIVE-19874 > URL: https://issues.apache.org/jira/browse/HIVE-19874 > Project: Hive > Issue Type: Bug > Components: Standalone Metastore > Affects Versions: 2.3.2 > Reporter: Dan Burkert > Assignee: Dan Burkert > Priority: Major > Attachments: HIVE-19874.0-branch-2.patch > > > The static {{JSONMessageFactory::getTObj}} internally uses the non-threadsafe > static field {{thriftDeSerializer}}, which causes concurrent calls to fail > with non-intuitive exceptions. For instance, here's the output from > concurrent {{ALTER TABLE RENAME}} operations on different tables: > > {code:java} > 2018-06-12T15:34:44,466 ERROR [pool-8-thread-2] metastore.HiveAlterHandler: > Failed to alter table default.table_b_0 > 2018-06-12T15:34:44,466 ERROR [pool-8-thread-3] metastore.HiveAlterHandler: > Failed to alter table default.table_a_0 > 2018-06-12T15:34:44,476 ERROR [pool-8-thread-2] metastore.RetryingHMSHandler: > MetaException(message:java.lang.IllegalStateException: Event not set up > correctly) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newMetaException(HiveMetaStore.java:6189) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.alter_table_core(HiveMetaStore.java:4063) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.alter_table_with_environment_context(HiveMetaStore.java:4020) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:148) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107) > at com.sun.proxy.$Proxy25.alter_table_with_environment_context(Unknown > Source) > at > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$alter_table_with_environment_context.getResult(ThriftHiveMetastore.java:11631) > at > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$alter_table_with_environment_context.getResult(ThriftHiveMetastore.java:11615) > at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) > at > org.apache.hadoop.hive.metastore.TUGIBasedProcessor.process(TUGIBasedProcessor.java:103) > at > org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > Caused by: java.lang.IllegalStateException: Event not set up correctly > at > org.apache.hadoop.hive.metastore.messaging.AlterTableMessage.checkValid(AlterTableMessage.java:49) > at > org.apache.hadoop.hive.metastore.messaging.json.JSONAlterTableMessage.<init>(JSONAlterTableMessage.java:57) > at > org.apache.hadoop.hive.metastore.messaging.json.JSONMessageFactory.buildAlterTableMessage(JSONMessageFactory.java:115) > at > org.apache.hive.hcatalog.listener.DbNotificationListener.onAlterTable(DbNotificationListener.java:187) > at > org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier$8.notify(MetaStoreListenerNotifier.java:107) > at > org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier.notifyEvent(MetaStoreListenerNotifier.java:175) > at > org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier.notifyEvent(MetaStoreListenerNotifier.java:205) > at > org.apache.hadoop.hive.metastore.HiveAlterHandler.alterTable(HiveAlterHandler.java:317) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.alter_table_core(HiveMetaStore.java:4049) > ... 16 more > Caused by: org.apache.thrift.protocol.TProtocolException: Unexpected > character:{ > at > org.apache.thrift.protocol.TJSONProtocol.readJSONSyntaxChar(TJSONProtocol.java:337) > at > org.apache.thrift.protocol.TJSONProtocol.readJSONInteger(TJSONProtocol.java:719) > at > org.apache.thrift.protocol.TJSONProtocol.readFieldBegin(TJSONProtocol.java:858) > at > org.apache.hadoop.hive.metastore.api.Table$TableStandardScheme.read(Table.java:1580) > at > org.apache.hadoop.hive.metastore.api.Table$TableStandardScheme.read(Table.java:1573) > at org.apache.hadoop.hive.metastore.api.Table.read(Table.java:1407) > at org.apache.thrift.TDeserializer.deserialize(TDeserializer.java:81) > at org.apache.thrift.TDeserializer.deserialize(TDeserializer.java:67) > at org.apache.thrift.TDeserializer.deserialize(TDeserializer.java:98) > at > org.apache.hadoop.hive.metastore.messaging.json.JSONMessageFactory.getTObj(JSONMessageFactory.java:270) > at > org.apache.hadoop.hive.metastore.messaging.json.JSONAlterTableMessage.getTableObjAfter(JSONAlterTableMessage.java:97) > at > org.apache.hadoop.hive.metastore.messaging.AlterTableMessage.checkValid(AlterTableMessage.java:41) > ... 24 more > 2018-06-12T15:34:44,476 ERROR [pool-8-thread-3] metastore.RetryingHMSHandler: > MetaException(message:java.lang.IllegalStateException: Event not set up > correctly) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newMetaException(HiveMetaStore.java:6189) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.alter_table_core(HiveMetaStore.java:4063) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.alter_table_with_environment_context(HiveMetaStore.java:4020) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:148) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107) > at com.sun.proxy.$Proxy25.alter_table_with_environment_context(Unknown > Source) > at > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$alter_table_with_environment_context.getResult(ThriftHiveMetastore.java:11631) > at > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$alter_table_with_environment_context.getResult(ThriftHiveMetastore.java:11615) > at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) > at > org.apache.hadoop.hive.metastore.TUGIBasedProcessor.process(TUGIBasedProcessor.java:103) > at > org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > Caused by: java.lang.IllegalStateException: Event not set up correctly > at > org.apache.hadoop.hive.metastore.messaging.AlterTableMessage.checkValid(AlterTableMessage.java:49) > at > org.apache.hadoop.hive.metastore.messaging.json.JSONAlterTableMessage.<init>(JSONAlterTableMessage.java:57) > at > org.apache.hadoop.hive.metastore.messaging.json.JSONMessageFactory.buildAlterTableMessage(JSONMessageFactory.java:115) > at > org.apache.hive.hcatalog.listener.DbNotificationListener.onAlterTable(DbNotificationListener.java:187) > at > org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier$8.notify(MetaStoreListenerNotifier.java:107) > at > org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier.notifyEvent(MetaStoreListenerNotifier.java:175) > at > org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier.notifyEvent(MetaStoreListenerNotifier.java:205) > at > org.apache.hadoop.hive.metastore.HiveAlterHandler.alterTable(HiveAlterHandler.java:317) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.alter_table_core(HiveMetaStore.java:4049) > ... 16 more > Caused by: org.apache.thrift.protocol.TProtocolException: Unexpected > character:" > at > org.apache.thrift.protocol.TJSONProtocol.readJSONSyntaxChar(TJSONProtocol.java:337) > at > org.apache.thrift.protocol.TJSONProtocol$JSONPairContext.read(TJSONProtocol.java:246) > at > org.apache.thrift.protocol.TJSONProtocol.readJSONObjectStart(TJSONProtocol.java:793) > at > org.apache.thrift.protocol.TJSONProtocol.readStructBegin(TJSONProtocol.java:840) > at > org.apache.hadoop.hive.metastore.api.Table$TableStandardScheme.read(Table.java:1577) > at > org.apache.hadoop.hive.metastore.api.Table$TableStandardScheme.read(Table.java:1573) > at org.apache.hadoop.hive.metastore.api.Table.read(Table.java:1407) > at org.apache.thrift.TDeserializer.deserialize(TDeserializer.java:81) > at org.apache.thrift.TDeserializer.deserialize(TDeserializer.java:67) > at org.apache.thrift.TDeserializer.deserialize(TDeserializer.java:98) > at > org.apache.hadoop.hive.metastore.messaging.json.JSONMessageFactory.getTObj(JSONMessageFactory.java:270) > at > org.apache.hadoop.hive.metastore.messaging.json.JSONAlterTableMessage.getTableObjAfter(JSONAlterTableMessage.java:97) > at > org.apache.hadoop.hive.metastore.messaging.AlterTableMessage.checkValid(AlterTableMessage.java:41) > ... 24 more > {code} > > This is the case on at least {{branch-2}}, I haven't checked any other > branches. Will attach a patch shortly. -- This message was sent by Atlassian JIRA (v7.6.3#76005)