[
https://issues.apache.org/jira/browse/HIVE-30205?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Stamatis Zampetakis resolved HIVE-30205.
----------------------------------------
Fix Version/s: 4.3.0
Assignee: Thomas Rebele
Resolution: Fixed
Fixed in
[https://github.com/apache/hive/commit/66fb11595f6307fc079b4c598c3388e6af6e0785]
Thanks for the PR [~thomas.rebele] !
> OpenCSVSerde without vectorization throws UnsupportedOperationException
> -----------------------------------------------------------------------
>
> Key: HIVE-30205
> URL: https://issues.apache.org/jira/browse/HIVE-30205
> Project: Hive
> Issue Type: Bug
> Components: Serializers/Deserializers
> Reporter: Thomas Rebele
> Assignee: Thomas Rebele
> Priority: Major
> Labels: pull-request-available
> Fix For: 4.3.0
>
>
> The OpenCSVSerde only stores string columns. HIVE-26388 introduced an
> auto-conversion, so that OpenCSVSerde can accept other types as input. It
> seems that this auto-conversion works only for vectorized execution. Ideally,
> the patch of HIVE-26388 would be extended so that tables created without
> vectorization also apply the auto-conversion.
> The following q file test:
> {code:java}
> set hive.vectorized.execution.enabled=true;
> create table tab1_vec(abc float);
> insert into table tab1_vec values (1.23);
> create table tab2_vec row format serde
> 'org.apache.hadoop.hive.serde2.OpenCSVSerde' with serdeproperties
> ("separatorChar" = ',' , "quoteChar" = '"') stored as textfile as select *
> from tab1_vec;
> select * from tab2_vec;
> set hive.vectorized.execution.enabled=false;
> create table tab1_novec(abc float);
> insert into table tab1_novec values (1.23);
> create table tab2_novec row format serde
> 'org.apache.hadoop.hive.serde2.OpenCSVSerde' with serdeproperties
> ("separatorChar" = ',' , "quoteChar" = '"') stored as textfile as select *
> from tab1_novec;
> select * from tab2_novec;
> {code}
> executed with {{mvn test -Drat.skip=true -Pitests -pl serde,itests/qtest
> -Dtest=TestMiniLlapLocalCliDriver -Dqfile=opencsvserde_vectorization.q
> -Dtest.output.overwrite=true -T1C}} on the most recent commit
> (32efdcd9e6cfdfbecd30a4bf454c66f76be6281d), leads to an
> {{UnsupportedOperationException: Column type of float is not supported with
> OpenCSVSerde}}
> Here the complete stack trace:
> {code:java}
> create table tab2_novec row format serde
> 'org.apache.hadoop.hive.serde2.OpenCSVSerde' with serdeproperties
> ("separatorChar" = ',' , "quoteChar" = '"') stored as textfile as select *
> from tab1_novec
> fname=opencsvserde_vectorization.q
> See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log,
> or check ./ql/target/surefire-reports or
> ./itests/qtest/target/surefire-reports/ for specific test cases logs.
> org.apache.hadoop.hive.ql.exec.tez.TezRuntimeException: Vertex failed,
> vertexName=Map 1, vertexId=vertex_1792447139701_0001_4_00, diagnostics=[Task
> failed, taskId=task_1792447139701_0001_4_00_000000, diagnostics=[TaskAttempt
> 0 failed, info=[Error: Error while running task ( failure ) :
> attempt_1792447139701_0001_4_00_000000_0:java.lang.RuntimeException:
> java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
> Hive Runtime Error while processing row
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:342)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:277)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:381)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:86)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:72)
> at
> java.base/java.security.AccessController.doPrivileged(AccessController.java:714)
> at java.base/javax.security.auth.Subject.doAs(Subject.java:525)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:72)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:42)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at
> org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:112)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
> at java.base/java.lang.Thread.run(Thread.java:1583)
> Caused by: java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:111)
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:84)
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:418)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:294)
> ... 15 more
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row
> at
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:580)
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:102)
> ... 18 more
> Caused by: java.lang.UnsupportedOperationException: Column type of float is
> not supported with OpenCSVSerde
> at
> org.apache.hadoop.hive.serde2.OpenCSVSerde.doSerialize(OpenCSVSerde.java:136)
> at
> org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.serialize(AbstractEncodingAwareSerDe.java:54)
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:1166)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:889)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:889)
> at
> org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:182)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:157)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:559)
> ... 19 more
> ], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
> attempt_1792447139701_0001_4_00_000000_1:java.lang.RuntimeException:
> java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
> Hive Runtime Error while processing row
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:342)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:277)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:381)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:86)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:72)
> at
> java.base/java.security.AccessController.doPrivileged(AccessController.java:714)
> at java.base/javax.security.auth.Subject.doAs(Subject.java:525)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:72)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:42)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at
> org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:112)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
> at java.base/java.lang.Thread.run(Thread.java:1583)
> Caused by: java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:111)
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:84)
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:418)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:294)
> ... 15 more
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row
> at
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:580)
> at
> org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:102)
> ... 18 more
> Caused by: java.lang.UnsupportedOperationException: Column type of float is
> not supported with OpenCSVSerde
> at
> org.apache.hadoop.hive.serde2.OpenCSVSerde.doSerialize(OpenCSVSerde.java:136)
> at
> org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.serialize(AbstractEncodingAwareSerDe.java:54)
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:1166)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:889)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:889)
> at
> org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:182)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:157)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:559)
> ... 19 more
> ]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
> killedTasks:0, Vertex vertex_1792447139701_0001_4_00 [Map 1] killed/failed
> due to:OWN_TASK_FAILURE]Vertex killed, vertexName=Reducer 2,
> vertexId=vertex_1792447139701_0001_4_01, diagnostics=[Vertex received Kill
> while in RUNNING state., Vertex did not succeed due to OTHER_VERTEX_FAILURE,
> failedTasks:0 killedTasks:1, Vertex vertex_1792447139701_0001_4_01 [Reducer
> 2] killed/failed due to:OTHER_VERTEX_FAILURE]DAG did not succeed due to
> VERTEX_FAILURE. failedVertices:1 killedVertices:1
> at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:287)
> at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:215)
> at
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:106)
> at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:355)
> at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:328)
> at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:245)
> at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:106)
> at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:347)
> at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:192)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:144)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:139)
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:191)
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:236)
> at
> org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:260)
> at org.apache.hadoop.hive.cli.CliDriver.processCmd1(CliDriver.java:204)
> at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:130)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:433)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:361)
> at
> org.apache.hadoop.hive.ql.QTestUtil.executeClientInternal(QTestUtil.java:795)
> at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:765)
> at
> org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:116)
> at
> org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:140)
> at
> org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver(TestMiniLlapLocalCliDriver.java:63)
> at
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
> at java.base/java.lang.reflect.Method.invoke(Method.java:580)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.apache.hadoop.hive.cli.control.CliAdapter$2$1.evaluate(CliAdapter.java:119)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at
> org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
> at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
> at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
> at org.junit.runners.Suite.runChild(Suite.java:128)
> at org.junit.runners.Suite.runChild(Suite.java:27)
> at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
> at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
> at
> org.apache.hadoop.hive.cli.control.CliAdapter$1$1.evaluate(CliAdapter.java:90)
> at org.junit.rules.RunRules.evaluate(RunRules.java:20)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155)
> at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385)
> at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
> at
> org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507)
> at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495)
> {code}
--
This message was sent by Atlassian Jira
(v10.3.26#10030026)
