Juliusz Skowron created HIVE-18308:
--------------------------------------
Summary: Error inserting data into many partitions
Key: HIVE-18308
URL: https://issues.apache.org/jira/browse/HIVE-18308
Project: Hive
Issue Type: Bug
Components: Hive
Affects Versions: 2.3.1
Reporter: Juliusz Skowron
I am trying to partition data by a field of high cardinality (~75K). Is this
even possible?
Tried different ways and it keeps failing. Even on very small sample data (12MB
gzipped).
Below are queries and errors they throw. Anything I can do to work this out?
{noformat}
CREATE EXTERNAL TABLE IF NOT EXISTS history_sample_hdfs(
ts bigint,
sp float,
cr float,
id string)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE
LOCATION '/history_sample';
CREATE EXTERNAL TABLE IF NOT EXISTS history_sample_partitioned_hdfs(
ts bigint,
sp float,
cr float) PARTITIONED BY(id string)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE
LOCATION
'/history_sample_partitioned/';
{noformat}
{noformat}
set mapred.reduce.tasks=1000;
insert overwrite table history_sample_partitioned_hdfs
PARTITION(id)
select ts,
sp,
cr,
id
from history_sample_hdfs
distribute by id;
{noformat}
{noformat}
Vertex failed, vertexName=Reducer 2, vertexId=vertex_1513674323124_0019_5_01,
diagnostics=[Task failed, taskId=task_1513674323124_0019_5_01_000000,
diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task (
failure ) :
attempt_1513674323124_0019_5_01_000000_0:java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
attempt_1513674323124_0019_5_01_000000_1:java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
], TaskAttempt 2 failed, info=[Error: Error while running task ( failure ) :
attempt_1513674323124_0019_5_01_000000_2:java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
], TaskAttempt 3 failed, info=[Error: Error while running task ( failure ) :
attempt_1513674323124_0019_5_01_000000_3:java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
killedTasks:999, Vertex vertex_1513674323124_0019_5_01 [Reducer 2]
killed/failed due to:OWN_TASK_FAILURE]
DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0
FAILED: Execution Error, return code 2 from
org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Reducer
2, vertexId=vertex_1513674323124_0019_5_01, diagnostics=[Task failed,
taskId=task_1513674323124_0019_5_01_000000, diagnostics=[TaskAttempt 0 failed,
info=[Error: Error while running task ( failure ) :
attempt_1513674323124_0019_5_01_000000_0:java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
attempt_1513674323124_0019_5_01_000000_1:java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
], TaskAttempt 2 failed, info=[Error: Error while running task ( failure ) :
attempt_1513674323124_0019_5_01_000000_2:java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
], TaskAttempt 3 failed, info=[Error: Error while running task ( failure ) :
attempt_1513674323124_0019_5_01_000000_3:java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row (tag=0)
{"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
... 14 more
Caused by: java.lang.NullPointerException
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
... 17 more
]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
killedTasks:999, Vertex vertex_1513674323124_0019_5_01 [Reducer 2]
killed/failed due to:OWN_TASK_FAILURE]DAG did not succeed due to
VERTEX_FAILURE. failedVertices:1 killedVertices:0
{noformat}
{noformat}
set mapred.reduce.tasks=1; --back to default
insert overwrite table history_sample_partitioned_hdfs
PARTITION(id)
select ts,
sp,
cr,
id
from history_sample_hdfs;
{noformat}
{noformat}
Vertex failed, vertexName=Reducer 2, vertexId=vertex_1513674323124_0019_6_01,
diagnostics=[Task failed, taskId=task_1513674323124_0019_6_01_000000,
diagnostics=[TaskAttempt 0 failed, info=[Container
container_1513674323124_0019_01_000175 finished with diagnostics set to
[Container failed, exitCode=-104. Container
[pid=2746,containerID=container_1513674323124_0019_01_000175] is running beyond
physical memory limits. Current usage: 1.4 GB of 1.4 GB physical memory used;
4.2 GB of 7.0 GB virtual memory used. Killing container.
Dump of the process-tree for container_1513674323124_0019_01_000175 :
|- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS)
SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE
|- 2746 2744 2746 2746 (bash) 1 1 115818496 672 /bin/bash -c
/usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
-Dlog4j.configuration=tez-container-log4j.properties
-Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175
-Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN -Xmx1152m
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
-Dlog4j.configuration=tez-container-log4j.properties
-Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175
-Dtez.root.logger=INFO,CLA
-Djava.io.tmpdir=/mnt1/yarn/usercache/hadoop/appcache/application_1513674323124_0019/container_1513674323124_0019_01_000175/tmp
org.apache.tez.runtime.task.TezChild ip-10-22-1-155.eu-west-1.compute.internal
44339 container_1513674323124_0019_01_000175 application_1513674323124_0019 1
1>/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175/stdout
2>/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175/stderr
|- 2761 2746 2746 2746 (java) 4325 1233 4384346112 376642
/usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
-Dlog4j.configuration=tez-container-log4j.properties
-Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175
-Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN -Xmx1152m
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
-Dlog4j.configuration=tez-container-log4j.properties
-Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175
-Dtez.root.logger=INFO,CLA
-Djava.io.tmpdir=/mnt1/yarn/usercache/hadoop/appcache/application_1513674323124_0019/container_1513674323124_0019_01_000175/tmp
org.apache.tez.runtime.task.TezChild ip-10-22-1-155.eu-west-1.compute.internal
44339 container_1513674323124_0019_01_000175 application_1513674323124_0019 1
Container killed on request. Exit code is 143
Container exited with a non-zero exit code 143
]], TaskAttempt 1 failed, info=[Container
container_1513674323124_0019_01_000176 finished with diagnostics set to
[Container failed, exitCode=-104. Container
[pid=22630,containerID=container_1513674323124_0019_01_000176] is running
beyond physical memory limits. Current usage: 1.4 GB of 1.4 GB physical memory
used; 4.5 GB of 7.0 GB virtual memory used. Killing container.
Dump of the process-tree for container_1513674323124_0019_01_000176 :
|- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS)
SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE
|- 22630 22628 22630 22630 (bash) 0 1 115818496 674 /bin/bash -c
/usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
-Dlog4j.configuration=tez-container-log4j.properties
-Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176
-Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN -Xmx1152m
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
-Dlog4j.configuration=tez-container-log4j.properties
-Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176
-Dtez.root.logger=INFO,CLA
-Djava.io.tmpdir=/mnt1/yarn/usercache/hadoop/appcache/application_1513674323124_0019/container_1513674323124_0019_01_000176/tmp
org.apache.tez.runtime.task.TezChild ip-10-22-1-155.eu-west-1.compute.internal
44339 container_1513674323124_0019_01_000176 application_1513674323124_0019 1
1>/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176/stdout
2>/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176/stderr
|- 22644 22630 22630 22630 (java) 4798 1395 4666630144 368835
/usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
-Dlog4j.configuration=tez-container-log4j.properties
-Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176
-Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true
-Dhadoop.metrics.log.level=WARN -Xmx1152m
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
-Dlog4j.configuration=tez-container-log4j.properties
-Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176
-Dtez.root.logger=INFO,CLA
-Djava.io.tmpdir=/mnt1/yarn/usercache/hadoop/appcache/application_1513674323124_0019/container_1513674323124_0019_01_000176/tmp
org.apache.tez.runtime.task.TezChild ip-10-22-1-155.eu-west-1.compute.internal
44339 container_1513674323124_0019_01_000176 application_1513674323124_0019 1
{noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
