[
https://issues.apache.org/jira/browse/DRILL-3070?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14542888#comment-14542888
]
Rahul Challapalli commented on DRILL-3070:
------------------------------------------
Log contents :
{code}
org.apache.drill.common.exceptions.UserException: SYSTEM ERROR:
org.apache.drill.exec.memory.OutOfMemoryRuntimeException: Failure while
reallocating buffer to 262152 bytes
Fragment 7:30
[Error Id: 65a09d7d-75f2-4bbb-b8f9-cfb0bdfc2655 on qa-node191.qa.lab:31010]
at
org.apache.drill.common.exceptions.UserException$Builder.build(UserException.java:521)
~[drill-common-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.sendFinalState(FragmentExecutor.java:286)
[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:254)
[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.common.SelfCleaningRunnable.run(SelfCleaningRunnable.java:38)
[drill-common-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[na:1.7.0_71]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[na:1.7.0_71]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_71]
Caused by: org.apache.drill.common.exceptions.DrillRuntimeException: Error in
parquet record reader.
Message:
Hadoop path:
/drill/testdata/sf100-dicenc/lineitem/6935686303030493694--5013194999741243212_105907970_data.8
Total records read: 9404416
Mock records read: 0
Records to read: 32768
Row group index: 0
Records in row group: 9887524
Parquet Metadata: ParquetMetaData{FileMetaData{schema: message schema {
optional int64 l_orderkey;
optional int64 l_partkey;
optional int64 l_suppkey;
optional int64 l_linenumber;
optional double l_quantity;
optional double l_extendedprice;
optional double l_discount;
optional double l_tax;
optional binary l_returnflag;
optional binary l_linestatus;
optional binary l_shipdate;
optional binary l_commitdate;
optional binary l_receiptdate;
optional binary l_shipinstruct;
optional binary l_shipmode;
optional binary l_comment;
}
, metadata: {}}, blocks: [BlockMetaData{9887524, 381279024
[ColumnMetaData{SNAPPY [l_orderkey] INT64 [PLAIN, RLE, PLAIN_DICTIONARY],
262856}, ColumnMetaData{SNAPPY [l_partkey] INT64 [PLAIN, RLE,
PLAIN_DICTIONARY], 16080945}, ColumnMetaData{SNAPPY [l_suppkey] INT64 [PLAIN,
RLE, PLAIN_DICTIONARY], 70143759}, ColumnMetaData{SNAPPY [l_linenumber] INT64
[PLAIN, RLE, PLAIN_DICTIONARY], 118843881}, ColumnMetaData{SNAPPY [l_quantity]
DOUBLE [PLAIN, RLE, PLAIN_DICTIONARY], 121286644}, ColumnMetaData{SNAPPY
[l_extendedprice] DOUBLE [PLAIN, RLE, PLAIN_DICTIONARY], 129105075},
ColumnMetaData{SNAPPY [l_discount] DOUBLE [PLAIN, RLE, PLAIN_DICTIONARY],
185915285}, ColumnMetaData{SNAPPY [l_tax] DOUBLE [PLAIN, RLE,
PLAIN_DICTIONARY], 190882198}, ColumnMetaData{SNAPPY [l_returnflag] BINARY
[PLAIN, RLE, PLAIN_DICTIONARY], 195849066}, ColumnMetaData{SNAPPY
[l_linestatus] BINARY [PLAIN, RLE, PLAIN_DICTIONARY], 198248545},
ColumnMetaData{SNAPPY [l_shipdate] BINARY [PLAIN, RLE, PLAIN_DICTIONARY],
199707246}, ColumnMetaData{SNAPPY [l_commitdate] BINARY [PLAIN, RLE,
PLAIN_DICTIONARY], 214578801}, ColumnMetaData{SNAPPY [l_receiptdate] BINARY
[PLAIN, RLE, PLAIN_DICTIONARY], 229450796}, ColumnMetaData{SNAPPY
[l_shipinstruct] BINARY [PLAIN, RLE, PLAIN_DICTIONARY], 244311610},
ColumnMetaData{SNAPPY [l_shipmode] BINARY [PLAIN, RLE, PLAIN_DICTIONARY],
246804974}, ColumnMetaData{SNAPPY [l_comment] BINARY [PLAIN, RLE,
PLAIN_DICTIONARY], 251410909}]}]}
at
org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader.handleAndRaise(ParquetRecordReader.java:339)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader.next(ParquetRecordReader.java:441)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.ScanBatch.next(ScanBatch.java:175)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.validate.IteratorValidatorBatchIterator.next(IteratorValidatorBatchIterator.java:118)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:105)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:95)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.innerNext(ProjectRecordBatch.java:129)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:146)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.validate.IteratorValidatorBatchIterator.next(IteratorValidatorBatchIterator.java:118)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:105)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:95)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:146)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.validate.IteratorValidatorBatchIterator.next(IteratorValidatorBatchIterator.java:118)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:105)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:95)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch.innerNext(RemovingRecordBatch.java:92)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:146)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.validate.IteratorValidatorBatchIterator.next(IteratorValidatorBatchIterator.java:118)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:105)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:95)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractSingleRecordBatch.innerNext(AbstractSingleRecordBatch.java:51)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.innerNext(ProjectRecordBatch.java:129)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:146)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.validate.IteratorValidatorBatchIterator.next(IteratorValidatorBatchIterator.java:118)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:83)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.SingleSenderCreator$SingleSenderRootExec.innerNext(SingleSenderCreator.java:91)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:73)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:223)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:217)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at java.security.AccessController.doPrivileged(Native Method)
~[na:1.7.0_71]
at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_71]
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1469)
~[hadoop-common-2.4.1-mapr-1408.jar:na]
at
org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:217)
[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
... 4 common frames omitted
Caused by: org.apache.drill.exec.memory.OutOfMemoryRuntimeException: Failure
while reallocating buffer to 262152 bytes
at
org.apache.drill.exec.vector.UInt4Vector.reAlloc(UInt4Vector.java:192)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.vector.UInt4Vector$Mutator.setSafe(UInt4Vector.java:375)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.vector.VarBinaryVector$Mutator.setSafe(VarBinaryVector.java:484)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.vector.NullableVarBinaryVector$Mutator.fillEmpties(NullableVarBinaryVector.java:405)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.vector.NullableVarBinaryVector$Mutator.setValueCount(NullableVarBinaryVector.java:510)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.store.parquet.columnreaders.VarLenBinaryReader.readFields(VarLenBinaryReader.java:78)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
at
org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader.next(ParquetRecordReader.java:425)
~[drill-java-exec-1.0.0-SNAPSHOT-rebuffed.jar:1.0.0-SNAPSHOT]
... 38 common frames omitted
{code}
> Memory Leak when we run out of memory
> -------------------------------------
>
> Key: DRILL-3070
> URL: https://issues.apache.org/jira/browse/DRILL-3070
> Project: Apache Drill
> Issue Type: Bug
> Components: Execution - Flow
> Reporter: Rahul Challapalli
> Assignee: Chris Westin
>
> git.commit.id.abbrev=d10769f
> No of Drillbits : 2
> DRILL_MAX_DIRECT_MEMORY="1G"
> I ran the below query on tpsh SF 100
> {code}
> select
> l.l_shipmode,
> sum(case
> when o.o_orderpriority = '1-URGENT'
> or o.o_orderpriority = '2-HIGH'
> then 1
> else 0
> end) as high_line_count,
> sum(case
> when o.o_orderpriority <> '1-URGENT'
> and o.o_orderpriority <> '2-HIGH'
> then 1
> else 0
> end) as low_line_count
> from
> orders o,
> lineitem l
> where
> o.o_orderkey = l.l_orderkey
> and l.l_shipmode in ('TRUCK', 'REG AIR')
> and l.l_commitdate < l.l_receiptdate
> and l.l_shipdate < l.l_commitdate
> and l.l_receiptdate >= date '1994-01-01'
> and l.l_receiptdate < date '1994-01-01' + interval '1' year
> group by
> l.l_shipmode
> order by
> l.l_shipmode;
> +------------+-----------------+----------------+
> | l_shipmode | high_line_count | low_line_count |
> +------------+-----------------+----------------+
> java.lang.RuntimeException: java.sql.SQLException: SYSTEM ERROR:
> org.apache.drill.exec.memory.OutOfMemoryRuntimeException: Failure while
> reallocating buffer to 262152 bytes
> Fragment 7:30
> [Error Id: 65a09d7d-75f2-4bbb-b8f9-cfb0bdfc2655 on qa-node191.qa.lab:31010]
> at sqlline.SqlLine$IncrementalRows.hasNext(SqlLine.java:2514)
> at sqlline.SqlLine$TableOutputFormat.print(SqlLine.java:2148)
> at sqlline.SqlLine.print(SqlLine.java:1809)
> at sqlline.SqlLine$Commands.execute(SqlLine.java:3766)
> at sqlline.SqlLine$Commands.sql(SqlLine.java:3663)
> at sqlline.SqlLine.dispatch(SqlLine.java:889)
> at sqlline.SqlLine.begin(SqlLine.java:763)
> at sqlline.SqlLine.start(SqlLine.java:498)
> at sqlline.SqlLine.main(SqlLine.java:460)
> {code}
> Before memory :
> {code}
> 0: jdbc:drill:schema=dfs_eea> select * from sys.memory;
> +------------+------------+--------------+------------+----------------+--------------------+------------+
> | hostname | user_port | heap_current | heap_max | direct_current |
> jvm_direct_current | direct_max |
> +------------+------------+--------------+------------+----------------+--------------------+------------+
> | qa-node190.qa.lab | 31010 | 126726408 | 4151836672 | 11799087
> | 100676940 | 1073741824 |
> | qa-node191.qa.lab | 31010 | 299617984 | 4151836672 | 7750365
> | 50345292 | 1073741824 |
> +------------+------------+--------------+------------+----------------+--------------------+------------+
> {code}
> After Memory :
> {code}
> 0: jdbc:drill:schema=dfs_eea> select * from sys.memory;
> +------------+------------+--------------+------------+----------------+--------------------+------------+
> | hostname | user_port | heap_current | heap_max | direct_current |
> jvm_direct_current | direct_max |
> +------------+------------+--------------+------------+----------------+--------------------+------------+
> | qa-node190.qa.lab | 31010 | 446765888 | 4151836672 | 161309493
> | 436221380 | 1073741824 |
> | qa-node191.qa.lab | 31010 | 357580248 | 4151836672 | 199260917
> | 603993536 | 1073741824 |
> +------------+------------+--------------+------------+----------------+--------------------+------------+
> {code}
> There seems to be a bump in directory_memory on qa-node191.qa.lab. Not sure
> if this is normal
> - Rahul
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
