DRILL-496: drill stuck in loop when query parquet file and eventually ran out of memory
Added condition in NullableColumnReader to break out of loop when there are no more pages to read. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/7ec86696 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/7ec86696 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/7ec86696 Branch: refs/heads/master Commit: 7ec866968b8691f346523e583252462515279844 Parents: 171217d Author: Steven Phillips <[email protected]> Authored: Wed Apr 9 15:14:49 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sat Apr 19 21:07:28 2014 -0700 ---------------------------------------------------------------------- .../apache/drill/exec/store/parquet/NullableColumnReader.java | 4 +++- .../java/org/apache/drill/exec/store/parquet/PageReadStatus.java | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7ec86696/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java index 9be8266..b6ae715 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java @@ -116,7 +116,9 @@ abstract class NullableColumnReader extends ColumnReader{ totalValuesRead += recordsReadInThisIteration; pageReadStatus.valuesRead += recordsReadInThisIteration; if (readStartInBytes + readLength >= pageReadStatus.byteLength && bitsUsed == 0) { - pageReadStatus.next(); + if (!pageReadStatus.next()) { + break; + } } else { pageReadStatus.readPosInBytes = readStartInBytes + readLength; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7ec86696/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java index 1aef7b4..67262f6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java @@ -76,6 +76,8 @@ final class PageReadStatus { */ public boolean next() throws IOException { + currentPage = null; + if(!dataReader.hasRemainder()) return false; // next, we need to decompress the bytes
