deniskuzZ commented on code in PR #6680:
URL: https://github.com/apache/hive/pull/6680#discussion_r3764693556
##########
ql/src/java/org/apache/hadoop/hive/ql/io/SkippingTextInputFormat.java:
##########
@@ -193,17 +193,72 @@ private long getCachedEndIndex(Path path) throws
IOException {
// there were not enough lines in the file to consume all footer
rows.
endIndexForFile = Long.MIN_VALUE;
}
- } finally {
- if (fis != null) {
- fis.close();
- }
}
}
endIndexMap.put(path, endIndexForFile);
}
return endIndexForFile;
}
+ /**
+ * Reads lines while counting bytes consumed, so offsets can be computed
without
+ * {@link FSDataInputStream#getPos()} -- which is unreliable after {@code
readLine()}:
+ * a lone {@code '\r'} makes it swap in a non-Seekable {@code
PushbackInputStream}, so
+ * the next {@code getPos()} throws {@code ClassCastException}. Handles
{@code '\n'},
+ * {@code '\r\n'} and lone {@code '\r'}; after {@link #readLine()},
+ * {@link #getBytesConsumed()} is the offset just past the line's terminator.
+ */
+ static final class ByteCountingLineReader {
+ private final InputStream in;
+ private long bytesConsumed;
+ // Look-ahead byte past a '\r' (belongs to the next line, not yet
counted); -1 = empty.
+ private int pushedBack = -1;
+
+ ByteCountingLineReader(InputStream in) {
+ this.in = in;
+ }
+
+ long getBytesConsumed() {
+ return bytesConsumed;
+ }
+
+ private int nextByte() throws IOException {
+ if (pushedBack != -1) {
Review Comment:
could we replace the pushedBack field with Java's existing
PushbackInputStream ?
this.in = new PushbackInputStream(in, 1);
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
