ayushtkn commented on code in PR #3531:
URL: https://github.com/apache/hive/pull/3531#discussion_r951177365
##########
ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java:
##########
@@ -1503,24 +1503,31 @@ public static Map<Path, HdfsDirSnapshot>
getHdfsDirSnapshotsForCleaner(final Fil
stack.push(fs.listStatusIterator(path));
while (!stack.isEmpty()) {
RemoteIterator<FileStatus> itr = stack.pop();
- while (itr.hasNext()) {
- FileStatus fStatus = itr.next();
- Path fPath = fStatus.getPath();
- if (acidHiddenFileFilter.accept(fPath)) {
- if (baseFileFilter.accept(fPath) ||
- deltaFileFilter.accept(fPath) ||
- deleteEventDeltaDirFilter.accept(fPath)) {
- addToSnapshoot(dirToSnapshots, fPath);
- } else {
- if (fStatus.isDirectory()) {
- stack.push(fs.listStatusIterator(fPath));
+ try {
+ while (itr.hasNext()) {
+ FileStatus fStatus = itr.next();
+ Path fPath = fStatus.getPath();
+ if (acidHiddenFileFilter.accept(fPath)) {
+ if (baseFileFilter.accept(fPath) ||
+ deltaFileFilter.accept(fPath) ||
+ deleteEventDeltaDirFilter.accept(fPath)) {
+ addToSnapshoot(dirToSnapshots, fPath);
} else {
- // Found an original file
- HdfsDirSnapshot hdfsDirSnapshot = addToSnapshoot(dirToSnapshots,
fPath.getParent());
- hdfsDirSnapshot.addFile(fStatus);
+ if (fStatus.isDirectory()) {
+ stack.push(fs.listStatusIterator(fPath));
+ } else {
+ // Found an original file
+ HdfsDirSnapshot hdfsDirSnapshot =
addToSnapshoot(dirToSnapshots, fPath.getParent());
+ hdfsDirSnapshot.addFile(fStatus);
+ }
}
}
}
+ }catch(FileNotFoundException fne){
+ //Ignore....
+ //As current FS API doesn't provide the ability to supply a PathFilter
to ignore the staging dirs,
+ // need to catch this exception
Review Comment:
There is something like
RemoteIterators.filteringRemoteIterator(fs.listStatusIterator(path), <filter
Condition>) provided by Hadoop for such use cases.
https://github.com/apache/hadoop/blob/7f176d080c2576e512cbd401fce1a8d935b18ca7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/RemoteIterators.java#L160-L177
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
