Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/21408#discussion_r190196148
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
---
@@ -294,9 +294,12 @@ object InMemoryFileIndex extends Logging {
if (filter != null) allFiles.filter(f => filter.accept(f.getPath))
else allFiles
}
- allLeafStatuses.filterNot(status =>
shouldFilterOut(status.getPath.getName)).map {
+ val missingFiles = mutable.ArrayBuffer.empty[String]
+ val filteredLeafStatuses = allLeafStatuses.filterNot(
+ status => shouldFilterOut(status.getPath.getName))
--- End diff --
I made this var to reduce the diff.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
