Github user yhuai commented on a diff in the pull request: https://github.com/apache/spark/pull/12856#discussion_r61836088 --- Diff: sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/fileSourceInterfaces.scala --- @@ -443,6 +453,22 @@ class HDFSFileCatalog( } } + /** List of files to consider when there is not inferred partitioning scheme */ + private def unpartitionedDataFiles(): Seq[FileStatus] = { + // For each of the input paths, get the list of files inside them + paths.flatMap { path => + // Make the path qualified (consistent with listLeafFiles and listLeafFilesInParallel). + val fs = path.getFileSystem(hadoopConf) + val qualifiedPath = path.makeQualified(fs.getUri, fs.getWorkingDirectory) + + // If it is a directory (i.e. exists in leafDirToChildrenFiles), return its children files + // Or if it is a file (i.e. exists in leafFiles), return the path itself + leafDirToChildrenFiles.get(qualifiedPath).orElse { --- End diff -- nvm. This logic is correct.
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org