Copilot commented on code in PR #13631:
URL: https://github.com/apache/gluten/pull/13631#discussion_r4550874065


##########
gluten-substrait/src/main/scala/org/apache/spark/sql/hive/HivePartitionConverter.scala:
##########
@@ -62,12 +62,16 @@ class HivePartitionConverter(hadoopConf: Configuration, 
session: SparkSession)
       codec.getClass.getSimpleName)
   }
 
+  private def isVisiblePath(f: FileStatus): Boolean = {
+    val name = f.getPath.getName

Review Comment:
   The visibility rule is subtle (especially the `startsWith(\"_\")` exception 
when the name contains `=`). Since this method is now used to decide whether to 
descend into directories, please add a brief comment explaining the rationale 
(e.g., allowing partition directories for columns that start with `_`, like 
`_p=1`, while still excluding hidden/system directories such as `_temporary`). 
This will reduce the risk of future changes accidentally breaking partition 
discovery.



##########
gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveTableScanSuite.scala:
##########
@@ -32,6 +40,55 @@ class GlutenHiveTableScanSuite
     "Spark-4077: timestamp query for null value"
   )
 
+  testGluten("Hive recursive file listing excludes hidden directories") {
+    withTempDir {
+      dir =>
+        Seq(
+          "root-data",
+          "visible/nested-data",
+          "_temporary/attempt/uncommitted-data",
+          ".backup/backup-data",
+          "visible/_temporary/nested-uncommitted-data",
+          "visible/.backup/nested-backup-data",
+          "_hidden-file",
+          ".hidden-file"
+        ).foreach {
+          relativePath =>
+            val file = dir.toPath.resolve(relativePath)
+            Files.createDirectories(file.getParent)
+            Files.write(file, "1\n".getBytes(StandardCharsets.UTF_8))
+        }
+
+        Seq(false, true).foreach {
+          recursive =>
+            val conf = new Configuration(spark.sessionState.newHadoopConf())
+            conf.setBoolean(FileInputFormat.INPUT_DIR_RECURSIVE, recursive)
+            // Ignore visible subdirectories in the non-recursive Hadoop 
control case.
+            conf.setBoolean(
+              
"mapreduce.input.fileinputformat.input.dir.nonrecursive.ignore.subdirs",
+              true)

Review Comment:
   The Hadoop config key for ignoring subdirs is a hard-coded string, which is 
easy to mistype and can drift across Hadoop APIs/versions (mapred vs 
mapreduce). Prefer referencing a defined constant (if available) or defining a 
local `val` for this key so it’s centrally managed and less error-prone.



##########
gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveTableScanSuite.scala:
##########
@@ -32,6 +40,55 @@ class GlutenHiveTableScanSuite
     "Spark-4077: timestamp query for null value"
   )
 
+  testGluten("Hive recursive file listing excludes hidden directories") {
+    withTempDir {
+      dir =>
+        Seq(
+          "root-data",
+          "visible/nested-data",
+          "_temporary/attempt/uncommitted-data",
+          ".backup/backup-data",
+          "visible/_temporary/nested-uncommitted-data",
+          "visible/.backup/nested-backup-data",
+          "_hidden-file",
+          ".hidden-file"
+        ).foreach {
+          relativePath =>
+            val file = dir.toPath.resolve(relativePath)
+            Files.createDirectories(file.getParent)
+            Files.write(file, "1\n".getBytes(StandardCharsets.UTF_8))
+        }
+
+        Seq(false, true).foreach {
+          recursive =>
+            val conf = new Configuration(spark.sessionState.newHadoopConf())
+            conf.setBoolean(FileInputFormat.INPUT_DIR_RECURSIVE, recursive)
+            // Ignore visible subdirectories in the non-recursive Hadoop 
control case.
+            conf.setBoolean(
+              
"mapreduce.input.fileinputformat.input.dir.nonrecursive.ignore.subdirs",
+              true)
+            val job = new JobConf(conf)
+            FileInputFormat.setInputPaths(job, new Path(dir.toURI))
+            val inputFormat = new TextInputFormat
+            inputFormat.configure(job)
+            val hadoopFiles = inputFormat
+              .getSplits(job, 1)
+              .map(_.asInstanceOf[FileSplit].getPath.getName)
+              .sorted
+              .toSeq
+            val expected = if (recursive) Seq("nested-data", "root-data") else 
Seq("root-data")
+            assert(hadoopFiles == expected)
+
+            val converterFiles = new HivePartitionConverter(conf, spark)
+              .createFilePartition(dir.toURI)
+              .flatMap(_.files)
+              .map(file => new Path(file.filePath.toString).getName)

Review Comment:
   The test compares only `getName` (base filename). This can mask real 
regressions if two different directories contain the same filename (the 
comparison would treat them as identical). Consider comparing normalized 
relative paths (relative to `dir`) for both Hadoop and converter results, so 
the assertion is robust to filename collisions.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to