Github user HyukjinKwon commented on a diff in the pull request: https://github.com/apache/spark/pull/19464#discussion_r144197159 --- Diff: core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala --- @@ -196,7 +196,10 @@ class HadoopRDD[K, V]( // add the credentials here as this can be called before SparkContext initialized SparkHadoopUtil.get.addCredentials(jobConf) val inputFormat = getInputFormat(jobConf) - val inputSplits = inputFormat.getSplits(jobConf, minPartitions) + var inputSplits = inputFormat.getSplits(jobConf, minPartitions) + if (sparkContext.getConf.getBoolean("spark.hadoop.filterOutEmptySplit", false)) { --- End diff -- I'd use `spark.files` prefix, taken after `spark.files.ignoreCorruptFiles`, `spark.files.maxPartitionBytes` and `spark.files.openCostInBytes`.
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org