Repository: spark Updated Branches: refs/heads/branch-2.1 906d82c4c -> da66b9742
[SPARK-18583][SQL] Fix nullability of InputFileName. ## What changes were proposed in this pull request? The nullability of `InputFileName` should be `false`. ## How was this patch tested? Existing tests. Author: Takuya UESHIN <ues...@happy-camper.st> Closes #16007 from ueshin/issues/SPARK-18583. (cherry picked from commit a88329d4553b40c45ebf9eacf229db7839d46769) Signed-off-by: Reynold Xin <r...@databricks.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/da66b974 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/da66b974 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/da66b974 Branch: refs/heads/branch-2.1 Commit: da66b9742eabb2654b369f634eb05910220a6441 Parents: 906d82c Author: Takuya UESHIN <ues...@happy-camper.st> Authored: Fri Nov 25 20:25:29 2016 -0800 Committer: Reynold Xin <r...@databricks.com> Committed: Fri Nov 25 20:25:39 2016 -0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/rdd/InputFileNameHolder.scala | 10 +++++++++- .../spark/sql/catalyst/expressions/InputFileName.scala | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/da66b974/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala b/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala index f40d4c8..960c91a 100644 --- a/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala +++ b/core/src/main/scala/org/apache/spark/rdd/InputFileNameHolder.scala @@ -22,6 +22,8 @@ import org.apache.spark.unsafe.types.UTF8String /** * This holds file names of the current Spark task. This is used in HadoopRDD, * FileScanRDD, NewHadoopRDD and InputFileName function in Spark SQL. + * + * The returned value should never be null but empty string if it is unknown. */ private[spark] object InputFileNameHolder { /** @@ -32,9 +34,15 @@ private[spark] object InputFileNameHolder { override protected def initialValue(): UTF8String = UTF8String.fromString("") } + /** + * Returns the holding file name or empty string if it is unknown. + */ def getInputFileName(): UTF8String = inputFileName.get() - private[spark] def setInputFileName(file: String) = inputFileName.set(UTF8String.fromString(file)) + private[spark] def setInputFileName(file: String) = { + require(file != null, "The input file name cannot be null") + inputFileName.set(UTF8String.fromString(file)) + } private[spark] def unsetInputFileName(): Unit = inputFileName.remove() http://git-wip-us.apache.org/repos/asf/spark/blob/da66b974/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala index b7fb285..d412336 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala @@ -30,7 +30,7 @@ import org.apache.spark.unsafe.types.UTF8String usage = "_FUNC_() - Returns the name of the current file being read if available.") case class InputFileName() extends LeafExpression with Nondeterministic { - override def nullable: Boolean = true + override def nullable: Boolean = false override def dataType: DataType = StringType --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org