[ https://issues.apache.org/jira/browse/SPARK-24839?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
marios iliofotou updated SPARK-24839: ------------------------------------- Description: The problem shows up when joining a column that has constant (lit) value. As seen from the exception in the logical plan, the literal column gets dropped, which results in joining two DF on a column that does not exist (which then correctly results in a Cartesian join). {code:java} scala> val df1 = spark.createDataFrame(Seq((1, 2), (2, 4))).withColumn("index", lit("a")) scala> df1.show +---+---+-----+ | _1| _2|index| +---+---+-----+ | 1| 2| a| | 2| 4| a| +---+---+-----+ scala> val df2 = spark.createDataFrame(Seq(("a", 1),("b", 2))).toDF("index", "someval") scala> df2.show() +-----+-------+ |index|someval| +-----+-------+ | a| 1| | b| 2| +-----+-------+ scala> df1.join(df2).show() org.apache.spark.sql.AnalysisException: Detected implicit cartesian product for INNER join between logical plans LocalRelation [_1#370, _2#371|#370, _2#371] and LocalRelation [index#335, someval#336|#335, someval#336] Join condition is missing or trivial. Either: use the CROSS JOIN syntax to allow cartesian products between these relations, or: enable implicit cartesian products by setting the configuration variable spark.sql.crossJoin.enabled=true; at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$21.applyOrElse(Optimizer.scala:1124) at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$21.applyOrElse(Optimizer.scala:1121) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:266) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:256) at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$.apply(Optimizer.scala:1121) at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$.apply(Optimizer.scala:1103) at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:87) at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:84) at scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:57) at scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:66) at scala.collection.mutable.WrappedArray.foldLeft(WrappedArray.scala:35) at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:84) at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:76) at scala.collection.immutable.List.foreach(List.scala:392) at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:76) at org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:66) at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:66) at org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute(QueryExecution.scala:72) at org.apache.spark.sql.execution.QueryExecution.sparkPlan(QueryExecution.scala:68) at org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:77) at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:77) at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3249) at org.apache.spark.sql.Dataset.head(Dataset.scala:2484) at org.apache.spark.sql.Dataset.take(Dataset.scala:2698) at org.apache.spark.sql.Dataset.showString(Dataset.scala:254) at org.apache.spark.sql.Dataset.show(Dataset.scala:723) at org.apache.spark.sql.Dataset.show(Dataset.scala:682) at org.apache.spark.sql.Dataset.show(Dataset.scala:691) ... 42 elided {code} Same error shows up even when we are not using a natural join: {code:java} df1.join(df2, df1("index") === df2("index") ).show(){code} Same if you change the joining order: {code:java} scala> df2.join(df1, df2("index") === df1("index"), "left_outer").show() org.apache.spark.sql.AnalysisException: Detected implicit cartesian product for LEFT OUTER join between logical plans LocalRelation [index#12, someval#13] and LocalRelation [_1#0, _2#1, index#4] Join condition is missing or trivial. Either: use the CROSS JOIN syntax to allow cartesian products between these relations, or: enable implicit cartesian products by setting the configuration variable spark.sql.crossJoin.enabled=true; {code} Same if you have the literal column to not match any of the keys in the second DataFrame: {code:java} scala> val df1 = spark.createDataFrame(Seq((1, 2), (2, 4))).withColumn("index", lit("x")) df1: org.apache.spark.sql.DataFrame = [_1: int, _2: int ... 1 more field] scala> val df2 = spark.createDataFrame(Seq(("a", 1),("b", 2))).toDF("index", "someval") df2: org.apache.spark.sql.DataFrame = [index: string, someval: int] scala> df2.join(df1, df2("index") === df1("index"), "left_outer").show() org.apache.spark.sql.AnalysisException: Detected implicit cartesian product for LEFT OUTER join between logical plans LocalRelation [index#142, someval#143] and LocalRelation [_1#130, _2#131, index#134] Join condition is missing or trivial. {code} was: The problem shows up when joining a column that has constant value. As seen from the exception in the logical plan, the literal column gets dropped, which results in joining two DF on a column that does not exist, which correctly results in a Cartesian join. {code:java} scala> val df1 = spark.createDataFrame(Seq((1, 2), (2, 4))).withColumn("index", lit("a")) scala> df1.show +---+---+-----+ | _1| _2|index| +---+---+-----+ | 1| 2| a| | 2| 4| a| +---+---+-----+ scala> val df2 = spark.createDataFrame(Seq(("a", 1),("b", 2))).toDF("index", "someval") scala> df2.show() +-----+-------+ |index|someval| +-----+-------+ | a| 1| | b| 2| +-----+-------+ scala> df1.join(df2).show() org.apache.spark.sql.AnalysisException: Detected implicit cartesian product for INNER join between logical plans LocalRelation [_1#370, _2#371|#370, _2#371] and LocalRelation [index#335, someval#336|#335, someval#336] Join condition is missing or trivial. Either: use the CROSS JOIN syntax to allow cartesian products between these relations, or: enable implicit cartesian products by setting the configuration variable spark.sql.crossJoin.enabled=true; at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$21.applyOrElse(Optimizer.scala:1124) at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$21.applyOrElse(Optimizer.scala:1121) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:266) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:256) at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$.apply(Optimizer.scala:1121) at org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$.apply(Optimizer.scala:1103) at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:87) at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:84) at scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:57) at scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:66) at scala.collection.mutable.WrappedArray.foldLeft(WrappedArray.scala:35) at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:84) at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:76) at scala.collection.immutable.List.foreach(List.scala:392) at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:76) at org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:66) at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:66) at org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute(QueryExecution.scala:72) at org.apache.spark.sql.execution.QueryExecution.sparkPlan(QueryExecution.scala:68) at org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:77) at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:77) at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3249) at org.apache.spark.sql.Dataset.head(Dataset.scala:2484) at org.apache.spark.sql.Dataset.take(Dataset.scala:2698) at org.apache.spark.sql.Dataset.showString(Dataset.scala:254) at org.apache.spark.sql.Dataset.show(Dataset.scala:723) at org.apache.spark.sql.Dataset.show(Dataset.scala:682) at org.apache.spark.sql.Dataset.show(Dataset.scala:691) ... 42 elided {code} Same error shows up even when we are not using a natural join: {code:java} df1.join(df2, df1("index") === df2("index") ).show(){code} Same if you change the joining order: {code:java} scala> df2.join(df1, df2("index") === df1("index"), "left_outer").show() org.apache.spark.sql.AnalysisException: Detected implicit cartesian product for LEFT OUTER join between logical plans LocalRelation [index#12, someval#13] and LocalRelation [_1#0, _2#1, index#4] Join condition is missing or trivial. Either: use the CROSS JOIN syntax to allow cartesian products between these relations, or: enable implicit cartesian products by setting the configuration variable spark.sql.crossJoin.enabled=true; {code} Same if you have the literal column to not match any of the keys in the second DataFrame: {code:java} scala> val df1 = spark.createDataFrame(Seq((1, 2), (2, 4))).withColumn("index", lit("x")) df1: org.apache.spark.sql.DataFrame = [_1: int, _2: int ... 1 more field] scala> val df2 = spark.createDataFrame(Seq(("a", 1),("b", 2))).toDF("index", "someval") df2: org.apache.spark.sql.DataFrame = [index: string, someval: int] scala> df2.join(df1, df2("index") === df1("index"), "left_outer").show() org.apache.spark.sql.AnalysisException: Detected implicit cartesian product for LEFT OUTER join between logical plans LocalRelation [index#142, someval#143] and LocalRelation [_1#130, _2#131, index#134] Join condition is missing or trivial. {code} > Incorrect drop of lit() column results in cross join > ---------------------------------------------------- > > Key: SPARK-24839 > URL: https://issues.apache.org/jira/browse/SPARK-24839 > Project: Spark > Issue Type: Bug > Components: Optimizer > Affects Versions: 2.3.1 > Reporter: marios iliofotou > Priority: Major > > The problem shows up when joining a column that has constant (lit) value. As > seen from the exception in the logical plan, the literal column gets dropped, > which results in joining two DF on a column that does not exist (which then > correctly results in a Cartesian join). > > {code:java} > scala> val df1 = spark.createDataFrame(Seq((1, 2), (2, > 4))).withColumn("index", lit("a")) > scala> df1.show > +---+---+-----+ > | _1| _2|index| > +---+---+-----+ > | 1| 2| a| > | 2| 4| a| > +---+---+-----+ > > scala> val df2 = spark.createDataFrame(Seq(("a", 1),("b", 2))).toDF("index", > "someval") > scala> df2.show() > +-----+-------+ > |index|someval| > +-----+-------+ > | a| 1| > | b| 2| > +-----+-------+ > scala> df1.join(df2).show() > org.apache.spark.sql.AnalysisException: Detected implicit cartesian product > for INNER join between logical plans > LocalRelation [_1#370, _2#371|#370, _2#371] > and > LocalRelation [index#335, someval#336|#335, someval#336] > Join condition is missing or trivial. > Either: use the CROSS JOIN syntax to allow cartesian products between these > relations, or: enable implicit cartesian products by setting the > configuration > variable spark.sql.crossJoin.enabled=true; > at > org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$21.applyOrElse(Optimizer.scala:1124) > at > org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$21.applyOrElse(Optimizer.scala:1121) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267) > at > org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70) > at > org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:266) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) > at > org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) > at > org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) > at > org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) > at > org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) > at > org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) > at > org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306) > at > org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) > at > org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304) > at > org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272) > at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:256) > at > org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$.apply(Optimizer.scala:1121) > at > org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$.apply(Optimizer.scala:1103) > at > org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:87) > at > org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:84) > at > scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:57) > at > scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:66) > at scala.collection.mutable.WrappedArray.foldLeft(WrappedArray.scala:35) > at > org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:84) > at > org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:76) > at scala.collection.immutable.List.foreach(List.scala:392) > at > org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:76) > at > org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:66) > at > org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:66) > at > org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute(QueryExecution.scala:72) > at > org.apache.spark.sql.execution.QueryExecution.sparkPlan(QueryExecution.scala:68) > at > org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:77) > at > org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:77) > at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3249) > at org.apache.spark.sql.Dataset.head(Dataset.scala:2484) > at org.apache.spark.sql.Dataset.take(Dataset.scala:2698) > at org.apache.spark.sql.Dataset.showString(Dataset.scala:254) > at org.apache.spark.sql.Dataset.show(Dataset.scala:723) > at org.apache.spark.sql.Dataset.show(Dataset.scala:682) > at org.apache.spark.sql.Dataset.show(Dataset.scala:691) > ... 42 elided > {code} > > Same error shows up even when we are not using a natural join: > {code:java} > df1.join(df2, df1("index") === df2("index") ).show(){code} > > Same if you change the joining order: > > {code:java} > scala> df2.join(df1, df2("index") === df1("index"), "left_outer").show() > org.apache.spark.sql.AnalysisException: Detected implicit cartesian product > for LEFT OUTER join between logical plans > LocalRelation [index#12, someval#13] > and > LocalRelation [_1#0, _2#1, index#4] > Join condition is missing or trivial. > Either: use the CROSS JOIN syntax to allow cartesian products between these > relations, or: enable implicit cartesian products by setting the configuration > variable spark.sql.crossJoin.enabled=true; > {code} > > Same if you have the literal column to not match any of the keys in the > second DataFrame: > {code:java} > scala> val df1 = spark.createDataFrame(Seq((1, 2), (2, > 4))).withColumn("index", lit("x")) > df1: org.apache.spark.sql.DataFrame = [_1: int, _2: int ... 1 more field] > scala> val df2 = spark.createDataFrame(Seq(("a", 1),("b", 2))).toDF("index", > "someval") > df2: org.apache.spark.sql.DataFrame = [index: string, someval: int] > scala> df2.join(df1, df2("index") === df1("index"), "left_outer").show() > org.apache.spark.sql.AnalysisException: Detected implicit cartesian product > for LEFT OUTER join between logical plans > LocalRelation [index#142, someval#143] > and > LocalRelation [_1#130, _2#131, index#134] > Join condition is missing or trivial. > {code} > -- This message was sent by Atlassian JIRA (v7.6.3#76005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org