Repository: spark Updated Branches: refs/heads/master e88bff127 -> 18bcbbdd8
[SPARK-13270][SQL] Remove extra new lines in whole stage codegen and include pipeline plan in comments. Author: Nong Li <n...@databricks.com> Closes #11155 from nongli/spark-13270. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/18bcbbdd Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/18bcbbdd Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/18bcbbdd Branch: refs/heads/master Commit: 18bcbbdd84e80222d1d29530831c6d68d02e7593 Parents: e88bff1 Author: Nong Li <n...@databricks.com> Authored: Wed Feb 10 23:52:19 2016 -0800 Committer: Reynold Xin <r...@databricks.com> Committed: Wed Feb 10 23:52:19 2016 -0800 ---------------------------------------------------------------------- .../catalyst/expressions/codegen/CodeFormatter.scala | 14 ++++++++++++++ .../spark/sql/execution/WholeStageCodegen.scala | 8 ++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/18bcbbdd/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala index 9b8b638..9d99bbf 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala @@ -25,6 +25,20 @@ package org.apache.spark.sql.catalyst.expressions.codegen */ object CodeFormatter { def format(code: String): String = new CodeFormatter().addLines(code).result() + def stripExtraNewLines(input: String): String = { + val code = new StringBuilder + var lastLine: String = "dummy" + input.split('\n').foreach { l => + val line = l.trim() + val skip = line == "" && (lastLine == "" || lastLine.endsWith("{")) + if (!skip) { + code.append(line) + code.append("\n") + } + lastLine = line + } + code.result() + } } private class CodeFormatter { http://git-wip-us.apache.org/repos/asf/spark/blob/18bcbbdd/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala index b200239..30f74fc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala @@ -237,6 +237,9 @@ case class WholeStageCodegen(plan: CodegenSupport, children: Seq[SparkPlan]) return new GeneratedIterator(references); } + /** Codegened pipeline for: + * ${plan.treeString.trim} + */ class GeneratedIterator extends org.apache.spark.sql.execution.BufferedRowIterator { private Object[] references; @@ -256,8 +259,9 @@ case class WholeStageCodegen(plan: CodegenSupport, children: Seq[SparkPlan]) """ // try to compile, helpful for debug - // println(s"${CodeFormatter.format(source)}") - CodeGenerator.compile(source) + val cleanedSource = CodeFormatter.stripExtraNewLines(source) + // println(s"${CodeFormatter.format(cleanedSource)}") + CodeGenerator.compile(cleanedSource) plan.upstream().mapPartitions { iter => --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org