spark git commit: [SPARK-10178] [SQL] HiveComparisionTest should print out dependent tables
Repository: spark
Updated Branches:
refs/heads/branch-1.5 c99f4160b - 2239a2036
[SPARK-10178] [SQL] HiveComparisionTest should print out dependent tables
In `HiveComparisionTest`s it is possible to fail a query of the form `SELECT *
FROM dest1`, where `dest1` is the query that is actually computing the
incorrect results. To aid debugging this patch improves the harness to also
print these query plans and their results.
Author: Michael Armbrust mich...@databricks.com
Closes #8388 from marmbrus/generatedTables.
(cherry picked from commit 5175ca0c85b10045d12c3fb57b1e52278a413ecf)
Signed-off-by: Reynold Xin r...@databricks.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2239a203
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2239a203
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2239a203
Branch: refs/heads/branch-1.5
Commit: 2239a20368b7833ffe0059941478924c7be87bbe
Parents: c99f416
Author: Michael Armbrust mich...@databricks.com
Authored: Mon Aug 24 23:15:27 2015 -0700
Committer: Reynold Xin r...@databricks.com
Committed: Mon Aug 24 23:15:34 2015 -0700
--
.../sql/hive/execution/HiveComparisonTest.scala | 36
1 file changed, 36 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2239a203/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index 2bdb0e1..4d45249 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.hive.execution
import java.io._
+import scala.util.control.NonFatal
+
import org.scalatest.{BeforeAndAfterAll, GivenWhenThen}
import org.apache.spark.{Logging, SparkFunSuite}
@@ -386,11 +388,45 @@ abstract class HiveComparisonTest
hiveCacheFiles.foreach(_.delete())
}
+ // If this query is reading other tables that were created
during this test run
+ // also print out the query plans and results for those.
+ val computedTablesMessages: String = try {
+val tablesRead = new
TestHive.QueryExecution(query).executedPlan.collect {
+ case ts: HiveTableScan = ts.relation.tableName
+}.toSet
+
+TestHive.reset()
+val executions = queryList.map(new TestHive.QueryExecution(_))
+executions.foreach(_.toRdd)
+val tablesGenerated = queryList.zip(executions).flatMap {
+ case (q, e) = e.executedPlan.collect {
+case i: InsertIntoHiveTable if tablesRead contains
i.table.tableName =
+ (q, e, i)
+ }
+}
+
+tablesGenerated.map { case (hiveql, execution, insert) =
+ s
+ |=== Generated Table ===
+ |$hiveql
+ |$execution
+ |== Results ==
+ |${insert.child.execute().collect().mkString(\n)}
+ .stripMargin
+}.mkString(\n)
+
+ } catch {
+case NonFatal(e) =
+ logError(Failed to compute generated tables, e)
+ sCouldn't compute dependent tables: $e
+ }
+
val errorMessage =
s
|Results do not match for $testCaseName:
|$hiveQuery\n${hiveQuery.analyzed.output.map(_.name).mkString(\t)}
|$resultComparison
+ |$computedTablesMessages
.stripMargin
stringToFile(new File(wrongDirectory, testCaseName),
errorMessage + consoleTestCase)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-10178] [SQL] HiveComparisionTest should print out dependent tables
Repository: spark
Updated Branches:
refs/heads/master a0c0aae1d - 5175ca0c8
[SPARK-10178] [SQL] HiveComparisionTest should print out dependent tables
In `HiveComparisionTest`s it is possible to fail a query of the form `SELECT *
FROM dest1`, where `dest1` is the query that is actually computing the
incorrect results. To aid debugging this patch improves the harness to also
print these query plans and their results.
Author: Michael Armbrust mich...@databricks.com
Closes #8388 from marmbrus/generatedTables.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5175ca0c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5175ca0c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5175ca0c
Branch: refs/heads/master
Commit: 5175ca0c85b10045d12c3fb57b1e52278a413ecf
Parents: a0c0aae
Author: Michael Armbrust mich...@databricks.com
Authored: Mon Aug 24 23:15:27 2015 -0700
Committer: Reynold Xin r...@databricks.com
Committed: Mon Aug 24 23:15:27 2015 -0700
--
.../sql/hive/execution/HiveComparisonTest.scala | 36
1 file changed, 36 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5175ca0c/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index 2bdb0e1..4d45249 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.hive.execution
import java.io._
+import scala.util.control.NonFatal
+
import org.scalatest.{BeforeAndAfterAll, GivenWhenThen}
import org.apache.spark.{Logging, SparkFunSuite}
@@ -386,11 +388,45 @@ abstract class HiveComparisonTest
hiveCacheFiles.foreach(_.delete())
}
+ // If this query is reading other tables that were created
during this test run
+ // also print out the query plans and results for those.
+ val computedTablesMessages: String = try {
+val tablesRead = new
TestHive.QueryExecution(query).executedPlan.collect {
+ case ts: HiveTableScan = ts.relation.tableName
+}.toSet
+
+TestHive.reset()
+val executions = queryList.map(new TestHive.QueryExecution(_))
+executions.foreach(_.toRdd)
+val tablesGenerated = queryList.zip(executions).flatMap {
+ case (q, e) = e.executedPlan.collect {
+case i: InsertIntoHiveTable if tablesRead contains
i.table.tableName =
+ (q, e, i)
+ }
+}
+
+tablesGenerated.map { case (hiveql, execution, insert) =
+ s
+ |=== Generated Table ===
+ |$hiveql
+ |$execution
+ |== Results ==
+ |${insert.child.execute().collect().mkString(\n)}
+ .stripMargin
+}.mkString(\n)
+
+ } catch {
+case NonFatal(e) =
+ logError(Failed to compute generated tables, e)
+ sCouldn't compute dependent tables: $e
+ }
+
val errorMessage =
s
|Results do not match for $testCaseName:
|$hiveQuery\n${hiveQuery.analyzed.output.map(_.name).mkString(\t)}
|$resultComparison
+ |$computedTablesMessages
.stripMargin
stringToFile(new File(wrongDirectory, testCaseName),
errorMessage + consoleTestCase)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
