spark git commit: [SPARK-5454] More robust handling of self joins
Repository: spark
Updated Branches:
refs/heads/master 03bf704bf -> a60d2b70a
[SPARK-5454] More robust handling of self joins
Also I fix a bunch of bad output in test cases.
Author: Michael Armbrust
Closes #4520 from marmbrus/selfJoin and squashes the following commits:
4f4a85c [Michael Armbrust] comments
49c8e26 [Michael Armbrust] fix tests
6fc38de [Michael Armbrust] fix style
55d64b3 [Michael Armbrust] fix dataframe selfjoins
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a60d2b70
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a60d2b70
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a60d2b70
Branch: refs/heads/master
Commit: a60d2b70adff3a8fb3bdfac226b1d86fdb443da4
Parents: 03bf704
Author: Michael Armbrust
Authored: Wed Feb 11 12:31:56 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 11 12:31:56 2015 -0800
--
.../spark/sql/catalyst/analysis/Analyzer.scala | 26 +---
.../analysis/MultiInstanceRelation.scala| 21 +---
.../spark/sql/catalyst/plans/PlanTest.scala | 4 +--
.../scala/org/apache/spark/sql/SQLContext.scala | 2 ++
sql/core/src/test/resources/log4j.properties| 3 +++
.../org/apache/spark/sql/DataFrameSuite.scala | 10
.../spark/sql/catalyst/plans/PlanTest.scala | 4 +--
7 files changed, 40 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a60d2b70/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 3f0d77a..2d1fa10 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -53,14 +53,11 @@ class Analyzer(catalog: Catalog,
val extendedRules: Seq[Rule[LogicalPlan]] = Nil
lazy val batches: Seq[Batch] = Seq(
-Batch("MultiInstanceRelations", Once,
- NewRelationInstances),
Batch("Resolution", fixedPoint,
- ResolveReferences ::
ResolveRelations ::
+ ResolveReferences ::
ResolveGroupingAnalytics ::
ResolveSortReferences ::
- NewRelationInstances ::
ImplicitGenerate ::
ResolveFunctions ::
GlobalAggregates ::
@@ -285,6 +282,27 @@ class Analyzer(catalog: Catalog,
}
)
+ // Special handling for cases when self-join introduce duplicate
expression ids.
+ case j @ Join(left, right, _, _) if
left.outputSet.intersect(right.outputSet).nonEmpty =>
+val conflictingAttributes = left.outputSet.intersect(right.outputSet)
+
+val (oldRelation, newRelation, attributeRewrites) = right.collect {
+ case oldVersion: MultiInstanceRelation
+ if
oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
+val newVersion = oldVersion.newInstance()
+val newAttributes =
AttributeMap(oldVersion.output.zip(newVersion.output))
+(oldVersion, newVersion, newAttributes)
+}.head // Only handle first case found, others will be fixed on the
next pass.
+
+val newRight = right transformUp {
+ case r if r == oldRelation => newRelation
+ case other => other transformExpressions {
+case a: Attribute => attributeRewrites.get(a).getOrElse(a)
+ }
+}
+
+j.copy(right = newRight)
+
case q: LogicalPlan =>
logTrace(s"Attempting to resolve ${q.simpleString}")
q transformExpressionsUp {
http://git-wip-us.apache.org/repos/asf/spark/blob/a60d2b70/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
index 4c5fb3f..894c350 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/MultiInstanceRelation.scala
@@ -26,28 +26,9 @@ import
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
* produced by distinct operators in a query tree as this breaks the guarantee
that expression
* ids, which are used to differentiate attributes, are unique.
*
- * Before analysis, all operators that include this trait will be asked to
p
spark git commit: [SPARK-3688][SQL]LogicalPlan can't resolve column correctlly
-
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-0-c6d02549aec166e16bfc44d5905fa33a
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-0-c6d02549aec166e16bfc44d5905fa33a
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/44b2311d/sql/hive/src/test/resources/golden/test
ambiguousReferences resolved as hive-1-a8987ff8c7b9ca95bf8b32314694ed1f
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-1-a8987ff8c7b9ca95bf8b32314694ed1f
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-1-a8987ff8c7b9ca95bf8b32314694ed1f
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/44b2311d/sql/hive/src/test/resources/golden/test
ambiguousReferences resolved as hive-2-26f54240cf5b909086fc34a34d7fdb56
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-2-26f54240cf5b909086fc34a34d7fdb56
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-2-26f54240cf5b909086fc34a34d7fdb56
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/44b2311d/sql/hive/src/test/resources/golden/test
ambiguousReferences resolved as hive-3-d08d5280027adea681001ad82a5a6974
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-3-d08d5280027adea681001ad82a5a6974
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-3-d08d5280027adea681001ad82a5a6974
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/44b2311d/sql/hive/src/test/resources/golden/test
ambiguousReferences resolved as hive-4-22eb25b5be6daf72a6649adfe5041749
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-4-22eb25b5be6daf72a6649adfe5041749
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-4-22eb25b5be6daf72a6649adfe5041749
new file mode 100644
index 000..d00491f
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-4-22eb25b5be6daf72a6649adfe5041749
@@ -0,0 +1 @@
+1
http://git-wip-us.apache.org/repos/asf/spark/blob/44b2311d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
index ab5f9cd..029c36a 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
@@ -99,6 +99,15 @@ class HiveResolutionSuite extends HiveComparisonTest {
assert(sql("SELECT nestedArray[0].a FROM
nestedRepeatedTest").collect().head(0) === 1)
}
+ createQueryTest("test ambiguousReferences resolved as hive",
+"""
+ |CREATE TABLE t1(x INT);
+ |CREATE TABLE t2(a STRUCT, k INT);
+ |INSERT OVERWRITE TABLE t1 SELECT 1 FROM src LIMIT 1;
+ |INSERT OVERWRITE TABLE t2 SELECT named_struct("x",1),1 FROM src LIMIT 1;
+ |SELECT a.x FROM t1 a JOIN t2 b ON a.x = b.k;
+""".stripMargin)
+
/**
* Negative examples. Currently only left here for documentation purposes.
* TODO(marmbrus): Test that catalyst fails on these queries.
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-3688][SQL]LogicalPlan can't resolve column correctlly
rces/golden/test
ambiguousReferences resolved as hive-0-c6d02549aec166e16bfc44d5905fa33a
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-0-c6d02549aec166e16bfc44d5905fa33a
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-0-c6d02549aec166e16bfc44d5905fa33a
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/e136f477/sql/hive/src/test/resources/golden/test
ambiguousReferences resolved as hive-1-a8987ff8c7b9ca95bf8b32314694ed1f
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-1-a8987ff8c7b9ca95bf8b32314694ed1f
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-1-a8987ff8c7b9ca95bf8b32314694ed1f
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/e136f477/sql/hive/src/test/resources/golden/test
ambiguousReferences resolved as hive-2-26f54240cf5b909086fc34a34d7fdb56
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-2-26f54240cf5b909086fc34a34d7fdb56
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-2-26f54240cf5b909086fc34a34d7fdb56
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/e136f477/sql/hive/src/test/resources/golden/test
ambiguousReferences resolved as hive-3-d08d5280027adea681001ad82a5a6974
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-3-d08d5280027adea681001ad82a5a6974
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-3-d08d5280027adea681001ad82a5a6974
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/e136f477/sql/hive/src/test/resources/golden/test
ambiguousReferences resolved as hive-4-22eb25b5be6daf72a6649adfe5041749
--
diff --git a/sql/hive/src/test/resources/golden/test ambiguousReferences
resolved as hive-4-22eb25b5be6daf72a6649adfe5041749
b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-4-22eb25b5be6daf72a6649adfe5041749
new file mode 100644
index 000..d00491f
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/test ambiguousReferences resolved as
hive-4-22eb25b5be6daf72a6649adfe5041749
@@ -0,0 +1 @@
+1
http://git-wip-us.apache.org/repos/asf/spark/blob/e136f477/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
index ab5f9cd..029c36a 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
@@ -99,6 +99,15 @@ class HiveResolutionSuite extends HiveComparisonTest {
assert(sql("SELECT nestedArray[0].a FROM
nestedRepeatedTest").collect().head(0) === 1)
}
+ createQueryTest("test ambiguousReferences resolved as hive",
+"""
+ |CREATE TABLE t1(x INT);
+ |CREATE TABLE t2(a STRUCT, k INT);
+ |INSERT OVERWRITE TABLE t1 SELECT 1 FROM src LIMIT 1;
+ |INSERT OVERWRITE TABLE t2 SELECT named_struct("x",1),1 FROM src LIMIT 1;
+ |SELECT a.x FROM t1 a JOIN t2 b ON a.x = b.k;
+""".stripMargin)
+
/**
* Negative examples. Currently only left here for documentation purposes.
* TODO(marmbrus): Test that catalyst fails on these queries.
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SQL] Make dataframe more tolerant of being serialized
Repository: spark
Updated Branches:
refs/heads/master d931b01dc -> a38e23c30
[SQL] Make dataframe more tolerant of being serialized
Eases use in the spark-shell.
Author: Michael Armbrust
Closes #4545 from marmbrus/serialization and squashes the following commits:
04748e6 [Michael Armbrust] @scala.annotation.varargs
b36e219 [Michael Armbrust] moreFixes
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a38e23c3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a38e23c3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a38e23c3
Branch: refs/heads/master
Commit: a38e23c30fb5d12f8f46a119d91a0620036e6800
Parents: d931b01
Author: Michael Armbrust
Authored: Wed Feb 11 19:05:49 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 11 19:05:49 2015 -0800
--
.../src/main/scala/org/apache/spark/sql/DataFrame.scala | 2 +-
.../src/main/scala/org/apache/spark/sql/DataFrameImpl.scala | 4 ++--
sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala | 9 +
.../src/main/scala/org/apache/spark/sql/SQLContext.scala| 4 +++-
4 files changed, 15 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a38e23c3/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 327cf87..13aff76 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -76,7 +76,7 @@ private[sql] object DataFrame {
*/
// TODO: Improve documentation.
@Experimental
-trait DataFrame extends RDDApi[Row] {
+trait DataFrame extends RDDApi[Row] with Serializable {
val sqlContext: SQLContext
http://git-wip-us.apache.org/repos/asf/spark/blob/a38e23c3/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
index 3863df5..4c6e19c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
@@ -44,8 +44,8 @@ import org.apache.spark.sql.types.{NumericType, StructType}
* Internal implementation of [[DataFrame]]. Users of the API should use
[[DataFrame]] directly.
*/
private[sql] class DataFrameImpl protected[sql](
-override val sqlContext: SQLContext,
-val queryExecution: SQLContext#QueryExecution)
+@transient override val sqlContext: SQLContext,
+@transient val queryExecution: SQLContext#QueryExecution)
extends DataFrame {
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/a38e23c3/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
index 6bf21dd..7bc7683 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
@@ -167,6 +167,15 @@ object Dsl {
//
/**
+ * Returns the first column that is not null.
+ * {{{
+ * df.select(coalesce(df("a"), df("b")))
+ * }}}
+ */
+ @scala.annotation.varargs
+ def coalesce(e: Column*): Column = Coalesce(e.map(_.expr))
+
+ /**
* Unary minus, i.e. negate the expression.
* {{{
* // Select the amount column and negates all values.
http://git-wip-us.apache.org/repos/asf/spark/blob/a38e23c3/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index ca5e62f..8aae222 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -129,9 +129,11 @@ class SQLContext(@transient val sparkContext: SparkContext)
* A collection of methods that are considered experimental, but can be used
to hook into
* the query planner for advanced functionalities.
*/
+ @transient
val experimental: ExperimentalMethods = new ExperimentalMethods(this)
/** Returns a [[DataFrame]] with no rows or columns. */
+ @transient
lazy val emptyDataFrame = DataFrame(this, NoRelation)
/**
@@ -178,7 +18
spark git commit: [SQL] Make dataframe more tolerant of being serialized
Repository: spark
Updated Branches:
refs/heads/branch-1.3 bcb13827c -> 3c1b9bf65
[SQL] Make dataframe more tolerant of being serialized
Eases use in the spark-shell.
Author: Michael Armbrust
Closes #4545 from marmbrus/serialization and squashes the following commits:
04748e6 [Michael Armbrust] @scala.annotation.varargs
b36e219 [Michael Armbrust] moreFixes
(cherry picked from commit a38e23c30fb5d12f8f46a119d91a0620036e6800)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3c1b9bf6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3c1b9bf6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3c1b9bf6
Branch: refs/heads/branch-1.3
Commit: 3c1b9bf65290cc1fd690a5c5c252667e4576
Parents: bcb1382
Author: Michael Armbrust
Authored: Wed Feb 11 19:05:49 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 11 19:06:05 2015 -0800
--
.../src/main/scala/org/apache/spark/sql/DataFrame.scala | 2 +-
.../src/main/scala/org/apache/spark/sql/DataFrameImpl.scala | 4 ++--
sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala | 9 +
.../src/main/scala/org/apache/spark/sql/SQLContext.scala| 4 +++-
4 files changed, 15 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/3c1b9bf6/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 327cf87..13aff76 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -76,7 +76,7 @@ private[sql] object DataFrame {
*/
// TODO: Improve documentation.
@Experimental
-trait DataFrame extends RDDApi[Row] {
+trait DataFrame extends RDDApi[Row] with Serializable {
val sqlContext: SQLContext
http://git-wip-us.apache.org/repos/asf/spark/blob/3c1b9bf6/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
index 3863df5..4c6e19c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
@@ -44,8 +44,8 @@ import org.apache.spark.sql.types.{NumericType, StructType}
* Internal implementation of [[DataFrame]]. Users of the API should use
[[DataFrame]] directly.
*/
private[sql] class DataFrameImpl protected[sql](
-override val sqlContext: SQLContext,
-val queryExecution: SQLContext#QueryExecution)
+@transient override val sqlContext: SQLContext,
+@transient val queryExecution: SQLContext#QueryExecution)
extends DataFrame {
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/3c1b9bf6/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
index 6bf21dd..7bc7683 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
@@ -167,6 +167,15 @@ object Dsl {
//
/**
+ * Returns the first column that is not null.
+ * {{{
+ * df.select(coalesce(df("a"), df("b")))
+ * }}}
+ */
+ @scala.annotation.varargs
+ def coalesce(e: Column*): Column = Coalesce(e.map(_.expr))
+
+ /**
* Unary minus, i.e. negate the expression.
* {{{
* // Select the amount column and negates all values.
http://git-wip-us.apache.org/repos/asf/spark/blob/3c1b9bf6/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index ca5e62f..8aae222 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -129,9 +129,11 @@ class SQLContext(@transient val sparkContext: SparkContext)
* A collection of methods that are considered experimental, but can be used
to hook into
* the query planner for advanced functionalities.
*/
+ @transient
val experimental: ExperimentalMethods = new ExperimentalMethods(this)
/** Returns a [[DataFrame]]
spark git commit: [SQL] Improve error messages
Repository: spark
Updated Branches:
refs/heads/master 6a1be026c -> aa4ca8b87
[SQL] Improve error messages
Author: Michael Armbrust
Author: wangfei
Closes #4558 from marmbrus/errorMessages and squashes the following commits:
5e5ab50 [Michael Armbrust] Merge pull request #15 from scwf/errorMessages
fa38881 [wangfei] fix for grouping__id
f279a71 [wangfei] make right references for ScriptTransformation
d29fbde [Michael Armbrust] extra case
1a797b4 [Michael Armbrust] comments
d4e9015 [Michael Armbrust] add comment
af9e668 [Michael Armbrust] no braces
34eb3a4 [Michael Armbrust] more work
6197cd5 [Michael Armbrust] [SQL] Better error messages for analysis failures
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/aa4ca8b8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/aa4ca8b8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/aa4ca8b8
Branch: refs/heads/master
Commit: aa4ca8b873fd83e64e5faea6f7febcc830e30b02
Parents: 6a1be02
Author: Michael Armbrust
Authored: Thu Feb 12 13:11:28 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 13:11:28 2015 -0800
--
.../spark/sql/catalyst/analysis/Analyzer.scala | 123 +++
.../sql/catalyst/expressions/AttributeSet.scala | 6 +-
.../catalyst/expressions/namedExpressions.scala | 2 +-
.../plans/logical/ScriptTransformation.scala| 6 +-
.../spark/sql/catalyst/trees/TreeNode.scala | 9 ++
.../sql/catalyst/analysis/AnalysisSuite.scala | 79
.../optimizer/BooleanSimplificationSuite.scala | 4 +-
.../optimizer/ConstantFoldingSuite.scala| 4 +-
.../optimizer/FilterPushdownSuite.scala | 12 +-
.../catalyst/optimizer/OptimizeInSuite.scala| 4 +-
.../catalyst/optimizer/UnionPushdownSuite.scala | 4 +-
.../org/apache/spark/sql/SQLQuerySuite.scala| 6 +-
.../org/apache/spark/sql/hive/HiveContext.scala | 4 +-
.../spark/sql/hive/execution/commands.scala | 4 +-
14 files changed, 164 insertions(+), 103 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/aa4ca8b8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 2d1fa10..58a7003 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.errors.TreeNodeException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{ArrayType, StructField, StructType,
IntegerType}
+import org.apache.spark.sql.types._
/**
* A trivial [[Analyzer]] with an [[EmptyCatalog]] and
[[EmptyFunctionRegistry]]. Used for testing
@@ -66,32 +66,82 @@ class Analyzer(catalog: Catalog,
typeCoercionRules ++
extendedRules : _*),
Batch("Check Analysis", Once,
- CheckResolution ::
- CheckAggregation ::
- Nil: _*),
-Batch("AnalysisOperators", fixedPoint,
- EliminateAnalysisOperators)
+ CheckResolution),
+Batch("Remove SubQueries", fixedPoint,
+ EliminateSubQueries)
)
/**
* Makes sure all attributes and logical plans have been resolved.
*/
object CheckResolution extends Rule[LogicalPlan] {
+def failAnalysis(msg: String) = { throw new AnalysisException(msg) }
+
def apply(plan: LogicalPlan): LogicalPlan = {
- plan.transformUp {
-case p if p.expressions.exists(!_.resolved) =>
- val missing =
p.expressions.filterNot(_.resolved).map(_.prettyString).mkString(",")
- val from = p.inputSet.map(_.name).mkString("{", ", ", "}")
-
- throw new AnalysisException(s"Cannot resolve '$missing' given input
columns $from")
-case p if !p.resolved && p.childrenResolved =>
- throw new AnalysisException(s"Unresolved operator in the query plan
${p.simpleString}")
- } match {
-// As a backstop, use the root node to check that the entire plan tree
is resolved.
-case p if !p.resolved =>
- throw new AnalysisException(s"Unresolved operator in the query plan
${p.simpleString}")
-case p => p
+ // We transform up and order the rules so as to catch the first possible
failure instead
+ // of the result of cascading resolution failures.
+ p
spark git commit: [SQL] Improve error messages
Repository: spark
Updated Branches:
refs/heads/branch-1.3 cbd659e5f -> e3a975d45
[SQL] Improve error messages
Author: Michael Armbrust
Author: wangfei
Closes #4558 from marmbrus/errorMessages and squashes the following commits:
5e5ab50 [Michael Armbrust] Merge pull request #15 from scwf/errorMessages
fa38881 [wangfei] fix for grouping__id
f279a71 [wangfei] make right references for ScriptTransformation
d29fbde [Michael Armbrust] extra case
1a797b4 [Michael Armbrust] comments
d4e9015 [Michael Armbrust] add comment
af9e668 [Michael Armbrust] no braces
34eb3a4 [Michael Armbrust] more work
6197cd5 [Michael Armbrust] [SQL] Better error messages for analysis failures
(cherry picked from commit aa4ca8b873fd83e64e5faea6f7febcc830e30b02)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e3a975d4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e3a975d4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e3a975d4
Branch: refs/heads/branch-1.3
Commit: e3a975d45a960ddbfe03051a6ae8b614e63cde6b
Parents: cbd659e
Author: Michael Armbrust
Authored: Thu Feb 12 13:11:28 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 13:11:39 2015 -0800
--
.../spark/sql/catalyst/analysis/Analyzer.scala | 123 +++
.../sql/catalyst/expressions/AttributeSet.scala | 6 +-
.../catalyst/expressions/namedExpressions.scala | 2 +-
.../plans/logical/ScriptTransformation.scala| 6 +-
.../spark/sql/catalyst/trees/TreeNode.scala | 9 ++
.../sql/catalyst/analysis/AnalysisSuite.scala | 79
.../optimizer/BooleanSimplificationSuite.scala | 4 +-
.../optimizer/ConstantFoldingSuite.scala| 4 +-
.../optimizer/FilterPushdownSuite.scala | 12 +-
.../catalyst/optimizer/OptimizeInSuite.scala| 4 +-
.../catalyst/optimizer/UnionPushdownSuite.scala | 4 +-
.../org/apache/spark/sql/SQLQuerySuite.scala| 6 +-
.../org/apache/spark/sql/hive/HiveContext.scala | 4 +-
.../spark/sql/hive/execution/commands.scala | 4 +-
14 files changed, 164 insertions(+), 103 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e3a975d4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 2d1fa10..58a7003 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.errors.TreeNodeException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{ArrayType, StructField, StructType,
IntegerType}
+import org.apache.spark.sql.types._
/**
* A trivial [[Analyzer]] with an [[EmptyCatalog]] and
[[EmptyFunctionRegistry]]. Used for testing
@@ -66,32 +66,82 @@ class Analyzer(catalog: Catalog,
typeCoercionRules ++
extendedRules : _*),
Batch("Check Analysis", Once,
- CheckResolution ::
- CheckAggregation ::
- Nil: _*),
-Batch("AnalysisOperators", fixedPoint,
- EliminateAnalysisOperators)
+ CheckResolution),
+Batch("Remove SubQueries", fixedPoint,
+ EliminateSubQueries)
)
/**
* Makes sure all attributes and logical plans have been resolved.
*/
object CheckResolution extends Rule[LogicalPlan] {
+def failAnalysis(msg: String) = { throw new AnalysisException(msg) }
+
def apply(plan: LogicalPlan): LogicalPlan = {
- plan.transformUp {
-case p if p.expressions.exists(!_.resolved) =>
- val missing =
p.expressions.filterNot(_.resolved).map(_.prettyString).mkString(",")
- val from = p.inputSet.map(_.name).mkString("{", ", ", "}")
-
- throw new AnalysisException(s"Cannot resolve '$missing' given input
columns $from")
-case p if !p.resolved && p.childrenResolved =>
- throw new AnalysisException(s"Unresolved operator in the query plan
${p.simpleString}")
- } match {
-// As a backstop, use the root node to check that the entire plan tree
is resolved.
-case p if !p.resolved =>
- throw new AnalysisException(s"Unresolved operator in the query plan
${p.simpleString}")
-case p => p
+ // We transform up and order the rul
spark git commit: [SPARK-5758][SQL] Use LongType as the default type for integers in JSON schema inference.
Repository: spark
Updated Branches:
refs/heads/master 0bf031582 -> c352ffbdb
[SPARK-5758][SQL] Use LongType as the default type for integers in JSON schema
inference.
Author: Yin Huai
Closes #4544 from yhuai/jsonUseLongTypeByDefault and squashes the following
commits:
6e2ffc2 [Yin Huai] Use LongType as the default type for integers in JSON schema
inference.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c352ffbd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c352ffbd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c352ffbd
Branch: refs/heads/master
Commit: c352ffbdb9112714c176a747edff6115e9369e58
Parents: 0bf0315
Author: Yin Huai
Authored: Thu Feb 12 15:17:25 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 15:17:25 2015 -0800
--
.../scala/org/apache/spark/sql/json/JsonRDD.scala | 12
.../spark/sql/api/java/JavaApplySchemaSuite.java| 2 +-
.../scala/org/apache/spark/sql/json/JsonSuite.scala | 16
3 files changed, 17 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c352ffbd/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index 1043eef..7dfb304 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -17,13 +17,12 @@
package org.apache.spark.sql.json
-import java.io.StringWriter
-import java.sql.{Date, Timestamp}
+import java.sql.Timestamp
import scala.collection.Map
import scala.collection.convert.Wrappers.{JMapWrapper, JListWrapper}
-import com.fasterxml.jackson.core.{JsonGenerator, JsonProcessingException,
JsonFactory}
+import com.fasterxml.jackson.core.{JsonGenerator, JsonProcessingException}
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.spark.rdd.RDD
@@ -178,7 +177,12 @@ private[sql] object JsonRDD extends Logging {
}
private def typeOfPrimitiveValue: PartialFunction[Any, DataType] = {
-ScalaReflection.typeOfObject orElse {
+// For Integer values, use LongType by default.
+val useLongType: PartialFunction[Any, DataType] = {
+ case value: IntegerType.JvmType => LongType
+}
+
+useLongType orElse ScalaReflection.typeOfObject orElse {
// Since we do not have a data type backed by BigInteger,
// when we see a Java BigInteger, we use DecimalType.
case value: java.math.BigInteger => DecimalType.Unlimited
http://git-wip-us.apache.org/repos/asf/spark/blob/c352ffbd/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
--
diff --git
a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
index 2e6e977..643b891 100644
---
a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
+++
b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
@@ -164,7 +164,7 @@ public class JavaApplySchemaSuite implements Serializable {
fields.add(DataTypes.createStructField("bigInteger",
DataTypes.createDecimalType(), true));
fields.add(DataTypes.createStructField("boolean", DataTypes.BooleanType,
true));
fields.add(DataTypes.createStructField("double", DataTypes.DoubleType,
true));
-fields.add(DataTypes.createStructField("integer", DataTypes.IntegerType,
true));
+fields.add(DataTypes.createStructField("integer", DataTypes.LongType,
true));
fields.add(DataTypes.createStructField("long", DataTypes.LongType, true));
fields.add(DataTypes.createStructField("null", DataTypes.StringType,
true));
fields.add(DataTypes.createStructField("string", DataTypes.StringType,
true));
http://git-wip-us.apache.org/repos/asf/spark/blob/c352ffbd/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
--
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
index fde4b47..b5f13f8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
@@ -222,7 +222,7 @@ class JsonSuite extends QueryTest {
StructField("bigInteger", DecimalType.Unlimited, true) ::
StructField("boolean", BooleanType, true) ::
StructField("double", DoubleType, true) ::
- StructFie
spark git commit: [SPARK-5758][SQL] Use LongType as the default type for integers in JSON schema inference.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 bf0d15c52 -> b0c79daf4
[SPARK-5758][SQL] Use LongType as the default type for integers in JSON schema
inference.
Author: Yin Huai
Closes #4544 from yhuai/jsonUseLongTypeByDefault and squashes the following
commits:
6e2ffc2 [Yin Huai] Use LongType as the default type for integers in JSON schema
inference.
(cherry picked from commit c352ffbdb9112714c176a747edff6115e9369e58)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b0c79daf
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b0c79daf
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b0c79daf
Branch: refs/heads/branch-1.3
Commit: b0c79daf4a24739963726dfecedff9a4b129f3c0
Parents: bf0d15c
Author: Yin Huai
Authored: Thu Feb 12 15:17:25 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 15:17:35 2015 -0800
--
.../scala/org/apache/spark/sql/json/JsonRDD.scala | 12
.../spark/sql/api/java/JavaApplySchemaSuite.java| 2 +-
.../scala/org/apache/spark/sql/json/JsonSuite.scala | 16
3 files changed, 17 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b0c79daf/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index 1043eef..7dfb304 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -17,13 +17,12 @@
package org.apache.spark.sql.json
-import java.io.StringWriter
-import java.sql.{Date, Timestamp}
+import java.sql.Timestamp
import scala.collection.Map
import scala.collection.convert.Wrappers.{JMapWrapper, JListWrapper}
-import com.fasterxml.jackson.core.{JsonGenerator, JsonProcessingException,
JsonFactory}
+import com.fasterxml.jackson.core.{JsonGenerator, JsonProcessingException}
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.spark.rdd.RDD
@@ -178,7 +177,12 @@ private[sql] object JsonRDD extends Logging {
}
private def typeOfPrimitiveValue: PartialFunction[Any, DataType] = {
-ScalaReflection.typeOfObject orElse {
+// For Integer values, use LongType by default.
+val useLongType: PartialFunction[Any, DataType] = {
+ case value: IntegerType.JvmType => LongType
+}
+
+useLongType orElse ScalaReflection.typeOfObject orElse {
// Since we do not have a data type backed by BigInteger,
// when we see a Java BigInteger, we use DecimalType.
case value: java.math.BigInteger => DecimalType.Unlimited
http://git-wip-us.apache.org/repos/asf/spark/blob/b0c79daf/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
--
diff --git
a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
index 2e6e977..643b891 100644
---
a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
+++
b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
@@ -164,7 +164,7 @@ public class JavaApplySchemaSuite implements Serializable {
fields.add(DataTypes.createStructField("bigInteger",
DataTypes.createDecimalType(), true));
fields.add(DataTypes.createStructField("boolean", DataTypes.BooleanType,
true));
fields.add(DataTypes.createStructField("double", DataTypes.DoubleType,
true));
-fields.add(DataTypes.createStructField("integer", DataTypes.IntegerType,
true));
+fields.add(DataTypes.createStructField("integer", DataTypes.LongType,
true));
fields.add(DataTypes.createStructField("long", DataTypes.LongType, true));
fields.add(DataTypes.createStructField("null", DataTypes.StringType,
true));
fields.add(DataTypes.createStructField("string", DataTypes.StringType,
true));
http://git-wip-us.apache.org/repos/asf/spark/blob/b0c79daf/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
--
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
index fde4b47..b5f13f8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
@@ -222,7 +222,7 @@ class JsonSuite extends QueryTest {
StructField("bigInteger", DecimalType.Unlimited, true) ::
spark git commit: [SPARK-5573][SQL] Add explode to dataframes
Repository: spark
Updated Branches:
refs/heads/branch-1.3 b0c79daf4 -> c7eb9ee2c
[SPARK-5573][SQL] Add explode to dataframes
Author: Michael Armbrust
Closes #4546 from marmbrus/explode and squashes the following commits:
eefd33a [Michael Armbrust] whitespace
a8d496c [Michael Armbrust] Merge remote-tracking branch 'apache/master' into
explode
4af740e [Michael Armbrust] Merge remote-tracking branch 'origin/master' into
explode
dc86a5c [Michael Armbrust] simple version
d633d01 [Michael Armbrust] add scala specific
950707a [Michael Armbrust] fix comments
ba8854c [Michael Armbrust] [SPARK-5573][SQL] Add explode to dataframes
(cherry picked from commit ee04a8b19be8330bfc48f470ef365622162c915f)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c7eb9ee2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c7eb9ee2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c7eb9ee2
Branch: refs/heads/branch-1.3
Commit: c7eb9ee2ccd93211c9ec125fd2baae267b35d3d4
Parents: b0c79da
Author: Michael Armbrust
Authored: Thu Feb 12 15:19:19 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 15:19:32 2015 -0800
--
.../sql/catalyst/expressions/generators.scala | 19 ++
.../scala/org/apache/spark/sql/DataFrame.scala | 38
.../org/apache/spark/sql/DataFrameImpl.scala| 30 ++--
.../apache/spark/sql/IncomputableColumn.scala | 9 +
.../org/apache/spark/sql/DataFrameSuite.scala | 25 +
5 files changed, 119 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c7eb9ee2/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 43b6482..0983d27 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -74,6 +74,25 @@ abstract class Generator extends Expression {
}
/**
+ * A generator that produces its output using the provided lambda function.
+ */
+case class UserDefinedGenerator(
+schema: Seq[Attribute],
+function: Row => TraversableOnce[Row],
+children: Seq[Expression])
+ extends Generator{
+
+ override protected def makeOutput(): Seq[Attribute] = schema
+
+ override def eval(input: Row): TraversableOnce[Row] = {
+val inputRow = new InterpretedProjection(children)
+function(inputRow(input))
+ }
+
+ override def toString = s"UserDefinedGenerator(${children.mkString(",")})"
+}
+
+/**
* Given an input array produces a sequence of rows for each value in the
array.
*/
case class Explode(attributeNames: Seq[String], child: Expression)
http://git-wip-us.apache.org/repos/asf/spark/blob/c7eb9ee2/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 13aff76..6525788 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql
import scala.collection.JavaConversions._
import scala.reflect.ClassTag
+import scala.reflect.runtime.universe.TypeTag
import scala.util.control.NonFatal
import org.apache.spark.annotation.{DeveloperApi, Experimental}
@@ -441,6 +442,43 @@ trait DataFrame extends RDDApi[Row] with Serializable {
sample(withReplacement, fraction, Utils.random.nextLong)
}
+ /**
+ * (Scala-specific) Returns a new [[DataFrame]] where each row has been
expanded to zero or more
+ * rows by the provided function. This is similar to a `LATERAL VIEW` in
HiveQL. The columns of
+ * the input row are implicitly joined with each row that is output by the
function.
+ *
+ * The following example uses this function to count the number of books
which contain
+ * a given word:
+ *
+ * {{{
+ * case class Book(title: String, words: String)
+ * val df: RDD[Book]
+ *
+ * case class Word(word: String)
+ * val allWords = df.explode('words) {
+ * case Row(words: String) => words.split(" ").map(Word(_))
+ * }
+ *
+ * val bookCountPerWord =
allWords.groupBy("word").agg(countDistinct("title"))
+ * }}}
+ */
+ def explo
spark git commit: [SPARK-5573][SQL] Add explode to dataframes
Repository: spark
Updated Branches:
refs/heads/master c352ffbdb -> ee04a8b19
[SPARK-5573][SQL] Add explode to dataframes
Author: Michael Armbrust
Closes #4546 from marmbrus/explode and squashes the following commits:
eefd33a [Michael Armbrust] whitespace
a8d496c [Michael Armbrust] Merge remote-tracking branch 'apache/master' into
explode
4af740e [Michael Armbrust] Merge remote-tracking branch 'origin/master' into
explode
dc86a5c [Michael Armbrust] simple version
d633d01 [Michael Armbrust] add scala specific
950707a [Michael Armbrust] fix comments
ba8854c [Michael Armbrust] [SPARK-5573][SQL] Add explode to dataframes
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ee04a8b1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ee04a8b1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ee04a8b1
Branch: refs/heads/master
Commit: ee04a8b19be8330bfc48f470ef365622162c915f
Parents: c352ffb
Author: Michael Armbrust
Authored: Thu Feb 12 15:19:19 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 15:19:19 2015 -0800
--
.../sql/catalyst/expressions/generators.scala | 19 ++
.../scala/org/apache/spark/sql/DataFrame.scala | 38
.../org/apache/spark/sql/DataFrameImpl.scala| 30 ++--
.../apache/spark/sql/IncomputableColumn.scala | 9 +
.../org/apache/spark/sql/DataFrameSuite.scala | 25 +
5 files changed, 119 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ee04a8b1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 43b6482..0983d27 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -74,6 +74,25 @@ abstract class Generator extends Expression {
}
/**
+ * A generator that produces its output using the provided lambda function.
+ */
+case class UserDefinedGenerator(
+schema: Seq[Attribute],
+function: Row => TraversableOnce[Row],
+children: Seq[Expression])
+ extends Generator{
+
+ override protected def makeOutput(): Seq[Attribute] = schema
+
+ override def eval(input: Row): TraversableOnce[Row] = {
+val inputRow = new InterpretedProjection(children)
+function(inputRow(input))
+ }
+
+ override def toString = s"UserDefinedGenerator(${children.mkString(",")})"
+}
+
+/**
* Given an input array produces a sequence of rows for each value in the
array.
*/
case class Explode(attributeNames: Seq[String], child: Expression)
http://git-wip-us.apache.org/repos/asf/spark/blob/ee04a8b1/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 13aff76..6525788 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql
import scala.collection.JavaConversions._
import scala.reflect.ClassTag
+import scala.reflect.runtime.universe.TypeTag
import scala.util.control.NonFatal
import org.apache.spark.annotation.{DeveloperApi, Experimental}
@@ -441,6 +442,43 @@ trait DataFrame extends RDDApi[Row] with Serializable {
sample(withReplacement, fraction, Utils.random.nextLong)
}
+ /**
+ * (Scala-specific) Returns a new [[DataFrame]] where each row has been
expanded to zero or more
+ * rows by the provided function. This is similar to a `LATERAL VIEW` in
HiveQL. The columns of
+ * the input row are implicitly joined with each row that is output by the
function.
+ *
+ * The following example uses this function to count the number of books
which contain
+ * a given word:
+ *
+ * {{{
+ * case class Book(title: String, words: String)
+ * val df: RDD[Book]
+ *
+ * case class Word(word: String)
+ * val allWords = df.explode('words) {
+ * case Row(words: String) => words.split(" ").map(Word(_))
+ * }
+ *
+ * val bookCountPerWord =
allWords.groupBy("word").agg(countDistinct("title"))
+ * }}}
+ */
+ def explode[A <: Product : TypeTag](input: Column*)(f: Row =>
TraversableOnce[A]): DataFrame
+
+
+ /**
+ * (Scala-specific) Returns
spark git commit: [SPARK-5755] [SQL] remove unnecessary Add
Repository: spark
Updated Branches:
refs/heads/master ee04a8b19 -> d5fc51491
[SPARK-5755] [SQL] remove unnecessary Add
explain extended select +key from src;
before:
== Parsed Logical Plan ==
'Project [(0 + 'key) AS _c0#8]
'UnresolvedRelation [src], None
== Analyzed Logical Plan ==
Project [(0 + key#10) AS _c0#8]
MetastoreRelation test, src, None
== Optimized Logical Plan ==
Project [(0 + key#10) AS _c0#8]
MetastoreRelation test, src, None
== Physical Plan ==
Project [(0 + key#10) AS _c0#8]
HiveTableScan [key#10], (MetastoreRelation test, src, None), None
after this patch:
== Parsed Logical Plan ==
'Project ['key]
'UnresolvedRelation [src], None
== Analyzed Logical Plan ==
Project [key#10]
MetastoreRelation test, src, None
== Optimized Logical Plan ==
Project [key#10]
MetastoreRelation test, src, None
== Physical Plan ==
HiveTableScan [key#10], (MetastoreRelation test, src, None), None
Author: Daoyuan Wang
Closes #4551 from adrian-wang/positive and squashes the following commits:
0821ae4 [Daoyuan Wang] remove unnecessary Add
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d5fc5149
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d5fc5149
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d5fc5149
Branch: refs/heads/master
Commit: d5fc51491808630d0328a5937dbf349e00de361f
Parents: ee04a8b
Author: Daoyuan Wang
Authored: Thu Feb 12 15:22:07 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 15:22:07 2015 -0800
--
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d5fc5149/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
--
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index f3c9e63..5269460 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -1099,7 +1099,7 @@
https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
Cast(nodeToExpr(arg), DateType)
/* Arithmetic */
-case Token("+", child :: Nil) => Add(Literal(0), nodeToExpr(child))
+case Token("+", child :: Nil) => nodeToExpr(child)
case Token("-", child :: Nil) => UnaryMinus(nodeToExpr(child))
case Token("~", child :: Nil) => BitwiseNot(nodeToExpr(child))
case Token("+", left :: right:: Nil) => Add(nodeToExpr(left),
nodeToExpr(right))
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5755] [SQL] remove unnecessary Add
Repository: spark
Updated Branches:
refs/heads/branch-1.3 c7eb9ee2c -> f7103b343
[SPARK-5755] [SQL] remove unnecessary Add
explain extended select +key from src;
before:
== Parsed Logical Plan ==
'Project [(0 + 'key) AS _c0#8]
'UnresolvedRelation [src], None
== Analyzed Logical Plan ==
Project [(0 + key#10) AS _c0#8]
MetastoreRelation test, src, None
== Optimized Logical Plan ==
Project [(0 + key#10) AS _c0#8]
MetastoreRelation test, src, None
== Physical Plan ==
Project [(0 + key#10) AS _c0#8]
HiveTableScan [key#10], (MetastoreRelation test, src, None), None
after this patch:
== Parsed Logical Plan ==
'Project ['key]
'UnresolvedRelation [src], None
== Analyzed Logical Plan ==
Project [key#10]
MetastoreRelation test, src, None
== Optimized Logical Plan ==
Project [key#10]
MetastoreRelation test, src, None
== Physical Plan ==
HiveTableScan [key#10], (MetastoreRelation test, src, None), None
Author: Daoyuan Wang
Closes #4551 from adrian-wang/positive and squashes the following commits:
0821ae4 [Daoyuan Wang] remove unnecessary Add
(cherry picked from commit d5fc51491808630d0328a5937dbf349e00de361f)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f7103b34
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f7103b34
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f7103b34
Branch: refs/heads/branch-1.3
Commit: f7103b3437363bd81e4f4cfa282229019fcdcdad
Parents: c7eb9ee
Author: Daoyuan Wang
Authored: Thu Feb 12 15:22:07 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 15:22:16 2015 -0800
--
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f7103b34/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
--
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index f3c9e63..5269460 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -1099,7 +1099,7 @@
https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
Cast(nodeToExpr(arg), DateType)
/* Arithmetic */
-case Token("+", child :: Nil) => Add(Literal(0), nodeToExpr(child))
+case Token("+", child :: Nil) => nodeToExpr(child)
case Token("-", child :: Nil) => UnaryMinus(nodeToExpr(child))
case Token("~", child :: Nil) => BitwiseNot(nodeToExpr(child))
case Token("+", left :: right:: Nil) => Add(nodeToExpr(left),
nodeToExpr(right))
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SQL] Move SaveMode to SQL package.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 5c9db4e75 -> 925fd84a1
[SQL] Move SaveMode to SQL package.
Author: Yin Huai
Closes #4542 from yhuai/moveSaveMode and squashes the following commits:
65a4425 [Yin Huai] Move SaveMode to sql package.
(cherry picked from commit c025a468826e9b9f62032e207daa9d42d9dba3ca)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/925fd84a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/925fd84a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/925fd84a
Branch: refs/heads/branch-1.3
Commit: 925fd84a1d25fd453af8ad457569dddb54938388
Parents: 5c9db4e
Author: Yin Huai
Authored: Thu Feb 12 15:32:17 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 15:32:25 2015 -0800
--
python/pyspark/sql/dataframe.py | 2 +-
.../java/org/apache/spark/sql/SaveMode.java | 45
.../org/apache/spark/sql/sources/SaveMode.java | 45
.../scala/org/apache/spark/sql/DataFrame.scala | 1 -
.../apache/spark/sql/IncomputableColumn.scala | 1 -
.../apache/spark/sql/json/JSONRelation.scala| 2 +-
.../apache/spark/sql/parquet/newParquet.scala | 3 +-
.../org/apache/spark/sql/sources/ddl.scala | 2 +-
.../apache/spark/sql/sources/interfaces.scala | 2 +-
.../spark/sql/sources/SaveLoadSuite.scala | 2 +-
.../spark/sql/hive/execution/commands.scala | 2 +-
.../sql/hive/JavaMetastoreDataSourcesSuite.java | 2 +-
.../sql/hive/MetastoreDataSourcesSuite.scala| 2 -
13 files changed, 53 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/925fd84a/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 3eb56ed..b6f052e 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -149,7 +149,7 @@ class DataFrame(object):
def _java_save_mode(self, mode):
"""Returns the Java save mode based on the Python save mode
represented by a string.
"""
-jSaveMode = self._sc._jvm.org.apache.spark.sql.sources.SaveMode
+jSaveMode = self._sc._jvm.org.apache.spark.sql.SaveMode
jmode = jSaveMode.ErrorIfExists
mode = mode.lower()
if mode == "append":
http://git-wip-us.apache.org/repos/asf/spark/blob/925fd84a/sql/core/src/main/java/org/apache/spark/sql/SaveMode.java
--
diff --git a/sql/core/src/main/java/org/apache/spark/sql/SaveMode.java
b/sql/core/src/main/java/org/apache/spark/sql/SaveMode.java
new file mode 100644
index 000..a40be52
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/SaveMode.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql;
+
+/**
+ * SaveMode is used to specify the expected behavior of saving a DataFrame to
a data source.
+ */
+public enum SaveMode {
+ /**
+ * Append mode means that when saving a DataFrame to a data source, if
data/table already exists,
+ * contents of the DataFrame are expected to be appended to existing data.
+ */
+ Append,
+ /**
+ * Overwrite mode means that when saving a DataFrame to a data source,
+ * if data/table already exists, existing data is expected to be overwritten
by the contents of
+ * the DataFrame.
+ */
+ Overwrite,
+ /**
+ * ErrorIfExists mode means that when saving a DataFrame to a data source,
if data already exists,
+ * an exception is expected to be thrown.
+ */
+ ErrorIfExists,
+ /**
+ * Ignore mode means that when saving a DataFrame to a data source, if data
already exists,
+ * the save operation is expected to not save the contents of the DataFrame
and to not
+ * change the existing data.
+ */
+ Ignore
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/925fd84a/sql/core/src/main/java/org/apache/spark/sql/
spark git commit: [SQL] Move SaveMode to SQL package.
Repository: spark
Updated Branches:
refs/heads/master ada993e95 -> c025a4688
[SQL] Move SaveMode to SQL package.
Author: Yin Huai
Closes #4542 from yhuai/moveSaveMode and squashes the following commits:
65a4425 [Yin Huai] Move SaveMode to sql package.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c025a468
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c025a468
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c025a468
Branch: refs/heads/master
Commit: c025a468826e9b9f62032e207daa9d42d9dba3ca
Parents: ada993e
Author: Yin Huai
Authored: Thu Feb 12 15:32:17 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 15:32:17 2015 -0800
--
python/pyspark/sql/dataframe.py | 2 +-
.../java/org/apache/spark/sql/SaveMode.java | 45
.../org/apache/spark/sql/sources/SaveMode.java | 45
.../scala/org/apache/spark/sql/DataFrame.scala | 1 -
.../apache/spark/sql/IncomputableColumn.scala | 1 -
.../apache/spark/sql/json/JSONRelation.scala| 2 +-
.../apache/spark/sql/parquet/newParquet.scala | 3 +-
.../org/apache/spark/sql/sources/ddl.scala | 2 +-
.../apache/spark/sql/sources/interfaces.scala | 2 +-
.../spark/sql/sources/SaveLoadSuite.scala | 2 +-
.../spark/sql/hive/execution/commands.scala | 2 +-
.../sql/hive/JavaMetastoreDataSourcesSuite.java | 2 +-
.../sql/hive/MetastoreDataSourcesSuite.scala| 2 -
13 files changed, 53 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c025a468/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 3eb56ed..b6f052e 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -149,7 +149,7 @@ class DataFrame(object):
def _java_save_mode(self, mode):
"""Returns the Java save mode based on the Python save mode
represented by a string.
"""
-jSaveMode = self._sc._jvm.org.apache.spark.sql.sources.SaveMode
+jSaveMode = self._sc._jvm.org.apache.spark.sql.SaveMode
jmode = jSaveMode.ErrorIfExists
mode = mode.lower()
if mode == "append":
http://git-wip-us.apache.org/repos/asf/spark/blob/c025a468/sql/core/src/main/java/org/apache/spark/sql/SaveMode.java
--
diff --git a/sql/core/src/main/java/org/apache/spark/sql/SaveMode.java
b/sql/core/src/main/java/org/apache/spark/sql/SaveMode.java
new file mode 100644
index 000..a40be52
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/SaveMode.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql;
+
+/**
+ * SaveMode is used to specify the expected behavior of saving a DataFrame to
a data source.
+ */
+public enum SaveMode {
+ /**
+ * Append mode means that when saving a DataFrame to a data source, if
data/table already exists,
+ * contents of the DataFrame are expected to be appended to existing data.
+ */
+ Append,
+ /**
+ * Overwrite mode means that when saving a DataFrame to a data source,
+ * if data/table already exists, existing data is expected to be overwritten
by the contents of
+ * the DataFrame.
+ */
+ Overwrite,
+ /**
+ * ErrorIfExists mode means that when saving a DataFrame to a data source,
if data already exists,
+ * an exception is expected to be thrown.
+ */
+ ErrorIfExists,
+ /**
+ * Ignore mode means that when saving a DataFrame to a data source, if data
already exists,
+ * the save operation is expected to not save the contents of the DataFrame
and to not
+ * change the existing data.
+ */
+ Ignore
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c025a468/sql/core/src/main/java/org/apache/spark/sql/sources/SaveMode.java
--
diff --git a/sql/c
spark git commit: [SPARK-3299][SQL]Public API in SQLContext to list tables
Repository: spark
Updated Branches:
refs/heads/master c025a4688 -> 1d0596a16
[SPARK-3299][SQL]Public API in SQLContext to list tables
https://issues.apache.org/jira/browse/SPARK-3299
Author: Yin Huai
Closes #4547 from yhuai/tables and squashes the following commits:
6c8f92e [Yin Huai] Add tableNames.
acbb281 [Yin Huai] Update Python test.
7793dcb [Yin Huai] Fix scala test.
572870d [Yin Huai] Address comments.
aba2e88 [Yin Huai] Format.
12c86df [Yin Huai] Add tables() to SQLContext to return a DataFrame containing
existing tables.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1d0596a1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1d0596a1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1d0596a1
Branch: refs/heads/master
Commit: 1d0596a16e1d3add2631f5d8169aeec2876a1362
Parents: c025a46
Author: Yin Huai
Authored: Thu Feb 12 18:08:01 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 18:08:01 2015 -0800
--
python/pyspark/sql/context.py | 34 +
.../spark/sql/catalyst/analysis/Catalog.scala | 37 ++
.../scala/org/apache/spark/sql/SQLContext.scala | 36 +
.../org/apache/spark/sql/ListTablesSuite.scala | 76 +++
.../spark/sql/hive/HiveMetastoreCatalog.scala | 5 ++
.../apache/spark/sql/hive/ListTablesSuite.scala | 77
6 files changed, 265 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1d0596a1/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index db4bcbe..082f1b6 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -621,6 +621,40 @@ class SQLContext(object):
"""
return DataFrame(self._ssql_ctx.table(tableName), self)
+def tables(self, dbName=None):
+"""Returns a DataFrame containing names of tables in the given
database.
+
+If `dbName` is not specified, the current database will be used.
+
+The returned DataFrame has two columns, tableName and isTemporary
+(a column with BooleanType indicating if a table is a temporary one or
not).
+
+>>> sqlCtx.registerRDDAsTable(df, "table1")
+>>> df2 = sqlCtx.tables()
+>>> df2.filter("tableName = 'table1'").first()
+Row(tableName=u'table1', isTemporary=True)
+"""
+if dbName is None:
+return DataFrame(self._ssql_ctx.tables(), self)
+else:
+return DataFrame(self._ssql_ctx.tables(dbName), self)
+
+def tableNames(self, dbName=None):
+"""Returns a list of names of tables in the database `dbName`.
+
+If `dbName` is not specified, the current database will be used.
+
+>>> sqlCtx.registerRDDAsTable(df, "table1")
+>>> "table1" in sqlCtx.tableNames()
+True
+>>> "table1" in sqlCtx.tableNames("db")
+True
+"""
+if dbName is None:
+return [name for name in self._ssql_ctx.tableNames()]
+else:
+return [name for name in self._ssql_ctx.tableNames(dbName)]
+
def cacheTable(self, tableName):
"""Caches the specified table in-memory."""
self._ssql_ctx.cacheTable(tableName)
http://git-wip-us.apache.org/repos/asf/spark/blob/1d0596a1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
index df8d03b..f57eab2 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
@@ -34,6 +34,12 @@ trait Catalog {
tableIdentifier: Seq[String],
alias: Option[String] = None): LogicalPlan
+ /**
+ * Returns tuples of (tableName, isTemporary) for all tables in the given
database.
+ * isTemporary is a Boolean value indicates if a table is a temporary or not.
+ */
+ def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
+
def registerTable(tableIdentifier: Seq[String], plan: LogicalPlan): Unit
def unregisterTable(tableIdentifier: Seq[String]): Unit
@@ -101,6 +107,12 @@ class SimpleCatalog(val caseSensitive: Boolean) extends
Catalog {
// properly qualified with this alias.
alias.map(a => Subquery(a,
tableWithQualifiers)).getOrElse(tableWithQualifiers)
}
+
+ override def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
= {
+tables.map {
+ case (name, _) => (name, true)
+
spark git commit: [SPARK-3299][SQL]Public API in SQLContext to list tables
Repository: spark
Updated Branches:
refs/heads/branch-1.3 925fd84a1 -> edbac178d
[SPARK-3299][SQL]Public API in SQLContext to list tables
https://issues.apache.org/jira/browse/SPARK-3299
Author: Yin Huai
Closes #4547 from yhuai/tables and squashes the following commits:
6c8f92e [Yin Huai] Add tableNames.
acbb281 [Yin Huai] Update Python test.
7793dcb [Yin Huai] Fix scala test.
572870d [Yin Huai] Address comments.
aba2e88 [Yin Huai] Format.
12c86df [Yin Huai] Add tables() to SQLContext to return a DataFrame containing
existing tables.
(cherry picked from commit 1d0596a16e1d3add2631f5d8169aeec2876a1362)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/edbac178
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/edbac178
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/edbac178
Branch: refs/heads/branch-1.3
Commit: edbac178d186f6936408b211385a5fea9e4f4603
Parents: 925fd84
Author: Yin Huai
Authored: Thu Feb 12 18:08:01 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 18:08:19 2015 -0800
--
python/pyspark/sql/context.py | 34 +
.../spark/sql/catalyst/analysis/Catalog.scala | 37 ++
.../scala/org/apache/spark/sql/SQLContext.scala | 36 +
.../org/apache/spark/sql/ListTablesSuite.scala | 76 +++
.../spark/sql/hive/HiveMetastoreCatalog.scala | 5 ++
.../apache/spark/sql/hive/ListTablesSuite.scala | 77
6 files changed, 265 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/edbac178/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index db4bcbe..082f1b6 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -621,6 +621,40 @@ class SQLContext(object):
"""
return DataFrame(self._ssql_ctx.table(tableName), self)
+def tables(self, dbName=None):
+"""Returns a DataFrame containing names of tables in the given
database.
+
+If `dbName` is not specified, the current database will be used.
+
+The returned DataFrame has two columns, tableName and isTemporary
+(a column with BooleanType indicating if a table is a temporary one or
not).
+
+>>> sqlCtx.registerRDDAsTable(df, "table1")
+>>> df2 = sqlCtx.tables()
+>>> df2.filter("tableName = 'table1'").first()
+Row(tableName=u'table1', isTemporary=True)
+"""
+if dbName is None:
+return DataFrame(self._ssql_ctx.tables(), self)
+else:
+return DataFrame(self._ssql_ctx.tables(dbName), self)
+
+def tableNames(self, dbName=None):
+"""Returns a list of names of tables in the database `dbName`.
+
+If `dbName` is not specified, the current database will be used.
+
+>>> sqlCtx.registerRDDAsTable(df, "table1")
+>>> "table1" in sqlCtx.tableNames()
+True
+>>> "table1" in sqlCtx.tableNames("db")
+True
+"""
+if dbName is None:
+return [name for name in self._ssql_ctx.tableNames()]
+else:
+return [name for name in self._ssql_ctx.tableNames(dbName)]
+
def cacheTable(self, tableName):
"""Caches the specified table in-memory."""
self._ssql_ctx.cacheTable(tableName)
http://git-wip-us.apache.org/repos/asf/spark/blob/edbac178/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
index df8d03b..f57eab2 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
@@ -34,6 +34,12 @@ trait Catalog {
tableIdentifier: Seq[String],
alias: Option[String] = None): LogicalPlan
+ /**
+ * Returns tuples of (tableName, isTemporary) for all tables in the given
database.
+ * isTemporary is a Boolean value indicates if a table is a temporary or not.
+ */
+ def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
+
def registerTable(tableIdentifier: Seq[String], plan: LogicalPlan): Unit
def unregisterTable(tableIdentifier: Seq[String]): Unit
@@ -101,6 +107,12 @@ class SimpleCatalog(val caseSensitive: Boolean) extends
Catalog {
// properly qualified with this alias.
alias.map(a => Subquery(a,
tableWithQualifiers)).getOrElse(tableWithQualifiers)
}
+
+ override def getTables(databa
spark git commit: [SQL] Fix docs of SQLContext.tables
Repository: spark
Updated Branches:
refs/heads/master 1d0596a16 -> 2aea892eb
[SQL] Fix docs of SQLContext.tables
Author: Yin Huai
Closes #4579 from yhuai/tablesDoc and squashes the following commits:
7f8964c [Yin Huai] Fix doc.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2aea892e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2aea892e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2aea892e
Branch: refs/heads/master
Commit: 2aea892ebd4d6c802defeef35ef7ebfe42c06eba
Parents: 1d0596a
Author: Yin Huai
Authored: Thu Feb 12 20:37:55 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 12 20:37:55 2015 -0800
--
.../main/scala/org/apache/spark/sql/SQLContext.scala| 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2aea892e/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 0f8af75..2165949 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -775,8 +775,8 @@ class SQLContext(@transient val sparkContext: SparkContext)
DataFrame(this, catalog.lookupRelation(Seq(tableName)))
/**
- * Returns a [[DataFrame]] containing names of existing tables in the given
database.
- * The returned DataFrame has two columns, tableName and isTemporary (a
column with BooleanType
+ * Returns a [[DataFrame]] containing names of existing tables in the
current database.
+ * The returned DataFrame has two columns, tableName and isTemporary (a
Boolean
* indicating if a table is a temporary one or not).
*/
def tables(): DataFrame = {
@@ -784,8 +784,8 @@ class SQLContext(@transient val sparkContext: SparkContext)
}
/**
- * Returns a [[DataFrame]] containing names of existing tables in the
current database.
- * The returned DataFrame has two columns, tableName and isTemporary (a
column with BooleanType
+ * Returns a [[DataFrame]] containing names of existing tables in the given
database.
+ * The returned DataFrame has two columns, tableName and isTemporary (a
Boolean
* indicating if a table is a temporary one or not).
*/
def tables(databaseName: String): DataFrame = {
@@ -793,7 +793,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
}
/**
- * Returns an array of names of tables in the current database.
+ * Returns the names of tables in the current database as an array.
*/
def tableNames(): Array[String] = {
catalog.getTables(None).map {
@@ -802,7 +802,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
}
/**
- * Returns an array of names of tables in the given database.
+ * Returns the names of tables in the given database as an array.
*/
def tableNames(databaseName: String): Array[String] = {
catalog.getTables(Some(databaseName)).map {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5642] [SQL] Apply column pruning on unused aggregation fields
Repository: spark
Updated Branches:
refs/heads/master 5d3cc6b3d -> 2cbb3e433
[SPARK-5642] [SQL] Apply column pruning on unused aggregation fields
select k from (select key k, max(value) v from src group by k) t
Author: Daoyuan Wang
Author: Michael Armbrust
Closes #4415 from adrian-wang/groupprune and squashes the following commits:
5d2d8a3 [Daoyuan Wang] address Michael's comments
61f8ef7 [Daoyuan Wang] add a unit test
80ddcc6 [Daoyuan Wang] keep project
b69d385 [Daoyuan Wang] add a prune rule for grouping set
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2cbb3e43
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2cbb3e43
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2cbb3e43
Branch: refs/heads/master
Commit: 2cbb3e433ae334d5c318f05b987af314c854fbcc
Parents: 5d3cc6b
Author: Daoyuan Wang
Authored: Fri Feb 13 13:46:50 2015 -0800
Committer: Michael Armbrust
Committed: Fri Feb 13 13:48:39 2015 -0800
--
.../sql/catalyst/optimizer/Optimizer.scala | 9 +
.../optimizer/FilterPushdownSuite.scala | 37 ++--
2 files changed, 44 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2cbb3e43/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 0da081e..1a75fcf 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -119,6 +119,15 @@ object ColumnPruning extends Rule[LogicalPlan] {
case a @ Aggregate(_, _, child) if (child.outputSet --
a.references).nonEmpty =>
a.copy(child = Project(a.references.toSeq, child))
+case p @ Project(projectList, a @ Aggregate(groupingExpressions,
aggregateExpressions, child))
+if (a.outputSet -- p.references).nonEmpty =>
+ Project(
+projectList,
+Aggregate(
+ groupingExpressions,
+ aggregateExpressions.filter(e => p.references.contains(e)),
+ child))
+
// Eliminate unneeded attributes from either side of a Join.
case Project(projectList, Join(left, right, joinType, condition)) =>
// Collect the list of all references required either above or to
evaluate the condition.
http://git-wip-us.apache.org/repos/asf/spark/blob/2cbb3e43/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index 0b74bac..55c6766 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.catalyst.analysis.EliminateSubQueries
-import org.apache.spark.sql.catalyst.expressions.Explode
+import org.apache.spark.sql.catalyst.expressions.{Count, Explode}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.plans.{PlanTest, LeftOuter, RightOuter}
import org.apache.spark.sql.catalyst.rules._
@@ -37,7 +37,8 @@ class FilterPushdownSuite extends PlanTest {
CombineFilters,
PushPredicateThroughProject,
PushPredicateThroughJoin,
-PushPredicateThroughGenerate) :: Nil
+PushPredicateThroughGenerate,
+ColumnPruning) :: Nil
}
val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
@@ -58,6 +59,38 @@ class FilterPushdownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+ test("column pruning for group") {
+val originalQuery =
+ testRelation
+.groupBy('a)('a, Count('b))
+.select('a)
+
+val optimized = Optimize(originalQuery.analyze)
+val correctAnswer =
+ testRelation
+.select('a)
+.groupBy('a)('a)
+.select('a).analyze
+
+comparePlans(optimized, correctAnswer)
+ }
+
+ test("column pruning for group with alias") {
+val originalQuery =
+ testRelation
+.groupBy('a)('a as 'c, Count('b))
+.select('c)
+
+val optimized = Optimize(originalQuery.analyze)
+val correctAnswer =
+ testRelation
+.select
spark git commit: [SPARK-5642] [SQL] Apply column pruning on unused aggregation fields
Repository: spark
Updated Branches:
refs/heads/branch-1.3 41603717a -> efffc2e42
[SPARK-5642] [SQL] Apply column pruning on unused aggregation fields
select k from (select key k, max(value) v from src group by k) t
Author: Daoyuan Wang
Author: Michael Armbrust
Closes #4415 from adrian-wang/groupprune and squashes the following commits:
5d2d8a3 [Daoyuan Wang] address Michael's comments
61f8ef7 [Daoyuan Wang] add a unit test
80ddcc6 [Daoyuan Wang] keep project
b69d385 [Daoyuan Wang] add a prune rule for grouping set
(cherry picked from commit 2cbb3e433ae334d5c318f05b987af314c854fbcc)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/efffc2e4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/efffc2e4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/efffc2e4
Branch: refs/heads/branch-1.3
Commit: efffc2e428b1e867a586749685da90875f6bcfc4
Parents: 4160371
Author: Daoyuan Wang
Authored: Fri Feb 13 13:46:50 2015 -0800
Committer: Michael Armbrust
Committed: Fri Feb 13 13:49:21 2015 -0800
--
.../sql/catalyst/optimizer/Optimizer.scala | 9 +
.../optimizer/FilterPushdownSuite.scala | 37 ++--
2 files changed, 44 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/efffc2e4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 0da081e..1a75fcf 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -119,6 +119,15 @@ object ColumnPruning extends Rule[LogicalPlan] {
case a @ Aggregate(_, _, child) if (child.outputSet --
a.references).nonEmpty =>
a.copy(child = Project(a.references.toSeq, child))
+case p @ Project(projectList, a @ Aggregate(groupingExpressions,
aggregateExpressions, child))
+if (a.outputSet -- p.references).nonEmpty =>
+ Project(
+projectList,
+Aggregate(
+ groupingExpressions,
+ aggregateExpressions.filter(e => p.references.contains(e)),
+ child))
+
// Eliminate unneeded attributes from either side of a Join.
case Project(projectList, Join(left, right, joinType, condition)) =>
// Collect the list of all references required either above or to
evaluate the condition.
http://git-wip-us.apache.org/repos/asf/spark/blob/efffc2e4/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index 0b74bac..55c6766 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.catalyst.analysis.EliminateSubQueries
-import org.apache.spark.sql.catalyst.expressions.Explode
+import org.apache.spark.sql.catalyst.expressions.{Count, Explode}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.plans.{PlanTest, LeftOuter, RightOuter}
import org.apache.spark.sql.catalyst.rules._
@@ -37,7 +37,8 @@ class FilterPushdownSuite extends PlanTest {
CombineFilters,
PushPredicateThroughProject,
PushPredicateThroughJoin,
-PushPredicateThroughGenerate) :: Nil
+PushPredicateThroughGenerate,
+ColumnPruning) :: Nil
}
val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
@@ -58,6 +59,38 @@ class FilterPushdownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+ test("column pruning for group") {
+val originalQuery =
+ testRelation
+.groupBy('a)('a, Count('b))
+.select('a)
+
+val optimized = Optimize(originalQuery.analyze)
+val correctAnswer =
+ testRelation
+.select('a)
+.groupBy('a)('a)
+.select('a).analyze
+
+comparePlans(optimized, correctAnswer)
+ }
+
+ test("column pruning for group with alias") {
+val originalQuery =
+ testRelation
+.groupBy('a)('a as 'c, Count('b))
+.select('c)
+
+
spark git commit: [SPARK-5789][SQL]Throw a better error message if JsonRDD.parseJson encounters unrecoverable parsing errors.
Repository: spark
Updated Branches:
refs/heads/master 2cbb3e433 -> 2e0c08452
[SPARK-5789][SQL]Throw a better error message if JsonRDD.parseJson encounters
unrecoverable parsing errors.
Author: Yin Huai
Closes #4582 from yhuai/jsonErrorMessage and squashes the following commits:
152dbd4 [Yin Huai] Update error message.
1466256 [Yin Huai] Throw a better error message when a JSON object in the input
dataset span multiple records (lines for files or strings for an RDD of
strings).
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2e0c0845
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2e0c0845
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2e0c0845
Branch: refs/heads/master
Commit: 2e0c084528409e1c565e6945521a33c0835ebbee
Parents: 2cbb3e4
Author: Yin Huai
Authored: Fri Feb 13 13:51:06 2015 -0800
Committer: Michael Armbrust
Committed: Fri Feb 13 13:51:06 2015 -0800
--
sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2e0c0845/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index 7dfb304..3b8dde1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -306,6 +306,10 @@ private[sql] object JsonRDD extends Logging {
val parsed = mapper.readValue(record, classOf[Object]) match {
case map: java.util.Map[_, _] =>
scalafy(map).asInstanceOf[Map[String, Any]] :: Nil
case list: java.util.List[_] =>
scalafy(list).asInstanceOf[Seq[Map[String, Any]]]
+case _ =>
+ sys.error(
+s"Failed to parse record $record. Please make sure that each
line of the file " +
+"(or each string in the RDD) is a valid JSON object or an
array of JSON objects.")
}
parsed
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5789][SQL]Throw a better error message if JsonRDD.parseJson encounters unrecoverable parsing errors.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 efffc2e42 -> d9d0250fc
[SPARK-5789][SQL]Throw a better error message if JsonRDD.parseJson encounters
unrecoverable parsing errors.
Author: Yin Huai
Closes #4582 from yhuai/jsonErrorMessage and squashes the following commits:
152dbd4 [Yin Huai] Update error message.
1466256 [Yin Huai] Throw a better error message when a JSON object in the input
dataset span multiple records (lines for files or strings for an RDD of
strings).
(cherry picked from commit 2e0c084528409e1c565e6945521a33c0835ebbee)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d9d0250f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d9d0250f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d9d0250f
Branch: refs/heads/branch-1.3
Commit: d9d0250fc5dfe529bebd4f67f945f4d7c3fc4106
Parents: efffc2e
Author: Yin Huai
Authored: Fri Feb 13 13:51:06 2015 -0800
Committer: Michael Armbrust
Committed: Fri Feb 13 13:51:34 2015 -0800
--
sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d9d0250f/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index 7dfb304..3b8dde1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -306,6 +306,10 @@ private[sql] object JsonRDD extends Logging {
val parsed = mapper.readValue(record, classOf[Object]) match {
case map: java.util.Map[_, _] =>
scalafy(map).asInstanceOf[Map[String, Any]] :: Nil
case list: java.util.List[_] =>
scalafy(list).asInstanceOf[Seq[Map[String, Any]]]
+case _ =>
+ sys.error(
+s"Failed to parse record $record. Please make sure that each
line of the file " +
+"(or each string in the RDD) is a valid JSON object or an
array of JSON objects.")
}
parsed
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SQL] [Minor] Update the SpecificMutableRow.copy
Repository: spark
Updated Branches:
refs/heads/master 8e25373ce -> cc552e042
[SQL] [Minor] Update the SpecificMutableRow.copy
When profiling the Join / Aggregate queries via VisualVM, I noticed lots of
`SpecificMutableRow` objects created, as well as the `MutableValue`, since the
`SpecificMutableRow` are mostly used in data source implementation, but the
`copy` method could be called multiple times in upper modules (e.g. in Join /
aggregation etc.), duplicated instances created should be avoid.
Author: Cheng Hao
Closes #4619 from chenghao-intel/specific_mutable_row and squashes the
following commits:
9300d23 [Cheng Hao] update the SpecificMutableRow.copy
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cc552e04
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cc552e04
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cc552e04
Branch: refs/heads/master
Commit: cc552e042896350e21eec9b78593de25006ecc70
Parents: 8e25373
Author: Cheng Hao
Authored: Mon Feb 16 12:21:08 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:21:08 2015 -0800
--
.../spark/sql/catalyst/expressions/SpecificMutableRow.scala | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/cc552e04/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
index 7434165..21d714c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
@@ -220,13 +220,14 @@ final class SpecificMutableRow(val values:
Array[MutableValue]) extends MutableR
override def isNullAt(i: Int): Boolean = values(i).isNull
override def copy(): Row = {
-val newValues = new Array[MutableValue](values.length)
+val newValues = new Array[Any](values.length)
var i = 0
while (i < values.length) {
- newValues(i) = values(i).copy()
+ newValues(i) = values(i).boxed
i += 1
}
-new SpecificMutableRow(newValues)
+
+new GenericRow(newValues)
}
override def update(ordinal: Int, value: Any): Unit = {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SQL] [Minor] Update the SpecificMutableRow.copy
Repository: spark
Updated Branches:
refs/heads/branch-1.3 fef2267cd -> 1a8895560
[SQL] [Minor] Update the SpecificMutableRow.copy
When profiling the Join / Aggregate queries via VisualVM, I noticed lots of
`SpecificMutableRow` objects created, as well as the `MutableValue`, since the
`SpecificMutableRow` are mostly used in data source implementation, but the
`copy` method could be called multiple times in upper modules (e.g. in Join /
aggregation etc.), duplicated instances created should be avoid.
Author: Cheng Hao
Closes #4619 from chenghao-intel/specific_mutable_row and squashes the
following commits:
9300d23 [Cheng Hao] update the SpecificMutableRow.copy
(cherry picked from commit cc552e042896350e21eec9b78593de25006ecc70)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1a889556
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1a889556
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1a889556
Branch: refs/heads/branch-1.3
Commit: 1a8895560f668faed33e99bcb88cafefd64fef03
Parents: fef2267
Author: Cheng Hao
Authored: Mon Feb 16 12:21:08 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:21:29 2015 -0800
--
.../spark/sql/catalyst/expressions/SpecificMutableRow.scala | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1a889556/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
index 7434165..21d714c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SpecificMutableRow.scala
@@ -220,13 +220,14 @@ final class SpecificMutableRow(val values:
Array[MutableValue]) extends MutableR
override def isNullAt(i: Int): Boolean = values(i).isNull
override def copy(): Row = {
-val newValues = new Array[MutableValue](values.length)
+val newValues = new Array[Any](values.length)
var i = 0
while (i < values.length) {
- newValues(i) = values(i).copy()
+ newValues(i) = values(i).boxed
i += 1
}
-new SpecificMutableRow(newValues)
+
+new GenericRow(newValues)
}
override def update(ordinal: Int, value: Any): Unit = {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5824] [SQL] add null format in ctas and set default col comment to null
Repository: spark
Updated Branches:
refs/heads/master cc552e042 -> 275a0c081
[SPARK-5824] [SQL] add null format in ctas and set default col comment to null
Author: Daoyuan Wang
Closes #4609 from adrian-wang/ctas and squashes the following commits:
0a75d5a [Daoyuan Wang] reorder import
93d1863 [Daoyuan Wang] add null format in ctas and set default col comment to
null
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/275a0c08
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/275a0c08
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/275a0c08
Branch: refs/heads/master
Commit: 275a0c08134dea1896eab73a8e017256900fb1db
Parents: cc552e0
Author: Daoyuan Wang
Authored: Mon Feb 16 12:31:36 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:31:36 2015 -0800
--
.../hive/execution/HiveCompatibilitySuite.scala | 1 +
.../spark/sql/hive/HiveMetastoreCatalog.scala | 3 ++-
...llformatCTAS-0-36f9196395758cebfed837a1c391a1e | 0
...lformatCTAS-1-b5a31d4cb34218b8de1ac3fed59fa75b | 0
...formatCTAS-10-7f4f04b87c7ef9653b4646949b24cf0b | 10 ++
...formatCTAS-11-4a4c16b53c612d00012d338c97bf5281 | 0
...formatCTAS-12-7f4f04b87c7ef9653b4646949b24cf0b | 10 ++
...formatCTAS-13-2e59caa113585495d8684fee69d88bc0 | 0
...formatCTAS-14-ad9fe9d68c2cf492259af4f6167c1b12 | 0
...lformatCTAS-2-aa2bdbd93668dceae43d1a02f2ede68d | 0
...lformatCTAS-3-b0057150f237050f38c1efa1f2d6b273 | 6 ++
...lformatCTAS-4-16c7086f39d6458b6c5cf2479f0473bd | 0
...lformatCTAS-5-183d77b734ce6a373de5b3ebe1cd04c9 | 0
...lformatCTAS-6-159fff36b548e00ee952d1df8ef19833 | 0
...lformatCTAS-7-46900b082b02ce3e58087d1f41128f65 | 4
...lformatCTAS-8-7f26cbd6be5631a3acce26f667d1c5d8 | 18 ++
...lformatCTAS-9-22e1b3899de7087b39c24d9d8f643b47 | 1 +
.../scala/org/apache/spark/sql/hive/Shim12.scala | 2 ++
.../scala/org/apache/spark/sql/hive/Shim13.scala | 7 +++
19 files changed, 61 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/275a0c08/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
--
diff --git
a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index e443e5b..133f2d3 100644
---
a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -640,6 +640,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with
BeforeAndAfter {
"nonblock_op_deduplicate",
"notable_alias1",
"notable_alias2",
+"nullformatCTAS",
"nullgroup",
"nullgroup2",
"nullgroup3",
http://git-wip-us.apache.org/repos/asf/spark/blob/275a0c08/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 6d794d0..f82778c 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -240,7 +240,7 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
val hiveSchema: JList[FieldSchema] = if (schema == null || schema.isEmpty)
{
crtTbl.getCols
} else {
- schema.map(attr => new FieldSchema(attr.name,
toMetastoreType(attr.dataType), ""))
+ schema.map(attr => new FieldSchema(attr.name,
toMetastoreType(attr.dataType), null))
}
tbl.setFields(hiveSchema)
@@ -314,6 +314,7 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
if (crtTbl != null && crtTbl.getLineDelim() != null) {
tbl.setSerdeParam(serdeConstants.LINE_DELIM, crtTbl.getLineDelim())
}
+HiveShim.setTblNullFormat(crtTbl, tbl)
if (crtTbl != null && crtTbl.getSerdeProps() != null) {
val iter = crtTbl.getSerdeProps().entrySet().iterator()
http://git-wip-us.apache.org/repos/asf/spark/blob/275a0c08/sql/hive/src/test/resources/golden/nullformatCTAS-0-36f9196395758cebfed837a1c391a1e
--
diff --git
a/sql/hive/src/test/resources/golden/nullformatCTAS-0-36f9196395758cebfed837a1c391a1e
b/sql/hive/src/test/resources/golden/nullformatCTAS-0-36f9196395758cebfed837a1c391a1e
new
spark git commit: [SQL] Initial support for reporting location of error in sql string
Repository: spark
Updated Branches:
refs/heads/branch-1.3 c2eaaea9f -> 63fa123f1
[SQL] Initial support for reporting location of error in sql string
Author: Michael Armbrust
Closes #4587 from marmbrus/position and squashes the following commits:
0810052 [Michael Armbrust] fix tests
395c019 [Michael Armbrust] Merge remote-tracking branch 'marmbrus/position'
into position
e155dce [Michael Armbrust] more errors
f3efa51 [Michael Armbrust] Update AnalysisException.scala
d45ff60 [Michael Armbrust] [SQL] Initial support for reporting location of
error in sql string
(cherry picked from commit 104b2c45805ce0a9c86e2823f402de6e9f0aee81)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/63fa123f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/63fa123f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/63fa123f
Branch: refs/heads/branch-1.3
Commit: 63fa123f1c2113caea74a7cf9a7293f256441dc7
Parents: c2eaaea
Author: Michael Armbrust
Authored: Mon Feb 16 12:32:56 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:33:11 2015 -0800
--
.../apache/spark/sql/AnalysisException.scala| 17 +-
.../spark/sql/catalyst/analysis/Analyzer.scala | 19 ++-
.../spark/sql/catalyst/analysis/Catalog.scala | 6 +
.../spark/sql/catalyst/analysis/package.scala | 10 ++
.../spark/sql/catalyst/trees/TreeNode.scala | 60 ++-
.../sql/catalyst/trees/TreeNodeSuite.scala | 14 ++
.../spark/sql/hive/HiveMetastoreCatalog.scala | 9 +-
.../org/apache/spark/sql/hive/HiveQl.scala | 47 --
.../spark/sql/hive/CachedTableSuite.scala | 4 +-
.../spark/sql/hive/ErrorPositionSuite.scala | 163 +++
.../sql/hive/execution/SQLQuerySuite.scala | 4 +-
11 files changed, 314 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/63fa123f/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
index 871d560..15add84 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
@@ -17,7 +17,22 @@
package org.apache.spark.sql
+import org.apache.spark.annotation.DeveloperApi
+
/**
+ * :: DeveloperApi ::
* Thrown when a query fails to analyze, usually because the query itself is
invalid.
*/
-class AnalysisException(message: String) extends Exception(message) with
Serializable
+@DeveloperApi
+class AnalysisException protected[sql] (
+val message: String,
+val line: Option[Int] = None,
+val startPosition: Option[Int] = None)
+ extends Exception with Serializable {
+
+ override def getMessage: String = {
+val lineAnnotation = line.map(l => s" line $l").getOrElse("")
+val positionAnnotation = startPosition.map(p => s" pos $p").getOrElse("")
+s"$message;$lineAnnotation$positionAnnotation"
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/63fa123f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 58a7003..aa4320b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -85,7 +85,7 @@ class Analyzer(catalog: Catalog,
operator transformExpressionsUp {
case a: Attribute if !a.resolved =>
val from = operator.inputSet.map(_.name).mkString(", ")
- failAnalysis(s"cannot resolve '${a.prettyString}' given input
columns $from")
+ a.failAnalysis(s"cannot resolve '${a.prettyString}' given input
columns $from")
case c: Cast if !c.resolved =>
failAnalysis(
@@ -246,12 +246,21 @@ class Analyzer(catalog: Catalog,
* Replaces [[UnresolvedRelation]]s with concrete relations from the catalog.
*/
object ResolveRelations extends Rule[LogicalPlan] {
+def getTable(u: UnresolvedRelation) = {
+ try {
+catalog.lookupRelation(u.tableIdentifier, u.alias)
+ } catch {
+case _: NoSuchTableException =>
+ u.failAnalysis(s"no such table ${u.t
spark git commit: [SPARK-5824] [SQL] add null format in ctas and set default col comment to null
Repository: spark
Updated Branches:
refs/heads/branch-1.3 1a8895560 -> c2eaaea9f
[SPARK-5824] [SQL] add null format in ctas and set default col comment to null
Author: Daoyuan Wang
Closes #4609 from adrian-wang/ctas and squashes the following commits:
0a75d5a [Daoyuan Wang] reorder import
93d1863 [Daoyuan Wang] add null format in ctas and set default col comment to
null
(cherry picked from commit 275a0c08134dea1896eab73a8e017256900fb1db)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c2eaaea9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c2eaaea9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c2eaaea9
Branch: refs/heads/branch-1.3
Commit: c2eaaea9f9f77662a4c9405b2796aa6bd362466e
Parents: 1a88955
Author: Daoyuan Wang
Authored: Mon Feb 16 12:31:36 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:31:44 2015 -0800
--
.../hive/execution/HiveCompatibilitySuite.scala | 1 +
.../spark/sql/hive/HiveMetastoreCatalog.scala | 3 ++-
...llformatCTAS-0-36f9196395758cebfed837a1c391a1e | 0
...lformatCTAS-1-b5a31d4cb34218b8de1ac3fed59fa75b | 0
...formatCTAS-10-7f4f04b87c7ef9653b4646949b24cf0b | 10 ++
...formatCTAS-11-4a4c16b53c612d00012d338c97bf5281 | 0
...formatCTAS-12-7f4f04b87c7ef9653b4646949b24cf0b | 10 ++
...formatCTAS-13-2e59caa113585495d8684fee69d88bc0 | 0
...formatCTAS-14-ad9fe9d68c2cf492259af4f6167c1b12 | 0
...lformatCTAS-2-aa2bdbd93668dceae43d1a02f2ede68d | 0
...lformatCTAS-3-b0057150f237050f38c1efa1f2d6b273 | 6 ++
...lformatCTAS-4-16c7086f39d6458b6c5cf2479f0473bd | 0
...lformatCTAS-5-183d77b734ce6a373de5b3ebe1cd04c9 | 0
...lformatCTAS-6-159fff36b548e00ee952d1df8ef19833 | 0
...lformatCTAS-7-46900b082b02ce3e58087d1f41128f65 | 4
...lformatCTAS-8-7f26cbd6be5631a3acce26f667d1c5d8 | 18 ++
...lformatCTAS-9-22e1b3899de7087b39c24d9d8f643b47 | 1 +
.../scala/org/apache/spark/sql/hive/Shim12.scala | 2 ++
.../scala/org/apache/spark/sql/hive/Shim13.scala | 7 +++
19 files changed, 61 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c2eaaea9/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
--
diff --git
a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index e443e5b..133f2d3 100644
---
a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -640,6 +640,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with
BeforeAndAfter {
"nonblock_op_deduplicate",
"notable_alias1",
"notable_alias2",
+"nullformatCTAS",
"nullgroup",
"nullgroup2",
"nullgroup3",
http://git-wip-us.apache.org/repos/asf/spark/blob/c2eaaea9/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 6d794d0..f82778c 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -240,7 +240,7 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
val hiveSchema: JList[FieldSchema] = if (schema == null || schema.isEmpty)
{
crtTbl.getCols
} else {
- schema.map(attr => new FieldSchema(attr.name,
toMetastoreType(attr.dataType), ""))
+ schema.map(attr => new FieldSchema(attr.name,
toMetastoreType(attr.dataType), null))
}
tbl.setFields(hiveSchema)
@@ -314,6 +314,7 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
if (crtTbl != null && crtTbl.getLineDelim() != null) {
tbl.setSerdeParam(serdeConstants.LINE_DELIM, crtTbl.getLineDelim())
}
+HiveShim.setTblNullFormat(crtTbl, tbl)
if (crtTbl != null && crtTbl.getSerdeProps() != null) {
val iter = crtTbl.getSerdeProps().entrySet().iterator()
http://git-wip-us.apache.org/repos/asf/spark/blob/c2eaaea9/sql/hive/src/test/resources/golden/nullformatCTAS-0-36f9196395758cebfed837a1c391a1e
--
diff --git
a/sql/hive/src/test/resources/golden/nullformatCTAS-0-36f919639575
spark git commit: [SQL] Initial support for reporting location of error in sql string
Repository: spark
Updated Branches:
refs/heads/master 275a0c081 -> 104b2c458
[SQL] Initial support for reporting location of error in sql string
Author: Michael Armbrust
Closes #4587 from marmbrus/position and squashes the following commits:
0810052 [Michael Armbrust] fix tests
395c019 [Michael Armbrust] Merge remote-tracking branch 'marmbrus/position'
into position
e155dce [Michael Armbrust] more errors
f3efa51 [Michael Armbrust] Update AnalysisException.scala
d45ff60 [Michael Armbrust] [SQL] Initial support for reporting location of
error in sql string
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/104b2c45
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/104b2c45
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/104b2c45
Branch: refs/heads/master
Commit: 104b2c45805ce0a9c86e2823f402de6e9f0aee81
Parents: 275a0c0
Author: Michael Armbrust
Authored: Mon Feb 16 12:32:56 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:32:56 2015 -0800
--
.../apache/spark/sql/AnalysisException.scala| 17 +-
.../spark/sql/catalyst/analysis/Analyzer.scala | 19 ++-
.../spark/sql/catalyst/analysis/Catalog.scala | 6 +
.../spark/sql/catalyst/analysis/package.scala | 10 ++
.../spark/sql/catalyst/trees/TreeNode.scala | 60 ++-
.../sql/catalyst/trees/TreeNodeSuite.scala | 14 ++
.../spark/sql/hive/HiveMetastoreCatalog.scala | 9 +-
.../org/apache/spark/sql/hive/HiveQl.scala | 47 --
.../spark/sql/hive/CachedTableSuite.scala | 4 +-
.../spark/sql/hive/ErrorPositionSuite.scala | 163 +++
.../sql/hive/execution/SQLQuerySuite.scala | 4 +-
11 files changed, 314 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/104b2c45/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
index 871d560..15add84 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/AnalysisException.scala
@@ -17,7 +17,22 @@
package org.apache.spark.sql
+import org.apache.spark.annotation.DeveloperApi
+
/**
+ * :: DeveloperApi ::
* Thrown when a query fails to analyze, usually because the query itself is
invalid.
*/
-class AnalysisException(message: String) extends Exception(message) with
Serializable
+@DeveloperApi
+class AnalysisException protected[sql] (
+val message: String,
+val line: Option[Int] = None,
+val startPosition: Option[Int] = None)
+ extends Exception with Serializable {
+
+ override def getMessage: String = {
+val lineAnnotation = line.map(l => s" line $l").getOrElse("")
+val positionAnnotation = startPosition.map(p => s" pos $p").getOrElse("")
+s"$message;$lineAnnotation$positionAnnotation"
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/104b2c45/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 58a7003..aa4320b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -85,7 +85,7 @@ class Analyzer(catalog: Catalog,
operator transformExpressionsUp {
case a: Attribute if !a.resolved =>
val from = operator.inputSet.map(_.name).mkString(", ")
- failAnalysis(s"cannot resolve '${a.prettyString}' given input
columns $from")
+ a.failAnalysis(s"cannot resolve '${a.prettyString}' given input
columns $from")
case c: Cast if !c.resolved =>
failAnalysis(
@@ -246,12 +246,21 @@ class Analyzer(catalog: Catalog,
* Replaces [[UnresolvedRelation]]s with concrete relations from the catalog.
*/
object ResolveRelations extends Rule[LogicalPlan] {
+def getTable(u: UnresolvedRelation) = {
+ try {
+catalog.lookupRelation(u.tableIdentifier, u.alias)
+ } catch {
+case _: NoSuchTableException =>
+ u.failAnalysis(s"no such table ${u.tableIdentifier}")
+ }
+}
+
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-
spark git commit: [SQL] Add fetched row count in SparkSQLCLIDriver
Repository: spark
Updated Branches:
refs/heads/branch-1.3 63fa123f1 -> 0368494c5
[SQL] Add fetched row count in SparkSQLCLIDriver
before this change:
```scala
Time taken: 0.619 seconds
```
after this change :
```scala
Time taken: 0.619 seconds, Fetched: 4 row(s)
```
Author: OopsOutOfMemory
Closes #4604 from OopsOutOfMemory/rowcount and squashes the following commits:
7252dea [OopsOutOfMemory] add fetched row count
(cherry picked from commit b4d7c7032d755de42951f92d9535287ef6230b9b)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0368494c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0368494c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0368494c
Branch: refs/heads/branch-1.3
Commit: 0368494c502c33c05f806d106ff2042acad91cee
Parents: 63fa123
Author: OopsOutOfMemory
Authored: Mon Feb 16 12:34:09 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:34:22 2015 -0800
--
.../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 12 ++--
1 file changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0368494c/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
--
diff --git
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index bb19ac2..401e97b 100755
---
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -292,9 +292,13 @@ private[hive] class SparkSQLCLIDriver extends CliDriver
with Logging {
}
}
+ var counter = 0
try {
while (!out.checkError() && driver.getResults(res)) {
- res.foreach(out.println)
+ res.foreach{ l =>
+counter += 1
+out.println(l)
+ }
res.clear()
}
} catch {
@@ -311,7 +315,11 @@ private[hive] class SparkSQLCLIDriver extends CliDriver
with Logging {
ret = cret
}
- console.printInfo(s"Time taken: $timeTaken seconds", null)
+ var responseMsg = s"Time taken: $timeTaken seconds"
+ if (counter != 0) {
+responseMsg += s", Fetched $counter row(s)"
+ }
+ console.printInfo(responseMsg , null)
// Destroy the driver to release all the locks.
driver.destroy()
} else {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SQL] Add fetched row count in SparkSQLCLIDriver
Repository: spark
Updated Branches:
refs/heads/master 104b2c458 -> b4d7c7032
[SQL] Add fetched row count in SparkSQLCLIDriver
before this change:
```scala
Time taken: 0.619 seconds
```
after this change :
```scala
Time taken: 0.619 seconds, Fetched: 4 row(s)
```
Author: OopsOutOfMemory
Closes #4604 from OopsOutOfMemory/rowcount and squashes the following commits:
7252dea [OopsOutOfMemory] add fetched row count
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b4d7c703
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b4d7c703
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b4d7c703
Branch: refs/heads/master
Commit: b4d7c7032d755de42951f92d9535287ef6230b9b
Parents: 104b2c4
Author: OopsOutOfMemory
Authored: Mon Feb 16 12:34:09 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:34:09 2015 -0800
--
.../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 12 ++--
1 file changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b4d7c703/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
--
diff --git
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index bb19ac2..401e97b 100755
---
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -292,9 +292,13 @@ private[hive] class SparkSQLCLIDriver extends CliDriver
with Logging {
}
}
+ var counter = 0
try {
while (!out.checkError() && driver.getResults(res)) {
- res.foreach(out.println)
+ res.foreach{ l =>
+counter += 1
+out.println(l)
+ }
res.clear()
}
} catch {
@@ -311,7 +315,11 @@ private[hive] class SparkSQLCLIDriver extends CliDriver
with Logging {
ret = cret
}
- console.printInfo(s"Time taken: $timeTaken seconds", null)
+ var responseMsg = s"Time taken: $timeTaken seconds"
+ if (counter != 0) {
+responseMsg += s", Fetched $counter row(s)"
+ }
+ console.printInfo(responseMsg , null)
// Destroy the driver to release all the locks.
driver.destroy()
} else {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5296] [SQL] Add more filter types for data sources API
Repository: spark
Updated Branches:
refs/heads/master b4d7c7032 -> 6f54dee66
[SPARK-5296] [SQL] Add more filter types for data sources API
This PR adds the following filter types for data sources API:
- `IsNull`
- `IsNotNull`
- `Not`
- `And`
- `Or`
The code which converts Catalyst predicate expressions to data sources filters
is very similar to filter conversion logics in `ParquetFilters` which converts
Catalyst predicates to Parquet filter predicates. In this way we can support
nested AND/OR/NOT predicates without changing current `BaseScan` type hierarchy.
[https://reviewable.io/review_button.png"; height=40 alt="Review on
Reviewable"/>](https://reviewable.io/reviews/apache/spark/4623)
Author: Cheng Lian
This patch had conflicts when merged, resolved by
Committer: Michael Armbrust
Closes #4623 from liancheng/more-fiters and squashes the following commits:
1b296f4 [Cheng Lian] Add more filter types for data sources API
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6f54dee6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6f54dee6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6f54dee6
Branch: refs/heads/master
Commit: 6f54dee66100e5e58f6649158db257eb5009bd6a
Parents: b4d7c70
Author: Cheng Lian
Authored: Mon Feb 16 12:48:55 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:48:55 2015 -0800
--
.../scala/org/apache/spark/sql/SQLContext.scala | 9 ++-
.../apache/spark/sql/parquet/newParquet.scala | 5 +-
.../spark/sql/sources/DataSourceStrategy.scala | 81 ++--
.../org/apache/spark/sql/sources/filters.scala | 5 ++
.../spark/sql/sources/FilteredScanSuite.scala | 34 +++-
5 files changed, 103 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6f54dee6/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index b42a52e..1442250 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -28,16 +28,16 @@ import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.annotation.{AlphaComponent, DeveloperApi, Experimental}
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.optimizer.{DefaultOptimizer, Optimizer}
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation,
LogicalPlan, NoRelation}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
-import org.apache.spark.sql.execution._
+import org.apache.spark.sql.catalyst.{ScalaReflection, expressions}
+import org.apache.spark.sql.execution.{Filter, _}
import org.apache.spark.sql.jdbc.{JDBCPartition, JDBCPartitioningInfo,
JDBCRelation}
import org.apache.spark.sql.json._
-import org.apache.spark.sql.sources.{BaseRelation, DDLParser,
DataSourceStrategy, LogicalRelation, _}
+import org.apache.spark.sql.sources._
import org.apache.spark.sql.types._
import org.apache.spark.util.Utils
import org.apache.spark.{Partition, SparkContext}
@@ -867,7 +867,8 @@ class SQLContext(@transient val sparkContext: SparkContext)
val projectSet = AttributeSet(projectList.flatMap(_.references))
val filterSet = AttributeSet(filterPredicates.flatMap(_.references))
- val filterCondition =
prunePushedDownFilters(filterPredicates).reduceLeftOption(And)
+ val filterCondition =
+
prunePushedDownFilters(filterPredicates).reduceLeftOption(expressions.And)
// Right now we still use a projection even if the only evaluation is
applying an alias
// to a column. Since this is a no-op, it could be avoided. However,
using this
http://git-wip-us.apache.org/repos/asf/spark/blob/6f54dee6/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
index 9279f5a..9bb34e2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
@@ -32,6 +32,7 @@ import org.apache.hadoop.io.Writable
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
import org.apache.hadoop.mapred
spark git commit: [SPARK-5296] [SQL] Add more filter types for data sources API
Repository: spark
Updated Branches:
refs/heads/branch-1.3 0368494c5 -> 363a9a7d5
[SPARK-5296] [SQL] Add more filter types for data sources API
This PR adds the following filter types for data sources API:
- `IsNull`
- `IsNotNull`
- `Not`
- `And`
- `Or`
The code which converts Catalyst predicate expressions to data sources filters
is very similar to filter conversion logics in `ParquetFilters` which converts
Catalyst predicates to Parquet filter predicates. In this way we can support
nested AND/OR/NOT predicates without changing current `BaseScan` type hierarchy.
[https://reviewable.io/review_button.png"; height=40 alt="Review on
Reviewable"/>](https://reviewable.io/reviews/apache/spark/4623)
Author: Cheng Lian
This patch had conflicts when merged, resolved by
Committer: Michael Armbrust
Closes #4623 from liancheng/more-fiters and squashes the following commits:
1b296f4 [Cheng Lian] Add more filter types for data sources API
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/363a9a7d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/363a9a7d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/363a9a7d
Branch: refs/heads/branch-1.3
Commit: 363a9a7d5ad682f828288f792a836c2c0b5e2f89
Parents: 0368494
Author: Cheng Lian
Authored: Mon Feb 16 12:48:55 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:50:10 2015 -0800
--
.../scala/org/apache/spark/sql/SQLContext.scala | 9 ++-
.../apache/spark/sql/parquet/newParquet.scala | 5 +-
.../spark/sql/sources/DataSourceStrategy.scala | 81 ++--
.../org/apache/spark/sql/sources/filters.scala | 5 ++
.../spark/sql/sources/FilteredScanSuite.scala | 34 +++-
5 files changed, 103 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/363a9a7d/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index b42a52e..1442250 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -28,16 +28,16 @@ import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.annotation.{AlphaComponent, DeveloperApi, Experimental}
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.optimizer.{DefaultOptimizer, Optimizer}
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation,
LogicalPlan, NoRelation}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
-import org.apache.spark.sql.execution._
+import org.apache.spark.sql.catalyst.{ScalaReflection, expressions}
+import org.apache.spark.sql.execution.{Filter, _}
import org.apache.spark.sql.jdbc.{JDBCPartition, JDBCPartitioningInfo,
JDBCRelation}
import org.apache.spark.sql.json._
-import org.apache.spark.sql.sources.{BaseRelation, DDLParser,
DataSourceStrategy, LogicalRelation, _}
+import org.apache.spark.sql.sources._
import org.apache.spark.sql.types._
import org.apache.spark.util.Utils
import org.apache.spark.{Partition, SparkContext}
@@ -867,7 +867,8 @@ class SQLContext(@transient val sparkContext: SparkContext)
val projectSet = AttributeSet(projectList.flatMap(_.references))
val filterSet = AttributeSet(filterPredicates.flatMap(_.references))
- val filterCondition =
prunePushedDownFilters(filterPredicates).reduceLeftOption(And)
+ val filterCondition =
+
prunePushedDownFilters(filterPredicates).reduceLeftOption(expressions.And)
// Right now we still use a projection even if the only evaluation is
applying an alias
// to a column. Since this is a no-op, it could be avoided. However,
using this
http://git-wip-us.apache.org/repos/asf/spark/blob/363a9a7d/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
index 9279f5a..9bb34e2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
@@ -32,6 +32,7 @@ import org.apache.hadoop.io.Writable
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
import org.apache.hadoo
spark git commit: [SPARK-5833] [SQL] Adds REFRESH TABLE command
Repository: spark
Updated Branches:
refs/heads/master 6f54dee66 -> c51ab37fa
[SPARK-5833] [SQL] Adds REFRESH TABLE command
Lifts `HiveMetastoreCatalog.refreshTable` to `Catalog`. Adds `RefreshTable`
command to refresh (possibly cached) metadata in external data sources tables.
[https://reviewable.io/review_button.png"; height=40 alt="Review on
Reviewable"/>](https://reviewable.io/reviews/apache/spark/4624)
Author: Cheng Lian
Closes #4624 from liancheng/refresh-table and squashes the following commits:
8d1aa4c [Cheng Lian] Adds REFRESH TABLE command
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c51ab37f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c51ab37f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c51ab37f
Branch: refs/heads/master
Commit: c51ab37faddf4ede23243058dfb388e74a192552
Parents: 6f54dee
Author: Cheng Lian
Authored: Mon Feb 16 12:52:05 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:52:05 2015 -0800
--
.../spark/sql/catalyst/analysis/Catalog.scala | 10
.../org/apache/spark/sql/sources/ddl.scala | 52 +++-
.../spark/sql/hive/HiveMetastoreCatalog.scala | 2 +-
.../sql/hive/MetastoreDataSourcesSuite.scala| 2 +-
4 files changed, 42 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c51ab37f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
index bf97215..9e6e291 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
@@ -46,6 +46,8 @@ trait Catalog {
*/
def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
+ def refreshTable(databaseName: String, tableName: String): Unit
+
def registerTable(tableIdentifier: Seq[String], plan: LogicalPlan): Unit
def unregisterTable(tableIdentifier: Seq[String]): Unit
@@ -119,6 +121,10 @@ class SimpleCatalog(val caseSensitive: Boolean) extends
Catalog {
case (name, _) => (name, true)
}.toSeq
}
+
+ override def refreshTable(databaseName: String, tableName: String): Unit = {
+throw new UnsupportedOperationException
+ }
}
/**
@@ -224,4 +230,8 @@ object EmptyCatalog extends Catalog {
}
override def unregisterAllTables(): Unit = {}
+
+ override def refreshTable(databaseName: String, tableName: String): Unit = {
+throw new UnsupportedOperationException
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c51ab37f/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
index 8cac9c0..1b5e8c2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.{SaveMode, DataFrame, SQLContext}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.AbstractSparkSQLParser
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.expressions.{Row, AttributeReference}
import org.apache.spark.sql.execution.RunnableCommand
import org.apache.spark.sql.types._
import org.apache.spark.util.Utils
@@ -66,6 +66,7 @@ private[sql] class DDLParser extends AbstractSparkSQLParser
with Logging {
protected val EXTENDED = Keyword("EXTENDED")
protected val AS = Keyword("AS")
protected val COMMENT = Keyword("COMMENT")
+ protected val REFRESH = Keyword("REFRESH")
// Data types.
protected val STRING = Keyword("STRING")
@@ -85,7 +86,7 @@ private[sql] class DDLParser extends AbstractSparkSQLParser
with Logging {
protected val MAP = Keyword("MAP")
protected val STRUCT = Keyword("STRUCT")
- protected lazy val ddl: Parser[LogicalPlan] = createTable | describeTable
+ protected lazy val ddl: Parser[LogicalPlan] = createTable | describeTable |
refreshTable
protected def start: Parser[LogicalPlan] = ddl
@@ -104,9 +105,8 @@ private[sql] class DDLParser extends AbstractSparkSQLParser
with Logging {
* AS SELECT ...
*/
protected lazy val createTable: Parser[LogicalPlan] =
- (
-(CREATE ~> TEMPORARY.? <~ TABLE) ~ (IF ~> NOT <~ EXISTS).? ~ ident
- ~ (tableCo
spark git commit: [SPARK-5833] [SQL] Adds REFRESH TABLE command
Repository: spark
Updated Branches:
refs/heads/branch-1.3 363a9a7d5 -> 864d77e0d
[SPARK-5833] [SQL] Adds REFRESH TABLE command
Lifts `HiveMetastoreCatalog.refreshTable` to `Catalog`. Adds `RefreshTable`
command to refresh (possibly cached) metadata in external data sources tables.
[https://reviewable.io/review_button.png"; height=40 alt="Review on
Reviewable"/>](https://reviewable.io/reviews/apache/spark/4624)
Author: Cheng Lian
Closes #4624 from liancheng/refresh-table and squashes the following commits:
8d1aa4c [Cheng Lian] Adds REFRESH TABLE command
(cherry picked from commit c51ab37faddf4ede23243058dfb388e74a192552)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/864d77e0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/864d77e0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/864d77e0
Branch: refs/heads/branch-1.3
Commit: 864d77e0d23b974943a1875b7372de05b3595bd5
Parents: 363a9a7
Author: Cheng Lian
Authored: Mon Feb 16 12:52:05 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 12:52:28 2015 -0800
--
.../spark/sql/catalyst/analysis/Catalog.scala | 10
.../org/apache/spark/sql/sources/ddl.scala | 52 +++-
.../spark/sql/hive/HiveMetastoreCatalog.scala | 2 +-
.../sql/hive/MetastoreDataSourcesSuite.scala| 2 +-
4 files changed, 42 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/864d77e0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
index bf97215..9e6e291 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
@@ -46,6 +46,8 @@ trait Catalog {
*/
def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
+ def refreshTable(databaseName: String, tableName: String): Unit
+
def registerTable(tableIdentifier: Seq[String], plan: LogicalPlan): Unit
def unregisterTable(tableIdentifier: Seq[String]): Unit
@@ -119,6 +121,10 @@ class SimpleCatalog(val caseSensitive: Boolean) extends
Catalog {
case (name, _) => (name, true)
}.toSeq
}
+
+ override def refreshTable(databaseName: String, tableName: String): Unit = {
+throw new UnsupportedOperationException
+ }
}
/**
@@ -224,4 +230,8 @@ object EmptyCatalog extends Catalog {
}
override def unregisterAllTables(): Unit = {}
+
+ override def refreshTable(databaseName: String, tableName: String): Unit = {
+throw new UnsupportedOperationException
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/864d77e0/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
index 16ea952..c762568 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.{SaveMode, DataFrame, SQLContext}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.AbstractSparkSQLParser
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.expressions.{Row, AttributeReference}
import org.apache.spark.sql.execution.RunnableCommand
import org.apache.spark.sql.types._
import org.apache.spark.util.Utils
@@ -66,6 +66,7 @@ private[sql] class DDLParser extends AbstractSparkSQLParser
with Logging {
protected val EXTENDED = Keyword("EXTENDED")
protected val AS = Keyword("AS")
protected val COMMENT = Keyword("COMMENT")
+ protected val REFRESH = Keyword("REFRESH")
// Data types.
protected val STRING = Keyword("STRING")
@@ -85,7 +86,7 @@ private[sql] class DDLParser extends AbstractSparkSQLParser
with Logging {
protected val MAP = Keyword("MAP")
protected val STRUCT = Keyword("STRUCT")
- protected lazy val ddl: Parser[LogicalPlan] = createTable | describeTable
+ protected lazy val ddl: Parser[LogicalPlan] = createTable | describeTable |
refreshTable
protected def start: Parser[LogicalPlan] = ddl
@@ -104,9 +105,8 @@ private[sql] class DDLParser extends AbstractSparkSQLParser
with Logging {
* AS SELECT ...
*/
protected lazy val createTable: Parse
spark git commit: [SPARK-5746][SQL] Check invalid cases for the write path of data source API
Repository: spark
Updated Branches:
refs/heads/master 04b401da8 -> 5b6cd65cd
[SPARK-5746][SQL] Check invalid cases for the write path of data source API
JIRA: https://issues.apache.org/jira/browse/SPARK-5746
liancheng marmbrus
Author: Yin Huai
Closes #4617 from yhuai/insertOverwrite and squashes the following commits:
8e3019d [Yin Huai] Fix compilation error.
499e8e7 [Yin Huai] Merge remote-tracking branch 'upstream/master' into
insertOverwrite
e76e85a [Yin Huai] Address comments.
ac31b3c [Yin Huai] Merge remote-tracking branch 'upstream/master' into
insertOverwrite
f30bdad [Yin Huai] Use toDF.
99da57e [Yin Huai] Merge remote-tracking branch 'upstream/master' into
insertOverwrite
6b7545c [Yin Huai] Add a pre write check to the data source API.
a88c516 [Yin Huai] DDLParser will take a parsering function to take care CTAS
statements.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5b6cd65c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5b6cd65c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5b6cd65c
Branch: refs/heads/master
Commit: 5b6cd65cd611b1a46a7d5eb33139c6224b96264e
Parents: 04b401d
Author: Yin Huai
Authored: Mon Feb 16 15:51:59 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 15:51:59 2015 -0800
--
.../spark/sql/catalyst/analysis/Analyzer.scala | 13 +-
.../org/apache/spark/sql/DataFrameImpl.scala| 8 +-
.../scala/org/apache/spark/sql/SQLContext.scala | 5 +-
.../spark/sql/execution/SparkStrategies.scala | 10 +-
.../spark/sql/sources/DataSourceStrategy.scala | 5 +-
.../org/apache/spark/sql/sources/ddl.scala | 29 ++-
.../org/apache/spark/sql/sources/rules.scala| 72 +-
.../spark/sql/parquet/ParquetQuerySuite.scala | 13 +-
.../sql/sources/CreateTableAsSelectSuite.scala | 28 +++
.../spark/sql/sources/DataSourceTest.scala | 3 +-
.../spark/sql/sources/InsertIntoSuite.scala | 176 ---
.../apache/spark/sql/sources/InsertSuite.scala | 218 +++
.../org/apache/spark/sql/hive/HiveContext.scala | 12 +-
.../spark/sql/hive/HiveMetastoreCatalog.scala | 2 +-
.../apache/spark/sql/hive/HiveStrategies.scala | 8 +-
15 files changed, 371 insertions(+), 231 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5b6cd65c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index aa4320b..fc37b8c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -50,7 +50,13 @@ class Analyzer(catalog: Catalog,
/**
* Override to provide additional rules for the "Resolution" batch.
*/
- val extendedRules: Seq[Rule[LogicalPlan]] = Nil
+ val extendedResolutionRules: Seq[Rule[LogicalPlan]] = Nil
+
+ /**
+ * Override to provide additional rules for the "Check Analysis" batch.
+ * These rules will be evaluated after our built-in check rules.
+ */
+ val extendedCheckRules: Seq[Rule[LogicalPlan]] = Nil
lazy val batches: Seq[Batch] = Seq(
Batch("Resolution", fixedPoint,
@@ -64,9 +70,10 @@ class Analyzer(catalog: Catalog,
UnresolvedHavingClauseAttributes ::
TrimGroupingAliases ::
typeCoercionRules ++
- extendedRules : _*),
+ extendedResolutionRules : _*),
Batch("Check Analysis", Once,
- CheckResolution),
+ CheckResolution +:
+ extendedCheckRules: _*),
Batch("Remove SubQueries", fixedPoint,
EliminateSubQueries)
)
http://git-wip-us.apache.org/repos/asf/spark/blob/5b6cd65c/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
index 500e3c9..3c1cf8d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
@@ -67,7 +67,11 @@ private[sql] class DataFrameImpl protected[sql](
@transient protected[sql] override val logicalPlan: LogicalPlan =
queryExecution.logical match {
// For various commands (like DDL) and queries with side effects, we force
query optimization to
// happen right away to let these side effects take place eagerly.
-case _: Command | _: I
spark git commit: [SPARK-5746][SQL] Check invalid cases for the write path of data source API
Repository: spark
Updated Branches:
refs/heads/branch-1.3 ad8fd4fb3 -> 419865475
[SPARK-5746][SQL] Check invalid cases for the write path of data source API
JIRA: https://issues.apache.org/jira/browse/SPARK-5746
liancheng marmbrus
Author: Yin Huai
Closes #4617 from yhuai/insertOverwrite and squashes the following commits:
8e3019d [Yin Huai] Fix compilation error.
499e8e7 [Yin Huai] Merge remote-tracking branch 'upstream/master' into
insertOverwrite
e76e85a [Yin Huai] Address comments.
ac31b3c [Yin Huai] Merge remote-tracking branch 'upstream/master' into
insertOverwrite
f30bdad [Yin Huai] Use toDF.
99da57e [Yin Huai] Merge remote-tracking branch 'upstream/master' into
insertOverwrite
6b7545c [Yin Huai] Add a pre write check to the data source API.
a88c516 [Yin Huai] DDLParser will take a parsering function to take care CTAS
statements.
(cherry picked from commit 5b6cd65cd611b1a46a7d5eb33139c6224b96264e)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/41986547
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/41986547
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/41986547
Branch: refs/heads/branch-1.3
Commit: 4198654758ba262177010c4651d8c0782576ff67
Parents: ad8fd4f
Author: Yin Huai
Authored: Mon Feb 16 15:51:59 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 15:52:07 2015 -0800
--
.../spark/sql/catalyst/analysis/Analyzer.scala | 13 +-
.../org/apache/spark/sql/DataFrameImpl.scala| 8 +-
.../scala/org/apache/spark/sql/SQLContext.scala | 5 +-
.../spark/sql/execution/SparkStrategies.scala | 10 +-
.../spark/sql/sources/DataSourceStrategy.scala | 5 +-
.../org/apache/spark/sql/sources/ddl.scala | 29 ++-
.../org/apache/spark/sql/sources/rules.scala| 72 +-
.../spark/sql/parquet/ParquetQuerySuite.scala | 13 +-
.../sql/sources/CreateTableAsSelectSuite.scala | 28 +++
.../spark/sql/sources/DataSourceTest.scala | 3 +-
.../spark/sql/sources/InsertIntoSuite.scala | 176 ---
.../apache/spark/sql/sources/InsertSuite.scala | 218 +++
.../org/apache/spark/sql/hive/HiveContext.scala | 12 +-
.../spark/sql/hive/HiveMetastoreCatalog.scala | 2 +-
.../apache/spark/sql/hive/HiveStrategies.scala | 8 +-
15 files changed, 371 insertions(+), 231 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/41986547/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index aa4320b..fc37b8c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -50,7 +50,13 @@ class Analyzer(catalog: Catalog,
/**
* Override to provide additional rules for the "Resolution" batch.
*/
- val extendedRules: Seq[Rule[LogicalPlan]] = Nil
+ val extendedResolutionRules: Seq[Rule[LogicalPlan]] = Nil
+
+ /**
+ * Override to provide additional rules for the "Check Analysis" batch.
+ * These rules will be evaluated after our built-in check rules.
+ */
+ val extendedCheckRules: Seq[Rule[LogicalPlan]] = Nil
lazy val batches: Seq[Batch] = Seq(
Batch("Resolution", fixedPoint,
@@ -64,9 +70,10 @@ class Analyzer(catalog: Catalog,
UnresolvedHavingClauseAttributes ::
TrimGroupingAliases ::
typeCoercionRules ++
- extendedRules : _*),
+ extendedResolutionRules : _*),
Batch("Check Analysis", Once,
- CheckResolution),
+ CheckResolution +:
+ extendedCheckRules: _*),
Batch("Remove SubQueries", fixedPoint,
EliminateSubQueries)
)
http://git-wip-us.apache.org/repos/asf/spark/blob/41986547/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
index 9eb0c13..bc19210 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
@@ -67,7 +67,11 @@ private[sql] class DataFrameImpl protected[sql](
@transient protected[sql] override val logicalPlan: LogicalPlan =
queryExecution.logical match {
// For various commands (like DDL) and queries with side effects, we force
query optimization to
// h
spark git commit: [SPARK-5839][SQL]HiveMetastoreCatalog does not recognize table names and aliases of data source tables.
Repository: spark
Updated Branches:
refs/heads/master 5b6cd65cd -> f3ff1eb29
[SPARK-5839][SQL]HiveMetastoreCatalog does not recognize table names and
aliases of data source tables.
JIRA: https://issues.apache.org/jira/browse/SPARK-5839
Author: Yin Huai
Closes #4626 from yhuai/SPARK-5839 and squashes the following commits:
f779d85 [Yin Huai] Use subqeury to wrap replaced ParquetRelation.
2695f13 [Yin Huai] Merge remote-tracking branch 'upstream/master' into
SPARK-5839
f1ba6ca [Yin Huai] Address comment.
2c7fa08 [Yin Huai] Use Subqueries to wrap a data source table.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f3ff1eb2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f3ff1eb2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f3ff1eb2
Branch: refs/heads/master
Commit: f3ff1eb2985ff3e1567645b898f6b42e4b01f237
Parents: 5b6cd65
Author: Yin Huai
Authored: Mon Feb 16 15:54:01 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 15:54:01 2015 -0800
--
.../spark/sql/parquet/ParquetQuerySuite.scala | 5 +--
.../spark/sql/hive/HiveMetastoreCatalog.scala | 18 +--
.../sql/hive/MetastoreDataSourcesSuite.scala| 34
3 files changed, 53 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f3ff1eb2/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
index 9318c15..8b4d05e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
@@ -30,9 +30,10 @@ import org.apache.spark.sql.test.TestSQLContext._
class ParquetQuerySuiteBase extends QueryTest with ParquetTest {
val sqlContext = TestSQLContext
- test("simple projection") {
+ test("simple select queries") {
withParquetTable((0 until 10).map(i => (i, i.toString)), "t") {
- checkAnswer(sql("SELECT _1 FROM t"), (0 until 10).map(Row.apply(_)))
+ checkAnswer(sql("SELECT _1 FROM t where t._1 > 5"), (6 until
10).map(Row.apply(_)))
+ checkAnswer(sql("SELECT _1 FROM t as tmp where tmp._1 < 5"), (0 until
5).map(Row.apply(_)))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f3ff1eb2/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 72211fe..87bc9fe 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -160,7 +160,15 @@ private[hive] class HiveMetastoreCatalog(hive:
HiveContext) extends Catalog with
}
if (table.getProperty("spark.sql.sources.provider") != null) {
- cachedDataSourceTables(QualifiedTableName(databaseName,
tblName).toLowerCase)
+ val dataSourceTable =
+cachedDataSourceTables(QualifiedTableName(databaseName,
tblName).toLowerCase)
+ // Then, if alias is specified, wrap the table with a Subquery using the
alias.
+ // Othersie, wrap the table with a Subquery using the table name.
+ val withAlias =
+alias.map(a => Subquery(a, dataSourceTable)).getOrElse(
+ Subquery(tableIdent.last, dataSourceTable))
+
+ withAlias
} else if (table.isView) {
// if the unresolved relation is from hive view
// parse the text into logic node.
@@ -433,7 +441,13 @@ private[hive] class HiveMetastoreCatalog(hive:
HiveContext) extends Catalog with
val attributedRewrites =
AttributeMap(relation.output.zip(parquetRelation.output))
lastPlan.transformUp {
- case r: MetastoreRelation if r == relation => parquetRelation
+ case r: MetastoreRelation if r == relation => {
+val withAlias =
+ r.alias.map(a => Subquery(a, parquetRelation)).getOrElse(
+Subquery(r.tableName, parquetRelation))
+
+withAlias
+ }
case other => other.transformExpressions {
case a: Attribute if a.resolved => attributedRewrites.getOrElse(a,
a)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f3ff1eb2/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
--
diff --git
a/sql/hive/src/test/scala/o
spark git commit: [SPARK-5839][SQL]HiveMetastoreCatalog does not recognize table names and aliases of data source tables.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 419865475 -> a15a0a02c
[SPARK-5839][SQL]HiveMetastoreCatalog does not recognize table names and
aliases of data source tables.
JIRA: https://issues.apache.org/jira/browse/SPARK-5839
Author: Yin Huai
Closes #4626 from yhuai/SPARK-5839 and squashes the following commits:
f779d85 [Yin Huai] Use subqeury to wrap replaced ParquetRelation.
2695f13 [Yin Huai] Merge remote-tracking branch 'upstream/master' into
SPARK-5839
f1ba6ca [Yin Huai] Address comment.
2c7fa08 [Yin Huai] Use Subqueries to wrap a data source table.
(cherry picked from commit f3ff1eb2985ff3e1567645b898f6b42e4b01f237)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a15a0a02
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a15a0a02
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a15a0a02
Branch: refs/heads/branch-1.3
Commit: a15a0a02c654cd70bb84f6d610e1c178c9cb98a6
Parents: 4198654
Author: Yin Huai
Authored: Mon Feb 16 15:54:01 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 15:54:13 2015 -0800
--
.../spark/sql/parquet/ParquetQuerySuite.scala | 5 +--
.../spark/sql/hive/HiveMetastoreCatalog.scala | 18 +--
.../sql/hive/MetastoreDataSourcesSuite.scala| 34
3 files changed, 53 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a15a0a02/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
index 9318c15..8b4d05e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetQuerySuite.scala
@@ -30,9 +30,10 @@ import org.apache.spark.sql.test.TestSQLContext._
class ParquetQuerySuiteBase extends QueryTest with ParquetTest {
val sqlContext = TestSQLContext
- test("simple projection") {
+ test("simple select queries") {
withParquetTable((0 until 10).map(i => (i, i.toString)), "t") {
- checkAnswer(sql("SELECT _1 FROM t"), (0 until 10).map(Row.apply(_)))
+ checkAnswer(sql("SELECT _1 FROM t where t._1 > 5"), (6 until
10).map(Row.apply(_)))
+ checkAnswer(sql("SELECT _1 FROM t as tmp where tmp._1 < 5"), (0 until
5).map(Row.apply(_)))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a15a0a02/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 72211fe..87bc9fe 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -160,7 +160,15 @@ private[hive] class HiveMetastoreCatalog(hive:
HiveContext) extends Catalog with
}
if (table.getProperty("spark.sql.sources.provider") != null) {
- cachedDataSourceTables(QualifiedTableName(databaseName,
tblName).toLowerCase)
+ val dataSourceTable =
+cachedDataSourceTables(QualifiedTableName(databaseName,
tblName).toLowerCase)
+ // Then, if alias is specified, wrap the table with a Subquery using the
alias.
+ // Othersie, wrap the table with a Subquery using the table name.
+ val withAlias =
+alias.map(a => Subquery(a, dataSourceTable)).getOrElse(
+ Subquery(tableIdent.last, dataSourceTable))
+
+ withAlias
} else if (table.isView) {
// if the unresolved relation is from hive view
// parse the text into logic node.
@@ -433,7 +441,13 @@ private[hive] class HiveMetastoreCatalog(hive:
HiveContext) extends Catalog with
val attributedRewrites =
AttributeMap(relation.output.zip(parquetRelation.output))
lastPlan.transformUp {
- case r: MetastoreRelation if r == relation => parquetRelation
+ case r: MetastoreRelation if r == relation => {
+val withAlias =
+ r.alias.map(a => Subquery(a, parquetRelation)).getOrElse(
+Subquery(r.tableName, parquetRelation))
+
+withAlias
+ }
case other => other.transformExpressions {
case a: Attribute if a.resolved => attributedRewrites.getOrElse(a,
a)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a15a0a02/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
spark git commit: [SQL] Optimize arithmetic and predicate operators
Repository: spark Updated Branches: refs/heads/master f3ff1eb29 -> cb6c48c87 [SQL] Optimize arithmetic and predicate operators Existing implementation of arithmetic operators and BinaryComparison operators have redundant type checking codes, e.g.: Expression.n2 is used by Add/Subtract/Multiply. (1) n2 always checks left.dataType == right.dataType. However, this checking should be done once when we resolve expression types; (2) n2 requires dataType is a NumericType. This can be done once. This PR optimizes arithmetic and predicate operators by removing such redundant type-checking codes. Some preliminary benchmarking on 10G TPC-H data over 5 r3.2xlarge EC2 machines shows that this PR can reduce the query time by 5.5% to 11%. The benchmark queries follow the template below, where OP is plus/minus/times/divide/remainder/bitwise and/bitwise or/bitwise xor. SELECT l_returnflag, l_linestatus, SUM(l_quantity OP cnt1), SUM(l_quantity OP cnt2), , SUM(l_quantity OP cnt700) FROM ( SELECT l_returnflag, l_linestatus, l_quantity, 1 AS cnt1, 2 AS cnt2, ..., 700 AS cnt700 FROM lineitem WHERE l_shipdate <= '1998-09-01' ) GROUP BY l_returnflag, l_linestatus; Author: kai Closes #4472 from kai-zeng/arithmetic-optimize and squashes the following commits: fef0cf1 [kai] Merge branch 'master' of github.com:apache/spark into arithmetic-optimize 4b3a1bb [kai] chmod a-x 5a41e49 [kai] chmod a-x Expression.scala cb37c94 [kai] rebase onto spark master 7f6e968 [kai] chmod 100755 -> 100644 6cddb46 [kai] format 7490dbc [kai] fix unresolved-expression exception for EqualTo 9c40bc0 [kai] fix bitwisenot 3cbd363 [kai] clean up test code ca47801 [kai] override evalInternal for bitwise ops 8fa84a1 [kai] add bitwise or and xor 6892fc4 [kai] revert override evalInternal f8eba24 [kai] override evalInternal 31ccdd4 [kai] rewrite all bitwise op and remove evalInternal 86297e2 [kai] generalized cb92ae1 [kai] bitwise-and: override eval 97a7d6c [kai] bitwise-and: override evalInternal using and func 0906c39 [kai] add bitwise test 62abbbc [kai] clean up predicate and arithmetic b34d58d [kai] add caching and benmark option 12c5b32 [kai] override eval 1cd7571 [kai] fix sqrt and maxof 03fd0c3 [kai] fix predicate 16fd84c [kai] optimize + - * / % -(unary) abs < > <= >= fd95823 [kai] remove unnecessary type checking 24d062f [kai] test suite Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cb6c48c8 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cb6c48c8 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cb6c48c8 Branch: refs/heads/master Commit: cb6c48c874af2bd78ee73c1dc8a44fd28ecc0991 Parents: f3ff1eb Author: kai Authored: Mon Feb 16 15:58:05 2015 -0800 Committer: Michael Armbrust Committed: Mon Feb 16 15:58:05 2015 -0800 -- .../apache/spark/sql/catalyst/SqlParser.scala | 0 .../apache/spark/sql/catalyst/dsl/package.scala | 0 .../sql/catalyst/expressions/Expression.scala | 200 --- .../sql/catalyst/expressions/aggregates.scala | 0 .../sql/catalyst/expressions/arithmetic.scala | 243 ++- .../sql/catalyst/expressions/predicates.scala | 107 +++- .../org/apache/spark/sql/types/Metadata.scala | 0 .../spark/sql/catalyst/util/MetadataSuite.scala | 0 .../apache/spark/sql/execution/Aggregate.scala | 0 .../hive/thriftserver/SparkSQLCLIDriver.scala | 0 10 files changed, 290 insertions(+), 260 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/cb6c48c8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala old mode 100755 new mode 100644 http://git-wip-us.apache.org/repos/asf/spark/blob/cb6c48c8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala old mode 100755 new mode 100644 http://git-wip-us.apache.org/repos/asf/spark/blob/cb6c48c8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala index c32a4b8..6ad39b8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala +++ b/sql/cata
spark git commit: [SQL] Optimize arithmetic and predicate operators
Repository: spark Updated Branches: refs/heads/branch-1.3 a15a0a02c -> 639a3c2fd [SQL] Optimize arithmetic and predicate operators Existing implementation of arithmetic operators and BinaryComparison operators have redundant type checking codes, e.g.: Expression.n2 is used by Add/Subtract/Multiply. (1) n2 always checks left.dataType == right.dataType. However, this checking should be done once when we resolve expression types; (2) n2 requires dataType is a NumericType. This can be done once. This PR optimizes arithmetic and predicate operators by removing such redundant type-checking codes. Some preliminary benchmarking on 10G TPC-H data over 5 r3.2xlarge EC2 machines shows that this PR can reduce the query time by 5.5% to 11%. The benchmark queries follow the template below, where OP is plus/minus/times/divide/remainder/bitwise and/bitwise or/bitwise xor. SELECT l_returnflag, l_linestatus, SUM(l_quantity OP cnt1), SUM(l_quantity OP cnt2), , SUM(l_quantity OP cnt700) FROM ( SELECT l_returnflag, l_linestatus, l_quantity, 1 AS cnt1, 2 AS cnt2, ..., 700 AS cnt700 FROM lineitem WHERE l_shipdate <= '1998-09-01' ) GROUP BY l_returnflag, l_linestatus; Author: kai Closes #4472 from kai-zeng/arithmetic-optimize and squashes the following commits: fef0cf1 [kai] Merge branch 'master' of github.com:apache/spark into arithmetic-optimize 4b3a1bb [kai] chmod a-x 5a41e49 [kai] chmod a-x Expression.scala cb37c94 [kai] rebase onto spark master 7f6e968 [kai] chmod 100755 -> 100644 6cddb46 [kai] format 7490dbc [kai] fix unresolved-expression exception for EqualTo 9c40bc0 [kai] fix bitwisenot 3cbd363 [kai] clean up test code ca47801 [kai] override evalInternal for bitwise ops 8fa84a1 [kai] add bitwise or and xor 6892fc4 [kai] revert override evalInternal f8eba24 [kai] override evalInternal 31ccdd4 [kai] rewrite all bitwise op and remove evalInternal 86297e2 [kai] generalized cb92ae1 [kai] bitwise-and: override eval 97a7d6c [kai] bitwise-and: override evalInternal using and func 0906c39 [kai] add bitwise test 62abbbc [kai] clean up predicate and arithmetic b34d58d [kai] add caching and benmark option 12c5b32 [kai] override eval 1cd7571 [kai] fix sqrt and maxof 03fd0c3 [kai] fix predicate 16fd84c [kai] optimize + - * / % -(unary) abs < > <= >= fd95823 [kai] remove unnecessary type checking 24d062f [kai] test suite (cherry picked from commit cb6c48c874af2bd78ee73c1dc8a44fd28ecc0991) Signed-off-by: Michael Armbrust Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/639a3c2f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/639a3c2f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/639a3c2f Branch: refs/heads/branch-1.3 Commit: 639a3c2fdf302a34ad5f62199abee2b96023669b Parents: a15a0a0 Author: kai Authored: Mon Feb 16 15:58:05 2015 -0800 Committer: Michael Armbrust Committed: Mon Feb 16 15:58:18 2015 -0800 -- .../apache/spark/sql/catalyst/SqlParser.scala | 0 .../apache/spark/sql/catalyst/dsl/package.scala | 0 .../sql/catalyst/expressions/Expression.scala | 200 --- .../sql/catalyst/expressions/aggregates.scala | 0 .../sql/catalyst/expressions/arithmetic.scala | 243 ++- .../sql/catalyst/expressions/predicates.scala | 107 +++- .../org/apache/spark/sql/types/Metadata.scala | 0 .../spark/sql/catalyst/util/MetadataSuite.scala | 0 .../apache/spark/sql/execution/Aggregate.scala | 0 .../hive/thriftserver/SparkSQLCLIDriver.scala | 0 10 files changed, 290 insertions(+), 260 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/639a3c2f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala old mode 100755 new mode 100644 http://git-wip-us.apache.org/repos/asf/spark/blob/639a3c2f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala old mode 100755 new mode 100644 http://git-wip-us.apache.org/repos/asf/spark/blob/639a3c2f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala index c32a4b8..6ad39b8 10064
spark git commit: [SPARK-4865][SQL]Include temporary tables in SHOW TABLES
Repository: spark
Updated Branches:
refs/heads/master cb6c48c87 -> e189cbb05
[SPARK-4865][SQL]Include temporary tables in SHOW TABLES
This PR adds a `ShowTablesCommand` to support `SHOW TABLES [IN databaseName]`
SQL command. The result of `SHOW TABLE` has two columns, `tableName` and
`isTemporary`. For temporary tables, the value of `isTemporary` column will be
`false`.
JIRA: https://issues.apache.org/jira/browse/SPARK-4865
Author: Yin Huai
Closes #4618 from yhuai/showTablesCommand and squashes the following commits:
0c09791 [Yin Huai] Use ShowTablesCommand.
85ee76d [Yin Huai] Since SHOW TABLES is not a Hive native command any more and
we will not see "OK" (originally generated by Hive's driver), use SHOW
DATABASES in the test.
94bacac [Yin Huai] Add SHOW TABLES to the list of noExplainCommands.
d71ed09 [Yin Huai] Fix test.
a4a6ec3 [Yin Huai] Add SHOW TABLE command.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e189cbb0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e189cbb0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e189cbb0
Branch: refs/heads/master
Commit: e189cbb052d59eb499dd4312403925fdd72f5718
Parents: cb6c48c
Author: Yin Huai
Authored: Mon Feb 16 15:59:23 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 15:59:23 2015 -0800
--
.../scala/org/apache/spark/sql/SQLContext.scala | 4 +-
.../org/apache/spark/sql/SparkSQLParser.scala | 12 -
.../apache/spark/sql/execution/commands.scala | 34 ++-
.../org/apache/spark/sql/ListTablesSuite.scala | 37 ++--
.../spark/sql/hive/thriftserver/CliSuite.scala | 2 +-
.../spark/sql/hive/HiveMetastoreCatalog.scala | 10 -
.../org/apache/spark/sql/hive/HiveQl.scala | 2 +-
.../apache/spark/sql/hive/ListTablesSuite.scala | 46 +++-
.../sql/hive/execution/HiveQuerySuite.scala | 14 +++---
9 files changed, 111 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e189cbb0/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index d08c2d1..aa0fc3e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -794,7 +794,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
* indicating if a table is a temporary one or not).
*/
def tables(): DataFrame = {
-createDataFrame(catalog.getTables(None)).toDF("tableName", "isTemporary")
+DataFrame(this, ShowTablesCommand(None))
}
/**
@@ -803,7 +803,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
* indicating if a table is a temporary one or not).
*/
def tables(databaseName: String): DataFrame = {
-createDataFrame(catalog.getTables(Some(databaseName))).toDF("tableName",
"isTemporary")
+DataFrame(this, ShowTablesCommand(Some(databaseName)))
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/e189cbb0/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
index f1a4053..00e19da 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
@@ -23,7 +23,7 @@ import scala.util.parsing.combinator.RegexParsers
import org.apache.spark.sql.catalyst.AbstractSparkSQLParser
import org.apache.spark.sql.catalyst.expressions.{Attribute,
AttributeReference}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.{UncacheTableCommand, CacheTableCommand,
SetCommand}
+import org.apache.spark.sql.execution.{ShowTablesCommand, UncacheTableCommand,
CacheTableCommand, SetCommand}
import org.apache.spark.sql.types.StringType
@@ -57,12 +57,15 @@ private[sql] class SparkSQLParser(fallback: String =>
LogicalPlan) extends Abstr
protected val AS = Keyword("AS")
protected val CACHE = Keyword("CACHE")
+ protected val IN = Keyword("IN")
protected val LAZY= Keyword("LAZY")
protected val SET = Keyword("SET")
+ protected val SHOW= Keyword("SHOW")
protected val TABLE = Keyword("TABLE")
+ protected val TABLES = Keyword("TABLES")
protected val UNCACHE = Keyword("UNCACHE")
- override protected lazy val start: Parser[LogicalPlan] = cache | uncache |
set | others
+ override protected lazy val start: Parser[L
spark git commit: [SPARK-4865][SQL]Include temporary tables in SHOW TABLES
Repository: spark
Updated Branches:
refs/heads/branch-1.3 639a3c2fd -> 8a94bf76b
[SPARK-4865][SQL]Include temporary tables in SHOW TABLES
This PR adds a `ShowTablesCommand` to support `SHOW TABLES [IN databaseName]`
SQL command. The result of `SHOW TABLE` has two columns, `tableName` and
`isTemporary`. For temporary tables, the value of `isTemporary` column will be
`false`.
JIRA: https://issues.apache.org/jira/browse/SPARK-4865
Author: Yin Huai
Closes #4618 from yhuai/showTablesCommand and squashes the following commits:
0c09791 [Yin Huai] Use ShowTablesCommand.
85ee76d [Yin Huai] Since SHOW TABLES is not a Hive native command any more and
we will not see "OK" (originally generated by Hive's driver), use SHOW
DATABASES in the test.
94bacac [Yin Huai] Add SHOW TABLES to the list of noExplainCommands.
d71ed09 [Yin Huai] Fix test.
a4a6ec3 [Yin Huai] Add SHOW TABLE command.
(cherry picked from commit e189cbb052d59eb499dd4312403925fdd72f5718)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8a94bf76
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8a94bf76
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8a94bf76
Branch: refs/heads/branch-1.3
Commit: 8a94bf76b883ec59b6142fab781a05c1ee985262
Parents: 639a3c2
Author: Yin Huai
Authored: Mon Feb 16 15:59:23 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 16 15:59:41 2015 -0800
--
.../scala/org/apache/spark/sql/SQLContext.scala | 4 +-
.../org/apache/spark/sql/SparkSQLParser.scala | 12 -
.../apache/spark/sql/execution/commands.scala | 34 ++-
.../org/apache/spark/sql/ListTablesSuite.scala | 37 ++--
.../spark/sql/hive/thriftserver/CliSuite.scala | 2 +-
.../spark/sql/hive/HiveMetastoreCatalog.scala | 10 -
.../org/apache/spark/sql/hive/HiveQl.scala | 2 +-
.../apache/spark/sql/hive/ListTablesSuite.scala | 46 +++-
.../sql/hive/execution/HiveQuerySuite.scala | 14 +++---
9 files changed, 111 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/8a94bf76/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index d08c2d1..aa0fc3e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -794,7 +794,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
* indicating if a table is a temporary one or not).
*/
def tables(): DataFrame = {
-createDataFrame(catalog.getTables(None)).toDF("tableName", "isTemporary")
+DataFrame(this, ShowTablesCommand(None))
}
/**
@@ -803,7 +803,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
* indicating if a table is a temporary one or not).
*/
def tables(databaseName: String): DataFrame = {
-createDataFrame(catalog.getTables(Some(databaseName))).toDF("tableName",
"isTemporary")
+DataFrame(this, ShowTablesCommand(Some(databaseName)))
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/8a94bf76/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
index f1a4053..00e19da 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
@@ -23,7 +23,7 @@ import scala.util.parsing.combinator.RegexParsers
import org.apache.spark.sql.catalyst.AbstractSparkSQLParser
import org.apache.spark.sql.catalyst.expressions.{Attribute,
AttributeReference}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.{UncacheTableCommand, CacheTableCommand,
SetCommand}
+import org.apache.spark.sql.execution.{ShowTablesCommand, UncacheTableCommand,
CacheTableCommand, SetCommand}
import org.apache.spark.sql.types.StringType
@@ -57,12 +57,15 @@ private[sql] class SparkSQLParser(fallback: String =>
LogicalPlan) extends Abstr
protected val AS = Keyword("AS")
protected val CACHE = Keyword("CACHE")
+ protected val IN = Keyword("IN")
protected val LAZY= Keyword("LAZY")
protected val SET = Keyword("SET")
+ protected val SHOW= Keyword("SHOW")
protected val TABLE = Keyword("TABLE")
+ protected val TABLES = Keyword("TABLES")
protected val UNCACHE = Keyword("UNCACHE")
- override protected lazy
[2/2] spark git commit: [SPARK-5166][SPARK-5247][SPARK-5258][SQL] API Cleanup / Documentation
[SPARK-5166][SPARK-5247][SPARK-5258][SQL] API Cleanup / Documentation
Author: Michael Armbrust
Closes #4642 from marmbrus/docs and squashes the following commits:
d291c34 [Michael Armbrust] python tests
9be66e3 [Michael Armbrust] comments
d56afc2 [Michael Armbrust] fix style
f004747 [Michael Armbrust] fix build
c4a907b [Michael Armbrust] fix tests
42e2b73 [Michael Armbrust] [SQL] Documentation / API Clean-up.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c74b07fa
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c74b07fa
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c74b07fa
Branch: refs/heads/master
Commit: c74b07fa94a8da50437d952ae05cf6ac70fbb93e
Parents: c76da36
Author: Michael Armbrust
Authored: Tue Feb 17 10:21:17 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 10:21:17 2015 -0800
--
project/SparkBuild.scala| 12 +-
python/pyspark/sql/context.py | 28 +--
.../org/apache/spark/sql/jdbc/JDBCUtils.java| 59 --
.../scala/org/apache/spark/sql/DataFrame.scala | 153 +-
.../org/apache/spark/sql/DataFrameImpl.scala| 33 ++-
.../apache/spark/sql/ExperimentalMethods.scala | 5 +
.../apache/spark/sql/IncomputableColumn.scala | 4 +
.../scala/org/apache/spark/sql/SQLContext.scala | 200 +++
.../apache/spark/sql/UserDefinedFunction.scala | 3 +-
.../org/apache/spark/sql/api/package.scala | 23 +++
.../apache/spark/sql/execution/commands.scala | 2 +-
.../spark/sql/execution/debug/package.scala | 10 +-
.../spark/sql/jdbc/JavaJDBCTrampoline.scala | 30 ---
.../scala/org/apache/spark/sql/jdbc/jdbc.scala | 74 ++-
.../sql/parquet/ParquetTableOperations.scala| 4 +-
.../apache/spark/sql/parquet/ParquetTest.scala | 4 +-
.../apache/spark/sql/parquet/newParquet.scala | 6 +-
.../spark/sql/parquet/timestamp/NanoTime.scala | 2 +-
.../org/apache/spark/sql/sources/ddl.scala | 4 +-
.../org/apache/spark/sql/jdbc/JavaJDBCTest.java | 102 --
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 7 +-
.../apache/spark/sql/jdbc/JDBCWriteSuite.scala | 20 +-
.../spark/sql/jdbc/MySQLIntegration.scala | 14 +-
.../spark/sql/jdbc/PostgresIntegration.scala| 4 +-
.../execution/DescribeHiveTableCommand.scala| 4 +-
.../spark/sql/hive/execution/commands.scala | 8 +
.../spark/sql/hive/execution/package.scala | 25 +++
.../org/apache/spark/sql/hive/package.scala | 10 +
.../sql/hive/parquet/FakeParquetSerDe.scala | 56 --
.../org/apache/spark/sql/hive/Shim12.scala | 9 +-
.../org/apache/spark/sql/hive/Shim13.scala | 9 +-
31 files changed, 501 insertions(+), 423 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c74b07fa/project/SparkBuild.scala
--
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 8fb1239..e4b1b96 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -361,9 +361,16 @@ object Unidoc {
publish := {},
unidocProjectFilter in(ScalaUnidoc, unidoc) :=
- inAnyProject -- inProjects(OldDeps.project, repl, examples, tools,
catalyst, streamingFlumeSink, yarn),
+ inAnyProject -- inProjects(OldDeps.project, repl, examples, tools,
streamingFlumeSink, yarn),
unidocProjectFilter in(JavaUnidoc, unidoc) :=
- inAnyProject -- inProjects(OldDeps.project, repl, bagel, examples,
tools, catalyst, streamingFlumeSink, yarn),
+ inAnyProject -- inProjects(OldDeps.project, repl, bagel, examples,
tools, streamingFlumeSink, yarn),
+
+// Skip actual catalyst, but include the subproject.
+// Catalyst is not public API and contains quasiquotes which break
scaladoc.
+unidocAllSources in (ScalaUnidoc, unidoc) := {
+ (unidocAllSources in (ScalaUnidoc, unidoc)).value
+.map(_.filterNot(_.getCanonicalPath.contains("sql/catalyst")))
+},
// Skip class names containing $ and some internal packages in Javadocs
unidocAllSources in (JavaUnidoc, unidoc) := {
@@ -376,6 +383,7 @@ object Unidoc {
.map(_.filterNot(_.getCanonicalPath.contains("executor")))
.map(_.filterNot(_.getCanonicalPath.contains("python")))
.map(_.filterNot(_.getCanonicalPath.contains("collection")))
+.map(_.filterNot(_.getCanonicalPath.contains("sql/catalyst")))
},
// Javadoc options: create a window title, and group key packages on index
page
http://git-wip-us.apache.org/repos/asf/spark/blob/c74b07fa/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/contex
spark git commit: [SPARK-5859] [PySpark] [SQL] fix DataFrame Python API
Repository: spark
Updated Branches:
refs/heads/master c74b07fa9 -> d8adefefc
[SPARK-5859] [PySpark] [SQL] fix DataFrame Python API
1. added explain()
2. add isLocal()
3. do not call show() in __repl__
4. add foreach() and foreachPartition()
5. add distinct()
6. fix functions.col()/column()/lit()
7. fix unit tests in sql/functions.py
8. fix unicode in showString()
Author: Davies Liu
Closes #4645 from davies/df6 and squashes the following commits:
6b46a2c [Davies Liu] fix DataFrame Python API
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d8adefef
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d8adefef
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d8adefef
Branch: refs/heads/master
Commit: d8adefefcc2a4af32295440ed1d4917a6968f017
Parents: c74b07f
Author: Davies Liu
Authored: Tue Feb 17 10:22:48 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 10:22:48 2015 -0800
--
python/pyspark/sql/dataframe.py | 65 ++--
python/pyspark/sql/functions.py | 12 +++
2 files changed, 59 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d8adefef/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 28a59e7..8417240 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -238,6 +238,22 @@ class DataFrame(object):
"""
print (self._jdf.schema().treeString())
+def explain(self, extended=False):
+"""
+Prints the plans (logical and physical) to the console for
+debugging purpose.
+
+If extended is False, only prints the physical plan.
+"""
+self._jdf.explain(extended)
+
+def isLocal(self):
+"""
+Returns True if the `collect` and `take` methods can be run locally
+(without any Spark executors).
+"""
+return self._jdf.isLocal()
+
def show(self):
"""
Print the first 20 rows.
@@ -247,14 +263,12 @@ class DataFrame(object):
2 Alice
5 Bob
>>> df
-age name
-2 Alice
-5 Bob
+DataFrame[age: int, name: string]
"""
-print (self)
+print self._jdf.showString().encode('utf8', 'ignore')
def __repr__(self):
-return self._jdf.showString()
+return "DataFrame[%s]" % (", ".join("%s: %s" % c for c in self.dtypes))
def count(self):
"""Return the number of elements in this RDD.
@@ -336,6 +350,8 @@ class DataFrame(object):
"""
Return a new RDD by applying a function to each partition.
+It's a shorthand for df.rdd.mapPartitions()
+
>>> rdd = sc.parallelize([1, 2, 3, 4], 4)
>>> def f(iterator): yield 1
>>> rdd.mapPartitions(f).sum()
@@ -343,6 +359,31 @@ class DataFrame(object):
"""
return self.rdd.mapPartitions(f, preservesPartitioning)
+def foreach(self, f):
+"""
+Applies a function to all rows of this DataFrame.
+
+It's a shorthand for df.rdd.foreach()
+
+>>> def f(person):
+... print person.name
+>>> df.foreach(f)
+"""
+return self.rdd.foreach(f)
+
+def foreachPartition(self, f):
+"""
+Applies a function to each partition of this DataFrame.
+
+It's a shorthand for df.rdd.foreachPartition()
+
+>>> def f(people):
+... for person in people:
+... print person.name
+>>> df.foreachPartition(f)
+"""
+return self.rdd.foreachPartition(f)
+
def cache(self):
""" Persist with the default storage level (C{MEMORY_ONLY_SER}).
"""
@@ -377,8 +418,13 @@ class DataFrame(object):
""" Return a new :class:`DataFrame` that has exactly `numPartitions`
partitions.
"""
-rdd = self._jdf.repartition(numPartitions, None)
-return DataFrame(rdd, self.sql_ctx)
+return DataFrame(self._jdf.repartition(numPartitions, None),
self.sql_ctx)
+
+def distinct(self):
+"""
+Return a new :class:`DataFrame` containing the distinct rows in this
DataFrame.
+"""
+return DataFrame(self._jdf.distinct(), self.sql_ctx)
def sample(self, withReplacement, fraction, seed=None):
"""
@@ -957,10 +1003,7 @@ class Column(DataFrame):
return Column(jc, self.sql_ctx)
def __repr__(self):
-if self._jdf.isComputable():
-return self._jdf.samples()
-else:
-return 'Column<%s>' % self._jdf.toString()
+return 'Column<%s>' % self._jdf.toString().encode('utf8')
[1/2] spark git commit: [SPARK-5166][SPARK-5247][SPARK-5258][SQL] API Cleanup / Documentation
Repository: spark
Updated Branches:
refs/heads/master c76da36c2 -> c74b07fa9
http://git-wip-us.apache.org/repos/asf/spark/blob/c74b07fa/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
index 89920f2..4f38110 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
@@ -143,7 +143,7 @@ class MySQLDatabase {
}
test("Basic test") {
-val rdd = TestSQLContext.jdbcRDD(url(ip, "foo"), "tbl")
+val rdd = TestSQLContext.jdbc(url(ip, "foo"), "tbl")
val rows = rdd.collect
assert(rows.length == 2)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -153,7 +153,7 @@ class MySQLDatabase {
}
test("Numeric types") {
-val rdd = TestSQLContext.jdbcRDD(url(ip, "foo"), "numbers")
+val rdd = TestSQLContext.jdbc(url(ip, "foo"), "numbers")
val rows = rdd.collect
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -181,7 +181,7 @@ class MySQLDatabase {
}
test("Date types") {
-val rdd = TestSQLContext.jdbcRDD(url(ip, "foo"), "dates")
+val rdd = TestSQLContext.jdbc(url(ip, "foo"), "dates")
val rows = rdd.collect
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -199,7 +199,7 @@ class MySQLDatabase {
}
test("String types") {
-val rdd = TestSQLContext.jdbcRDD(url(ip, "foo"), "strings")
+val rdd = TestSQLContext.jdbc(url(ip, "foo"), "strings")
val rows = rdd.collect
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -225,9 +225,9 @@ class MySQLDatabase {
}
test("Basic write test") {
-val rdd1 = TestSQLContext.jdbcRDD(url(ip, "foo"), "numbers")
-val rdd2 = TestSQLContext.jdbcRDD(url(ip, "foo"), "dates")
-val rdd3 = TestSQLContext.jdbcRDD(url(ip, "foo"), "strings")
+val rdd1 = TestSQLContext.jdbc(url(ip, "foo"), "numbers")
+val rdd2 = TestSQLContext.jdbc(url(ip, "foo"), "dates")
+val rdd3 = TestSQLContext.jdbc(url(ip, "foo"), "strings")
rdd1.createJDBCTable(url(ip, "foo"), "numberscopy", false)
rdd2.createJDBCTable(url(ip, "foo"), "datescopy", false)
rdd3.createJDBCTable(url(ip, "foo"), "stringscopy", false)
http://git-wip-us.apache.org/repos/asf/spark/blob/c74b07fa/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
index c174d7a..7b47fee 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
@@ -113,7 +113,7 @@ class PostgresDatabase {
}
test("Type mapping for various types") {
-val rdd = TestSQLContext.jdbcRDD(url(db.ip), "public.bar")
+val rdd = TestSQLContext.jdbc(url(db.ip), "public.bar")
val rows = rdd.collect
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -142,7 +142,7 @@ class PostgresDatabase {
}
test("Basic write test") {
-val rdd = TestSQLContext.jdbcRDD(url(db.ip), "public.bar")
+val rdd = TestSQLContext.jdbc(url(db.ip), "public.bar")
rdd.createJDBCTable(url(db.ip), "public.barcopy", false)
// Test only that it doesn't bomb out.
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c74b07fa/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
index bfacc51..07b5a84 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
@@ -29,9 +29,9 @@ import org.apache.spark.sql.hive.HiveShim
import org.apache.spark.sql.SQLContext
/**
- * Implementation for "describe [extended] table".
- *
* :: DeveloperApi ::
+ *
+ * Implementation for "describe [extended] table".
*/
@DeveloperApi
case class DescribeHiveTableCommand(
http://git-wip-us.apache.org/repos/asf/spark/blob/c74b07fa/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/
[2/2] spark git commit: [SPARK-5166][SPARK-5247][SPARK-5258][SQL] API Cleanup / Documentation
[SPARK-5166][SPARK-5247][SPARK-5258][SQL] API Cleanup / Documentation
Author: Michael Armbrust
Closes #4642 from marmbrus/docs and squashes the following commits:
d291c34 [Michael Armbrust] python tests
9be66e3 [Michael Armbrust] comments
d56afc2 [Michael Armbrust] fix style
f004747 [Michael Armbrust] fix build
c4a907b [Michael Armbrust] fix tests
42e2b73 [Michael Armbrust] [SQL] Documentation / API Clean-up.
(cherry picked from commit c74b07fa94a8da50437d952ae05cf6ac70fbb93e)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cd3d4158
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cd3d4158
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cd3d4158
Branch: refs/heads/branch-1.3
Commit: cd3d4158721b5c3cc35df47675f4f4d9540be6f1
Parents: 97cb568
Author: Michael Armbrust
Authored: Tue Feb 17 10:21:17 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 10:21:33 2015 -0800
--
project/SparkBuild.scala| 12 +-
python/pyspark/sql/context.py | 28 +--
.../org/apache/spark/sql/jdbc/JDBCUtils.java| 59 --
.../scala/org/apache/spark/sql/DataFrame.scala | 153 +-
.../org/apache/spark/sql/DataFrameImpl.scala| 33 ++-
.../apache/spark/sql/ExperimentalMethods.scala | 5 +
.../apache/spark/sql/IncomputableColumn.scala | 4 +
.../scala/org/apache/spark/sql/SQLContext.scala | 200 +++
.../apache/spark/sql/UserDefinedFunction.scala | 3 +-
.../org/apache/spark/sql/api/package.scala | 23 +++
.../apache/spark/sql/execution/commands.scala | 2 +-
.../spark/sql/execution/debug/package.scala | 10 +-
.../spark/sql/jdbc/JavaJDBCTrampoline.scala | 30 ---
.../scala/org/apache/spark/sql/jdbc/jdbc.scala | 74 ++-
.../sql/parquet/ParquetTableOperations.scala| 4 +-
.../apache/spark/sql/parquet/ParquetTest.scala | 4 +-
.../apache/spark/sql/parquet/newParquet.scala | 6 +-
.../spark/sql/parquet/timestamp/NanoTime.scala | 2 +-
.../org/apache/spark/sql/sources/ddl.scala | 4 +-
.../org/apache/spark/sql/jdbc/JavaJDBCTest.java | 102 --
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 7 +-
.../apache/spark/sql/jdbc/JDBCWriteSuite.scala | 20 +-
.../spark/sql/jdbc/MySQLIntegration.scala | 14 +-
.../spark/sql/jdbc/PostgresIntegration.scala| 4 +-
.../execution/DescribeHiveTableCommand.scala| 4 +-
.../spark/sql/hive/execution/commands.scala | 8 +
.../spark/sql/hive/execution/package.scala | 25 +++
.../org/apache/spark/sql/hive/package.scala | 10 +
.../sql/hive/parquet/FakeParquetSerDe.scala | 56 --
.../org/apache/spark/sql/hive/Shim12.scala | 9 +-
.../org/apache/spark/sql/hive/Shim13.scala | 9 +-
31 files changed, 501 insertions(+), 423 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/cd3d4158/project/SparkBuild.scala
--
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 8fb1239..e4b1b96 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -361,9 +361,16 @@ object Unidoc {
publish := {},
unidocProjectFilter in(ScalaUnidoc, unidoc) :=
- inAnyProject -- inProjects(OldDeps.project, repl, examples, tools,
catalyst, streamingFlumeSink, yarn),
+ inAnyProject -- inProjects(OldDeps.project, repl, examples, tools,
streamingFlumeSink, yarn),
unidocProjectFilter in(JavaUnidoc, unidoc) :=
- inAnyProject -- inProjects(OldDeps.project, repl, bagel, examples,
tools, catalyst, streamingFlumeSink, yarn),
+ inAnyProject -- inProjects(OldDeps.project, repl, bagel, examples,
tools, streamingFlumeSink, yarn),
+
+// Skip actual catalyst, but include the subproject.
+// Catalyst is not public API and contains quasiquotes which break
scaladoc.
+unidocAllSources in (ScalaUnidoc, unidoc) := {
+ (unidocAllSources in (ScalaUnidoc, unidoc)).value
+.map(_.filterNot(_.getCanonicalPath.contains("sql/catalyst")))
+},
// Skip class names containing $ and some internal packages in Javadocs
unidocAllSources in (JavaUnidoc, unidoc) := {
@@ -376,6 +383,7 @@ object Unidoc {
.map(_.filterNot(_.getCanonicalPath.contains("executor")))
.map(_.filterNot(_.getCanonicalPath.contains("python")))
.map(_.filterNot(_.getCanonicalPath.contains("collection")))
+.map(_.filterNot(_.getCanonicalPath.contains("sql/catalyst")))
},
// Javadoc options: create a window title, and group key packages on index
page
http://git-wip-us.apache.org/repos/asf
[1/2] spark git commit: [SPARK-5166][SPARK-5247][SPARK-5258][SQL] API Cleanup / Documentation
Repository: spark
Updated Branches:
refs/heads/branch-1.3 97cb568a2 -> cd3d41587
http://git-wip-us.apache.org/repos/asf/spark/blob/cd3d4158/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
index 89920f2..4f38110 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegration.scala
@@ -143,7 +143,7 @@ class MySQLDatabase {
}
test("Basic test") {
-val rdd = TestSQLContext.jdbcRDD(url(ip, "foo"), "tbl")
+val rdd = TestSQLContext.jdbc(url(ip, "foo"), "tbl")
val rows = rdd.collect
assert(rows.length == 2)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -153,7 +153,7 @@ class MySQLDatabase {
}
test("Numeric types") {
-val rdd = TestSQLContext.jdbcRDD(url(ip, "foo"), "numbers")
+val rdd = TestSQLContext.jdbc(url(ip, "foo"), "numbers")
val rows = rdd.collect
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -181,7 +181,7 @@ class MySQLDatabase {
}
test("Date types") {
-val rdd = TestSQLContext.jdbcRDD(url(ip, "foo"), "dates")
+val rdd = TestSQLContext.jdbc(url(ip, "foo"), "dates")
val rows = rdd.collect
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -199,7 +199,7 @@ class MySQLDatabase {
}
test("String types") {
-val rdd = TestSQLContext.jdbcRDD(url(ip, "foo"), "strings")
+val rdd = TestSQLContext.jdbc(url(ip, "foo"), "strings")
val rows = rdd.collect
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -225,9 +225,9 @@ class MySQLDatabase {
}
test("Basic write test") {
-val rdd1 = TestSQLContext.jdbcRDD(url(ip, "foo"), "numbers")
-val rdd2 = TestSQLContext.jdbcRDD(url(ip, "foo"), "dates")
-val rdd3 = TestSQLContext.jdbcRDD(url(ip, "foo"), "strings")
+val rdd1 = TestSQLContext.jdbc(url(ip, "foo"), "numbers")
+val rdd2 = TestSQLContext.jdbc(url(ip, "foo"), "dates")
+val rdd3 = TestSQLContext.jdbc(url(ip, "foo"), "strings")
rdd1.createJDBCTable(url(ip, "foo"), "numberscopy", false)
rdd2.createJDBCTable(url(ip, "foo"), "datescopy", false)
rdd3.createJDBCTable(url(ip, "foo"), "stringscopy", false)
http://git-wip-us.apache.org/repos/asf/spark/blob/cd3d4158/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
index c174d7a..7b47fee 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegration.scala
@@ -113,7 +113,7 @@ class PostgresDatabase {
}
test("Type mapping for various types") {
-val rdd = TestSQLContext.jdbcRDD(url(db.ip), "public.bar")
+val rdd = TestSQLContext.jdbc(url(db.ip), "public.bar")
val rows = rdd.collect
assert(rows.length == 1)
val types = rows(0).toSeq.map(x => x.getClass.toString)
@@ -142,7 +142,7 @@ class PostgresDatabase {
}
test("Basic write test") {
-val rdd = TestSQLContext.jdbcRDD(url(db.ip), "public.bar")
+val rdd = TestSQLContext.jdbc(url(db.ip), "public.bar")
rdd.createJDBCTable(url(db.ip), "public.barcopy", false)
// Test only that it doesn't bomb out.
}
http://git-wip-us.apache.org/repos/asf/spark/blob/cd3d4158/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
index bfacc51..07b5a84 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
@@ -29,9 +29,9 @@ import org.apache.spark.sql.hive.HiveShim
import org.apache.spark.sql.SQLContext
/**
- * Implementation for "describe [extended] table".
- *
* :: DeveloperApi ::
+ *
+ * Implementation for "describe [extended] table".
*/
@DeveloperApi
case class DescribeHiveTableCommand(
http://git-wip-us.apache.org/repos/asf/spark/blob/cd3d4158/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/
spark git commit: [SPARK-5859] [PySpark] [SQL] fix DataFrame Python API
Repository: spark
Updated Branches:
refs/heads/branch-1.3 cd3d41587 -> 4a581aa3f
[SPARK-5859] [PySpark] [SQL] fix DataFrame Python API
1. added explain()
2. add isLocal()
3. do not call show() in __repl__
4. add foreach() and foreachPartition()
5. add distinct()
6. fix functions.col()/column()/lit()
7. fix unit tests in sql/functions.py
8. fix unicode in showString()
Author: Davies Liu
Closes #4645 from davies/df6 and squashes the following commits:
6b46a2c [Davies Liu] fix DataFrame Python API
(cherry picked from commit d8adefefcc2a4af32295440ed1d4917a6968f017)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4a581aa3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4a581aa3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4a581aa3
Branch: refs/heads/branch-1.3
Commit: 4a581aa3f9144732cc0f6dab6e46f72035e072f4
Parents: cd3d415
Author: Davies Liu
Authored: Tue Feb 17 10:22:48 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 10:22:56 2015 -0800
--
python/pyspark/sql/dataframe.py | 65 ++--
python/pyspark/sql/functions.py | 12 +++
2 files changed, 59 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/4a581aa3/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 28a59e7..8417240 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -238,6 +238,22 @@ class DataFrame(object):
"""
print (self._jdf.schema().treeString())
+def explain(self, extended=False):
+"""
+Prints the plans (logical and physical) to the console for
+debugging purpose.
+
+If extended is False, only prints the physical plan.
+"""
+self._jdf.explain(extended)
+
+def isLocal(self):
+"""
+Returns True if the `collect` and `take` methods can be run locally
+(without any Spark executors).
+"""
+return self._jdf.isLocal()
+
def show(self):
"""
Print the first 20 rows.
@@ -247,14 +263,12 @@ class DataFrame(object):
2 Alice
5 Bob
>>> df
-age name
-2 Alice
-5 Bob
+DataFrame[age: int, name: string]
"""
-print (self)
+print self._jdf.showString().encode('utf8', 'ignore')
def __repr__(self):
-return self._jdf.showString()
+return "DataFrame[%s]" % (", ".join("%s: %s" % c for c in self.dtypes))
def count(self):
"""Return the number of elements in this RDD.
@@ -336,6 +350,8 @@ class DataFrame(object):
"""
Return a new RDD by applying a function to each partition.
+It's a shorthand for df.rdd.mapPartitions()
+
>>> rdd = sc.parallelize([1, 2, 3, 4], 4)
>>> def f(iterator): yield 1
>>> rdd.mapPartitions(f).sum()
@@ -343,6 +359,31 @@ class DataFrame(object):
"""
return self.rdd.mapPartitions(f, preservesPartitioning)
+def foreach(self, f):
+"""
+Applies a function to all rows of this DataFrame.
+
+It's a shorthand for df.rdd.foreach()
+
+>>> def f(person):
+... print person.name
+>>> df.foreach(f)
+"""
+return self.rdd.foreach(f)
+
+def foreachPartition(self, f):
+"""
+Applies a function to each partition of this DataFrame.
+
+It's a shorthand for df.rdd.foreachPartition()
+
+>>> def f(people):
+... for person in people:
+... print person.name
+>>> df.foreachPartition(f)
+"""
+return self.rdd.foreachPartition(f)
+
def cache(self):
""" Persist with the default storage level (C{MEMORY_ONLY_SER}).
"""
@@ -377,8 +418,13 @@ class DataFrame(object):
""" Return a new :class:`DataFrame` that has exactly `numPartitions`
partitions.
"""
-rdd = self._jdf.repartition(numPartitions, None)
-return DataFrame(rdd, self.sql_ctx)
+return DataFrame(self._jdf.repartition(numPartitions, None),
self.sql_ctx)
+
+def distinct(self):
+"""
+Return a new :class:`DataFrame` containing the distinct rows in this
DataFrame.
+"""
+return DataFrame(self._jdf.distinct(), self.sql_ctx)
def sample(self, withReplacement, fraction, seed=None):
"""
@@ -957,10 +1003,7 @@ class Column(DataFrame):
return Column(jc, self.sql_ctx)
def __repr__(self):
-if self._jdf.isComputable():
-return self._jdf.samples()
-else:
-ret
spark git commit: [Minor] fix typo in SQL document
Repository: spark Updated Branches: refs/heads/master fc4eb9505 -> 31efb39c1 [Minor] fix typo in SQL document Author: CodingCat Closes #4656 from CodingCat/fix_typo and squashes the following commits: b41d15c [CodingCat] recover 689fe46 [CodingCat] fix typo Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/31efb39c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/31efb39c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/31efb39c Branch: refs/heads/master Commit: 31efb39c1deb253032b38e8fbafde4b2b1dde1f6 Parents: fc4eb95 Author: CodingCat Authored: Tue Feb 17 12:16:52 2015 -0800 Committer: Michael Armbrust Committed: Tue Feb 17 12:16:52 2015 -0800 -- docs/sql-programming-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/31efb39c/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index 8022c5e..0146a4e 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -159,7 +159,7 @@ you to construct DataFrames when the columns and their types are not known until -The Scala interaface for Spark SQL supports automatically converting an RDD containing case classes +The Scala interface for Spark SQL supports automatically converting an RDD containing case classes to a DataFrame. The case class defines the schema of the table. The names of the arguments to the case class are read using reflection and become the names of the columns. Case classes can also be nested or contain complex - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [Minor] fix typo in SQL document
Repository: spark Updated Branches: refs/heads/branch-1.3 71cf6e295 -> 5636c4a58 [Minor] fix typo in SQL document Author: CodingCat Closes #4656 from CodingCat/fix_typo and squashes the following commits: b41d15c [CodingCat] recover 689fe46 [CodingCat] fix typo (cherry picked from commit 31efb39c1deb253032b38e8fbafde4b2b1dde1f6) Signed-off-by: Michael Armbrust Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5636c4a5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5636c4a5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5636c4a5 Branch: refs/heads/branch-1.3 Commit: 5636c4a583bd28a5b54e14930d72fd9265c301de Parents: 71cf6e2 Author: CodingCat Authored: Tue Feb 17 12:16:52 2015 -0800 Committer: Michael Armbrust Committed: Tue Feb 17 12:17:05 2015 -0800 -- docs/sql-programming-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/5636c4a5/docs/sql-programming-guide.md -- diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index 8022c5e..0146a4e 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -159,7 +159,7 @@ you to construct DataFrames when the columns and their types are not known until -The Scala interaface for Spark SQL supports automatically converting an RDD containing case classes +The Scala interface for Spark SQL supports automatically converting an RDD containing case classes to a DataFrame. The case class defines the schema of the table. The names of the arguments to the case class are read using reflection and become the names of the columns. Case classes can also be nested or contain complex - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5862][SQL] Only transformUp the given plan once in HiveMetastoreCatalog
Repository: spark
Updated Branches:
refs/heads/master 31efb39c1 -> 4611de1ce
[SPARK-5862][SQL] Only transformUp the given plan once in HiveMetastoreCatalog
Current `ParquetConversions` in `HiveMetastoreCatalog` will transformUp the
given plan multiple times if there are many Metastore Parquet tables. Since the
transformUp operation is recursive, it should be better to only perform it once.
Author: Liang-Chi Hsieh
Closes #4651 from viirya/parquet_atonce and squashes the following commits:
c1ed29d [Liang-Chi Hsieh] Fix bug.
e0f919b [Liang-Chi Hsieh] Only transformUp the given plan once.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4611de1c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4611de1c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4611de1c
Branch: refs/heads/master
Commit: 4611de1cef7363bc71ec608560dfd866ae477747
Parents: 31efb39
Author: Liang-Chi Hsieh
Authored: Tue Feb 17 12:23:18 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 12:23:18 2015 -0800
--
.../spark/sql/hive/HiveMetastoreCatalog.scala | 37 +++-
1 file changed, 20 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/4611de1c/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 0e43faa..cfd6f27 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -430,33 +430,36 @@ private[hive] class HiveMetastoreCatalog(hive:
HiveContext) extends Catalog with
hive.convertMetastoreParquet &&
hive.conf.parquetUseDataSourceApi &&
relation.tableDesc.getSerdeClassName.toLowerCase.contains("parquet") =>
- relation
+ val parquetRelation = convertToParquetRelation(relation)
+ val attributedRewrites = relation.output.zip(parquetRelation.output)
+ (relation, parquetRelation, attributedRewrites)
// Read path
case p @ PhysicalOperation(_, _, relation: MetastoreRelation)
if hive.convertMetastoreParquet &&
hive.conf.parquetUseDataSourceApi &&
relation.tableDesc.getSerdeClassName.toLowerCase.contains("parquet") =>
- relation
+ val parquetRelation = convertToParquetRelation(relation)
+ val attributedRewrites = relation.output.zip(parquetRelation.output)
+ (relation, parquetRelation, attributedRewrites)
}
+ val relationMap = toBeReplaced.map(r => (r._1, r._2)).toMap
+ val attributedRewrites = AttributeMap(toBeReplaced.map(_._3).fold(Nil)(_
++: _))
+
// Replaces all `MetastoreRelation`s with corresponding
`ParquetRelation2`s, and fixes
// attribute IDs referenced in other nodes.
- toBeReplaced.distinct.foldLeft(plan) { (lastPlan, relation) =>
-val parquetRelation = convertToParquetRelation(relation)
-val attributedRewrites =
AttributeMap(relation.output.zip(parquetRelation.output))
-
-lastPlan.transformUp {
- case r: MetastoreRelation if r == relation => {
-val withAlias =
- r.alias.map(a => Subquery(a, parquetRelation)).getOrElse(
-Subquery(r.tableName, parquetRelation))
-
-withAlias
- }
- case other => other.transformExpressions {
-case a: Attribute if a.resolved => attributedRewrites.getOrElse(a,
a)
- }
+ plan.transformUp {
+case r: MetastoreRelation if relationMap.contains(r) => {
+ val parquetRelation = relationMap(r)
+ val withAlias =
+r.alias.map(a => Subquery(a, parquetRelation)).getOrElse(
+ Subquery(r.tableName, parquetRelation))
+
+ withAlias
+}
+case other => other.transformExpressions {
+ case a: Attribute if a.resolved => attributedRewrites.getOrElse(a, a)
}
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5862][SQL] Only transformUp the given plan once in HiveMetastoreCatalog
Repository: spark
Updated Branches:
refs/heads/branch-1.3 5636c4a58 -> 62063b7a3
[SPARK-5862][SQL] Only transformUp the given plan once in HiveMetastoreCatalog
Current `ParquetConversions` in `HiveMetastoreCatalog` will transformUp the
given plan multiple times if there are many Metastore Parquet tables. Since the
transformUp operation is recursive, it should be better to only perform it once.
Author: Liang-Chi Hsieh
Closes #4651 from viirya/parquet_atonce and squashes the following commits:
c1ed29d [Liang-Chi Hsieh] Fix bug.
e0f919b [Liang-Chi Hsieh] Only transformUp the given plan once.
(cherry picked from commit 4611de1cef7363bc71ec608560dfd866ae477747)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/62063b7a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/62063b7a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/62063b7a
Branch: refs/heads/branch-1.3
Commit: 62063b7a3e2db9fc7320739d3b900a7840c2dee7
Parents: 5636c4a
Author: Liang-Chi Hsieh
Authored: Tue Feb 17 12:23:18 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 12:23:26 2015 -0800
--
.../spark/sql/hive/HiveMetastoreCatalog.scala | 37 +++-
1 file changed, 20 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/62063b7a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 0e43faa..cfd6f27 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -430,33 +430,36 @@ private[hive] class HiveMetastoreCatalog(hive:
HiveContext) extends Catalog with
hive.convertMetastoreParquet &&
hive.conf.parquetUseDataSourceApi &&
relation.tableDesc.getSerdeClassName.toLowerCase.contains("parquet") =>
- relation
+ val parquetRelation = convertToParquetRelation(relation)
+ val attributedRewrites = relation.output.zip(parquetRelation.output)
+ (relation, parquetRelation, attributedRewrites)
// Read path
case p @ PhysicalOperation(_, _, relation: MetastoreRelation)
if hive.convertMetastoreParquet &&
hive.conf.parquetUseDataSourceApi &&
relation.tableDesc.getSerdeClassName.toLowerCase.contains("parquet") =>
- relation
+ val parquetRelation = convertToParquetRelation(relation)
+ val attributedRewrites = relation.output.zip(parquetRelation.output)
+ (relation, parquetRelation, attributedRewrites)
}
+ val relationMap = toBeReplaced.map(r => (r._1, r._2)).toMap
+ val attributedRewrites = AttributeMap(toBeReplaced.map(_._3).fold(Nil)(_
++: _))
+
// Replaces all `MetastoreRelation`s with corresponding
`ParquetRelation2`s, and fixes
// attribute IDs referenced in other nodes.
- toBeReplaced.distinct.foldLeft(plan) { (lastPlan, relation) =>
-val parquetRelation = convertToParquetRelation(relation)
-val attributedRewrites =
AttributeMap(relation.output.zip(parquetRelation.output))
-
-lastPlan.transformUp {
- case r: MetastoreRelation if r == relation => {
-val withAlias =
- r.alias.map(a => Subquery(a, parquetRelation)).getOrElse(
-Subquery(r.tableName, parquetRelation))
-
-withAlias
- }
- case other => other.transformExpressions {
-case a: Attribute if a.resolved => attributedRewrites.getOrElse(a,
a)
- }
+ plan.transformUp {
+case r: MetastoreRelation if relationMap.contains(r) => {
+ val parquetRelation = relationMap(r)
+ val withAlias =
+r.alias.map(a => Subquery(a, parquetRelation)).getOrElse(
+ Subquery(r.tableName, parquetRelation))
+
+ withAlias
+}
+case other => other.transformExpressions {
+ case a: Attribute if a.resolved => attributedRewrites.getOrElse(a, a)
}
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [Minor][SQL] Use same function to check path parameter in JSONRelation
Repository: spark
Updated Branches:
refs/heads/branch-1.3 62063b7a3 -> d74d5e86a
[Minor][SQL] Use same function to check path parameter in JSONRelation
Author: Liang-Chi Hsieh
Closes #4649 from viirya/use_checkpath and squashes the following commits:
0f9a1a1 [Liang-Chi Hsieh] Use same function to check path parameter.
(cherry picked from commit ac506b7c2846f656e03839bbd0e93827c7cc613e)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d74d5e86
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d74d5e86
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d74d5e86
Branch: refs/heads/branch-1.3
Commit: d74d5e86abcb2ee4f142bb76f641d66cb4ffeb42
Parents: 62063b7
Author: Liang-Chi Hsieh
Authored: Tue Feb 17 12:24:13 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 12:24:21 2015 -0800
--
.../src/main/scala/org/apache/spark/sql/json/JSONRelation.scala | 4 ++--
.../org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala| 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d74d5e86/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
index 2484863..3b68b7c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
@@ -37,7 +37,7 @@ private[sql] class DefaultSource
override def createRelation(
sqlContext: SQLContext,
parameters: Map[String, String]): BaseRelation = {
-val path = parameters.getOrElse("path", sys.error("Option 'path' not
specified"))
+val path = checkPath(parameters)
val samplingRatio =
parameters.get("samplingRatio").map(_.toDouble).getOrElse(1.0)
JSONRelation(path, samplingRatio, None)(sqlContext)
@@ -48,7 +48,7 @@ private[sql] class DefaultSource
sqlContext: SQLContext,
parameters: Map[String, String],
schema: StructType): BaseRelation = {
-val path = parameters.getOrElse("path", sys.error("Option 'path' not
specified"))
+val path = checkPath(parameters)
val samplingRatio =
parameters.get("samplingRatio").map(_.toDouble).getOrElse(1.0)
JSONRelation(path, samplingRatio, Some(schema))(sqlContext)
http://git-wip-us.apache.org/repos/asf/spark/blob/d74d5e86/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index 0263e3b..485d5c9 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -547,7 +547,7 @@ class MetastoreDataSourcesSuite extends QueryTest with
BeforeAndAfterEach {
Map.empty[String, String])
}.getMessage
assert(
- message.contains("Option 'path' not specified"),
+ message.contains("'path' must be specified for json data."),
"We should complain that path is not specified.")
sql("DROP TABLE savedJsonTable")
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SQL] [Minor] Update the HiveContext Unittest
Repository: spark
Updated Branches:
refs/heads/master ac506b7c2 -> 9d281fa56
[SQL] [Minor] Update the HiveContext Unittest
In unit test, the table src(key INT, value STRING) is not the same as HIVE
src(key STRING, value STRING)
https://github.com/apache/hive/blob/branch-0.13/data/scripts/q_test_init.sql
And in the reflect.q, test failed for expression `reflect("java.lang.Integer",
"valueOf", key, 16)`, which expect the argument `key` as STRING not INT.
This PR doesn't aim to change the `src` schema, we can do that after 1.3
released, however, we probably need to re-generate all the golden files.
Author: Cheng Hao
Closes #4584 from chenghao-intel/reflect and squashes the following commits:
e5bdc3a [Cheng Hao] Move the test case reflect into blacklist
184abfd [Cheng Hao] revert the change to table src1
d9bcf92 [Cheng Hao] Update the HiveContext Unittest
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9d281fa5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9d281fa5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9d281fa5
Branch: refs/heads/master
Commit: 9d281fa56022800dc008a3de233fec44379a2bd7
Parents: ac506b7
Author: Cheng Hao
Authored: Tue Feb 17 12:25:35 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 12:25:35 2015 -0800
--
.../spark/sql/hive/execution/HiveCompatibilitySuite.scala | 6 ++
.../main/scala/org/apache/spark/sql/hive/test/TestHive.scala | 1 +
.../golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada | 1 +
.../golden/udf_reflect2-1-7bec330c7bc6f71cbaf9bf1883d1b184 | 1 +
.../golden/udf_reflect2-2-c5a05379f482215a5a484bed0299bf19 | 3 +++
.../golden/udf_reflect2-3-effc057c78c00b0af26a4ac0f5f116ca | 0
.../golden/udf_reflect2-4-73d466e70e96e9e5f0cd373b37d4e1f4 | 5 +
7 files changed, 17 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9d281fa5/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
--
diff --git
a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index 133f2d3..c6ead45 100644
---
a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -225,6 +225,11 @@ class HiveCompatibilitySuite extends HiveQueryFileTest
with BeforeAndAfter {
// Needs constant object inspectors
"udf_round",
+// the table src(key INT, value STRING) is not the same as HIVE unittest.
In Hive
+// is src(key STRING, value STRING), and in the reflect.q, it failed in
+// Integer.valueOf, which expect the first argument passed as STRING type
not INT.
+"udf_reflect",
+
// Sort with Limit clause causes failure.
"ctas",
"ctas_hadoop20",
@@ -886,6 +891,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with
BeforeAndAfter {
"udf_power",
"udf_radians",
"udf_rand",
+"udf_reflect2",
"udf_regexp",
"udf_regexp_extract",
"udf_regexp_replace",
http://git-wip-us.apache.org/repos/asf/spark/blob/9d281fa5/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 840fbc1..a2d99f1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -196,6 +196,7 @@ class TestHiveContext(sc: SparkContext) extends
HiveContext(sc) {
// The test tables that are defined in the Hive QTestUtil.
// /itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+ //
https://github.com/apache/hive/blob/branch-0.13/data/scripts/q_test_init.sql
val hiveQTestUtilTables = Seq(
TestTable("src",
"CREATE TABLE src (key INT, value STRING)".cmd,
http://git-wip-us.apache.org/repos/asf/spark/blob/9d281fa5/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada
--
diff --git
a/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada
b/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada
new file mode 100644
index 000..573541a
--- /dev/null
+++
b/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a
spark git commit: [SQL] [Minor] Update the HiveContext Unittest
Repository: spark
Updated Branches:
refs/heads/branch-1.3 d74d5e86a -> 01356514e
[SQL] [Minor] Update the HiveContext Unittest
In unit test, the table src(key INT, value STRING) is not the same as HIVE
src(key STRING, value STRING)
https://github.com/apache/hive/blob/branch-0.13/data/scripts/q_test_init.sql
And in the reflect.q, test failed for expression `reflect("java.lang.Integer",
"valueOf", key, 16)`, which expect the argument `key` as STRING not INT.
This PR doesn't aim to change the `src` schema, we can do that after 1.3
released, however, we probably need to re-generate all the golden files.
Author: Cheng Hao
Closes #4584 from chenghao-intel/reflect and squashes the following commits:
e5bdc3a [Cheng Hao] Move the test case reflect into blacklist
184abfd [Cheng Hao] revert the change to table src1
d9bcf92 [Cheng Hao] Update the HiveContext Unittest
(cherry picked from commit 9d281fa56022800dc008a3de233fec44379a2bd7)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/01356514
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/01356514
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/01356514
Branch: refs/heads/branch-1.3
Commit: 01356514ef42321e09e9a67ba08366bb2bd5af4b
Parents: d74d5e8
Author: Cheng Hao
Authored: Tue Feb 17 12:25:35 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 12:25:43 2015 -0800
--
.../spark/sql/hive/execution/HiveCompatibilitySuite.scala | 6 ++
.../main/scala/org/apache/spark/sql/hive/test/TestHive.scala | 1 +
.../golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada | 1 +
.../golden/udf_reflect2-1-7bec330c7bc6f71cbaf9bf1883d1b184 | 1 +
.../golden/udf_reflect2-2-c5a05379f482215a5a484bed0299bf19 | 3 +++
.../golden/udf_reflect2-3-effc057c78c00b0af26a4ac0f5f116ca | 0
.../golden/udf_reflect2-4-73d466e70e96e9e5f0cd373b37d4e1f4 | 5 +
7 files changed, 17 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/01356514/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
--
diff --git
a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index 133f2d3..c6ead45 100644
---
a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -225,6 +225,11 @@ class HiveCompatibilitySuite extends HiveQueryFileTest
with BeforeAndAfter {
// Needs constant object inspectors
"udf_round",
+// the table src(key INT, value STRING) is not the same as HIVE unittest.
In Hive
+// is src(key STRING, value STRING), and in the reflect.q, it failed in
+// Integer.valueOf, which expect the first argument passed as STRING type
not INT.
+"udf_reflect",
+
// Sort with Limit clause causes failure.
"ctas",
"ctas_hadoop20",
@@ -886,6 +891,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with
BeforeAndAfter {
"udf_power",
"udf_radians",
"udf_rand",
+"udf_reflect2",
"udf_regexp",
"udf_regexp_extract",
"udf_regexp_replace",
http://git-wip-us.apache.org/repos/asf/spark/blob/01356514/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 840fbc1..a2d99f1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -196,6 +196,7 @@ class TestHiveContext(sc: SparkContext) extends
HiveContext(sc) {
// The test tables that are defined in the Hive QTestUtil.
// /itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+ //
https://github.com/apache/hive/blob/branch-0.13/data/scripts/q_test_init.sql
val hiveQTestUtilTables = Seq(
TestTable("src",
"CREATE TABLE src (key INT, value STRING)".cmd,
http://git-wip-us.apache.org/repos/asf/spark/blob/01356514/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada
--
diff --git
a/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada
b/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada
new file mode 100
spark git commit: [Minor][SQL] Use same function to check path parameter in JSONRelation
Repository: spark
Updated Branches:
refs/heads/master 4611de1ce -> ac506b7c2
[Minor][SQL] Use same function to check path parameter in JSONRelation
Author: Liang-Chi Hsieh
Closes #4649 from viirya/use_checkpath and squashes the following commits:
0f9a1a1 [Liang-Chi Hsieh] Use same function to check path parameter.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ac506b7c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ac506b7c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ac506b7c
Branch: refs/heads/master
Commit: ac506b7c2846f656e03839bbd0e93827c7cc613e
Parents: 4611de1
Author: Liang-Chi Hsieh
Authored: Tue Feb 17 12:24:13 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 12:24:13 2015 -0800
--
.../src/main/scala/org/apache/spark/sql/json/JSONRelation.scala | 4 ++--
.../org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala| 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ac506b7c/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
index 2484863..3b68b7c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
@@ -37,7 +37,7 @@ private[sql] class DefaultSource
override def createRelation(
sqlContext: SQLContext,
parameters: Map[String, String]): BaseRelation = {
-val path = parameters.getOrElse("path", sys.error("Option 'path' not
specified"))
+val path = checkPath(parameters)
val samplingRatio =
parameters.get("samplingRatio").map(_.toDouble).getOrElse(1.0)
JSONRelation(path, samplingRatio, None)(sqlContext)
@@ -48,7 +48,7 @@ private[sql] class DefaultSource
sqlContext: SQLContext,
parameters: Map[String, String],
schema: StructType): BaseRelation = {
-val path = parameters.getOrElse("path", sys.error("Option 'path' not
specified"))
+val path = checkPath(parameters)
val samplingRatio =
parameters.get("samplingRatio").map(_.toDouble).getOrElse(1.0)
JSONRelation(path, samplingRatio, Some(schema))(sqlContext)
http://git-wip-us.apache.org/repos/asf/spark/blob/ac506b7c/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index 0263e3b..485d5c9 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -547,7 +547,7 @@ class MetastoreDataSourcesSuite extends QueryTest with
BeforeAndAfterEach {
Map.empty[String, String])
}.getMessage
assert(
- message.contains("Option 'path' not specified"),
+ message.contains("'path' must be specified for json data."),
"We should complain that path is not specified.")
sql("DROP TABLE savedJsonTable")
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5868][SQL] Fix python UDFs in HiveContext and checks in SQLContext
Repository: spark
Updated Branches:
refs/heads/branch-1.3 01356514e -> e65dc1fd5
[SPARK-5868][SQL] Fix python UDFs in HiveContext and checks in SQLContext
Author: Michael Armbrust
Closes #4657 from marmbrus/pythonUdfs and squashes the following commits:
a7823a8 [Michael Armbrust] [SPARK-5868][SQL] Fix python UDFs in HiveContext and
checks in SQLContext
(cherry picked from commit de4836f8f12c36c1b350cef288a75b5e59155735)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e65dc1fd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e65dc1fd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e65dc1fd
Branch: refs/heads/branch-1.3
Commit: e65dc1fd58e4f9445247c9fd4e94b34550e992fb
Parents: 0135651
Author: Michael Armbrust
Authored: Tue Feb 17 13:23:45 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 13:23:56 2015 -0800
--
sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 3 ++-
.../main/scala/org/apache/spark/sql/execution/pythonUdfs.scala| 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e65dc1fd/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 31afa0e..709b350 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -113,6 +113,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
protected[sql] lazy val analyzer: Analyzer =
new Analyzer(catalog, functionRegistry, caseSensitive = true) {
override val extendedResolutionRules =
+ExtractPythonUdfs ::
sources.PreWriteCheck(catalog) ::
sources.PreInsertCastAndRename ::
Nil
@@ -1059,7 +1060,7 @@ class SQLContext(@transient val sparkContext:
SparkContext)
@DeveloperApi
protected[sql] class QueryExecution(val logical: LogicalPlan) {
-lazy val analyzed: LogicalPlan = ExtractPythonUdfs(analyzer(logical))
+lazy val analyzed: LogicalPlan = analyzer(logical)
lazy val withCachedData: LogicalPlan = cacheManager.useCachedData(analyzed)
lazy val optimizedPlan: LogicalPlan = optimizer(withCachedData)
http://git-wip-us.apache.org/repos/asf/spark/blob/e65dc1fd/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
index 3a2f8d7..69de4d1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
@@ -205,6 +205,9 @@ case class EvaluatePython(
extends logical.UnaryNode {
def output = child.output :+ resultAttribute
+
+ // References should not include the produced attribute.
+ override def references = udf.references
}
/**
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5868][SQL] Fix python UDFs in HiveContext and checks in SQLContext
Repository: spark
Updated Branches:
refs/heads/master 9d281fa56 -> de4836f8f
[SPARK-5868][SQL] Fix python UDFs in HiveContext and checks in SQLContext
Author: Michael Armbrust
Closes #4657 from marmbrus/pythonUdfs and squashes the following commits:
a7823a8 [Michael Armbrust] [SPARK-5868][SQL] Fix python UDFs in HiveContext and
checks in SQLContext
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/de4836f8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/de4836f8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/de4836f8
Branch: refs/heads/master
Commit: de4836f8f12c36c1b350cef288a75b5e59155735
Parents: 9d281fa
Author: Michael Armbrust
Authored: Tue Feb 17 13:23:45 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 13:23:45 2015 -0800
--
sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 3 ++-
.../main/scala/org/apache/spark/sql/execution/pythonUdfs.scala| 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/de4836f8/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 31afa0e..709b350 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -113,6 +113,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
protected[sql] lazy val analyzer: Analyzer =
new Analyzer(catalog, functionRegistry, caseSensitive = true) {
override val extendedResolutionRules =
+ExtractPythonUdfs ::
sources.PreWriteCheck(catalog) ::
sources.PreInsertCastAndRename ::
Nil
@@ -1059,7 +1060,7 @@ class SQLContext(@transient val sparkContext:
SparkContext)
@DeveloperApi
protected[sql] class QueryExecution(val logical: LogicalPlan) {
-lazy val analyzed: LogicalPlan = ExtractPythonUdfs(analyzer(logical))
+lazy val analyzed: LogicalPlan = analyzer(logical)
lazy val withCachedData: LogicalPlan = cacheManager.useCachedData(analyzed)
lazy val optimizedPlan: LogicalPlan = optimizer(withCachedData)
http://git-wip-us.apache.org/repos/asf/spark/blob/de4836f8/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
index 3a2f8d7..69de4d1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
@@ -205,6 +205,9 @@ case class EvaluatePython(
extends logical.UnaryNode {
def output = child.output :+ resultAttribute
+
+ // References should not include the produced attribute.
+ override def references = udf.references
}
/**
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5871] output explain in Python
Repository: spark Updated Branches: refs/heads/master 445a755b8 -> 3df85dccb [SPARK-5871] output explain in Python Author: Davies Liu Closes #4658 from davies/explain and squashes the following commits: db87ea2 [Davies Liu] output explain in Python Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3df85dcc Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3df85dcc Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3df85dcc Branch: refs/heads/master Commit: 3df85dccbc8fd1ba19bbcdb8d359c073b1494d98 Parents: 445a755 Author: Davies Liu Authored: Tue Feb 17 13:48:38 2015 -0800 Committer: Michael Armbrust Committed: Tue Feb 17 13:48:38 2015 -0800 -- python/pyspark/sql/dataframe.py | 23 --- 1 file changed, 20 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3df85dcc/python/pyspark/sql/dataframe.py -- diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 8417240..388033d 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -244,8 +244,25 @@ class DataFrame(object): debugging purpose. If extended is False, only prints the physical plan. -""" -self._jdf.explain(extended) + +>>> df.explain() +PhysicalRDD [age#0,name#1], MapPartitionsRDD[...] at mapPartitions at SQLContext.scala:... + +>>> df.explain(True) +== Parsed Logical Plan == +... +== Analyzed Logical Plan == +... +== Optimized Logical Plan == +... +== Physical Plan == +... +== RDD == +""" +if extended: +print self._jdf.queryExecution().toString() +else: +print self._jdf.queryExecution().executedPlan().toString() def isLocal(self): """ @@ -1034,7 +1051,7 @@ def _test(): Row(name='Bob', age=5, height=85)]).toDF() (failure_count, test_count) = doctest.testmod( pyspark.sql.dataframe, globs=globs, -optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE) +optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF) globs['sc'].stop() if failure_count: exit(-1) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5871] output explain in Python
Repository: spark Updated Branches: refs/heads/branch-1.3 35e23ff14 -> cb061603c [SPARK-5871] output explain in Python Author: Davies Liu Closes #4658 from davies/explain and squashes the following commits: db87ea2 [Davies Liu] output explain in Python (cherry picked from commit 3df85dccbc8fd1ba19bbcdb8d359c073b1494d98) Signed-off-by: Michael Armbrust Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cb061603 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cb061603 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cb061603 Branch: refs/heads/branch-1.3 Commit: cb061603c3ca4cd5162a36fc32de15779614e854 Parents: 35e23ff Author: Davies Liu Authored: Tue Feb 17 13:48:38 2015 -0800 Committer: Michael Armbrust Committed: Tue Feb 17 13:51:00 2015 -0800 -- python/pyspark/sql/dataframe.py | 23 --- 1 file changed, 20 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/cb061603/python/pyspark/sql/dataframe.py -- diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 8417240..388033d 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -244,8 +244,25 @@ class DataFrame(object): debugging purpose. If extended is False, only prints the physical plan. -""" -self._jdf.explain(extended) + +>>> df.explain() +PhysicalRDD [age#0,name#1], MapPartitionsRDD[...] at mapPartitions at SQLContext.scala:... + +>>> df.explain(True) +== Parsed Logical Plan == +... +== Analyzed Logical Plan == +... +== Optimized Logical Plan == +... +== Physical Plan == +... +== RDD == +""" +if extended: +print self._jdf.queryExecution().toString() +else: +print self._jdf.queryExecution().executedPlan().toString() def isLocal(self): """ @@ -1034,7 +1051,7 @@ def _test(): Row(name='Bob', age=5, height=85)]).toDF() (failure_count, test_count) = doctest.testmod( pyspark.sql.dataframe, globs=globs, -optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE) +optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF) globs['sc'].stop() if failure_count: exit(-1) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5872] [SQL] create a sqlCtx in pyspark shell
Repository: spark
Updated Branches:
refs/heads/master 3df85dccb -> 4d4cc760f
[SPARK-5872] [SQL] create a sqlCtx in pyspark shell
The sqlCtx will be HiveContext if hive is built in assembly jar, or SQLContext
if not.
It also skip the Hive tests in pyspark.sql.tests if no hive is available.
Author: Davies Liu
Closes #4659 from davies/sqlctx and squashes the following commits:
0e6629a [Davies Liu] sqlCtx in pyspark
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4d4cc760
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4d4cc760
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4d4cc760
Branch: refs/heads/master
Commit: 4d4cc760fa9687ce563320094557ef9144488676
Parents: 3df85dc
Author: Davies Liu
Authored: Tue Feb 17 15:44:37 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 15:44:37 2015 -0800
--
python/pyspark/shell.py | 13 -
python/pyspark/sql/tests.py | 12 ++--
2 files changed, 22 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/4d4cc760/python/pyspark/shell.py
--
diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py
index 4cf4b89..1a02fec 100644
--- a/python/pyspark/shell.py
+++ b/python/pyspark/shell.py
@@ -31,8 +31,12 @@ if sys.version_info[0] != 2:
import atexit
import os
import platform
+
+import py4j
+
import pyspark
from pyspark.context import SparkContext
+from pyspark.sql import SQLContext, HiveContext
from pyspark.storagelevel import StorageLevel
# this is the deprecated equivalent of ADD_JARS
@@ -46,6 +50,13 @@ if os.environ.get("SPARK_EXECUTOR_URI"):
sc = SparkContext(appName="PySparkShell", pyFiles=add_files)
atexit.register(lambda: sc.stop())
+try:
+# Try to access HiveConf, it will raise exception if Hive is not added
+sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
+sqlCtx = HiveContext(sc)
+except py4j.protocol.Py4JError:
+sqlCtx = SQLContext(sc)
+
print("""Welcome to
__
/ __/__ ___ _/ /__
@@ -57,7 +68,7 @@ print("Using Python version %s (%s, %s)" % (
platform.python_version(),
platform.python_build()[0],
platform.python_build()[1]))
-print("SparkContext available as sc.")
+print("SparkContext available as sc, %s available as sqlCtx." %
sqlCtx.__class__.__name__)
if add_files is not None:
print("Warning: ADD_FILES environment variable is deprecated, use
--py-files argument instead")
http://git-wip-us.apache.org/repos/asf/spark/blob/4d4cc760/python/pyspark/sql/tests.py
--
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index aa80bca..52f7e65 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -25,6 +25,8 @@ import pydoc
import shutil
import tempfile
+import py4j
+
if sys.version_info[:2] <= (2, 6):
try:
import unittest2 as unittest
@@ -329,9 +331,12 @@ class HiveContextSQLTests(ReusedPySparkTestCase):
def setUpClass(cls):
ReusedPySparkTestCase.setUpClass()
cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
+try:
+cls.sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
+except py4j.protocol.Py4JError:
+cls.sqlCtx = None
+return
os.unlink(cls.tempdir.name)
-print "type", type(cls.sc)
-print "type", type(cls.sc._jsc)
_scala_HiveContext =\
cls.sc._jvm.org.apache.spark.sql.hive.test.TestHiveContext(cls.sc._jsc.sc())
cls.sqlCtx = HiveContext(cls.sc, _scala_HiveContext)
@@ -344,6 +349,9 @@ class HiveContextSQLTests(ReusedPySparkTestCase):
shutil.rmtree(cls.tempdir.name, ignore_errors=True)
def test_save_and_load_table(self):
+if self.sqlCtx is None:
+return # no hive available, skipped
+
df = self.df
tmpPath = tempfile.mkdtemp()
shutil.rmtree(tmpPath)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5872] [SQL] create a sqlCtx in pyspark shell
Repository: spark
Updated Branches:
refs/heads/branch-1.3 cb061603c -> 0dba382ee
[SPARK-5872] [SQL] create a sqlCtx in pyspark shell
The sqlCtx will be HiveContext if hive is built in assembly jar, or SQLContext
if not.
It also skip the Hive tests in pyspark.sql.tests if no hive is available.
Author: Davies Liu
Closes #4659 from davies/sqlctx and squashes the following commits:
0e6629a [Davies Liu] sqlCtx in pyspark
(cherry picked from commit 4d4cc760fa9687ce563320094557ef9144488676)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0dba382e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0dba382e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0dba382e
Branch: refs/heads/branch-1.3
Commit: 0dba382ee65694969704384c4968e3a656b3c833
Parents: cb06160
Author: Davies Liu
Authored: Tue Feb 17 15:44:37 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 15:44:45 2015 -0800
--
python/pyspark/shell.py | 13 -
python/pyspark/sql/tests.py | 12 ++--
2 files changed, 22 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0dba382e/python/pyspark/shell.py
--
diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py
index 4cf4b89..1a02fec 100644
--- a/python/pyspark/shell.py
+++ b/python/pyspark/shell.py
@@ -31,8 +31,12 @@ if sys.version_info[0] != 2:
import atexit
import os
import platform
+
+import py4j
+
import pyspark
from pyspark.context import SparkContext
+from pyspark.sql import SQLContext, HiveContext
from pyspark.storagelevel import StorageLevel
# this is the deprecated equivalent of ADD_JARS
@@ -46,6 +50,13 @@ if os.environ.get("SPARK_EXECUTOR_URI"):
sc = SparkContext(appName="PySparkShell", pyFiles=add_files)
atexit.register(lambda: sc.stop())
+try:
+# Try to access HiveConf, it will raise exception if Hive is not added
+sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
+sqlCtx = HiveContext(sc)
+except py4j.protocol.Py4JError:
+sqlCtx = SQLContext(sc)
+
print("""Welcome to
__
/ __/__ ___ _/ /__
@@ -57,7 +68,7 @@ print("Using Python version %s (%s, %s)" % (
platform.python_version(),
platform.python_build()[0],
platform.python_build()[1]))
-print("SparkContext available as sc.")
+print("SparkContext available as sc, %s available as sqlCtx." %
sqlCtx.__class__.__name__)
if add_files is not None:
print("Warning: ADD_FILES environment variable is deprecated, use
--py-files argument instead")
http://git-wip-us.apache.org/repos/asf/spark/blob/0dba382e/python/pyspark/sql/tests.py
--
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index aa80bca..52f7e65 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -25,6 +25,8 @@ import pydoc
import shutil
import tempfile
+import py4j
+
if sys.version_info[:2] <= (2, 6):
try:
import unittest2 as unittest
@@ -329,9 +331,12 @@ class HiveContextSQLTests(ReusedPySparkTestCase):
def setUpClass(cls):
ReusedPySparkTestCase.setUpClass()
cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
+try:
+cls.sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
+except py4j.protocol.Py4JError:
+cls.sqlCtx = None
+return
os.unlink(cls.tempdir.name)
-print "type", type(cls.sc)
-print "type", type(cls.sc._jsc)
_scala_HiveContext =\
cls.sc._jvm.org.apache.spark.sql.hive.test.TestHiveContext(cls.sc._jsc.sc())
cls.sqlCtx = HiveContext(cls.sc, _scala_HiveContext)
@@ -344,6 +349,9 @@ class HiveContextSQLTests(ReusedPySparkTestCase):
shutil.rmtree(cls.tempdir.name, ignore_errors=True)
def test_save_and_load_table(self):
+if self.sqlCtx is None:
+return # no hive available, skipped
+
df = self.df
tmpPath = tempfile.mkdtemp()
shutil.rmtree(tmpPath)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5852][SQL]Fail to convert a newly created empty metastore parquet table to a data source parquet table.
Repository: spark
Updated Branches:
refs/heads/master 4d4cc760f -> 117121a4e
[SPARK-5852][SQL]Fail to convert a newly created empty metastore parquet table
to a data source parquet table.
The problem is that after we create an empty hive metastore parquet table (e.g.
`CREATE TABLE test (a int) STORED AS PARQUET`), Hive will create an empty dir
for us, which cause our data source `ParquetRelation2` fail to get the schema
of the table. See JIRA for the case to reproduce the bug and the exception.
This PR is based on #4562 from chenghao-intel.
JIRA: https://issues.apache.org/jira/browse/SPARK-5852
Author: Yin Huai
Author: Cheng Hao
Closes #4655 from yhuai/CTASParquet and squashes the following commits:
b8b3450 [Yin Huai] Update tests.
2ac94f7 [Yin Huai] Update tests.
3db3d20 [Yin Huai] Minor update.
d7e2308 [Yin Huai] Revert changes in HiveMetastoreCatalog.scala.
36978d1 [Cheng Hao] Update the code as feedback
a04930b [Cheng Hao] fix bug of scan an empty parquet based table
442ffe0 [Cheng Hao] passdown the schema for Parquet File in HiveContext
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/117121a4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/117121a4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/117121a4
Branch: refs/heads/master
Commit: 117121a4ecaadda156a82255333670775e7727db
Parents: 4d4cc76
Author: Yin Huai
Authored: Tue Feb 17 15:47:59 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 15:47:59 2015 -0800
--
.../apache/spark/sql/parquet/newParquet.scala | 18 ++-
.../sql/hive/MetastoreDataSourcesSuite.scala| 38 +++
.../spark/sql/parquet/parquetSuites.scala | 114 ++-
3 files changed, 164 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/117121a4/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
index 95bea92..16b7713 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
@@ -287,7 +287,16 @@ private[sql] case class ParquetRelation2(
}
}
- parquetSchema = maybeSchema.getOrElse(readSchema())
+ // To get the schema. We first try to get the schema defined in
maybeSchema.
+ // If maybeSchema is not defined, we will try to get the schema from
existing parquet data
+ // (through readSchema). If data does not exist, we will try to get the
schema defined in
+ // maybeMetastoreSchema (defined in the options of the data source).
+ // Finally, if we still could not get the schema. We throw an error.
+ parquetSchema =
+maybeSchema
+ .orElse(readSchema())
+ .orElse(maybeMetastoreSchema)
+ .getOrElse(sys.error("Failed to get the schema."))
partitionKeysIncludedInParquetSchema =
isPartitioned &&
@@ -308,7 +317,7 @@ private[sql] case class ParquetRelation2(
}
}
-private def readSchema(): StructType = {
+private def readSchema(): Option[StructType] = {
// Sees which file(s) we need to touch in order to figure out the schema.
val filesToTouch =
// Always tries the summary files first if users don't require a merged
schema. In this case,
@@ -611,7 +620,8 @@ private[sql] object ParquetRelation2 {
// internally.
private[sql] val METASTORE_SCHEMA = "metastoreSchema"
- private[parquet] def readSchema(footers: Seq[Footer], sqlContext:
SQLContext): StructType = {
+ private[parquet] def readSchema(
+ footers: Seq[Footer], sqlContext: SQLContext): Option[StructType] = {
footers.map { footer =>
val metadata = footer.getParquetMetadata.getFileMetaData
val parquetSchema = metadata.getSchema
@@ -630,7 +640,7 @@ private[sql] object ParquetRelation2 {
sqlContext.conf.isParquetBinaryAsString,
sqlContext.conf.isParquetINT96AsTimestamp))
}
-}.reduce { (left, right) =>
+}.reduceOption { (left, right) =>
try left.merge(right) catch { case e: Throwable =>
throw new SparkException(s"Failed to merge incompatible schemas $left
and $right", e)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/117121a4/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuit
spark git commit: [SPARK-5852][SQL]Fail to convert a newly created empty metastore parquet table to a data source parquet table.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 0dba382ee -> 07d8ef9e7
[SPARK-5852][SQL]Fail to convert a newly created empty metastore parquet table
to a data source parquet table.
The problem is that after we create an empty hive metastore parquet table (e.g.
`CREATE TABLE test (a int) STORED AS PARQUET`), Hive will create an empty dir
for us, which cause our data source `ParquetRelation2` fail to get the schema
of the table. See JIRA for the case to reproduce the bug and the exception.
This PR is based on #4562 from chenghao-intel.
JIRA: https://issues.apache.org/jira/browse/SPARK-5852
Author: Yin Huai
Author: Cheng Hao
Closes #4655 from yhuai/CTASParquet and squashes the following commits:
b8b3450 [Yin Huai] Update tests.
2ac94f7 [Yin Huai] Update tests.
3db3d20 [Yin Huai] Minor update.
d7e2308 [Yin Huai] Revert changes in HiveMetastoreCatalog.scala.
36978d1 [Cheng Hao] Update the code as feedback
a04930b [Cheng Hao] fix bug of scan an empty parquet based table
442ffe0 [Cheng Hao] passdown the schema for Parquet File in HiveContext
(cherry picked from commit 117121a4ecaadda156a82255333670775e7727db)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/07d8ef9e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/07d8ef9e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/07d8ef9e
Branch: refs/heads/branch-1.3
Commit: 07d8ef9e730251b66f962fa7acdaf7d7eacc62a1
Parents: 0dba382
Author: Yin Huai
Authored: Tue Feb 17 15:47:59 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 15:48:23 2015 -0800
--
.../apache/spark/sql/parquet/newParquet.scala | 18 ++-
.../sql/hive/MetastoreDataSourcesSuite.scala| 38 +++
.../spark/sql/parquet/parquetSuites.scala | 114 ++-
3 files changed, 164 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/07d8ef9e/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
index 95bea92..16b7713 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
@@ -287,7 +287,16 @@ private[sql] case class ParquetRelation2(
}
}
- parquetSchema = maybeSchema.getOrElse(readSchema())
+ // To get the schema. We first try to get the schema defined in
maybeSchema.
+ // If maybeSchema is not defined, we will try to get the schema from
existing parquet data
+ // (through readSchema). If data does not exist, we will try to get the
schema defined in
+ // maybeMetastoreSchema (defined in the options of the data source).
+ // Finally, if we still could not get the schema. We throw an error.
+ parquetSchema =
+maybeSchema
+ .orElse(readSchema())
+ .orElse(maybeMetastoreSchema)
+ .getOrElse(sys.error("Failed to get the schema."))
partitionKeysIncludedInParquetSchema =
isPartitioned &&
@@ -308,7 +317,7 @@ private[sql] case class ParquetRelation2(
}
}
-private def readSchema(): StructType = {
+private def readSchema(): Option[StructType] = {
// Sees which file(s) we need to touch in order to figure out the schema.
val filesToTouch =
// Always tries the summary files first if users don't require a merged
schema. In this case,
@@ -611,7 +620,8 @@ private[sql] object ParquetRelation2 {
// internally.
private[sql] val METASTORE_SCHEMA = "metastoreSchema"
- private[parquet] def readSchema(footers: Seq[Footer], sqlContext:
SQLContext): StructType = {
+ private[parquet] def readSchema(
+ footers: Seq[Footer], sqlContext: SQLContext): Option[StructType] = {
footers.map { footer =>
val metadata = footer.getParquetMetadata.getFileMetaData
val parquetSchema = metadata.getSchema
@@ -630,7 +640,7 @@ private[sql] object ParquetRelation2 {
sqlContext.conf.isParquetBinaryAsString,
sqlContext.conf.isParquetINT96AsTimestamp))
}
-}.reduce { (left, right) =>
+}.reduceOption { (left, right) =>
try left.merge(right) catch { case e: Throwable =>
throw new SparkException(s"Failed to merge incompatible schemas $left
and $right", e)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/07d8ef9e/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive
spark git commit: [SPARK-5875][SQL]logical.Project should not be resolved if it contains aggregates or generators
Repository: spark
Updated Branches:
refs/heads/master a51fc7ef9 -> d5f12bfe8
[SPARK-5875][SQL]logical.Project should not be resolved if it contains
aggregates or generators
https://issues.apache.org/jira/browse/SPARK-5875 has a case to reproduce the
bug and explain the root cause.
Author: Yin Huai
Closes #4663 from yhuai/projectResolved and squashes the following commits:
472f7b6 [Yin Huai] If a logical.Project has any AggregateExpression or
Generator, it's resolved field should be false.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d5f12bfe
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d5f12bfe
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d5f12bfe
Branch: refs/heads/master
Commit: d5f12bfe8f0a98d6fee114bb24376668ebe2898e
Parents: a51fc7e
Author: Yin Huai
Authored: Tue Feb 17 17:50:39 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 17:50:39 2015 -0800
--
.../catalyst/plans/logical/basicOperators.scala | 10 ++
.../sql/catalyst/analysis/AnalysisSuite.scala | 13 +++-
.../sql/hive/execution/SQLQuerySuite.scala | 32 +++-
3 files changed, 53 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d5f12bfe/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
index 9628e93..89544ad 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
@@ -23,6 +23,16 @@ import org.apache.spark.sql.types._
case class Project(projectList: Seq[NamedExpression], child: LogicalPlan)
extends UnaryNode {
def output = projectList.map(_.toAttribute)
+
+ override lazy val resolved: Boolean = {
+val containsAggregatesOrGenerators = projectList.exists ( _.collect {
+case agg: AggregateExpression => agg
+case generator: Generator => generator
+ }.nonEmpty
+)
+
+!expressions.exists(!_.resolved) && childrenResolved &&
!containsAggregatesOrGenerators
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/d5f12bfe/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index e70c651..aec7847 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.analysis
import org.scalatest.{BeforeAndAfter, FunSuite}
import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.{Literal, Alias,
AttributeReference}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.types._
@@ -58,6 +58,17 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
assert(caseInsensitiveAnalyze(plan).resolved)
}
+ test("check project's resolved") {
+assert(Project(testRelation.output, testRelation).resolved)
+
+assert(!Project(Seq(UnresolvedAttribute("a")), testRelation).resolved)
+
+val explode = Explode(Nil, AttributeReference("a", IntegerType, nullable =
true)())
+assert(!Project(Seq(Alias(explode, "explode")()), testRelation).resolved)
+
+assert(!Project(Seq(Alias(Count(Literal(1)), "count")()),
testRelation).resolved)
+ }
+
test("analyze project") {
assert(
caseSensitiveAnalyze(Project(Seq(UnresolvedAttribute("a")),
testRelation)) ===
http://git-wip-us.apache.org/repos/asf/spark/blob/d5f12bfe/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index e8d9eec..ae03bc5 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.hive.execution
-import or
spark git commit: [SPARK-5875][SQL]logical.Project should not be resolved if it contains aggregates or generators
Repository: spark
Updated Branches:
refs/heads/branch-1.3 7320605ad -> e8284b29d
[SPARK-5875][SQL]logical.Project should not be resolved if it contains
aggregates or generators
https://issues.apache.org/jira/browse/SPARK-5875 has a case to reproduce the
bug and explain the root cause.
Author: Yin Huai
Closes #4663 from yhuai/projectResolved and squashes the following commits:
472f7b6 [Yin Huai] If a logical.Project has any AggregateExpression or
Generator, it's resolved field should be false.
(cherry picked from commit d5f12bfe8f0a98d6fee114bb24376668ebe2898e)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e8284b29
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e8284b29
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e8284b29
Branch: refs/heads/branch-1.3
Commit: e8284b29df0445bdf92f5ea2c74402a111718792
Parents: 7320605
Author: Yin Huai
Authored: Tue Feb 17 17:50:39 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 17:50:51 2015 -0800
--
.../catalyst/plans/logical/basicOperators.scala | 10 ++
.../sql/catalyst/analysis/AnalysisSuite.scala | 13 +++-
.../sql/hive/execution/SQLQuerySuite.scala | 32 +++-
3 files changed, 53 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e8284b29/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
index 9628e93..89544ad 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
@@ -23,6 +23,16 @@ import org.apache.spark.sql.types._
case class Project(projectList: Seq[NamedExpression], child: LogicalPlan)
extends UnaryNode {
def output = projectList.map(_.toAttribute)
+
+ override lazy val resolved: Boolean = {
+val containsAggregatesOrGenerators = projectList.exists ( _.collect {
+case agg: AggregateExpression => agg
+case generator: Generator => generator
+ }.nonEmpty
+)
+
+!expressions.exists(!_.resolved) && childrenResolved &&
!containsAggregatesOrGenerators
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/e8284b29/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index e70c651..aec7847 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.analysis
import org.scalatest.{BeforeAndAfter, FunSuite}
import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.{Literal, Alias,
AttributeReference}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.types._
@@ -58,6 +58,17 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
assert(caseInsensitiveAnalyze(plan).resolved)
}
+ test("check project's resolved") {
+assert(Project(testRelation.output, testRelation).resolved)
+
+assert(!Project(Seq(UnresolvedAttribute("a")), testRelation).resolved)
+
+val explode = Explode(Nil, AttributeReference("a", IntegerType, nullable =
true)())
+assert(!Project(Seq(Alias(explode, "explode")()), testRelation).resolved)
+
+assert(!Project(Seq(Alias(Count(Literal(1)), "count")()),
testRelation).resolved)
+ }
+
test("analyze project") {
assert(
caseSensitiveAnalyze(Project(Seq(UnresolvedAttribute("a")),
testRelation)) ===
http://git-wip-us.apache.org/repos/asf/spark/blob/e8284b29/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index e8d9eec..ae03bc5 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/h
spark git commit: [SPARK-5723][SQL]Change the default file format to Parquet for CTAS statements.
Repository: spark
Updated Branches:
refs/heads/master d5f12bfe8 -> e50934f11
[SPARK-5723][SQL]Change the default file format to Parquet for CTAS statements.
JIRA: https://issues.apache.org/jira/browse/SPARK-5723
Author: Yin Huai
This patch had conflicts when merged, resolved by
Committer: Michael Armbrust
Closes #4639 from yhuai/defaultCTASFileFormat and squashes the following
commits:
a568137 [Yin Huai] Merge remote-tracking branch 'upstream/master' into
defaultCTASFileFormat
ad2b07d [Yin Huai] Update tests and error messages.
8af5b2a [Yin Huai] Update conf key and unit test.
5a67903 [Yin Huai] Use data source write path for Hive's CTAS statements when
no storage format/handler is specified.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e50934f1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e50934f1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e50934f1
Branch: refs/heads/master
Commit: e50934f11e1e3ded21a631e5ab69db3c79467137
Parents: d5f12bf
Author: Yin Huai
Authored: Tue Feb 17 18:14:33 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 18:14:33 2015 -0800
--
.../org/apache/spark/sql/hive/HiveContext.scala | 15
.../spark/sql/hive/HiveMetastoreCatalog.scala | 75
.../spark/sql/hive/execution/commands.scala | 17 +++--
.../sql/hive/MetastoreDataSourcesSuite.scala| 6 +-
.../sql/hive/execution/SQLQuerySuite.scala | 70 ++
5 files changed, 158 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e50934f1/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index 6c55bc6..d3365b1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -61,6 +61,21 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
protected[sql] def convertMetastoreParquet: Boolean =
getConf("spark.sql.hive.convertMetastoreParquet", "true") == "true"
+ /**
+ * When true, a table created by a Hive CTAS statement (no USING clause)
will be
+ * converted to a data source table, using the data source set by
spark.sql.sources.default.
+ * The table in CTAS statement will be converted when it meets any of the
following conditions:
+ * - The CTAS does not specify any of a SerDe (ROW FORMAT SERDE), a File
Format (STORED AS), or
+ * a Storage Hanlder (STORED BY), and the value of
hive.default.fileformat in hive-site.xml
+ * is either TextFile or SequenceFile.
+ * - The CTAS statement specifies TextFile (STORED AS TEXTFILE) as the
file format and no SerDe
+ * is specified (no ROW FORMAT SERDE clause).
+ * - The CTAS statement specifies SequenceFile (STORED AS SEQUENCEFILE) as
the file format
+ * and no SerDe is specified (no ROW FORMAT SERDE clause).
+ */
+ protected[sql] def convertCTAS: Boolean =
+getConf("spark.sql.hive.convertCTAS", "false").toBoolean
+
override protected[sql] def executePlan(plan: LogicalPlan):
this.QueryExecution =
new this.QueryExecution(plan)
http://git-wip-us.apache.org/repos/asf/spark/blob/e50934f1/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index cfd6f27..f7ad2ef 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.serde2.{Deserializer,
SerDeException}
import org.apache.hadoop.util.ReflectionUtils
import org.apache.spark.Logging
-import org.apache.spark.sql.{AnalysisException, SQLContext}
+import org.apache.spark.sql.{SaveMode, AnalysisException, SQLContext}
import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, Catalog,
OverrideCatalog}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.planning.PhysicalOperation
@@ -39,7 +39,7 @@ import org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.parquet.{ParquetRelation2, Partition =>
ParquetPartition, PartitionSpec}
-import org.apache.spark.sql.sources.{DDLParser, LogicalRelation,
ResolvedData
spark git commit: [SPARK-5723][SQL]Change the default file format to Parquet for CTAS statements.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 2ab0ba04f -> 6e82c46bf
[SPARK-5723][SQL]Change the default file format to Parquet for CTAS statements.
JIRA: https://issues.apache.org/jira/browse/SPARK-5723
Author: Yin Huai
This patch had conflicts when merged, resolved by
Committer: Michael Armbrust
Closes #4639 from yhuai/defaultCTASFileFormat and squashes the following
commits:
a568137 [Yin Huai] Merge remote-tracking branch 'upstream/master' into
defaultCTASFileFormat
ad2b07d [Yin Huai] Update tests and error messages.
8af5b2a [Yin Huai] Update conf key and unit test.
5a67903 [Yin Huai] Use data source write path for Hive's CTAS statements when
no storage format/handler is specified.
(cherry picked from commit e50934f11e1e3ded21a631e5ab69db3c79467137)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e82c46b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e82c46b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e82c46b
Branch: refs/heads/branch-1.3
Commit: 6e82c46bf6cdfd227b5680f114e029219204e641
Parents: 2ab0ba0
Author: Yin Huai
Authored: Tue Feb 17 18:14:33 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 17 18:14:45 2015 -0800
--
.../org/apache/spark/sql/hive/HiveContext.scala | 15
.../spark/sql/hive/HiveMetastoreCatalog.scala | 75
.../spark/sql/hive/execution/commands.scala | 17 +++--
.../sql/hive/MetastoreDataSourcesSuite.scala| 6 +-
.../sql/hive/execution/SQLQuerySuite.scala | 70 ++
5 files changed, 158 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6e82c46b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index 6c55bc6..d3365b1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -61,6 +61,21 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
protected[sql] def convertMetastoreParquet: Boolean =
getConf("spark.sql.hive.convertMetastoreParquet", "true") == "true"
+ /**
+ * When true, a table created by a Hive CTAS statement (no USING clause)
will be
+ * converted to a data source table, using the data source set by
spark.sql.sources.default.
+ * The table in CTAS statement will be converted when it meets any of the
following conditions:
+ * - The CTAS does not specify any of a SerDe (ROW FORMAT SERDE), a File
Format (STORED AS), or
+ * a Storage Hanlder (STORED BY), and the value of
hive.default.fileformat in hive-site.xml
+ * is either TextFile or SequenceFile.
+ * - The CTAS statement specifies TextFile (STORED AS TEXTFILE) as the
file format and no SerDe
+ * is specified (no ROW FORMAT SERDE clause).
+ * - The CTAS statement specifies SequenceFile (STORED AS SEQUENCEFILE) as
the file format
+ * and no SerDe is specified (no ROW FORMAT SERDE clause).
+ */
+ protected[sql] def convertCTAS: Boolean =
+getConf("spark.sql.hive.convertCTAS", "false").toBoolean
+
override protected[sql] def executePlan(plan: LogicalPlan):
this.QueryExecution =
new this.QueryExecution(plan)
http://git-wip-us.apache.org/repos/asf/spark/blob/6e82c46b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index cfd6f27..f7ad2ef 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.serde2.{Deserializer,
SerDeException}
import org.apache.hadoop.util.ReflectionUtils
import org.apache.spark.Logging
-import org.apache.spark.sql.{AnalysisException, SQLContext}
+import org.apache.spark.sql.{SaveMode, AnalysisException, SQLContext}
import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, Catalog,
OverrideCatalog}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.planning.PhysicalOperation
@@ -39,7 +39,7 @@ import org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.parquet.{ParquetRelation2, Partition =>
P
spark git commit: [SPARK-4903][SQL]Backport the bug fix for SPARK-4903
Repository: spark
Updated Branches:
refs/heads/branch-1.2 068ba45cf -> 36e15b48e
[SPARK-4903][SQL]Backport the bug fix for SPARK-4903
The original fix was a part of https://issues.apache.org/jira/browse/SPARK-4912
(commit
https://github.com/apache/spark/commit/6463e0b9e8067cce70602c5c9006a2546856a9d6).
Author: Yin Huai
Closes #4671 from yhuai/SPARK-4903-1.2 and squashes the following commits:
3168b4b [Yin Huai] Uncache table in drop table command.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/36e15b48
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/36e15b48
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/36e15b48
Branch: refs/heads/branch-1.2
Commit: 36e15b48ead360f8b870606add5cf0f7d639f126
Parents: 068ba45
Author: Yin Huai
Authored: Wed Feb 18 13:59:55 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 18 13:59:55 2015 -0800
--
.../org/apache/spark/sql/hive/execution/commands.scala | 9 +
1 file changed, 9 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/36e15b48/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
index a42a7c8..9d52a57 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
@@ -55,6 +55,15 @@ case class DropTable(tableName: String, ifExists: Boolean)
extends LeafNode with
override protected lazy val sideEffectResult: Seq[Row] = {
val ifExistsClause = if (ifExists) "IF EXISTS " else ""
+try {
+ hiveContext.tryUncacheQuery(hiveContext.table(tableName))
+} catch {
+ // This table's metadata is not in
+ case _: org.apache.hadoop.hive.ql.metadata.InvalidTableException =>
+ // Got an error during table lookup or uncache the query. We log the
exception message.
+ // Users should be able to drop such kinds of tables regardless if there
is an exception.
+ case e: Exception => log.warn(s"${e.getMessage}")
+}
hiveContext.runSqlHive(s"DROP TABLE $ifExistsClause$tableName")
hiveContext.catalog.unregisterTable(Seq(tableName))
Seq.empty[Row]
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5840][SQL] HiveContext cannot be serialized due to tuple extraction
Repository: spark
Updated Branches:
refs/heads/master a8eb92dcb -> f0e3b7107
[SPARK-5840][SQL] HiveContext cannot be serialized due to tuple extraction
Also added test cases for checking the serializability of HiveContext and
SQLContext.
Author: Reynold Xin
Closes #4628 from rxin/SPARK-5840 and squashes the following commits:
ecb3bcd [Reynold Xin] test cases and reviews.
55eb822 [Reynold Xin] [SPARK-5840][SQL] HiveContext cannot be serialized due to
tuple extraction.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f0e3b710
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f0e3b710
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f0e3b710
Branch: refs/heads/master
Commit: f0e3b71077a6c28aba29a7a75e901a9e0911b9f0
Parents: a8eb92d
Author: Reynold Xin
Authored: Wed Feb 18 14:02:32 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 18 14:02:32 2015 -0800
--
.../apache/spark/sql/SerializationSuite.scala | 32 ++
.../org/apache/spark/sql/hive/HiveContext.scala | 35 +++-
.../spark/sql/hive/SerializationSuite.scala | 33 ++
3 files changed, 84 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f0e3b710/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
new file mode 100644
index 000..6f6d3c9
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.scalatest.FunSuite
+
+import org.apache.spark.SparkConf
+import org.apache.spark.serializer.JavaSerializer
+import org.apache.spark.sql.test.TestSQLContext
+
+class SerializationSuite extends FunSuite {
+
+ test("[SPARK-5235] SQLContext should be serializable") {
+val sqlContext = new SQLContext(TestSQLContext.sparkContext)
+new JavaSerializer(new SparkConf()).newInstance().serialize(sqlContext)
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/f0e3b710/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index d3365b1..2e205e6 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -222,22 +222,25 @@ class HiveContext(sc: SparkContext) extends
SQLContext(sc) {
*SQLConf. Additionally, any properties set by set() or a SET command
inside sql() will be
*set in the SQLConf *as well as* in the HiveConf.
*/
- @transient protected[hive] lazy val (hiveconf, sessionState) =
-Option(SessionState.get())
- .orElse {
-val newState = new SessionState(new HiveConf(classOf[SessionState]))
-// Only starts newly created `SessionState` instance. Any existing
`SessionState` instance
-// returned by `SessionState.get()` must be the most recently started
one.
-SessionState.start(newState)
-Some(newState)
- }
- .map { state =>
-setConf(state.getConf.getAllProperties)
-if (state.out == null) state.out = new PrintStream(outputBuffer, true,
"UTF-8")
-if (state.err == null) state.err = new PrintStream(outputBuffer, true,
"UTF-8")
-(state.getConf, state)
- }
- .get
+ @transient protected[hive] lazy val sessionState: SessionState = {
+var state = SessionState.get()
+if (state == null) {
+ state = new SessionState(new HiveConf(classOf[SessionState]))
+ SessionState.start(state)
+}
+if (state.out == null) {
+ state.out =
spark git commit: [SPARK-5840][SQL] HiveContext cannot be serialized due to tuple extraction
Repository: spark
Updated Branches:
refs/heads/branch-1.3 56f8f295c -> b86e44cd9
[SPARK-5840][SQL] HiveContext cannot be serialized due to tuple extraction
Also added test cases for checking the serializability of HiveContext and
SQLContext.
Author: Reynold Xin
Closes #4628 from rxin/SPARK-5840 and squashes the following commits:
ecb3bcd [Reynold Xin] test cases and reviews.
55eb822 [Reynold Xin] [SPARK-5840][SQL] HiveContext cannot be serialized due to
tuple extraction.
(cherry picked from commit f0e3b71077a6c28aba29a7a75e901a9e0911b9f0)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b86e44cd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b86e44cd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b86e44cd
Branch: refs/heads/branch-1.3
Commit: b86e44cd9e7658abd2f8021120cb927bdc6ba693
Parents: 56f8f29
Author: Reynold Xin
Authored: Wed Feb 18 14:02:32 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 18 14:02:42 2015 -0800
--
.../apache/spark/sql/SerializationSuite.scala | 32 ++
.../org/apache/spark/sql/hive/HiveContext.scala | 35 +++-
.../spark/sql/hive/SerializationSuite.scala | 33 ++
3 files changed, 84 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b86e44cd/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
new file mode 100644
index 000..6f6d3c9
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SerializationSuite.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.scalatest.FunSuite
+
+import org.apache.spark.SparkConf
+import org.apache.spark.serializer.JavaSerializer
+import org.apache.spark.sql.test.TestSQLContext
+
+class SerializationSuite extends FunSuite {
+
+ test("[SPARK-5235] SQLContext should be serializable") {
+val sqlContext = new SQLContext(TestSQLContext.sparkContext)
+new JavaSerializer(new SparkConf()).newInstance().serialize(sqlContext)
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/b86e44cd/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index d3365b1..2e205e6 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -222,22 +222,25 @@ class HiveContext(sc: SparkContext) extends
SQLContext(sc) {
*SQLConf. Additionally, any properties set by set() or a SET command
inside sql() will be
*set in the SQLConf *as well as* in the HiveConf.
*/
- @transient protected[hive] lazy val (hiveconf, sessionState) =
-Option(SessionState.get())
- .orElse {
-val newState = new SessionState(new HiveConf(classOf[SessionState]))
-// Only starts newly created `SessionState` instance. Any existing
`SessionState` instance
-// returned by `SessionState.get()` must be the most recently started
one.
-SessionState.start(newState)
-Some(newState)
- }
- .map { state =>
-setConf(state.getConf.getAllProperties)
-if (state.out == null) state.out = new PrintStream(outputBuffer, true,
"UTF-8")
-if (state.err == null) state.err = new PrintStream(outputBuffer, true,
"UTF-8")
-(state.getConf, state)
- }
- .get
+ @transient protected[hive] lazy val sessionState: SessionState = {
+var state = SessionState.get()
+if (state == null) {
+ state = new SessionState(new HiveConf(cl
spark git commit: [SPARK-5722] [SQL] [PySpark] infer int as LongType
Repository: spark
Updated Branches:
refs/heads/master f0e3b7107 -> aa8f10e82
[SPARK-5722] [SQL] [PySpark] infer int as LongType
The `int` is 64-bit on 64-bit machine (very common now), we should infer it as
LongType for it in Spark SQL.
Also, LongType in SQL will come back as `int`.
Author: Davies Liu
Closes #4666 from davies/long and squashes the following commits:
6bc6cc4 [Davies Liu] infer int as LongType
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/aa8f10e8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/aa8f10e8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/aa8f10e8
Branch: refs/heads/master
Commit: aa8f10e82a743d59ce87348af19c0177eb618a66
Parents: f0e3b71
Author: Davies Liu
Authored: Wed Feb 18 14:17:04 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 18 14:17:04 2015 -0800
--
python/pyspark/sql/dataframe.py | 14 +++--
python/pyspark/sql/tests.py | 22 +++-
python/pyspark/sql/types.py | 8 +++
.../scala/org/apache/spark/sql/SQLContext.scala | 1 +
.../apache/spark/sql/execution/pythonUdfs.scala | 1 +
5 files changed, 35 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/aa8f10e8/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 52bd75b..c68c97e 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -803,7 +803,7 @@ class GroupedData(object):
>>> df.groupBy().mean('age').collect()
[Row(AVG(age#0)=3.5)]
>>> df3.groupBy().mean('age', 'height').collect()
-[Row(AVG(age#4)=3.5, AVG(height#5)=82.5)]
+[Row(AVG(age#4L)=3.5, AVG(height#5L)=82.5)]
"""
@df_varargs_api
@@ -814,7 +814,7 @@ class GroupedData(object):
>>> df.groupBy().avg('age').collect()
[Row(AVG(age#0)=3.5)]
>>> df3.groupBy().avg('age', 'height').collect()
-[Row(AVG(age#4)=3.5, AVG(height#5)=82.5)]
+[Row(AVG(age#4L)=3.5, AVG(height#5L)=82.5)]
"""
@df_varargs_api
@@ -825,7 +825,7 @@ class GroupedData(object):
>>> df.groupBy().max('age').collect()
[Row(MAX(age#0)=5)]
>>> df3.groupBy().max('age', 'height').collect()
-[Row(MAX(age#4)=5, MAX(height#5)=85)]
+[Row(MAX(age#4L)=5, MAX(height#5L)=85)]
"""
@df_varargs_api
@@ -836,7 +836,7 @@ class GroupedData(object):
>>> df.groupBy().min('age').collect()
[Row(MIN(age#0)=2)]
>>> df3.groupBy().min('age', 'height').collect()
-[Row(MIN(age#4)=2, MIN(height#5)=80)]
+[Row(MIN(age#4L)=2, MIN(height#5L)=80)]
"""
@df_varargs_api
@@ -847,7 +847,7 @@ class GroupedData(object):
>>> df.groupBy().sum('age').collect()
[Row(SUM(age#0)=7)]
>>> df3.groupBy().sum('age', 'height').collect()
-[Row(SUM(age#4)=7, SUM(height#5)=165)]
+[Row(SUM(age#4L)=7, SUM(height#5L)=165)]
"""
@@ -1051,7 +1051,9 @@ def _test():
sc = SparkContext('local[4]', 'PythonTest')
globs['sc'] = sc
globs['sqlCtx'] = SQLContext(sc)
-globs['df'] = sc.parallelize([Row(name='Alice', age=2), Row(name='Bob',
age=5)]).toDF()
+globs['df'] = sc.parallelize([(2, 'Alice'), (5, 'Bob')])\
+.toDF(StructType([StructField('age', IntegerType()),
+ StructField('name', StringType())]))
globs['df2'] = sc.parallelize([Row(name='Tom', height=80), Row(name='Bob',
height=85)]).toDF()
globs['df3'] = sc.parallelize([Row(name='Alice', age=2, height=80),
Row(name='Bob', age=5, height=85)]).toDF()
http://git-wip-us.apache.org/repos/asf/spark/blob/aa8f10e8/python/pyspark/sql/tests.py
--
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 52f7e65..8e1bb36 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -38,7 +38,7 @@ else:
from pyspark.sql import SQLContext, HiveContext, Column
from pyspark.sql.types import IntegerType, Row, ArrayType, StructType,
StructField, \
-UserDefinedType, DoubleType, LongType, StringType
+UserDefinedType, DoubleType, LongType, StringType, _infer_type
from pyspark.tests import ReusedPySparkTestCase
@@ -324,6 +324,26 @@ class SQLTests(ReusedPySparkTestCase):
pydoc.render_doc(df.foo)
pydoc.render_doc(df.take(1))
+def test_infer_long_type(self):
+longrow = [Row(f1='a', f2=100)]
+df = self.sc.parallelize(longrow).toDF()
+self.assertEqual(df.sch
spark git commit: [SPARK-5722] [SQL] [PySpark] infer int as LongType
Repository: spark
Updated Branches:
refs/heads/branch-1.3 b86e44cd9 -> 470cba82c
[SPARK-5722] [SQL] [PySpark] infer int as LongType
The `int` is 64-bit on 64-bit machine (very common now), we should infer it as
LongType for it in Spark SQL.
Also, LongType in SQL will come back as `int`.
Author: Davies Liu
Closes #4666 from davies/long and squashes the following commits:
6bc6cc4 [Davies Liu] infer int as LongType
(cherry picked from commit aa8f10e82a743d59ce87348af19c0177eb618a66)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/470cba82
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/470cba82
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/470cba82
Branch: refs/heads/branch-1.3
Commit: 470cba82c021f1ae78ae8a597e2518d9e4045e77
Parents: b86e44c
Author: Davies Liu
Authored: Wed Feb 18 14:17:04 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 18 14:17:14 2015 -0800
--
python/pyspark/sql/dataframe.py | 14 +++--
python/pyspark/sql/tests.py | 22 +++-
python/pyspark/sql/types.py | 8 +++
.../scala/org/apache/spark/sql/SQLContext.scala | 1 +
.../apache/spark/sql/execution/pythonUdfs.scala | 1 +
5 files changed, 35 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/470cba82/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 52bd75b..c68c97e 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -803,7 +803,7 @@ class GroupedData(object):
>>> df.groupBy().mean('age').collect()
[Row(AVG(age#0)=3.5)]
>>> df3.groupBy().mean('age', 'height').collect()
-[Row(AVG(age#4)=3.5, AVG(height#5)=82.5)]
+[Row(AVG(age#4L)=3.5, AVG(height#5L)=82.5)]
"""
@df_varargs_api
@@ -814,7 +814,7 @@ class GroupedData(object):
>>> df.groupBy().avg('age').collect()
[Row(AVG(age#0)=3.5)]
>>> df3.groupBy().avg('age', 'height').collect()
-[Row(AVG(age#4)=3.5, AVG(height#5)=82.5)]
+[Row(AVG(age#4L)=3.5, AVG(height#5L)=82.5)]
"""
@df_varargs_api
@@ -825,7 +825,7 @@ class GroupedData(object):
>>> df.groupBy().max('age').collect()
[Row(MAX(age#0)=5)]
>>> df3.groupBy().max('age', 'height').collect()
-[Row(MAX(age#4)=5, MAX(height#5)=85)]
+[Row(MAX(age#4L)=5, MAX(height#5L)=85)]
"""
@df_varargs_api
@@ -836,7 +836,7 @@ class GroupedData(object):
>>> df.groupBy().min('age').collect()
[Row(MIN(age#0)=2)]
>>> df3.groupBy().min('age', 'height').collect()
-[Row(MIN(age#4)=2, MIN(height#5)=80)]
+[Row(MIN(age#4L)=2, MIN(height#5L)=80)]
"""
@df_varargs_api
@@ -847,7 +847,7 @@ class GroupedData(object):
>>> df.groupBy().sum('age').collect()
[Row(SUM(age#0)=7)]
>>> df3.groupBy().sum('age', 'height').collect()
-[Row(SUM(age#4)=7, SUM(height#5)=165)]
+[Row(SUM(age#4L)=7, SUM(height#5L)=165)]
"""
@@ -1051,7 +1051,9 @@ def _test():
sc = SparkContext('local[4]', 'PythonTest')
globs['sc'] = sc
globs['sqlCtx'] = SQLContext(sc)
-globs['df'] = sc.parallelize([Row(name='Alice', age=2), Row(name='Bob',
age=5)]).toDF()
+globs['df'] = sc.parallelize([(2, 'Alice'), (5, 'Bob')])\
+.toDF(StructType([StructField('age', IntegerType()),
+ StructField('name', StringType())]))
globs['df2'] = sc.parallelize([Row(name='Tom', height=80), Row(name='Bob',
height=85)]).toDF()
globs['df3'] = sc.parallelize([Row(name='Alice', age=2, height=80),
Row(name='Bob', age=5, height=85)]).toDF()
http://git-wip-us.apache.org/repos/asf/spark/blob/470cba82/python/pyspark/sql/tests.py
--
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 52f7e65..8e1bb36 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -38,7 +38,7 @@ else:
from pyspark.sql import SQLContext, HiveContext, Column
from pyspark.sql.types import IntegerType, Row, ArrayType, StructType,
StructField, \
-UserDefinedType, DoubleType, LongType, StringType
+UserDefinedType, DoubleType, LongType, StringType, _infer_type
from pyspark.tests import ReusedPySparkTestCase
@@ -324,6 +324,26 @@ class SQLTests(ReusedPySparkTestCase):
pydoc.render_doc(df.foo)
pydoc.render_doc(df.take(1))
+def test_infer_long_type(self):
+longrow = [Row(
spark git commit: [SPARK-5904][SQL] DataFrame API fixes.
Repository: spark
Updated Branches:
refs/heads/master 94cdb05ff -> 8ca3418e1
[SPARK-5904][SQL] DataFrame API fixes.
1. Column is no longer a DataFrame to simplify class hierarchy.
2. Don't use varargs on abstract methods (see Scala compiler bug SI-9013).
Author: Reynold Xin
Closes #4686 from rxin/SPARK-5904 and squashes the following commits:
fd9b199 [Reynold Xin] Fixed Python tests.
df25cef [Reynold Xin] Non final.
5221530 [Reynold Xin] [SPARK-5904][SQL] DataFrame API fixes.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8ca3418e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8ca3418e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8ca3418e
Branch: refs/heads/master
Commit: 8ca3418e1b3e2687e75a08c185d17045a97279fb
Parents: 94cdb05
Author: Reynold Xin
Authored: Thu Feb 19 12:09:44 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 19 12:09:44 2015 -0800
--
python/pyspark/sql/dataframe.py | 56 +--
.../scala/org/apache/spark/sql/Column.scala | 223 +++--
.../org/apache/spark/sql/ComputableColumn.scala | 33 --
.../scala/org/apache/spark/sql/DataFrame.scala | 420
.../org/apache/spark/sql/DataFrameImpl.scala| 483 ---
.../org/apache/spark/sql/GroupedData.scala | 2 +-
.../apache/spark/sql/IncomputableColumn.scala | 183 ---
.../spark/sql/ColumnExpressionSuite.scala | 44 +-
.../org/apache/spark/sql/DataFrameSuite.scala | 7 +-
9 files changed, 427 insertions(+), 1024 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/8ca3418e/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index c68c97e..010c38f 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -546,7 +546,7 @@ class DataFrame(object):
def __getitem__(self, item):
""" Return the column by given name
->>> df['age'].collect()
+>>> df.select(df['age']).collect()
[Row(age=2), Row(age=5)]
>>> df[ ["name", "age"]].collect()
[Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]
@@ -555,7 +555,7 @@ class DataFrame(object):
"""
if isinstance(item, basestring):
jc = self._jdf.apply(item)
-return Column(jc, self.sql_ctx)
+return Column(jc)
elif isinstance(item, Column):
return self.filter(item)
elif isinstance(item, list):
@@ -566,13 +566,13 @@ class DataFrame(object):
def __getattr__(self, name):
""" Return the column by given name
->>> df.age.collect()
+>>> df.select(df.age).collect()
[Row(age=2), Row(age=5)]
"""
if name.startswith("__"):
raise AttributeError(name)
jc = self._jdf.apply(name)
-return Column(jc, self.sql_ctx)
+return Column(jc)
def select(self, *cols):
""" Selecting a set of expressions.
@@ -698,7 +698,7 @@ class DataFrame(object):
>>> df.withColumnRenamed('age', 'age2').collect()
[Row(age2=2, name=u'Alice'), Row(age2=5, name=u'Bob')]
"""
-cols = [Column(_to_java_column(c), self.sql_ctx).alias(new)
+cols = [Column(_to_java_column(c)).alias(new)
if c == existing else c
for c in self.columns]
return self.select(*cols)
@@ -873,15 +873,16 @@ def _unary_op(name, doc="unary operator"):
""" Create a method for given unary operator """
def _(self):
jc = getattr(self._jc, name)()
-return Column(jc, self.sql_ctx)
+return Column(jc)
_.__doc__ = doc
return _
def _func_op(name, doc=''):
def _(self):
-jc = getattr(self._sc._jvm.functions, name)(self._jc)
-return Column(jc, self.sql_ctx)
+sc = SparkContext._active_spark_context
+jc = getattr(sc._jvm.functions, name)(self._jc)
+return Column(jc)
_.__doc__ = doc
return _
@@ -892,7 +893,7 @@ def _bin_op(name, doc="binary operator"):
def _(self, other):
jc = other._jc if isinstance(other, Column) else other
njc = getattr(self._jc, name)(jc)
-return Column(njc, self.sql_ctx)
+return Column(njc)
_.__doc__ = doc
return _
@@ -903,12 +904,12 @@ def _reverse_op(name, doc="binary operator"):
def _(self, other):
jother = _create_column_from_literal(other)
jc = getattr(jother, name)(self._jc)
-return Column(jc, self.sql_ctx)
+return Column(jc)
_.__doc__ = doc
return _
-class Column(DataFrame):
+class Column(object):
"""
A column in a DataFrame.
@@ -924,9 +925,8
spark git commit: [SPARK-5904][SQL] DataFrame API fixes.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 fe00eb66e -> 55d91d92b
[SPARK-5904][SQL] DataFrame API fixes.
1. Column is no longer a DataFrame to simplify class hierarchy.
2. Don't use varargs on abstract methods (see Scala compiler bug SI-9013).
Author: Reynold Xin
Closes #4686 from rxin/SPARK-5904 and squashes the following commits:
fd9b199 [Reynold Xin] Fixed Python tests.
df25cef [Reynold Xin] Non final.
5221530 [Reynold Xin] [SPARK-5904][SQL] DataFrame API fixes.
Conflicts:
sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/55d91d92
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/55d91d92
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/55d91d92
Branch: refs/heads/branch-1.3
Commit: 55d91d92bcdd56c479138cbdd9d2354bb8db52a7
Parents: fe00eb6
Author: Reynold Xin
Authored: Thu Feb 19 12:09:44 2015 -0800
Committer: Michael Armbrust
Committed: Thu Feb 19 12:15:18 2015 -0800
--
python/pyspark/sql/dataframe.py | 56 +--
.../scala/org/apache/spark/sql/Column.scala | 223 +++--
.../org/apache/spark/sql/ComputableColumn.scala | 33 --
.../scala/org/apache/spark/sql/DataFrame.scala | 420
.../org/apache/spark/sql/DataFrameImpl.scala| 483 ---
.../org/apache/spark/sql/GroupedData.scala | 2 +-
.../apache/spark/sql/IncomputableColumn.scala | 183 ---
.../spark/sql/ColumnExpressionSuite.scala | 44 +-
.../org/apache/spark/sql/DataFrameSuite.scala | 7 +-
9 files changed, 427 insertions(+), 1024 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/55d91d92/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index c68c97e..010c38f 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -546,7 +546,7 @@ class DataFrame(object):
def __getitem__(self, item):
""" Return the column by given name
->>> df['age'].collect()
+>>> df.select(df['age']).collect()
[Row(age=2), Row(age=5)]
>>> df[ ["name", "age"]].collect()
[Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]
@@ -555,7 +555,7 @@ class DataFrame(object):
"""
if isinstance(item, basestring):
jc = self._jdf.apply(item)
-return Column(jc, self.sql_ctx)
+return Column(jc)
elif isinstance(item, Column):
return self.filter(item)
elif isinstance(item, list):
@@ -566,13 +566,13 @@ class DataFrame(object):
def __getattr__(self, name):
""" Return the column by given name
->>> df.age.collect()
+>>> df.select(df.age).collect()
[Row(age=2), Row(age=5)]
"""
if name.startswith("__"):
raise AttributeError(name)
jc = self._jdf.apply(name)
-return Column(jc, self.sql_ctx)
+return Column(jc)
def select(self, *cols):
""" Selecting a set of expressions.
@@ -698,7 +698,7 @@ class DataFrame(object):
>>> df.withColumnRenamed('age', 'age2').collect()
[Row(age2=2, name=u'Alice'), Row(age2=5, name=u'Bob')]
"""
-cols = [Column(_to_java_column(c), self.sql_ctx).alias(new)
+cols = [Column(_to_java_column(c)).alias(new)
if c == existing else c
for c in self.columns]
return self.select(*cols)
@@ -873,15 +873,16 @@ def _unary_op(name, doc="unary operator"):
""" Create a method for given unary operator """
def _(self):
jc = getattr(self._jc, name)()
-return Column(jc, self.sql_ctx)
+return Column(jc)
_.__doc__ = doc
return _
def _func_op(name, doc=''):
def _(self):
-jc = getattr(self._sc._jvm.functions, name)(self._jc)
-return Column(jc, self.sql_ctx)
+sc = SparkContext._active_spark_context
+jc = getattr(sc._jvm.functions, name)(self._jc)
+return Column(jc)
_.__doc__ = doc
return _
@@ -892,7 +893,7 @@ def _bin_op(name, doc="binary operator"):
def _(self, other):
jc = other._jc if isinstance(other, Column) else other
njc = getattr(self._jc, name)(jc)
-return Column(njc, self.sql_ctx)
+return Column(njc)
_.__doc__ = doc
return _
@@ -903,12 +904,12 @@ def _reverse_op(name, doc="binary operator"):
def _(self, other):
jother = _create_column_from_literal(other)
jc = getattr(jother, name)(self._jc)
-return Column(jc, self.sql_ctx)
+return Column(jc)
_.__doc__ = doc
return _
-class Column(
spark git commit: [SPARK-5898] [SPARK-5896] [SQL] [PySpark] create DataFrame from pandas and tuple/list
Repository: spark
Updated Branches:
refs/heads/master 4a17eedb1 -> 5b0a42cb1
[SPARK-5898] [SPARK-5896] [SQL] [PySpark] create DataFrame from pandas and
tuple/list
Fix createDataFrame() from pandas DataFrame (not tested by jenkins, depends on
SPARK-5693).
It also support to create DataFrame from plain tuple/list without column names,
`_1`, `_2` will be used as column names.
Author: Davies Liu
Closes #4679 from davies/pandas and squashes the following commits:
c0cbe0b [Davies Liu] fix tests
8466d1d [Davies Liu] fix create DataFrame from pandas
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5b0a42cb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5b0a42cb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5b0a42cb
Branch: refs/heads/master
Commit: 5b0a42cb17b840c82d3f8a5ad061d99e261ceadf
Parents: 4a17eed
Author: Davies Liu
Authored: Fri Feb 20 15:35:05 2015 -0800
Committer: Michael Armbrust
Committed: Fri Feb 20 15:35:05 2015 -0800
--
python/pyspark/sql/context.py | 12 ++--
python/pyspark/sql/tests.py | 2 +-
python/pyspark/sql/types.py | 26 +-
3 files changed, 20 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5b0a42cb/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 3f168f7..313f15e 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -351,6 +351,8 @@ class SQLContext(object):
:return: a DataFrame
>>> l = [('Alice', 1)]
+>>> sqlCtx.createDataFrame(l).collect()
+[Row(_1=u'Alice', _2=1)]
>>> sqlCtx.createDataFrame(l, ['name', 'age']).collect()
[Row(name=u'Alice', age=1)]
@@ -359,6 +361,8 @@ class SQLContext(object):
[Row(age=1, name=u'Alice')]
>>> rdd = sc.parallelize(l)
+>>> sqlCtx.createDataFrame(rdd).collect()
+[Row(_1=u'Alice', _2=1)]
>>> df = sqlCtx.createDataFrame(rdd, ['name', 'age'])
>>> df.collect()
[Row(name=u'Alice', age=1)]
@@ -377,14 +381,17 @@ class SQLContext(object):
>>> df3 = sqlCtx.createDataFrame(rdd, schema)
>>> df3.collect()
[Row(name=u'Alice', age=1)]
+
+>>> sqlCtx.createDataFrame(df.toPandas()).collect() # doctest: +SKIP
+[Row(name=u'Alice', age=1)]
"""
if isinstance(data, DataFrame):
raise TypeError("data is already a DataFrame")
if has_pandas and isinstance(data, pandas.DataFrame):
-data = self._sc.parallelize(data.to_records(index=False))
if schema is None:
schema = list(data.columns)
+data = [r.tolist() for r in data.to_records(index=False)]
if not isinstance(data, RDD):
try:
@@ -399,7 +406,8 @@ class SQLContext(object):
if isinstance(schema, (list, tuple)):
first = data.first()
if not isinstance(first, (list, tuple)):
-raise ValueError("each row in `rdd` should be list or tuple")
+raise ValueError("each row in `rdd` should be list or tuple, "
+ "but got %r" % type(first))
row_cls = Row(*schema)
schema = self._inferSchema(data.map(lambda r: row_cls(*r)),
samplingRatio)
http://git-wip-us.apache.org/repos/asf/spark/blob/5b0a42cb/python/pyspark/sql/tests.py
--
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 8e1bb36..39071e7 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -186,7 +186,7 @@ class SQLTests(ReusedPySparkTestCase):
self.assertEqual("2", row.d)
def test_infer_schema(self):
-d = [Row(l=[], d={}),
+d = [Row(l=[], d={}, s=None),
Row(l=[Row(a=1, b='s')], d={"key": Row(c=1.0, d="2")}, s="")]
rdd = self.sc.parallelize(d)
df = self.sqlCtx.createDataFrame(rdd)
http://git-wip-us.apache.org/repos/asf/spark/blob/5b0a42cb/python/pyspark/sql/types.py
--
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 9409c6f..b6e41cf 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -604,7 +604,7 @@ def _infer_type(obj):
ExamplePointUDT
"""
if obj is None:
-raise ValueError("Can not infer type for None")
+return NullType()
if hasattr(obj, '__UDT__'):
return obj.__UDT__
@@ -637,15 +637,14 @@ def _infer_schema(row):
if isinstance(row, dict):
items = sorted(row.
spark git commit: [SPARK-5898] [SPARK-5896] [SQL] [PySpark] create DataFrame from pandas and tuple/list
Repository: spark
Updated Branches:
refs/heads/branch-1.3 8c12f3114 -> 913562ae7
[SPARK-5898] [SPARK-5896] [SQL] [PySpark] create DataFrame from pandas and
tuple/list
Fix createDataFrame() from pandas DataFrame (not tested by jenkins, depends on
SPARK-5693).
It also support to create DataFrame from plain tuple/list without column names,
`_1`, `_2` will be used as column names.
Author: Davies Liu
Closes #4679 from davies/pandas and squashes the following commits:
c0cbe0b [Davies Liu] fix tests
8466d1d [Davies Liu] fix create DataFrame from pandas
(cherry picked from commit 5b0a42cb17b840c82d3f8a5ad061d99e261ceadf)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/913562ae
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/913562ae
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/913562ae
Branch: refs/heads/branch-1.3
Commit: 913562ae7c3141b2d02419828b9e364867e85d85
Parents: 8c12f31
Author: Davies Liu
Authored: Fri Feb 20 15:35:05 2015 -0800
Committer: Michael Armbrust
Committed: Fri Feb 20 15:35:21 2015 -0800
--
python/pyspark/sql/context.py | 12 ++--
python/pyspark/sql/tests.py | 2 +-
python/pyspark/sql/types.py | 26 +-
3 files changed, 20 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/913562ae/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 2e2309f..c5b361a 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -351,6 +351,8 @@ class SQLContext(object):
:return: a DataFrame
>>> l = [('Alice', 1)]
+>>> sqlCtx.createDataFrame(l).collect()
+[Row(_1=u'Alice', _2=1)]
>>> sqlCtx.createDataFrame(l, ['name', 'age']).collect()
[Row(name=u'Alice', age=1)]
@@ -359,6 +361,8 @@ class SQLContext(object):
[Row(age=1, name=u'Alice')]
>>> rdd = sc.parallelize(l)
+>>> sqlCtx.createDataFrame(rdd).collect()
+[Row(_1=u'Alice', _2=1)]
>>> df = sqlCtx.createDataFrame(rdd, ['name', 'age'])
>>> df.collect()
[Row(name=u'Alice', age=1)]
@@ -377,14 +381,17 @@ class SQLContext(object):
>>> df3 = sqlCtx.createDataFrame(rdd, schema)
>>> df3.collect()
[Row(name=u'Alice', age=1)]
+
+>>> sqlCtx.createDataFrame(df.toPandas()).collect() # doctest: +SKIP
+[Row(name=u'Alice', age=1)]
"""
if isinstance(data, DataFrame):
raise TypeError("data is already a DataFrame")
if has_pandas and isinstance(data, pandas.DataFrame):
-data = self._sc.parallelize(data.to_records(index=False))
if schema is None:
schema = list(data.columns)
+data = [r.tolist() for r in data.to_records(index=False)]
if not isinstance(data, RDD):
try:
@@ -399,7 +406,8 @@ class SQLContext(object):
if isinstance(schema, (list, tuple)):
first = data.first()
if not isinstance(first, (list, tuple)):
-raise ValueError("each row in `rdd` should be list or tuple")
+raise ValueError("each row in `rdd` should be list or tuple, "
+ "but got %r" % type(first))
row_cls = Row(*schema)
schema = self._inferSchema(data.map(lambda r: row_cls(*r)),
samplingRatio)
http://git-wip-us.apache.org/repos/asf/spark/blob/913562ae/python/pyspark/sql/tests.py
--
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 8e1bb36..39071e7 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -186,7 +186,7 @@ class SQLTests(ReusedPySparkTestCase):
self.assertEqual("2", row.d)
def test_infer_schema(self):
-d = [Row(l=[], d={}),
+d = [Row(l=[], d={}, s=None),
Row(l=[Row(a=1, b='s')], d={"key": Row(c=1.0, d="2")}, s="")]
rdd = self.sc.parallelize(d)
df = self.sqlCtx.createDataFrame(rdd)
http://git-wip-us.apache.org/repos/asf/spark/blob/913562ae/python/pyspark/sql/types.py
--
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 9409c6f..b6e41cf 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -604,7 +604,7 @@ def _infer_type(obj):
ExamplePointUDT
"""
if obj is None:
-raise ValueError("Can not infer type for None")
+return NullType()
if hasattr(obj, '__UDT__'):
return obj.__
spark git commit: [SPARK-5909][SQL] Add a clearCache command to Spark SQL's cache manager
Repository: spark
Updated Branches:
refs/heads/branch-1.3 913562ae7 -> b9a6c5c84
[SPARK-5909][SQL] Add a clearCache command to Spark SQL's cache manager
JIRA: https://issues.apache.org/jira/browse/SPARK-5909
Author: Yin Huai
Closes #4694 from yhuai/clearCache and squashes the following commits:
397ecc4 [Yin Huai] Address comments.
a2702fc [Yin Huai] Update parser.
3a54506 [Yin Huai] add isEmpty to CacheManager.
6d14460 [Yin Huai] Python clearCache.
f7b8dbd [Yin Huai] Add clear cache command.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b9a6c5c8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b9a6c5c8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b9a6c5c8
Branch: refs/heads/branch-1.3
Commit: b9a6c5c840be1cb4ec4c256920424afbe09c9b37
Parents: 913562a
Author: Yin Huai
Authored: Fri Feb 20 16:20:02 2015 +0800
Committer: Michael Armbrust
Committed: Fri Feb 20 15:37:38 2015 -0800
--
python/pyspark/sql/context.py | 4
.../scala/org/apache/spark/sql/CacheManager.scala | 6 ++
.../scala/org/apache/spark/sql/SQLContext.scala | 5 +
.../scala/org/apache/spark/sql/SparkSQLParser.scala | 11 +++
.../org/apache/spark/sql/execution/commands.scala | 15 +++
.../org/apache/spark/sql/CachedTableSuite.scala | 16
6 files changed, 53 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b9a6c5c8/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index c5b361a..313f15e 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -695,6 +695,10 @@ class SQLContext(object):
"""Removes the specified table from the in-memory cache."""
self._ssql_ctx.uncacheTable(tableName)
+def clearCache(self):
+"""Removes all cached tables from the in-memory cache. """
+self._ssql_ctx.clearCache()
+
class HiveContext(SQLContext):
http://git-wip-us.apache.org/repos/asf/spark/blob/b9a6c5c8/sql/core/src/main/scala/org/apache/spark/sql/CacheManager.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/CacheManager.scala
b/sql/core/src/main/scala/org/apache/spark/sql/CacheManager.scala
index f1949aa..ca4a127 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/CacheManager.scala
@@ -71,11 +71,17 @@ private[sql] class CacheManager(sqlContext: SQLContext)
extends Logging {
}
}
+ /** Clears all cached tables. */
private[sql] def clearCache(): Unit = writeLock {
cachedData.foreach(_.cachedRepresentation.cachedColumnBuffers.unpersist())
cachedData.clear()
}
+ /** Checks if the cache is empty. */
+ private[sql] def isEmpty: Boolean = readLock {
+cachedData.isEmpty
+ }
+
/**
* Caches the data produced by the logical representation of the given
schema rdd. Unlike
* `RDD.cache()`, the default storage level is set to be `MEMORY_AND_DISK`
because recomputing
http://git-wip-us.apache.org/repos/asf/spark/blob/b9a6c5c8/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index a6cf3cd..4bdaa02 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -217,6 +217,11 @@ class SQLContext(@transient val sparkContext: SparkContext)
*/
def uncacheTable(tableName: String): Unit =
cacheManager.uncacheTable(tableName)
+ /**
+ * Removes all cached tables from the in-memory cache.
+ */
+ def clearCache(): Unit = cacheManager.clearCache()
+
// scalastyle:off
// Disable style checker so "implicits" object can start with lowercase i
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/b9a6c5c8/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
index 00e19da..5921eaf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSQLParser.scala
@@ -23,7 +23,7 @@ import scala.util.parsing.combinator.RegexParsers
import org.apache.spark.sql.catalyst.AbstractSparkSQLParser
import org.apache.spark.sql.ca
spark git commit: [SPARK-5935][SQL] Accept MapType in the schema provided to a JSON dataset.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 ae9704010 -> 33ccad20e
[SPARK-5935][SQL] Accept MapType in the schema provided to a JSON dataset.
JIRA: https://issues.apache.org/jira/browse/SPARK-5935
Author: Yin Huai
Author: Yin Huai
Closes #4710 from yhuai/jsonMapType and squashes the following commits:
3e40390 [Yin Huai] Remove unnecessary changes.
f8e6267 [Yin Huai] Fix test.
baa36e3 [Yin Huai] Accept MapType in the schema provided to jsonFile/jsonRDD.
(cherry picked from commit 48376bfe9c97bf31279918def6c6615849c88f4d)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/33ccad20
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/33ccad20
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/33ccad20
Branch: refs/heads/branch-1.3
Commit: 33ccad20ef2a4501ea8a8c5983f88f256c3ed478
Parents: ae97040
Author: Yin Huai
Authored: Mon Feb 23 17:16:34 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 23 17:16:42 2015 -0800
--
.../org/apache/spark/sql/json/JsonRDD.scala | 3 ++
.../org/apache/spark/sql/json/JsonSuite.scala | 56
.../apache/spark/sql/json/TestJsonData.scala| 17 ++
3 files changed, 76 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/33ccad20/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index 3b8dde1..d83bdc2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -416,6 +416,9 @@ private[sql] object JsonRDD extends Logging {
case NullType => null
case ArrayType(elementType, _) =>
value.asInstanceOf[Seq[Any]].map(enforceCorrectType(_, elementType))
+case MapType(StringType, valueType, _) =>
+ val map = value.asInstanceOf[Map[String, Any]]
+ map.mapValues(enforceCorrectType(_, valueType)).map(identity)
case struct: StructType => asRow(value.asInstanceOf[Map[String, Any]],
struct)
case DateType => toDate(value)
case TimestampType => toTimestamp(value)
http://git-wip-us.apache.org/repos/asf/spark/blob/33ccad20/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
--
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
index c94e44b..005f20b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
@@ -657,6 +657,62 @@ class JsonSuite extends QueryTest {
)
}
+ test("Applying schemas with MapType") {
+val schemaWithSimpleMap = StructType(
+ StructField("map", MapType(StringType, IntegerType, true), false) :: Nil)
+val jsonWithSimpleMap = jsonRDD(mapType1, schemaWithSimpleMap)
+
+jsonWithSimpleMap.registerTempTable("jsonWithSimpleMap")
+
+checkAnswer(
+ sql("select map from jsonWithSimpleMap"),
+ Row(Map("a" -> 1)) ::
+ Row(Map("b" -> 2)) ::
+ Row(Map("c" -> 3)) ::
+ Row(Map("c" -> 1, "d" -> 4)) ::
+ Row(Map("e" -> null)) :: Nil
+)
+
+checkAnswer(
+ sql("select map['c'] from jsonWithSimpleMap"),
+ Row(null) ::
+ Row(null) ::
+ Row(3) ::
+ Row(1) ::
+ Row(null) :: Nil
+)
+
+val innerStruct = StructType(
+ StructField("field1", ArrayType(IntegerType, true), true) ::
+ StructField("field2", IntegerType, true) :: Nil)
+val schemaWithComplexMap = StructType(
+ StructField("map", MapType(StringType, innerStruct, true), false) :: Nil)
+
+val jsonWithComplexMap = jsonRDD(mapType2, schemaWithComplexMap)
+
+jsonWithComplexMap.registerTempTable("jsonWithComplexMap")
+
+checkAnswer(
+ sql("select map from jsonWithComplexMap"),
+ Row(Map("a" -> Row(Seq(1, 2, 3, null), null))) ::
+ Row(Map("b" -> Row(null, 2))) ::
+ Row(Map("c" -> Row(Seq(), 4))) ::
+ Row(Map("c" -> Row(null, 3), "d" -> Row(Seq(null), null))) ::
+ Row(Map("e" -> null)) ::
+ Row(Map("f" -> Row(null, null))) :: Nil
+)
+
+checkAnswer(
+ sql("select map['a'].field1, map['c'].field2 from jsonWithComplexMap"),
+ Row(Seq(1, 2, 3, null), null) ::
+ Row(null, null) ::
+ Row(null, 4) ::
+ Row(null, 3) ::
+ Row(null, null) ::
+ Row(null, null) :: Nil
+)
+ }
+
test("SPARK-2096 Correctly parse dot notations") {
val jsonDF = jsonRDD(complexField
spark git commit: [SPARK-5935][SQL] Accept MapType in the schema provided to a JSON dataset.
Repository: spark
Updated Branches:
refs/heads/master 59536cc87 -> 48376bfe9
[SPARK-5935][SQL] Accept MapType in the schema provided to a JSON dataset.
JIRA: https://issues.apache.org/jira/browse/SPARK-5935
Author: Yin Huai
Author: Yin Huai
Closes #4710 from yhuai/jsonMapType and squashes the following commits:
3e40390 [Yin Huai] Remove unnecessary changes.
f8e6267 [Yin Huai] Fix test.
baa36e3 [Yin Huai] Accept MapType in the schema provided to jsonFile/jsonRDD.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/48376bfe
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/48376bfe
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/48376bfe
Branch: refs/heads/master
Commit: 48376bfe9c97bf31279918def6c6615849c88f4d
Parents: 59536cc
Author: Yin Huai
Authored: Mon Feb 23 17:16:34 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 23 17:16:34 2015 -0800
--
.../org/apache/spark/sql/json/JsonRDD.scala | 3 ++
.../org/apache/spark/sql/json/JsonSuite.scala | 56
.../apache/spark/sql/json/TestJsonData.scala| 17 ++
3 files changed, 76 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/48376bfe/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index 3b8dde1..d83bdc2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -416,6 +416,9 @@ private[sql] object JsonRDD extends Logging {
case NullType => null
case ArrayType(elementType, _) =>
value.asInstanceOf[Seq[Any]].map(enforceCorrectType(_, elementType))
+case MapType(StringType, valueType, _) =>
+ val map = value.asInstanceOf[Map[String, Any]]
+ map.mapValues(enforceCorrectType(_, valueType)).map(identity)
case struct: StructType => asRow(value.asInstanceOf[Map[String, Any]],
struct)
case DateType => toDate(value)
case TimestampType => toTimestamp(value)
http://git-wip-us.apache.org/repos/asf/spark/blob/48376bfe/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
--
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
index c94e44b..005f20b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
@@ -657,6 +657,62 @@ class JsonSuite extends QueryTest {
)
}
+ test("Applying schemas with MapType") {
+val schemaWithSimpleMap = StructType(
+ StructField("map", MapType(StringType, IntegerType, true), false) :: Nil)
+val jsonWithSimpleMap = jsonRDD(mapType1, schemaWithSimpleMap)
+
+jsonWithSimpleMap.registerTempTable("jsonWithSimpleMap")
+
+checkAnswer(
+ sql("select map from jsonWithSimpleMap"),
+ Row(Map("a" -> 1)) ::
+ Row(Map("b" -> 2)) ::
+ Row(Map("c" -> 3)) ::
+ Row(Map("c" -> 1, "d" -> 4)) ::
+ Row(Map("e" -> null)) :: Nil
+)
+
+checkAnswer(
+ sql("select map['c'] from jsonWithSimpleMap"),
+ Row(null) ::
+ Row(null) ::
+ Row(3) ::
+ Row(1) ::
+ Row(null) :: Nil
+)
+
+val innerStruct = StructType(
+ StructField("field1", ArrayType(IntegerType, true), true) ::
+ StructField("field2", IntegerType, true) :: Nil)
+val schemaWithComplexMap = StructType(
+ StructField("map", MapType(StringType, innerStruct, true), false) :: Nil)
+
+val jsonWithComplexMap = jsonRDD(mapType2, schemaWithComplexMap)
+
+jsonWithComplexMap.registerTempTable("jsonWithComplexMap")
+
+checkAnswer(
+ sql("select map from jsonWithComplexMap"),
+ Row(Map("a" -> Row(Seq(1, 2, 3, null), null))) ::
+ Row(Map("b" -> Row(null, 2))) ::
+ Row(Map("c" -> Row(Seq(), 4))) ::
+ Row(Map("c" -> Row(null, 3), "d" -> Row(Seq(null), null))) ::
+ Row(Map("e" -> null)) ::
+ Row(Map("f" -> Row(null, null))) :: Nil
+)
+
+checkAnswer(
+ sql("select map['a'].field1, map['c'].field2 from jsonWithComplexMap"),
+ Row(Seq(1, 2, 3, null), null) ::
+ Row(null, null) ::
+ Row(null, 4) ::
+ Row(null, 3) ::
+ Row(null, null) ::
+ Row(null, null) :: Nil
+)
+ }
+
test("SPARK-2096 Correctly parse dot notations") {
val jsonDF = jsonRDD(complexFieldAndType2)
jsonDF.registerTempTable("jsonTable")
http://git-wip-us.apache.org/repos/asf/spark/blob/48376bf
spark git commit: [SPARK-5722] [SQL] [PySpark] infer int as LongType in Python (for 1.2 branch)
Repository: spark
Updated Branches:
refs/heads/branch-1.2 5cea859fd -> 71173de7a
[SPARK-5722] [SQL] [PySpark] infer int as LongType in Python (for 1.2 branch)
This PR change to use LongType for int in Python, when inferSchema(), because
IntegerType in SQL is not enough for int in Python (which is 64-bit on 64-bit
machines).
Closes #4521
cc dondrake marmbrus
Author: Davies Liu
Closes #4681 from davies/long2 and squashes the following commits:
05ef1c8 [Davies Liu] infer LongType for int in Python
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/71173de7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/71173de7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/71173de7
Branch: refs/heads/branch-1.2
Commit: 71173de7ac1bde60afb4e8473817ec766eda013a
Parents: 5cea859
Author: Davies Liu
Authored: Mon Feb 23 17:29:25 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 23 17:29:25 2015 -0800
--
python/pyspark/sql.py | 8 +++
python/pyspark/tests.py | 23 +++-
.../scala/org/apache/spark/sql/SQLContext.scala | 1 +
.../apache/spark/sql/execution/pythonUdfs.scala | 1 +
4 files changed, 28 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/71173de7/python/pyspark/sql.py
--
diff --git a/python/pyspark/sql.py b/python/pyspark/sql.py
index ae28847..aa5af1b 100644
--- a/python/pyspark/sql.py
+++ b/python/pyspark/sql.py
@@ -577,7 +577,7 @@ def _parse_datatype_json_value(json_value):
_type_mappings = {
type(None): NullType,
bool: BooleanType,
-int: IntegerType,
+int: LongType,
long: LongType,
float: DoubleType,
str: StringType,
@@ -926,11 +926,11 @@ def _infer_schema_type(obj, dataType):
>>> schema = _parse_schema_abstract("a b c d")
>>> row = (1, 1.0, "str", datetime.date(2014, 10, 10))
>>> _infer_schema_type(row, schema)
-StructType...IntegerType...DoubleType...StringType...DateType...
+StructType...LongType...DoubleType...StringType...DateType...
>>> row = [[1], {"key": (1, 2.0)}]
>>> schema = _parse_schema_abstract("a[] b{c d}")
>>> _infer_schema_type(row, schema)
-StructType...a,ArrayType...b,MapType(StringType,...c,IntegerType...
+StructType...a,ArrayType...b,MapType(StringType,...c,LongType...
"""
if dataType is None:
return _infer_type(obj)
@@ -985,7 +985,7 @@ def _verify_type(obj, dataType):
>>> _verify_type(None, StructType([]))
>>> _verify_type("", StringType())
->>> _verify_type(0, IntegerType())
+>>> _verify_type(0, LongType())
>>> _verify_type(range(3), ArrayType(ShortType()))
>>> _verify_type(set(), ArrayType(StringType())) # doctest:
+IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
http://git-wip-us.apache.org/repos/asf/spark/blob/71173de7/python/pyspark/tests.py
--
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
index 1349384..1fc690a 100644
--- a/python/pyspark/tests.py
+++ b/python/pyspark/tests.py
@@ -51,7 +51,7 @@ from pyspark.serializers import read_int, BatchedSerializer,
MarshalSerializer,
CloudPickleSerializer, CompressedSerializer
from pyspark.shuffle import Aggregator, InMemoryMerger, ExternalMerger,
ExternalSorter
from pyspark.sql import SQLContext, IntegerType, Row, ArrayType, StructType,
StructField, \
-UserDefinedType, DoubleType
+UserDefinedType, DoubleType, LongType, _infer_type
from pyspark import shuffle
_have_scipy = False
@@ -985,6 +985,27 @@ class SQLTests(ReusedPySparkTestCase):
point = srdd1.first().point
self.assertEquals(point, ExamplePoint(1.0, 2.0))
+def test_infer_long_type(self):
+longrow = [Row(f1='a', f2=100)]
+rdd = self.sc.parallelize(longrow)
+srdd = self.sqlCtx.inferSchema(rdd)
+self.assertEqual(srdd.schema().fields[1].dataType, LongType())
+
+# this saving as Parquet caused issues as well.
+output_dir = os.path.join(self.tempdir.name, "infer_long_type")
+srdd.saveAsParquetFile(output_dir)
+df1 = self.sqlCtx.parquetFile(output_dir)
+self.assertEquals('a', df1.first().f1)
+self.assertEquals(100, df1.first().f2)
+
+self.assertEqual(_infer_type(1), LongType())
+self.assertEqual(_infer_type(2**10), LongType())
+self.assertEqual(_infer_type(2**20), LongType(
spark git commit: [SPARK-5873][SQL] Allow viewing of partially analyzed plans in queryExecution
Repository: spark
Updated Branches:
refs/heads/branch-1.3 33ccad20e -> 2d7786ed1
[SPARK-5873][SQL] Allow viewing of partially analyzed plans in queryExecution
Author: Michael Armbrust
Closes #4684 from marmbrus/explainAnalysis and squashes the following commits:
afbaa19 [Michael Armbrust] fix python
d93278c [Michael Armbrust] fix hive
e5fa0a4 [Michael Armbrust] Merge remote-tracking branch 'origin/master' into
explainAnalysis
52119f2 [Michael Armbrust] more tests
82a5431 [Michael Armbrust] fix tests
25753d2 [Michael Armbrust] Merge remote-tracking branch 'origin/master' into
explainAnalysis
aee1e6a [Michael Armbrust] fix hive
b23a844 [Michael Armbrust] newline
de8dc51 [Michael Armbrust] more comments
acf620a [Michael Armbrust] [SPARK-5873][SQL] Show partially analyzed plans in
query execution
(cherry picked from commit 1ed57086d402c38d95cda6c3d9d7aea806609bf9)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2d7786ed
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2d7786ed
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2d7786ed
Branch: refs/heads/branch-1.3
Commit: 2d7786ed1e008b33e8b171a8f2ea30e19426ba1f
Parents: 33ccad2
Author: Michael Armbrust
Authored: Mon Feb 23 17:34:54 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 23 17:35:04 2015 -0800
--
python/pyspark/sql/context.py | 30 +++---
.../apache/spark/sql/catalyst/SqlParser.scala | 2 +
.../spark/sql/catalyst/analysis/Analyzer.scala | 83 ---
.../sql/catalyst/analysis/CheckAnalysis.scala | 105 +++
.../sql/catalyst/analysis/AnalysisSuite.scala | 35 ---
.../scala/org/apache/spark/sql/DataFrame.scala | 2 +-
.../scala/org/apache/spark/sql/SQLConf.scala| 5 +-
.../scala/org/apache/spark/sql/SQLContext.scala | 14 ++-
.../org/apache/spark/sql/sources/rules.scala| 10 +-
.../spark/sql/sources/DataSourceTest.scala | 1 -
.../apache/spark/sql/sources/InsertSuite.scala | 2 +-
.../org/apache/spark/sql/hive/HiveContext.scala | 1 -
12 files changed, 164 insertions(+), 126 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2d7786ed/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 313f15e..125933c 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -267,20 +267,20 @@ class SQLContext(object):
... StructField("byte2", ByteType(), False),
... StructField("short1", ShortType(), False),
... StructField("short2", ShortType(), False),
-... StructField("int", IntegerType(), False),
-... StructField("float", FloatType(), False),
-... StructField("date", DateType(), False),
-... StructField("time", TimestampType(), False),
-... StructField("map",
+... StructField("int1", IntegerType(), False),
+... StructField("float1", FloatType(), False),
+... StructField("date1", DateType(), False),
+... StructField("time1", TimestampType(), False),
+... StructField("map1",
... MapType(StringType(), IntegerType(), False), False),
-... StructField("struct",
+... StructField("struct1",
... StructType([StructField("b", ShortType(), False)]), False),
-... StructField("list", ArrayType(ByteType(), False), False),
-... StructField("null", DoubleType(), True)])
+... StructField("list1", ArrayType(ByteType(), False), False),
+... StructField("null1", DoubleType(), True)])
>>> df = sqlCtx.applySchema(rdd, schema)
>>> results = df.map(
-... lambda x: (x.byte1, x.byte2, x.short1, x.short2, x.int,
x.float, x.date,
-... x.time, x.map["a"], x.struct.b, x.list, x.null))
+... lambda x: (x.byte1, x.byte2, x.short1, x.short2, x.int1,
x.float1, x.date1,
+... x.time1, x.map1["a"], x.struct1.b, x.list1, x.null1))
>>> results.collect()[0] # doctest: +NORMALIZE_WHITESPACE
(127, -128, -32768, 32767, 2147483647, 1.0, datetime.date(2010, 1, 1),
datetime.datetime(2010, 1, 1, 1, 1, 1), 1, 2, [1, 2, 3], None)
@@ -288,20 +288,20 @@ class SQLContext(object):
>>> df.registerTempTable("table2")
>
spark git commit: [SPARK-5873][SQL] Allow viewing of partially analyzed plans in queryExecution
Repository: spark
Updated Branches:
refs/heads/master 48376bfe9 -> 1ed57086d
[SPARK-5873][SQL] Allow viewing of partially analyzed plans in queryExecution
Author: Michael Armbrust
Closes #4684 from marmbrus/explainAnalysis and squashes the following commits:
afbaa19 [Michael Armbrust] fix python
d93278c [Michael Armbrust] fix hive
e5fa0a4 [Michael Armbrust] Merge remote-tracking branch 'origin/master' into
explainAnalysis
52119f2 [Michael Armbrust] more tests
82a5431 [Michael Armbrust] fix tests
25753d2 [Michael Armbrust] Merge remote-tracking branch 'origin/master' into
explainAnalysis
aee1e6a [Michael Armbrust] fix hive
b23a844 [Michael Armbrust] newline
de8dc51 [Michael Armbrust] more comments
acf620a [Michael Armbrust] [SPARK-5873][SQL] Show partially analyzed plans in
query execution
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1ed57086
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1ed57086
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1ed57086
Branch: refs/heads/master
Commit: 1ed57086d402c38d95cda6c3d9d7aea806609bf9
Parents: 48376bf
Author: Michael Armbrust
Authored: Mon Feb 23 17:34:54 2015 -0800
Committer: Michael Armbrust
Committed: Mon Feb 23 17:34:54 2015 -0800
--
python/pyspark/sql/context.py | 30 +++---
.../apache/spark/sql/catalyst/SqlParser.scala | 2 +
.../spark/sql/catalyst/analysis/Analyzer.scala | 83 ---
.../sql/catalyst/analysis/CheckAnalysis.scala | 105 +++
.../sql/catalyst/analysis/AnalysisSuite.scala | 35 ---
.../scala/org/apache/spark/sql/DataFrame.scala | 2 +-
.../scala/org/apache/spark/sql/SQLConf.scala| 5 +-
.../scala/org/apache/spark/sql/SQLContext.scala | 14 ++-
.../org/apache/spark/sql/sources/rules.scala| 10 +-
.../spark/sql/sources/DataSourceTest.scala | 1 -
.../apache/spark/sql/sources/InsertSuite.scala | 2 +-
.../org/apache/spark/sql/hive/HiveContext.scala | 1 -
12 files changed, 164 insertions(+), 126 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1ed57086/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 313f15e..125933c 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -267,20 +267,20 @@ class SQLContext(object):
... StructField("byte2", ByteType(), False),
... StructField("short1", ShortType(), False),
... StructField("short2", ShortType(), False),
-... StructField("int", IntegerType(), False),
-... StructField("float", FloatType(), False),
-... StructField("date", DateType(), False),
-... StructField("time", TimestampType(), False),
-... StructField("map",
+... StructField("int1", IntegerType(), False),
+... StructField("float1", FloatType(), False),
+... StructField("date1", DateType(), False),
+... StructField("time1", TimestampType(), False),
+... StructField("map1",
... MapType(StringType(), IntegerType(), False), False),
-... StructField("struct",
+... StructField("struct1",
... StructType([StructField("b", ShortType(), False)]), False),
-... StructField("list", ArrayType(ByteType(), False), False),
-... StructField("null", DoubleType(), True)])
+... StructField("list1", ArrayType(ByteType(), False), False),
+... StructField("null1", DoubleType(), True)])
>>> df = sqlCtx.applySchema(rdd, schema)
>>> results = df.map(
-... lambda x: (x.byte1, x.byte2, x.short1, x.short2, x.int,
x.float, x.date,
-... x.time, x.map["a"], x.struct.b, x.list, x.null))
+... lambda x: (x.byte1, x.byte2, x.short1, x.short2, x.int1,
x.float1, x.date1,
+... x.time1, x.map1["a"], x.struct1.b, x.list1, x.null1))
>>> results.collect()[0] # doctest: +NORMALIZE_WHITESPACE
(127, -128, -32768, 32767, 2147483647, 1.0, datetime.date(2010, 1, 1),
datetime.datetime(2010, 1, 1, 1, 1, 1), 1, 2, [1, 2, 3], None)
@@ -288,20 +288,20 @@ class SQLContext(object):
>>> df.registerTempTable("table2")
>>> sqlCtx.sql(
... "SELECT byte1 - 1 AS byte1, byte2 + 1 AS byte2, " +
-..
spark git commit: [SPARK-5968] [SQL] Suppresses ParquetOutputCommitter WARN logs
Repository: spark
Updated Branches:
refs/heads/master cf2e41653 -> 840333133
[SPARK-5968] [SQL] Suppresses ParquetOutputCommitter WARN logs
Please refer to the [JIRA ticket] [1] for the motivation.
[1]: https://issues.apache.org/jira/browse/SPARK-5968
[https://reviewable.io/review_button.png"; height=40 alt="Review on
Reviewable"/>](https://reviewable.io/reviews/apache/spark/4744)
Author: Cheng Lian
Closes #4744 from liancheng/spark-5968 and squashes the following commits:
caac6a8 [Cheng Lian] Suppresses ParquetOutputCommitter WARN logs
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/84033313
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/84033313
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/84033313
Branch: refs/heads/master
Commit: 840333133396d443e747f62fce9967f7681fb276
Parents: cf2e416
Author: Cheng Lian
Authored: Tue Feb 24 10:45:38 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 10:45:38 2015 -0800
--
.../org/apache/spark/sql/parquet/ParquetRelation.scala | 12 +---
1 file changed, 9 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/84033313/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
index b0db994..a0d1005 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
@@ -18,11 +18,12 @@
package org.apache.spark.sql.parquet
import java.io.IOException
+import java.util.logging.Level
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.permission.FsAction
-import parquet.hadoop.ParquetOutputFormat
+import parquet.hadoop.{ParquetOutputCommitter, ParquetOutputFormat}
import parquet.hadoop.metadata.CompressionCodecName
import parquet.schema.MessageType
@@ -91,7 +92,7 @@ private[sql] object ParquetRelation {
// checks first to see if there's any handlers already set
// and if not it creates them. If this method executes prior
// to that class being loaded then:
-// 1) there's no handlers installed so there's none to
+// 1) there's no handlers installed so there's none to
// remove. But when it IS finally loaded the desired affect
// of removing them is circumvented.
// 2) The parquet.Log static initializer calls setUseParentHanders(false)
@@ -99,7 +100,7 @@ private[sql] object ParquetRelation {
//
// Therefore we need to force the class to be loaded.
// This should really be resolved by Parquet.
-Class.forName(classOf[parquet.Log].getName())
+Class.forName(classOf[parquet.Log].getName)
// Note: Logger.getLogger("parquet") has a default logger
// that appends to Console which needs to be cleared.
@@ -108,6 +109,11 @@ private[sql] object ParquetRelation {
// TODO(witgo): Need to set the log level ?
// if(parquetLogger.getLevel != null) parquetLogger.setLevel(null)
if (!parquetLogger.getUseParentHandlers)
parquetLogger.setUseParentHandlers(true)
+
+// Disables WARN log message in ParquetOutputCommitter.
+// See https://issues.apache.org/jira/browse/SPARK-5968 for details
+Class.forName(classOf[ParquetOutputCommitter].getName)
+
java.util.logging.Logger.getLogger(classOf[ParquetOutputCommitter].getName).setLevel(Level.OFF)
}
// The element type for the RDDs that this relation maps to.
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5968] [SQL] Suppresses ParquetOutputCommitter WARN logs
Repository: spark
Updated Branches:
refs/heads/branch-1.3 dd4255850 -> 2b562b043
[SPARK-5968] [SQL] Suppresses ParquetOutputCommitter WARN logs
Please refer to the [JIRA ticket] [1] for the motivation.
[1]: https://issues.apache.org/jira/browse/SPARK-5968
[https://reviewable.io/review_button.png"; height=40 alt="Review on
Reviewable"/>](https://reviewable.io/reviews/apache/spark/4744)
Author: Cheng Lian
Closes #4744 from liancheng/spark-5968 and squashes the following commits:
caac6a8 [Cheng Lian] Suppresses ParquetOutputCommitter WARN logs
(cherry picked from commit 840333133396d443e747f62fce9967f7681fb276)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2b562b04
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2b562b04
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2b562b04
Branch: refs/heads/branch-1.3
Commit: 2b562b04324229e43a48803fe9aff65ee4af21e8
Parents: dd42558
Author: Cheng Lian
Authored: Tue Feb 24 10:45:38 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 10:47:02 2015 -0800
--
.../org/apache/spark/sql/parquet/ParquetRelation.scala | 12 +---
1 file changed, 9 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2b562b04/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
index b0db994..a0d1005 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetRelation.scala
@@ -18,11 +18,12 @@
package org.apache.spark.sql.parquet
import java.io.IOException
+import java.util.logging.Level
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.permission.FsAction
-import parquet.hadoop.ParquetOutputFormat
+import parquet.hadoop.{ParquetOutputCommitter, ParquetOutputFormat}
import parquet.hadoop.metadata.CompressionCodecName
import parquet.schema.MessageType
@@ -91,7 +92,7 @@ private[sql] object ParquetRelation {
// checks first to see if there's any handlers already set
// and if not it creates them. If this method executes prior
// to that class being loaded then:
-// 1) there's no handlers installed so there's none to
+// 1) there's no handlers installed so there's none to
// remove. But when it IS finally loaded the desired affect
// of removing them is circumvented.
// 2) The parquet.Log static initializer calls setUseParentHanders(false)
@@ -99,7 +100,7 @@ private[sql] object ParquetRelation {
//
// Therefore we need to force the class to be loaded.
// This should really be resolved by Parquet.
-Class.forName(classOf[parquet.Log].getName())
+Class.forName(classOf[parquet.Log].getName)
// Note: Logger.getLogger("parquet") has a default logger
// that appends to Console which needs to be cleared.
@@ -108,6 +109,11 @@ private[sql] object ParquetRelation {
// TODO(witgo): Need to set the log level ?
// if(parquetLogger.getLevel != null) parquetLogger.setLevel(null)
if (!parquetLogger.getUseParentHandlers)
parquetLogger.setUseParentHandlers(true)
+
+// Disables WARN log message in ParquetOutputCommitter.
+// See https://issues.apache.org/jira/browse/SPARK-5968 for details
+Class.forName(classOf[ParquetOutputCommitter].getName)
+
java.util.logging.Logger.getLogger(classOf[ParquetOutputCommitter].getName).setLevel(Level.OFF)
}
// The element type for the RDDs that this relation maps to.
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5910][SQL] Support for as in selectExpr
Repository: spark
Updated Branches:
refs/heads/master 840333133 -> 0a59e45e2
[SPARK-5910][SQL] Support for as in selectExpr
Author: Michael Armbrust
Closes #4736 from marmbrus/asExprs and squashes the following commits:
5ba97e4 [Michael Armbrust] [SPARK-5910][SQL] Support for as in selectExpr
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0a59e45e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0a59e45e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0a59e45e
Branch: refs/heads/master
Commit: 0a59e45e2f2e6f00ccd5f10c79f629fb796fd8d0
Parents: 8403331
Author: Michael Armbrust
Authored: Tue Feb 24 10:49:51 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 10:49:51 2015 -0800
--
.../main/scala/org/apache/spark/sql/catalyst/SqlParser.scala | 2 +-
.../src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 6 ++
2 files changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0a59e45e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
index b16aff9..c363a5e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
@@ -40,7 +40,7 @@ class SqlParser extends AbstractSparkSQLParser {
def parseExpression(input: String): Expression = {
// Initialize the Keywords.
lexical.initialize(reservedWords)
-phrase(expression)(new lexical.Scanner(input)) match {
+phrase(projection)(new lexical.Scanner(input)) match {
case Success(plan, _) => plan
case failureOrError => sys.error(failureOrError.toString)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/0a59e45e/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
--
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 6b9b3a8..e71e9be 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -130,6 +130,12 @@ class DataFrameSuite extends QueryTest {
testData.collect().map(row => Row(math.abs(row.getInt(0)),
row.getString(1))).toSeq)
}
+ test("selectExpr with alias") {
+checkAnswer(
+ testData.selectExpr("key as k").select("k"),
+ testData.select("key").collect().toSeq)
+ }
+
test("filterExpr") {
checkAnswer(
testData.filter("key > 90"),
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5910][SQL] Support for as in selectExpr
Repository: spark
Updated Branches:
refs/heads/branch-1.3 2b562b043 -> ba5d60dda
[SPARK-5910][SQL] Support for as in selectExpr
Author: Michael Armbrust
Closes #4736 from marmbrus/asExprs and squashes the following commits:
5ba97e4 [Michael Armbrust] [SPARK-5910][SQL] Support for as in selectExpr
(cherry picked from commit 0a59e45e2f2e6f00ccd5f10c79f629fb796fd8d0)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ba5d60dd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ba5d60dd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ba5d60dd
Branch: refs/heads/branch-1.3
Commit: ba5d60dda770fe1c1f8034149495009ef65749e2
Parents: 2b562b0
Author: Michael Armbrust
Authored: Tue Feb 24 10:49:51 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 10:49:58 2015 -0800
--
.../main/scala/org/apache/spark/sql/catalyst/SqlParser.scala | 2 +-
.../src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 6 ++
2 files changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ba5d60dd/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
index b16aff9..c363a5e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
@@ -40,7 +40,7 @@ class SqlParser extends AbstractSparkSQLParser {
def parseExpression(input: String): Expression = {
// Initialize the Keywords.
lexical.initialize(reservedWords)
-phrase(expression)(new lexical.Scanner(input)) match {
+phrase(projection)(new lexical.Scanner(input)) match {
case Success(plan, _) => plan
case failureOrError => sys.error(failureOrError.toString)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/ba5d60dd/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
--
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 6b9b3a8..e71e9be 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -130,6 +130,12 @@ class DataFrameSuite extends QueryTest {
testData.collect().map(row => Row(math.abs(row.getInt(0)),
row.getString(1))).toSeq)
}
+ test("selectExpr with alias") {
+checkAnswer(
+ testData.selectExpr("key as k").select("k"),
+ testData.select("key").collect().toSeq)
+ }
+
test("filterExpr") {
checkAnswer(
testData.filter("key > 90"),
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5532][SQL] Repartition should not use external rdd representation
Repository: spark
Updated Branches:
refs/heads/master 0a59e45e2 -> 201236628
[SPARK-5532][SQL] Repartition should not use external rdd representation
Author: Michael Armbrust
Closes #4738 from marmbrus/udtRepart and squashes the following commits:
c06d7b5 [Michael Armbrust] fix compilation
91c8829 [Michael Armbrust] [SQL][SPARK-5532] Repartition should not use
external rdd representation
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/20123662
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/20123662
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/20123662
Branch: refs/heads/master
Commit: 201236628a344194f7c20ba8e9afeeaefbe9318c
Parents: 0a59e45
Author: Michael Armbrust
Authored: Tue Feb 24 10:52:18 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 10:52:18 2015 -0800
--
.../main/scala/org/apache/spark/sql/DataFrame.scala | 5 +++--
.../apache/spark/sql/execution/debug/package.scala | 1 +
.../org/apache/spark/sql/UserDefinedTypeSuite.scala | 16 +++-
3 files changed, 19 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/20123662/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 27ac398..04bf5d9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -799,14 +799,15 @@ class DataFrame protected[sql](
* Returns the number of rows in the [[DataFrame]].
* @group action
*/
- override def count(): Long = groupBy().count().rdd.collect().head.getLong(0)
+ override def count(): Long = groupBy().count().collect().head.getLong(0)
/**
* Returns a new [[DataFrame]] that has exactly `numPartitions` partitions.
* @group rdd
*/
override def repartition(numPartitions: Int): DataFrame = {
-sqlContext.createDataFrame(rdd.repartition(numPartitions), schema)
+sqlContext.createDataFrame(
+ queryExecution.toRdd.map(_.copy()).repartition(numPartitions), schema)
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/20123662/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
index 73162b2..ffe388c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
@@ -168,6 +168,7 @@ package object debug {
case (_: Short, ShortType) =>
case (_: Boolean, BooleanType) =>
case (_: Double, DoubleType) =>
+ case (v, udt: UserDefinedType[_]) => typeCheck(v, udt.sqlType)
case (d, t) => sys.error(s"Invalid data found: got $d (${d.getClass})
expected $t")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/20123662/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index 5f21d99..9c098df 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -17,10 +17,11 @@
package org.apache.spark.sql
+import java.io.File
+
import scala.beans.{BeanInfo, BeanProperty}
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.functions._
import org.apache.spark.sql.test.TestSQLContext
import org.apache.spark.sql.test.TestSQLContext.{sparkContext, sql}
import org.apache.spark.sql.test.TestSQLContext.implicits._
@@ -91,4 +92,17 @@ class UserDefinedTypeSuite extends QueryTest {
sql("SELECT testType(features) from points"),
Seq(Row(true), Row(true)))
}
+
+
+ test("UDTs with Parquet") {
+val tempDir = File.createTempFile("parquet", "test")
+tempDir.delete()
+pointsRDD.saveAsParquetFile(tempDir.getCanonicalPath)
+ }
+
+ test("Repartition UDTs with Parquet") {
+val tempDir = File.createTempFile("parquet", "test")
+tempDir.delete()
+pointsRDD.repartition(1).saveAsParquetFile(tempDir.getCanonicalPath)
+ }
}
-
To un
spark git commit: [SPARK-5532][SQL] Repartition should not use external rdd representation
Repository: spark
Updated Branches:
refs/heads/branch-1.3 ba5d60dda -> e46096b1e
[SPARK-5532][SQL] Repartition should not use external rdd representation
Author: Michael Armbrust
Closes #4738 from marmbrus/udtRepart and squashes the following commits:
c06d7b5 [Michael Armbrust] fix compilation
91c8829 [Michael Armbrust] [SQL][SPARK-5532] Repartition should not use
external rdd representation
(cherry picked from commit 201236628a344194f7c20ba8e9afeeaefbe9318c)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e46096b1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e46096b1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e46096b1
Branch: refs/heads/branch-1.3
Commit: e46096b1e9f173c9f65425c3980b6f32edb3bf99
Parents: ba5d60d
Author: Michael Armbrust
Authored: Tue Feb 24 10:52:18 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 10:52:32 2015 -0800
--
.../main/scala/org/apache/spark/sql/DataFrame.scala | 5 +++--
.../apache/spark/sql/execution/debug/package.scala | 1 +
.../org/apache/spark/sql/UserDefinedTypeSuite.scala | 16 +++-
3 files changed, 19 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e46096b1/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 27ac398..04bf5d9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -799,14 +799,15 @@ class DataFrame protected[sql](
* Returns the number of rows in the [[DataFrame]].
* @group action
*/
- override def count(): Long = groupBy().count().rdd.collect().head.getLong(0)
+ override def count(): Long = groupBy().count().collect().head.getLong(0)
/**
* Returns a new [[DataFrame]] that has exactly `numPartitions` partitions.
* @group rdd
*/
override def repartition(numPartitions: Int): DataFrame = {
-sqlContext.createDataFrame(rdd.repartition(numPartitions), schema)
+sqlContext.createDataFrame(
+ queryExecution.toRdd.map(_.copy()).repartition(numPartitions), schema)
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/e46096b1/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
index 73162b2..ffe388c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
@@ -168,6 +168,7 @@ package object debug {
case (_: Short, ShortType) =>
case (_: Boolean, BooleanType) =>
case (_: Double, DoubleType) =>
+ case (v, udt: UserDefinedType[_]) => typeCheck(v, udt.sqlType)
case (d, t) => sys.error(s"Invalid data found: got $d (${d.getClass})
expected $t")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/e46096b1/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index 5f21d99..9c098df 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -17,10 +17,11 @@
package org.apache.spark.sql
+import java.io.File
+
import scala.beans.{BeanInfo, BeanProperty}
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.functions._
import org.apache.spark.sql.test.TestSQLContext
import org.apache.spark.sql.test.TestSQLContext.{sparkContext, sql}
import org.apache.spark.sql.test.TestSQLContext.implicits._
@@ -91,4 +92,17 @@ class UserDefinedTypeSuite extends QueryTest {
sql("SELECT testType(features) from points"),
Seq(Row(true), Row(true)))
}
+
+
+ test("UDTs with Parquet") {
+val tempDir = File.createTempFile("parquet", "test")
+tempDir.delete()
+pointsRDD.saveAsParquetFile(tempDir.getCanonicalPath)
+ }
+
+ test("Repartition UDTs with Parquet") {
+val tempDir = File.createTempFile("parquet", "test")
+tempDir.delete()
+pointsRDD.repartition(1).saveAsParquetFile(tempDir.ge
spark git commit: [SPARK-5952][SQL] Lock when using hive metastore client
Repository: spark
Updated Branches:
refs/heads/master c5ba975ee -> a2b913792
[SPARK-5952][SQL] Lock when using hive metastore client
Author: Michael Armbrust
Closes #4746 from marmbrus/hiveLock and squashes the following commits:
8b871cf [Michael Armbrust] [SPARK-5952][SQL] Lock when using hive metastore
client
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a2b91379
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a2b91379
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a2b91379
Branch: refs/heads/master
Commit: a2b9137923e0ba328da8fff2fbbfcf2abf50b033
Parents: c5ba975
Author: Michael Armbrust
Authored: Tue Feb 24 13:39:29 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 13:39:29 2015 -0800
--
.../spark/sql/hive/HiveMetastoreCatalog.scala | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a2b91379/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index f7ad2ef..2cc8d65 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -52,6 +52,7 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
/** Connection to hive metastore. Usages should lock on `this`. */
protected[hive] val client = Hive.get(hive.hiveconf)
+ /** Usages should lock on `this`. */
protected[hive] lazy val hiveWarehouse = new Warehouse(hive.hiveconf)
// TODO: Use this everywhere instead of tuples or databaseName, tableName,.
@@ -65,7 +66,9 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
val cacheLoader = new CacheLoader[QualifiedTableName, LogicalPlan]() {
override def load(in: QualifiedTableName): LogicalPlan = {
logDebug(s"Creating new cached data source for $in")
-val table = client.getTable(in.database, in.name)
+val table = synchronized {
+ client.getTable(in.database, in.name)
+}
val schemaString = table.getProperty("spark.sql.sources.schema")
val userSpecifiedSchema =
if (schemaString == null) {
@@ -134,15 +137,18 @@ private[hive] class HiveMetastoreCatalog(hive:
HiveContext) extends Catalog with
}
}
- def hiveDefaultTableFilePath(tableName: String): String = {
+ def hiveDefaultTableFilePath(tableName: String): String = synchronized {
val currentDatabase =
client.getDatabase(hive.sessionState.getCurrentDatabase)
+
hiveWarehouse.getTablePath(currentDatabase, tableName).toString
}
- def tableExists(tableIdentifier: Seq[String]): Boolean = {
+ def tableExists(tableIdentifier: Seq[String]): Boolean = synchronized {
val tableIdent = processTableIdentifier(tableIdentifier)
-val databaseName = tableIdent.lift(tableIdent.size - 2).getOrElse(
- hive.sessionState.getCurrentDatabase)
+val databaseName =
+ tableIdent
+.lift(tableIdent.size - 2)
+.getOrElse(hive.sessionState.getCurrentDatabase)
val tblName = tableIdent.last
client.getTable(databaseName, tblName, false) != null
}
@@ -219,7 +225,7 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
}
}
- override def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
= {
+ override def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
= synchronized {
val dbName = if (!caseSensitive) {
if (databaseName.isDefined) Some(databaseName.get.toLowerCase) else None
} else {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5952][SQL] Lock when using hive metastore client
Repository: spark
Updated Branches:
refs/heads/branch-1.3 a4ff445a9 -> 641423dbf
[SPARK-5952][SQL] Lock when using hive metastore client
Author: Michael Armbrust
Closes #4746 from marmbrus/hiveLock and squashes the following commits:
8b871cf [Michael Armbrust] [SPARK-5952][SQL] Lock when using hive metastore
client
(cherry picked from commit a2b9137923e0ba328da8fff2fbbfcf2abf50b033)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/641423db
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/641423db
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/641423db
Branch: refs/heads/branch-1.3
Commit: 641423dbffd9333ea8d989d0afa7b78426bd3979
Parents: a4ff445
Author: Michael Armbrust
Authored: Tue Feb 24 13:39:29 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 13:39:42 2015 -0800
--
.../spark/sql/hive/HiveMetastoreCatalog.scala | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/641423db/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index f7ad2ef..2cc8d65 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -52,6 +52,7 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
/** Connection to hive metastore. Usages should lock on `this`. */
protected[hive] val client = Hive.get(hive.hiveconf)
+ /** Usages should lock on `this`. */
protected[hive] lazy val hiveWarehouse = new Warehouse(hive.hiveconf)
// TODO: Use this everywhere instead of tuples or databaseName, tableName,.
@@ -65,7 +66,9 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
val cacheLoader = new CacheLoader[QualifiedTableName, LogicalPlan]() {
override def load(in: QualifiedTableName): LogicalPlan = {
logDebug(s"Creating new cached data source for $in")
-val table = client.getTable(in.database, in.name)
+val table = synchronized {
+ client.getTable(in.database, in.name)
+}
val schemaString = table.getProperty("spark.sql.sources.schema")
val userSpecifiedSchema =
if (schemaString == null) {
@@ -134,15 +137,18 @@ private[hive] class HiveMetastoreCatalog(hive:
HiveContext) extends Catalog with
}
}
- def hiveDefaultTableFilePath(tableName: String): String = {
+ def hiveDefaultTableFilePath(tableName: String): String = synchronized {
val currentDatabase =
client.getDatabase(hive.sessionState.getCurrentDatabase)
+
hiveWarehouse.getTablePath(currentDatabase, tableName).toString
}
- def tableExists(tableIdentifier: Seq[String]): Boolean = {
+ def tableExists(tableIdentifier: Seq[String]): Boolean = synchronized {
val tableIdent = processTableIdentifier(tableIdentifier)
-val databaseName = tableIdent.lift(tableIdent.size - 2).getOrElse(
- hive.sessionState.getCurrentDatabase)
+val databaseName =
+ tableIdent
+.lift(tableIdent.size - 2)
+.getOrElse(hive.sessionState.getCurrentDatabase)
val tblName = tableIdent.last
client.getTable(databaseName, tblName, false) != null
}
@@ -219,7 +225,7 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext)
extends Catalog with
}
}
- override def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
= {
+ override def getTables(databaseName: Option[String]): Seq[(String, Boolean)]
= synchronized {
val dbName = if (!caseSensitive) {
if (databaseName.isDefined) Some(databaseName.get.toLowerCase) else None
} else {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5904][SQL] DataFrame Java API test suites.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 17ee2460a -> 78a1781a9
[SPARK-5904][SQL] DataFrame Java API test suites.
Added a new test suite to make sure Java DF programs can use varargs properly.
Also moved all suites into test.org.apache.spark package to make sure the
suites also test for method visibility.
Author: Reynold Xin
Closes #4751 from rxin/df-tests and squashes the following commits:
1e8b8e4 [Reynold Xin] Fixed imports and renamed JavaAPISuite.
a6ca53b [Reynold Xin] [SPARK-5904][SQL] DataFrame Java API test suites.
(cherry picked from commit 53a1ebf33b5c349ae3a40d7eebf357b839b363af)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/78a1781a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/78a1781a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/78a1781a
Branch: refs/heads/branch-1.3
Commit: 78a1781a96df476a63e1cd61e89107fa4ed7bf55
Parents: 17ee246
Author: Reynold Xin
Authored: Tue Feb 24 18:51:41 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 18:51:53 2015 -0800
--
.../apache/spark/sql/api/java/JavaAPISuite.java | 86
.../sql/api/java/JavaApplySchemaSuite.java | 206 ---
.../org/apache/spark/sql/api/java/JavaDsl.java | 120 ---
.../apache/spark/sql/api/java/JavaRowSuite.java | 179
.../spark/sql/sources/JavaSaveLoadSuite.java| 97 -
.../apache/spark/sql/JavaApplySchemaSuite.java | 204 ++
.../apache/spark/sql/JavaDataFrameSuite.java| 84
.../test/org/apache/spark/sql/JavaRowSuite.java | 179
.../test/org/apache/spark/sql/JavaUDFSuite.java | 88
.../spark/sql/sources/JavaSaveLoadSuite.java| 98 +
.../org/apache/spark/sql/DataFrameSuite.scala | 4 +-
11 files changed, 655 insertions(+), 690 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/78a1781a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaAPISuite.java
--
diff --git
a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaAPISuite.java
b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaAPISuite.java
deleted file mode 100644
index a21a154..000
--- a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaAPISuite.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java;
-
-import java.io.Serializable;
-
-import org.apache.spark.sql.test.TestSQLContext$;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.types.DataTypes;
-
-// The test suite itself is Serializable so that anonymous Function
implementations can be
-// serialized, as an alternative to converting these anonymous classes to
static inner classes;
-// see http://stackoverflow.com/questions/758570/.
-public class JavaAPISuite implements Serializable {
- private transient JavaSparkContext sc;
- private transient SQLContext sqlContext;
-
- @Before
- public void setUp() {
-sqlContext = TestSQLContext$.MODULE$;
-sc = new JavaSparkContext(sqlContext.sparkContext());
- }
-
- @After
- public void tearDown() {
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void udf1Test() {
-// With Java 8 lambdas:
-// sqlContext.registerFunction(
-// "stringLengthTest", (String str) -> str.length(),
DataType.IntegerType);
-
-sqlContext.udf().register("stringLengthTest", new UDF1() {
- @Override
- public Integer call(String str) throws Exception {
-return str.length();
- }
-}, DataTypes.IntegerType);
-
-Row result = sqlContext.sql("SELECT stringLengthTest('test')").head();
-assert(result.getInt(0) == 4);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void udf2Test() {
spark git commit: [SPARK-5904][SQL] DataFrame Java API test suites.
Repository: spark
Updated Branches:
refs/heads/master f816e7390 -> 53a1ebf33
[SPARK-5904][SQL] DataFrame Java API test suites.
Added a new test suite to make sure Java DF programs can use varargs properly.
Also moved all suites into test.org.apache.spark package to make sure the
suites also test for method visibility.
Author: Reynold Xin
Closes #4751 from rxin/df-tests and squashes the following commits:
1e8b8e4 [Reynold Xin] Fixed imports and renamed JavaAPISuite.
a6ca53b [Reynold Xin] [SPARK-5904][SQL] DataFrame Java API test suites.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/53a1ebf3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/53a1ebf3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/53a1ebf3
Branch: refs/heads/master
Commit: 53a1ebf33b5c349ae3a40d7eebf357b839b363af
Parents: f816e73
Author: Reynold Xin
Authored: Tue Feb 24 18:51:41 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 18:51:41 2015 -0800
--
.../apache/spark/sql/api/java/JavaAPISuite.java | 86
.../sql/api/java/JavaApplySchemaSuite.java | 206 ---
.../org/apache/spark/sql/api/java/JavaDsl.java | 120 ---
.../apache/spark/sql/api/java/JavaRowSuite.java | 179
.../spark/sql/sources/JavaSaveLoadSuite.java| 97 -
.../apache/spark/sql/JavaApplySchemaSuite.java | 204 ++
.../apache/spark/sql/JavaDataFrameSuite.java| 84
.../test/org/apache/spark/sql/JavaRowSuite.java | 179
.../test/org/apache/spark/sql/JavaUDFSuite.java | 88
.../spark/sql/sources/JavaSaveLoadSuite.java| 98 +
.../org/apache/spark/sql/DataFrameSuite.scala | 4 +-
11 files changed, 655 insertions(+), 690 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/53a1ebf3/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaAPISuite.java
--
diff --git
a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaAPISuite.java
b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaAPISuite.java
deleted file mode 100644
index a21a154..000
--- a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaAPISuite.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java;
-
-import java.io.Serializable;
-
-import org.apache.spark.sql.test.TestSQLContext$;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.types.DataTypes;
-
-// The test suite itself is Serializable so that anonymous Function
implementations can be
-// serialized, as an alternative to converting these anonymous classes to
static inner classes;
-// see http://stackoverflow.com/questions/758570/.
-public class JavaAPISuite implements Serializable {
- private transient JavaSparkContext sc;
- private transient SQLContext sqlContext;
-
- @Before
- public void setUp() {
-sqlContext = TestSQLContext$.MODULE$;
-sc = new JavaSparkContext(sqlContext.sparkContext());
- }
-
- @After
- public void tearDown() {
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void udf1Test() {
-// With Java 8 lambdas:
-// sqlContext.registerFunction(
-// "stringLengthTest", (String str) -> str.length(),
DataType.IntegerType);
-
-sqlContext.udf().register("stringLengthTest", new UDF1() {
- @Override
- public Integer call(String str) throws Exception {
-return str.length();
- }
-}, DataTypes.IntegerType);
-
-Row result = sqlContext.sql("SELECT stringLengthTest('test')").head();
-assert(result.getInt(0) == 4);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void udf2Test() {
-// With Java 8 lambdas:
-// sqlContext.registerFunction(
-// "stringLengthTest",
-// (Str
spark git commit: [SPARK-5985][SQL] DataFrame sortBy -> orderBy in Python.
Repository: spark
Updated Branches:
refs/heads/master 53a1ebf33 -> fba11c2f5
[SPARK-5985][SQL] DataFrame sortBy -> orderBy in Python.
Also added desc/asc function for constructing sorting expressions more
conveniently. And added a small fix to lift alias out of cast expression.
Author: Reynold Xin
Closes #4752 from rxin/SPARK-5985 and squashes the following commits:
aeda5ae [Reynold Xin] Added Experimental flag to ColumnName.
047ad03 [Reynold Xin] Lift alias out of cast.
c9cf17c [Reynold Xin] [SPARK-5985][SQL] DataFrame sortBy -> orderBy in Python.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fba11c2f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fba11c2f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fba11c2f
Branch: refs/heads/master
Commit: fba11c2f55dd81e4f6230e7edca3c7b2e01ccd9d
Parents: 53a1ebf
Author: Reynold Xin
Authored: Tue Feb 24 18:59:23 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 18:59:23 2015 -0800
--
python/pyspark/sql/dataframe.py | 11 ++--
python/pyspark/sql/functions.py | 3 ++
.../scala/org/apache/spark/sql/Column.scala | 13 +++--
.../scala/org/apache/spark/sql/functions.scala | 29
.../spark/sql/ColumnExpressionSuite.scala | 4 +++
.../org/apache/spark/sql/DataFrameSuite.scala | 4 +++
6 files changed, 59 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/fba11c2f/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 010c38f..6f746d1 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -504,13 +504,18 @@ class DataFrame(object):
return DataFrame(jdf, self.sql_ctx)
def sort(self, *cols):
-""" Return a new :class:`DataFrame` sorted by the specified column.
+""" Return a new :class:`DataFrame` sorted by the specified column(s).
:param cols: The columns or expressions used for sorting
>>> df.sort(df.age.desc()).collect()
[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
->>> df.sortBy(df.age.desc()).collect()
+>>> df.orderBy(df.age.desc()).collect()
+[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
+>>> from pyspark.sql.functions import *
+>>> df.sort(asc("age")).collect()
+[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
+>>> df.orderBy(desc("age"), "name").collect()
[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
"""
if not cols:
@@ -520,7 +525,7 @@ class DataFrame(object):
jdf = self._jdf.sort(self._sc._jvm.PythonUtils.toSeq(jcols))
return DataFrame(jdf, self.sql_ctx)
-sortBy = sort
+orderBy = sort
def head(self, n=None):
""" Return the first `n` rows or the first row if n is None.
http://git-wip-us.apache.org/repos/asf/spark/blob/fba11c2f/python/pyspark/sql/functions.py
--
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index fc61162..8aa4476 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -48,6 +48,9 @@ _functions = {
'lit': 'Creates a :class:`Column` of literal value.',
'col': 'Returns a :class:`Column` based on the given column name.',
'column': 'Returns a :class:`Column` based on the given column name.',
+'asc': 'Returns a sort expression based on the ascending order of the
given column name.',
+'desc': 'Returns a sort expression based on the descending order of the
given column name.',
+
'upper': 'Converts a string expression to upper case.',
'lower': 'Converts a string expression to upper case.',
'sqrt': 'Computes the square root of the specified float value.',
http://git-wip-us.apache.org/repos/asf/spark/blob/fba11c2f/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 9807543..a2cc9a9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -600,7 +600,11 @@ class Column(protected[sql] val expr: Expression) {
*
* @group expr_ops
*/
- def cast(to: DataType): Column = Cast(expr, to)
+ def cast(to: DataType): Column = expr match {
+// Lift alias out of cast so we can support
col.as("name").cast(IntegerType)
+case Alias(childExpr, name) => Alias(Cast(childExpr
spark git commit: [SPARK-5985][SQL] DataFrame sortBy -> orderBy in Python.
Repository: spark
Updated Branches:
refs/heads/branch-1.3 78a1781a9 -> 5e233b2c7
[SPARK-5985][SQL] DataFrame sortBy -> orderBy in Python.
Also added desc/asc function for constructing sorting expressions more
conveniently. And added a small fix to lift alias out of cast expression.
Author: Reynold Xin
Closes #4752 from rxin/SPARK-5985 and squashes the following commits:
aeda5ae [Reynold Xin] Added Experimental flag to ColumnName.
047ad03 [Reynold Xin] Lift alias out of cast.
c9cf17c [Reynold Xin] [SPARK-5985][SQL] DataFrame sortBy -> orderBy in Python.
(cherry picked from commit fba11c2f55dd81e4f6230e7edca3c7b2e01ccd9d)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5e233b2c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5e233b2c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5e233b2c
Branch: refs/heads/branch-1.3
Commit: 5e233b2c7ec67ca87182230f9bb5fd8121f26419
Parents: 78a1781
Author: Reynold Xin
Authored: Tue Feb 24 18:59:23 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 18:59:35 2015 -0800
--
python/pyspark/sql/dataframe.py | 11 ++--
python/pyspark/sql/functions.py | 3 ++
.../scala/org/apache/spark/sql/Column.scala | 13 +++--
.../scala/org/apache/spark/sql/functions.scala | 29
.../spark/sql/ColumnExpressionSuite.scala | 4 +++
.../org/apache/spark/sql/DataFrameSuite.scala | 4 +++
6 files changed, 59 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5e233b2c/python/pyspark/sql/dataframe.py
--
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 010c38f..6f746d1 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -504,13 +504,18 @@ class DataFrame(object):
return DataFrame(jdf, self.sql_ctx)
def sort(self, *cols):
-""" Return a new :class:`DataFrame` sorted by the specified column.
+""" Return a new :class:`DataFrame` sorted by the specified column(s).
:param cols: The columns or expressions used for sorting
>>> df.sort(df.age.desc()).collect()
[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
->>> df.sortBy(df.age.desc()).collect()
+>>> df.orderBy(df.age.desc()).collect()
+[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
+>>> from pyspark.sql.functions import *
+>>> df.sort(asc("age")).collect()
+[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
+>>> df.orderBy(desc("age"), "name").collect()
[Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]
"""
if not cols:
@@ -520,7 +525,7 @@ class DataFrame(object):
jdf = self._jdf.sort(self._sc._jvm.PythonUtils.toSeq(jcols))
return DataFrame(jdf, self.sql_ctx)
-sortBy = sort
+orderBy = sort
def head(self, n=None):
""" Return the first `n` rows or the first row if n is None.
http://git-wip-us.apache.org/repos/asf/spark/blob/5e233b2c/python/pyspark/sql/functions.py
--
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index fc61162..8aa4476 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -48,6 +48,9 @@ _functions = {
'lit': 'Creates a :class:`Column` of literal value.',
'col': 'Returns a :class:`Column` based on the given column name.',
'column': 'Returns a :class:`Column` based on the given column name.',
+'asc': 'Returns a sort expression based on the ascending order of the
given column name.',
+'desc': 'Returns a sort expression based on the descending order of the
given column name.',
+
'upper': 'Converts a string expression to upper case.',
'lower': 'Converts a string expression to upper case.',
'sqrt': 'Computes the square root of the specified float value.',
http://git-wip-us.apache.org/repos/asf/spark/blob/5e233b2c/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 9807543..a2cc9a9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -600,7 +600,11 @@ class Column(protected[sql] val expr: Expression) {
*
* @group expr_ops
*/
- def cast(to: DataType): Column = Cast(expr, to)
+ def cast(to: DataType): Column = expr match {
+// Lift alias out of ca
spark git commit: [SPARK-5286][SQL] SPARK-5286 followup
Repository: spark
Updated Branches:
refs/heads/master 922b43b3c -> 769e092bd
[SPARK-5286][SQL] SPARK-5286 followup
https://issues.apache.org/jira/browse/SPARK-5286
Author: Yin Huai
Closes #4755 from yhuai/SPARK-5286-throwable and squashes the following commits:
4c0c450 [Yin Huai] Catch Throwable instead of Exception.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/769e092b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/769e092b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/769e092b
Branch: refs/heads/master
Commit: 769e092bdc51582372093f76dbaece27149cc4ea
Parents: 922b43b
Author: Yin Huai
Authored: Tue Feb 24 19:51:36 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 19:51:36 2015 -0800
--
.../scala/org/apache/spark/sql/hive/execution/commands.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/769e092b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
index c88d0e6..9934a5d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
@@ -63,10 +63,10 @@ case class DropTable(
} catch {
// This table's metadata is not in
case _: org.apache.hadoop.hive.ql.metadata.InvalidTableException =>
- // Other exceptions can be caused by users providing wrong parameters in
OPTIONS
+ // Other Throwables can be caused by users providing wrong parameters in
OPTIONS
// (e.g. invalid paths). We catch it and log a warning message.
- // Users should be able to drop such kinds of tables regardless if there
is an exception.
- case e: Exception => log.warn(s"${e.getMessage}")
+ // Users should be able to drop such kinds of tables regardless if there
is an error.
+ case e: Throwable => log.warn(s"${e.getMessage}")
}
hiveContext.invalidateTable(tableName)
hiveContext.runSqlHive(s"DROP TABLE $ifExistsClause$tableName")
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5286][SQL] SPARK-5286 followup
Repository: spark
Updated Branches:
refs/heads/branch-1.3 1e9489422 -> e7a748ecf
[SPARK-5286][SQL] SPARK-5286 followup
https://issues.apache.org/jira/browse/SPARK-5286
Author: Yin Huai
Closes #4755 from yhuai/SPARK-5286-throwable and squashes the following commits:
4c0c450 [Yin Huai] Catch Throwable instead of Exception.
(cherry picked from commit 769e092bdc51582372093f76dbaece27149cc4ea)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e7a748ec
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e7a748ec
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e7a748ec
Branch: refs/heads/branch-1.3
Commit: e7a748ecfdb22bd7b64d95bbb19347c36ca9ceb2
Parents: 1e94894
Author: Yin Huai
Authored: Tue Feb 24 19:51:36 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 19:51:48 2015 -0800
--
.../scala/org/apache/spark/sql/hive/execution/commands.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e7a748ec/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
index c88d0e6..9934a5d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
@@ -63,10 +63,10 @@ case class DropTable(
} catch {
// This table's metadata is not in
case _: org.apache.hadoop.hive.ql.metadata.InvalidTableException =>
- // Other exceptions can be caused by users providing wrong parameters in
OPTIONS
+ // Other Throwables can be caused by users providing wrong parameters in
OPTIONS
// (e.g. invalid paths). We catch it and log a warning message.
- // Users should be able to drop such kinds of tables regardless if there
is an exception.
- case e: Exception => log.warn(s"${e.getMessage}")
+ // Users should be able to drop such kinds of tables regardless if there
is an error.
+ case e: Throwable => log.warn(s"${e.getMessage}")
}
hiveContext.invalidateTable(tableName)
hiveContext.runSqlHive(s"DROP TABLE $ifExistsClause$tableName")
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5994] [SQL] Python DataFrame documentation fixes
Repository: spark
Updated Branches:
refs/heads/master 769e092bd -> d641fbb39
[SPARK-5994] [SQL] Python DataFrame documentation fixes
select empty should NOT be the same as select. make sure selectExpr is behaving
the same.
join param documentation
link to source doesn't work in jekyll generated file
cross reference of columns (i.e. enabling linking)
show(): move df example before df.show()
move tests in SQLContext out of docstring otherwise doc is too long
Column.desc and .asc doesn't have any documentation
in documentation, sort functions.*)
Author: Davies Liu
Closes #4756 from davies/df_docs and squashes the following commits:
f30502c [Davies Liu] fix doc
32f0d46 [Davies Liu] fix DataFrame docs
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d641fbb3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d641fbb3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d641fbb3
Branch: refs/heads/master
Commit: d641fbb39c90b1d734cc55396ca43d7e98788975
Parents: 769e092
Author: Davies Liu
Authored: Tue Feb 24 20:51:55 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 20:51:55 2015 -0800
--
docs/_config.yml| 1 +
python/docs/pyspark.sql.rst | 3 -
python/pyspark/sql/context.py | 182 +++
python/pyspark/sql/dataframe.py | 56 +--
python/pyspark/sql/functions.py | 1 +
python/pyspark/sql/tests.py | 68 -
python/pyspark/sql/types.py | 2 +-
7 files changed, 130 insertions(+), 183 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d641fbb3/docs/_config.yml
--
diff --git a/docs/_config.yml b/docs/_config.yml
index e2db274..0652927 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -10,6 +10,7 @@ kramdown:
include:
- _static
+ - _modules
# These allow the documentation to be updated with newer releases
# of Spark, Scala, and Mesos.
http://git-wip-us.apache.org/repos/asf/spark/blob/d641fbb3/python/docs/pyspark.sql.rst
--
diff --git a/python/docs/pyspark.sql.rst b/python/docs/pyspark.sql.rst
index e03379e..2e3f69b 100644
--- a/python/docs/pyspark.sql.rst
+++ b/python/docs/pyspark.sql.rst
@@ -7,7 +7,6 @@ Module Context
.. automodule:: pyspark.sql
:members:
:undoc-members:
-:show-inheritance:
pyspark.sql.types module
@@ -15,7 +14,6 @@ pyspark.sql.types module
.. automodule:: pyspark.sql.types
:members:
:undoc-members:
-:show-inheritance:
pyspark.sql.functions module
@@ -23,4 +21,3 @@ pyspark.sql.functions module
.. automodule:: pyspark.sql.functions
:members:
:undoc-members:
-:show-inheritance:
http://git-wip-us.apache.org/repos/asf/spark/blob/d641fbb3/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 125933c..5d7aeb6 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -129,6 +129,7 @@ class SQLContext(object):
>>> sqlCtx.registerFunction("stringLengthString", lambda x: len(x))
>>> sqlCtx.sql("SELECT stringLengthString('test')").collect()
[Row(c0=u'4')]
+
>>> from pyspark.sql.types import IntegerType
>>> sqlCtx.registerFunction("stringLengthInt", lambda x: len(x),
IntegerType())
>>> sqlCtx.sql("SELECT stringLengthInt('test')").collect()
@@ -197,31 +198,6 @@ class SQLContext(object):
>>> df = sqlCtx.inferSchema(rdd)
>>> df.collect()[0]
Row(field1=1, field2=u'row1')
-
->>> NestedRow = Row("f1", "f2")
->>> nestedRdd1 = sc.parallelize([
-... NestedRow(array('i', [1, 2]), {"row1": 1.0}),
-... NestedRow(array('i', [2, 3]), {"row2": 2.0})])
->>> df = sqlCtx.inferSchema(nestedRdd1)
->>> df.collect()
-[Row(f1=[1, 2], f2={u'row1': 1.0}), ..., f2={u'row2': 2.0})]
-
->>> nestedRdd2 = sc.parallelize([
-... NestedRow([[1, 2], [2, 3]], [1, 2]),
-... NestedRow([[2, 3], [3, 4]], [2, 3])])
->>> df = sqlCtx.inferSchema(nestedRdd2)
->>> df.collect()
-[Row(f1=[[1, 2], [2, 3]], f2=[1, 2]), ..., f2=[2, 3])]
-
->>> from collections import namedtuple
->>> CustomRow = namedtuple('CustomRow', 'field1 field2')
->>> rdd = sc.parallelize(
-... [CustomRow(field1=1, field2="row1"),
-... CustomRow(field1=2, field2="row2"),
-... CustomRow(field1=3, field2="row3")])
->>> df = sqlCtx.inferSchema(rdd)
->>> df.collect()[0]
-Row(field1=1, field2=u'row1')
"""
spark git commit: [SPARK-5994] [SQL] Python DataFrame documentation fixes
Repository: spark
Updated Branches:
refs/heads/branch-1.3 e7a748ecf -> 5c421e030
[SPARK-5994] [SQL] Python DataFrame documentation fixes
select empty should NOT be the same as select. make sure selectExpr is behaving
the same.
join param documentation
link to source doesn't work in jekyll generated file
cross reference of columns (i.e. enabling linking)
show(): move df example before df.show()
move tests in SQLContext out of docstring otherwise doc is too long
Column.desc and .asc doesn't have any documentation
in documentation, sort functions.*)
Author: Davies Liu
Closes #4756 from davies/df_docs and squashes the following commits:
f30502c [Davies Liu] fix doc
32f0d46 [Davies Liu] fix DataFrame docs
(cherry picked from commit d641fbb39c90b1d734cc55396ca43d7e98788975)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5c421e03
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5c421e03
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5c421e03
Branch: refs/heads/branch-1.3
Commit: 5c421e030b656b143cf962466157832723d8d1f5
Parents: e7a748e
Author: Davies Liu
Authored: Tue Feb 24 20:51:55 2015 -0800
Committer: Michael Armbrust
Committed: Tue Feb 24 20:52:05 2015 -0800
--
docs/_config.yml| 1 +
python/docs/pyspark.sql.rst | 3 -
python/pyspark/sql/context.py | 182 +++
python/pyspark/sql/dataframe.py | 56 +--
python/pyspark/sql/functions.py | 1 +
python/pyspark/sql/tests.py | 68 -
python/pyspark/sql/types.py | 2 +-
7 files changed, 130 insertions(+), 183 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5c421e03/docs/_config.yml
--
diff --git a/docs/_config.yml b/docs/_config.yml
index 50124c5..ef6da03 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -10,6 +10,7 @@ kramdown:
include:
- _static
+ - _modules
# These allow the documentation to be updated with newer releases
# of Spark, Scala, and Mesos.
http://git-wip-us.apache.org/repos/asf/spark/blob/5c421e03/python/docs/pyspark.sql.rst
--
diff --git a/python/docs/pyspark.sql.rst b/python/docs/pyspark.sql.rst
index e03379e..2e3f69b 100644
--- a/python/docs/pyspark.sql.rst
+++ b/python/docs/pyspark.sql.rst
@@ -7,7 +7,6 @@ Module Context
.. automodule:: pyspark.sql
:members:
:undoc-members:
-:show-inheritance:
pyspark.sql.types module
@@ -15,7 +14,6 @@ pyspark.sql.types module
.. automodule:: pyspark.sql.types
:members:
:undoc-members:
-:show-inheritance:
pyspark.sql.functions module
@@ -23,4 +21,3 @@ pyspark.sql.functions module
.. automodule:: pyspark.sql.functions
:members:
:undoc-members:
-:show-inheritance:
http://git-wip-us.apache.org/repos/asf/spark/blob/5c421e03/python/pyspark/sql/context.py
--
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 125933c..5d7aeb6 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -129,6 +129,7 @@ class SQLContext(object):
>>> sqlCtx.registerFunction("stringLengthString", lambda x: len(x))
>>> sqlCtx.sql("SELECT stringLengthString('test')").collect()
[Row(c0=u'4')]
+
>>> from pyspark.sql.types import IntegerType
>>> sqlCtx.registerFunction("stringLengthInt", lambda x: len(x),
IntegerType())
>>> sqlCtx.sql("SELECT stringLengthInt('test')").collect()
@@ -197,31 +198,6 @@ class SQLContext(object):
>>> df = sqlCtx.inferSchema(rdd)
>>> df.collect()[0]
Row(field1=1, field2=u'row1')
-
->>> NestedRow = Row("f1", "f2")
->>> nestedRdd1 = sc.parallelize([
-... NestedRow(array('i', [1, 2]), {"row1": 1.0}),
-... NestedRow(array('i', [2, 3]), {"row2": 2.0})])
->>> df = sqlCtx.inferSchema(nestedRdd1)
->>> df.collect()
-[Row(f1=[1, 2], f2={u'row1': 1.0}), ..., f2={u'row2': 2.0})]
-
->>> nestedRdd2 = sc.parallelize([
-... NestedRow([[1, 2], [2, 3]], [1, 2]),
-... NestedRow([[2, 3], [3, 4]], [2, 3])])
->>> df = sqlCtx.inferSchema(nestedRdd2)
->>> df.collect()
-[Row(f1=[[1, 2], [2, 3]], f2=[1, 2]), ..., f2=[2, 3])]
-
->>> from collections import namedtuple
->>> CustomRow = namedtuple('CustomRow', 'field1 field2')
->>> rdd = sc.parallelize(
-... [CustomRow(field1=1, field2="row1"),
-... CustomRow(field1=2, field2="row2"),
-... CustomRow(field1=3, field2="row3")])
->>> df
spark git commit: [SPARK-5944] [PySpark] fix version in Python API docs
Repository: spark
Updated Branches:
refs/heads/branch-1.3 791df93cd -> 9aca3c688
[SPARK-5944] [PySpark] fix version in Python API docs
use RELEASE_VERSION when building the Python API docs
Author: Davies Liu
Closes #4731 from davies/api_version and squashes the following commits:
c9744c9 [Davies Liu] Update create-release.sh
08cbc3f [Davies Liu] fix python docs
(cherry picked from commit f3f4c87b3d944c10d1200dfe49091ebb2a149be6)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9aca3c68
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9aca3c68
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9aca3c68
Branch: refs/heads/branch-1.3
Commit: 9aca3c688ad9a51bc8e14053ed4daac168028913
Parents: 791df93
Author: Davies Liu
Authored: Wed Feb 25 15:13:34 2015 -0800
Committer: Michael Armbrust
Committed: Wed Feb 25 15:13:56 2015 -0800
--
dev/create-release/create-release.sh | 2 +-
python/docs/conf.py | 6 +++---
python/docs/pyspark.sql.rst | 2 +-
python/pyspark/rdd.py| 4
4 files changed, 9 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9aca3c68/dev/create-release/create-release.sh
--
diff --git a/dev/create-release/create-release.sh
b/dev/create-release/create-release.sh
index 607ce1c..da15ce3 100755
--- a/dev/create-release/create-release.sh
+++ b/dev/create-release/create-release.sh
@@ -237,7 +237,7 @@ if [[ ! "$@" =~ --skip-package ]]; then
sbt/sbt clean
cd docs
# Compile docs with Java 7 to use nicer format
- JAVA_HOME=$JAVA_7_HOME PRODUCTION=1 jekyll build
+ JAVA_HOME="$JAVA_7_HOME" PRODUCTION=1 RELEASE_VERSION="$RELEASE_VERSION"
jekyll build
echo "Copying release documentation"
rc_docs_folder=${rc_folder}-docs
ssh $asf_usern...@people.apache.org \
http://git-wip-us.apache.org/repos/asf/spark/blob/9aca3c68/python/docs/conf.py
--
diff --git a/python/docs/conf.py b/python/docs/conf.py
index cbbf7ff..163987d 100644
--- a/python/docs/conf.py
+++ b/python/docs/conf.py
@@ -48,16 +48,16 @@ master_doc = 'index'
# General information about the project.
project = u'PySpark'
-copyright = u'2014, Author'
+copyright = u''
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
-version = '1.3-SNAPSHOT'
+version = 'master'
# The full version, including alpha/beta/rc tags.
-release = '1.3-SNAPSHOT'
+release = os.environ.get('RELEASE_VERSION', version)
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
http://git-wip-us.apache.org/repos/asf/spark/blob/9aca3c68/python/docs/pyspark.sql.rst
--
diff --git a/python/docs/pyspark.sql.rst b/python/docs/pyspark.sql.rst
index 2e3f69b..6259379 100644
--- a/python/docs/pyspark.sql.rst
+++ b/python/docs/pyspark.sql.rst
@@ -17,7 +17,7 @@ pyspark.sql.types module
pyspark.sql.functions module
-
+
.. automodule:: pyspark.sql.functions
:members:
:undoc-members:
http://git-wip-us.apache.org/repos/asf/spark/blob/9aca3c68/python/pyspark/rdd.py
--
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index d3148de..cb12fed 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -2111,6 +2111,7 @@ class RDD(object):
def countApprox(self, timeout, confidence=0.95):
"""
.. note:: Experimental
+
Approximate version of count() that returns a potentially incomplete
result within a timeout, even if not all tasks have finished.
@@ -2124,6 +2125,7 @@ class RDD(object):
def sumApprox(self, timeout, confidence=0.95):
"""
.. note:: Experimental
+
Approximate operation to return the sum within a timeout
or meet the confidence.
@@ -2140,6 +2142,7 @@ class RDD(object):
def meanApprox(self, timeout, confidence=0.95):
"""
.. note:: Experimental
+
Approximate operation to return the mean within a timeout
or meet the confidence.
@@ -2156,6 +2159,7 @@ class RDD(object):
def countApproxDistinct(self, relativeSD=0.05):
"""
.. note:: Experimental
+
Return approximate number of distinct elements in the RDD.
The algorithm used is based on streamlib's implementation of
---
