spark git commit: [SPARK-16726][SQL] Improve `Union/Intersect/Except` error messages on incompatible types
Repository: spark
Updated Branches:
refs/heads/master 579fbcf3b -> 64d8f37c7
[SPARK-16726][SQL] Improve `Union/Intersect/Except` error messages on
incompatible types
## What changes were proposed in this pull request?
Currently, `UNION` queries on incompatible types show misleading error
messages, i.e., `unresolved operator Union`. We had better show a more correct
message. This will help users in the situation of
[SPARK-16704](https://issues.apache.org/jira/browse/SPARK-16704).
**Before**
```scala
scala> sql("select 1,2,3 union (select 1,array(2),3)")
org.apache.spark.sql.AnalysisException: unresolved operator 'Union;
scala> sql("select 1,2,3 intersect (select 1,array(2),3)")
org.apache.spark.sql.AnalysisException: unresolved operator 'Intersect;
scala> sql("select 1,2,3 except (select 1,array(2),3)")
org.apache.spark.sql.AnalysisException: unresolved operator 'Except;
```
**After**
```scala
scala> sql("select 1,2,3 union (select 1,array(2),3)")
org.apache.spark.sql.AnalysisException: Union can only be performed on tables
with the compatible column types. ArrayType(IntegerType,false) <> IntegerType
at the second column of the second table;
scala> sql("select 1,2,3 intersect (select 1,array(2),3)")
org.apache.spark.sql.AnalysisException: Intersect can only be performed on
tables with the compatible column types. ArrayType(IntegerType,false) <>
IntegerType at the second column of the second table;
scala> sql("select 1,2,3 except (select array(1),array(2),3)")
org.apache.spark.sql.AnalysisException: Except can only be performed on tables
with the compatible column types. ArrayType(IntegerType,false) <> IntegerType
at the first column of the second table;
```
## How was this patch tested?
Pass the Jenkins test with a new test case.
Author: Dongjoon Hyun
Closes #14355 from dongjoon-hyun/SPARK-16726.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/64d8f37c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/64d8f37c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/64d8f37c
Branch: refs/heads/master
Commit: 64d8f37c717cbc9c1c3649cae4c7cc4e628cd72d
Parents: 579fbcf
Author: Dongjoon Hyun
Authored: Mon Aug 1 11:12:58 2016 +0200
Committer: Herman van Hovell
Committed: Mon Aug 1 11:12:58 2016 +0200
--
.../sql/catalyst/analysis/CheckAnalysis.scala | 44 ++--
.../catalyst/analysis/AnalysisErrorSuite.scala | 15 +++
2 files changed, 46 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/64d8f37c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 8b87a4e..41b7e62 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -253,19 +253,6 @@ trait CheckAnalysis extends PredicateHelper {
}
}
- case s @ SetOperation(left, right) if left.output.length !=
right.output.length =>
-failAnalysis(
- s"${s.nodeName} can only be performed on tables with the same
number of columns, " +
-s"but the left table has ${left.output.length} columns and the
right has " +
-s"${right.output.length}")
-
- case s: Union if s.children.exists(_.output.length !=
s.children.head.output.length) =>
-val firstError = s.children.find(_.output.length !=
s.children.head.output.length).get
-failAnalysis(
- s"Unions can only be performed on tables with the same number of
columns, " +
-s"but one table has '${firstError.output.length}' columns and
another table has " +
-s"'${s.children.head.output.length}' columns")
-
case GlobalLimit(limitExpr, _) => checkLimitClause(limitExpr)
case LocalLimit(limitExpr, _) => checkLimitClause(limitExpr)
@@ -280,6 +267,37 @@ trait CheckAnalysis extends PredicateHelper {
case p if
p.expressions.exists(PredicateSubquery.hasPredicateSubquery) =>
failAnalysis(s"Predicate sub-queries can only be used in a Filter:
$p")
+ case _: Union | _: SetOperation if operator.children.length > 1 =>
+def dataTypes(plan: LogicalPlan): Seq[DataType] =
plan.output.map(_.dataType)
+def ordinalNumber(i: Int): String = i match {
+ case 0 => "first"
+ case 1 => "second"
+ case i => s"${i}th"
+}
+
spark git commit: [SPARK-16485][DOC][ML] Remove useless latex in a log messge.
Repository: spark
Updated Branches:
refs/heads/master 64d8f37c7 -> 2a0de7dc9
[SPARK-16485][DOC][ML] Remove useless latex in a log messge.
## What changes were proposed in this pull request?
Removed useless latex in a log messge.
## How was this patch tested?
Check generated scaladoc.
Author: Shuai Lin
Closes #14380 from lins05/fix-docs-formatting.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2a0de7dc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2a0de7dc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2a0de7dc
Branch: refs/heads/master
Commit: 2a0de7dc995844984d7dbb4238418967c6bbac70
Parents: 64d8f37
Author: Shuai Lin
Authored: Mon Aug 1 06:54:18 2016 -0700
Committer: Sean Owen
Committed: Mon Aug 1 06:54:18 2016 -0700
--
.../org/apache/spark/mllib/optimization/GradientDescentSuite.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2a0de7dc/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
--
diff --git
a/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
b/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
index 1c9b7c7..37eb794 100644
---
a/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
@@ -131,7 +131,7 @@ class GradientDescentSuite extends SparkFunSuite with
MLlibTestSparkContext with
assert(
loss1(0) ~= (loss0(0) + (math.pow(initialWeightsWithIntercept(0), 2) +
math.pow(initialWeightsWithIntercept(1), 2)) / 2) absTol 1E-5,
- """For non-zero weights, the regVal should be \frac{1}{2}\sum_i
w_i^2.""")
+ """For non-zero weights, the regVal should be 0.5 * sum(w_i ^ 2).""")
assert(
(newWeights1(0) ~= (newWeights0(0) - initialWeightsWithIntercept(0))
absTol 1E-5) &&
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-16778][SQL][TRIVIAL] Fix deprecation warning with SQLContext
Repository: spark
Updated Branches:
refs/heads/branch-2.0 c651ff53a -> 4bdc55898
[SPARK-16778][SQL][TRIVIAL] Fix deprecation warning with SQLContext
## What changes were proposed in this pull request?
Change to non-deprecated constructor for SQLContext.
## How was this patch tested?
Existing tests
Author: Holden Karau
Closes #14406 from
holdenk/SPARK-16778-fix-use-of-deprecated-SQLContext-constructor.
(cherry picked from commit 1e9b59b73bdb8aacf5a85e0eed29efc6485a3bc3)
Signed-off-by: Sean Owen
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4bdc5589
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4bdc5589
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4bdc5589
Branch: refs/heads/branch-2.0
Commit: 4bdc558989ef4a9490ca42e7330c10136151134b
Parents: c651ff5
Author: Holden Karau
Authored: Mon Aug 1 06:55:31 2016 -0700
Committer: Sean Owen
Committed: Mon Aug 1 06:55:47 2016 -0700
--
.../test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/4bdc5589/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index 2cd3f47..5d34804 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -28,7 +28,7 @@ class SQLConfSuite extends QueryTest with SharedSQLContext {
test("propagate from spark conf") {
// We create a new context here to avoid order dependence with other tests
that might call
// clear().
-val newContext = new SQLContext(sparkContext)
+val newContext = new
SQLContext(SparkSession.builder().sparkContext(sparkContext).getOrCreate())
assert(newContext.getConf("spark.sql.testkey", "false") === "true")
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-16778][SQL][TRIVIAL] Fix deprecation warning with SQLContext
Repository: spark
Updated Branches:
refs/heads/master 2a0de7dc9 -> 1e9b59b73
[SPARK-16778][SQL][TRIVIAL] Fix deprecation warning with SQLContext
## What changes were proposed in this pull request?
Change to non-deprecated constructor for SQLContext.
## How was this patch tested?
Existing tests
Author: Holden Karau
Closes #14406 from
holdenk/SPARK-16778-fix-use-of-deprecated-SQLContext-constructor.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1e9b59b7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1e9b59b7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1e9b59b7
Branch: refs/heads/master
Commit: 1e9b59b73bdb8aacf5a85e0eed29efc6485a3bc3
Parents: 2a0de7d
Author: Holden Karau
Authored: Mon Aug 1 06:55:31 2016 -0700
Committer: Sean Owen
Committed: Mon Aug 1 06:55:31 2016 -0700
--
.../test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1e9b59b7/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index 4454cad..7424e17 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -28,7 +28,7 @@ class SQLConfSuite extends QueryTest with SharedSQLContext {
test("propagate from spark conf") {
// We create a new context here to avoid order dependence with other tests
that might call
// clear().
-val newContext = new SQLContext(sparkContext)
+val newContext = new
SQLContext(SparkSession.builder().sparkContext(sparkContext).getOrCreate())
assert(newContext.getConf("spark.sql.testkey", "false") === "true")
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-16776][STREAMING] Replace deprecated API in KafkaTestUtils for 0.10.0.
Repository: spark
Updated Branches:
refs/heads/branch-2.0 4bdc55898 -> b49091e10
[SPARK-16776][STREAMING] Replace deprecated API in KafkaTestUtils for 0.10.0.
## What changes were proposed in this pull request?
This PR replaces the old Kafka API to 0.10.0 ones in `KafkaTestUtils`.
The change include:
- `Producer` to `KafkaProducer`
- Change configurations to equalvant ones. (I referred
[here](http://kafka.apache.org/documentation.html#producerconfigs) for 0.10.0
and [here](http://kafka.apache.org/082/documentation.html#producerconfigs
) for old, 0.8.2).
This PR will remove the build warning as below:
```scala
[WARNING]
.../spark/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala:71:
class Producer in package producer is deprecated: This class has been
deprecated and will be removed in a future release. Please use
org.apache.kafka.clients.producer.KafkaProducer instead.
[WARNING] private var producer: Producer[String, String] = _
[WARNING] ^
[WARNING]
.../spark/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala:181:
class Producer in package producer is deprecated: This class has been
deprecated and will be removed in a future release. Please use
org.apache.kafka.clients.producer.KafkaProducer instead.
[WARNING] producer = new Producer[String, String](new
ProducerConfig(producerConfiguration))
[WARNING]^
[WARNING] .../spark/streaming/kafka010/KafkaTestUtils.scala:181: class
ProducerConfig in package producer is deprecated: This class has been
deprecated and will be removed in a future release. Please use
org.apache.kafka.clients.producer.ProducerConfig instead.
[WARNING] producer = new Producer[String, String](new
ProducerConfig(producerConfiguration))
[WARNING] ^
[WARNING]
.../spark/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala:182:
class KeyedMessage in package producer is deprecated: This class has been
deprecated and will be removed in a future release. Please use
org.apache.kafka.clients.producer.ProducerRecord instead.
[WARNING] producer.send(messages.map { new KeyedMessage[String,
String](topic, _ ) }: _*)
[WARNING] ^
[WARNING] four warnings found
[WARNING] warning: [options] bootstrap class path not set in conjunction with
-source 1.7
[WARNING] 1 warning
```
## How was this patch tested?
Existing tests that use `KafkaTestUtils` should cover this.
Author: hyukjinkwon
Closes #14416 from HyukjinKwon/SPARK-16776.
(cherry picked from commit f93ad4fe7c9728c8dd67a8095de3d39fad21d03f)
Signed-off-by: Sean Owen
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b49091e1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b49091e1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b49091e1
Branch: refs/heads/branch-2.0
Commit: b49091e10100dfbefeabdd2dfe0b64cdf613a052
Parents: 4bdc558
Author: hyukjinkwon
Authored: Mon Aug 1 06:56:52 2016 -0700
Committer: Sean Owen
Committed: Mon Aug 1 06:57:09 2016 -0700
--
.../streaming/kafka010/KafkaTestUtils.scala | 20
1 file changed, 12 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b49091e1/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
--
diff --git
a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
index 19192e4b..ecabe1c 100644
---
a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
+++
b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
@@ -30,10 +30,10 @@ import scala.util.control.NonFatal
import kafka.admin.AdminUtils
import kafka.api.Request
-import kafka.producer.{KeyedMessage, Producer, ProducerConfig}
-import kafka.serializer.StringEncoder
import kafka.server.{KafkaConfig, KafkaServer}
import kafka.utils.ZkUtils
+import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
+import org.apache.kafka.common.serialization.StringSerializer
import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
import org.apache.spark.SparkConf
@@ -68,7 +68,7 @@ private[kafka010] class KafkaTestUtils extends Logging {
private var server: KafkaServer = _
// Kafka producer
- private var producer: Producer[String, String] = _
+ private var producer: KafkaProducer[String, String] = _
// Flag to test whether the system
spark git commit: [SPARK-16776][STREAMING] Replace deprecated API in KafkaTestUtils for 0.10.0.
Repository: spark
Updated Branches:
refs/heads/master 1e9b59b73 -> f93ad4fe7
[SPARK-16776][STREAMING] Replace deprecated API in KafkaTestUtils for 0.10.0.
## What changes were proposed in this pull request?
This PR replaces the old Kafka API to 0.10.0 ones in `KafkaTestUtils`.
The change include:
- `Producer` to `KafkaProducer`
- Change configurations to equalvant ones. (I referred
[here](http://kafka.apache.org/documentation.html#producerconfigs) for 0.10.0
and [here](http://kafka.apache.org/082/documentation.html#producerconfigs
) for old, 0.8.2).
This PR will remove the build warning as below:
```scala
[WARNING]
.../spark/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala:71:
class Producer in package producer is deprecated: This class has been
deprecated and will be removed in a future release. Please use
org.apache.kafka.clients.producer.KafkaProducer instead.
[WARNING] private var producer: Producer[String, String] = _
[WARNING] ^
[WARNING]
.../spark/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala:181:
class Producer in package producer is deprecated: This class has been
deprecated and will be removed in a future release. Please use
org.apache.kafka.clients.producer.KafkaProducer instead.
[WARNING] producer = new Producer[String, String](new
ProducerConfig(producerConfiguration))
[WARNING]^
[WARNING] .../spark/streaming/kafka010/KafkaTestUtils.scala:181: class
ProducerConfig in package producer is deprecated: This class has been
deprecated and will be removed in a future release. Please use
org.apache.kafka.clients.producer.ProducerConfig instead.
[WARNING] producer = new Producer[String, String](new
ProducerConfig(producerConfiguration))
[WARNING] ^
[WARNING]
.../spark/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala:182:
class KeyedMessage in package producer is deprecated: This class has been
deprecated and will be removed in a future release. Please use
org.apache.kafka.clients.producer.ProducerRecord instead.
[WARNING] producer.send(messages.map { new KeyedMessage[String,
String](topic, _ ) }: _*)
[WARNING] ^
[WARNING] four warnings found
[WARNING] warning: [options] bootstrap class path not set in conjunction with
-source 1.7
[WARNING] 1 warning
```
## How was this patch tested?
Existing tests that use `KafkaTestUtils` should cover this.
Author: hyukjinkwon
Closes #14416 from HyukjinKwon/SPARK-16776.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f93ad4fe
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f93ad4fe
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f93ad4fe
Branch: refs/heads/master
Commit: f93ad4fe7c9728c8dd67a8095de3d39fad21d03f
Parents: 1e9b59b
Author: hyukjinkwon
Authored: Mon Aug 1 06:56:52 2016 -0700
Committer: Sean Owen
Committed: Mon Aug 1 06:56:52 2016 -0700
--
.../streaming/kafka010/KafkaTestUtils.scala | 20
1 file changed, 12 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f93ad4fe/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
--
diff --git
a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
index 19192e4b..ecabe1c 100644
---
a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
+++
b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaTestUtils.scala
@@ -30,10 +30,10 @@ import scala.util.control.NonFatal
import kafka.admin.AdminUtils
import kafka.api.Request
-import kafka.producer.{KeyedMessage, Producer, ProducerConfig}
-import kafka.serializer.StringEncoder
import kafka.server.{KafkaConfig, KafkaServer}
import kafka.utils.ZkUtils
+import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
+import org.apache.kafka.common.serialization.StringSerializer
import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
import org.apache.spark.SparkConf
@@ -68,7 +68,7 @@ private[kafka010] class KafkaTestUtils extends Logging {
private var server: KafkaServer = _
// Kafka producer
- private var producer: Producer[String, String] = _
+ private var producer: KafkaProducer[String, String] = _
// Flag to test whether the system is correctly started
private var zkReady = false
@@ -178,8 +178,10 @@ private[kafka010] class KafkaT
spark git commit: [SPARK-16791][SQL] cast struct with timestamp field fails
Repository: spark
Updated Branches:
refs/heads/master f93ad4fe7 -> 338a98d65
[SPARK-16791][SQL] cast struct with timestamp field fails
## What changes were proposed in this pull request?
a failing test case + fix to SPARK-16791
(https://issues.apache.org/jira/browse/SPARK-16791)
## How was this patch tested?
added a failing test case to CastSuit, then fixed the Cast code and rerun the
entire CastSuit
Author: eyal farago
Author: Eyal Farago
Closes #14400 from eyalfa/SPARK-16791_cast_struct_with_timestamp_field_fails.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/338a98d6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/338a98d6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/338a98d6
Branch: refs/heads/master
Commit: 338a98d65c8efe0c41f39a8dddeab7040dcda125
Parents: f93ad4f
Author: eyal farago
Authored: Mon Aug 1 22:43:32 2016 +0800
Committer: Wenchen Fan
Committed: Mon Aug 1 22:43:32 2016 +0800
--
.../org/apache/spark/sql/catalyst/expressions/Cast.scala | 2 +-
.../apache/spark/sql/catalyst/expressions/CastSuite.scala | 10 ++
2 files changed, 11 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/338a98d6/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index c452765..70fff51 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -416,7 +416,7 @@ case class Cast(child: Expression, dataType: DataType)
extends UnaryExpression w
}
private[this] def cast(from: DataType, to: DataType): Any => Any = to match {
-case dt if dt == child.dataType => identity[Any]
+case dt if dt == from => identity[Any]
case StringType => castToString(from)
case BinaryType => castToBinary(from)
case DateType => castToDate(from)
http://git-wip-us.apache.org/repos/asf/spark/blob/338a98d6/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index 5ae0527..5c35baa 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -727,6 +727,16 @@ class CastSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
+ test("cast struct with a timestamp field") {
+val originalSchema = new StructType().add("tsField", TimestampType,
nullable = false)
+// nine out of ten times I'm casting a struct, it's to normalize its
fields nullability
+val targetSchema = new StructType().add("tsField", TimestampType, nullable
= true)
+
+val inp = Literal.create(InternalRow(0L), originalSchema)
+val expected = InternalRow(0L)
+checkEvaluation(cast(inp, targetSchema), expected)
+ }
+
test("complex casting") {
val complex = Literal.create(
Row(
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-16791][SQL] cast struct with timestamp field fails
Repository: spark
Updated Branches:
refs/heads/branch-2.0 b49091e10 -> 1523bf69a
[SPARK-16791][SQL] cast struct with timestamp field fails
## What changes were proposed in this pull request?
a failing test case + fix to SPARK-16791
(https://issues.apache.org/jira/browse/SPARK-16791)
## How was this patch tested?
added a failing test case to CastSuit, then fixed the Cast code and rerun the
entire CastSuit
Author: eyal farago
Author: Eyal Farago
Closes #14400 from eyalfa/SPARK-16791_cast_struct_with_timestamp_field_fails.
(cherry picked from commit 338a98d65c8efe0c41f39a8dddeab7040dcda125)
Signed-off-by: Wenchen Fan
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1523bf69
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1523bf69
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1523bf69
Branch: refs/heads/branch-2.0
Commit: 1523bf69a0ef87f36b0b3995ce2d7a33aaff6046
Parents: b49091e
Author: eyal farago
Authored: Mon Aug 1 22:43:32 2016 +0800
Committer: Wenchen Fan
Committed: Mon Aug 1 22:44:23 2016 +0800
--
.../org/apache/spark/sql/catalyst/expressions/Cast.scala | 2 +-
.../apache/spark/sql/catalyst/expressions/CastSuite.scala | 10 ++
2 files changed, 11 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1523bf69/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index c452765..70fff51 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -416,7 +416,7 @@ case class Cast(child: Expression, dataType: DataType)
extends UnaryExpression w
}
private[this] def cast(from: DataType, to: DataType): Any => Any = to match {
-case dt if dt == child.dataType => identity[Any]
+case dt if dt == from => identity[Any]
case StringType => castToString(from)
case BinaryType => castToBinary(from)
case DateType => castToDate(from)
http://git-wip-us.apache.org/repos/asf/spark/blob/1523bf69/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index 5ae0527..5c35baa 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -727,6 +727,16 @@ class CastSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
+ test("cast struct with a timestamp field") {
+val originalSchema = new StructType().add("tsField", TimestampType,
nullable = false)
+// nine out of ten times I'm casting a struct, it's to normalize its
fields nullability
+val targetSchema = new StructType().add("tsField", TimestampType, nullable
= true)
+
+val inp = Literal.create(InternalRow(0L), originalSchema)
+val expected = InternalRow(0L)
+checkEvaluation(cast(inp, targetSchema), expected)
+ }
+
test("complex casting") {
val complex = Literal.create(
Row(
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark-website git commit: Fix main links to streaming examples
Repository: spark-website Updated Branches: refs/heads/asf-site 5e869909e -> 612383440 Fix main links to streaming examples Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/61238344 Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/61238344 Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/61238344 Branch: refs/heads/asf-site Commit: 612383440bfa727a9980cf83460b8f32df8d666c Parents: 5e86990 Author: Isaac Levy Authored: Sat Jul 30 10:29:20 2016 -0400 Committer: Isaac Levy Committed: Sat Jul 30 10:29:20 2016 -0400 -- site/streaming/index.html | 2 +- streaming/index.md| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark-website/blob/61238344/site/streaming/index.html -- diff --git a/site/streaming/index.html b/site/streaming/index.html index ddd5c33..85f9c8b 100644 --- a/site/streaming/index.html +++ b/site/streaming/index.html @@ -307,7 +307,7 @@ Download Spark. It includes Streaming as a module. Read the Spark Streaming programming guide, which includes a tutorial and describes system architecture, configuration and high availability. - Check out example programs in https://github.com/apache/incubator-spark/tree/master/examples/src/main/scala/org/apache/spark/streaming/examples";>Scala and https://github.com/apache/incubator-spark/tree/master/examples/src/main/java/org/apache/spark/streaming/examples";>Java. + Check out example programs in https://github.com/apache/spark/tree/master/examples/src/main/scala/org/apache/spark/examples/streaming";>Scala and https://github.com/apache/spark/tree/master/examples/src/main/java/org/apache/spark/examples/streaming";>Java. http://git-wip-us.apache.org/repos/asf/spark-website/blob/61238344/streaming/index.md -- diff --git a/streaming/index.md b/streaming/index.md index 691ba36..459d84f 100644 --- a/streaming/index.md +++ b/streaming/index.md @@ -130,7 +130,7 @@ subproject: Streaming Download Spark. It includes Streaming as a module. Read the Spark Streaming programming guide, which includes a tutorial and describes system architecture, configuration and high availability. - Check out example programs in https://github.com/apache/incubator-spark/tree/master/examples/src/main/scala/org/apache/spark/streaming/examples";>Scala and https://github.com/apache/incubator-spark/tree/master/examples/src/main/java/org/apache/spark/streaming/examples";>Java. + Check out example programs in https://github.com/apache/spark/tree/master/examples/src/main/scala/org/apache/spark/examples/streaming";>Scala and https://github.com/apache/spark/tree/master/examples/src/main/java/org/apache/spark/examples/streaming";>Java. - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-16774][SQL] Fix use of deprecated timestamp constructor & improve timezone handling
Repository: spark
Updated Branches:
refs/heads/master 338a98d65 -> ab1e761f9
[SPARK-16774][SQL] Fix use of deprecated timestamp constructor & improve
timezone handling
## What changes were proposed in this pull request?
Removes the deprecated timestamp constructor and incidentally fixes the use
which was using system timezone rather than the one specified when working near
DST.
This change also causes the roundtrip tests to fail since it now actually uses
all the timezones near DST boundaries where it didn't before.
Note: this is only a partial the solution, longer term we should follow up with
https://issues.apache.org/jira/browse/SPARK-16788 to avoid this problem &
simplify our timezone handling code.
## How was this patch tested?
New tests for two timezones added so even if user timezone happens to coincided
with one, the other tests should still fail. Important note: this (temporarily)
disables the round trip tests until we can fix the issue more thoroughly.
Author: Holden Karau
Closes #14398 from
holdenk/SPARK-16774-fix-use-of-deprecated-timestamp-constructor.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ab1e761f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ab1e761f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ab1e761f
Branch: refs/heads/master
Commit: ab1e761f9691b41385e2ed2202c5a671c63c963d
Parents: 338a98d
Author: Holden Karau
Authored: Mon Aug 1 13:57:05 2016 -0700
Committer: Sean Owen
Committed: Mon Aug 1 13:57:05 2016 -0700
--
.../spark/sql/catalyst/util/DateTimeUtils.scala | 14 --
.../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 3 ++-
2 files changed, 10 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ab1e761f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index df480a1..0b643a5 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -852,8 +852,10 @@ object DateTimeUtils {
/**
* Lookup the offset for given millis seconds since 1970-01-01 00:00:00 in
given timezone.
+ * TODO: Improve handling of normalization differences.
+ * TODO: Replace with JSR-310 or similar system - see SPARK-16788
*/
- private def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long
= {
+ private[sql] def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone):
Long = {
var guess = tz.getRawOffset
// the actual offset should be calculated based on milliseconds in UTC
val offset = tz.getOffset(millisLocal - guess)
@@ -875,11 +877,11 @@ object DateTimeUtils {
val hh = seconds / 3600
val mm = seconds / 60 % 60
val ss = seconds % 60
-val nano = millisOfDay % 1000 * 100
-
-// create a Timestamp to get the unix timestamp (in UTC)
-val timestamp = new Timestamp(year - 1900, month - 1, day, hh, mm, ss,
nano)
-guess = (millisLocal - timestamp.getTime).toInt
+val ms = millisOfDay % 1000
+val calendar = Calendar.getInstance(tz)
+calendar.set(year, month - 1, day, hh, mm, ss)
+calendar.set(Calendar.MILLISECOND, ms)
+guess = (millisLocal - calendar.getTimeInMillis()).toInt
}
}
guess
http://git-wip-us.apache.org/repos/asf/spark/blob/ab1e761f/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 059a5b7..4f516d0 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -551,7 +551,8 @@ class DateTimeUtilsSuite extends SparkFunSuite {
val skipped = skipped_days.getOrElse(tz.getID, Int.MinValue)
(-2 to 2).foreach { d =>
if (d != skipped) {
-assert(millisToDays(daysToMillis(d)) === d)
+assert(millisToDays(daysToMillis(d)) === d,
+ s"Round trip of ${d} did not work in tz ${tz}")
}
}
}
-
To unsubscribe, e-mail: commits-unsubscr..
spark git commit: [SPARK-16774][SQL] Fix use of deprecated timestamp constructor & improve timezone handling
Repository: spark
Updated Branches:
refs/heads/branch-2.0 1523bf69a -> 4e73cb8eb
[SPARK-16774][SQL] Fix use of deprecated timestamp constructor & improve
timezone handling
## What changes were proposed in this pull request?
Removes the deprecated timestamp constructor and incidentally fixes the use
which was using system timezone rather than the one specified when working near
DST.
This change also causes the roundtrip tests to fail since it now actually uses
all the timezones near DST boundaries where it didn't before.
Note: this is only a partial the solution, longer term we should follow up with
https://issues.apache.org/jira/browse/SPARK-16788 to avoid this problem &
simplify our timezone handling code.
## How was this patch tested?
New tests for two timezones added so even if user timezone happens to coincided
with one, the other tests should still fail. Important note: this (temporarily)
disables the round trip tests until we can fix the issue more thoroughly.
Author: Holden Karau
Closes #14398 from
holdenk/SPARK-16774-fix-use-of-deprecated-timestamp-constructor.
(cherry picked from commit ab1e761f9691b41385e2ed2202c5a671c63c963d)
Signed-off-by: Sean Owen
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4e73cb8e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4e73cb8e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4e73cb8e
Branch: refs/heads/branch-2.0
Commit: 4e73cb8ebdb0dcb1be4dce562bac9214e9905b8e
Parents: 1523bf6
Author: Holden Karau
Authored: Mon Aug 1 13:57:05 2016 -0700
Committer: Sean Owen
Committed: Mon Aug 1 13:57:19 2016 -0700
--
.../spark/sql/catalyst/util/DateTimeUtils.scala | 14 --
.../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 3 ++-
2 files changed, 10 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/4e73cb8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index df480a1..0b643a5 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -852,8 +852,10 @@ object DateTimeUtils {
/**
* Lookup the offset for given millis seconds since 1970-01-01 00:00:00 in
given timezone.
+ * TODO: Improve handling of normalization differences.
+ * TODO: Replace with JSR-310 or similar system - see SPARK-16788
*/
- private def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long
= {
+ private[sql] def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone):
Long = {
var guess = tz.getRawOffset
// the actual offset should be calculated based on milliseconds in UTC
val offset = tz.getOffset(millisLocal - guess)
@@ -875,11 +877,11 @@ object DateTimeUtils {
val hh = seconds / 3600
val mm = seconds / 60 % 60
val ss = seconds % 60
-val nano = millisOfDay % 1000 * 100
-
-// create a Timestamp to get the unix timestamp (in UTC)
-val timestamp = new Timestamp(year - 1900, month - 1, day, hh, mm, ss,
nano)
-guess = (millisLocal - timestamp.getTime).toInt
+val ms = millisOfDay % 1000
+val calendar = Calendar.getInstance(tz)
+calendar.set(year, month - 1, day, hh, mm, ss)
+calendar.set(Calendar.MILLISECOND, ms)
+guess = (millisLocal - calendar.getTimeInMillis()).toInt
}
}
guess
http://git-wip-us.apache.org/repos/asf/spark/blob/4e73cb8e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 059a5b7..4f516d0 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -551,7 +551,8 @@ class DateTimeUtilsSuite extends SparkFunSuite {
val skipped = skipped_days.getOrElse(tz.getID, Int.MinValue)
(-2 to 2).foreach { d =>
if (d != skipped) {
-assert(millisToDays(daysToMillis(d)) === d)
+assert(millisToDays(daysToMillis(d)) === d,
+ s"Round trip of ${d} did not work in tz ${tz}")
}
}
}
spark git commit: [SPARK-15869][STREAMING] Fix a potential NPE in StreamingJobProgressListener.getBatchUIData
Repository: spark Updated Branches: refs/heads/master ab1e761f9 -> 03d46aafe [SPARK-15869][STREAMING] Fix a potential NPE in StreamingJobProgressListener.getBatchUIData ## What changes were proposed in this pull request? Moved `asScala` to a `map` to avoid NPE. ## How was this patch tested? Existing unit tests. Author: Shixiong Zhu Closes #14443 from zsxwing/SPARK-15869. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/03d46aaf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/03d46aaf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/03d46aaf Branch: refs/heads/master Commit: 03d46aafe561b03e25f4e25cf01e631c18dd827c Parents: ab1e761 Author: Shixiong Zhu Authored: Mon Aug 1 14:41:22 2016 -0700 Committer: Shixiong Zhu Committed: Mon Aug 1 14:41:22 2016 -0700 -- .../apache/spark/streaming/ui/StreamingJobProgressListener.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/03d46aaf/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala -- diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index c086df4..61f852a 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -259,7 +259,7 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) // We use an Iterable rather than explicitly converting to a seq so that updates // will propagate val outputOpIdToSparkJobIds: Iterable[OutputOpIdAndSparkJobId] = -Option(batchTimeToOutputOpIdSparkJobIdPair.get(batchTime).asScala) + Option(batchTimeToOutputOpIdSparkJobIdPair.get(batchTime)).map(_.asScala) .getOrElse(Seq.empty) _batchUIData.outputOpIdSparkJobIdPairs = outputOpIdToSparkJobIds } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15869][STREAMING] Fix a potential NPE in StreamingJobProgressListener.getBatchUIData
Repository: spark Updated Branches: refs/heads/branch-2.0 4e73cb8eb -> 1813bbd9b [SPARK-15869][STREAMING] Fix a potential NPE in StreamingJobProgressListener.getBatchUIData ## What changes were proposed in this pull request? Moved `asScala` to a `map` to avoid NPE. ## How was this patch tested? Existing unit tests. Author: Shixiong Zhu Closes #14443 from zsxwing/SPARK-15869. (cherry picked from commit 03d46aafe561b03e25f4e25cf01e631c18dd827c) Signed-off-by: Shixiong Zhu Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1813bbd9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1813bbd9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1813bbd9 Branch: refs/heads/branch-2.0 Commit: 1813bbd9bf7cb9afd29e1385f0dc52e8fcc4f132 Parents: 4e73cb8 Author: Shixiong Zhu Authored: Mon Aug 1 14:41:22 2016 -0700 Committer: Shixiong Zhu Committed: Mon Aug 1 14:41:34 2016 -0700 -- .../apache/spark/streaming/ui/StreamingJobProgressListener.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1813bbd9/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala -- diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index c086df4..61f852a 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -259,7 +259,7 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) // We use an Iterable rather than explicitly converting to a seq so that updates // will propagate val outputOpIdToSparkJobIds: Iterable[OutputOpIdAndSparkJobId] = -Option(batchTimeToOutputOpIdSparkJobIdPair.get(batchTime).asScala) + Option(batchTimeToOutputOpIdSparkJobIdPair.get(batchTime)).map(_.asScala) .getOrElse(Seq.empty) _batchUIData.outputOpIdSparkJobIdPairs = outputOpIdToSparkJobIds } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-16828][SQL] remove MaxOf and MinOf
Repository: spark
Updated Branches:
refs/heads/master 03d46aafe -> 2eedc00b0
[SPARK-16828][SQL] remove MaxOf and MinOf
## What changes were proposed in this pull request?
These 2 expressions are not needed anymore after we have `Greatest` and
`Least`. This PR removes them and related tests.
## How was this patch tested?
N/A
Author: Wenchen Fan
Closes #14434 from cloud-fan/minor1.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2eedc00b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2eedc00b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2eedc00b
Branch: refs/heads/master
Commit: 2eedc00b04ef8ca771ff64c4f834c25f835f5f44
Parents: 03d46aa
Author: Wenchen Fan
Authored: Mon Aug 1 17:54:41 2016 -0700
Committer: Yin Huai
Committed: Mon Aug 1 17:54:41 2016 -0700
--
.../sql/catalyst/expressions/arithmetic.scala | 110 ---
.../sql/catalyst/optimizer/Optimizer.scala | 4 -
.../analysis/ExpressionTypeCheckingSuite.scala | 7 --
.../expressions/ArithmeticExpressionSuite.scala | 54 -
4 files changed, 175 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2eedc00b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 7ff8795..77d40a5 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -361,116 +361,6 @@ case class Remainder(left: Expression, right: Expression)
}
}
-case class MaxOf(left: Expression, right: Expression)
- extends BinaryArithmetic with NonSQLExpression {
-
- // TODO: Remove MaxOf and MinOf, and replace its usage with Greatest and
Least.
-
- override def inputType: AbstractDataType = TypeCollection.Ordered
-
- override def nullable: Boolean = left.nullable && right.nullable
-
- private lazy val ordering = TypeUtils.getInterpretedOrdering(dataType)
-
- override def eval(input: InternalRow): Any = {
-val input1 = left.eval(input)
-val input2 = right.eval(input)
-if (input1 == null) {
- input2
-} else if (input2 == null) {
- input1
-} else {
- if (ordering.compare(input1, input2) < 0) {
-input2
- } else {
-input1
- }
-}
- }
-
- override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-val eval1 = left.genCode(ctx)
-val eval2 = right.genCode(ctx)
-val compCode = ctx.genComp(dataType, eval1.value, eval2.value)
-
-ev.copy(code = eval1.code + eval2.code + s"""
- boolean ${ev.isNull} = false;
- ${ctx.javaType(left.dataType)} ${ev.value} =
-${ctx.defaultValue(left.dataType)};
-
- if (${eval1.isNull}) {
-${ev.isNull} = ${eval2.isNull};
-${ev.value} = ${eval2.value};
- } else if (${eval2.isNull}) {
-${ev.isNull} = ${eval1.isNull};
-${ev.value} = ${eval1.value};
- } else {
-if ($compCode > 0) {
- ${ev.value} = ${eval1.value};
-} else {
- ${ev.value} = ${eval2.value};
-}
- }""")
- }
-
- override def symbol: String = "max"
-}
-
-case class MinOf(left: Expression, right: Expression)
- extends BinaryArithmetic with NonSQLExpression {
-
- // TODO: Remove MaxOf and MinOf, and replace its usage with Greatest and
Least.
-
- override def inputType: AbstractDataType = TypeCollection.Ordered
-
- override def nullable: Boolean = left.nullable && right.nullable
-
- private lazy val ordering = TypeUtils.getInterpretedOrdering(dataType)
-
- override def eval(input: InternalRow): Any = {
-val input1 = left.eval(input)
-val input2 = right.eval(input)
-if (input1 == null) {
- input2
-} else if (input2 == null) {
- input1
-} else {
- if (ordering.compare(input1, input2) < 0) {
-input1
- } else {
-input2
- }
-}
- }
-
- override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-val eval1 = left.genCode(ctx)
-val eval2 = right.genCode(ctx)
-val compCode = ctx.genComp(dataType, eval1.value, eval2.value)
-
-ev.copy(code = eval1.code + eval2.code + s"""
- boolean ${ev.isNull} = false;
- ${ctx.javaType(left.dataType)} ${ev.value} =
-${ctx.defaultValue(left.dataType)};
-
- if (${eval1.isNull}) {
-${ev.isNull} = ${eval2.isNull};
-${ev.value} = ${eval2.value};
- } else if (${eval2.isNull}) {
-${ev.isNull} = ${eval1.isNull};
-${ev.valu
spark git commit: [SPARK-16818] Exchange reuse incorrectly reuses scans over different sets of partitions
Repository: spark
Updated Branches:
refs/heads/branch-2.0 1813bbd9b -> 5fbf5f93e
[SPARK-16818] Exchange reuse incorrectly reuses scans over different sets of
partitions
https://github.com/apache/spark/pull/14425 rebased for branch-2.0
Author: Eric Liang
Closes #14427 from ericl/spark-16818-br-2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5fbf5f93
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5fbf5f93
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5fbf5f93
Branch: refs/heads/branch-2.0
Commit: 5fbf5f93ee5aa4d1aca0fa0c8fb769a085dd7b93
Parents: 1813bbd
Author: Eric Liang
Authored: Mon Aug 1 19:46:20 2016 -0700
Committer: Reynold Xin
Committed: Mon Aug 1 19:46:20 2016 -0700
--
.../datasources/FileSourceStrategy.scala| 2 ++
.../datasources/FileSourceStrategySuite.scala | 35 +++-
2 files changed, 36 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5fbf5f93/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
index 13a86bf..8af9562 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
@@ -202,7 +202,9 @@ private[sql] object FileSourceStrategy extends Strategy
with Logging {
partitions
}
+ // These metadata values make scan plans uniquely identifiable for
equality checking.
val meta = Map(
+"PartitionFilters" -> partitionKeyFilters.mkString("[", ", ", "]"),
"Format" -> files.fileFormat.toString,
"ReadSchema" -> prunedDataSchema.simpleString,
PUSHED_FILTERS -> pushedDownFilters.mkString("[", ", ", "]"),
http://git-wip-us.apache.org/repos/asf/spark/blob/5fbf5f93/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
index 8d8a18f..7a24f21 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategySuite.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionSet,
PredicateHelper}
import org.apache.spark.sql.catalyst.util
-import org.apache.spark.sql.execution.DataSourceScanExec
+import org.apache.spark.sql.execution.{DataSourceScanExec, SparkPlan}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources._
@@ -407,6 +407,39 @@ class FileSourceStrategySuite extends QueryTest with
SharedSQLContext with Predi
}
}
+ test("[SPARK-16818] partition pruned file scans implement sameResult
correctly") {
+withTempPath { path =>
+ val tempDir = path.getCanonicalPath
+ spark.range(100)
+.selectExpr("id", "id as b")
+.write
+.partitionBy("id")
+.parquet(tempDir)
+ val df = spark.read.parquet(tempDir)
+ def getPlan(df: DataFrame): SparkPlan = {
+df.queryExecution.executedPlan
+ }
+ assert(getPlan(df.where("id = 2")).sameResult(getPlan(df.where("id =
2"
+ assert(!getPlan(df.where("id = 2")).sameResult(getPlan(df.where("id =
3"
+}
+ }
+
+ test("[SPARK-16818] exchange reuse respects differences in partition
pruning") {
+spark.conf.set("spark.sql.exchange.reuse", true)
+withTempPath { path =>
+ val tempDir = path.getCanonicalPath
+ spark.range(10)
+.selectExpr("id % 2 as a", "id % 3 as b", "id as c")
+.write
+.partitionBy("a")
+.parquet(tempDir)
+ val df = spark.read.parquet(tempDir)
+ val df1 = df.where("a = 0").groupBy("b").agg("c" -> "sum")
+ val df2 = df.where("a = 1").groupBy("b").agg("c" -> "sum")
+ checkAnswer(df1.join(df2, "b"), Row(0, 6, 12) :: Row(1, 4, 8) :: Row(2,
10, 5) :: Nil)
+}
+ }
+
// Helpers for checking the arguments passed to the FileFormat.
protected val checkPartitionSchema =
-
To u
spark git commit: [SPARK-16793][SQL] Set the temporary warehouse path to sc'conf in TestHive.
Repository: spark
Updated Branches:
refs/heads/master 2eedc00b0 -> 5184df06b
[SPARK-16793][SQL] Set the temporary warehouse path to sc'conf in TestHive.
## What changes were proposed in this pull request?
With SPARK-15034, we could use the value of spark.sql.warehouse.dir to set the
warehouse location. In TestHive, we can now simply set the temporary warehouse
path in sc's conf, and thus, param "warehousePath" could be removed.
## How was this patch tested?
exsiting testsuites.
Author: jiangxingbo
Closes #14401 from jiangxb1987/warehousePath.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5184df06
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5184df06
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5184df06
Branch: refs/heads/master
Commit: 5184df06b347f86776c8ac87415b8002a5942a35
Parents: 2eedc00
Author: jiangxingbo
Authored: Mon Aug 1 23:08:06 2016 -0700
Committer: Reynold Xin
Committed: Mon Aug 1 23:08:06 2016 -0700
--
.../apache/spark/sql/hive/test/TestHive.scala | 42 +---
.../sql/hive/execution/HiveQuerySuite.scala | 2 +-
.../spark/sql/sources/BucketedReadSuite.scala | 2 +-
3 files changed, 21 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5184df06/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 7f89204..fbacd59 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -54,6 +54,7 @@ object TestHive
.set("spark.sql.test", "")
.set("spark.sql.hive.metastore.barrierPrefixes",
"org.apache.spark.sql.hive.execution.PairSerDe")
+.set("spark.sql.warehouse.dir",
TestHiveContext.makeWarehouseDir().toURI.getPath)
// SPARK-8910
.set("spark.ui.enabled", "false")))
@@ -111,7 +112,6 @@ class TestHiveContext(
* A [[SparkSession]] used in [[TestHiveContext]].
*
* @param sc SparkContext
- * @param warehousePath path to the Hive warehouse directory
* @param scratchDirPath scratch directory used by Hive's metastore client
* @param metastoreTemporaryConf configuration options for Hive's metastore
* @param existingSharedState optional [[TestHiveSharedState]]
@@ -120,23 +120,15 @@ class TestHiveContext(
*/
private[hive] class TestHiveSparkSession(
@transient private val sc: SparkContext,
-val warehousePath: File,
scratchDirPath: File,
metastoreTemporaryConf: Map[String, String],
@transient private val existingSharedState: Option[TestHiveSharedState],
private val loadTestTables: Boolean)
extends SparkSession(sc) with Logging { self =>
- // TODO: We need to set the temp warehouse path to sc's conf.
- // Right now, In SparkSession, we will set the warehouse path to the default
one
- // instead of the temp one. Then, we override the setting in
TestHiveSharedState
- // when we creating metadataHive. This flow is not easy to follow and can
introduce
- // confusion when a developer is debugging an issue. We need to refactor
this part
- // to just set the temp warehouse path in sc's conf.
def this(sc: SparkContext, loadTestTables: Boolean) {
this(
sc,
- Utils.createTempDir(namePrefix = "warehouse"),
TestHiveContext.makeScratchDir(),
HiveUtils.newTemporaryConfiguration(useInMemoryDerby = false),
None,
@@ -151,16 +143,16 @@ private[hive] class TestHiveSparkSession(
@transient
override lazy val sharedState: TestHiveSharedState = {
existingSharedState.getOrElse(
- new TestHiveSharedState(sc, warehousePath, scratchDirPath,
metastoreTemporaryConf))
+ new TestHiveSharedState(sc, scratchDirPath, metastoreTemporaryConf))
}
@transient
override lazy val sessionState: TestHiveSessionState =
-new TestHiveSessionState(self, warehousePath)
+new TestHiveSessionState(self)
override def newSession(): TestHiveSparkSession = {
new TestHiveSparkSession(
- sc, warehousePath, scratchDirPath, metastoreTemporaryConf,
Some(sharedState), loadTestTables)
+ sc, scratchDirPath, metastoreTemporaryConf, Some(sharedState),
loadTestTables)
}
private var cacheTables: Boolean = false
@@ -199,6 +191,12 @@ private[hive] class TestHiveSparkSession(
new
File(Thread.currentThread().getContextClassLoader.getResource(path).getFile)
}
+ def getWarehousePath(): String = {
+val tempConf = new SQLConf
+sc.conf.getAll.foreach { case (k, v) => tempConf.setConfString(k, v) }
