spark git commit: [SPARK-15436][SQL] Remove DescribeFunction and ShowFunctions
Repository: spark Updated Branches: refs/heads/branch-2.0 1dad1a891 -> f63ba2210 [SPARK-15436][SQL] Remove DescribeFunction and ShowFunctions ## What changes were proposed in this pull request? This patch removes the last two commands defined in the catalyst module: DescribeFunction and ShowFunctions. They were unnecessary since the parser could just generate DescribeFunctionCommand and ShowFunctionsCommand directly. ## How was this patch tested? Created a new SparkSqlParserSuite. Author: Reynold Xin Closes #13292 from rxin/SPARK-15436. (cherry picked from commit 4f27b8dd58a66fca7ddd4c239e02b90c34b1cebd) Signed-off-by: Herman van Hovell Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f63ba221 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f63ba221 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f63ba221 Branch: refs/heads/branch-2.0 Commit: f63ba2210a27df1843f5007d367036dae0e0139f Parents: 1dad1a8 Author: Reynold Xin Authored: Wed May 25 19:17:53 2016 +0200 Committer: Herman van Hovell Committed: Wed May 25 19:18:32 2016 +0200 -- .../spark/sql/catalyst/parser/AstBuilder.scala | 32 + .../sql/catalyst/plans/logical/Command.scala| 25 +++ .../sql/catalyst/plans/logical/commands.scala | 55 .../analysis/UnsupportedOperationsSuite.scala | 9 ++- .../sql/catalyst/parser/PlanParserSuite.scala | 30 +++-- .../spark/sql/execution/SparkSqlParser.scala| 33 +- .../spark/sql/execution/SparkStrategies.scala | 6 -- .../sql/execution/SparkSqlParserSuite.scala | 68 .../sql/execution/command/DDLCommandSuite.scala | 2 +- .../sql/hive/execution/HiveComparisonTest.scala | 6 +- 10 files changed, 145 insertions(+), 121 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f63ba221/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index a13c03a..3473fee 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.catalyst.parser import java.sql.{Date, Timestamp} @@ -82,37 +83,6 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { protected def plan(tree: ParserRuleContext): LogicalPlan = typedVisit(tree) /** - * Create a plan for a SHOW FUNCTIONS command. - */ - override def visitShowFunctions(ctx: ShowFunctionsContext): LogicalPlan = withOrigin(ctx) { -import ctx._ -if (qualifiedName != null) { - val name = visitFunctionName(qualifiedName) - ShowFunctions(name.database, Some(name.funcName)) -} else if (pattern != null) { - ShowFunctions(None, Some(string(pattern))) -} else { - ShowFunctions(None, None) -} - } - - /** - * Create a plan for a DESCRIBE FUNCTION command. - */ - override def visitDescribeFunction(ctx: DescribeFunctionContext): LogicalPlan = withOrigin(ctx) { -import ctx._ -val functionName = - if (describeFuncName.STRING() != null) { -FunctionIdentifier(string(describeFuncName.STRING()), database = None) - } else if (describeFuncName.qualifiedName() != null) { -visitFunctionName(describeFuncName.qualifiedName) - } else { -FunctionIdentifier(describeFuncName.getText, database = None) - } -DescribeFunction(functionName, EXTENDED != null) - } - - /** * Create a top-level plan with Common Table Expressions. */ override def visitQuery(ctx: QueryContext): LogicalPlan = withOrigin(ctx) { http://git-wip-us.apache.org/repos/asf/spark/blob/f63ba221/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala new file mode 100644 index 000..75a5b10 --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. +
spark git commit: [SPARK-15436][SQL] Remove DescribeFunction and ShowFunctions
Repository: spark Updated Branches: refs/heads/master 9082b7968 -> 4f27b8dd5 [SPARK-15436][SQL] Remove DescribeFunction and ShowFunctions ## What changes were proposed in this pull request? This patch removes the last two commands defined in the catalyst module: DescribeFunction and ShowFunctions. They were unnecessary since the parser could just generate DescribeFunctionCommand and ShowFunctionsCommand directly. ## How was this patch tested? Created a new SparkSqlParserSuite. Author: Reynold Xin Closes #13292 from rxin/SPARK-15436. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4f27b8dd Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4f27b8dd Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4f27b8dd Branch: refs/heads/master Commit: 4f27b8dd58a66fca7ddd4c239e02b90c34b1cebd Parents: 9082b79 Author: Reynold Xin Authored: Wed May 25 19:17:53 2016 +0200 Committer: Herman van Hovell Committed: Wed May 25 19:17:53 2016 +0200 -- .../spark/sql/catalyst/parser/AstBuilder.scala | 32 + .../sql/catalyst/plans/logical/Command.scala| 25 +++ .../sql/catalyst/plans/logical/commands.scala | 55 .../analysis/UnsupportedOperationsSuite.scala | 9 ++- .../sql/catalyst/parser/PlanParserSuite.scala | 30 +++-- .../spark/sql/execution/SparkSqlParser.scala| 33 +- .../spark/sql/execution/SparkStrategies.scala | 6 -- .../sql/execution/SparkSqlParserSuite.scala | 68 .../sql/execution/command/DDLCommandSuite.scala | 2 +- .../sql/hive/execution/HiveComparisonTest.scala | 6 +- 10 files changed, 145 insertions(+), 121 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/4f27b8dd/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index a13c03a..3473fee 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.catalyst.parser import java.sql.{Date, Timestamp} @@ -82,37 +83,6 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { protected def plan(tree: ParserRuleContext): LogicalPlan = typedVisit(tree) /** - * Create a plan for a SHOW FUNCTIONS command. - */ - override def visitShowFunctions(ctx: ShowFunctionsContext): LogicalPlan = withOrigin(ctx) { -import ctx._ -if (qualifiedName != null) { - val name = visitFunctionName(qualifiedName) - ShowFunctions(name.database, Some(name.funcName)) -} else if (pattern != null) { - ShowFunctions(None, Some(string(pattern))) -} else { - ShowFunctions(None, None) -} - } - - /** - * Create a plan for a DESCRIBE FUNCTION command. - */ - override def visitDescribeFunction(ctx: DescribeFunctionContext): LogicalPlan = withOrigin(ctx) { -import ctx._ -val functionName = - if (describeFuncName.STRING() != null) { -FunctionIdentifier(string(describeFuncName.STRING()), database = None) - } else if (describeFuncName.qualifiedName() != null) { -visitFunctionName(describeFuncName.qualifiedName) - } else { -FunctionIdentifier(describeFuncName.getText, database = None) - } -DescribeFunction(functionName, EXTENDED != null) - } - - /** * Create a top-level plan with Common Table Expressions. */ override def visitQuery(ctx: QueryContext): LogicalPlan = withOrigin(ctx) { http://git-wip-us.apache.org/repos/asf/spark/blob/4f27b8dd/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala new file mode 100644 index 000..75a5b10 --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Command.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use t
spark git commit: [SPARK-9044] Fix "Storage" tab in UI so that it reflects RDD name change.
Repository: spark Updated Branches: refs/heads/master 4f27b8dd5 -> b120fba6a [SPARK-9044] Fix "Storage" tab in UI so that it reflects RDD name change. ## What changes were proposed in this pull request? 1. Making 'name' field of RDDInfo mutable. 2. In StorageListener: catching the fact that RDD's name was changed and updating it in RDDInfo. ## How was this patch tested? 1. Manual verification - the 'Storage' tab now behaves as expected. 2. The commit also contains a new unit test which verifies this. Author: Lukasz Closes #13264 from lgieron/SPARK-9044. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b120fba6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b120fba6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b120fba6 Branch: refs/heads/master Commit: b120fba6ae26186b3fa0dfbb1637046f4e76c2b0 Parents: 4f27b8d Author: Lukasz Authored: Wed May 25 10:24:21 2016 -0700 Committer: Shixiong Zhu Committed: Wed May 25 10:24:21 2016 -0700 -- .../scala/org/apache/spark/storage/RDDInfo.scala | 2 +- .../org/apache/spark/ui/storage/StorageTab.scala | 2 +- .../apache/spark/ui/storage/StorageTabSuite.scala | 17 + 3 files changed, 19 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b120fba6/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala -- diff --git a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala index 083d78b..e5abbf7 100644 --- a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala +++ b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala @@ -24,7 +24,7 @@ import org.apache.spark.util.Utils @DeveloperApi class RDDInfo( val id: Int, -val name: String, +var name: String, val numPartitions: Int, var storageLevel: StorageLevel, val parentIds: Seq[Int], http://git-wip-us.apache.org/repos/asf/spark/blob/b120fba6/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala index 5009583..c212362 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala @@ -59,7 +59,7 @@ class StorageListener(storageStatusListener: StorageStatusListener) extends Bloc override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized { val rddInfos = stageSubmitted.stageInfo.rddInfos -rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info) } +rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info).name = info.name } } override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized { http://git-wip-us.apache.org/repos/asf/spark/blob/b120fba6/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala b/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala index 7d77dee..411a0dd 100644 --- a/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala @@ -179,6 +179,23 @@ class StorageTabSuite extends SparkFunSuite with BeforeAndAfter { assert(storageListener.rddInfoList.size === 2) } + test("verify StorageTab still contains a renamed RDD") { +val rddInfo = new RDDInfo(0, "original_name", 1, memOnly, Seq(4)) +val stageInfo0 = new StageInfo(0, 0, "stage0", 1, Seq(rddInfo), Seq.empty, "details") +bus.postToAll(SparkListenerBlockManagerAdded(1L, bm1, 1000L)) +bus.postToAll(SparkListenerStageSubmitted(stageInfo0)) +val blockUpdateInfos1 = Seq(BlockUpdatedInfo(bm1, RDDBlockId(0, 1), memOnly, 100L, 0L)) +postUpdateBlocks(bus, blockUpdateInfos1) +assert(storageListener.rddInfoList.size == 1) + +val newName = "new_name" +val rddInfoRenamed = new RDDInfo(0, newName, 1, memOnly, Seq(4)) +val stageInfo1 = new StageInfo(1, 0, "stage1", 1, Seq(rddInfoRenamed), Seq.empty, "details") +bus.postToAll(SparkListenerStageSubmitted(stageInfo1)) +assert(storageListener.rddInfoList.size == 1) +assert(storageListener.rddInfoList.head.name == newName) + } + private def postUpdateBlocks( bus: SparkListenerBus, blockUpdateInfos: Seq[BlockUpdatedInfo]): Unit = { blockUpdateInfos.foreach { blockUpdateInfo => ---
spark git commit: [SPARK-9044] Fix "Storage" tab in UI so that it reflects RDD name change.
Repository: spark Updated Branches: refs/heads/branch-2.0 f63ba2210 -> 69327667d [SPARK-9044] Fix "Storage" tab in UI so that it reflects RDD name change. ## What changes were proposed in this pull request? 1. Making 'name' field of RDDInfo mutable. 2. In StorageListener: catching the fact that RDD's name was changed and updating it in RDDInfo. ## How was this patch tested? 1. Manual verification - the 'Storage' tab now behaves as expected. 2. The commit also contains a new unit test which verifies this. Author: Lukasz Closes #13264 from lgieron/SPARK-9044. (cherry picked from commit b120fba6ae26186b3fa0dfbb1637046f4e76c2b0) Signed-off-by: Shixiong Zhu Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/69327667 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/69327667 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/69327667 Branch: refs/heads/branch-2.0 Commit: 69327667d5a14b12de8055d752fbe3abb8d6671c Parents: f63ba22 Author: Lukasz Authored: Wed May 25 10:24:21 2016 -0700 Committer: Shixiong Zhu Committed: Wed May 25 10:24:28 2016 -0700 -- .../scala/org/apache/spark/storage/RDDInfo.scala | 2 +- .../org/apache/spark/ui/storage/StorageTab.scala | 2 +- .../apache/spark/ui/storage/StorageTabSuite.scala | 17 + 3 files changed, 19 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/69327667/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala -- diff --git a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala index 083d78b..e5abbf7 100644 --- a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala +++ b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala @@ -24,7 +24,7 @@ import org.apache.spark.util.Utils @DeveloperApi class RDDInfo( val id: Int, -val name: String, +var name: String, val numPartitions: Int, var storageLevel: StorageLevel, val parentIds: Seq[Int], http://git-wip-us.apache.org/repos/asf/spark/blob/69327667/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala index 5009583..c212362 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala @@ -59,7 +59,7 @@ class StorageListener(storageStatusListener: StorageStatusListener) extends Bloc override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized { val rddInfos = stageSubmitted.stageInfo.rddInfos -rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info) } +rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info).name = info.name } } override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized { http://git-wip-us.apache.org/repos/asf/spark/blob/69327667/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala b/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala index 7d77dee..411a0dd 100644 --- a/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala @@ -179,6 +179,23 @@ class StorageTabSuite extends SparkFunSuite with BeforeAndAfter { assert(storageListener.rddInfoList.size === 2) } + test("verify StorageTab still contains a renamed RDD") { +val rddInfo = new RDDInfo(0, "original_name", 1, memOnly, Seq(4)) +val stageInfo0 = new StageInfo(0, 0, "stage0", 1, Seq(rddInfo), Seq.empty, "details") +bus.postToAll(SparkListenerBlockManagerAdded(1L, bm1, 1000L)) +bus.postToAll(SparkListenerStageSubmitted(stageInfo0)) +val blockUpdateInfos1 = Seq(BlockUpdatedInfo(bm1, RDDBlockId(0, 1), memOnly, 100L, 0L)) +postUpdateBlocks(bus, blockUpdateInfos1) +assert(storageListener.rddInfoList.size == 1) + +val newName = "new_name" +val rddInfoRenamed = new RDDInfo(0, newName, 1, memOnly, Seq(4)) +val stageInfo1 = new StageInfo(1, 0, "stage1", 1, Seq(rddInfoRenamed), Seq.empty, "details") +bus.postToAll(SparkListenerStageSubmitted(stageInfo1)) +assert(storageListener.rddInfoList.size == 1) +assert(storageListener.rddInfoList.head.name == newName) + } + private def postUpdateBlocks( bus: SparkListenerBus,
spark git commit: [SPARK-15345][SQL][PYSPARK] SparkSession's conf doesn't take effect when this already an existing SparkContext
Repository: spark Updated Branches: refs/heads/master b120fba6a -> 01e7b9c85 [SPARK-15345][SQL][PYSPARK] SparkSession's conf doesn't take effect when this already an existing SparkContext ## What changes were proposed in this pull request? Override the existing SparkContext is the provided SparkConf is different. PySpark part hasn't been fixed yet, will do that after the first round of review to ensure this is the correct approach. ## How was this patch tested? Manually verify it in spark-shell. rxin Please help review it, I think this is a very critical issue for spark 2.0 Author: Jeff Zhang Closes #13160 from zjffdu/SPARK-15345. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/01e7b9c8 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/01e7b9c8 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/01e7b9c8 Branch: refs/heads/master Commit: 01e7b9c85bb84924e279021f9748774dce9702c8 Parents: b120fba Author: Jeff Zhang Authored: Wed May 25 10:46:51 2016 -0700 Committer: Andrew Or Committed: Wed May 25 10:46:51 2016 -0700 -- .../main/scala/org/apache/spark/SparkContext.scala| 3 +++ .../scala/org/apache/spark/sql/SparkSession.scala | 14 -- .../apache/spark/sql/SparkSessionBuilderSuite.scala | 14 +- 3 files changed, 28 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/01e7b9c8/core/src/main/scala/org/apache/spark/SparkContext.scala -- diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 36aa3be..5018eb3 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -2254,6 +2254,9 @@ object SparkContext extends Logging { if (activeContext.get() == null) { setActiveContext(new SparkContext(config), allowMultipleContexts = false) } + if (config.getAll.nonEmpty) { +logWarning("Use an existing SparkContext, some configuration may not take effect.") + } activeContext.get() } } http://git-wip-us.apache.org/repos/asf/spark/blob/01e7b9c8/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 5c87c84..86c97b9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -636,7 +636,7 @@ object SparkSession { /** * Builder for [[SparkSession]]. */ - class Builder { + class Builder extends Logging { private[this] val options = new scala.collection.mutable.HashMap[String, String] @@ -753,6 +753,9 @@ object SparkSession { var session = activeThreadSession.get() if ((session ne null) && !session.sparkContext.isStopped) { options.foreach { case (k, v) => session.conf.set(k, v) } +if (options.nonEmpty) { + logWarning("Use an existing SparkSession, some configuration may not take effect.") +} return session } @@ -762,6 +765,9 @@ object SparkSession { session = defaultSession.get() if ((session ne null) && !session.sparkContext.isStopped) { options.foreach { case (k, v) => session.conf.set(k, v) } + if (options.nonEmpty) { +logWarning("Use an existing SparkSession, some configuration may not take effect.") + } return session } @@ -774,7 +780,11 @@ object SparkSession { val sparkConf = new SparkConf() options.foreach { case (k, v) => sparkConf.set(k, v) } - SparkContext.getOrCreate(sparkConf) + val sc = SparkContext.getOrCreate(sparkConf) + // maybe this is an existing SparkContext, update its SparkConf which maybe used + // by SparkSession + options.foreach { case (k, v) => sc.conf.set(k, v) } + sc } session = new SparkSession(sparkContext) options.foreach { case (k, v) => session.conf.set(k, v) } http://git-wip-us.apache.org/repos/asf/spark/blob/01e7b9c8/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala index ec6a2b3..786956d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSu
spark git commit: [SPARK-15345][SQL][PYSPARK] SparkSession's conf doesn't take effect when this already an existing SparkContext
Repository: spark Updated Branches: refs/heads/branch-2.0 69327667d -> 27f26a39d [SPARK-15345][SQL][PYSPARK] SparkSession's conf doesn't take effect when this already an existing SparkContext ## What changes were proposed in this pull request? Override the existing SparkContext is the provided SparkConf is different. PySpark part hasn't been fixed yet, will do that after the first round of review to ensure this is the correct approach. ## How was this patch tested? Manually verify it in spark-shell. rxin Please help review it, I think this is a very critical issue for spark 2.0 Author: Jeff Zhang Closes #13160 from zjffdu/SPARK-15345. (cherry picked from commit 01e7b9c85bb84924e279021f9748774dce9702c8) Signed-off-by: Andrew Or Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/27f26a39 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/27f26a39 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/27f26a39 Branch: refs/heads/branch-2.0 Commit: 27f26a39db021735bcf75a1f1b89b9481b199341 Parents: 6932766 Author: Jeff Zhang Authored: Wed May 25 10:46:51 2016 -0700 Committer: Andrew Or Committed: Wed May 25 10:47:00 2016 -0700 -- .../main/scala/org/apache/spark/SparkContext.scala| 3 +++ .../scala/org/apache/spark/sql/SparkSession.scala | 14 -- .../apache/spark/sql/SparkSessionBuilderSuite.scala | 14 +- 3 files changed, 28 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/27f26a39/core/src/main/scala/org/apache/spark/SparkContext.scala -- diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 36aa3be..5018eb3 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -2254,6 +2254,9 @@ object SparkContext extends Logging { if (activeContext.get() == null) { setActiveContext(new SparkContext(config), allowMultipleContexts = false) } + if (config.getAll.nonEmpty) { +logWarning("Use an existing SparkContext, some configuration may not take effect.") + } activeContext.get() } } http://git-wip-us.apache.org/repos/asf/spark/blob/27f26a39/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 5c87c84..86c97b9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -636,7 +636,7 @@ object SparkSession { /** * Builder for [[SparkSession]]. */ - class Builder { + class Builder extends Logging { private[this] val options = new scala.collection.mutable.HashMap[String, String] @@ -753,6 +753,9 @@ object SparkSession { var session = activeThreadSession.get() if ((session ne null) && !session.sparkContext.isStopped) { options.foreach { case (k, v) => session.conf.set(k, v) } +if (options.nonEmpty) { + logWarning("Use an existing SparkSession, some configuration may not take effect.") +} return session } @@ -762,6 +765,9 @@ object SparkSession { session = defaultSession.get() if ((session ne null) && !session.sparkContext.isStopped) { options.foreach { case (k, v) => session.conf.set(k, v) } + if (options.nonEmpty) { +logWarning("Use an existing SparkSession, some configuration may not take effect.") + } return session } @@ -774,7 +780,11 @@ object SparkSession { val sparkConf = new SparkConf() options.foreach { case (k, v) => sparkConf.set(k, v) } - SparkContext.getOrCreate(sparkConf) + val sc = SparkContext.getOrCreate(sparkConf) + // maybe this is an existing SparkContext, update its SparkConf which maybe used + // by SparkSession + options.foreach { case (k, v) => sc.conf.set(k, v) } + sc } session = new SparkSession(sparkContext) options.foreach { case (k, v) => session.conf.set(k, v) } http://git-wip-us.apache.org/repos/asf/spark/blob/27f26a39/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
spark git commit: [SPARK-15520][SQL] SparkSession builder in python should also allow overriding confs of existing sessions
Repository: spark Updated Branches: refs/heads/branch-2.0 27f26a39d -> c75ec5eaa [SPARK-15520][SQL] SparkSession builder in python should also allow overriding confs of existing sessions ## What changes were proposed in this pull request? This fixes the python SparkSession builder to allow setting confs correctly. This was a leftover TODO from https://github.com/apache/spark/pull/13200. ## How was this patch tested? Python doc tests. cc andrewor14 Author: Eric Liang Closes #13289 from ericl/spark-15520. (cherry picked from commit 8239fdcb9b54ab6d13c31ad9916b8334dd1462c2) Signed-off-by: Andrew Or Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c75ec5ea Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c75ec5ea Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c75ec5ea Branch: refs/heads/branch-2.0 Commit: c75ec5eaa6c95b3647c80b6595902d16ab3165fa Parents: 27f26a39 Author: Eric Liang Authored: Wed May 25 10:49:11 2016 -0700 Committer: Andrew Or Committed: Wed May 25 10:49:19 2016 -0700 -- python/pyspark/sql/session.py | 35 --- 1 file changed, 24 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c75ec5ea/python/pyspark/sql/session.py -- diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 2419475..52e7f3d 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -138,24 +138,37 @@ class SparkSession(object): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. -This method first checks whether there is a valid thread-local SparkSession, -and if yes, return that one. It then checks whether there is a valid global -default SparkSession, and if yes, return that one. If no valid global default -SparkSession exists, the method creates a new SparkSession and assigns the -newly created SparkSession as the global default. +This method first checks whether there is a valid global default SparkSession, and if +yes, return that one. If no valid global default SparkSession exists, the method +creates a new SparkSession and assigns the newly created SparkSession as the global +default. + +>>> s1 = SparkSession.builder.config("k1", "v1").getOrCreate() +>>> s1.conf.get("k1") == "v1" +True In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. + +>>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() +>>> s1.conf.get("k1") == s2.conf.get("k1") +True +>>> s1.conf.get("k2") == s2.conf.get("k2") +True """ with self._lock: -from pyspark.conf import SparkConf from pyspark.context import SparkContext -from pyspark.sql.context import SQLContext -sparkConf = SparkConf() +from pyspark.conf import SparkConf +session = SparkSession._instantiatedContext +if session is None: +sparkConf = SparkConf() +for key, value in self._options.items(): +sparkConf.set(key, value) +sc = SparkContext.getOrCreate(sparkConf) +session = SparkSession(sc) for key, value in self._options.items(): -sparkConf.set(key, value) -sparkContext = SparkContext.getOrCreate(sparkConf) -return SQLContext.getOrCreate(sparkContext).sparkSession +session.conf.set(key, value) +return session builder = Builder() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15520][SQL] SparkSession builder in python should also allow overriding confs of existing sessions
Repository: spark Updated Branches: refs/heads/master 01e7b9c85 -> 8239fdcb9 [SPARK-15520][SQL] SparkSession builder in python should also allow overriding confs of existing sessions ## What changes were proposed in this pull request? This fixes the python SparkSession builder to allow setting confs correctly. This was a leftover TODO from https://github.com/apache/spark/pull/13200. ## How was this patch tested? Python doc tests. cc andrewor14 Author: Eric Liang Closes #13289 from ericl/spark-15520. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8239fdcb Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8239fdcb Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8239fdcb Branch: refs/heads/master Commit: 8239fdcb9b54ab6d13c31ad9916b8334dd1462c2 Parents: 01e7b9c Author: Eric Liang Authored: Wed May 25 10:49:11 2016 -0700 Committer: Andrew Or Committed: Wed May 25 10:49:11 2016 -0700 -- python/pyspark/sql/session.py | 35 --- 1 file changed, 24 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/8239fdcb/python/pyspark/sql/session.py -- diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 2419475..52e7f3d 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -138,24 +138,37 @@ class SparkSession(object): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a new one based on the options set in this builder. -This method first checks whether there is a valid thread-local SparkSession, -and if yes, return that one. It then checks whether there is a valid global -default SparkSession, and if yes, return that one. If no valid global default -SparkSession exists, the method creates a new SparkSession and assigns the -newly created SparkSession as the global default. +This method first checks whether there is a valid global default SparkSession, and if +yes, return that one. If no valid global default SparkSession exists, the method +creates a new SparkSession and assigns the newly created SparkSession as the global +default. + +>>> s1 = SparkSession.builder.config("k1", "v1").getOrCreate() +>>> s1.conf.get("k1") == "v1" +True In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. + +>>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() +>>> s1.conf.get("k1") == s2.conf.get("k1") +True +>>> s1.conf.get("k2") == s2.conf.get("k2") +True """ with self._lock: -from pyspark.conf import SparkConf from pyspark.context import SparkContext -from pyspark.sql.context import SQLContext -sparkConf = SparkConf() +from pyspark.conf import SparkConf +session = SparkSession._instantiatedContext +if session is None: +sparkConf = SparkConf() +for key, value in self._options.items(): +sparkConf.set(key, value) +sc = SparkContext.getOrCreate(sparkConf) +session = SparkSession(sc) for key, value in self._options.items(): -sparkConf.set(key, value) -sparkContext = SparkContext.getOrCreate(sparkConf) -return SQLContext.getOrCreate(sparkContext).sparkSession +session.conf.set(key, value) +return session builder = Builder() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [MINOR][CORE] Fix a HadoopRDD log message and remove unused imports in rdd files.
Repository: spark Updated Branches: refs/heads/master 8239fdcb9 -> d6d3e5071 [MINOR][CORE] Fix a HadoopRDD log message and remove unused imports in rdd files. ## What changes were proposed in this pull request? This PR fixes the following typos in log message and comments of `HadoopRDD.scala`. Also, this removes unused imports. ```scala - logWarning("Caching NewHadoopRDDs as deserialized objects usually leads to undesired" + + logWarning("Caching HadoopRDDs as deserialized objects usually leads to undesired" + ... - // since its not removed yet + // since it's not removed yet ``` ## How was this patch tested? Manual. Author: Dongjoon Hyun Closes #13294 from dongjoon-hyun/minor_rdd_fix_log_message. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d6d3e507 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d6d3e507 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d6d3e507 Branch: refs/heads/master Commit: d6d3e50719b01005aa0e77349fc9a6ff88fecce3 Parents: 8239fdc Author: Dongjoon Hyun Authored: Wed May 25 10:51:33 2016 -0700 Committer: Andrew Or Committed: Wed May 25 10:51:33 2016 -0700 -- core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala| 2 +- core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala| 5 ++--- core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala | 1 - core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 +- core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala | 1 - 5 files changed, 4 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d6d3e507/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala -- diff --git a/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala b/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala index be0cb17..41832e8 100644 --- a/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala @@ -17,7 +17,7 @@ package org.apache.spark.rdd -import org.apache.hadoop.conf.{ Configurable, Configuration } +import org.apache.hadoop.conf.{Configurable, Configuration} import org.apache.hadoop.io.Writable import org.apache.hadoop.mapreduce._ import org.apache.hadoop.mapreduce.task.JobContextImpl http://git-wip-us.apache.org/repos/asf/spark/blob/d6d3e507/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala -- diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala index b22134a..515fd6f 100644 --- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala @@ -43,7 +43,6 @@ import org.apache.spark._ import org.apache.spark.annotation.DeveloperApi import org.apache.spark.broadcast.Broadcast import org.apache.spark.deploy.SparkHadoopUtil -import org.apache.spark.executor.DataReadMethod import org.apache.spark.internal.Logging import org.apache.spark.rdd.HadoopRDD.HadoopMapPartitionsWithSplitRDD import org.apache.spark.scheduler.{HDFSCacheTaskLocation, HostTaskLocation} @@ -70,7 +69,7 @@ private[spark] class HadoopPartition(rddId: Int, override val index: Int, s: Inp val envVars: Map[String, String] = if (inputSplit.value.isInstanceOf[FileSplit]) { val is: FileSplit = inputSplit.value.asInstanceOf[FileSplit] // map_input_file is deprecated in favor of mapreduce_map_input_file but set both - // since its not removed yet + // since it's not removed yet Map("map_input_file" -> is.getPath().toString(), "mapreduce_map_input_file" -> is.getPath().toString()) } else { @@ -335,7 +334,7 @@ class HadoopRDD[K, V]( override def persist(storageLevel: StorageLevel): this.type = { if (storageLevel.deserialized) { - logWarning("Caching NewHadoopRDDs as deserialized objects usually leads to undesired" + + logWarning("Caching HadoopRDDs as deserialized objects usually leads to undesired" + " behavior because Hadoop's RecordReader reuses the same Writable object for all records." + " Use a map transformation to make copies of the records.") } http://git-wip-us.apache.org/repos/asf/spark/blob/d6d3e507/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala -- diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala index ad7c221..189dc7b 100644 --- a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala +++ b/core/src/main/scala
spark git commit: [MINOR][CORE] Fix a HadoopRDD log message and remove unused imports in rdd files.
Repository: spark Updated Branches: refs/heads/branch-2.0 c75ec5eaa -> 4009ddafd [MINOR][CORE] Fix a HadoopRDD log message and remove unused imports in rdd files. ## What changes were proposed in this pull request? This PR fixes the following typos in log message and comments of `HadoopRDD.scala`. Also, this removes unused imports. ```scala - logWarning("Caching NewHadoopRDDs as deserialized objects usually leads to undesired" + + logWarning("Caching HadoopRDDs as deserialized objects usually leads to undesired" + ... - // since its not removed yet + // since it's not removed yet ``` ## How was this patch tested? Manual. Author: Dongjoon Hyun Closes #13294 from dongjoon-hyun/minor_rdd_fix_log_message. (cherry picked from commit d6d3e50719b01005aa0e77349fc9a6ff88fecce3) Signed-off-by: Andrew Or Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4009ddaf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4009ddaf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4009ddaf Branch: refs/heads/branch-2.0 Commit: 4009ddafd810f91f699e52d7822c8c959fe7761e Parents: c75ec5e Author: Dongjoon Hyun Authored: Wed May 25 10:51:33 2016 -0700 Committer: Andrew Or Committed: Wed May 25 10:51:41 2016 -0700 -- core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala| 2 +- core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala| 5 ++--- core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala | 1 - core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 +- core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala | 1 - 5 files changed, 4 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/4009ddaf/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala -- diff --git a/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala b/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala index be0cb17..41832e8 100644 --- a/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/BinaryFileRDD.scala @@ -17,7 +17,7 @@ package org.apache.spark.rdd -import org.apache.hadoop.conf.{ Configurable, Configuration } +import org.apache.hadoop.conf.{Configurable, Configuration} import org.apache.hadoop.io.Writable import org.apache.hadoop.mapreduce._ import org.apache.hadoop.mapreduce.task.JobContextImpl http://git-wip-us.apache.org/repos/asf/spark/blob/4009ddaf/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala -- diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala index b22134a..515fd6f 100644 --- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala @@ -43,7 +43,6 @@ import org.apache.spark._ import org.apache.spark.annotation.DeveloperApi import org.apache.spark.broadcast.Broadcast import org.apache.spark.deploy.SparkHadoopUtil -import org.apache.spark.executor.DataReadMethod import org.apache.spark.internal.Logging import org.apache.spark.rdd.HadoopRDD.HadoopMapPartitionsWithSplitRDD import org.apache.spark.scheduler.{HDFSCacheTaskLocation, HostTaskLocation} @@ -70,7 +69,7 @@ private[spark] class HadoopPartition(rddId: Int, override val index: Int, s: Inp val envVars: Map[String, String] = if (inputSplit.value.isInstanceOf[FileSplit]) { val is: FileSplit = inputSplit.value.asInstanceOf[FileSplit] // map_input_file is deprecated in favor of mapreduce_map_input_file but set both - // since its not removed yet + // since it's not removed yet Map("map_input_file" -> is.getPath().toString(), "mapreduce_map_input_file" -> is.getPath().toString()) } else { @@ -335,7 +334,7 @@ class HadoopRDD[K, V]( override def persist(storageLevel: StorageLevel): this.type = { if (storageLevel.deserialized) { - logWarning("Caching NewHadoopRDDs as deserialized objects usually leads to undesired" + + logWarning("Caching HadoopRDDs as deserialized objects usually leads to undesired" + " behavior because Hadoop's RecordReader reuses the same Writable object for all records." + " Use a map transformation to make copies of the records.") } http://git-wip-us.apache.org/repos/asf/spark/blob/4009ddaf/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala -- diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala index ad7c221..18
spark git commit: [MINOR][MLLIB][STREAMING][SQL] Fix typos
Repository: spark Updated Branches: refs/heads/branch-2.0 4009ddafd -> 6fc367e50 [MINOR][MLLIB][STREAMING][SQL] Fix typos fixed typos for source code for components [mllib] [streaming] and [SQL] None and obvious. Author: lfzCarlosC Closes #13298 from lfzCarlosC/master. (cherry picked from commit 02c8072eea72425e89256347e1f373a3e76e6eba) Signed-off-by: Andrew Or Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6fc367e5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6fc367e5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6fc367e5 Branch: refs/heads/branch-2.0 Commit: 6fc367e50c2de9f68237763919eae12241a2179f Parents: 4009dda Author: lfzCarlosC Authored: Wed May 25 10:53:53 2016 -0700 Committer: Andrew Or Committed: Wed May 25 10:54:25 2016 -0700 -- mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala| 2 +- .../org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala | 2 +- .../spark/sql/catalyst/plans/logical/basicLogicalOperators.scala | 2 +- .../org/apache/spark/sql/execution/vectorized/ColumnVector.java | 2 +- .../spark/sql/execution/streaming/state/StateStoreSuite.scala| 2 +- sql/hive-thriftserver/if/TCLIService.thrift | 4 ++-- .../java/org/apache/hive/service/ServiceStateChangeListener.java | 2 +- .../java/org/apache/hive/service/cli/operation/SQLOperation.java | 2 +- .../apache/hive/service/cli/session/HiveSessionHookContext.java | 2 +- .../main/scala/org/apache/spark/sql/hive/HiveSessionState.scala | 2 +- .../apache/spark/streaming/util/WriteAheadLogRecordHandle.java | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6fc367e5/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala index 9457c6e..bb4b37e 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala @@ -204,7 +204,7 @@ private object IDFModel { * Transforms a term frequency (TF) vector to a TF-IDF vector with a IDF vector * * @param idf an IDF vector - * @param v a term frequence vector + * @param v a term frequency vector * @return a TF-IDF vector */ def transform(idf: Vector, v: Vector): Vector = { http://git-wip-us.apache.org/repos/asf/spark/blob/6fc367e5/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala index 9748fbf..c3de5d7 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala @@ -45,7 +45,7 @@ import org.apache.spark.rdd.RDD * many elements are in each partition. Once these three values have been returned for every * partition, we can collect and operate locally. Locally, we can now adjust each distance by the * appropriate constant (the cumulative sum of number of elements in the prior partitions divided by - * thedata set size). Finally, we take the maximum absolute value, and this is the statistic. + * the data set size). Finally, we take the maximum absolute value, and this is the statistic. */ private[stat] object KolmogorovSmirnovTest extends Logging { http://git-wip-us.apache.org/repos/asf/spark/blob/6fc367e5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index 0a9250b..8b7e21b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -591,7 +591,7 @@ case class Expand( } // This operator can reuse attributes (for example making them null when doing a roll up) so - // the contraints of the child may no longer be valid. + // the constraints of the child may no longer be valid. override protected def validConstraints: Set[Expression] = Set.empty[Expression] } http://git-wip-us.apache.org/repos/asf/spark/blob
spark git commit: [MINOR][MLLIB][STREAMING][SQL] Fix typos
Repository: spark Updated Branches: refs/heads/master d6d3e5071 -> 02c8072ee [MINOR][MLLIB][STREAMING][SQL] Fix typos fixed typos for source code for components [mllib] [streaming] and [SQL] None and obvious. Author: lfzCarlosC Closes #13298 from lfzCarlosC/master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/02c8072e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/02c8072e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/02c8072e Branch: refs/heads/master Commit: 02c8072eea72425e89256347e1f373a3e76e6eba Parents: d6d3e50 Author: lfzCarlosC Authored: Wed May 25 10:53:53 2016 -0700 Committer: Andrew Or Committed: Wed May 25 10:53:57 2016 -0700 -- mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala| 2 +- .../org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala | 2 +- .../spark/sql/catalyst/plans/logical/basicLogicalOperators.scala | 2 +- .../org/apache/spark/sql/execution/vectorized/ColumnVector.java | 2 +- .../spark/sql/execution/streaming/state/StateStoreSuite.scala| 2 +- sql/hive-thriftserver/if/TCLIService.thrift | 4 ++-- .../java/org/apache/hive/service/ServiceStateChangeListener.java | 2 +- .../java/org/apache/hive/service/cli/operation/SQLOperation.java | 2 +- .../apache/hive/service/cli/session/HiveSessionHookContext.java | 2 +- .../main/scala/org/apache/spark/sql/hive/HiveSessionState.scala | 2 +- .../apache/spark/streaming/util/WriteAheadLogRecordHandle.java | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/02c8072e/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala index 9457c6e..bb4b37e 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/IDF.scala @@ -204,7 +204,7 @@ private object IDFModel { * Transforms a term frequency (TF) vector to a TF-IDF vector with a IDF vector * * @param idf an IDF vector - * @param v a term frequence vector + * @param v a term frequency vector * @return a TF-IDF vector */ def transform(idf: Vector, v: Vector): Vector = { http://git-wip-us.apache.org/repos/asf/spark/blob/02c8072e/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala index 9748fbf..c3de5d7 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala @@ -45,7 +45,7 @@ import org.apache.spark.rdd.RDD * many elements are in each partition. Once these three values have been returned for every * partition, we can collect and operate locally. Locally, we can now adjust each distance by the * appropriate constant (the cumulative sum of number of elements in the prior partitions divided by - * thedata set size). Finally, we take the maximum absolute value, and this is the statistic. + * the data set size). Finally, we take the maximum absolute value, and this is the statistic. */ private[stat] object KolmogorovSmirnovTest extends Logging { http://git-wip-us.apache.org/repos/asf/spark/blob/02c8072e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala -- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index 0a9250b..8b7e21b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -591,7 +591,7 @@ case class Expand( } // This operator can reuse attributes (for example making them null when doing a roll up) so - // the contraints of the child may no longer be valid. + // the constraints of the child may no longer be valid. override protected def validConstraints: Set[Expression] = Set.empty[Expression] } http://git-wip-us.apache.org/repos/asf/spark/blob/02c8072e/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVector.java ---
spark git commit: [SPARK-15500][DOC][ML][PYSPARK] Remove default value in Param doc field in ALS
Repository: spark Updated Branches: refs/heads/master 02c8072ee -> 1cb347fbc [SPARK-15500][DOC][ML][PYSPARK] Remove default value in Param doc field in ALS Remove "Default: MEMORY_AND_DISK" from `Param` doc field in ALS storage level params. This fixes up the output of `explainParam(s)` so that default values are not displayed twice. We can revisit in the case that [SPARK-15130](https://issues.apache.org/jira/browse/SPARK-15130) moves ahead with adding defaults in some way to PySpark param doc fields. Tests N/A. Author: Nick Pentreath Closes #13277 from MLnick/SPARK-15500-als-remove-default-storage-param. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1cb347fb Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1cb347fb Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1cb347fb Branch: refs/heads/master Commit: 1cb347fbc446092b478ae0578fc7d1b0626a9294 Parents: 02c8072 Author: Nick Pentreath Authored: Wed May 25 20:41:53 2016 +0200 Committer: Nick Pentreath Committed: Wed May 25 20:41:53 2016 +0200 -- .../main/scala/org/apache/spark/ml/recommendation/ALS.scala| 4 ++-- python/pyspark/ml/recommendation.py| 6 ++ 2 files changed, 4 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1cb347fb/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index f257382..8dc7437 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -180,7 +180,7 @@ private[recommendation] trait ALSParams extends ALSModelParams with HasMaxIter w * @group expertParam */ val intermediateStorageLevel = new Param[String](this, "intermediateStorageLevel", -"StorageLevel for intermediate datasets. Cannot be 'NONE'. Default: 'MEMORY_AND_DISK'.", +"StorageLevel for intermediate datasets. Cannot be 'NONE'.", (s: String) => Try(StorageLevel.fromString(s)).isSuccess && s != "NONE") /** @group expertGetParam */ @@ -194,7 +194,7 @@ private[recommendation] trait ALSParams extends ALSModelParams with HasMaxIter w * @group expertParam */ val finalStorageLevel = new Param[String](this, "finalStorageLevel", -"StorageLevel for ALS model factors. Default: 'MEMORY_AND_DISK'.", +"StorageLevel for ALS model factors.", (s: String) => Try(StorageLevel.fromString(s)).isSuccess) /** @group expertGetParam */ http://git-wip-us.apache.org/repos/asf/spark/blob/1cb347fb/python/pyspark/ml/recommendation.py -- diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py index bac2a30..1778bfe 100644 --- a/python/pyspark/ml/recommendation.py +++ b/python/pyspark/ml/recommendation.py @@ -120,12 +120,10 @@ class ALS(JavaEstimator, HasCheckpointInterval, HasMaxIter, HasPredictionCol, Ha "whether to use nonnegative constraint for least squares", typeConverter=TypeConverters.toBoolean) intermediateStorageLevel = Param(Params._dummy(), "intermediateStorageLevel", - "StorageLevel for intermediate datasets. Cannot be 'NONE'. " + - "Default: 'MEMORY_AND_DISK'.", + "StorageLevel for intermediate datasets. Cannot be 'NONE'.", typeConverter=TypeConverters.toString) finalStorageLevel = Param(Params._dummy(), "finalStorageLevel", - "StorageLevel for ALS model factors. " + - "Default: 'MEMORY_AND_DISK'.", + "StorageLevel for ALS model factors.", typeConverter=TypeConverters.toString) @keyword_only - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15500][DOC][ML][PYSPARK] Remove default value in Param doc field in ALS
Repository: spark Updated Branches: refs/heads/branch-2.0 6fc367e50 -> 409eb28f7 [SPARK-15500][DOC][ML][PYSPARK] Remove default value in Param doc field in ALS Remove "Default: MEMORY_AND_DISK" from `Param` doc field in ALS storage level params. This fixes up the output of `explainParam(s)` so that default values are not displayed twice. We can revisit in the case that [SPARK-15130](https://issues.apache.org/jira/browse/SPARK-15130) moves ahead with adding defaults in some way to PySpark param doc fields. Tests N/A. Author: Nick Pentreath Closes #13277 from MLnick/SPARK-15500-als-remove-default-storage-param. (cherry picked from commit 1cb347fbc446092b478ae0578fc7d1b0626a9294) Signed-off-by: Nick Pentreath Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/409eb28f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/409eb28f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/409eb28f Branch: refs/heads/branch-2.0 Commit: 409eb28f7219778d2aba36079c20c84c8d31b604 Parents: 6fc367e Author: Nick Pentreath Authored: Wed May 25 20:41:53 2016 +0200 Committer: Nick Pentreath Committed: Wed May 25 20:42:11 2016 +0200 -- .../main/scala/org/apache/spark/ml/recommendation/ALS.scala| 4 ++-- python/pyspark/ml/recommendation.py| 6 ++ 2 files changed, 4 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/409eb28f/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index f257382..8dc7437 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -180,7 +180,7 @@ private[recommendation] trait ALSParams extends ALSModelParams with HasMaxIter w * @group expertParam */ val intermediateStorageLevel = new Param[String](this, "intermediateStorageLevel", -"StorageLevel for intermediate datasets. Cannot be 'NONE'. Default: 'MEMORY_AND_DISK'.", +"StorageLevel for intermediate datasets. Cannot be 'NONE'.", (s: String) => Try(StorageLevel.fromString(s)).isSuccess && s != "NONE") /** @group expertGetParam */ @@ -194,7 +194,7 @@ private[recommendation] trait ALSParams extends ALSModelParams with HasMaxIter w * @group expertParam */ val finalStorageLevel = new Param[String](this, "finalStorageLevel", -"StorageLevel for ALS model factors. Default: 'MEMORY_AND_DISK'.", +"StorageLevel for ALS model factors.", (s: String) => Try(StorageLevel.fromString(s)).isSuccess) /** @group expertGetParam */ http://git-wip-us.apache.org/repos/asf/spark/blob/409eb28f/python/pyspark/ml/recommendation.py -- diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py index bac2a30..1778bfe 100644 --- a/python/pyspark/ml/recommendation.py +++ b/python/pyspark/ml/recommendation.py @@ -120,12 +120,10 @@ class ALS(JavaEstimator, HasCheckpointInterval, HasMaxIter, HasPredictionCol, Ha "whether to use nonnegative constraint for least squares", typeConverter=TypeConverters.toBoolean) intermediateStorageLevel = Param(Params._dummy(), "intermediateStorageLevel", - "StorageLevel for intermediate datasets. Cannot be 'NONE'. " + - "Default: 'MEMORY_AND_DISK'.", + "StorageLevel for intermediate datasets. Cannot be 'NONE'.", typeConverter=TypeConverters.toString) finalStorageLevel = Param(Params._dummy(), "finalStorageLevel", - "StorageLevel for ALS model factors. " + - "Default: 'MEMORY_AND_DISK'.", + "StorageLevel for ALS model factors.", typeConverter=TypeConverters.toString) @keyword_only - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15483][SQL] IncrementalExecution should use extra strategies.
Repository: spark Updated Branches: refs/heads/master 1cb347fbc -> 4b8806741 [SPARK-15483][SQL] IncrementalExecution should use extra strategies. ## What changes were proposed in this pull request? Extra strategies does not work for streams because `IncrementalExecution` uses modified planner with stateful operations but it does not include extra strategies. This pr fixes `IncrementalExecution` to include extra strategies to use them. ## How was this patch tested? I added a test to check if extra strategies work for streams. Author: Takuya UESHIN Closes #13261 from ueshin/issues/SPARK-15483. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4b880674 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4b880674 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4b880674 Branch: refs/heads/master Commit: 4b88067416ce922ae15a1445cf953fb9b5c43427 Parents: 1cb347f Author: Takuya UESHIN Authored: Wed May 25 12:02:07 2016 -0700 Committer: Michael Armbrust Committed: Wed May 25 12:02:07 2016 -0700 -- .../execution/streaming/IncrementalExecution.scala | 3 ++- .../org/apache/spark/sql/streaming/StreamSuite.scala | 15 +++ 2 files changed, 17 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/4b880674/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala index 8b96f65..fe5f36e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala @@ -36,7 +36,8 @@ class IncrementalExecution private[sql]( extends QueryExecution(sparkSession, logicalPlan) { // TODO: make this always part of planning. - val stateStrategy = sparkSession.sessionState.planner.StatefulAggregationStrategy :: Nil + val stateStrategy = sparkSession.sessionState.planner.StatefulAggregationStrategy +: +sparkSession.sessionState.experimentalMethods.extraStrategies // Modified planner with stateful operations. override def planner: SparkPlanner = http://git-wip-us.apache.org/repos/asf/spark/blob/4b880674/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala index b742206..ae89a68 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala @@ -220,6 +220,21 @@ class StreamSuite extends StreamTest with SharedSQLContext { CheckOffsetLogLatestBatchId(2), CheckSinkLatestBatchId(2)) } + + test("insert an extraStrategy") { +try { + spark.experimental.extraStrategies = TestStrategy :: Nil + + val inputData = MemoryStream[(String, Int)] + val df = inputData.toDS().map(_._1).toDF("a") + + testStream(df)( +AddData(inputData, ("so slow", 1)), +CheckAnswer("so fast")) +} finally { + spark.experimental.extraStrategies = Nil +} + } } /** - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15483][SQL] IncrementalExecution should use extra strategies.
Repository: spark Updated Branches: refs/heads/branch-2.0 409eb28f7 -> 20cc2eb1b [SPARK-15483][SQL] IncrementalExecution should use extra strategies. ## What changes were proposed in this pull request? Extra strategies does not work for streams because `IncrementalExecution` uses modified planner with stateful operations but it does not include extra strategies. This pr fixes `IncrementalExecution` to include extra strategies to use them. ## How was this patch tested? I added a test to check if extra strategies work for streams. Author: Takuya UESHIN Closes #13261 from ueshin/issues/SPARK-15483. (cherry picked from commit 4b88067416ce922ae15a1445cf953fb9b5c43427) Signed-off-by: Michael Armbrust Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/20cc2eb1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/20cc2eb1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/20cc2eb1 Branch: refs/heads/branch-2.0 Commit: 20cc2eb1b8bf9fbeda9d00a6d02da8f624973742 Parents: 409eb28 Author: Takuya UESHIN Authored: Wed May 25 12:02:07 2016 -0700 Committer: Michael Armbrust Committed: Wed May 25 12:02:16 2016 -0700 -- .../execution/streaming/IncrementalExecution.scala | 3 ++- .../org/apache/spark/sql/streaming/StreamSuite.scala | 15 +++ 2 files changed, 17 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/20cc2eb1/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala index 8b96f65..fe5f36e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala @@ -36,7 +36,8 @@ class IncrementalExecution private[sql]( extends QueryExecution(sparkSession, logicalPlan) { // TODO: make this always part of planning. - val stateStrategy = sparkSession.sessionState.planner.StatefulAggregationStrategy :: Nil + val stateStrategy = sparkSession.sessionState.planner.StatefulAggregationStrategy +: +sparkSession.sessionState.experimentalMethods.extraStrategies // Modified planner with stateful operations. override def planner: SparkPlanner = http://git-wip-us.apache.org/repos/asf/spark/blob/20cc2eb1/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala index b742206..ae89a68 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala @@ -220,6 +220,21 @@ class StreamSuite extends StreamTest with SharedSQLContext { CheckOffsetLogLatestBatchId(2), CheckSinkLatestBatchId(2)) } + + test("insert an extraStrategy") { +try { + spark.experimental.extraStrategies = TestStrategy :: Nil + + val inputData = MemoryStream[(String, Int)] + val df = inputData.toDS().map(_._1).toDF("a") + + testStream(df)( +AddData(inputData, ("so slow", 1)), +CheckAnswer("so fast")) +} finally { + spark.experimental.extraStrategies = Nil +} + } } /** - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15493][SQL] default QuoteEscapingEnabled flag to true when writing CSV
Repository: spark Updated Branches: refs/heads/master 4b8806741 -> c875d81a3 [SPARK-15493][SQL] default QuoteEscapingEnabled flag to true when writing CSV ## What changes were proposed in this pull request? Default QuoteEscapingEnabled flag to true when writing CSV and add an escapeQuotes option to be able to change this. See https://github.com/uniVocity/univocity-parsers/blob/f3eb2af26374940e60d91d1703bde54619f50c51/src/main/java/com/univocity/parsers/csv/CsvWriterSettings.java#L231-L247 This change is needed to be able to write RFC 4180 compatible CSV files (https://tools.ietf.org/html/rfc4180#section-2) https://issues.apache.org/jira/browse/SPARK-15493 ## How was this patch tested? Added a test that verifies the output is quoted correctly. Author: Jurriaan Pruis Closes #13267 from jurriaan/quote-escaping. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c875d81a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c875d81a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c875d81a Branch: refs/heads/master Commit: c875d81a3de3f209b9eb03adf96b7c740b2c7b52 Parents: 4b88067 Author: Jurriaan Pruis Authored: Wed May 25 12:40:16 2016 -0700 Committer: Reynold Xin Committed: Wed May 25 12:40:16 2016 -0700 -- dev/deps/spark-deps-hadoop-2.2 | 2 +- dev/deps/spark-deps-hadoop-2.3 | 2 +- dev/deps/spark-deps-hadoop-2.4 | 2 +- dev/deps/spark-deps-hadoop-2.6 | 2 +- dev/deps/spark-deps-hadoop-2.7 | 2 +- python/pyspark/sql/readwriter.py| 7 ++- sql/core/pom.xml| 2 +- .../org/apache/spark/sql/DataFrameWriter.scala | 3 ++ .../execution/datasources/csv/CSVOptions.scala | 2 + .../execution/datasources/csv/CSVParser.scala | 1 + .../execution/datasources/csv/CSVSuite.scala| 51 11 files changed, 69 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c875d81a/dev/deps/spark-deps-hadoop-2.2 -- diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index a9068da..0ac1c00 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -159,7 +159,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar validation-api-1.1.0.Final.jar xbean-asm5-shaded-4.4.jar xmlenc-0.52.jar http://git-wip-us.apache.org/repos/asf/spark/blob/c875d81a/dev/deps/spark-deps-hadoop-2.3 -- diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index 7e60a31..fa35fa7 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -167,7 +167,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar validation-api-1.1.0.Final.jar xbean-asm5-shaded-4.4.jar xmlenc-0.52.jar http://git-wip-us.apache.org/repos/asf/spark/blob/c875d81a/dev/deps/spark-deps-hadoop-2.4 -- diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index 70d33b4..99dffa9 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -167,7 +167,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar validation-api-1.1.0.Final.jar xbean-asm5-shaded-4.4.jar xmlenc-0.52.jar http://git-wip-us.apache.org/repos/asf/spark/blob/c875d81a/dev/deps/spark-deps-hadoop-2.6 -- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index a80f6bc..a3bee36 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -175,7 +175,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar validation-api-1.1.0.Final.jar xbean-asm5-shaded-4.4.jar xercesImpl-2.9.1.jar http://git-wip-us.apache.org/repos/asf/spark/blob/c875d81a/dev/deps/spark-deps-hadoop-2.7 -- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index c0b53f7..dbd7a8e 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -176,7 +176,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar val
spark git commit: [SPARK-15493][SQL] default QuoteEscapingEnabled flag to true when writing CSV
Repository: spark Updated Branches: refs/heads/branch-2.0 20cc2eb1b -> 8629537cc [SPARK-15493][SQL] default QuoteEscapingEnabled flag to true when writing CSV ## What changes were proposed in this pull request? Default QuoteEscapingEnabled flag to true when writing CSV and add an escapeQuotes option to be able to change this. See https://github.com/uniVocity/univocity-parsers/blob/f3eb2af26374940e60d91d1703bde54619f50c51/src/main/java/com/univocity/parsers/csv/CsvWriterSettings.java#L231-L247 This change is needed to be able to write RFC 4180 compatible CSV files (https://tools.ietf.org/html/rfc4180#section-2) https://issues.apache.org/jira/browse/SPARK-15493 ## How was this patch tested? Added a test that verifies the output is quoted correctly. Author: Jurriaan Pruis Closes #13267 from jurriaan/quote-escaping. (cherry picked from commit c875d81a3de3f209b9eb03adf96b7c740b2c7b52) Signed-off-by: Reynold Xin Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8629537c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8629537c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8629537c Branch: refs/heads/branch-2.0 Commit: 8629537cc7d89c97ef2038326f1138d166596315 Parents: 20cc2eb Author: Jurriaan Pruis Authored: Wed May 25 12:40:16 2016 -0700 Committer: Reynold Xin Committed: Wed May 25 12:40:26 2016 -0700 -- dev/deps/spark-deps-hadoop-2.2 | 2 +- dev/deps/spark-deps-hadoop-2.3 | 2 +- dev/deps/spark-deps-hadoop-2.4 | 2 +- dev/deps/spark-deps-hadoop-2.6 | 2 +- dev/deps/spark-deps-hadoop-2.7 | 2 +- python/pyspark/sql/readwriter.py| 7 ++- sql/core/pom.xml| 2 +- .../org/apache/spark/sql/DataFrameWriter.scala | 3 ++ .../execution/datasources/csv/CSVOptions.scala | 2 + .../execution/datasources/csv/CSVParser.scala | 1 + .../execution/datasources/csv/CSVSuite.scala| 51 11 files changed, 69 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/8629537c/dev/deps/spark-deps-hadoop-2.2 -- diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index a9068da..0ac1c00 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -159,7 +159,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar validation-api-1.1.0.Final.jar xbean-asm5-shaded-4.4.jar xmlenc-0.52.jar http://git-wip-us.apache.org/repos/asf/spark/blob/8629537c/dev/deps/spark-deps-hadoop-2.3 -- diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index 7e60a31..fa35fa7 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -167,7 +167,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar validation-api-1.1.0.Final.jar xbean-asm5-shaded-4.4.jar xmlenc-0.52.jar http://git-wip-us.apache.org/repos/asf/spark/blob/8629537c/dev/deps/spark-deps-hadoop-2.4 -- diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index 70d33b4..99dffa9 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -167,7 +167,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar validation-api-1.1.0.Final.jar xbean-asm5-shaded-4.4.jar xmlenc-0.52.jar http://git-wip-us.apache.org/repos/asf/spark/blob/8629537c/dev/deps/spark-deps-hadoop-2.6 -- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index a80f6bc..a3bee36 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -175,7 +175,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar stringtemplate-3.2.1.jar super-csv-2.2.0.jar -univocity-parsers-2.1.0.jar +univocity-parsers-2.1.1.jar validation-api-1.1.0.Final.jar xbean-asm5-shaded-4.4.jar xercesImpl-2.9.1.jar http://git-wip-us.apache.org/repos/asf/spark/blob/8629537c/dev/deps/spark-deps-hadoop-2.7 -- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index c0b53f7..dbd7a8e 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -176,7 +176,7 @@ stax-api-1.0.1.jar stream-2.7.0.jar st
spark git commit: [SPARK-14269][SCHEDULER] Eliminate unnecessary submitStage() call.
Repository: spark Updated Branches: refs/heads/master c875d81a3 -> 698ef762f [SPARK-14269][SCHEDULER] Eliminate unnecessary submitStage() call. ## What changes were proposed in this pull request? Currently a method `submitStage()` for waiting stages is called on every iteration of the event loop in `DAGScheduler` to submit all waiting stages, but most of them are not necessary because they are not related to Stage status. The case we should try to submit waiting stages is only when their parent stages are successfully completed. This elimination can improve `DAGScheduler` performance. ## How was this patch tested? Added some checks and other existing tests, and our projects. We have a project bottle-necked by `DAGScheduler`, having about 2000 stages. Before this patch the almost all execution time in `Driver` process was spent to process `submitStage()` of `dag-scheduler-event-loop` thread but after this patch the performance was improved as follows: || total execution time | `dag-scheduler-event-loop` thread time | `submitStage()` | ||-:|---:|:| | Before | 760 sec |710 sec | 667 sec | | After | 440 sec | 14 sec | 10 sec | Author: Takuya UESHIN Closes #12060 from ueshin/issues/SPARK-14269. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/698ef762 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/698ef762 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/698ef762 Branch: refs/heads/master Commit: 698ef762f80cf4c84bc7b7cf083aa97d44b87170 Parents: c875d81 Author: Takuya UESHIN Authored: Wed May 25 13:57:25 2016 -0700 Committer: Kay Ousterhout Committed: Wed May 25 13:57:25 2016 -0700 -- .../apache/spark/scheduler/DAGScheduler.scala | 35 ++-- .../spark/scheduler/DAGSchedulerSuite.scala | 7 2 files changed, 17 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/698ef762/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 5291b66..766e979 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -726,7 +726,6 @@ class DAGScheduler( reason = "as part of cancellation of all jobs")) activeJobs.clear() // These should already be empty by this point, jobIdToActiveJob.clear() // but just in case we lost track of some jobs... -submitWaitingStages() } /** @@ -752,23 +751,21 @@ class DAGScheduler( submitStage(stage) } } -submitWaitingStages() } /** * Check for waiting stages which are now eligible for resubmission. - * Ordinarily run on every iteration of the event loop. + * Submits stages that depend on the given parent stage. Called when the parent stage completes + * successfully. */ - private def submitWaitingStages() { -// TODO: We might want to run this less often, when we are sure that something has become -// runnable that wasn't before. -logTrace("Checking for newly runnable parent stages") + private def submitWaitingChildStages(parent: Stage) { +logTrace(s"Checking if any dependencies of $parent are now runnable") logTrace("running: " + runningStages) logTrace("waiting: " + waitingStages) logTrace("failed: " + failedStages) -val waitingStagesCopy = waitingStages.toArray -waitingStages.clear() -for (stage <- waitingStagesCopy.sortBy(_.firstJobId)) { +val childStages = waitingStages.filter(_.parents.contains(parent)).toArray +waitingStages --= childStages +for (stage <- childStages.sortBy(_.firstJobId)) { submitStage(stage) } } @@ -793,7 +790,6 @@ class DAGScheduler( } val jobIds = activeInGroup.map(_.jobId) jobIds.foreach(handleJobCancellation(_, "part of cancelled job group %s".format(groupId))) -submitWaitingStages() } private[scheduler] def handleBeginEvent(task: Task[_], taskInfo: TaskInfo) { @@ -801,7 +797,6 @@ class DAGScheduler( // In that case, we wouldn't have the stage anymore in stageIdToStage. val stageAttemptId = stageIdToStage.get(task.stageId).map(_.latestInfo.attemptId).getOrElse(-1) listenerBus.post(SparkListenerTaskStart(task.stageId, stageAttemptId, taskInfo)) -submitWaitingStages() } private[scheduler] def handleTaskSetFailed( @@ -809,7 +804,6 @@ class DAGSchedul
spark git commit: [MINOR] [PYSPARK] [EXAMPLES] Changed examples to use SparkSession.sparkContext instead of _sc
Repository: spark Updated Branches: refs/heads/master 698ef762f -> 9c297df3d [MINOR] [PYSPARK] [EXAMPLES] Changed examples to use SparkSession.sparkContext instead of _sc ## What changes were proposed in this pull request? Some PySpark examples need a SparkContext and get it by accessing _sc directly from the session. These examples should use the provided property `sparkContext` in `SparkSession` instead. ## How was this patch tested? Ran modified examples Author: Bryan Cutler Closes #13303 from BryanCutler/pyspark-session-sparkContext-MINOR. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9c297df3 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9c297df3 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9c297df3 Branch: refs/heads/master Commit: 9c297df3d4d5fa4bbfdffdaad15f362586db384b Parents: 698ef76 Author: Bryan Cutler Authored: Wed May 25 14:29:14 2016 -0700 Committer: Davies Liu Committed: Wed May 25 14:29:14 2016 -0700 -- examples/src/main/python/als.py | 2 +- examples/src/main/python/avro_inputformat.py| 2 +- examples/src/main/python/parquet_inputformat.py | 2 +- examples/src/main/python/pi.py | 2 +- examples/src/main/python/transitive_closure.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/9c297df3/examples/src/main/python/als.py -- diff --git a/examples/src/main/python/als.py b/examples/src/main/python/als.py index 81562e2..80290e7 100755 --- a/examples/src/main/python/als.py +++ b/examples/src/main/python/als.py @@ -67,7 +67,7 @@ if __name__ == "__main__": .appName("PythonALS")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext M = int(sys.argv[1]) if len(sys.argv) > 1 else 100 U = int(sys.argv[2]) if len(sys.argv) > 2 else 500 http://git-wip-us.apache.org/repos/asf/spark/blob/9c297df3/examples/src/main/python/avro_inputformat.py -- diff --git a/examples/src/main/python/avro_inputformat.py b/examples/src/main/python/avro_inputformat.py index 3f65e8f..4422f9e 100644 --- a/examples/src/main/python/avro_inputformat.py +++ b/examples/src/main/python/avro_inputformat.py @@ -70,7 +70,7 @@ if __name__ == "__main__": .appName("AvroKeyInputFormat")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext conf = None if len(sys.argv) == 3: http://git-wip-us.apache.org/repos/asf/spark/blob/9c297df3/examples/src/main/python/parquet_inputformat.py -- diff --git a/examples/src/main/python/parquet_inputformat.py b/examples/src/main/python/parquet_inputformat.py index 2f09f4d..29a1ac2 100644 --- a/examples/src/main/python/parquet_inputformat.py +++ b/examples/src/main/python/parquet_inputformat.py @@ -53,7 +53,7 @@ if __name__ == "__main__": .appName("ParquetInputFormat")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext parquet_rdd = sc.newAPIHadoopFile( path, http://git-wip-us.apache.org/repos/asf/spark/blob/9c297df3/examples/src/main/python/pi.py -- diff --git a/examples/src/main/python/pi.py b/examples/src/main/python/pi.py index 5db03e4..b39d710 100755 --- a/examples/src/main/python/pi.py +++ b/examples/src/main/python/pi.py @@ -32,7 +32,7 @@ if __name__ == "__main__": .appName("PythonPi")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2 n = 10 * partitions http://git-wip-us.apache.org/repos/asf/spark/blob/9c297df3/examples/src/main/python/transitive_closure.py -- diff --git a/examples/src/main/python/transitive_closure.py b/examples/src/main/python/transitive_closure.py index 37c41dc..d88ea94 100755 --- a/examples/src/main/python/transitive_closure.py +++ b/examples/src/main/python/transitive_closure.py @@ -46,7 +46,7 @@ if __name__ == "__main__": .appName("PythonTransitiveClosure")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2 tc = sc.parallelize(generateGraph(), partitions).cache() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [MINOR] [PYSPARK] [EXAMPLES] Changed examples to use SparkSession.sparkContext instead of _sc
Repository: spark Updated Branches: refs/heads/branch-2.0 8629537cc -> f7158c482 [MINOR] [PYSPARK] [EXAMPLES] Changed examples to use SparkSession.sparkContext instead of _sc ## What changes were proposed in this pull request? Some PySpark examples need a SparkContext and get it by accessing _sc directly from the session. These examples should use the provided property `sparkContext` in `SparkSession` instead. ## How was this patch tested? Ran modified examples Author: Bryan Cutler Closes #13303 from BryanCutler/pyspark-session-sparkContext-MINOR. (cherry picked from commit 9c297df3d4d5fa4bbfdffdaad15f362586db384b) Signed-off-by: Davies Liu Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f7158c48 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f7158c48 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f7158c48 Branch: refs/heads/branch-2.0 Commit: f7158c4828320e18aedd4369832da16c759b90fb Parents: 8629537 Author: Bryan Cutler Authored: Wed May 25 14:29:14 2016 -0700 Committer: Davies Liu Committed: Wed May 25 14:29:23 2016 -0700 -- examples/src/main/python/als.py | 2 +- examples/src/main/python/avro_inputformat.py| 2 +- examples/src/main/python/parquet_inputformat.py | 2 +- examples/src/main/python/pi.py | 2 +- examples/src/main/python/transitive_closure.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f7158c48/examples/src/main/python/als.py -- diff --git a/examples/src/main/python/als.py b/examples/src/main/python/als.py index 81562e2..80290e7 100755 --- a/examples/src/main/python/als.py +++ b/examples/src/main/python/als.py @@ -67,7 +67,7 @@ if __name__ == "__main__": .appName("PythonALS")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext M = int(sys.argv[1]) if len(sys.argv) > 1 else 100 U = int(sys.argv[2]) if len(sys.argv) > 2 else 500 http://git-wip-us.apache.org/repos/asf/spark/blob/f7158c48/examples/src/main/python/avro_inputformat.py -- diff --git a/examples/src/main/python/avro_inputformat.py b/examples/src/main/python/avro_inputformat.py index 3f65e8f..4422f9e 100644 --- a/examples/src/main/python/avro_inputformat.py +++ b/examples/src/main/python/avro_inputformat.py @@ -70,7 +70,7 @@ if __name__ == "__main__": .appName("AvroKeyInputFormat")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext conf = None if len(sys.argv) == 3: http://git-wip-us.apache.org/repos/asf/spark/blob/f7158c48/examples/src/main/python/parquet_inputformat.py -- diff --git a/examples/src/main/python/parquet_inputformat.py b/examples/src/main/python/parquet_inputformat.py index 2f09f4d..29a1ac2 100644 --- a/examples/src/main/python/parquet_inputformat.py +++ b/examples/src/main/python/parquet_inputformat.py @@ -53,7 +53,7 @@ if __name__ == "__main__": .appName("ParquetInputFormat")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext parquet_rdd = sc.newAPIHadoopFile( path, http://git-wip-us.apache.org/repos/asf/spark/blob/f7158c48/examples/src/main/python/pi.py -- diff --git a/examples/src/main/python/pi.py b/examples/src/main/python/pi.py index 5db03e4..b39d710 100755 --- a/examples/src/main/python/pi.py +++ b/examples/src/main/python/pi.py @@ -32,7 +32,7 @@ if __name__ == "__main__": .appName("PythonPi")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2 n = 10 * partitions http://git-wip-us.apache.org/repos/asf/spark/blob/f7158c48/examples/src/main/python/transitive_closure.py -- diff --git a/examples/src/main/python/transitive_closure.py b/examples/src/main/python/transitive_closure.py index 37c41dc..d88ea94 100755 --- a/examples/src/main/python/transitive_closure.py +++ b/examples/src/main/python/transitive_closure.py @@ -46,7 +46,7 @@ if __name__ == "__main__": .appName("PythonTransitiveClosure")\ .getOrCreate() -sc = spark._sc +sc = spark.sparkContext partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2 tc = sc.parallelize(generateGraph(), partitions).cache() - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.ap
spark git commit: Log warnings for numIterations * miniBatchFraction < 1.0
Repository: spark Updated Branches: refs/heads/master 9c297df3d -> 589cce93c Log warnings for numIterations * miniBatchFraction < 1.0 ## What changes were proposed in this pull request? Add a warning log for the case that `numIterations * miniBatchFraction <1.0` during gradient descent. If the product of those two numbers is less than `1.0`, then not all training examples will be used during optimization. To put this concretely, suppose that `numExamples = 100`, `miniBatchFraction = 0.2` and `numIterations = 3`. Then, 3 iterations will occur each sampling approximately 6 examples each. In the best case, each of the 6 examples are unique; hence 18/100 examples are used. This may be counter-intuitive to most users and led to the issue during the development of another Spark ML model: https://github.com/zhengruifeng/spark-libFM/issues/11. If a user actually does not require the training data set, it would be easier and more intuitive to use `RDD.sample`. ## How was this patch tested? `build/mvn -DskipTests clean package` build succeeds Author: Gio Borje Closes #13265 from Hydrotoast/master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/589cce93 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/589cce93 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/589cce93 Branch: refs/heads/master Commit: 589cce93c821ac28e9090a478f6e7465398b7c30 Parents: 9c297df Author: Gio Borje Authored: Wed May 25 16:52:31 2016 -0500 Committer: Sean Owen Committed: Wed May 25 16:52:31 2016 -0500 -- .../org/apache/spark/mllib/optimization/GradientDescent.scala | 5 + 1 file changed, 5 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/589cce93/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala index a67ea83..735e780 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala @@ -197,6 +197,11 @@ object GradientDescent extends Logging { "< 1.0 can be unstable because of the stochasticity in sampling.") } +if (numIterations * miniBatchFraction < 1.0) { + logWarning("Not all examples will be used if numIterations * miniBatchFraction < 1.0: " + +s"numIterations=$numIterations and miniBatchFraction=$miniBatchFraction") +} + val stochasticLossHistory = new ArrayBuffer[Double](numIterations) // Record previous weight and current one to calculate solution vector difference - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: Log warnings for numIterations * miniBatchFraction < 1.0
Repository: spark Updated Branches: refs/heads/branch-2.0 f7158c482 -> 0064a4dcb Log warnings for numIterations * miniBatchFraction < 1.0 ## What changes were proposed in this pull request? Add a warning log for the case that `numIterations * miniBatchFraction <1.0` during gradient descent. If the product of those two numbers is less than `1.0`, then not all training examples will be used during optimization. To put this concretely, suppose that `numExamples = 100`, `miniBatchFraction = 0.2` and `numIterations = 3`. Then, 3 iterations will occur each sampling approximately 6 examples each. In the best case, each of the 6 examples are unique; hence 18/100 examples are used. This may be counter-intuitive to most users and led to the issue during the development of another Spark ML model: https://github.com/zhengruifeng/spark-libFM/issues/11. If a user actually does not require the training data set, it would be easier and more intuitive to use `RDD.sample`. ## How was this patch tested? `build/mvn -DskipTests clean package` build succeeds Author: Gio Borje Closes #13265 from Hydrotoast/master. (cherry picked from commit 589cce93c821ac28e9090a478f6e7465398b7c30) Signed-off-by: Sean Owen Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0064a4dc Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0064a4dc Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0064a4dc Branch: refs/heads/branch-2.0 Commit: 0064a4dcbed1d91732a29c2cede464b8d148aeca Parents: f7158c4 Author: Gio Borje Authored: Wed May 25 16:52:31 2016 -0500 Committer: Sean Owen Committed: Wed May 25 16:52:48 2016 -0500 -- .../org/apache/spark/mllib/optimization/GradientDescent.scala | 5 + 1 file changed, 5 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/0064a4dc/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala index a67ea83..735e780 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala @@ -197,6 +197,11 @@ object GradientDescent extends Logging { "< 1.0 can be unstable because of the stochasticity in sampling.") } +if (numIterations * miniBatchFraction < 1.0) { + logWarning("Not all examples will be used if numIterations * miniBatchFraction < 1.0: " + +s"numIterations=$numIterations and miniBatchFraction=$miniBatchFraction") +} + val stochasticLossHistory = new ArrayBuffer[Double](numIterations) // Record previous weight and current one to calculate solution vector difference - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15534][SPARK-15535][SQL] Truncate table fixes
Repository: spark Updated Branches: refs/heads/branch-2.0 0064a4dcb -> 347acc4ea [SPARK-15534][SPARK-15535][SQL] Truncate table fixes ## What changes were proposed in this pull request? Two changes: - When things fail, `TRUNCATE TABLE` just returns nothing. Instead, we should throw exceptions. - Remove `TRUNCATE TABLE ... COLUMN`, which was never supported by either Spark or Hive. ## How was this patch tested? Jenkins. Author: Andrew Or Closes #13302 from andrewor14/truncate-table. (cherry picked from commit ee682fe293b47988056b540ee46ca49861309982) Signed-off-by: Andrew Or Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/347acc4e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/347acc4e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/347acc4e Branch: refs/heads/branch-2.0 Commit: 347acc4ea411cd430cfb1d8b8b596463f5b9dd3c Parents: 0064a4d Author: Andrew Or Authored: Wed May 25 15:08:39 2016 -0700 Committer: Andrew Or Committed: Wed May 25 15:08:59 2016 -0700 -- .../org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 3 +-- .../org/apache/spark/sql/execution/SparkSqlParser.scala | 7 +-- .../org/apache/spark/sql/execution/command/tables.scala | 7 --- .../spark/sql/hive/execution/HiveCommandSuite.scala | 12 4 files changed, 6 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/347acc4e/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 -- diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 403191a..b0e71c7 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -115,8 +115,7 @@ statement | CLEAR CACHE #clearCache | LOAD DATA LOCAL? INPATH path=STRING OVERWRITE? INTO TABLE tableIdentifier partitionSpec? #loadData -| TRUNCATE TABLE tableIdentifier partitionSpec? -(COLUMNS identifierList)? #truncateTable +| TRUNCATE TABLE tableIdentifier partitionSpec? #truncateTable | op=(ADD | LIST) identifier .*? #manageResource | SET ROLE .*? #failNativeCommand | SET .*? #setConfiguration http://git-wip-us.apache.org/repos/asf/spark/blob/347acc4e/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 57f534c..cfebfc6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -368,17 +368,12 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { * For example: * {{{ * TRUNCATE TABLE tablename [PARTITION (partcol1=val1, partcol2=val2 ...)] - * [COLUMNS (col1, col2)] * }}} */ override def visitTruncateTable(ctx: TruncateTableContext): LogicalPlan = withOrigin(ctx) { -if (ctx.identifierList != null) { - throw operationNotAllowed("TRUNCATE TABLE ... COLUMNS", ctx) -} TruncateTableCommand( visitTableIdentifier(ctx.tableIdentifier), - Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec) -) + Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec)) } /** http://git-wip-us.apache.org/repos/asf/spark/blob/347acc4e/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 13e63a1..bef4c92 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -278,7 +278,7 @@ case class LoadDataCommand( * * The syntax of this command is: * {{{ - * TRUNCATE TABLE tablename [PARTITION (partcol1=val1, partcol2=val2 ...)] + * TRUNCATE TABLE tablename [PARTITION (partcol1=val1, partcol2=val2 ...)] * }}} */ case class T
spark git commit: [SPARK-15534][SPARK-15535][SQL] Truncate table fixes
Repository: spark Updated Branches: refs/heads/master 589cce93c -> ee682fe29 [SPARK-15534][SPARK-15535][SQL] Truncate table fixes ## What changes were proposed in this pull request? Two changes: - When things fail, `TRUNCATE TABLE` just returns nothing. Instead, we should throw exceptions. - Remove `TRUNCATE TABLE ... COLUMN`, which was never supported by either Spark or Hive. ## How was this patch tested? Jenkins. Author: Andrew Or Closes #13302 from andrewor14/truncate-table. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ee682fe2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ee682fe2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ee682fe2 Branch: refs/heads/master Commit: ee682fe293b47988056b540ee46ca49861309982 Parents: 589cce9 Author: Andrew Or Authored: Wed May 25 15:08:39 2016 -0700 Committer: Andrew Or Committed: Wed May 25 15:08:39 2016 -0700 -- .../org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 3 +-- .../org/apache/spark/sql/execution/SparkSqlParser.scala | 7 +-- .../org/apache/spark/sql/execution/command/tables.scala | 7 --- .../spark/sql/hive/execution/HiveCommandSuite.scala | 12 4 files changed, 6 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ee682fe2/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 -- diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 403191a..b0e71c7 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -115,8 +115,7 @@ statement | CLEAR CACHE #clearCache | LOAD DATA LOCAL? INPATH path=STRING OVERWRITE? INTO TABLE tableIdentifier partitionSpec? #loadData -| TRUNCATE TABLE tableIdentifier partitionSpec? -(COLUMNS identifierList)? #truncateTable +| TRUNCATE TABLE tableIdentifier partitionSpec? #truncateTable | op=(ADD | LIST) identifier .*? #manageResource | SET ROLE .*? #failNativeCommand | SET .*? #setConfiguration http://git-wip-us.apache.org/repos/asf/spark/blob/ee682fe2/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 57f534c..cfebfc6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -368,17 +368,12 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { * For example: * {{{ * TRUNCATE TABLE tablename [PARTITION (partcol1=val1, partcol2=val2 ...)] - * [COLUMNS (col1, col2)] * }}} */ override def visitTruncateTable(ctx: TruncateTableContext): LogicalPlan = withOrigin(ctx) { -if (ctx.identifierList != null) { - throw operationNotAllowed("TRUNCATE TABLE ... COLUMNS", ctx) -} TruncateTableCommand( visitTableIdentifier(ctx.tableIdentifier), - Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec) -) + Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec)) } /** http://git-wip-us.apache.org/repos/asf/spark/blob/ee682fe2/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 13e63a1..bef4c92 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -278,7 +278,7 @@ case class LoadDataCommand( * * The syntax of this command is: * {{{ - * TRUNCATE TABLE tablename [PARTITION (partcol1=val1, partcol2=val2 ...)] + * TRUNCATE TABLE tablename [PARTITION (partcol1=val1, partcol2=val2 ...)] * }}} */ case class TruncateTableCommand( @@ -288,9 +288,10 @@ case class TruncateTableCommand( override def run(sparkSess
spark git commit: [SPARK-15525][SQL][BUILD] Upgrade ANTLR4 SBT plugin
Repository: spark Updated Branches: refs/heads/master ee682fe29 -> 527499b62 [SPARK-15525][SQL][BUILD] Upgrade ANTLR4 SBT plugin ## What changes were proposed in this pull request? The ANTLR4 SBT plugin has been moved from its own repo to one on bintray. The version was also changed from `0.7.10` to `0.7.11`. The latter actually broke our build (ihji has fixed this by also adding `0.7.10` and others to the bin-tray repo). This PR upgrades the SBT-ANTLR4 plugin and ANTLR4 to their most recent versions (`0.7.11`/`4.5.3`). I have also removed a few obsolete build configurations. ## How was this patch tested? Manually running SBT/Maven builds. Author: Herman van Hovell Closes #13299 from hvanhovell/SPARK-15525. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/527499b6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/527499b6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/527499b6 Branch: refs/heads/master Commit: 527499b624e743583fe0f93ea0b487031891ac3a Parents: ee682fe Author: Herman van Hovell Authored: Wed May 25 15:35:38 2016 -0700 Committer: Reynold Xin Committed: Wed May 25 15:35:38 2016 -0700 -- dev/deps/spark-deps-hadoop-2.2 | 2 +- dev/deps/spark-deps-hadoop-2.3 | 2 +- dev/deps/spark-deps-hadoop-2.4 | 2 +- dev/deps/spark-deps-hadoop-2.6 | 2 +- dev/deps/spark-deps-hadoop-2.7 | 2 +- pom.xml| 7 +-- project/plugins.sbt| 5 + 7 files changed, 7 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/527499b6/dev/deps/spark-deps-hadoop-2.2 -- diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index 0ac1c00..0d6b18e 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -3,7 +3,7 @@ RoaringBitmap-0.5.11.jar ST4-4.0.4.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/527499b6/dev/deps/spark-deps-hadoop-2.3 -- diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index fa35fa7..a3597f4 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -4,7 +4,7 @@ ST4-4.0.4.jar activation-1.1.1.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/527499b6/dev/deps/spark-deps-hadoop-2.4 -- diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index 99dffa9..3ca44c5 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -4,7 +4,7 @@ ST4-4.0.4.jar activation-1.1.1.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/527499b6/dev/deps/spark-deps-hadoop-2.6 -- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index a3bee36..01e7551 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -4,7 +4,7 @@ ST4-4.0.4.jar activation-1.1.1.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/527499b6/dev/deps/spark-deps-hadoop-2.7 -- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index dbd7a8e..402fd05 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -4,7 +4,7 @@ ST4-4.0.4.jar activation-1.1.1.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/527499b6/pom.xml -- diff --git a/pom.xml b/pom.xml index f28aa14..3fa0eeb 100644 --- a/pom.xml +++ b/pom.xml @@ -177,7 +177,7 @@ 3.5.2 1.3.9 0.9.2 -4.5.2-1 +4.5.3 1.1 2.52.0 2.8 @@ -1954,11 +1954,6 @@ org.antlr -
spark git commit: [SPARK-15525][SQL][BUILD] Upgrade ANTLR4 SBT plugin
Repository: spark Updated Branches: refs/heads/branch-2.0 347acc4ea -> 733cb44e3 [SPARK-15525][SQL][BUILD] Upgrade ANTLR4 SBT plugin ## What changes were proposed in this pull request? The ANTLR4 SBT plugin has been moved from its own repo to one on bintray. The version was also changed from `0.7.10` to `0.7.11`. The latter actually broke our build (ihji has fixed this by also adding `0.7.10` and others to the bin-tray repo). This PR upgrades the SBT-ANTLR4 plugin and ANTLR4 to their most recent versions (`0.7.11`/`4.5.3`). I have also removed a few obsolete build configurations. ## How was this patch tested? Manually running SBT/Maven builds. Author: Herman van Hovell Closes #13299 from hvanhovell/SPARK-15525. (cherry picked from commit 527499b624e743583fe0f93ea0b487031891ac3a) Signed-off-by: Reynold Xin Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/733cb44e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/733cb44e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/733cb44e Branch: refs/heads/branch-2.0 Commit: 733cb44e3321744d521f0791e241cc0fe062d01a Parents: 347acc4 Author: Herman van Hovell Authored: Wed May 25 15:35:38 2016 -0700 Committer: Reynold Xin Committed: Wed May 25 15:36:04 2016 -0700 -- dev/deps/spark-deps-hadoop-2.2 | 2 +- dev/deps/spark-deps-hadoop-2.3 | 2 +- dev/deps/spark-deps-hadoop-2.4 | 2 +- dev/deps/spark-deps-hadoop-2.6 | 2 +- dev/deps/spark-deps-hadoop-2.7 | 2 +- pom.xml| 7 +-- project/plugins.sbt| 5 + 7 files changed, 7 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/733cb44e/dev/deps/spark-deps-hadoop-2.2 -- diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index 0ac1c00..0d6b18e 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -3,7 +3,7 @@ RoaringBitmap-0.5.11.jar ST4-4.0.4.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/733cb44e/dev/deps/spark-deps-hadoop-2.3 -- diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index fa35fa7..a3597f4 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -4,7 +4,7 @@ ST4-4.0.4.jar activation-1.1.1.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/733cb44e/dev/deps/spark-deps-hadoop-2.4 -- diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index 99dffa9..3ca44c5 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -4,7 +4,7 @@ ST4-4.0.4.jar activation-1.1.1.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/733cb44e/dev/deps/spark-deps-hadoop-2.6 -- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index a3bee36..01e7551 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -4,7 +4,7 @@ ST4-4.0.4.jar activation-1.1.1.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/733cb44e/dev/deps/spark-deps-hadoop-2.7 -- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index dbd7a8e..402fd05 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -4,7 +4,7 @@ ST4-4.0.4.jar activation-1.1.1.jar antlr-2.7.7.jar antlr-runtime-3.4.jar -antlr4-runtime-4.5.2-1.jar +antlr4-runtime-4.5.3.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar apache-log4j-extras-1.2.17.jar http://git-wip-us.apache.org/repos/asf/spark/blob/733cb44e/pom.xml -- diff --git a/pom.xml b/pom.xml index f28aa14..3fa0eeb 100644 --- a/pom.xml +++ b/pom.xml @@ -177,7 +177,7 @@ 3.5.2 1.3.9 0.9.2 -4.5.2-1 +
spark git commit: [SPARK-15533][SQL] Deprecate Dataset.explode
Repository: spark Updated Branches: refs/heads/master 527499b62 -> 06ed1fa3e [SPARK-15533][SQL] Deprecate Dataset.explode ## What changes were proposed in this pull request? This patch deprecates `Dataset.explode` and documents appropriate workarounds to use `flatMap()` or `functions.explode()` instead. ## How was this patch tested? N/A Author: Sameer Agarwal Closes #13312 from sameeragarwal/deprecate. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/06ed1fa3 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/06ed1fa3 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/06ed1fa3 Branch: refs/heads/master Commit: 06ed1fa3e45adfc11b0f615cb8b97c99fadc735f Parents: 527499b Author: Sameer Agarwal Authored: Wed May 25 19:10:57 2016 -0700 Committer: Reynold Xin Committed: Wed May 25 19:10:57 2016 -0700 -- .../scala/org/apache/spark/sql/Dataset.scala| 33 +--- 1 file changed, 22 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/06ed1fa3/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index 78a167e..e5140fc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -1556,30 +1556,33 @@ class Dataset[T] private[sql]( } /** - * :: Experimental :: * (Scala-specific) Returns a new [[Dataset]] where each row has been expanded to zero or more * rows by the provided function. This is similar to a `LATERAL VIEW` in HiveQL. The columns of * the input row are implicitly joined with each row that is output by the function. * - * The following example uses this function to count the number of books which contain - * a given word: + * Given that this is deprecated, as an alternative, you can explode columns either using + * `functions.explode()` or `flatMap()`. The following example uses these alternatives to count + * the number of books that contain a given word: * * {{{ * case class Book(title: String, words: String) * val ds: Dataset[Book] * - * case class Word(word: String) - * val allWords = ds.explode('words) { - * case Row(words: String) => words.split(" ").map(Word(_)) - * } + * val allWords = ds.select('title, explode(split('words, " ")).as("word")) * * val bookCountPerWord = allWords.groupBy("word").agg(countDistinct("title")) * }}} * + * Using `flatMap()` this can similarly be exploded as: + * + * {{{ + * ds.flatMap(_.words.split(" ")) + * }}} + * * @group untypedrel * @since 2.0.0 */ - @Experimental + @deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0") def explode[A <: Product : TypeTag](input: Column*)(f: Row => TraversableOnce[A]): DataFrame = { val elementSchema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType] @@ -1596,19 +1599,27 @@ class Dataset[T] private[sql]( } /** - * :: Experimental :: * (Scala-specific) Returns a new [[Dataset]] where a single column has been expanded to zero * or more rows by the provided function. This is similar to a `LATERAL VIEW` in HiveQL. All * columns of the input row are implicitly joined with each value that is output by the function. * + * Given that this is deprecated, as an alternative, you can explode columns either using + * `functions.explode()`: + * + * {{{ + * ds.select(explode(split('words, " ")).as("word")) + * }}} + * + * or `flatMap()`: + * * {{{ - * ds.explode("words", "word") {words: String => words.split(" ")} + * ds.flatMap(_.words.split(" ")) * }}} * * @group untypedrel * @since 2.0.0 */ - @Experimental + @deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0") def explode[A, B : TypeTag](inputColumn: String, outputColumn: String)(f: A => TraversableOnce[B]) : DataFrame = { val dataType = ScalaReflection.schemaFor[B].dataType - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15533][SQL] Deprecate Dataset.explode
Repository: spark Updated Branches: refs/heads/branch-2.0 733cb44e3 -> 15a2dba66 [SPARK-15533][SQL] Deprecate Dataset.explode ## What changes were proposed in this pull request? This patch deprecates `Dataset.explode` and documents appropriate workarounds to use `flatMap()` or `functions.explode()` instead. ## How was this patch tested? N/A Author: Sameer Agarwal Closes #13312 from sameeragarwal/deprecate. (cherry picked from commit 06ed1fa3e45adfc11b0f615cb8b97c99fadc735f) Signed-off-by: Reynold Xin Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/15a2dba6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/15a2dba6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/15a2dba6 Branch: refs/heads/branch-2.0 Commit: 15a2dba6619438c5fb228437eaab50f399839473 Parents: 733cb44 Author: Sameer Agarwal Authored: Wed May 25 19:10:57 2016 -0700 Committer: Reynold Xin Committed: Wed May 25 19:11:10 2016 -0700 -- .../scala/org/apache/spark/sql/Dataset.scala| 33 +--- 1 file changed, 22 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/15a2dba6/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index 78a167e..e5140fc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -1556,30 +1556,33 @@ class Dataset[T] private[sql]( } /** - * :: Experimental :: * (Scala-specific) Returns a new [[Dataset]] where each row has been expanded to zero or more * rows by the provided function. This is similar to a `LATERAL VIEW` in HiveQL. The columns of * the input row are implicitly joined with each row that is output by the function. * - * The following example uses this function to count the number of books which contain - * a given word: + * Given that this is deprecated, as an alternative, you can explode columns either using + * `functions.explode()` or `flatMap()`. The following example uses these alternatives to count + * the number of books that contain a given word: * * {{{ * case class Book(title: String, words: String) * val ds: Dataset[Book] * - * case class Word(word: String) - * val allWords = ds.explode('words) { - * case Row(words: String) => words.split(" ").map(Word(_)) - * } + * val allWords = ds.select('title, explode(split('words, " ")).as("word")) * * val bookCountPerWord = allWords.groupBy("word").agg(countDistinct("title")) * }}} * + * Using `flatMap()` this can similarly be exploded as: + * + * {{{ + * ds.flatMap(_.words.split(" ")) + * }}} + * * @group untypedrel * @since 2.0.0 */ - @Experimental + @deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0") def explode[A <: Product : TypeTag](input: Column*)(f: Row => TraversableOnce[A]): DataFrame = { val elementSchema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType] @@ -1596,19 +1599,27 @@ class Dataset[T] private[sql]( } /** - * :: Experimental :: * (Scala-specific) Returns a new [[Dataset]] where a single column has been expanded to zero * or more rows by the provided function. This is similar to a `LATERAL VIEW` in HiveQL. All * columns of the input row are implicitly joined with each value that is output by the function. * + * Given that this is deprecated, as an alternative, you can explode columns either using + * `functions.explode()`: + * + * {{{ + * ds.select(explode(split('words, " ")).as("word")) + * }}} + * + * or `flatMap()`: + * * {{{ - * ds.explode("words", "word") {words: String => words.split(" ")} + * ds.flatMap(_.words.split(" ")) * }}} * * @group untypedrel * @since 2.0.0 */ - @Experimental + @deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0") def explode[A, B : TypeTag](inputColumn: String, outputColumn: String)(f: A => TraversableOnce[B]) : DataFrame = { val dataType = ScalaReflection.schemaFor[B].dataType - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15439][SPARKR] Failed to run unit test in SparkR
Repository: spark Updated Branches: refs/heads/master 06ed1fa3e -> 06bae8af1 [SPARK-15439][SPARKR] Failed to run unit test in SparkR ## What changes were proposed in this pull request? (Please fill in changes proposed in this fix) There are some failures when running SparkR unit tests. In this PR, I fixed two of these failures in test_context.R and test_sparkSQL.R The first one is due to different masked name. I added missed names in the expected arrays. The second one is because one PR removed the logic of a previous fix of missing subset method. The file privilege issue is still there. I am debugging it. SparkR shell can run the test case successfully. test_that("pipeRDD() on RDDs", { actual <- collect(pipeRDD(rdd, "more")) When using run-test script, it complains no such directories as below: cannot open file '/tmp/Rtmp4FQbah/filee2273f9d47f7': No such file or directory ## How was this patch tested? (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) Manually test it Author: wm...@hotmail.com Closes #13284 from wangmiao1981/R. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/06bae8af Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/06bae8af Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/06bae8af Branch: refs/heads/master Commit: 06bae8af17d9478c889d206a4556a697b5d629e7 Parents: 06ed1fa Author: wm...@hotmail.com Authored: Wed May 25 21:08:03 2016 -0700 Committer: Shivaram Venkataraman Committed: Wed May 25 21:08:03 2016 -0700 -- R/pkg/R/DataFrame.R | 6 +- R/pkg/inst/tests/testthat/test_context.R | 6 +- 2 files changed, 10 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/06bae8af/R/pkg/R/DataFrame.R -- diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R index 0c2a194..f719173 100644 --- a/R/pkg/R/DataFrame.R +++ b/R/pkg/R/DataFrame.R @@ -1445,7 +1445,11 @@ setMethod("[", signature(x = "SparkDataFrame"), #' } setMethod("subset", signature(x = "SparkDataFrame"), function(x, subset, select, drop = F, ...) { -x[subset, select, drop = drop] +if (missing(subset)) { +x[, select, drop = drop, ...] +} else { +x[subset, select, drop = drop, ...] +} }) #' Select http://git-wip-us.apache.org/repos/asf/spark/blob/06bae8af/R/pkg/inst/tests/testthat/test_context.R -- diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R index 0e5e15c..95258ba 100644 --- a/R/pkg/inst/tests/testthat/test_context.R +++ b/R/pkg/inst/tests/testthat/test_context.R @@ -27,6 +27,11 @@ test_that("Check masked functions", { namesOfMasked <- c("describe", "cov", "filter", "lag", "na.omit", "predict", "sd", "var", "colnames", "colnames<-", "intersect", "rank", "rbind", "sample", "subset", "summary", "transform", "drop", "window", "as.data.frame") + namesOfMaskedCompletely <- c("cov", "filter", "sample") + if (as.numeric(R.version$major) == 3 && as.numeric(R.version$minor) > 2) { +namesOfMasked <- c("endsWith", "startsWith", namesOfMasked) +namesOfMaskedCompletely <- c("endsWith", "startsWith", namesOfMaskedCompletely) + } expect_equal(length(maskedBySparkR), length(namesOfMasked)) expect_equal(sort(maskedBySparkR), sort(namesOfMasked)) # above are those reported as masked when `library(SparkR)` @@ -36,7 +41,6 @@ test_that("Check masked functions", { any(grepl("=\"ANY\"", capture.output(showMethods(x)[-1]))) })) maskedCompletely <- masked[!funcHasAny] - namesOfMaskedCompletely <- c("cov", "filter", "sample") expect_equal(length(maskedCompletely), length(namesOfMaskedCompletely)) expect_equal(sort(maskedCompletely), sort(namesOfMaskedCompletely)) }) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-15439][SPARKR] Failed to run unit test in SparkR
Repository: spark Updated Branches: refs/heads/branch-2.0 15a2dba66 -> bcad1d13f [SPARK-15439][SPARKR] Failed to run unit test in SparkR ## What changes were proposed in this pull request? (Please fill in changes proposed in this fix) There are some failures when running SparkR unit tests. In this PR, I fixed two of these failures in test_context.R and test_sparkSQL.R The first one is due to different masked name. I added missed names in the expected arrays. The second one is because one PR removed the logic of a previous fix of missing subset method. The file privilege issue is still there. I am debugging it. SparkR shell can run the test case successfully. test_that("pipeRDD() on RDDs", { actual <- collect(pipeRDD(rdd, "more")) When using run-test script, it complains no such directories as below: cannot open file '/tmp/Rtmp4FQbah/filee2273f9d47f7': No such file or directory ## How was this patch tested? (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) Manually test it Author: wm...@hotmail.com Closes #13284 from wangmiao1981/R. (cherry picked from commit 06bae8af17d9478c889d206a4556a697b5d629e7) Signed-off-by: Shivaram Venkataraman Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bcad1d13 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bcad1d13 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bcad1d13 Branch: refs/heads/branch-2.0 Commit: bcad1d13f58a119948e3374072824f70a14a6d34 Parents: 15a2dba Author: wm...@hotmail.com Authored: Wed May 25 21:08:03 2016 -0700 Committer: Shivaram Venkataraman Committed: Wed May 25 21:08:17 2016 -0700 -- R/pkg/R/DataFrame.R | 6 +- R/pkg/inst/tests/testthat/test_context.R | 6 +- 2 files changed, 10 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/bcad1d13/R/pkg/R/DataFrame.R -- diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R index 0c2a194..f719173 100644 --- a/R/pkg/R/DataFrame.R +++ b/R/pkg/R/DataFrame.R @@ -1445,7 +1445,11 @@ setMethod("[", signature(x = "SparkDataFrame"), #' } setMethod("subset", signature(x = "SparkDataFrame"), function(x, subset, select, drop = F, ...) { -x[subset, select, drop = drop] +if (missing(subset)) { +x[, select, drop = drop, ...] +} else { +x[subset, select, drop = drop, ...] +} }) #' Select http://git-wip-us.apache.org/repos/asf/spark/blob/bcad1d13/R/pkg/inst/tests/testthat/test_context.R -- diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R index 0e5e15c..95258ba 100644 --- a/R/pkg/inst/tests/testthat/test_context.R +++ b/R/pkg/inst/tests/testthat/test_context.R @@ -27,6 +27,11 @@ test_that("Check masked functions", { namesOfMasked <- c("describe", "cov", "filter", "lag", "na.omit", "predict", "sd", "var", "colnames", "colnames<-", "intersect", "rank", "rbind", "sample", "subset", "summary", "transform", "drop", "window", "as.data.frame") + namesOfMaskedCompletely <- c("cov", "filter", "sample") + if (as.numeric(R.version$major) == 3 && as.numeric(R.version$minor) > 2) { +namesOfMasked <- c("endsWith", "startsWith", namesOfMasked) +namesOfMaskedCompletely <- c("endsWith", "startsWith", namesOfMaskedCompletely) + } expect_equal(length(maskedBySparkR), length(namesOfMasked)) expect_equal(sort(maskedBySparkR), sort(namesOfMasked)) # above are those reported as masked when `library(SparkR)` @@ -36,7 +41,6 @@ test_that("Check masked functions", { any(grepl("=\"ANY\"", capture.output(showMethods(x)[-1]))) })) maskedCompletely <- masked[!funcHasAny] - namesOfMaskedCompletely <- c("cov", "filter", "sample") expect_equal(length(maskedCompletely), length(namesOfMaskedCompletely)) expect_equal(sort(maskedCompletely), sort(namesOfMaskedCompletely)) }) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-10372] [CORE] basic test framework for entire spark scheduler
Repository: spark Updated Branches: refs/heads/master 06bae8af1 -> dfc9fc02c [SPARK-10372] [CORE] basic test framework for entire spark scheduler This is a basic framework for testing the entire scheduler. The tests this adds aren't very interesting -- the point of this PR is just to setup the framework, to keep the initial change small, but it can be built upon to test more features (eg., speculation, killing tasks, blacklisting, etc.). Author: Imran Rashid Closes #8559 from squito/SPARK-10372-scheduler-integs. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dfc9fc02 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dfc9fc02 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dfc9fc02 Branch: refs/heads/master Commit: dfc9fc02ccbceb09213c394177d54b9ca56b6f24 Parents: 06bae8a Author: Imran Rashid Authored: Thu May 26 00:29:09 2016 -0500 Committer: Imran Rashid Committed: Thu May 26 00:29:09 2016 -0500 -- .../apache/spark/scheduler/DAGScheduler.scala | 2 +- .../org/apache/spark/scheduler/TaskResult.scala | 2 +- ...pache.spark.scheduler.ExternalClusterManager | 3 +- .../scheduler/BlacklistIntegrationSuite.scala | 130 + .../spark/scheduler/DAGSchedulerSuite.scala | 15 +- .../scheduler/ExternalClusterManagerSuite.scala | 25 +- .../scheduler/SchedulerIntegrationSuite.scala | 564 +++ 7 files changed, 728 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/dfc9fc02/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 766e979..a2eadbc 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -1396,7 +1396,7 @@ class DAGScheduler( stage.clearFailures() } else { stage.latestInfo.stageFailed(errorMessage.get) - logInfo("%s (%s) failed in %s s".format(stage, stage.name, serviceTime)) + logInfo(s"$stage (${stage.name}) failed in $serviceTime s due to ${errorMessage.get}") } outputCommitCoordinator.stageEnd(stage.id) http://git-wip-us.apache.org/repos/asf/spark/blob/dfc9fc02/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala index 80f2bf4..77fda6f 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResult.scala @@ -59,7 +59,7 @@ private[spark] class DirectTaskResult[T]( val numUpdates = in.readInt if (numUpdates == 0) { - accumUpdates = null + accumUpdates = Seq() } else { val _accumUpdates = new ArrayBuffer[AccumulatorV2[_, _]] for (i <- 0 until numUpdates) { http://git-wip-us.apache.org/repos/asf/spark/blob/dfc9fc02/core/src/test/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager -- diff --git a/core/src/test/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager b/core/src/test/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager index 3c570ff..757c6d2 100644 --- a/core/src/test/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager +++ b/core/src/test/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager @@ -1 +1,2 @@ -org.apache.spark.scheduler.DummyExternalClusterManager \ No newline at end of file +org.apache.spark.scheduler.DummyExternalClusterManager +org.apache.spark.scheduler.MockExternalClusterManager \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/dfc9fc02/core/src/test/scala/org/apache/spark/scheduler/BlacklistIntegrationSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/scheduler/BlacklistIntegrationSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BlacklistIntegrationSuite.scala new file mode 100644 index 000..6c9d4fb --- /dev/null +++ b/core/src/test/scala/org/apache/spark/scheduler/BlacklistIntegrationSuite.scala @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The AS
[4/4] spark git commit: [SPARK-15543][SQL] Rename DefaultSources to make them more self-describing
[SPARK-15543][SQL] Rename DefaultSources to make them more self-describing ## What changes were proposed in this pull request? This patch renames various DefaultSources to make their names more self-describing. The choice of "DefaultSource" was from the days when we did not have a good way to specify short names. They are now named: - LibSVMFileFormat - CSVFileFormat - JdbcRelationProvider - JsonFileFormat - ParquetFileFormat - TextFileFormat Backward compatibility is maintained through aliasing. ## How was this patch tested? Updated relevant test cases too. Author: Reynold Xin Closes #13311 from rxin/SPARK-15543. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/361ebc28 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/361ebc28 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/361ebc28 Branch: refs/heads/master Commit: 361ebc282b2d09dc6dcf21419a53c5c617b1b6bd Parents: dfc9fc0 Author: Reynold Xin Authored: Wed May 25 23:54:24 2016 -0700 Committer: Reynold Xin Committed: Wed May 25 23:54:24 2016 -0700 -- apache.spark.sql.sources.DataSourceRegister | 2 +- .../spark/ml/source/libsvm/LibSVMRelation.scala | 8 +- project/MimaExcludes.scala | 4 +- apache.spark.sql.sources.DataSourceRegister | 10 +- .../spark/sql/execution/ExistingRDD.scala | 2 +- .../sql/execution/datasources/DataSource.scala | 45 +- .../datasources/csv/CSVFileFormat.scala | 185 .../datasources/csv/DefaultSource.scala | 187 .../datasources/jdbc/DefaultSource.scala| 55 -- .../datasources/jdbc/JdbcRelationProvider.scala | 55 ++ .../datasources/json/JSONRelation.scala | 198 .../datasources/json/JsonFileFormat.scala | 198 .../datasources/parquet/ParquetFileFormat.scala | 923 +++ .../datasources/parquet/ParquetRelation.scala | 923 --- .../datasources/text/DefaultSource.scala| 141 --- .../datasources/text/TextFileFormat.scala | 141 +++ .../execution/datasources/json/JsonSuite.scala | 4 +- .../parquet/ParquetSchemaSuite.scala| 12 +- .../sql/sources/ResolvedDataSourceSuite.scala | 18 +- .../streaming/DataFrameReaderWriterSuite.scala | 542 --- .../sql/streaming/FileStreamSinkSuite.scala | 4 +- .../test/DataFrameReaderWriterSuite.scala | 541 +++ apache.spark.sql.sources.DataSourceRegister | 2 +- .../spark/sql/hive/HiveMetastoreCatalog.scala | 18 +- .../spark/sql/hive/orc/OrcFileFormat.scala | 375 .../apache/spark/sql/hive/orc/OrcRelation.scala | 371 .../sql/hive/orc/OrcHadoopFsRelationSuite.scala | 2 +- 27 files changed, 2498 insertions(+), 2468 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/361ebc28/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister -- diff --git a/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index f632dd6..a865cbe 100644 --- a/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -1 +1 @@ -org.apache.spark.ml.source.libsvm.DefaultSource +org.apache.spark.ml.source.libsvm.LibSVMFileFormat http://git-wip-us.apache.org/repos/asf/spark/blob/361ebc28/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala index 5ba768d..64ebf0c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala @@ -90,7 +90,7 @@ private[libsvm] class LibSVMOutputWriter( * .load("data/mllib/sample_libsvm_data.txt") * * // Java - * DataFrame df = spark.read().format("libsvm") + * Dataset df = spark.read().format("libsvm") * .option("numFeatures, "780") * .load("data/mllib/sample_libsvm_data.txt"); * }}} @@ -105,9 +105,13 @@ private[libsvm] class LibSVMOutputWriter( * - "vectorType": feature vector type, "sparse" (default) or "dense". * * @see [[https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/ LIBSVM datasets]] + * + * Note that this class is public for documentation purpose. Please don't use this class directly. + * Rather, use the data source API as illustrated abo
[2/4] spark git commit: [SPARK-15543][SQL] Rename DefaultSources to make them more self-describing
http://git-wip-us.apache.org/repos/asf/spark/blob/361ebc28/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala deleted file mode 100644 index f091615..000 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.text - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.io.{NullWritable, Text} -import org.apache.hadoop.mapreduce.{Job, RecordWriter, TaskAttemptContext} -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat - -import org.apache.spark.sql.{AnalysisException, Row, SparkSession} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.UnsafeRow -import org.apache.spark.sql.catalyst.expressions.codegen.{BufferHolder, UnsafeRowWriter} -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types.{StringType, StructType} -import org.apache.spark.util.SerializableConfiguration - -/** - * A data source for reading text files. - */ -class DefaultSource extends FileFormat with DataSourceRegister { - - override def shortName(): String = "text" - - private def verifySchema(schema: StructType): Unit = { -if (schema.size != 1) { - throw new AnalysisException( -s"Text data source supports only a single column, and you have ${schema.size} columns.") -} -val tpe = schema(0).dataType -if (tpe != StringType) { - throw new AnalysisException( -s"Text data source supports only a string column, but you have ${tpe.simpleString}.") -} - } - - override def inferSchema( - sparkSession: SparkSession, - options: Map[String, String], - files: Seq[FileStatus]): Option[StructType] = Some(new StructType().add("value", StringType)) - - override def prepareWrite( - sparkSession: SparkSession, - job: Job, - options: Map[String, String], - dataSchema: StructType): OutputWriterFactory = { -verifySchema(dataSchema) - -val conf = job.getConfiguration -val compressionCodec = options.get("compression").map(CompressionCodecs.getCodecClassName) -compressionCodec.foreach { codec => - CompressionCodecs.setCodecConfiguration(conf, codec) -} - -new OutputWriterFactory { - override def newInstance( - path: String, - bucketId: Option[Int], - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { -if (bucketId.isDefined) { - throw new AnalysisException("Text doesn't support bucketing") -} -new TextOutputWriter(path, dataSchema, context) - } -} - } - - override def buildReader( - sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { -val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - -(file: PartitionedFile) => { - val unsafeRow = new UnsafeRow(1) - val bufferHolder = new BufferHolder(unsafeRow) - val unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 1) - - new HadoopFileLinesReader(file, broadcastedHadoopConf.value.value).map { line => -// Writes to an UnsafeRow directly -bufferHolder.reset() -unsafeRowWriter.write(0, line.getBytes, 0, line.getLength) -unsafeRow.setTotalSize(bufferHolder.totalSize()) -unsafeRow - } -} - } -} - -class TextOutputWriter(path: String, dataSchema: StructType, context: TaskAttemptContext) - extends OutputWrit
[4/4] spark git commit: [SPARK-15543][SQL] Rename DefaultSources to make them more self-describing
[SPARK-15543][SQL] Rename DefaultSources to make them more self-describing ## What changes were proposed in this pull request? This patch renames various DefaultSources to make their names more self-describing. The choice of "DefaultSource" was from the days when we did not have a good way to specify short names. They are now named: - LibSVMFileFormat - CSVFileFormat - JdbcRelationProvider - JsonFileFormat - ParquetFileFormat - TextFileFormat Backward compatibility is maintained through aliasing. ## How was this patch tested? Updated relevant test cases too. Author: Reynold Xin Closes #13311 from rxin/SPARK-15543. (cherry picked from commit 361ebc282b2d09dc6dcf21419a53c5c617b1b6bd) Signed-off-by: Reynold Xin Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b3ee53b8 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b3ee53b8 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b3ee53b8 Branch: refs/heads/branch-2.0 Commit: b3ee53b8424bf7be4f3111744e07a6100a01e4da Parents: bcad1d1 Author: Reynold Xin Authored: Wed May 25 23:54:24 2016 -0700 Committer: Reynold Xin Committed: Wed May 25 23:54:30 2016 -0700 -- apache.spark.sql.sources.DataSourceRegister | 2 +- .../spark/ml/source/libsvm/LibSVMRelation.scala | 8 +- project/MimaExcludes.scala | 4 +- apache.spark.sql.sources.DataSourceRegister | 10 +- .../spark/sql/execution/ExistingRDD.scala | 2 +- .../sql/execution/datasources/DataSource.scala | 45 +- .../datasources/csv/CSVFileFormat.scala | 185 .../datasources/csv/DefaultSource.scala | 187 .../datasources/jdbc/DefaultSource.scala| 55 -- .../datasources/jdbc/JdbcRelationProvider.scala | 55 ++ .../datasources/json/JSONRelation.scala | 198 .../datasources/json/JsonFileFormat.scala | 198 .../datasources/parquet/ParquetFileFormat.scala | 923 +++ .../datasources/parquet/ParquetRelation.scala | 923 --- .../datasources/text/DefaultSource.scala| 141 --- .../datasources/text/TextFileFormat.scala | 141 +++ .../execution/datasources/json/JsonSuite.scala | 4 +- .../parquet/ParquetSchemaSuite.scala| 12 +- .../sql/sources/ResolvedDataSourceSuite.scala | 18 +- .../streaming/DataFrameReaderWriterSuite.scala | 542 --- .../sql/streaming/FileStreamSinkSuite.scala | 4 +- .../test/DataFrameReaderWriterSuite.scala | 541 +++ apache.spark.sql.sources.DataSourceRegister | 2 +- .../spark/sql/hive/HiveMetastoreCatalog.scala | 18 +- .../spark/sql/hive/orc/OrcFileFormat.scala | 375 .../apache/spark/sql/hive/orc/OrcRelation.scala | 371 .../sql/hive/orc/OrcHadoopFsRelationSuite.scala | 2 +- 27 files changed, 2498 insertions(+), 2468 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b3ee53b8/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister -- diff --git a/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index f632dd6..a865cbe 100644 --- a/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/mllib/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -1 +1 @@ -org.apache.spark.ml.source.libsvm.DefaultSource +org.apache.spark.ml.source.libsvm.LibSVMFileFormat http://git-wip-us.apache.org/repos/asf/spark/blob/b3ee53b8/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala -- diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala index 5ba768d..64ebf0c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala @@ -90,7 +90,7 @@ private[libsvm] class LibSVMOutputWriter( * .load("data/mllib/sample_libsvm_data.txt") * * // Java - * DataFrame df = spark.read().format("libsvm") + * Dataset df = spark.read().format("libsvm") * .option("numFeatures, "780") * .load("data/mllib/sample_libsvm_data.txt"); * }}} @@ -105,9 +105,13 @@ private[libsvm] class LibSVMOutputWriter( * - "vectorType": feature vector type, "sparse" (default) or "dense". * * @see [[https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/ LIBSVM datasets]] + * + * Note that this class is public for documentation
[3/4] spark git commit: [SPARK-15543][SQL] Rename DefaultSources to make them more self-describing
http://git-wip-us.apache.org/repos/asf/spark/blob/b3ee53b8/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala new file mode 100644 index 000..b47d41e --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala @@ -0,0 +1,923 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.parquet + +import java.net.URI +import java.util.logging.{Logger => JLogger} + +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.{Failure, Try} + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.mapreduce._ +import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit} +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl +import org.apache.parquet.{Log => ApacheParquetLog} +import org.apache.parquet.filter2.compat.FilterCompat +import org.apache.parquet.filter2.predicate.FilterApi +import org.apache.parquet.hadoop._ +import org.apache.parquet.hadoop.util.ContextUtil +import org.apache.parquet.schema.MessageType +import org.slf4j.bridge.SLF4JBridgeHandler + +import org.apache.spark.SparkException +import org.apache.spark.internal.Logging +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection +import org.apache.spark.sql.catalyst.parser.LegacyTypeStringParser +import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources._ +import org.apache.spark.sql.types._ +import org.apache.spark.util.SerializableConfiguration + +private[sql] class ParquetFileFormat + extends FileFormat + with DataSourceRegister + with Logging + with Serializable { + + override def shortName(): String = "parquet" + + override def toString: String = "ParquetFormat" + + override def hashCode(): Int = getClass.hashCode() + + override def equals(other: Any): Boolean = other.isInstanceOf[ParquetFileFormat] + + override def prepareWrite( + sparkSession: SparkSession, + job: Job, + options: Map[String, String], + dataSchema: StructType): OutputWriterFactory = { + +val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) + +val conf = ContextUtil.getConfiguration(job) + +val committerClass = + conf.getClass( +SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key, +classOf[ParquetOutputCommitter], +classOf[ParquetOutputCommitter]) + +if (conf.get(SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key) == null) { + logInfo("Using default output committer for Parquet: " + +classOf[ParquetOutputCommitter].getCanonicalName) +} else { + logInfo("Using user defined output committer for Parquet: " + committerClass.getCanonicalName) +} + +conf.setClass( + SQLConf.OUTPUT_COMMITTER_CLASS.key, + committerClass, + classOf[ParquetOutputCommitter]) + +// We're not really using `ParquetOutputFormat[Row]` for writing data here, because we override +// it in `ParquetOutputWriter` to support appending and dynamic partitioning. The reason why +// we set it here is to setup the output committer class to `ParquetOutputCommitter`, which is +// bundled with `ParquetOutputFormat[Row]`. +job.setOutputFormatClass(classOf[ParquetOutputFormat[Row]]) + +ParquetOutputFormat.setWriteSupportClass(job, classOf[CatalystWriteSupport]) + +// We want to clear this temporary metadata from saving into Parquet file. +// This metadata is only useful for detecting optional columns when pushdowning filters. +val dataSchemaToWrite = StructType.removeMetadata(StructType.metadataKeyForOptionalFi
[2/4] spark git commit: [SPARK-15543][SQL] Rename DefaultSources to make them more self-describing
http://git-wip-us.apache.org/repos/asf/spark/blob/b3ee53b8/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala deleted file mode 100644 index f091615..000 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.text - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.io.{NullWritable, Text} -import org.apache.hadoop.mapreduce.{Job, RecordWriter, TaskAttemptContext} -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat - -import org.apache.spark.sql.{AnalysisException, Row, SparkSession} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.UnsafeRow -import org.apache.spark.sql.catalyst.expressions.codegen.{BufferHolder, UnsafeRowWriter} -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types.{StringType, StructType} -import org.apache.spark.util.SerializableConfiguration - -/** - * A data source for reading text files. - */ -class DefaultSource extends FileFormat with DataSourceRegister { - - override def shortName(): String = "text" - - private def verifySchema(schema: StructType): Unit = { -if (schema.size != 1) { - throw new AnalysisException( -s"Text data source supports only a single column, and you have ${schema.size} columns.") -} -val tpe = schema(0).dataType -if (tpe != StringType) { - throw new AnalysisException( -s"Text data source supports only a string column, but you have ${tpe.simpleString}.") -} - } - - override def inferSchema( - sparkSession: SparkSession, - options: Map[String, String], - files: Seq[FileStatus]): Option[StructType] = Some(new StructType().add("value", StringType)) - - override def prepareWrite( - sparkSession: SparkSession, - job: Job, - options: Map[String, String], - dataSchema: StructType): OutputWriterFactory = { -verifySchema(dataSchema) - -val conf = job.getConfiguration -val compressionCodec = options.get("compression").map(CompressionCodecs.getCodecClassName) -compressionCodec.foreach { codec => - CompressionCodecs.setCodecConfiguration(conf, codec) -} - -new OutputWriterFactory { - override def newInstance( - path: String, - bucketId: Option[Int], - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { -if (bucketId.isDefined) { - throw new AnalysisException("Text doesn't support bucketing") -} -new TextOutputWriter(path, dataSchema, context) - } -} - } - - override def buildReader( - sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { -val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - -(file: PartitionedFile) => { - val unsafeRow = new UnsafeRow(1) - val bufferHolder = new BufferHolder(unsafeRow) - val unsafeRowWriter = new UnsafeRowWriter(bufferHolder, 1) - - new HadoopFileLinesReader(file, broadcastedHadoopConf.value.value).map { line => -// Writes to an UnsafeRow directly -bufferHolder.reset() -unsafeRowWriter.write(0, line.getBytes, 0, line.getLength) -unsafeRow.setTotalSize(bufferHolder.totalSize()) -unsafeRow - } -} - } -} - -class TextOutputWriter(path: String, dataSchema: StructType, context: TaskAttemptContext) - extends OutputWrit
[1/4] spark git commit: [SPARK-15543][SQL] Rename DefaultSources to make them more self-describing
Repository: spark Updated Branches: refs/heads/branch-2.0 bcad1d13f -> b3ee53b84 http://git-wip-us.apache.org/repos/asf/spark/blob/b3ee53b8/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala deleted file mode 100644 index 38f50c1..000 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hive.orc - -import java.net.URI -import java.util.Properties - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.hive.conf.HiveConf.ConfVars -import org.apache.hadoop.hive.ql.io.orc._ -import org.apache.hadoop.hive.serde2.objectinspector.{SettableStructObjectInspector, StructObjectInspector} -import org.apache.hadoop.hive.serde2.typeinfo.{StructTypeInfo, TypeInfoUtils} -import org.apache.hadoop.io.{NullWritable, Writable} -import org.apache.hadoop.mapred.{InputFormat => MapRedInputFormat, JobConf, OutputFormat => MapRedOutputFormat, RecordWriter, Reporter} -import org.apache.hadoop.mapreduce._ -import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit} - -import org.apache.spark.internal.Logging -import org.apache.spark.rdd.{HadoopRDD, RDD} -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.hive.{HiveInspectors, HiveShim} -import org.apache.spark.sql.sources.{Filter, _} -import org.apache.spark.sql.types.StructType -import org.apache.spark.util.SerializableConfiguration - -private[sql] class DefaultSource - extends FileFormat with DataSourceRegister with Serializable { - - override def shortName(): String = "orc" - - override def toString: String = "ORC" - - override def inferSchema( - sparkSession: SparkSession, - options: Map[String, String], - files: Seq[FileStatus]): Option[StructType] = { -OrcFileOperator.readSchema( - files.map(_.getPath.toUri.toString), - Some(sparkSession.sessionState.newHadoopConf()) -) - } - - override def prepareWrite( - sparkSession: SparkSession, - job: Job, - options: Map[String, String], - dataSchema: StructType): OutputWriterFactory = { -val orcOptions = new OrcOptions(options) - -val configuration = job.getConfiguration - -configuration.set(OrcRelation.ORC_COMPRESSION, orcOptions.compressionCodec) -configuration match { - case conf: JobConf => -conf.setOutputFormat(classOf[OrcOutputFormat]) - case conf => -conf.setClass( - "mapred.output.format.class", - classOf[OrcOutputFormat], - classOf[MapRedOutputFormat[_, _]]) -} - -new OutputWriterFactory { - override def newInstance( - path: String, - bucketId: Option[Int], - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { -new OrcOutputWriter(path, bucketId, dataSchema, context) - } -} - } - - override def buildReader( - sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): (PartitionedFile) => Iterator[InternalRow] = { -if (sparkSession.sessionState.conf.orcFilterPushDown) { - // Sets pushed predicates - OrcFilters.createFilter(requiredSchema, filters.toArray).foreach { f => -hadoopConf.set(OrcTableScan.SARG_PUSHDOWN, f.toKryo) -hadoopConf.setBoolean(ConfVars.HIVEOPTINDEXFILTER.varname, true) - } -} - -val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - -(file: PartitionedFile) => { - val conf = broadcastedHadoopConf.value.value
[3/4] spark git commit: [SPARK-15543][SQL] Rename DefaultSources to make them more self-describing
http://git-wip-us.apache.org/repos/asf/spark/blob/361ebc28/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala -- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala new file mode 100644 index 000..b47d41e --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala @@ -0,0 +1,923 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.parquet + +import java.net.URI +import java.util.logging.{Logger => JLogger} + +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.{Failure, Try} + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.mapreduce._ +import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit} +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl +import org.apache.parquet.{Log => ApacheParquetLog} +import org.apache.parquet.filter2.compat.FilterCompat +import org.apache.parquet.filter2.predicate.FilterApi +import org.apache.parquet.hadoop._ +import org.apache.parquet.hadoop.util.ContextUtil +import org.apache.parquet.schema.MessageType +import org.slf4j.bridge.SLF4JBridgeHandler + +import org.apache.spark.SparkException +import org.apache.spark.internal.Logging +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection +import org.apache.spark.sql.catalyst.parser.LegacyTypeStringParser +import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources._ +import org.apache.spark.sql.types._ +import org.apache.spark.util.SerializableConfiguration + +private[sql] class ParquetFileFormat + extends FileFormat + with DataSourceRegister + with Logging + with Serializable { + + override def shortName(): String = "parquet" + + override def toString: String = "ParquetFormat" + + override def hashCode(): Int = getClass.hashCode() + + override def equals(other: Any): Boolean = other.isInstanceOf[ParquetFileFormat] + + override def prepareWrite( + sparkSession: SparkSession, + job: Job, + options: Map[String, String], + dataSchema: StructType): OutputWriterFactory = { + +val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) + +val conf = ContextUtil.getConfiguration(job) + +val committerClass = + conf.getClass( +SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key, +classOf[ParquetOutputCommitter], +classOf[ParquetOutputCommitter]) + +if (conf.get(SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key) == null) { + logInfo("Using default output committer for Parquet: " + +classOf[ParquetOutputCommitter].getCanonicalName) +} else { + logInfo("Using user defined output committer for Parquet: " + committerClass.getCanonicalName) +} + +conf.setClass( + SQLConf.OUTPUT_COMMITTER_CLASS.key, + committerClass, + classOf[ParquetOutputCommitter]) + +// We're not really using `ParquetOutputFormat[Row]` for writing data here, because we override +// it in `ParquetOutputWriter` to support appending and dynamic partitioning. The reason why +// we set it here is to setup the output committer class to `ParquetOutputCommitter`, which is +// bundled with `ParquetOutputFormat[Row]`. +job.setOutputFormatClass(classOf[ParquetOutputFormat[Row]]) + +ParquetOutputFormat.setWriteSupportClass(job, classOf[CatalystWriteSupport]) + +// We want to clear this temporary metadata from saving into Parquet file. +// This metadata is only useful for detecting optional columns when pushdowning filters. +val dataSchemaToWrite = StructType.removeMetadata(StructType.metadataKeyForOptionalFi
[1/4] spark git commit: [SPARK-15543][SQL] Rename DefaultSources to make them more self-describing
Repository: spark Updated Branches: refs/heads/master dfc9fc02c -> 361ebc282 http://git-wip-us.apache.org/repos/asf/spark/blob/361ebc28/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala -- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala deleted file mode 100644 index 38f50c1..000 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hive.orc - -import java.net.URI -import java.util.Properties - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.hive.conf.HiveConf.ConfVars -import org.apache.hadoop.hive.ql.io.orc._ -import org.apache.hadoop.hive.serde2.objectinspector.{SettableStructObjectInspector, StructObjectInspector} -import org.apache.hadoop.hive.serde2.typeinfo.{StructTypeInfo, TypeInfoUtils} -import org.apache.hadoop.io.{NullWritable, Writable} -import org.apache.hadoop.mapred.{InputFormat => MapRedInputFormat, JobConf, OutputFormat => MapRedOutputFormat, RecordWriter, Reporter} -import org.apache.hadoop.mapreduce._ -import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit} - -import org.apache.spark.internal.Logging -import org.apache.spark.rdd.{HadoopRDD, RDD} -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.hive.{HiveInspectors, HiveShim} -import org.apache.spark.sql.sources.{Filter, _} -import org.apache.spark.sql.types.StructType -import org.apache.spark.util.SerializableConfiguration - -private[sql] class DefaultSource - extends FileFormat with DataSourceRegister with Serializable { - - override def shortName(): String = "orc" - - override def toString: String = "ORC" - - override def inferSchema( - sparkSession: SparkSession, - options: Map[String, String], - files: Seq[FileStatus]): Option[StructType] = { -OrcFileOperator.readSchema( - files.map(_.getPath.toUri.toString), - Some(sparkSession.sessionState.newHadoopConf()) -) - } - - override def prepareWrite( - sparkSession: SparkSession, - job: Job, - options: Map[String, String], - dataSchema: StructType): OutputWriterFactory = { -val orcOptions = new OrcOptions(options) - -val configuration = job.getConfiguration - -configuration.set(OrcRelation.ORC_COMPRESSION, orcOptions.compressionCodec) -configuration match { - case conf: JobConf => -conf.setOutputFormat(classOf[OrcOutputFormat]) - case conf => -conf.setClass( - "mapred.output.format.class", - classOf[OrcOutputFormat], - classOf[MapRedOutputFormat[_, _]]) -} - -new OutputWriterFactory { - override def newInstance( - path: String, - bucketId: Option[Int], - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { -new OrcOutputWriter(path, bucketId, dataSchema, context) - } -} - } - - override def buildReader( - sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): (PartitionedFile) => Iterator[InternalRow] = { -if (sparkSession.sessionState.conf.orcFilterPushDown) { - // Sets pushed predicates - OrcFilters.createFilter(requiredSchema, filters.toArray).foreach { f => -hadoopConf.set(OrcTableScan.SARG_PUSHDOWN, f.toKryo) -hadoopConf.setBoolean(ConfVars.HIVEOPTINDEXFILTER.varname, true) - } -} - -val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - -(file: PartitionedFile) => { - val conf = broadcastedHadoopConf.value.value - -