spark git commit: [SPARK-19265][SQL] make table relation cache general and does not depend on hive
Repository: spark
Updated Branches:
refs/heads/master 0c9231858 -> 2e6256002
[SPARK-19265][SQL] make table relation cache general and does not depend on hive
## What changes were proposed in this pull request?
We have a table relation plan cache in `HiveMetastoreCatalog`, which caches a
lot of things: file status, resolved data source, inferred schema, etc.
However, it doesn't make sense to limit this cache with hive support, we should
move it to SQL core module so that users can use this cache without hive
support.
It can also reduce the size of `HiveMetastoreCatalog`, so that it's easier to
remove it eventually.
main changes:
1. move the table relation cache to `SessionCatalog`
2. `SessionCatalog.lookupRelation` will return `SimpleCatalogRelation` and the
analyzer will convert it to `LogicalRelation` or `MetastoreRelation` later,
then `HiveSessionCatalog` doesn't need to override `lookupRelation` anymore
3. `FindDataSourceTable` will read/write the table relation cache.
## How was this patch tested?
existing tests.
Author: Wenchen Fan
Closes #16621 from cloud-fan/plan-cache.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2e625600
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2e625600
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2e625600
Branch: refs/heads/master
Commit: 2e62560024999c215cf2373fc9a8070bb2ad5c58
Parents: 0c92318
Author: Wenchen Fan
Authored: Thu Jan 19 00:07:48 2017 -0800
Committer: gatorsmile
Committed: Thu Jan 19 00:07:48 2017 -0800
--
.../sql/catalyst/catalog/SessionCatalog.scala | 33 +--
.../apache/spark/sql/catalyst/identifiers.scala | 4 +-
.../catalyst/catalog/SessionCatalogSuite.scala | 6 +-
.../org/apache/spark/sql/DataFrameWriter.scala | 4 +-
.../command/AnalyzeColumnCommand.scala | 3 +-
.../execution/command/AnalyzeTableCommand.scala | 3 +-
.../spark/sql/execution/command/tables.scala| 2 +-
.../datasources/DataSourceStrategy.scala| 60 ++--
.../apache/spark/sql/internal/CatalogImpl.scala | 2 +-
.../org/apache/spark/sql/DataFrameSuite.scala | 11 ---
.../spark/sql/execution/command/DDLSuite.scala | 3 +-
.../spark/sql/hive/HiveMetastoreCatalog.scala | 98 ++--
.../spark/sql/hive/HiveSessionCatalog.scala | 37 +---
.../spark/sql/hive/HiveSessionState.scala | 2 +
.../apache/spark/sql/hive/HiveStrategies.scala | 17 +++-
.../CreateHiveTableAsSelectCommand.scala| 8 +-
.../apache/spark/sql/hive/test/TestHive.scala | 2 +-
.../sql/hive/MetastoreDataSourcesSuite.scala| 22 +
.../apache/spark/sql/hive/StatisticsSuite.scala | 8 +-
.../sql/hive/execution/SQLQuerySuite.scala | 17 ++--
20 files changed, 144 insertions(+), 198 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2e625600/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 8008fcd..e9543f7 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -21,13 +21,13 @@ import javax.annotation.concurrent.GuardedBy
import scala.collection.mutable
+import com.google.common.cache.{Cache, CacheBuilder}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.internal.Logging
import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf}
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst._
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo}
@@ -118,6 +118,14 @@ class SessionCatalog(
}
/**
+ * A cache of qualified table name to table relation plan.
+ */
+ val tableRelationCache: Cache[QualifiedTableName, LogicalPlan] = {
+// TODO: create a config instead of hardcode 1000 here.
+CacheBuilder.newBuilder().maximumSize(1000).build[QualifiedTableName,
LogicalPlan]()
+ }
+
+ /**
* This method is used to make the given path qualified before we
* store this path in the underlying external catalog. So, when a path
* does not contain a scheme, this path will not be changed after the default
@@ -573,7 +581,7 @@ class SessionCatalog(
val relationAlias = alias.getOrEls
spark-website git commit: [SPARK-19249][DOC] Update download page to describe archive location
Repository: spark-website
Updated Branches:
refs/heads/asf-site 03485ecc8 -> 0fce54176
[SPARK-19249][DOC] Update download page to describe archive location
Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/0fce5417
Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/0fce5417
Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/0fce5417
Branch: refs/heads/asf-site
Commit: 0fce54176020f0a1adcc704cdd2d47e4415ebbb4
Parents: 03485ec
Author: Luciano Resende
Authored: Mon Jan 16 11:08:59 2017 -0800
Committer: Luciano Resende
Committed: Mon Jan 16 20:34:40 2017 -0800
--
downloads.md| 4
site/downloads.html | 6 +-
2 files changed, 9 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark-website/blob/0fce5417/downloads.md
--
diff --git a/downloads.md b/downloads.md
index cfbc6e4..3654d93 100644
--- a/downloads.md
+++ b/downloads.md
@@ -68,6 +68,10 @@ Once you've downloaded Spark, you can find instructions for
installing and build
+### Archived Releases
+
+As new Spark releases come out for each development stream, previous ones will
be archived, but they are still available at [Spark release
archives](https://archive.apache.org/dist/spark/).
+
### Nightly Packages and Artifacts
For developers, Spark maintains nightly builds and SNAPSHOT artifacts. More
information is available on the [the Developer Tools
page](/developer-tools.html#nightly-builds).
http://git-wip-us.apache.org/repos/asf/spark-website/blob/0fce5417/site/downloads.html
--
diff --git a/site/downloads.html b/site/downloads.html
index 24394ab..343751d 100644
--- a/site/downloads.html
+++ b/site/downloads.html
@@ -199,7 +199,7 @@ $(document).ready(function() {
});
-Download Apache Spark™
+Download Apache Spark™
@@ -263,6 +263,10 @@ git clone git://github.com/apache/spark.git -b branch-2.1
+Archived Releases
+
+As new Spark releases come out for each development stream, previous ones
will be archived, but they are still available at https://archive.apache.org/dist/spark/";>Spark release archives.
+
Nightly Packages and Artifacts
For developers, Spark maintains nightly builds and SNAPSHOT artifacts. More
information is available on the the Developer Tools page.
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-14272][ML] Add Loglikelihood in GaussianMixtureSummary
Repository: spark
Updated Branches:
refs/heads/master 2e6256002 -> 8ccca9170
[SPARK-14272][ML] Add Loglikelihood in GaussianMixtureSummary
## What changes were proposed in this pull request?
add loglikelihood in GMM.summary
## How was this patch tested?
added tests
Author: Zheng RuiFeng
Author: Ruifeng Zheng
Closes #12064 from zhengruifeng/gmm_metric.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8ccca917
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8ccca917
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8ccca917
Branch: refs/heads/master
Commit: 8ccca9170f983f74a7482f67206dae070c77b419
Parents: 2e62560
Author: Zheng RuiFeng
Authored: Thu Jan 19 03:46:37 2017 -0800
Committer: Yanbo Liang
Committed: Thu Jan 19 03:46:37 2017 -0800
--
.../org/apache/spark/ml/clustering/GaussianMixture.scala | 7 +--
.../main/scala/org/apache/spark/mllib/util/MLUtils.scala | 2 +-
.../apache/spark/ml/clustering/GaussianMixtureSuite.scala | 7 +++
project/MimaExcludes.scala| 5 -
python/pyspark/ml/clustering.py | 10 ++
5 files changed, 27 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/8ccca917/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
--
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index a7bb413..db5fff5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -416,7 +416,7 @@ class GaussianMixture @Since("2.0.0") (
val model = copyValues(new GaussianMixtureModel(uid, weights,
gaussianDists)).setParent(this)
val summary = new GaussianMixtureSummary(model.transform(dataset),
- $(predictionCol), $(probabilityCol), $(featuresCol), $(k))
+ $(predictionCol), $(probabilityCol), $(featuresCol), $(k), logLikelihood)
model.setSummary(Some(summary))
instr.logSuccess(model)
model
@@ -674,6 +674,7 @@ private class ExpectationAggregator(
*in `predictions`.
* @param featuresCol Name for column of features in `predictions`.
* @param k Number of clusters.
+ * @param logLikelihood Total log-likelihood for this model on the given data.
*/
@Since("2.0.0")
@Experimental
@@ -682,7 +683,9 @@ class GaussianMixtureSummary private[clustering] (
predictionCol: String,
@Since("2.0.0") val probabilityCol: String,
featuresCol: String,
-k: Int) extends ClusteringSummary(predictions, predictionCol, featuresCol,
k) {
+k: Int,
+@Since("2.2.0") val logLikelihood: Double)
+ extends ClusteringSummary(predictions, predictionCol, featuresCol, k) {
/**
* Probability of each cluster.
http://git-wip-us.apache.org/repos/asf/spark/blob/8ccca917/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
--
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
index de66c7c..95f904d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
@@ -34,7 +34,7 @@ import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.random.BernoulliCellSampler
/**
- * Helper methods to load, save and pre-process data used in ML Lib.
+ * Helper methods to load, save and pre-process data used in MLLib.
*/
@Since("0.8.0")
object MLUtils extends Logging {
http://git-wip-us.apache.org/repos/asf/spark/blob/8ccca917/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
--
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
index a362aee..e54eb27 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
@@ -207,6 +207,10 @@ class GaussianMixtureSuite extends SparkFunSuite with
MLlibTestSparkContext
[,1] [,2]
[1,] 0.2961543 0.160783
[2,] 0.1607830 1.008878
+
+ model$loglik
+
+ [1] -46.89499
*/
val weights = Array(0.533, 0.467)
val means = Array(Vectors.dense(10.363673, 9.897081),
Vectors.dense(0.11731091, -0.06192351))
@@ -219,6
spark git commit: [SPARK-19059][SQL] Unable to retrieve data from parquet table whose name startswith underscore
Repository: spark
Updated Branches:
refs/heads/master 8ccca9170 -> 064fadd2a
[SPARK-19059][SQL] Unable to retrieve data from parquet table whose name
startswith underscore
## What changes were proposed in this pull request?
The initial shouldFilterOut() method invocation filter the root path name(table
name in the intial call) and remove if it contains _. I moved the check one
level below, so it first list files/directories in the given root path and then
apply filter.
(Please fill in changes proposed in this fix)
## How was this patch tested?
Added new test case for this scenario
(Please explain how this patch was tested. E.g. unit tests, integration tests,
manual tests)
(If this patch involves UI changes, please attach a screenshot; otherwise,
remove this)
Please review http://spark.apache.org/contributing.html before opening a pull
request.
Author: jayadevanmurali
Author: jayadevan
Closes #16635 from jayadevanmurali/branch-0.1-SPARK-19059.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/064fadd2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/064fadd2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/064fadd2
Branch: refs/heads/master
Commit: 064fadd2a25d1c118e062e505a0ed56be31bdf34
Parents: 8ccca91
Author: jayadevanmurali
Authored: Thu Jan 19 20:07:52 2017 +0800
Committer: Wenchen Fan
Committed: Thu Jan 19 20:07:52 2017 +0800
--
.../PartitioningAwareFileIndex.scala| 91 ++--
.../org/apache/spark/sql/SQLQuerySuite.scala| 8 ++
2 files changed, 53 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/064fadd2/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
index 82c1599..fe9c657 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
@@ -385,55 +385,54 @@ object PartitioningAwareFileIndex extends Logging {
logTrace(s"Listing $path")
val fs = path.getFileSystem(hadoopConf)
val name = path.getName.toLowerCase
-if (shouldFilterOut(name)) {
- Seq.empty[FileStatus]
-} else {
- // [SPARK-17599] Prevent InMemoryFileIndex from failing if path doesn't
exist
- // Note that statuses only include FileStatus for the files and dirs
directly under path,
- // and does not include anything else recursively.
- val statuses = try fs.listStatus(path) catch {
-case _: FileNotFoundException =>
- logWarning(s"The directory $path was not found. Was it deleted very
recently?")
- Array.empty[FileStatus]
- }
- val allLeafStatuses = {
-val (dirs, topLevelFiles) = statuses.partition(_.isDirectory)
-val nestedFiles: Seq[FileStatus] = sessionOpt match {
- case Some(session) =>
-bulkListLeafFiles(dirs.map(_.getPath), hadoopConf, filter,
session).flatMap(_._2)
- case _ =>
-dirs.flatMap(dir => listLeafFiles(dir.getPath, hadoopConf, filter,
sessionOpt))
-}
-val allFiles = topLevelFiles ++ nestedFiles
-if (filter != null) allFiles.filter(f => filter.accept(f.getPath))
else allFiles
- }
+// [SPARK-17599] Prevent InMemoryFileIndex from failing if path doesn't
exist
+// Note that statuses only include FileStatus for the files and dirs
directly under path,
+// and does not include anything else recursively.
+val statuses = try fs.listStatus(path) catch {
+ case _: FileNotFoundException =>
+logWarning(s"The directory $path was not found. Was it deleted very
recently?")
+Array.empty[FileStatus]
+}
- allLeafStatuses.filterNot(status =>
shouldFilterOut(status.getPath.getName)).map {
-case f: LocatedFileStatus =>
- f
-
-// NOTE:
-//
-// - Although S3/S3A/S3N file system can be quite slow for remote file
metadata
-// operations, calling `getFileBlockLocations` does no harm here
since these file system
-// implementations don't actually issue RPC for this method.
-//
-// - Here we are calling `getFileBlockLocations` in a sequential
manner, but it should not
-// be a big deal since we always use to `listLeafFilesInParallel`
when the number of
-// paths exceeds threshold.
-case f =>
- // The other constructor
[4/4] spark git commit: [SPARK-16654][CORE] Add UI coverage for Application Level Blacklisting
[SPARK-16654][CORE] Add UI coverage for Application Level Blacklisting
Builds on top of work in SPARK-8425 to update Application Level Blacklisting in
the scheduler.
## What changes were proposed in this pull request?
Adds a UI to these patches by:
- defining new listener events for blacklisting and unblacklisting, nodes and
executors;
- sending said events at the relevant points in BlacklistTracker;
- adding JSON (de)serialization code for these events;
- augmenting the Executors UI page to show which, and how many, executors are
blacklisted;
- adding a unit test to make sure events are being fired;
- adding HistoryServerSuite coverage to verify that the SHS reads these events
correctly.
- updates the Executor UI to show Blacklisted/Active/Dead as a tri-state in
Executors Status
Updates .rat-excludes to pass tests.
username squito
## How was this patch tested?
./dev/run-tests
testOnly org.apache.spark.util.JsonProtocolSuite
testOnly org.apache.spark.scheduler.BlacklistTrackerSuite
testOnly org.apache.spark.deploy.history.HistoryServerSuite
https://github.com/jsoltren/jose-utils/blob/master/blacklist/test-blacklist.sh
Author: José Hiram Soltren
Closes #16346 from jsoltren/SPARK-16654-submit.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/640f9423
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/640f9423
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/640f9423
Branch: refs/heads/master
Commit: 640f942337e1ce87075195998bd051e19c4b50b9
Parents: 064fadd
Author: José Hiram Soltren
Authored: Thu Jan 19 09:08:18 2017 -0600
Committer: Imran Rashid
Committed: Thu Jan 19 09:08:18 2017 -0600
--
.../org/apache/spark/SparkFirehoseListener.java | 20
.../spark/ui/static/executorspage-template.html | 5 +
.../org/apache/spark/ui/static/executorspage.js | 39 --
.../spark/scheduler/BlacklistTracker.scala | 16 ++-
.../spark/scheduler/EventLoggingListener.scala | 16 +++
.../apache/spark/scheduler/SparkListener.scala | 54 +
.../spark/scheduler/SparkListenerBus.scala | 8 ++
.../spark/scheduler/TaskSchedulerImpl.scala | 10 +-
.../org/apache/spark/status/api/v1/api.scala| 1 +
.../scala/org/apache/spark/ui/ToolTips.scala| 3 +
.../apache/spark/ui/exec/ExecutorsPage.scala| 2 +
.../org/apache/spark/ui/exec/ExecutorsTab.scala | 57 -
.../apache/spark/ui/jobs/ExecutorTable.scala| 6 +
.../scala/org/apache/spark/ui/jobs/UIData.scala | 2 +
.../application_list_json_expectation.json | 44 +--
.../completed_app_list_json_expectation.json| 44 +--
.../executor_list_json_expectation.json | 1 +
.../executor_node_blacklisting_expectation.json | 118 +++
...blacklisting_unblacklisting_expectation.json | 118 +++
.../limit_app_list_json_expectation.json| 64 --
.../maxDate2_app_list_json_expectation.json | 2 +-
.../maxDate_app_list_json_expectation.json | 4 +-
.../minDate_app_list_json_expectation.json | 40 ++-
.../one_app_json_expectation.json | 2 +-
.../one_app_multi_attempt_json_expectation.json | 4 +-
.../one_stage_attempt_json_expectation.json | 80 ++---
.../one_stage_json_expectation.json | 80 ++---
...summary_w__custom_quantiles_expectation.json | 2 +-
...stage_with_accumulable_json_expectation.json | 100
.../spark-events/app-20161115172038-| 75
.../spark-events/app-20161116163331-| 68 +++
.../deploy/history/HistoryServerSuite.scala | 4 +-
.../spark/scheduler/BlacklistTrackerSuite.scala | 42 ++-
.../spark/scheduler/TaskSetManagerSuite.scala | 3 +-
.../apache/spark/util/JsonProtocolSuite.scala | 49
dev/.rat-excludes | 2 +
36 files changed, 950 insertions(+), 235 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
--
diff --git a/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
b/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
index 97eed61..9fe97b4 100644
--- a/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
+++ b/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
@@ -114,6 +114,26 @@ public class SparkFirehoseListener implements
SparkListenerInterface {
}
@Override
+public final void onExecutorBlacklisted(SparkListenerExecutorBlacklisted
executorBlacklisted) {
+on
[2/4] spark git commit: [SPARK-16654][CORE] Add UI coverage for Application Level Blacklisting
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/spark-events/app-20161116163331-
--
diff --git a/core/src/test/resources/spark-events/app-20161116163331-
b/core/src/test/resources/spark-events/app-20161116163331-
new file mode 100755
index 000..7566c9f
--- /dev/null
+++ b/core/src/test/resources/spark-events/app-20161116163331-
@@ -0,0 +1,68 @@
+{"Event":"SparkListenerLogStart","Spark Version":"2.1.0-SNAPSHOT"}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor
ID":"driver","Host":"172.22.0.167","Port":51475},"Maximum
Memory":384093388,"Timestamp":1479335611477}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java
Home":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre","Java
Version":"1.8.0_92 (Oracle Corporation)","Scala Version":"version
2.11.8"},"Spark
Properties":{"spark.blacklist.task.maxTaskAttemptsPerExecutor":"3","spark.blacklist.enabled":"TRUE","spark.driver.host":"172.22.0.167","spark.blacklist.task.maxTaskAttemptsPerNode":"3","spark.eventLog.enabled":"TRUE","spark.driver.port":"51459","spark.repl.class.uri":"spark://172.22.0.167:51459/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/l4/d46wlzj16593f3d812vk49twgp/T/spark-1cbc97d0-7fe6-4c9f-8c2c-f6fe51ee3cf2/repl-39929169-ac4c-4c6d-b116-f648e4dd62ed","spark.app.name":"Spark
shell","spark.blacklist.stage.maxFailedExecutorsPerNode":"3","spark.scheduler.mode":"FIFO","spark.eventLog.overwrite":"TRUE","spark.blacklist.stage.maxFailedTasksPerExecutor":"3","spark.executor.id":"driver","spark.blacklist.application.maxFailedEx
ecutorsPerNode":"2","spark.submit.deployMode":"client","spark.master":"local-cluster[4,4,1024]","spark.home":"/Users/Jose/IdeaProjects/spark","spark.eventLog.dir":"/Users/jose/logs","spark.sql.catalogImplementation":"in-memory","spark.eventLog.compress":"FALSE","spark.blacklist.application.maxFailedTasksPerExecutor":"1","spark.blacklist.timeout":"100","spark.app.id":"app-20161116163331-","spark.task.maxFailures":"4"},"System
Properties":{"java.io.tmpdir":"/var/folders/l4/d46wlzj16593f3d812vk49twgp/T/","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot
64-Bit Tiered
Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java
Virtual Machine Specification","java.vendor":"Oracle
Corporation","java.vm.specification.version":"1.8","user.home":"/Users/Jose","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","ftp.nonProxyHosts":"local|*.local|169.254/16|*.169.254/16","sun.arch.dat
a.model":"64","sun.boot.library.path":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib","user.dir":"/Users/Jose/IdeaProjects/spark","java.library.path":"/Users/Jose/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:.","sun.cpu.isalist":"","os.arch":"x86_64","java.vm.version":"25.92-b14","java.endorsed.dirs":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/endorsed","java.runtime.version":"1.8.0_92-b14","java.vm.info":"mixed
mode","java.ext.dirs":"/Users/Jose/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java","java.runtime.name":"Java(TM)
SE Runtime
Environment","file.separator":"/","io.netty.maxDirectMemory":"0","java.class.version":"52.0","scala.usejavacp":"true","java.specification.name":"Java
Platform API
Specification","sun.boot.class.path":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Chicago","java.specification.vendor":"Oracle
Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"10.11.6","sun.os.patch.level":"unknown","gopherProxySet":"false","java.vm.specification.vendor":"Oracle
Corporation","user.country":"US","sun.jnu.e
ncoding":"UTF-8","http.nonProxyHosts":"local|*.local|169.254/16|*.169.254/16","user.language":"en","socksNonProxyHosts":"local|*.local|169.254/16|*.169.254/16","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.lwawt.macosx.CPrinterJob","jav
[1/4] spark git commit: [SPARK-16654][CORE] Add UI coverage for Application Level Blacklisting
Repository: spark
Updated Branches:
refs/heads/master 064fadd2a -> 640f94233
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index d3b79dd..cd3959a 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -143,7 +143,9 @@ class HistoryServerSuite extends SparkFunSuite with
BeforeAndAfter with Matchers
"stage task list from multi-attempt app json(2)" ->
"applications/local-1426533911241/2/stages/0/0/taskList",
-"rdd list storage json" -> "applications/local-1422981780767/storage/rdd"
+"rdd list storage json" -> "applications/local-1422981780767/storage/rdd",
+"executor node blacklisting" ->
"applications/app-20161116163331-/executors",
+"executor node blacklisting unblacklisting" ->
"applications/app-20161115172038-/executors"
// Todo: enable this test when logging the even of onBlockUpdated. See:
SPARK-13845
// "one rdd storage json" ->
"applications/local-1422981780767/storage/rdd/0"
)
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
index 6b314d2..ead6955 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.scheduler
-import org.mockito.Mockito.when
+import org.mockito.Mockito.{verify, when}
import org.scalatest.BeforeAndAfterEach
import org.scalatest.mock.MockitoSugar
@@ -31,6 +31,7 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
private val clock = new ManualClock(0)
private var blacklist: BlacklistTracker = _
+ private var listenerBusMock: LiveListenerBus = _
private var scheduler: TaskSchedulerImpl = _
private var conf: SparkConf = _
@@ -40,7 +41,9 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
scheduler = mockTaskSchedWithConf(conf)
clock.setTime(0)
-blacklist = new BlacklistTracker(conf, clock)
+
+listenerBusMock = mock[LiveListenerBus]
+blacklist = new BlacklistTracker(listenerBusMock, conf, clock)
}
override def afterEach(): Unit = {
@@ -112,6 +115,8 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set())
} else {
assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1"))
+verify(listenerBusMock).post(
+ SparkListenerExecutorBlacklisted(0, "1", failuresUntilBlacklisted))
}
}
}
@@ -147,6 +152,7 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
// and it should be blacklisted for the entire application.
blacklist.updateBlacklistForSuccessfulTaskSet(0, 0,
taskSetBlacklist.execToFailures)
assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1"))
+verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "1",
numFailures))
} else {
// The task set failed, so we don't count these failures against the
executor for other
// stages.
@@ -166,6 +172,7 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
assert(blacklist.nodeBlacklist() === Set())
assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set())
assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1"))
+verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "1", 4))
val taskSetBlacklist1 = createTaskSetBlacklist(stageId = 1)
// Fail 4 tasks in one task set on executor 2, so that executor gets
blacklisted for the whole
@@ -177,15 +184,21 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
blacklist.updateBlacklistForSuccessfulTaskSet(0, 0,
taskSetBlacklist1.execToFailures)
assert(blacklist.nodeBlacklist() === Set("hostA"))
assertEquivalentToSet(blacklist.isNodeBlacklisted(_), Set("hostA"))
+verify(listenerBusMock).post(SparkListenerNodeBlacklisted(0, "hostA", 2))
assertEquivalentToSet(blacklist.isExecutorBlacklisted(_), Set("1", "2"))
+verify(listenerBusMock).post(SparkListenerExecutorBlacklisted(0, "2", 4))
// Advance the
[3/4] spark git commit: [SPARK-16654][CORE] Add UI coverage for Application Level Blacklisting
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/spark-events/app-20161115172038-
--
diff --git a/core/src/test/resources/spark-events/app-20161115172038-
b/core/src/test/resources/spark-events/app-20161115172038-
new file mode 100755
index 000..3af0451
--- /dev/null
+++ b/core/src/test/resources/spark-events/app-20161115172038-
@@ -0,0 +1,75 @@
+{"Event":"SparkListenerLogStart","Spark Version":"2.1.0-SNAPSHOT"}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor
ID":"driver","Host":"172.22.0.111","Port":64527},"Maximum
Memory":384093388,"Timestamp":1479252038836}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java
Home":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre","Java
Version":"1.8.0_92 (Oracle Corporation)","Scala Version":"version
2.11.8"},"Spark
Properties":{"spark.blacklist.task.maxTaskAttemptsPerExecutor":"3","spark.blacklist.enabled":"TRUE","spark.driver.host":"172.22.0.111","spark.blacklist.task.maxTaskAttemptsPerNode":"3","spark.eventLog.enabled":"TRUE","spark.driver.port":"64511","spark.repl.class.uri":"spark://172.22.0.111:64511/classes","spark.jars":"","spark.repl.class.outputDir":"/private/var/folders/l4/d46wlzj16593f3d812vk49twgp/T/spark-f09ef9e2-7f15-433f-a5d1-30138d8764ca/repl-28d60911-dbc3-465f-b7b3-ee55c071595e","spark.app.name":"Spark
shell","spark.blacklist.stage.maxFailedExecutorsPerNode":"3","spark.scheduler.mode":"FIFO","spark.eventLog.overwrite":"TRUE","spark.blacklist.stage.maxFailedTasksPerExecutor":"3","spark.executor.id":"driver","spark.blacklist.application.maxFailedEx
ecutorsPerNode":"2","spark.submit.deployMode":"client","spark.master":"local-cluster[4,4,1024]","spark.home":"/Users/Jose/IdeaProjects/spark","spark.eventLog.dir":"/Users/jose/logs","spark.sql.catalogImplementation":"in-memory","spark.eventLog.compress":"FALSE","spark.blacklist.application.maxFailedTasksPerExecutor":"1","spark.blacklist.timeout":"1","spark.app.id":"app-20161115172038-","spark.task.maxFailures":"4"},"System
Properties":{"java.io.tmpdir":"/var/folders/l4/d46wlzj16593f3d812vk49twgp/T/","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot
64-Bit Tiered
Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java
Virtual Machine Specification","java.vendor":"Oracle
Corporation","java.vm.specification.version":"1.8","user.home":"/Users/Jose","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","ftp.nonProxyHosts":"local|*.local|169.254/16|*.169.254/16","sun.arch.data.
model":"64","sun.boot.library.path":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib","user.dir":"/Users/Jose/IdeaProjects/spark","java.library.path":"/Users/Jose/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:.","sun.cpu.isalist":"","os.arch":"x86_64","java.vm.version":"25.92-b14","java.endorsed.dirs":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/endorsed","java.runtime.version":"1.8.0_92-b14","java.vm.info":"mixed
mode","java.ext.dirs":"/Users/Jose/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java","java.runtime.name":"Java(TM)
SE Runtime
Environment","file.separator":"/","io.netty.maxDirectMemory":"0","java.class.version":"52.0","scala.usejavacp":"true","java.specification.name":"Java
Platform API S
pecification","sun.boot.class.path":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Chicago","java.specification.vendor":"Oracle
Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"10.11.6","sun.os.patch.level":"unknown","gopherProxySet":"false","java.vm.specification.vendor":"Oracle
Corporation","user.country":"US","sun.jnu.enc
oding":"UTF-8","http.nonProxyHosts":"local|*.local|169.254/16|*.169.254/16","user.language":"en","socksNonProxyHosts":"local|*.local|169.254/16|*.169.254/16","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.lwawt.macosx.CPrinterJob","java.
spark git commit: [SPARK-19295][SQL] IsolatedClientLoader's downloadVersion should log the location of downloaded metastore client jars
Repository: spark
Updated Branches:
refs/heads/master 640f94233 -> 63d839028
[SPARK-19295][SQL] IsolatedClientLoader's downloadVersion should log the
location of downloaded metastore client jars
## What changes were proposed in this pull request?
This will help the users to know the location of those downloaded jars when
`spark.sql.hive.metastore.jars` is set to `maven`.
## How was this patch tested?
jenkins
Author: Yin Huai
Closes #16649 from yhuai/SPARK-19295.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/63d83902
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/63d83902
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/63d83902
Branch: refs/heads/master
Commit: 63d839028a6e03644febc360519fa8e01c5534cf
Parents: 640f942
Author: Yin Huai
Authored: Thu Jan 19 14:23:36 2017 -0800
Committer: Yin Huai
Committed: Thu Jan 19 14:23:36 2017 -0800
--
.../org/apache/spark/sql/hive/client/IsolatedClientLoader.scala | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/63d83902/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index 26b2de8..63fdd6b 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -122,6 +122,7 @@ private[hive] object IsolatedClientLoader extends Logging {
// TODO: Remove copy logic.
val tempDir = Utils.createTempDir(namePrefix = s"hive-${version}")
allFiles.foreach(f => FileUtils.copyFileToDirectory(f, tempDir))
+logInfo(s"Downloaded metastore jars to ${tempDir.getCanonicalPath}")
tempDir.listFiles().map(_.toURI.toURL)
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-17912] [SQL] Refactor code generation to get data for ColumnVector/ColumnarBatch
Repository: spark
Updated Branches:
refs/heads/master 63d839028 -> 148a84b37
[SPARK-17912] [SQL] Refactor code generation to get data for
ColumnVector/ColumnarBatch
## What changes were proposed in this pull request?
This PR refactors the code generation part to get data from `ColumnarVector`
and `ColumnarBatch` by using a trait `ColumnarBatchScan` for ease of reuse.
This is because this part will be reused by several components (e.g. parquet
reader, Dataset.cache, and others) since `ColumnarBatch` will be first citizen.
This PR is a part of https://github.com/apache/spark/pull/15219. In advance,
this PR makes the code generation for `ColumnarVector` and `ColumnarBatch`
reuseable as a trait. In general, this is very useful for other components from
the reuseability view, too.
## How was this patch tested?
tested existing test suites
Author: Kazuaki Ishizaki
Closes #15467 from kiszk/columnarrefactor.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/148a84b3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/148a84b3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/148a84b3
Branch: refs/heads/master
Commit: 148a84b37082697c7f61c6a621010abe4b12f2eb
Parents: 63d8390
Author: Kazuaki Ishizaki
Authored: Thu Jan 19 15:16:05 2017 -0800
Committer: Davies Liu
Committed: Thu Jan 19 15:16:05 2017 -0800
--
.../spark/sql/execution/ColumnarBatchScan.scala | 133 +++
.../sql/execution/DataSourceScanExec.scala | 86 +---
2 files changed, 135 insertions(+), 84 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/148a84b3/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
new file mode 100644
index 000..04fba17
--- /dev/null
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext,
ExprCode}
+import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
+import org.apache.spark.sql.execution.metric.SQLMetrics
+import org.apache.spark.sql.execution.vectorized.{ColumnarBatch, ColumnVector}
+import org.apache.spark.sql.types.DataType
+
+
+/**
+ * Helper trait for abstracting scan functionality using
+ * [[org.apache.spark.sql.execution.vectorized.ColumnarBatch]]es.
+ */
+private[sql] trait ColumnarBatchScan extends CodegenSupport {
+
+ val inMemoryTableScan: InMemoryTableScanExec = null
+
+ override lazy val metrics = Map(
+"numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output
rows"),
+"scanTime" -> SQLMetrics.createTimingMetric(sparkContext, "scan time"))
+
+ /**
+ * Generate [[ColumnVector]] expressions for our parent to consume as rows.
+ * This is called once per [[ColumnarBatch]].
+ */
+ private def genCodeColumnVector(
+ ctx: CodegenContext,
+ columnVar: String,
+ ordinal: String,
+ dataType: DataType,
+ nullable: Boolean): ExprCode = {
+val javaType = ctx.javaType(dataType)
+val value = ctx.getValue(columnVar, dataType, ordinal)
+val isNullVar = if (nullable) { ctx.freshName("isNull") } else { "false" }
+val valueVar = ctx.freshName("value")
+val str = s"columnVector[$columnVar, $ordinal, ${dataType.simpleString}]"
+val code = s"${ctx.registerComment(str)}\n" + (if (nullable) {
+ s"""
+boolean $isNullVar = $columnVar.isNullAt($ordinal);
+$javaType $valueVar = $isNullVar ? ${ctx.defaultValue(dataType)} :
($value);
+ """
+} else {
+ s"$javaType $valueVar = $value;"
+}).trim
+ExprCode(code, isNullVar, valueV
spark git commit: [SPARK-18899][SPARK-18912][SPARK-18913][SQL] refactor the error checking when append data to an existing table
Repository: spark
Updated Branches:
refs/heads/branch-2.1 4cff0b504 -> 7bc3e9ba7
[SPARK-18899][SPARK-18912][SPARK-18913][SQL] refactor the error checking when
append data to an existing table
## What changes were proposed in this pull request?
When we append data to an existing table with `DataFrameWriter.saveAsTable`, we
will do various checks to make sure the appended data is consistent with the
existing data.
However, we get the information of the existing table by matching the table
relation, instead of looking at the table metadata. This is error-prone, e.g.
we only check the number of columns for `HadoopFsRelation`, we forget to check
bucketing, etc.
This PR refactors the error checking by looking at the metadata of the existing
table, and fix several bugs:
* SPARK-18899: We forget to check if the specified bucketing matched the
existing table, which may lead to a problematic table that has different
bucketing in different data files.
* SPARK-18912: We forget to check the number of columns for non-file-based data
source table
* SPARK-18913: We don't support append data to a table with special column
names.
## How was this patch tested?
new regression test.
Author: Wenchen Fan
Closes #16313 from cloud-fan/bug1.
(cherry picked from commit f923c849e5b8f7e7aeafee59db598a9bf4970f50)
Signed-off-by: Wenchen Fan
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7bc3e9ba
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7bc3e9ba
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7bc3e9ba
Branch: refs/heads/branch-2.1
Commit: 7bc3e9ba73869c0c6cb8e754e41dbdd4740cfd07
Parents: 4cff0b5
Author: Wenchen Fan
Authored: Mon Dec 19 20:03:33 2016 -0800
Committer: Wenchen Fan
Committed: Fri Jan 20 09:52:20 2017 +0800
--
.../catalyst/catalog/ExternalCatalogUtils.scala | 37 +++
.../spark/sql/catalyst/catalog/interface.scala | 10 ++
.../command/createDataSourceTables.scala| 110 ---
.../spark/sql/execution/datasources/rules.scala | 53 +++--
.../spark/sql/execution/command/DDLSuite.scala | 4 +-
.../sql/test/DataFrameReaderWriterSuite.scala | 38 ++-
.../sql/hive/MetastoreDataSourcesSuite.scala| 17 ++-
.../sql/sources/HadoopFsRelationTest.scala | 2 +-
8 files changed, 180 insertions(+), 91 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/7bc3e9ba/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
index 817c1ab..4331841 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.catalyst.catalog
import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.Shell
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
object ExternalCatalogUtils {
@@ -133,4 +135,39 @@ object CatalogUtils {
case o => o
}
}
+
+ def normalizePartCols(
+ tableName: String,
+ tableCols: Seq[String],
+ partCols: Seq[String],
+ resolver: Resolver): Seq[String] = {
+partCols.map(normalizeColumnName(tableName, tableCols, _, "partition",
resolver))
+ }
+
+ def normalizeBucketSpec(
+ tableName: String,
+ tableCols: Seq[String],
+ bucketSpec: BucketSpec,
+ resolver: Resolver): BucketSpec = {
+val BucketSpec(numBuckets, bucketColumnNames, sortColumnNames) = bucketSpec
+val normalizedBucketCols = bucketColumnNames.map { colName =>
+ normalizeColumnName(tableName, tableCols, colName, "bucket", resolver)
+}
+val normalizedSortCols = sortColumnNames.map { colName =>
+ normalizeColumnName(tableName, tableCols, colName, "sort", resolver)
+}
+BucketSpec(numBuckets, normalizedBucketCols, normalizedSortCols)
+ }
+
+ private def normalizeColumnName(
+ tableName: String,
+ tableCols: Seq[String],
+ colName: String,
+ colType: String,
+ resolver: Resolver): String = {
+tableCols.find(resolver(_, colName)).getOrElse {
+ throw new AnalysisException(s"$colType column $colName is not defined in
table $tableName, " +
+s"defined table columns are: ${tableCols.mkString(", ")}")
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7bc3e9ba/sql/catalyst/src/main/scala/o
spark git commit: [SPARK-19292][SQL] filter with partition columns should be case-insensitive on Hive tables
Repository: spark
Updated Branches:
refs/heads/master 148a84b37 -> 0bf605c2c
[SPARK-19292][SQL] filter with partition columns should be case-insensitive on
Hive tables
## What changes were proposed in this pull request?
When we query a table with a filter on partitioned columns, we will push the
partition filter to the metastore to get matched partitions directly.
In `HiveExternalCatalog.listPartitionsByFilter`, we assume the column names in
partition filter are already normalized and we don't need to consider case
sensitivity. However, `HiveTableScanExec` doesn't follow this assumption. This
PR fixes it.
## How was this patch tested?
new regression test
Author: Wenchen Fan
Closes #16647 from cloud-fan/bug.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0bf605c2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0bf605c2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0bf605c2
Branch: refs/heads/master
Commit: 0bf605c2c67ca361cd4aa3a3b4492bef4aef76b9
Parents: 148a84b
Author: Wenchen Fan
Authored: Thu Jan 19 20:09:48 2017 -0800
Committer: gatorsmile
Committed: Thu Jan 19 20:09:48 2017 -0800
--
.../sql/execution/datasources/FileSourceStrategy.scala | 2 +-
.../spark/sql/hive/execution/HiveTableScanExec.scala | 12 +++-
.../spark/sql/hive/execution/SQLQuerySuite.scala | 13 +
3 files changed, 25 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0bf605c2/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
--
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
index 6d0671d..26e1380 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
@@ -62,7 +62,7 @@ object FileSourceStrategy extends Strategy with Logging {
val filterSet = ExpressionSet(filters)
// The attribute name of predicate could be different than the one in
schema in case of
- // case insensitive, we should change them to match the one in schema,
so we donot need to
+ // case insensitive, we should change them to match the one in schema,
so we do not need to
// worry about case sensitivity anymore.
val normalizedFilters = filters.map { e =>
e transform {
http://git-wip-us.apache.org/repos/asf/spark/blob/0bf605c2/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
--
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
index 7ee5fc5..def6ef3 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
@@ -146,9 +146,19 @@ case class HiveTableScanExec(
hadoopReader.makeRDDForTable(relation.hiveQlTable)
}
} else {
+ // The attribute name of predicate could be different than the one in
schema in case of
+ // case insensitive, we should change them to match the one in schema,
so we do not need to
+ // worry about case sensitivity anymore.
+ val normalizedFilters = partitionPruningPred.map { e =>
+e transform {
+ case a: AttributeReference =>
+a.withName(relation.output.find(_.semanticEquals(a)).get.name)
+}
+ }
+
Utils.withDummyCallSite(sqlContext.sparkContext) {
hadoopReader.makeRDDForPartitionedTable(
- prunePartitions(relation.getHiveQlPartitions(partitionPruningPred)))
+ prunePartitions(relation.getHiveQlPartitions(normalizedFilters)))
}
}
val numOutputRows = longMetric("numOutputRows")
http://git-wip-us.apache.org/repos/asf/spark/blob/0bf605c2/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 104b525..1a28c4c 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2014,4 +2014,17 @@ class SQLQuerySuit
spark git commit: [SPARK-19271][SQL] Change non-cbo estimation of aggregate
Repository: spark
Updated Branches:
refs/heads/master 0bf605c2c -> 039ed9fe8
[SPARK-19271][SQL] Change non-cbo estimation of aggregate
## What changes were proposed in this pull request?
Change non-cbo estimation behavior of aggregate:
- If groupExpression is empty, we can know row count (=1) and the corresponding
size;
- otherwise, estimation falls back to UnaryNode's computeStats method, which
should not propagate rowCount and attributeStats in Statistics because they are
not estimated in that method.
## How was this patch tested?
Added test case
Author: wangzhenhua
Closes #16631 from wzhfy/aggNoCbo.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/039ed9fe
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/039ed9fe
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/039ed9fe
Branch: refs/heads/master
Commit: 039ed9fe8a2fdcd99e0561af64cda8fe3406bc12
Parents: 0bf605c
Author: wangzhenhua
Authored: Thu Jan 19 22:18:47 2017 -0800
Committer: gatorsmile
Committed: Thu Jan 19 22:18:47 2017 -0800
--
.../catalyst/plans/logical/LogicalPlan.scala| 3 ++-
.../plans/logical/basicLogicalOperators.scala | 7 --
.../statsEstimation/AggregateEstimation.scala | 2 +-
.../statsEstimation/EstimationUtils.scala | 4 ++--
.../statsEstimation/ProjectEstimation.scala | 2 +-
.../AggregateEstimationSuite.scala | 24 +++-
.../StatsEstimationTestBase.scala | 7 +++---
7 files changed, 38 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/039ed9fe/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
index 0587a59..93550e1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
@@ -344,7 +344,8 @@ abstract class UnaryNode extends LogicalPlan {
sizeInBytes = 1
}
-child.stats(conf).copy(sizeInBytes = sizeInBytes)
+// Don't propagate rowCount and attributeStats, since they are not
estimated here.
+Statistics(sizeInBytes = sizeInBytes, isBroadcastable =
child.stats(conf).isBroadcastable)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/039ed9fe/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 3bd3143..432097d 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogTable,
CatalogTypes}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
import org.apache.spark.sql.catalyst.plans._
-import
org.apache.spark.sql.catalyst.plans.logical.statsEstimation.{AggregateEstimation,
ProjectEstimation}
+import
org.apache.spark.sql.catalyst.plans.logical.statsEstimation.{AggregateEstimation,
EstimationUtils, ProjectEstimation}
import org.apache.spark.sql.types._
import org.apache.spark.util.Utils
@@ -541,7 +541,10 @@ case class Aggregate(
override def computeStats(conf: CatalystConf): Statistics = {
def simpleEstimation: Statistics = {
if (groupingExpressions.isEmpty) {
-super.computeStats(conf).copy(sizeInBytes = 1)
+Statistics(
+ sizeInBytes = EstimationUtils.getOutputSize(output, outputRowCount =
1),
+ rowCount = Some(1),
+ isBroadcastable = child.stats(conf).isBroadcastable)
} else {
super.computeStats(conf)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/039ed9fe/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
i
