svn commit: r31918 - in /dev/spark/2.4.1-SNAPSHOT-2019_01_11_23_17-bb97459-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sat Jan 12 07:34:36 2019 New Revision: 31918 Log: Apache Spark 2.4.1-SNAPSHOT-2019_01_11_23_17-bb97459 docs [This commit notification would consist of 1476 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31917 - in /dev/spark/2.3.3-SNAPSHOT-2019_01_11_23_17-b6c4649-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sat Jan 12 07:32:34 2019 New Revision: 31917 Log: Apache Spark 2.3.3-SNAPSHOT-2019_01_11_23_17-b6c4649 docs [This commit notification would consist of 1443 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.3 updated: [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-2.3 by this push: new b6c4649 [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite b6c4649 is described below commit b6c464997e179bf648250435dfe6295c84e76784 Author: Dongjoon Hyun AuthorDate: Fri Jan 11 22:53:58 2019 -0800 [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite The vote of final release of `branch-2.2` passed and the branch goes EOL. This PR removes Spark 2.2.x from the testing coverage. Pass the Jenkins. Closes #23526 from dongjoon-hyun/SPARK-26607. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun (cherry picked from commit 3587a9a2275615b82492b89204b141636542ce52) Signed-off-by: Dongjoon Hyun --- .../org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index af15da6..6522f77 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -203,7 +203,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { object PROCESS_TABLES extends QueryTest with SQLTestUtils { // Tests the latest version of every release line. - val testingVersions = Seq("2.1.3", "2.2.2", "2.3.2") + val testingVersions = Seq("2.3.2") protected var spark: SparkSession = _ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new 3587a9a [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite 3587a9a is described below commit 3587a9a2275615b82492b89204b141636542ce52 Author: Dongjoon Hyun AuthorDate: Fri Jan 11 22:53:58 2019 -0800 [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite ## What changes were proposed in this pull request? The vote of final release of `branch-2.2` passed and the branch goes EOL. This PR removes Spark 2.2.x from the testing coverage. ## How was this patch tested? Pass the Jenkins. Closes #23526 from dongjoon-hyun/SPARK-26607. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index f1e8423..dd0e1bd 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -206,7 +206,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { object PROCESS_TABLES extends QueryTest with SQLTestUtils { // Tests the latest version of every release line. - val testingVersions = Seq("2.2.2", "2.3.2", "2.4.0") + val testingVersions = Seq("2.3.2", "2.4.0") protected var spark: SparkSession = _ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new bb97459 [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite bb97459 is described below commit bb9745904f4c439be21fa435f2bb5c66de164f61 Author: Dongjoon Hyun AuthorDate: Fri Jan 11 22:53:58 2019 -0800 [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite ## What changes were proposed in this pull request? The vote of final release of `branch-2.2` passed and the branch goes EOL. This PR removes Spark 2.2.x from the testing coverage. ## How was this patch tested? Pass the Jenkins. Closes #23526 from dongjoon-hyun/SPARK-26607. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun (cherry picked from commit 3587a9a2275615b82492b89204b141636542ce52) Signed-off-by: Dongjoon Hyun --- .../org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index cce2f20..632a21a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -203,7 +203,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { object PROCESS_TABLES extends QueryTest with SQLTestUtils { // Tests the latest version of every release line. - val testingVersions = Seq("2.2.2", "2.3.2", "2.4.0") + val testingVersions = Seq("2.3.2", "2.4.0") protected var spark: SparkSession = _ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26482][K8S][TEST][FOLLOWUP] Fix compile failure
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new e00ebd5 [SPARK-26482][K8S][TEST][FOLLOWUP] Fix compile failure e00ebd5 is described below commit e00ebd5c72e719500f3deb676afc5c99be20a95e Author: Dongjoon Hyun AuthorDate: Fri Jan 11 21:58:06 2019 -0800 [SPARK-26482][K8S][TEST][FOLLOWUP] Fix compile failure ## What changes were proposed in this pull request? This fixes K8S integration test compilation failure introduced by #23423 . ```scala $ build/sbt -Pkubernetes-integration-tests test:package ... [error] /Users/dongjoon/APACHE/spark/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala:71: type mismatch; [error] found : org.apache.spark.internal.config.OptionalConfigEntry[Boolean] [error] required: String [error] .set(IS_TESTING, false) [error]^ [error] /Users/dongjoon/APACHE/spark/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala:71: type mismatch; [error] found : Boolean(false) [error] required: String [error] .set(IS_TESTING, false) [error]^ [error] two errors found ``` ## How was this patch tested? Pass the K8S integration test. Closes #23527 from dongjoon-hyun/SPARK-26482. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index d90c306..c869803 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -68,8 +68,8 @@ private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesCl .set("spark.executor.cores", "1") .set("spark.executors.instances", "1") .set("spark.app.name", "spark-test-app") - .set(IS_TESTING, false) - .set(UI_ENABLED, true) + .set(IS_TESTING.key, "false") + .set(UI_ENABLED.key, "true") .set("spark.kubernetes.submission.waitAppCompletion", "false") .set("spark.kubernetes.authenticate.driver.serviceAccountName", serviceAccountName) } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31916 - in /dev/spark/3.0.0-SNAPSHOT-2019_01_11_21_14-19e17ac-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sat Jan 12 05:27:11 2019 New Revision: 31916 Log: Apache Spark 3.0.0-SNAPSHOT-2019_01_11_21_14-19e17ac docs [This commit notification would consist of 1775 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated (ae382c9 -> 19e17ac)
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git. from ae382c9 [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs add 19e17ac [SPARK-25692][TEST] Increase timeout in fetchBothChunks test No new revisions were added by this update. Summary of changes: .../test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31914 - /dev/spark/v2.2.3-rc1-docs/
Author: dongjoon Date: Sat Jan 12 01:00:38 2019 New Revision: 31914 Log: Removing RC artifacts. Removed: dev/spark/v2.2.3-rc1-docs/ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] tag v2.2.3 created (now 4acb6ba)
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a change to tag v2.2.3 in repository https://gitbox.apache.org/repos/asf/spark.git. at 4acb6ba (commit) No new revisions were added by this update. - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31912 - /release/spark/KEYS
Author: dbtsai Date: Sat Jan 12 00:38:01 2019 New Revision: 31912 Log: Update KEYS Modified: release/spark/KEYS Modified: release/spark/KEYS == --- release/spark/KEYS (original) +++ release/spark/KEYS Sat Jan 12 00:38:01 2019 @@ -760,3 +760,130 @@ qmd2/zxUPGuOvX/e79ATOM8KMQFbGikLTcjW9gcU DFZYf6DnKNwDnBzBuAx2S+P0 =a0mL -END PGP PUBLIC KEY BLOCK- + +pub 4096R/34F0FC5C 2015-05-05 +uid Dongjoon Hyun (CODE SIGNING KEY) +sub 4096R/727240E9 2015-05-05 + +-BEGIN PGP PUBLIC KEY BLOCK- +Version: GnuPG v2.0.19 (FreeBSD) + +mQINBFVIx4YBEADCRN5gDmObS8L3e2YNFNlCrXxSVcUgJ6UOYt12wi337dEudEqk +s+0jZ6p9k3mz1hD3uTim+lT/O2ODRDNWYBfT2cF6E26QGieXrKTxeApwFrXyWuSm +49rlhgxi/ECGA59ovPMQvxrN1b8IFChWfXR9lmIllUKJ02zNnpossEPdHs6k2Efu +dgMlxhm0qFn5JULDfr/+ump9n9IeiojpQ+bsD/iZ/7DneK2BqtHA7NGOKCMjRzhe +OzG2lxwyLxB4zR3d2AUMlLDJUMAjR7MOEs7B/d9ES+ka2HI/CNc19F5cD5soT3EE +D7B6HwCRNPSyeytwYCafLCaxveKpDWfehUDNFrSqBaxyDh0/i8x50GrR++AsItZU +gh/qcYF2ju0uKZ4t5UVvoOOCBc11njzdk6VeE6VTDoRFXZ6QUP8HWR+F0PdLZKWf +e12Pp0dMwDcWppJRC6zzbfGizz/209VSkF4atIU9rJmB9T+rm2RxL4+l5QjADWRP +1U4NMC6Fh3aTR75KOtL50NMjk1H4o33KX33Ec2k6eJ+9kDcqhRb1+lA4I1JZB2pX ++crvehzPDUh9i+nVXPyxUhgC1SY3+I5IIdGGgz69KPU8da7l+MRLxL1YIZOnqQnt +3KdBRwpFzDXGK6bAhrboj4Hab+GIHBRpJ89pJz8uG5CaMFfWvapGbC85DQARAQAB +tDZEb25nam9vbiBIeXVuIChDT0RFIFNJR05JTkcgS0VZKSA8ZG9uZ2pvb25AYXBh +Y2hlLm9yZz6JAjcEEwEKACECGwMCHgECF4AFAlVQK2QFCwkIBwMFFQoJCAsFFgID +AQAACgkQ7aAM6DTw/FxSGA//eAzyfDp8RCD5IpPj3UeLs4J3zDsVMCj5IQNK0stv +XjMy7SBC4jhz8HdriYQC8qSj/fbnMfZuqtr2XV2lC8StDXFe8/G+JQFd25+BbrfR ++fNv97adR3lp97GxehXRLF4Aw+Am2EsbI0ej+ZhW/H6XS6jjSa/yxsLUtEIGDB4k +3wnlZ0gqH49nryHitXC7cCpXh7qY1K5ObXjkhnsdsIiKjVfsQBN4MzdknGuEhTfM +Aez4OkaNf9T7btmFI56ijTP6ClM7FaIbofLCJr8EVojR9HoErgdCO9byD4SIerc7 +rhuAyiyVFodMsqc/jvNvv8Usu1gFifP3YvuxL8TRrL04rJhOyg8oYU4gGTnipzii +AHcCvDP0zSwmhlL2wht8aExc3or8RvQluh7HSqGjDq+dUKIdqysIoxev7kOM4Gs3 +wBltyyV4Pwa3kzn3D2JjTm5E0rFEmP//3a58mbQL5zRXxg9iX9lGQ357GLRjz7se +eAuTpW+z9ZgmOhTRdFf1grVjiOmOCWMP7+Mqkq9R3/wvqkk1jjRJeSpwmR8ZtTP6 +IqduYhE609esFX65SeTTtbqpQIZ5J80wBybIYT/XPYWuFZdn/pgWpKVJWs+uEYjM +U7pONs6avNKguu92HLbpHCclQS5NgcjGb7PvuNIkJb8yQixeVoDZQXz/B6YN5iWS +uLuJAjgEEwECACIFAlVIx4YCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJ +EO2gDOg08Pxc4OMP/jTvfPE9Z+n/Veddf/i8Gdxurf2LiYgzHFu84CfKE8vLxo13 +PP8DN4ZNBIRx2+a3OQ3/PLTpDn/D3Pf9uFriKXHNGksu153TAqMO5x81Hc2vaRbP +StUZrWLcIsBmGlZVR4rzuSF1ypFJ80m7bGyVYQFL4rq9sd2E9aPwy9zWGBjqZAQu +IfldLnOfkpnDS+X5J9/12kB8By7nvDydQIgTCJGxiLGh71cfmZDtTJuHLdfmxw+o +iqfXEthy6UqDayLJRGqeMJDCXZGj0HPQZHZZTDtF8czH0w0ezj6WePD0dnpm5w/Y +g3dwQ2jWfkRAe04Q3um/bTzcADWASQtoAOPCCSe0/JT5fXo86uwdYUBDivx9OABz +bgQZlq1Xj4YbRNF5QUvAq/k76yVhftNGaUBT7ubWyxBWBeh43W5g/r5ggvO9XfQj +r+Ox6efblvvJLMnYZe8Gmd2lbuFWfUpShYXqf33rfsKHcUkQuTEKFTVOtUsgDMng +Fg/K3R/Kg9pCX1BHRha1o89mBcwaTW0KaZ9sTZ6pjo58f7f7oWEKBfpGVWRFlQcq +iHTzCJVVWnk4u1AMk7+M5xqSHspX0YOHKFl4EXRs9bOtX+NUu2+7vbY2B3D1fzTY +IqEGdZ3SGh6yMyJgzi7/k+jMmx1O/rq9WNspO630JOBDH89PcxibDaVw5pPQuQIN +BFVIx4YBEADKRaIlOp9qlYhID8n7esMy75XfOliKPAxk9yeuZWQ+Bat2ZznfPVmL +kqnY4xSTqVVvFiJqKmaFnVa93YT5Z9MoIkd5z0A5gsSukOiYe4856FrluGzNsMJ+ +0i5rSQlWC1YozOQCBWKgfIW5Snam6rUrvIVnVEoZaTMS0QqIYndGuw0GkaCLn04V +ycu2W9w/1Mj5ypYdHMZERqIaCju70PDYS1+MVGKcv08D8jBOy45ZNKhIHny2N6NQ +f8+WmBqG+hTl7avE+E3mRx2Wy/XF6sHS6jsqwgSHBE80/bK1H2UMXNO9qHSFDxsZ +bcWYqC2zFFy0rM5poGlTk+9G6Dw0iUY1oKJ5hxTkgIN1yLz9qah8l5quMgAKcc+t +5H89xoK1gnb9CZJrNVD52phKZwDU30gh/CasRxVNSWmotwuJRwx+adBhklclAmjt +QGJQomlmBK6LzQcbl987BwWLQJfPx+CJRH4VqYaojP62vKQttQtCG/P/IQjux59I +5RwkiSYIRgi7/RvXfCEDPZRE+GP9b77TY7Rj7NW43ouGpTwP6i78CLVQ/XhJLA63 +6o1e0xNfAdtLT1pZbsnCT0sp4BovBfeiZGHdoy0oiAVH78bZH1W09oKK3HXnsNa5 +83VD6h+6L4NJS4cxagl3TYI5/YMnX/AVQxvZXW42tAFTDmqqmf2ioQARAQABiQIf +BBgBAgAJBQJVSMeGAhsMAAoJEO2gDOg08PxcKsYP/2etfI6tUOVK0PWEsnrwrPAf +C4hObPvQBjS1ccbMO12fgtaQlZkeUgBfkKEhimeWW2s3a2VnwCN3BCqTW2dE6Pii +84FrL9I5VQ11lWBMG5bq3+NEZKzzkTUUCg3blvSReDSUowTxwTzIF2/opT2IDRdB +/c40WOb/7mwgGb0zxxm4ye4X+Q3d8uxFhKQGUqssYz37Dg8PnNfqHjyOWWVPPXzP +HuKwaZlE7+6Jzd7mTuCghViwliVT88e0oCM85t6Dm0u9PgPgQnbTzS4Gf3Z9iZeZ +ecTc4VoXBFpFxr52luXJmvy/b3lXTgBwF0SsTfHg7VY4K3DGtcijg/fSVD4jCHej +/jIcbFgFD5MRxwvGbOG/c5F/KKqSW0/9E9cr8pJXJX+m5llCU4C+ag4CU41SFhxO +G8Q9v60ZdfXlUqVJp7Xxe9lXeRKZPHCjLNepZLDc4HwRtyXcWVreq2w3U/mN3Lqh +oQXTzdUCO+hwbMVC87C4Afzkt4LPEeNAdmwd00hb5eduva1BCUMo9zXsfYPmUd1w +ZQwuVzdFAiXHmPZIIb1u8MJPm8wrqPqZmCawLa7xACSGo8VuAfFL2jdzdV61sKrs +aI9kX8V9gl5PZLw+LchGX5H7HKoRxZM3UbPkY5Mv21YPQ5C1EVM47ERc5mUtl5O0 +ZIAzEigXrrsePyvHGf6H +=6YJg +-END PGP PUBLIC KEY BLOCK- + +pub rsa4096 2019-01-10 [SC] + 0E9925082727075EEE83D4B06EC5F1052DF08FF4 +uid [ultimate] Takeshi Yamamuro (CODE SIGNING KEY) +sub rsa4096 2019-01-10 [E] + +-BEGIN PGP PUBLIC KEY BLOCK- + +mQINBFw2q20BEADLW2BZbJO2YHmAmAumggCTm4aVWFRYH+NX0zqEX2bynA0GM5hR +euvLL6w5vq44S6zU+39o1s9wSDcBAqLNpPB2eDL8qqXKZa/AQTwCiitk9aDB1KZB +DzejoqtrtCK1WnCW7oB7mQIq+/txSyLgv1UgFijh2aAx0ChmMnb2WbeZAQz/5ids +ixMfZiRofZVJIjdNNe5kIBcc9uthoyLw3x16nLT3zrATtBSDA
svn commit: r31911 - /dev/spark/v2.2.3-rc1-bin/ /release/spark/spark-2.2.3/
Author: dbtsai Date: Sat Jan 12 00:14:38 2019 New Revision: 31911 Log: Publish Apache Spark 2.2.3 Added: release/spark/spark-2.2.3/ - copied from r31910, dev/spark/v2.2.3-rc1-bin/ Removed: dev/spark/v2.2.3-rc1-bin/ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31910 - /dev/spark/v2.4.0-rc5-docs/
Author: dongjoon Date: Fri Jan 11 23:29:23 2019 New Revision: 31910 Log: Remove old 2.4.0-rc5 docs Removed: dev/spark/v2.4.0-rc5-docs/ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31909 - in /dev/spark/2.4.1-SNAPSHOT-2019_01_11_15_03-bbf61eb-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 23:18:01 2019 New Revision: 31909 Log: Apache Spark 2.4.1-SNAPSHOT-2019_01_11_15_03-bbf61eb docs [This commit notification would consist of 1476 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31904 - in /dev/spark/3.0.0-SNAPSHOT-2019_01_11_12_56-ae382c9-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 21:08:21 2019 New Revision: 31904 Log: Apache Spark 3.0.0-SNAPSHOT-2019_01_11_12_56-ae382c9 docs [This commit notification would consist of 1775 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs
This is an automated email from the ASF dual-hosted git repository. zsxwing pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new bbf61eb [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs bbf61eb is described below commit bbf61eb4222d7b46e71dc91eeedf82d27226fc2c Author: Mukul Murthy AuthorDate: Fri Jan 11 11:46:14 2019 -0800 [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs ## What changes were proposed in this pull request? Fix race condition where streams can have unexpected conf values. New streaming queries should run with isolated SparkSessions so that they aren't affected by conf updates after they are started. In StreamExecution, the parent SparkSession is cloned and used to run each batch, but this cloning happens in a separate thread and may happen after DataStreamWriter.start() returns. If a stream is started and a conf key is set immediately after, the stream is likely to have the new value. ## How was this patch tested? New unit test that fails prior to the production change and passes with it. Please review http://spark.apache.org/contributing.html before opening a pull request. Closes #23513 from mukulmurthy/26586. Authored-by: Mukul Murthy Signed-off-by: Shixiong Zhu (cherry picked from commit ae382c94dd10ff494dde4de44e66182bf6dbe8f8) Signed-off-by: Shixiong Zhu --- .../sql/execution/streaming/StreamExecution.scala | 5 +++-- .../test/DataStreamReaderWriterSuite.scala | 24 ++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala index de33844..c1aa98a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala @@ -176,6 +176,9 @@ abstract class StreamExecution( lazy val streamMetrics = new MetricsReporter( this, s"spark.streaming.${Option(name).getOrElse(id)}") + /** Isolated spark session to run the batches with. */ + private val sparkSessionForStream = sparkSession.cloneSession() + /** * The thread that runs the micro-batches of this stream. Note that this thread must be * [[org.apache.spark.util.UninterruptibleThread]] to workaround KAFKA-1894: interrupting a @@ -265,8 +268,6 @@ abstract class StreamExecution( // force initialization of the logical plan so that the sources can be created logicalPlan - // Isolated spark session to run the batches with. - val sparkSessionForStream = sparkSession.cloneSession() // Adaptive execution can change num shuffle partitions, disallow sparkSessionForStream.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false") // Disable cost-based join optimization as we do not want stateful operations to be rearranged diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala index 8212fb9..569114a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.streaming.test import java.io.File +import java.util.ConcurrentModificationException import java.util.Locale import java.util.concurrent.TimeUnit @@ -651,4 +652,27 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter { LastOptions.clear() } + + test("SPARK-26586: Streams should have isolated confs") { +import testImplicits._ +val input = MemoryStream[Int] +input.addData(1 to 10) +spark.conf.set("testKey1", 0) +val queries = (1 to 10).map { i => + spark.conf.set("testKey1", i) + input.toDF().writeStream +.foreachBatch { (df: Dataset[Row], id: Long) => + val v = df.sparkSession.conf.get("testKey1").toInt + if (i != v) { +throw new ConcurrentModificationException(s"Stream $i has the wrong conf value $v") + } +} +.start() +} +try { + queries.foreach(_.processAllAvailable()) +} finally { + queries.foreach(_.stop()) +} + } } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs
This is an automated email from the ASF dual-hosted git repository. zsxwing pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new ae382c9 [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs ae382c9 is described below commit ae382c94dd10ff494dde4de44e66182bf6dbe8f8 Author: Mukul Murthy AuthorDate: Fri Jan 11 11:46:14 2019 -0800 [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs ## What changes were proposed in this pull request? Fix race condition where streams can have unexpected conf values. New streaming queries should run with isolated SparkSessions so that they aren't affected by conf updates after they are started. In StreamExecution, the parent SparkSession is cloned and used to run each batch, but this cloning happens in a separate thread and may happen after DataStreamWriter.start() returns. If a stream is started and a conf key is set immediately after, the stream is likely to have the new value. ## How was this patch tested? New unit test that fails prior to the production change and passes with it. Please review http://spark.apache.org/contributing.html before opening a pull request. Closes #23513 from mukulmurthy/26586. Authored-by: Mukul Murthy Signed-off-by: Shixiong Zhu --- .../sql/execution/streaming/StreamExecution.scala | 5 +++-- .../test/DataStreamReaderWriterSuite.scala | 24 ++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala index 83824f4..90f7b47 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala @@ -181,6 +181,9 @@ abstract class StreamExecution( lazy val streamMetrics = new MetricsReporter( this, s"spark.streaming.${Option(name).getOrElse(id)}") + /** Isolated spark session to run the batches with. */ + private val sparkSessionForStream = sparkSession.cloneSession() + /** * The thread that runs the micro-batches of this stream. Note that this thread must be * [[org.apache.spark.util.UninterruptibleThread]] to workaround KAFKA-1894: interrupting a @@ -270,8 +273,6 @@ abstract class StreamExecution( // force initialization of the logical plan so that the sources can be created logicalPlan - // Isolated spark session to run the batches with. - val sparkSessionForStream = sparkSession.cloneSession() // Adaptive execution can change num shuffle partitions, disallow sparkSessionForStream.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false") // Disable cost-based join optimization as we do not want stateful operations to be rearranged diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala index 4d3a54a..74ea0bf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.streaming.test import java.io.File +import java.util.ConcurrentModificationException import java.util.Locale import java.util.concurrent.TimeUnit @@ -651,4 +652,27 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter { LastOptions.clear() } + + test("SPARK-26586: Streams should have isolated confs") { +import testImplicits._ +val input = MemoryStream[Int] +input.addData(1 to 10) +spark.conf.set("testKey1", 0) +val queries = (1 to 10).map { i => + spark.conf.set("testKey1", i) + input.toDF().writeStream +.foreachBatch { (df: Dataset[Row], id: Long) => + val v = df.sparkSession.conf.get("testKey1").toInt + if (i != v) { +throw new ConcurrentModificationException(s"Stream $i has the wrong conf value $v") + } +} +.start() +} +try { + queries.foreach(_.processAllAvailable()) +} finally { + queries.foreach(_.stop()) +} + } } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26551][SQL] Fix schema pruning error when selecting one complex field and having is not null predicate on another one
This is an automated email from the ASF dual-hosted git repository. dbtsai pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new 0e5b316 [SPARK-26551][SQL] Fix schema pruning error when selecting one complex field and having is not null predicate on another one 0e5b316 is described below commit 0e5b316dec8e3439a4020b3e5def2f5f2c0be5f2 Author: Liang-Chi Hsieh AuthorDate: Fri Jan 11 19:23:32 2019 + [SPARK-26551][SQL] Fix schema pruning error when selecting one complex field and having is not null predicate on another one ## What changes were proposed in this pull request? Schema pruning has errors when selecting one complex field and having is not null predicate on another one: ```scala val query = sql("select * from contacts") .where("name.middle is not null") .select( "id", "name.first", "name.middle", "name.last" ) .where("last = 'Jones'") .select(count("id")) ``` ``` java.lang.IllegalArgumentException: middle does not exist. Available: last [info] at org.apache.spark.sql.types.StructType.$anonfun$fieldIndex$1(StructType.scala:303) [info] at scala.collection.immutable.Map$Map1.getOrElse(Map.scala:119) [info] at org.apache.spark.sql.types.StructType.fieldIndex(StructType.scala:302) [info] at org.apache.spark.sql.execution.ProjectionOverSchema.$anonfun$getProjection$6(ProjectionOverSchema.scala:58) [info] at scala.Option.map(Option.scala:163) [info] at org.apache.spark.sql.execution.ProjectionOverSchema.getProjection(ProjectionOverSchema.scala:56) [info] at org.apache.spark.sql.execution.ProjectionOverSchema.unapply(ProjectionOverSchema.scala:32) [info] at org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaPruning$$anonfun$$nestedInanonfun$buildNewProjection$1$1.applyOrElse(Parque tSchemaPruning.scala:153) ``` ## How was this patch tested? Added tests. Closes #23474 from viirya/SPARK-26551. Authored-by: Liang-Chi Hsieh Signed-off-by: DB Tsai (cherry picked from commit 50ebf3a43b84c8538ec60437189221c2c527990b) Signed-off-by: DB Tsai --- .../datasources/parquet/ParquetSchemaPruning.scala | 34 +++- .../parquet/ParquetSchemaPruningSuite.scala| 36 ++ 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala index 91080b1..840fcae 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala @@ -116,10 +116,28 @@ private[sql] object ParquetSchemaPruning extends Rule[LogicalPlan] { // For example, for a query `SELECT name.first FROM contacts WHERE name IS NOT NULL`, // we don't need to read nested fields of `name` struct other than `first` field. val (rootFields, optRootFields) = (projectionRootFields ++ filterRootFields) - .distinct.partition(_.contentAccessed) + .distinct.partition(!_.prunedIfAnyChildAccessed) optRootFields.filter { opt => - !rootFields.exists(_.field.name == opt.field.name) + !rootFields.exists { root => +root.field.name == opt.field.name && { + // Checking if current optional root field can be pruned. + // For each required root field, we merge it with the optional root field: + // 1. If this optional root field has nested fields and any nested field of it is used + //in the query, the merged field type must equal to the optional root field type. + //We can prune this optional root field. For example, for optional root field + //`struct>`, if its field + //`struct>` is used, we don't need to add this optional + //root field. + // 2. If this optional root field has no nested fields, the merged field type equals + //to the optional root field only if they are the same. If they are, we can prune + //this optional root field too. + val rootFieldType = StructType(Array(root.field)) + val optFieldType = StructType(Array(opt.field)) + val merged = optFieldType.merge(rootFieldType) + merged.sameType(optFieldType) +} + } } ++ rootFields } @@ -213,11 +231,11 @@ private[sql] object ParquetSchemaPruning extends Rule[LogicalPlan] { // don't actually use any nested fields. These root field accesses might be excluded later // if there are any nested fields acce
[spark] branch master updated: [SPARK-26551][SQL] Fix schema pruning error when selecting one complex field and having is not null predicate on another one
This is an automated email from the ASF dual-hosted git repository. dbtsai pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new 50ebf3a [SPARK-26551][SQL] Fix schema pruning error when selecting one complex field and having is not null predicate on another one 50ebf3a is described below commit 50ebf3a43b84c8538ec60437189221c2c527990b Author: Liang-Chi Hsieh AuthorDate: Fri Jan 11 19:23:32 2019 + [SPARK-26551][SQL] Fix schema pruning error when selecting one complex field and having is not null predicate on another one ## What changes were proposed in this pull request? Schema pruning has errors when selecting one complex field and having is not null predicate on another one: ```scala val query = sql("select * from contacts") .where("name.middle is not null") .select( "id", "name.first", "name.middle", "name.last" ) .where("last = 'Jones'") .select(count("id")) ``` ``` java.lang.IllegalArgumentException: middle does not exist. Available: last [info] at org.apache.spark.sql.types.StructType.$anonfun$fieldIndex$1(StructType.scala:303) [info] at scala.collection.immutable.Map$Map1.getOrElse(Map.scala:119) [info] at org.apache.spark.sql.types.StructType.fieldIndex(StructType.scala:302) [info] at org.apache.spark.sql.execution.ProjectionOverSchema.$anonfun$getProjection$6(ProjectionOverSchema.scala:58) [info] at scala.Option.map(Option.scala:163) [info] at org.apache.spark.sql.execution.ProjectionOverSchema.getProjection(ProjectionOverSchema.scala:56) [info] at org.apache.spark.sql.execution.ProjectionOverSchema.unapply(ProjectionOverSchema.scala:32) [info] at org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaPruning$$anonfun$$nestedInanonfun$buildNewProjection$1$1.applyOrElse(Parque tSchemaPruning.scala:153) ``` ## How was this patch tested? Added tests. Closes #23474 from viirya/SPARK-26551. Authored-by: Liang-Chi Hsieh Signed-off-by: DB Tsai --- .../datasources/parquet/ParquetSchemaPruning.scala | 34 +++- .../parquet/ParquetSchemaPruningSuite.scala| 36 ++ 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala index 91080b1..840fcae 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala @@ -116,10 +116,28 @@ private[sql] object ParquetSchemaPruning extends Rule[LogicalPlan] { // For example, for a query `SELECT name.first FROM contacts WHERE name IS NOT NULL`, // we don't need to read nested fields of `name` struct other than `first` field. val (rootFields, optRootFields) = (projectionRootFields ++ filterRootFields) - .distinct.partition(_.contentAccessed) + .distinct.partition(!_.prunedIfAnyChildAccessed) optRootFields.filter { opt => - !rootFields.exists(_.field.name == opt.field.name) + !rootFields.exists { root => +root.field.name == opt.field.name && { + // Checking if current optional root field can be pruned. + // For each required root field, we merge it with the optional root field: + // 1. If this optional root field has nested fields and any nested field of it is used + //in the query, the merged field type must equal to the optional root field type. + //We can prune this optional root field. For example, for optional root field + //`struct>`, if its field + //`struct>` is used, we don't need to add this optional + //root field. + // 2. If this optional root field has no nested fields, the merged field type equals + //to the optional root field only if they are the same. If they are, we can prune + //this optional root field too. + val rootFieldType = StructType(Array(root.field)) + val optFieldType = StructType(Array(opt.field)) + val merged = optFieldType.merge(rootFieldType) + merged.sameType(optFieldType) +} + } } ++ rootFields } @@ -213,11 +231,11 @@ private[sql] object ParquetSchemaPruning extends Rule[LogicalPlan] { // don't actually use any nested fields. These root field accesses might be excluded later // if there are any nested fields accesses in the query plan. case IsNotNull(SelectedField(field)) => -RootField(field, derivedFromA
svn commit: r31902 - in /dev/spark/2.4.1-SNAPSHOT-2019_01_11_10_40-b9eb0e8-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 18:57:29 2019 New Revision: 31902 Log: Apache Spark 2.4.1-SNAPSHOT-2019_01_11_10_40-b9eb0e8 docs [This commit notification would consist of 1476 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31901 - in /dev/spark/2.3.3-SNAPSHOT-2019_01_11_10_40-87c2c11-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 18:55:32 2019 New Revision: 31901 Log: Apache Spark 2.3.3-SNAPSHOT-2019_01_11_10_40-87c2c11 docs [This commit notification would consist of 1443 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26482][CORE] Use ConfigEntry for hardcoded configs for ui categories
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new d9e4cf6 [SPARK-26482][CORE] Use ConfigEntry for hardcoded configs for ui categories d9e4cf6 is described below commit d9e4cf67c06b2d6daa4cd24b056e33dfb5eb35f5 Author: Jungtaek Lim (HeartSaVioR) AuthorDate: Fri Jan 11 10:18:07 2019 -0800 [SPARK-26482][CORE] Use ConfigEntry for hardcoded configs for ui categories ## What changes were proposed in this pull request? The PR makes hardcoded configs below to use `ConfigEntry`. * spark.ui * spark.ssl * spark.authenticate * spark.master.rest * spark.master.ui * spark.metrics * spark.admin * spark.modify.acl This patch doesn't change configs which are not relevant to SparkConf (e.g. system properties). ## How was this patch tested? Existing tests. Closes #23423 from HeartSaVioR/SPARK-26466. Authored-by: Jungtaek Lim (HeartSaVioR) Signed-off-by: Marcelo Vanzin --- .../scala/org/apache/spark/SecurityManager.scala | 54 +++ .../main/scala/org/apache/spark/SparkContext.scala | 5 +- .../apache/spark/deploy/LocalSparkCluster.scala| 4 +- .../org/apache/spark/deploy/SparkSubmit.scala | 1 + .../spark/deploy/history/FsHistoryProvider.scala | 47 +++--- .../spark/deploy/history/HistoryServer.scala | 10 +- .../org/apache/spark/deploy/master/Master.scala| 10 +- .../spark/deploy/master/MasterArguments.scala | 5 +- .../spark/deploy/master/ui/MasterWebUI.scala | 3 +- .../spark/deploy/rest/StandaloneRestServer.scala | 2 +- .../spark/deploy/worker/ExecutorRunner.scala | 3 +- .../org/apache/spark/deploy/worker/Worker.scala| 3 +- .../org/apache/spark/internal/config/History.scala | 12 +- .../org/apache/spark/internal/config/UI.scala | 145 +++ .../org/apache/spark/internal/config/package.scala | 50 +++ .../org/apache/spark/metrics/MetricsConfig.scala | 3 +- .../org/apache/spark/ui/ConsoleProgressBar.scala | 4 +- .../org/apache/spark/ui/HttpSecurityFilter.scala | 2 +- .../scala/org/apache/spark/ui/JettyUtils.scala | 4 +- .../main/scala/org/apache/spark/ui/SparkUI.scala | 6 +- .../org/apache/spark/ui/exec/ExecutorsTab.scala| 3 +- .../scala/org/apache/spark/ui/jobs/StagePage.scala | 3 +- .../main/scala/org/apache/spark/util/Utils.scala | 4 +- .../scala/org/apache/spark/CheckpointSuite.scala | 3 +- .../org/apache/spark/SecurityManagerSuite.scala| 160 +++-- .../scala/org/apache/spark/SparkContextSuite.scala | 5 +- .../org/apache/spark/deploy/SparkSubmitSuite.scala | 13 +- .../deploy/history/FsHistoryProviderSuite.scala| 23 +-- .../spark/deploy/history/HistoryServerSuite.scala | 7 +- .../apache/spark/deploy/master/MasterSuite.scala | 10 +- .../org/apache/spark/executor/ExecutorSuite.scala | 3 +- .../spark/launcher/LauncherBackendSuite.scala | 3 +- .../apache/spark/metrics/MetricsConfigSuite.scala | 9 +- .../apache/spark/metrics/MetricsSystemSuite.scala | 2 +- .../netty/NettyBlockTransferSecuritySuite.scala| 27 ++-- .../scala/org/apache/spark/rpc/RpcEnvSuite.scala | 29 ++-- .../apache/spark/scheduler/TaskContextSuite.scala | 3 +- .../storage/BlockManagerReplicationSuite.scala | 4 +- .../apache/spark/ui/HttpSecurityFilterSuite.scala | 10 +- .../org/apache/spark/ui/UISeleniumSuite.scala | 13 +- .../test/scala/org/apache/spark/ui/UISuite.scala | 3 +- .../k8s/features/BasicDriverFeatureStep.scala | 1 + .../k8s/features/BasicDriverFeatureStepSuite.scala | 3 +- .../integrationtest/KubernetesTestComponents.scala | 5 +- .../spark/deploy/yarn/ApplicationMaster.scala | 5 +- .../scheduler/cluster/YarnSchedulerBackend.scala | 5 +- .../spark/deploy/yarn/YarnClusterSuite.scala | 4 +- .../deploy/yarn/YarnSparkHadoopUtilSuite.scala | 10 +- .../spark/sql/SparkSessionBuilderSuite.scala | 3 +- .../sql/execution/ExchangeCoordinatorSuite.scala | 3 +- .../benchmark/DataSourceReadBenchmark.scala| 3 +- .../benchmark/FilterPushdownBenchmark.scala| 3 +- .../sql/hive/thriftserver/HiveThriftServer2.scala | 5 +- .../org/apache/spark/sql/hive/test/TestHive.scala | 3 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 23 +-- .../sql/hive/execution/ConcurrentHiveSuite.scala | 3 +- .../org/apache/spark/streaming/Checkpoint.scala| 3 +- .../apache/spark/streaming/StreamingContext.scala | 3 +- .../org/apache/spark/streaming/ReceiverSuite.scala | 3 +- .../apache/spark/streaming/UISeleniumSuite.scala | 3 +- 60 files changed, 496 insertions(+), 305 deletions(-) diff --git a/core/src/main/scala/org/apache/spar
[spark] branch branch-2.3 updated: [SPARK-26576][SQL] Broadcast hint not applied to partitioned table
This is an automated email from the ASF dual-hosted git repository. lixiao pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-2.3 by this push: new 87c2c11 [SPARK-26576][SQL] Broadcast hint not applied to partitioned table 87c2c11 is described below commit 87c2c11e742a8b35699f68ec2002f817c56bef87 Author: John Zhuge AuthorDate: Fri Jan 11 09:21:13 2019 -0800 [SPARK-26576][SQL] Broadcast hint not applied to partitioned table ## What changes were proposed in this pull request? Make sure broadcast hint is applied to partitioned tables. Since the issue exists in branch 2.0 to 2.4, but not in master, I created this PR for branch-2.4. ## How was this patch tested? - A new unit test in PruneFileSourcePartitionsSuite - Unit test suites touched by SPARK-14581: JoinOptimizationSuite, FilterPushdownSuite, ColumnPruningSuite, and PruneFiltersSuite cloud-fan davies rxin Closes #23507 from jzhuge/SPARK-26576. Authored-by: John Zhuge Signed-off-by: gatorsmile (cherry picked from commit b9eb0e85de3317a7f4c89a90082f7793b645c6ea) Signed-off-by: gatorsmile --- .../apache/spark/sql/catalyst/planning/patterns.scala | 3 --- .../execution/PruneFileSourcePartitionsSuite.scala| 19 ++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index cc391aa..a91063b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -65,9 +65,6 @@ object PhysicalOperation extends PredicateHelper { val substitutedCondition = substitute(aliases)(condition) (fields, filters ++ splitConjunctivePredicates(substitutedCondition), other, aliases) - case h: ResolvedHint => -collectProjectsAndFilters(h.child) - case other => (None, Nil, other, Map.empty) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala index 9438418..8a9adf7 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala @@ -17,15 +17,20 @@ package org.apache.spark.sql.hive.execution +import org.scalatest.Matchers._ + import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ -import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, ResolvedHint} import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec +import org.apache.spark.sql.functions.broadcast import org.apache.spark.sql.hive.test.TestHiveSingleton +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SQLTestUtils import org.apache.spark.sql.types.StructType @@ -91,4 +96,16 @@ class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with Te assert(size2 < tableStats.get.sizeInBytes) } } + + test("SPARK-26576 Broadcast hint not applied to partitioned table") { +withTable("tbl") { + withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { +spark.range(10).selectExpr("id", "id % 3 as p").write.partitionBy("p").saveAsTable("tbl") +val df = spark.table("tbl") +val qe = df.join(broadcast(df), "p").queryExecution +qe.optimizedPlan.collect { case _: ResolvedHint => } should have size 1 +qe.sparkPlan.collect { case j: BroadcastHashJoinExec => j } should have size 1 + } +} + } } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26576][SQL] Broadcast hint not applied to partitioned table
This is an automated email from the ASF dual-hosted git repository. lixiao pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-2.4 by this push: new b9eb0e8 [SPARK-26576][SQL] Broadcast hint not applied to partitioned table b9eb0e8 is described below commit b9eb0e85de3317a7f4c89a90082f7793b645c6ea Author: John Zhuge AuthorDate: Fri Jan 11 09:21:13 2019 -0800 [SPARK-26576][SQL] Broadcast hint not applied to partitioned table ## What changes were proposed in this pull request? Make sure broadcast hint is applied to partitioned tables. Since the issue exists in branch 2.0 to 2.4, but not in master, I created this PR for branch-2.4. ## How was this patch tested? - A new unit test in PruneFileSourcePartitionsSuite - Unit test suites touched by SPARK-14581: JoinOptimizationSuite, FilterPushdownSuite, ColumnPruningSuite, and PruneFiltersSuite cloud-fan davies rxin Closes #23507 from jzhuge/SPARK-26576. Authored-by: John Zhuge Signed-off-by: gatorsmile --- .../apache/spark/sql/catalyst/planning/patterns.scala | 3 --- .../execution/PruneFileSourcePartitionsSuite.scala| 19 ++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index 84be677..d91b890 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -66,9 +66,6 @@ object PhysicalOperation extends PredicateHelper { val substitutedCondition = substitute(aliases)(condition) (fields, filters ++ splitConjunctivePredicates(substitutedCondition), other, aliases) - case h: ResolvedHint => -collectProjectsAndFilters(h.child) - case other => (None, Nil, other, Map.empty) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala index 9438418..8a9adf7 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala @@ -17,15 +17,20 @@ package org.apache.spark.sql.hive.execution +import org.scalatest.Matchers._ + import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ -import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, ResolvedHint} import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec +import org.apache.spark.sql.functions.broadcast import org.apache.spark.sql.hive.test.TestHiveSingleton +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SQLTestUtils import org.apache.spark.sql.types.StructType @@ -91,4 +96,16 @@ class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with Te assert(size2 < tableStats.get.sizeInBytes) } } + + test("SPARK-26576 Broadcast hint not applied to partitioned table") { +withTable("tbl") { + withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { +spark.range(10).selectExpr("id", "id % 3 as p").write.partitionBy("p").saveAsTable("tbl") +val df = spark.table("tbl") +val qe = df.join(broadcast(df), "p").queryExecution +qe.optimizedPlan.collect { case _: ResolvedHint => } should have size 1 +qe.sparkPlan.collect { case j: BroadcastHashJoinExec => j } should have size 1 + } +} + } } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31900 - in /dev/spark/3.0.0-SNAPSHOT-2019_01_11_08_30-51a6ba0-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 16:43:16 2019 New Revision: 31900 Log: Apache Spark 3.0.0-SNAPSHOT-2019_01_11_08_30-51a6ba0 docs [This commit notification would consist of 1775 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26503][CORE] Get rid of spark.sql.legacy.timeParser.enabled
This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new 51a6ba0 [SPARK-26503][CORE] Get rid of spark.sql.legacy.timeParser.enabled 51a6ba0 is described below commit 51a6ba0181a013f2b62b47184785a8b6f6a78f12 Author: Sean Owen AuthorDate: Fri Jan 11 08:53:12 2019 -0600 [SPARK-26503][CORE] Get rid of spark.sql.legacy.timeParser.enabled ## What changes were proposed in this pull request? Per discussion in #23391 (comment) this proposes to just remove the old pre-Spark-3 time parsing behavior. This is a rebase of https://github.com/apache/spark/pull/23411 ## How was this patch tested? Existing tests. Closes #23495 from srowen/SPARK-26503.2. Authored-by: Sean Owen Signed-off-by: Sean Owen --- .../spark/sql/catalyst/util/DateFormatter.scala| 43 + .../sql/catalyst/util/TimestampFormatter.scala | 37 + .../org/apache/spark/sql/internal/SQLConf.scala| 9 - .../sql/catalyst/json/JsonInferSchemaSuite.scala | 77 - .../sql/execution/datasources/json/JsonSuite.scala | 183 - .../spark/sql/sources/HadoopFsRelationTest.scala | 107 ++-- 6 files changed, 120 insertions(+), 336 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala index db92552..c47b087 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala @@ -20,12 +20,6 @@ package org.apache.spark.sql.catalyst.util import java.time.{Instant, ZoneId} import java.util.Locale -import scala.util.Try - -import org.apache.commons.lang3.time.FastDateFormat - -import org.apache.spark.sql.internal.SQLConf - sealed trait DateFormatter extends Serializable { def parse(s: String): Int // returns days since epoch def format(days: Int): String @@ -56,43 +50,8 @@ class Iso8601DateFormatter( } } -class LegacyDateFormatter(pattern: String, locale: Locale) extends DateFormatter { - @transient - private lazy val format = FastDateFormat.getInstance(pattern, locale) - - override def parse(s: String): Int = { -val milliseconds = format.parse(s).getTime -DateTimeUtils.millisToDays(milliseconds) - } - - override def format(days: Int): String = { -val date = DateTimeUtils.toJavaDate(days) -format.format(date) - } -} - -class LegacyFallbackDateFormatter( -pattern: String, -locale: Locale) extends LegacyDateFormatter(pattern, locale) { - override def parse(s: String): Int = { -Try(super.parse(s)).orElse { - // If it fails to parse, then tries the way used in 2.0 and 1.x for backwards - // compatibility. - Try(DateTimeUtils.millisToDays(DateTimeUtils.stringToTime(s).getTime)) -}.getOrElse { - // In Spark 1.5.0, we store the data as number of days since epoch in string. - // So, we just convert it to Int. - s.toInt -} - } -} - object DateFormatter { def apply(format: String, locale: Locale): DateFormatter = { -if (SQLConf.get.legacyTimeParserEnabled) { - new LegacyFallbackDateFormatter(format, locale) -} else { - new Iso8601DateFormatter(format, locale) -} +new Iso8601DateFormatter(format, locale) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala index 8042099..10c73b2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala @@ -23,12 +23,6 @@ import java.time.format.DateTimeParseException import java.time.temporal.TemporalQueries import java.util.{Locale, TimeZone} -import scala.util.Try - -import org.apache.commons.lang3.time.FastDateFormat - -import org.apache.spark.sql.internal.SQLConf - sealed trait TimestampFormatter extends Serializable { /** * Parses a timestamp in a string and converts it to microseconds. @@ -79,37 +73,8 @@ class Iso8601TimestampFormatter( } } -class LegacyTimestampFormatter( -pattern: String, -timeZone: TimeZone, -locale: Locale) extends TimestampFormatter { - @transient - private lazy val format = FastDateFormat.getInstance(pattern, timeZone, locale) - - protected def toMillis(s: String): Long = format.parse(s).getTime - - override def parse(s: String): Long = toMillis(s) * DateTimeUtils.MICROS_PER_MILLIS - - override def format(us: Long): String = { -format.format(DateTimeUtils.toJavaTimestamp(us)) - } -} - -class Lega