svn commit: r35381 - /dev/spark/KEYS

2019-08-26 Thread kiszk
Author: kiszk
Date: Mon Aug 26 17:18:45 2019
New Revision: 35381

Log:
Update KEYS

Modified:
dev/spark/KEYS

Modified: dev/spark/KEYS
==
--- dev/spark/KEYS (original)
+++ dev/spark/KEYS Mon Aug 26 17:18:45 2019
@@ -993,12 +993,12 @@ ZTFPNYvCMMHM8A==
 =PEdD
 -END PGP PUBLIC KEY BLOCK-
 
-pub   rsa4096/7F0FEF75 2019-08-19 [SC]
-uid [ultimate] Kazuaki Ishizaki (CODE SIGNING KEY) 
-sub   rsa4096/7C3AEC68 2019-08-19 [E]
+pub   4096R/7F0FEF75 2019-08-19
+uid  Kazuaki Ishizaki (CODE SIGNING KEY) 
+sub   4096R/7C3AEC68 2019-08-19
 
 -BEGIN PGP PUBLIC KEY BLOCK-
-Version: GnuPG v2
+Version: GnuPG v1
 
 mQINBF1a3YcBEAC7I6f1jWpY9WlJBkbwvLneYBjnD2BRwG1eKjkz49aUXVKkx4Du
 XB7b+agbhWL7EIPjQHVJf0RVGochOujKfcPxOz5bZwAV078EbsJpiAYIAeVEimQF
@@ -1049,3 +1049,4 @@ au2shXGZFmo4V56uCJ5HqZTJJZaMceQx7u8uqZbh
 XJ5Dp1pqv9DC6cl9vLSHctRrM2kG
 =mQLW
 -END PGP PUBLIC KEY BLOCK-
+



-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org



svn commit: r35371 - in /dev/spark/v2.3.4-rc1-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _site/api/java/org/apache/spark

2019-08-26 Thread kiszk
Author: kiszk
Date: Mon Aug 26 09:54:45 2019
New Revision: 35371

Log:
Apache Spark v2.3.4-rc1 docs


[This commit notification would consist of 1447 parts, 
which exceeds the limit of 50 ones, so it was shortened to the summary.]

-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org



svn commit: r35370 - /dev/spark/v2.3.4-rc1-bin/

2019-08-26 Thread kiszk
Author: kiszk
Date: Mon Aug 26 09:00:20 2019
New Revision: 35370

Log:
Apache Spark v2.3.4-rc1

Added:
dev/spark/v2.3.4-rc1-bin/
dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz   (with props)
dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz.asc
dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz.sha512
dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz   (with props)
dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz.asc
dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz.sha512
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-hadoop2.6.tgz   (with props)
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-hadoop2.6.tgz.asc
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-hadoop2.6.tgz.sha512
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-hadoop2.7.tgz   (with props)
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-hadoop2.7.tgz.asc
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-hadoop2.7.tgz.sha512
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-without-hadoop.tgz   (with props)
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-without-hadoop.tgz.asc
dev/spark/v2.3.4-rc1-bin/spark-2.3.4-bin-without-hadoop.tgz.sha512
dev/spark/v2.3.4-rc1-bin/spark-2.3.4.tgz   (with props)
dev/spark/v2.3.4-rc1-bin/spark-2.3.4.tgz.asc
dev/spark/v2.3.4-rc1-bin/spark-2.3.4.tgz.sha512

Added: dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz
==
Binary file - no diff available.

Propchange: dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz
--
svn:mime-type = application/octet-stream

Added: dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz.asc
==
--- dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz.asc (added)
+++ dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz.asc Mon Aug 26 09:00:20 2019
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+
+iQJFBAABCgAvFiEEgFK1nI3grK++CMoV5JoEbH8P73UFAl1jmEMRHGtpc3prQGFw
+YWNoZS5vcmcACgkQ5JoEbH8P73Xmog//Qj/814bac4xbMnvsmEQyA9RfIRfv2i2T
+jJNh2jHiwUefV4Wd+vXy+5YXSW/A9y8MOgBHXRRbdsv+wzuaccy+SayFCg8gWXOb
+CihXw5gc3sUswIRFlxSsjwL0xkcqsxLkmPQtg7eOjIlq1LS3ynLzRPbnOov71que
+45dHOnZi1PIEonhQiIgwWEVQiEyUQk0cBjiWDgprrZe4sZStHm0IbTsPJNAmJ3qX
+KUZddOfEwmzm4u44oVYR1Z88YrRT/F7LOB8cNvCT/JLGNkn0Sf1DNN42E8gcSUyJ
+EWU8cgjy0j2kBYLVdO123Qo/V/HJ8XJUrz9fd3p89ZX6z+q66lCHVypg9Chku/OI
+CZ3pnTcBbaUKTMjB0R+r8Yj6OuIyEx95oMABoOi8ye98xrRSw7kEZ1CVIPHUiiDu
+oZdP8XQyg5sLda4qFAs/6AGY9jXTDojk46zE+MqJ7jefXVn8lvdwWKVhVaIyZYDs
+bDm9lGFTlXyakX0qxeMC7dCNkINMuXgQBZpMb+HMlUWDurneWA3IjwtzvJd2AfiU
+ZvBo7Gzv6eBjbcJ9eaG3UXEv25dt3sK56fV7/7Jh+9LVLIZDIIdNwV+YDDmVX4HF
+f7KHtaWIfQpy9lbHQqLuf6DikxntT3jIV1NUg7UbkWKrKg1wuBUozmiX4aqRTAnQ
+4MKVJZuZmzU=
+=h0w6
+-END PGP SIGNATURE-

Added: dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz.sha512
==
--- dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz.sha512 (added)
+++ dev/spark/v2.3.4-rc1-bin/SparkR_2.3.4.tar.gz.sha512 Mon Aug 26 09:00:20 2019
@@ -0,0 +1,3 @@
+SparkR_2.3.4.tar.gz: 09173710 547AFB95 417F908E 8057C0FC C78C41E7 17F64233
+ 440B8E58 B43AEB9F 15B9F5CC 1972750B 5A60D3BA AA702D22
+ 7AEF3D79 495C323A 803F9F54 7EE5DB13

Added: dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz
==
Binary file - no diff available.

Propchange: dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz
--
svn:mime-type = application/octet-stream

Added: dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz.asc
==
--- dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz.asc (added)
+++ dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz.asc Mon Aug 26 09:00:20 2019
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+
+iQJFBAABCgAvFiEEgFK1nI3grK++CMoV5JoEbH8P73UFAl1jmEURHGtpc3prQGFw
+YWNoZS5vcmcACgkQ5JoEbH8P73Vd5BAAmzqGMEWC50Eet0e8Jpl2IT77dRfY+6zz
+mj5Nf/4tAFZ8eys7rbr4qKkNoqV3+cfytmNQSC/va6hbb0ioOB19uhvQqUe+OXaF
+93enkUjV0FGFwUgh8dD6x+9V0hAQ8lFA6V0Y1NYBa53t5xJFAJSrpVcXv/Af4y0A
+p8vyZN9Fea15RQykBQBjszhaQuh8nMqZbZjd19Kmwk2Dfe+ABFRjljpwuZt/paaX
+qZaaRpgVj30JmxkbKtXfVeDW6IstcntBJdmCoA2wwcgZmn7vTu5Fu1dd4xXhLq/H
+LIlIJXTxzPEmZuHmt7kNMYrj/M1ulPj2GFI0Cm4zg0uw9wbA01VjQ79sFuS6n0HC
+cC2JGm8inG6CHmWrZ4peBM1BxefL7yhfWYROQm2jwhfRpeI5EcmHkUlhoK8w6+F6
+2i6H187IXizL0UQjMcQu8WiGHtlcvTPrMP3BHwKuALZlgnrXFfcIrXD+oE1AakK3
+vVwTSt48RxX7dp89pRGx3bxS8zaIsh5bG2GlgYVxx8EtAyq6hK9nzHulLAcY1hS9
+A/8j8lQKZlCtDmr+JkOhcGuZsiUtB2elMwsMJmFn+qBbu0R+AT08x5kAILBNDkp6
+iN8xRoOpgVvcqzHZvraz7a6OqxfoPtQ53A4xNtT8gFTDs1Kq7jLOvmjntZotreUs
+gJ3741FqslM=
+=hX41
+-END PGP SIGNATURE-

Added: dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4.tar.gz.sha512
==
--- dev/spark/v2.3.4-rc1-bin/pyspark-2.3.4

[spark] 01/01: Preparing Spark release v2.3.4-rc1

2019-08-25 Thread kiszk
This is an automated email from the ASF dual-hosted git repository.

kiszk pushed a commit to tag v2.3.4-rc1
in repository https://gitbox.apache.org/repos/asf/spark.git

commit 8c6f8150f3c6298ff4e1c7e06028f12d7eaf0210
Author: Kazuaki Ishizaki 
AuthorDate: Sun Aug 25 14:38:17 2019 +

Preparing Spark release v2.3.4-rc1
---
 assembly/pom.xml  | 2 +-
 common/kvstore/pom.xml| 2 +-
 common/network-common/pom.xml | 2 +-
 common/network-shuffle/pom.xml| 2 +-
 common/network-yarn/pom.xml   | 2 +-
 common/sketch/pom.xml | 2 +-
 common/tags/pom.xml   | 2 +-
 common/unsafe/pom.xml | 2 +-
 core/pom.xml  | 2 +-
 docs/_config.yml  | 2 +-
 examples/pom.xml  | 2 +-
 external/docker-integration-tests/pom.xml | 2 +-
 external/flume-assembly/pom.xml   | 2 +-
 external/flume-sink/pom.xml   | 2 +-
 external/flume/pom.xml| 2 +-
 external/kafka-0-10-assembly/pom.xml  | 2 +-
 external/kafka-0-10-sql/pom.xml   | 2 +-
 external/kafka-0-10/pom.xml   | 2 +-
 external/kafka-0-8-assembly/pom.xml   | 2 +-
 external/kafka-0-8/pom.xml| 2 +-
 external/kinesis-asl-assembly/pom.xml | 2 +-
 external/kinesis-asl/pom.xml  | 2 +-
 external/spark-ganglia-lgpl/pom.xml   | 2 +-
 graphx/pom.xml| 2 +-
 hadoop-cloud/pom.xml  | 2 +-
 launcher/pom.xml  | 2 +-
 mllib-local/pom.xml   | 2 +-
 mllib/pom.xml | 2 +-
 pom.xml   | 2 +-
 python/pyspark/version.py | 2 +-
 repl/pom.xml  | 2 +-
 resource-managers/kubernetes/core/pom.xml | 2 +-
 resource-managers/mesos/pom.xml   | 2 +-
 resource-managers/yarn/pom.xml| 2 +-
 sql/catalyst/pom.xml  | 2 +-
 sql/core/pom.xml  | 2 +-
 sql/hive-thriftserver/pom.xml | 2 +-
 sql/hive/pom.xml  | 2 +-
 streaming/pom.xml | 2 +-
 tools/pom.xml | 2 +-
 40 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/assembly/pom.xml b/assembly/pom.xml
index 612a1b8..583b1bf 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../pom.xml
   
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index 5547e97..29c2c58 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../../pom.xml
   
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 119dde2..224b229 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../../pom.xml
   
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index dba5224..c7f661e 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../../pom.xml
   
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 56902a3..f33fb99 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../../pom.xml
   
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 5302d95..a642cb2 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../../pom.xml
   
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 232ebfa..29bd7ba 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../../pom.xml
   
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index f0baa2a..03f9b77 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../../pom.xml
   
 
diff --git a/core/pom.xml b/core/pom.xml
index d4f5940..c9c1c7c 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4-SNAPSHOT
+2.3.4
 ../pom.xml
   
 
diff --git a/docs/_config.yml b/docs/_config.yml
index dd46965..f30ff62 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -14,7 +14,7 @@ include:
 
 # These allow

[spark] 01/01: Preparing development version 2.3.5-SNAPSHOT

2019-08-25 Thread kiszk
This is an automated email from the ASF dual-hosted git repository.

kiszk pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git

commit 3fb9e84c7a5ed6c7bde7a6c64cdeda974734dbc5
Author: Kazuaki Ishizaki 
AuthorDate: Sun Aug 25 14:38:22 2019 +

Preparing development version 2.3.5-SNAPSHOT
---
 R/pkg/DESCRIPTION | 2 +-
 assembly/pom.xml  | 2 +-
 common/kvstore/pom.xml| 2 +-
 common/network-common/pom.xml | 2 +-
 common/network-shuffle/pom.xml| 2 +-
 common/network-yarn/pom.xml   | 2 +-
 common/sketch/pom.xml | 2 +-
 common/tags/pom.xml   | 2 +-
 common/unsafe/pom.xml | 2 +-
 core/pom.xml  | 2 +-
 docs/_config.yml  | 4 ++--
 examples/pom.xml  | 2 +-
 external/docker-integration-tests/pom.xml | 2 +-
 external/flume-assembly/pom.xml   | 2 +-
 external/flume-sink/pom.xml   | 2 +-
 external/flume/pom.xml| 2 +-
 external/kafka-0-10-assembly/pom.xml  | 2 +-
 external/kafka-0-10-sql/pom.xml   | 2 +-
 external/kafka-0-10/pom.xml   | 2 +-
 external/kafka-0-8-assembly/pom.xml   | 2 +-
 external/kafka-0-8/pom.xml| 2 +-
 external/kinesis-asl-assembly/pom.xml | 2 +-
 external/kinesis-asl/pom.xml  | 2 +-
 external/spark-ganglia-lgpl/pom.xml   | 2 +-
 graphx/pom.xml| 2 +-
 hadoop-cloud/pom.xml  | 2 +-
 launcher/pom.xml  | 2 +-
 mllib-local/pom.xml   | 2 +-
 mllib/pom.xml | 2 +-
 pom.xml   | 2 +-
 python/pyspark/version.py | 2 +-
 repl/pom.xml  | 2 +-
 resource-managers/kubernetes/core/pom.xml | 2 +-
 resource-managers/mesos/pom.xml   | 2 +-
 resource-managers/yarn/pom.xml| 2 +-
 sql/catalyst/pom.xml  | 2 +-
 sql/core/pom.xml  | 2 +-
 sql/hive-thriftserver/pom.xml | 2 +-
 sql/hive/pom.xml  | 2 +-
 streaming/pom.xml | 2 +-
 tools/pom.xml | 2 +-
 41 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 9124a88..d14017e 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 2.3.4
+Version: 2.3.5
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' 
<https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 583b1bf..0c36ce2 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4
+2.3.5-SNAPSHOT
 ../pom.xml
   
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index 29c2c58..a9ab9d5 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4
+2.3.5-SNAPSHOT
 ../../pom.xml
   
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 224b229..f34618e 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4
+2.3.5-SNAPSHOT
 ../../pom.xml
   
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index c7f661e..62901b9 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4
+2.3.5-SNAPSHOT
 ../../pom.xml
   
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index f33fb99..8a64c64 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4
+2.3.5-SNAPSHOT
 ../../pom.xml
   
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index a642cb2..abb43d3 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4
+2.3.5-SNAPSHOT
 ../../pom.xml
   
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 29bd7ba..71e946a 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.spark
 spark-parent_2.11
-2.3.4
+2.3.5-SNAPSHOT
 ../../pom.xml
   
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 03f9b77..9fb92b7 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7

[spark] branch branch-2.3 updated (adb5255 -> 3fb9e84)

2019-08-25 Thread kiszk
This is an automated email from the ASF dual-hosted git repository.

kiszk pushed a change to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git.


from adb5255  [SPARK-26895][CORE][2.3] prepareSubmitEnvironment should be 
called within doAs for proxy users
 add 8c6f815  Preparing Spark release v2.3.4-rc1
 new 3fb9e84  Preparing development version 2.3.5-SNAPSHOT

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 R/pkg/DESCRIPTION | 2 +-
 assembly/pom.xml  | 2 +-
 common/kvstore/pom.xml| 2 +-
 common/network-common/pom.xml | 2 +-
 common/network-shuffle/pom.xml| 2 +-
 common/network-yarn/pom.xml   | 2 +-
 common/sketch/pom.xml | 2 +-
 common/tags/pom.xml   | 2 +-
 common/unsafe/pom.xml | 2 +-
 core/pom.xml  | 2 +-
 docs/_config.yml  | 4 ++--
 examples/pom.xml  | 2 +-
 external/docker-integration-tests/pom.xml | 2 +-
 external/flume-assembly/pom.xml   | 2 +-
 external/flume-sink/pom.xml   | 2 +-
 external/flume/pom.xml| 2 +-
 external/kafka-0-10-assembly/pom.xml  | 2 +-
 external/kafka-0-10-sql/pom.xml   | 2 +-
 external/kafka-0-10/pom.xml   | 2 +-
 external/kafka-0-8-assembly/pom.xml   | 2 +-
 external/kafka-0-8/pom.xml| 2 +-
 external/kinesis-asl-assembly/pom.xml | 2 +-
 external/kinesis-asl/pom.xml  | 2 +-
 external/spark-ganglia-lgpl/pom.xml   | 2 +-
 graphx/pom.xml| 2 +-
 hadoop-cloud/pom.xml  | 2 +-
 launcher/pom.xml  | 2 +-
 mllib-local/pom.xml   | 2 +-
 mllib/pom.xml | 2 +-
 pom.xml   | 2 +-
 python/pyspark/version.py | 2 +-
 repl/pom.xml  | 2 +-
 resource-managers/kubernetes/core/pom.xml | 2 +-
 resource-managers/mesos/pom.xml   | 2 +-
 resource-managers/yarn/pom.xml| 2 +-
 sql/catalyst/pom.xml  | 2 +-
 sql/core/pom.xml  | 2 +-
 sql/hive-thriftserver/pom.xml | 2 +-
 sql/hive/pom.xml  | 2 +-
 streaming/pom.xml | 2 +-
 tools/pom.xml | 2 +-
 41 files changed, 42 insertions(+), 42 deletions(-)


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org



[spark] tag v2.3.4-rc1 created (now 8c6f815)

2019-08-25 Thread kiszk
This is an automated email from the ASF dual-hosted git repository.

kiszk pushed a change to tag v2.3.4-rc1
in repository https://gitbox.apache.org/repos/asf/spark.git.


  at 8c6f815  (commit)
This tag includes the following new commits:

 new 8c6f815  Preparing Spark release v2.3.4-rc1

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org



svn commit: r35304 - /dev/spark/KEYS

2019-08-19 Thread kiszk
Author: kiszk
Date: Mon Aug 19 18:04:54 2019
New Revision: 35304

Log:
Update KEYS

Modified:
dev/spark/KEYS

Modified: dev/spark/KEYS
==
--- dev/spark/KEYS (original)
+++ dev/spark/KEYS Mon Aug 19 18:04:54 2019
@@ -991,4 +991,61 @@ QRMaCSG2MOvUAI8Zzk6i1Gi5InRlP5v8sQdrMYvS
 meyB5uExVklZg9yaoH2zAFXLkjG1pftpkCb57UIyC+Tk5KAMZXyS2vHNGxsnI3FG
 ZTFPNYvCMMHM8A==
 =PEdD
--END PGP PUBLIC KEY BLOCK-
\ No newline at end of file
+-END PGP PUBLIC KEY BLOCK-
+
+pub   rsa4096/7F0FEF75 2019-08-19 [SC]
+uid [ultimate] Kazuaki Ishizaki (CODE SIGNING KEY) 
+sub   rsa4096/7C3AEC68 2019-08-19 [E]
+
+-BEGIN PGP PUBLIC KEY BLOCK-
+Version: GnuPG v2
+
+mQINBF1a3YcBEAC7I6f1jWpY9WlJBkbwvLneYBjnD2BRwG1eKjkz49aUXVKkx4Du
+XB7b+agbhWL7EIPjQHVJf0RVGochOujKfcPxOz5bZwAV078EbsJpiAYIAeVEimQF
+Pv/uqaf9DbIjZAnJtZhKlyXJaXLpuZbqEwBimpfbgvF5ib4ii7a9kY7BO/YsSXXc
+ksLBIHKwNAeKSMIGmCQaxz/tNmRm1tAagFknCEoQ0CMsA8FesjXyS+U6nfJWdK3K
++678joAIhZvdn5k3f/bR94ifeDCh0QsY/zuG95er4Gp0rdr8EmRQbfJAUAwfkn8a
+viQD1FkTs+aJn4MSClb+FDXu7hNrPPdayA5CI6PSMdir//+Z7Haox92mvhQT5pBJ
+X21R4BDqF6bmL2d/RL3e2Zb1rmztDbTq43OL3Jm+x9R3OPg9UVwFJgHUy/xEirve
+Nah5Y6GzV3po/VSJbRIdM/p8OENv6YahFbLr5rT5O9iZns/PXHUpXYXLQDfdFJD2
+oCNFxlQmjfbxIL3PIcdS2gY2o1FmEbYuaLi6Bb9FDTm/J78vHYtR3wLvwufLh3PX
+5en9e6+g7o5w3jN/3J1skwXUUSOHK88mWBGt2B9ZwYS+7TQ0zWcgrXjwHQoi92nA
+JEADyvQSxTB/zd5usCVel8038FSKhawkhrmLBk2UoJR4prhnPC364MnjgQARAQAB
+tDZLYXp1YWtpIElzaGl6YWtpIChDT0RFIFNJR05JTkcgS0VZKSA8a2lzemtAYXBh
+Y2hlLm9yZz6JAjgEEwECACIFAl1a3YcCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4B
+AheAAAoJEOSaBGx/D+91w5AQALB6gff1BuaDyMSiSYaAGBGrBAxs1+ixQxlLX+ld
+KG9y/u41S3s8pBn0GXp1jthdURnPm+raLqJk1lVPUZ4JqNYot0FL/nGBIZjRRG6J
+TfmlWTza1AfgvzcROaO+7jVPMskBx/HZn8XxEOlMcnBv4P/v3m/QUW9/tH8j+6Bc
+JwfiqD3LIaWZTicAMxWE9r7MREDcgkrFROJDDJPMFxoVKomIcc3vzXJeI7BfVtkG
+5NHWYDVn4QTQygv+qes4ke9fcik7T5c9NcOjXgks6eF0z7Z/Rj6DUrIyVKleUwJZ
+AWpBJcbNc8crg623DRaXpGhXsGvnD5PxcPvVjJ9Jud7o884OhVr2abxQ++rIv/+m
+K5K99jbp2E/6Q6tR4ODEoPTGN6fSijziWfhuad26K/grN3878hayGmey57vPH3tx
+LsBkUfc9bz46HjcdhfaU1dS82YOMmrFLLmgBEL1PViK628gk0TR7C6N4kHKGWd1f
+tQz/bTFzoyXOTpS6bvceE88fZ2FSeepP0AgvZPZsUXxrHXo78oECZ9CAoO/q1P1J
+OrKr5oG5om9pB+4SI3FhD2PKxt/+ayMCyA6PVBlw8HDI2XLBmBi9YkiP2ws7gJcF
+A958J3CWc6Q7PstrU7LCmL0Apbl8T2Iqph7jB2Qiko2sOyxe5Vwkwh9vHYnhy1ox
+YZ2quQINBF1a3YcBEADfvUJtKQKQEHl6ug/c2PxDL5pfEhCXQfBIkfPScUgiQCO9
+aiSigMUReiYa/7cau2jmGUcBktjgLwlAGywX6YTGt/ZIWCkGRdK8K3mVRNssGwXs
++oWcNinRbzIV1cvZu9zndzM7lzIMFriIP/Shsi9QPg6SibK1XhgkYr2pTN8i1zmQ
+sd/FGnhEeGZxXDwW7wG6tPXvzQiAZgJEsUh90i9AbQzI/MWG2RqqjKGO423BcpQ8
+nHgUlj7JbgRI2knBjpnxAyKroDGw9dKXNBqYrGjQtbXcCkBTk6vDyOkXUWOz63Bc
+AtVfXwL5+RILvYjzn8bZne5jt8fkNK3z29XTv7N3Ee8HRwPnGp6Ny7jGR/f740gP
+3b8y4A6QI9YlyvOlp2SHIRPHEYKUQCLaTT1/b4DYN5SGtWwXA4GafCLBVBwD3fr+
+jIhCbInX0+MWOZwuTYuwpoE6nnsnWpsAd6ZOMJInULRyW1f7/zXoq2XvtFH8+IQN
+DYtF1lr2C8lm7WUKqSg2bmVy6+gV6KvYqj6oihLQBxlnmrKBQFhkBeOyNYxRW8rf
+c+nZZza/5QMZLD7mYL+BGmgHB2eycSuz7UkZ8H5DD0u7Wz74mmmHOg9EyJuJSa3z
+UXgg1VNtZCW/m7ha5jedQTiXSYX1R7HjjoX6vWm85mRLAFbyW7DaKnfbYlJvjwAR
+AQABiQIfBBgBAgAJBQJdWt2HAhsMAAoJEOSaBGx/D+91YNwQAIY41adyEUHRtwnP
+sT90VjheUdz9++rAet8jstwGK8M3wrnhDet18E7wTxt52Knkw7vMS2wqjm3jxeFs
+/pI/eA6Tq+AWLEySODegM9TGFxAtcP9TAR0bXGspw5LUWUKO+MJ17pyVs0M/0gb0
+GEjbVCjDn/h0Ozr3n81eokVDhvBZ8n2dUGoetmuZ77Wz1liPoV9G0paISKyLsj9d
+iQkE3ExZlGkvX6OiNbJMoo1pHMA4knAo9ch62THofPaoLX5mCKwhNgQDECYd4k89
+ww176ndkrllV8t1v/UDHXPwmDWGK+mLeLk4e+fDJ+bOQrZ543AYk6MB1gRyb94G7
+bQniuoc2YvB+Cn6qOB83ARhDz0zPUGVj/85P8xwmcsZJxlLGpiPAXEQJX2Zk6zFR
+1HLxy831IsHaEktglF9tBH+OxJqBg45fbRhuYclWfo724enVdm/rLtR1n93ybaJS
+eNmw1Lomks7IsX6qdBR36zVB2WgmIcsnxjtMee+YqfFiAbzbm27lV6A7aTDyIPzQ
+R2fSta747XADEy7rzYawV5zuCupmUHp/ZgfQK9xYDnZ+lJHHaipDgmIe4Mfe/3Je
+au2shXGZFmo4V56uCJ5HqZTJJZaMceQx7u8uqZbhtHG+lLhbvHXVylaxxEYpqf2O
+XJ5Dp1pqv9DC6cl9vLSHctRrM2kG
+=mQLW
+-END PGP PUBLIC KEY BLOCK-



-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org



spark git commit: [SPARK-25776][CORE]The disk write buffer size must be greater than 12

2018-11-04 Thread kiszk
Repository: spark
Updated Branches:
  refs/heads/master 463a67668 -> 6c9e5ac9d


[SPARK-25776][CORE]The disk write buffer size must be greater than 12

## What changes were proposed in this pull request?

 In `UnsafeSorterSpillWriter.java`, when we write a record to a spill file wtih 
` void write(Object baseObject, long baseOffset,  int recordLength, long 
keyPrefix)`, `recordLength` and `keyPrefix`  will be  written  the disk write 
buffer  first, and these will take 12 bytes, so the disk write buffer size must 
be greater than 12.

 If `diskWriteBufferSize` is  10, it will print this exception info:

_java.lang.ArrayIndexOutOfBoundsException: 10
   at 
org.apache.spark.util.collection.unsafe.sort.UnsafeSorterSpillWriter.writeLongToBuffer
 (UnsafeSorterSpillWriter.java:91)
at 
org.apache.spark.util.collection.unsafe.sort.UnsafeSorterSpillWriter.write(UnsafeSorterSpillWriter.java:123)
at 
org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter.spillIterator(UnsafeExternalSorter.java:498)
at 
org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter.spill(UnsafeExternalSorter.java:222)
at org.apache.spark.memory.MemoryConsumer.spill(MemoryConsumer.java:65)_

## How was this patch tested?
Existing UT in `UnsafeExternalSorterSuite`

Closes #22754 from 10110346/diskWriteBufferSize.

Authored-by: liuxian 
Signed-off-by: Kazuaki Ishizaki 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6c9e5ac9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6c9e5ac9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6c9e5ac9

Branch: refs/heads/master
Commit: 6c9e5ac9de3d0ae5ea86b768608b42b5feb46df4
Parents: 463a676
Author: liuxian 
Authored: Mon Nov 5 01:55:13 2018 +0900
Committer: Kazuaki Ishizaki 
Committed: Mon Nov 5 01:55:13 2018 +0900

--
 .../util/collection/unsafe/sort/UnsafeSorterSpillWriter.java   | 5 -
 .../main/scala/org/apache/spark/internal/config/package.scala  | 6 --
 2 files changed, 8 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/6c9e5ac9/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java
--
diff --git 
a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java
 
b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java
index 9399024..c1d71a2 100644
--- 
a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java
+++ 
b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java
@@ -42,7 +42,10 @@ public final class UnsafeSorterSpillWriter {
 
   private final SparkConf conf = new SparkConf();
 
-  /** The buffer size to use when writing the sorted records to an on-disk 
file */
+  /**
+   * The buffer size to use when writing the sorted records to an on-disk 
file, and
+   * this space used by prefix + len + recordLength must be greater than 4 + 8 
bytes.
+   */
   private final int diskWriteBufferSize =
 (int) (long) conf.get(package$.MODULE$.SHUFFLE_DISK_WRITE_BUFFER_SIZE());
 

http://git-wip-us.apache.org/repos/asf/spark/blob/6c9e5ac9/core/src/main/scala/org/apache/spark/internal/config/package.scala
--
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala 
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 034e5eb..c8993e1 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -21,6 +21,7 @@ import java.util.concurrent.TimeUnit
 
 import org.apache.spark.launcher.SparkLauncher
 import org.apache.spark.network.util.ByteUnit
+import org.apache.spark.unsafe.array.ByteArrayMethods
 import org.apache.spark.util.Utils
 
 package object config {
@@ -504,8 +505,9 @@ package object config {
 ConfigBuilder("spark.shuffle.spill.diskWriteBufferSize")
   .doc("The buffer size, in bytes, to use when writing the sorted records 
to an on-disk file.")
   .bytesConf(ByteUnit.BYTE)
-  .checkValue(v => v > 0 && v <= Int.MaxValue,
-s"The buffer size must be greater than 0 and less than 
${Int.MaxValue}.")
+  .checkValue(v => v > 12 && v <= 
ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
+s"The buffer size must be greater than 12 and less than or equal to " +
+  s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}.")
   .createWithDefault(1024 * 1024)
 
   private[spark] val UNROLL_MEMORY_CHECK_PERIOD =


-
To unsubscribe, e-mail: 

spark git commit: [SPARK-25497][SQL] Limit operation within whole stage codegen should not consume all the inputs

2018-10-09 Thread kiszk
Repository: spark
Updated Branches:
  refs/heads/master 46fe40838 -> e3133f4ab


[SPARK-25497][SQL] Limit operation within whole stage codegen should not 
consume all the inputs

## What changes were proposed in this pull request?

This PR is inspired by https://github.com/apache/spark/pull/22524, but proposes 
a safer fix.

The current limit whole stage codegen has 2 problems:
1. It's only applied to `InputAdapter`, many leaf nodes can't stop earlier 
w.r.t. limit.
2. It needs to override a method, which will break if we have more than one 
limit in the whole-stage.

The first problem is easy to fix, just figure out which nodes can stop earlier 
w.r.t. limit, and update them. This PR updates `RangeExec`, 
`ColumnarBatchScan`, `SortExec`, `HashAggregateExec`.

The second problem is hard to fix. This PR proposes to propagate the limit 
counter variable name upstream, so that the upstream leaf/blocking nodes can 
check the limit counter and quit the loop earlier.

For better performance, the implementation here follows 
`CodegenSupport.needStopCheck`, so that we only codegen the check only if there 
is limit in the query. For columnar node like range, we check the limit counter 
per-batch instead of per-row, to make the inner loop tight and fast.

Why this is safer?
1. the leaf/blocking nodes don't have to check the limit counter and stop 
earlier. It's only for performance. (this is same as before)
2. The blocking operators can stop propagating the limit counter name, because 
the counter of limit after blocking operators will never increase, before 
blocking operators consume all the data from upstream operators. So the 
upstream operators don't care about limit after blocking operators. This is 
also for performance only, it's OK if we forget to do it for some new blocking 
operators.

## How was this patch tested?

a new test

Closes #22630 from cloud-fan/limit.

Authored-by: Wenchen Fan 
Signed-off-by: Kazuaki Ishizaki 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e3133f4a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e3133f4a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e3133f4a

Branch: refs/heads/master
Commit: e3133f4abf1cd5667abe5f0d05fa0af0df3033ae
Parents: 46fe408
Author: Wenchen Fan 
Authored: Tue Oct 9 16:46:23 2018 +0900
Committer: Kazuaki Ishizaki 
Committed: Tue Oct 9 16:46:23 2018 +0900

--
 .../sql/execution/BufferedRowIterator.java  |  10 --
 .../spark/sql/execution/ColumnarBatchScan.scala |   4 +-
 .../apache/spark/sql/execution/SortExec.scala   |  12 +-
 .../sql/execution/WholeStageCodegenExec.scala   |  59 +-
 .../execution/aggregate/HashAggregateExec.scala |  22 +---
 .../sql/execution/basicPhysicalOperators.scala  |  91 +--
 .../org/apache/spark/sql/execution/limit.scala  |  31 --
 .../sql/execution/metric/SQLMetricsSuite.scala  | 111 ---
 8 files changed, 215 insertions(+), 125 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/e3133f4a/sql/core/src/main/java/org/apache/spark/sql/execution/BufferedRowIterator.java
--
diff --git 
a/sql/core/src/main/java/org/apache/spark/sql/execution/BufferedRowIterator.java
 
b/sql/core/src/main/java/org/apache/spark/sql/execution/BufferedRowIterator.java
index 74c9c05..3d0511b 100644
--- 
a/sql/core/src/main/java/org/apache/spark/sql/execution/BufferedRowIterator.java
+++ 
b/sql/core/src/main/java/org/apache/spark/sql/execution/BufferedRowIterator.java
@@ -74,16 +74,6 @@ public abstract class BufferedRowIterator {
   }
 
   /**
-   * Returns whether this iterator should stop fetching next row from 
[[CodegenSupport#inputRDDs]].
-   *
-   * If it returns true, the caller should exit the loop that [[InputAdapter]] 
generates.
-   * This interface is mainly used to limit the number of input rows.
-   */
-  public boolean stopEarly() {
-return false;
-  }
-
-  /**
* Returns whether `processNext()` should stop processing next row from 
`input` or not.
*
* If it returns true, the caller should exit the loop (return from 
processNext()).

http://git-wip-us.apache.org/repos/asf/spark/blob/e3133f4a/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
--
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
index 48abad9..9f6b593 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ColumnarBatchScan.scala
@@ -136,7 +136,7 @@ private[sql] trait ColumnarBatchScan extends 

spark git commit: [SPARK-25487][SQL][TEST] Refactor PrimitiveArrayBenchmark

2018-09-21 Thread kiszk
Repository: spark
Updated Branches:
  refs/heads/master 596af211a -> 1f4ca6f5c


[SPARK-25487][SQL][TEST] Refactor PrimitiveArrayBenchmark

## What changes were proposed in this pull request?
Refactor PrimitiveArrayBenchmark to use main method and print the output as a 
separate file.

Run blow command to generate benchmark results:

```
SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain 
org.apache.spark.sql.execution.benchmark.PrimitiveArrayBenchmark"
```

## How was this patch tested?
Manual tests.

Closes #22497 from seancxmao/SPARK-25487.

Authored-by: seancxmao 
Signed-off-by: Kazuaki Ishizaki 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1f4ca6f5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1f4ca6f5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1f4ca6f5

Branch: refs/heads/master
Commit: 1f4ca6f5c52560585ea977bddc69243a29bf67f2
Parents: 596af21
Author: seancxmao 
Authored: Fri Sep 21 15:04:47 2018 +0900
Committer: Kazuaki Ishizaki 
Committed: Fri Sep 21 15:04:47 2018 +0900

--
 .../PrimitiveArrayBenchmark-results.txt | 13 ++
 .../benchmark/PrimitiveArrayBenchmark.scala | 47 +---
 2 files changed, 35 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/1f4ca6f5/sql/core/benchmarks/PrimitiveArrayBenchmark-results.txt
--
diff --git a/sql/core/benchmarks/PrimitiveArrayBenchmark-results.txt 
b/sql/core/benchmarks/PrimitiveArrayBenchmark-results.txt
new file mode 100644
index 000..b06b5c0
--- /dev/null
+++ b/sql/core/benchmarks/PrimitiveArrayBenchmark-results.txt
@@ -0,0 +1,13 @@
+
+Write primitive arrays in dataset
+
+
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_131-b11 on Mac OS X 10.13.6
+Intel(R) Core(TM) i7-6820HQ CPU @ 2.70GHz
+
+Write an array in Dataset:   Best/Avg Time(ms)Rate(M/s)   Per 
Row(ns)   Relative
+
+Int437 /  529 19.2 
 52.1   1.0X
+Double 638 /  670 13.1 
 76.1   0.7X
+
+

http://git-wip-us.apache.org/repos/asf/spark/blob/1f4ca6f5/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
--
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
index e7c8f27..7f467d1 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala
@@ -17,21 +17,30 @@
 
 package org.apache.spark.sql.execution.benchmark
 
-import scala.concurrent.duration._
-
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.util.Benchmark
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.util.{Benchmark, BenchmarkBase => FileBenchmarkBase}
 
 /**
- * Benchmark [[PrimitiveArray]] for DataFrame and Dataset program using 
primitive array
- * To run this:
- *  1. replace ignore(...) with test(...)
- *  2. build/sbt "sql/test-only *benchmark.PrimitiveArrayBenchmark"
- *
- * Benchmarks in this file are skipped in normal builds.
+ * Benchmark primitive arrays via DataFrame and Dataset program using 
primitive arrays
+ * To run this benchmark:
+ * 1. without sbt: bin/spark-submit --class  
+ * 2. build/sbt "sql/test:runMain "
+ * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt 
"sql/test:runMain "
+ *Results will be written to 
"benchmarks/PrimitiveArrayBenchmark-results.txt".
  */
-class PrimitiveArrayBenchmark extends BenchmarkBase {
+object PrimitiveArrayBenchmark extends FileBenchmarkBase {
+  lazy val sparkSession = SparkSession.builder
+.master("local[1]")
+.appName("microbenchmark")
+.config("spark.sql.shuffle.partitions", 1)
+.config("spark.sql.autoBroadcastJoinThreshold", 1)
+.getOrCreate()
+
+  override def benchmark(): Unit = {
+runBenchmark("Write primitive arrays in dataset") {
+  writeDatasetArray(4)
+}
+  }
 
   def writeDatasetArray(iters: Int): Unit = {
 import sparkSession.implicits._
@@ -62,21 +71,9 @@ class PrimitiveArrayBenchmark extends BenchmarkBase {
   }
 }
 
-val benchmark = new 

spark-website git commit: Add Kazuaki Ishizaki to the list of committers

2018-09-18 Thread kiszk
Repository: spark-website
Updated Branches:
  refs/heads/asf-site 370d99580 -> 9b21d71d2


Add Kazuaki Ishizaki to the list of committers

Author: Kazuaki Ishizaki 

Closes #147 from kiszk/addCommitter.


Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/9b21d71d
Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/9b21d71d
Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/9b21d71d

Branch: refs/heads/asf-site
Commit: 9b21d71d226cc179779bbf0c9d072689fc69ea53
Parents: 370d995
Author: Kazuaki Ishizaki 
Authored: Wed Sep 19 14:13:42 2018 +0900
Committer: Kazuaki Ishizaki 
Committed: Wed Sep 19 14:13:42 2018 +0900

--
 committers.md| 1 +
 site/committers.html | 4 
 2 files changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/spark-website/blob/9b21d71d/committers.md
--
diff --git a/committers.md b/committers.md
index a25e10a..b64e278 100644
--- a/committers.md
+++ b/committers.md
@@ -34,6 +34,7 @@ navigation:
 |Yin Huai|Databricks|
 |Shane Huang|Intel|
 |Dongjoon Hyun|Hortonworks|
+|Kazuaki Ishizaki|IBM|
 |Holden Karau|Google|
 |Shane Knapp|UC Berkeley|
 |Cody Koeninger|Nexstar Digital|

http://git-wip-us.apache.org/repos/asf/spark-website/blob/9b21d71d/site/committers.html
--
diff --git a/site/committers.html b/site/committers.html
index 0884ad4..3ec90dd 100644
--- a/site/committers.html
+++ b/site/committers.html
@@ -307,6 +307,10 @@
   Hortonworks
 
 
+  Kazuaki Ishizaki
+  IBM
+
+
   Holden Karau
   Google
 


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org