[spark] branch master updated: [SPARK-28685][SQL][TEST] Test HMS 2.0.0+ in VersionsSuite/HiveClientSuites on JDK 11
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 58cc0df [SPARK-28685][SQL][TEST] Test HMS 2.0.0+ in
VersionsSuite/HiveClientSuites on JDK 11
58cc0df is described below
commit 58cc0df59e3dd8b8fc46706050ac4ca437a3f18f
Author: Yuming Wang
AuthorDate: Sat Aug 10 17:01:15 2019 -0700
[SPARK-28685][SQL][TEST] Test HMS 2.0.0+ in VersionsSuite/HiveClientSuites
on JDK 11
## What changes were proposed in this pull request?
It seems Datanucleus 3.x can not support JDK 11:
```java
[info] Cause: org.datanucleus.exceptions.NucleusException: The java type
java.lang.Long (jdbc-type="", sql-type="") cant be mapped for this datastore.
No mapping is available.
[info] at
org.datanucleus.store.rdbms.mapping.RDBMSMappingManager.getDatastoreMappingClass(RDBMSMappingManager.java:1215)
[info] at
org.datanucleus.store.rdbms.mapping.RDBMSMappingManager.createDatastoreMapping(RDBMSMappingManager.java:1378)
[info] at
org.datanucleus.store.rdbms.table.AbstractClassTable.addDatastoreId(AbstractClassTable.java:392)
[info] at
org.datanucleus.store.rdbms.table.ClassTable.initializePK(ClassTable.java:1087)
[info] at
org.datanucleus.store.rdbms.table.ClassTable.preInitialize(ClassTable.java:247)
```
Hive upgrade Datanucleus to 4.x from Hive
2.0([HIVE-6113](https://issues.apache.org/jira/browse/HIVE-6113)). This PR
makes it skip `0.12`, `0.13`, `0.14`, `1.0`, `1.1` and `1.2` when testing with
JDK 11.
Note that, this pr will not fix sql read hive materialized view. It's
another issue:
```
3.0: sql read hive materialized view *** FAILED *** (1 second, 521
milliseconds)
3.1: sql read hive materialized view *** FAILED *** (1 second, 536
milliseconds)
```
## How was this patch tested?
manual tests:
```shell
export JAVA_HOME="/usr/lib/jdk-11.0.3"
build/sbt "hive/test-only *.VersionsSuite *.HiveClientSuites" -Phive
-Phadoop-3.2
```
Closes #25405 from wangyum/SPARK-28685.
Authored-by: Yuming Wang
Signed-off-by: Dongjoon Hyun
---
.../apache/spark/sql/hive/client/HiveClientVersions.scala| 7 +--
.../org/apache/spark/sql/hive/client/VersionsSuite.scala | 12 +---
2 files changed, 14 insertions(+), 5 deletions(-)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientVersions.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientVersions.scala
index e9eebb4..d48d70f 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientVersions.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientVersions.scala
@@ -19,10 +19,13 @@ package org.apache.spark.sql.hive.client
import scala.collection.immutable.IndexedSeq
-import org.apache.spark.SparkFunSuite
+import org.apache.commons.lang3.{JavaVersion, SystemUtils}
private[client] trait HiveClientVersions {
- protected val versions =
+ protected val versions = if
(SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) {
+IndexedSeq("2.0", "2.1", "2.2", "2.3", "3.0", "3.1")
+ } else {
IndexedSeq("0.12", "0.13", "0.14", "1.0", "1.1", "1.2", "2.0", "2.1",
"2.2", "2.3", "3.0",
"3.1")
+ }
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index feb364e..34bdee4 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hive.client
import java.io.{ByteArrayOutputStream, File, PrintStream, PrintWriter}
import java.net.URI
+import org.apache.commons.lang3.{JavaVersion, SystemUtils}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hive.common.StatsSetupConst
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -102,8 +103,11 @@ class VersionsSuite extends SparkFunSuite with Logging {
assert(getNestedMessages(e) contains "Unknown column 'A0.OWNER_NAME' in
'field list'")
}
- private val versions =
+ private val versions = if
(SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) {
+Seq("2.0", "2.1", "2.2", "2.3", "3.0", "3.1")
+ } else {
Seq("0.12", "0.13", "0.14", "1.0", "1.1", "1.2", "2.0", "2.1", "2.2",
"2.3", "3.0", "3.1")
+ }
private var client: HiveClient = null
@@ -323,7 +327,8 @@ class VersionsSuite extends SparkFunSuite with Logging {
}
test(s"$version: dropTable") {
- val versionsWithoutPurge = versions.takeWhile(_ != "0.14")
+ val versionsWithoutPurge =
+if (versions.contains("0.14")) versions.takeWhile(_ !=
[spark] branch master updated: [SPARK-28675][SQL] Remove maskCredentials and use redactOptions
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 47af892 [SPARK-28675][SQL] Remove maskCredentials and use
redactOptions
47af892 is described below
commit 47af8925b60509d2a2c932e2bcf25394721c6f12
Author: Yuming Wang
AuthorDate: Sat Aug 10 16:45:59 2019 -0700
[SPARK-28675][SQL] Remove maskCredentials and use redactOptions
## What changes were proposed in this pull request?
This PR replaces `CatalogUtils.maskCredentials` with
`SQLConf.get.redactOptions` to match other redacts.
## How was this patch tested?
unit test and manual tests:
Before this PR:
```sql
spark-sql> DESC EXTENDED test_spark_28675;
id int NULL
# Detailed Table Information
Databasedefault
Table test_spark_28675
Owner root
Created TimeFri Aug 09 08:23:17 GMT-07:00 2019
Last Access Wed Dec 31 17:00:00 GMT-07:00 1969
Created By Spark 3.0.0-SNAPSHOT
TypeMANAGED
Providerorg.apache.spark.sql.jdbc
Locationfile:/user/hive/warehouse/test_spark_28675
Serde Library org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Storage Properties [url=###, driver=com.mysql.jdbc.Driver,
dbtable=test_spark_28675]
spark-sql> SHOW TABLE EXTENDED LIKE 'test_spark_28675';
default test_spark_28675false Database: default
Table: test_spark_28675
Owner: root
Created Time: Fri Aug 09 08:23:17 GMT-07:00 2019
Last Access: Wed Dec 31 17:00:00 GMT-07:00 1969
Created By: Spark 3.0.0-SNAPSHOT
Type: MANAGED
Provider: org.apache.spark.sql.jdbc
Location: file:/user/hive/warehouse/test_spark_28675
Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Storage Properties: [url=###, driver=com.mysql.jdbc.Driver,
dbtable=test_spark_28675]
Schema: root
|-- id: integer (nullable = true)
```
After this PR:
```sql
spark-sql> DESC EXTENDED test_spark_28675;
id int NULL
# Detailed Table Information
Databasedefault
Table test_spark_28675
Owner root
Created TimeFri Aug 09 08:19:49 GMT-07:00 2019
Last Access Wed Dec 31 17:00:00 GMT-07:00 1969
Created By Spark 3.0.0-SNAPSHOT
TypeMANAGED
Providerorg.apache.spark.sql.jdbc
Locationfile:/user/hive/warehouse/test_spark_28675
Serde Library org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Storage Properties [url=*(redacted), driver=com.mysql.jdbc.Driver,
dbtable=test_spark_28675]
spark-sql> SHOW TABLE EXTENDED LIKE 'test_spark_28675';
default test_spark_28675false Database: default
Table: test_spark_28675
Owner: root
Created Time: Fri Aug 09 08:19:49 GMT-07:00 2019
Last Access: Wed Dec 31 17:00:00 GMT-07:00 1969
Created By: Spark 3.0.0-SNAPSHOT
Type: MANAGED
Provider: org.apache.spark.sql.jdbc
Location: file:/user/hive/warehouse/test_spark_28675
Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Storage Properties: [url=*(redacted), driver=com.mysql.jdbc.Driver,
dbtable=test_spark_28675]
Schema: root
|-- id: integer (nullable = true)
```
Closes #25395 from wangyum/SPARK-28675.
Authored-by: Yuming Wang
Signed-off-by: Dongjoon Hyun
---
.../catalyst/catalog/ExternalCatalogUtils.scala| 15
.../spark/sql/catalyst/catalog/interface.scala | 2 +-
.../spark/sql/execution/datasources/ddl.scala | 5 +--
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 42 ++
4 files changed, 46 insertions(+), 18 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
index 50f32e8..4cff162 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
@@ -174,21 +174,6 @@ object ExternalCatalogUtils {
}
object CatalogUtils {
- /**
- *
[spark] branch master updated: [SPARK-28677][WEBUI] "Select All" checkbox in StagePage doesn't work properly
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new dd5599e [SPARK-28677][WEBUI] "Select All" checkbox in StagePage
doesn't work properly
dd5599e is described below
commit dd5599efaf42de8d5ce7db68e35debc3b40bfb20
Author: Kousuke Saruta
AuthorDate: Sat Aug 10 16:51:12 2019 -0500
[SPARK-28677][WEBUI] "Select All" checkbox in StagePage doesn't work
properly
## What changes were proposed in this pull request?
In StagePage, only the first optional column (Scheduler Delay, in this
case) appears even though "Select All" checkbox is checked.
The cause is that wrong method is used to manipulate multiple columns.
columns should have been used but column was used.
I've fixed this issue by replacing the `column` with `columns`.
## How was this patch tested?
Confirmed behavior of the check-box.
Closes #25397 from sarutak/fix-stagepage.js.
Authored-by: Kousuke Saruta
Signed-off-by: Sean Owen
---
.../main/resources/org/apache/spark/ui/static/stagepage.js | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index 4fe2cd9..3ef1a76 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -323,7 +323,7 @@ $(document).ready(function () {
"in this task. For SQL jobs, this only tracks all unsafe
operators, broadcast joins, and " +
"external sort.");
$('[data-toggle="tooltip"]').tooltip();
-tasksSummary = $("#parent-container");
+var tasksSummary = $("#parent-container");
getStandAloneAppId(function (appId) {
var endPoint = stageEndPoint(appId);
@@ -346,7 +346,7 @@ $(document).ready(function () {
}
// prepare data for executor summary table
-stageExecutorSummaryInfoKeys =
Object.keys(responseBody.executorSummary);
+var stageExecutorSummaryInfoKeys =
Object.keys(responseBody.executorSummary);
$.getJSON(createRESTEndPointForExecutorsPage(appId),
function(executorSummaryResponse, status, jqXHR) {
var executorDetailsMap = {};
@@ -877,7 +877,7 @@ $(document).ready(function () {
{ "visible": false, "targets": 16 },
{ "visible": false, "targets": 17 },
{ "visible": false, "targets": 18 }
-],
+]
};
taskTableSelector = $(taskTable).DataTable(taskConf);
$('#active-tasks-table_filter input').unbind();
@@ -897,14 +897,14 @@ $(document).ready(function () {
// Get the column
var para = $(this).attr('data-column');
if (para == "0") {
-var column = taskTableSelector.column(optionalColumns);
+var allColumns =
taskTableSelector.columns(optionalColumns);
if ($(this).is(":checked")) {
$(".toggle-vis").prop('checked', true);
-column.visible(true);
+allColumns.visible(true);
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray);
} else {
$(".toggle-vis").prop('checked', false);
-column.visible(false);
+allColumns.visible(false);
var taskSummaryMetricsTableFilteredArray =
taskSummaryMetricsTableArray.filter(row =>
row.checkboxId < 11);
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [MINOR] Fix typos in comments and replace an explicit type with <>
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8535df7 [MINOR] Fix typos in comments and replace an explicit type
with <>
8535df7 is described below
commit 8535df72614800ba789286e569a39ea6e84b3354
Author: younggyu chun
AuthorDate: Sat Aug 10 16:47:11 2019 -0500
[MINOR] Fix typos in comments and replace an explicit type with <>
## What changes were proposed in this pull request?
This PR fixed typos in comments and replace the explicit type with '<>' for
Java 8+.
## How was this patch tested?
Manually tested.
Closes #25338 from younggyuchun/younggyu.
Authored-by: younggyu chun
Signed-off-by: Sean Owen
---
appveyor.yml | 2 +-
.../java/org/apache/spark/network/ChunkFetchIntegrationSuite.java | 6 +++---
.../test/java/org/apache/spark/network/RpcIntegrationSuite.java | 8
.../org/apache/spark/network/TransportClientFactorySuite.java | 2 +-
.../spark/network/shuffle/ExternalShuffleIntegrationSuite.java| 6 +++---
.../test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java | 2 +-
.../org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java | 4 ++--
.../org/apache/spark/metrics/source/AccumulatorSourceSuite.scala | 2 +-
.../examples/sql/streaming/JavaStructuredSessionization.java | 2 +-
.../spark/streaming/kafka010/JavaConsumerStrategySuite.java | 2 +-
.../spark/streaming/kafka010/JavaDirectKafkaStreamSuite.java | 2 +-
.../java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java | 2 +-
.../test/java/org/apache/spark/ml/stat/JavaSummarizerSuite.java | 2 +-
.../apache/spark/mllib/regression/JavaRidgeRegressionSuite.java | 4 ++--
.../apache/spark/sql/execution/python/WindowInPandasExec.scala| 2 +-
.../test/org/apache/spark/sql/JavaDataFrameReaderWriterSuite.java | 2 +-
.../java/org/apache/spark/streaming/JavaMapWithStateSuite.java| 4 ++--
17 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/appveyor.yml b/appveyor.yml
index 8fb090c..b0e946c 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -55,7 +55,7 @@ build_script:
environment:
NOT_CRAN: true
- # See SPARK-27848. Currently installing some dependent packagess causes
+ # See SPARK-27848. Currently installing some dependent packages causes
# "(converted from warning) unable to identify current timezone 'C':" for an
unknown reason.
# This environment variable works around to test SparkR against a higher
version.
R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
diff --git
a/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java
b/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java
index 5999b62..a818fe4 100644
---
a/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java
+++
b/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java
@@ -151,9 +151,9 @@ public class ChunkFetchIntegrationSuite {
clientFactory.createClient(TestUtils.getLocalHost(), server.getPort())) {
final Semaphore sem = new Semaphore(0);
- res.successChunks = Collections.synchronizedSet(new HashSet());
- res.failedChunks = Collections.synchronizedSet(new HashSet());
- res.buffers = Collections.synchronizedList(new
LinkedList());
+ res.successChunks = Collections.synchronizedSet(new HashSet<>());
+ res.failedChunks = Collections.synchronizedSet(new HashSet<>());
+ res.buffers = Collections.synchronizedList(new LinkedList<>());
ChunkReceivedCallback callback = new ChunkReceivedCallback() {
@Override
diff --git
a/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
b/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
index 117f1e4..498dc51 100644
---
a/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
+++
b/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
@@ -175,8 +175,8 @@ public class RpcIntegrationSuite {
final Semaphore sem = new Semaphore(0);
final RpcResult res = new RpcResult();
-res.successMessages = Collections.synchronizedSet(new HashSet());
-res.errorMessages = Collections.synchronizedSet(new HashSet());
+res.successMessages = Collections.synchronizedSet(new HashSet<>());
+res.errorMessages = Collections.synchronizedSet(new HashSet<>());
RpcResponseCallback callback = new RpcResponseCallback() {
@Override
@@ -208,8 +208,8 @@ public class RpcIntegrationSuite {
TransportClient client =
clientFactory.createClient(TestUtils.getLocalHost(), server.getPort());
final Semaphore
