spark git commit: [SPARK-26069][TESTS] Fix flaky test: RpcIntegrationSuite.sendRpcWithStreamFailures
Repository: spark
Updated Branches:
refs/heads/branch-2.4 be42bfe5c -> 77c0629cb
[SPARK-26069][TESTS] Fix flaky test:
RpcIntegrationSuite.sendRpcWithStreamFailures
## What changes were proposed in this pull request?
The test failure is because `assertErrorAndClosed` misses one possible error
message: `java.nio.channels.ClosedChannelException`. This happens when the
second `uploadStream` is called after the channel has been closed. This can be
reproduced by adding `Thread.sleep(1000)` below this line:
https://github.com/apache/spark/blob/03306a6df39c9fd6cb581401c13c4dfc6bbd632e/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java#L217
This PR fixes the above issue and also improves the test failure messages of
`assertErrorAndClosed`.
## How was this patch tested?
Jenkins
Closes #23041 from zsxwing/SPARK-26069.
Authored-by: Shixiong Zhu
Signed-off-by: Shixiong Zhu
(cherry picked from commit 99cbc51b3250c07a3e8cc95c9b74e9d1725bac77)
Signed-off-by: Shixiong Zhu
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/77c0629c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/77c0629c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/77c0629c
Branch: refs/heads/branch-2.4
Commit: 77c0629cbac6046cc3f2ea1025c43d86af344d62
Parents: be42bfe
Author: Shixiong Zhu
Authored: Fri Nov 16 09:51:41 2018 -0800
Committer: Shixiong Zhu
Committed: Fri Nov 16 09:52:03 2018 -0800
--
.../apache/spark/network/RpcIntegrationSuite.java | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/77c0629c/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
--
diff --git
a/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
b/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
index 1f4d75c..45f4a18 100644
---
a/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
+++
b/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
@@ -371,7 +371,10 @@ public class RpcIntegrationSuite {
private void assertErrorAndClosed(RpcResult result, String expectedError) {
assertTrue("unexpected success: " + result.successMessages,
result.successMessages.isEmpty());
-// we expect 1 additional error, which contains *either* "closed" or
"Connection reset"
+// we expect 1 additional error, which should contain one of the follow
messages:
+// - "closed"
+// - "Connection reset"
+// - "java.nio.channels.ClosedChannelException"
Set errors = result.errorMessages;
assertEquals("Expected 2 errors, got " + errors.size() + "errors: " +
errors, 2, errors.size());
@@ -379,15 +382,18 @@ public class RpcIntegrationSuite {
Set containsAndClosed = Sets.newHashSet(expectedError);
containsAndClosed.add("closed");
containsAndClosed.add("Connection reset");
+containsAndClosed.add("java.nio.channels.ClosedChannelException");
Pair, Set> r = checkErrorsContain(errors,
containsAndClosed);
-Set errorsNotFound = r.getRight();
-assertEquals(1, errorsNotFound.size());
-String err = errorsNotFound.iterator().next();
-assertTrue(err.equals("closed") || err.equals("Connection reset"));
+assertTrue("Got a non-empty set " + r.getLeft(), r.getLeft().isEmpty());
-assertTrue(r.getLeft().isEmpty());
+Set errorsNotFound = r.getRight();
+assertEquals(
+"The size of " + errorsNotFound.toString() + " was not 2", 2,
errorsNotFound.size());
+for (String err: errorsNotFound) {
+ assertTrue("Found a wrong error " + err,
containsAndClosed.contains(err));
+}
}
private Pair, Set> checkErrorsContain(
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-26069][TESTS] Fix flaky test: RpcIntegrationSuite.sendRpcWithStreamFailures
Repository: spark
Updated Branches:
refs/heads/master a2fc48c28 -> 99cbc51b3
[SPARK-26069][TESTS] Fix flaky test:
RpcIntegrationSuite.sendRpcWithStreamFailures
## What changes were proposed in this pull request?
The test failure is because `assertErrorAndClosed` misses one possible error
message: `java.nio.channels.ClosedChannelException`. This happens when the
second `uploadStream` is called after the channel has been closed. This can be
reproduced by adding `Thread.sleep(1000)` below this line:
https://github.com/apache/spark/blob/03306a6df39c9fd6cb581401c13c4dfc6bbd632e/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java#L217
This PR fixes the above issue and also improves the test failure messages of
`assertErrorAndClosed`.
## How was this patch tested?
Jenkins
Closes #23041 from zsxwing/SPARK-26069.
Authored-by: Shixiong Zhu
Signed-off-by: Shixiong Zhu
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/99cbc51b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/99cbc51b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/99cbc51b
Branch: refs/heads/master
Commit: 99cbc51b3250c07a3e8cc95c9b74e9d1725bac77
Parents: a2fc48c
Author: Shixiong Zhu
Authored: Fri Nov 16 09:51:41 2018 -0800
Committer: Shixiong Zhu
Committed: Fri Nov 16 09:51:41 2018 -0800
--
.../apache/spark/network/RpcIntegrationSuite.java | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/99cbc51b/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
--
diff --git
a/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
b/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
index 1f4d75c..45f4a18 100644
---
a/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
+++
b/common/network-common/src/test/java/org/apache/spark/network/RpcIntegrationSuite.java
@@ -371,7 +371,10 @@ public class RpcIntegrationSuite {
private void assertErrorAndClosed(RpcResult result, String expectedError) {
assertTrue("unexpected success: " + result.successMessages,
result.successMessages.isEmpty());
-// we expect 1 additional error, which contains *either* "closed" or
"Connection reset"
+// we expect 1 additional error, which should contain one of the follow
messages:
+// - "closed"
+// - "Connection reset"
+// - "java.nio.channels.ClosedChannelException"
Set errors = result.errorMessages;
assertEquals("Expected 2 errors, got " + errors.size() + "errors: " +
errors, 2, errors.size());
@@ -379,15 +382,18 @@ public class RpcIntegrationSuite {
Set containsAndClosed = Sets.newHashSet(expectedError);
containsAndClosed.add("closed");
containsAndClosed.add("Connection reset");
+containsAndClosed.add("java.nio.channels.ClosedChannelException");
Pair, Set> r = checkErrorsContain(errors,
containsAndClosed);
-Set errorsNotFound = r.getRight();
-assertEquals(1, errorsNotFound.size());
-String err = errorsNotFound.iterator().next();
-assertTrue(err.equals("closed") || err.equals("Connection reset"));
+assertTrue("Got a non-empty set " + r.getLeft(), r.getLeft().isEmpty());
-assertTrue(r.getLeft().isEmpty());
+Set errorsNotFound = r.getRight();
+assertEquals(
+"The size of " + errorsNotFound.toString() + " was not 2", 2,
errorsNotFound.size());
+for (String err: errorsNotFound) {
+ assertTrue("Found a wrong error " + err,
containsAndClosed.contains(err));
+}
}
private Pair, Set> checkErrorsContain(
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
