Repository: spark Updated Branches: refs/heads/branch-2.0 ac2349199 -> a96fbd8b8
[SPARK-21551][PYTHON] Increase timeout for PythonRDD.serveIterator Backport of https://github.com/apache/spark/pull/18752 (https://issues.apache.org/jira/browse/SPARK-21551) (cherry picked from commit 9d3c6640f56e3e4fd195d3ad8cead09df67a72c7) Author: peay <p...@protonmail.com> Closes #19514 from FRosner/branch-2.0. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a96fbd8b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a96fbd8b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a96fbd8b Branch: refs/heads/branch-2.0 Commit: a96fbd8b868ef036f420b90964fb48a39c6b7108 Parents: ac23491 Author: peay <p...@protonmail.com> Authored: Sat Oct 21 09:53:09 2017 +0100 Committer: Sean Owen <so...@cloudera.com> Committed: Sat Oct 21 09:53:09 2017 +0100 ---------------------------------------------------------------------- .../src/main/scala/org/apache/spark/api/python/PythonRDD.scala | 6 +++--- python/pyspark/rdd.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/a96fbd8b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala index 2822eb5..2a9cd57 100644 --- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala @@ -678,7 +678,7 @@ private[spark] object PythonRDD extends Logging { * Create a socket server and a background thread to serve the data in `items`, * * The socket server can only accept one connection, or close if no connection - * in 3 seconds. + * in 15 seconds. * * Once a connection comes in, it tries to serialize all the data in `items` * and send them into this connection. @@ -687,8 +687,8 @@ private[spark] object PythonRDD extends Logging { */ def serveIterator[T](items: Iterator[T], threadName: String): Int = { val serverSocket = new ServerSocket(0, 1, InetAddress.getByName("localhost")) - // Close the socket if no connection in 3 seconds - serverSocket.setSoTimeout(3000) + // Close the socket if no connection in 15 seconds + serverSocket.setSoTimeout(15000) new Thread(threadName) { setDaemon(true) http://git-wip-us.apache.org/repos/asf/spark/blob/a96fbd8b/python/pyspark/rdd.py ---------------------------------------------------------------------- diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py index 751f0bf..8ab2965 100644 --- a/python/pyspark/rdd.py +++ b/python/pyspark/rdd.py @@ -128,7 +128,7 @@ def _load_from_socket(port, serializer): af, socktype, proto, canonname, sa = res sock = socket.socket(af, socktype, proto) try: - sock.settimeout(3) + sock.settimeout(15) sock.connect(sa) except socket.error: sock.close() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
