[
https://issues.apache.org/jira/browse/BEAM-5915?focusedWorklogId=163276&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-163276
]
ASF GitHub Bot logged work on BEAM-5915:
----------------------------------------
Author: ASF GitHub Bot
Created on: 06/Nov/18 23:53
Start Date: 06/Nov/18 23:53
Worklog Time Spent: 10m
Work Description: aaltay closed pull request #6922: [BEAM-5915] Timeout
http requests, if they got stuck, at dataflow job creation time
URL: https://github.com/apache/beam/pull/6922
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index a6c89900ebe..e561fac3afd 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -24,6 +24,7 @@
from builtins import object
import codecs
import getpass
+import httplib2
import json
import logging
import os
@@ -428,14 +429,19 @@ def __init__(self, options):
credentials = None
else:
credentials = get_service_credentials()
+
+ # Use 60 second socket timeout avoid hangs during network flakiness.
+ http_client = httplib2.Http(timeout=60)
self._client = dataflow.DataflowV1b3(
url=self.google_cloud_options.dataflow_endpoint,
credentials=credentials,
- get_credentials=(not self.google_cloud_options.no_auth))
+ get_credentials=(not self.google_cloud_options.no_auth),
+ http=http_client)
self._storage_client = storage.StorageV1(
url='https://www.googleapis.com/storage/v1',
credentials=credentials,
- get_credentials=(not self.google_cloud_options.no_auth))
+ get_credentials=(not self.google_cloud_options.no_auth),
+ http=http_client)
# TODO(silviuc): Refactor so that retry logic can be applied.
@retry.no_retries # Using no_retries marks this as an integration point.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 163276)
Time Spent: 1h (was: 50m)
> Python postcommit test_read_metrics times out
> ---------------------------------------------
>
> Key: BEAM-5915
> URL: https://issues.apache.org/jira/browse/BEAM-5915
> Project: Beam
> Issue Type: Bug
> Components: test-failures
> Reporter: Henning Rohde
> Assignee: Ahmet Altay
> Priority: Major
> Labels: currently-failing
> Time Spent: 1h
> Remaining Estimate: 0h
>
> Happened a couple of times:
> https://scans.gradle.com/s/u7z2b7mwyghwi/console-log?task=:beam-sdks-python:validatesRunnerBatchTests#L142
> https://scans.gradle.com/s/b3zz6oyq3ueuw/console-log?task=:beam-sdks-python:validatesRunnerBatchTests
> ======================================================================
> ERROR: test_read_metrics
> (apache_beam.transforms.ptransform_test.PTransformTest)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/nose/plugins/multiprocess.py",
> line 812, in run
> test(orig)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/nose/case.py",
> line 45, in __call__
> return self.run(*arg, **kwarg)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/nose/case.py",
> line 133, in run
> self.runTest(result)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/nose/case.py",
> line 151, in runTest
> test(result)
> File "/usr/lib/python2.7/unittest/case.py", line 393, in __call__
> return self.run(*args, **kwds)
> File "/usr/lib/python2.7/unittest/case.py", line 329, in run
> testMethod()
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/transforms/ptransform_test.py",
> line 211, in test_read_metrics
> res = pipeline.run()
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/testing/test_pipeline.py",
> line 107, in run
> else test_runner_api))
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/pipeline.py",
> line 405, in run
> self._options).run(False)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/pipeline.py",
> line 418, in run
> return self.runner.run_pipeline(self)
> File "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/
> src/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py", line
> 70, in run_pipeline
> self.result.cancel()
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",
> line 1208, in cancel
> return self.state
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",
> line 1148, in state
> self._update_job()
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",
> line 1104, in _update_job
> self._job = self._runner.dataflow_client.get_job(self.job_id())
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/utils/retry.py",
> line 184, in wrapper
> return fun(*args, **kwargs)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",
> line 635, in get_job
> response = self._client.projects_locations_jobs.Get(request)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py",
> line 604, in Get
> config, request, global_params=global_params)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/apitools/base/py/base_api.py",
> line 720, in _RunMethod
> http, http_request, **opts)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/apitools/base/py/http_wrapper.py",
> line 346, in MakeRequest
> check_response_func=check_response_func)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/apitools/base/py/http_wrapper.py",
> line 396, in _
> MakeRequestNoRetry
> redirections=redirections, connection_type=connection_type)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/oauth2client/transport.py",
> line 175, in new_request
> redirections, connection_type)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/oauth2client/transport.py",
> line 282, in request
> connection_type=connection_type)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/httplib2/__init__.py",
> line 1694, in request
> (response, content) = self._request(conn, authority, uri, request_uri,
> method, body, headers, redirections, cachekey)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/httplib2/__init__.py",
> line 1434, in _request
> (response, content) = self._conn_request(conn, request_uri, method, body,
> headers)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/lib/python2.7/site-packages/httplib2/__init__.py",
> line 1390, in _conn_request
> response = conn.getresponse()
> File "/usr/lib/python2.7/httplib.py", line 1136, in getresponse
> response.begin()
> File "/usr/lib/python2.7/httplib.py", line 453, in begin
> version, status, reason = self._read_status()
> File "/usr/lib/python2.7/httplib.py", line 409, in _read_status
> line = self.fp.readline(_MAXLINE + 1)
> File "/usr/lib/python2.7/socket.py", line 480, in readline
> data = self._sock.recv(self._rbufsize)
> File "/usr/lib/python2.7/ssl.py", line 756, in recv
> return self.read(buflen)
> File "/usr/lib/python2.7/ssl.py", line 643, in read
> v = self._sslobj.read(len)
> File
> "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/build/gradleenv/local/
> lib/python2.7/site-packages/nose/plugins/multiprocess.py", line 276, in
> signalhandler
> raise TimedOutException()
> TimedOutException: 'test_read_metrics
> (apache_beam.transforms.ptransform_test.PTransformTest)'
> ----------------------------------------------------------------------
> XML:
> /home/jenkins/jenkins-slave/workspace/beam_PostCommit_Py_VR_Dataflow/src/sdks/python/nosetests.xml
> ----------------------------------------------------------------------
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
