[ 
https://issues.apache.org/jira/browse/BEAM-6258?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16727103#comment-16727103
 ] 

Ankur Goenka commented on BEAM-6258:
------------------------------------

Upstream GRPC issue https://github.com/grpc/grpc-java/issues/5188

> Data channel failing after some time for 1G data input
> ------------------------------------------------------
>
>                 Key: BEAM-6258
>                 URL: https://issues.apache.org/jira/browse/BEAM-6258
>             Project: Beam
>          Issue Type: Bug
>          Components: sdk-py-harness
>            Reporter: Ankur Goenka
>            Assignee: Ankur Goenka
>            Priority: Major
>
> Data channel and logging channel are failing after some time with 1GB input 
> data for chicago taxi.
>  
> E1218 02:44:02.837680206 72 chttp2_transport.cc:1148] Received a GOAWAY with 
> error code ENHANCE_YOUR_CALM and debug data equal to "too_many_pings"
> Exception in thread read_grpc_client_inputs:
> Traceback (most recent call last):
>  File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
>  self.run()
>  File "/usr/local/lib/python2.7/threading.py", line 754, in run
>  self.__target(*self.__args, **self.__kwargs)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 273, in <lambda>
>  target=lambda: self._read_inputs(elements_iterator),
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 260, in _read_inputs
>  for elements in elements_iterator:
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in 
> next
>  return self._next()
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in 
> _next
>  raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with 
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 145, in _execute
>  response = task()
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 180, in <lambda>
>  self._execute(lambda: worker.do_instruction(work), work)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 253, in do_instruction
>  request.instruction_id)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 269, in process_bundle
>  bundle_processor.process_bundle(instruction_id)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
>  line 481, in process_bundle
>  instruction_id, expected_targets):
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 209, in input_elements
>  raise_(t, v, tb)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 260, in _read_inputs
>  for elements in elements_iterator:
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in 
> next
>  return self._next()
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in 
> _next
>  raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with 
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 145, in _execute
>  response = task()
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 180, in <lambda>
>  self._execute(lambda: worker.do_instruction(work), work)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 253, in do_instruction
>  request.instruction_id)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 269, in process_bundle
>  bundle_processor.process_bundle(instruction_id)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
>  line 481, in process_bundle
>  instruction_id, expected_targets):
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 209, in input_elements
>  raise_(t, v, tb)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 260, in _read_inputs
>  for elements in elements_iterator:
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in 
> next
>  return self._next()
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in 
> _next
>  raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with 
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 145, in _execute
>  response = task()
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 180, in <lambda>
>  self._execute(lambda: worker.do_instruction(work), work)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 253, in do_instruction
>  request.instruction_id)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 269, in process_bundle
>  bundle_processor.process_bundle(instruction_id)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
>  line 481, in process_bundle
>  instruction_id, expected_targets):
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 209, in input_elements
>  raise_(t, v, tb)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 260, in _read_inputs
>  for elements in elements_iterator:
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in 
> next
>  return self._next()
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in 
> _next
>  raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with 
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 145, in _execute
>  response = task()
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 180, in <lambda>
>  self._execute(lambda: worker.do_instruction(work), work)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 253, in do_instruction
>  request.instruction_id)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 269, in process_bundle
>  bundle_processor.process_bundle(instruction_id)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
>  line 481, in process_bundle
>  instruction_id, expected_targets):
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 209, in input_elements
>  raise_(t, v, tb)
>  File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
>  line 260, in _read_inputs
>  for elements in elements_iterator:
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in 
> next
>  return self._next()
>  File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in 
> _next
>  raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with 
> (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to