[ https://issues.apache.org/jira/browse/BEAM-6258?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16727103#comment-16727103 ]
Ankur Goenka commented on BEAM-6258: ------------------------------------ Upstream GRPC issue https://github.com/grpc/grpc-java/issues/5188 > Data channel failing after some time for 1G data input > ------------------------------------------------------ > > Key: BEAM-6258 > URL: https://issues.apache.org/jira/browse/BEAM-6258 > Project: Beam > Issue Type: Bug > Components: sdk-py-harness > Reporter: Ankur Goenka > Assignee: Ankur Goenka > Priority: Major > > Data channel and logging channel are failing after some time with 1GB input > data for chicago taxi. > > E1218 02:44:02.837680206 72 chttp2_transport.cc:1148] Received a GOAWAY with > error code ENHANCE_YOUR_CALM and debug data equal to "too_many_pings" > Exception in thread read_grpc_client_inputs: > Traceback (most recent call last): > File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner > self.run() > File "/usr/local/lib/python2.7/threading.py", line 754, in run > self.__target(*self.__args, **self.__kwargs) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 273, in <lambda> > target=lambda: self._read_inputs(elements_iterator), > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 260, in _read_inputs > for elements in elements_iterator: > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in > next > return self._next() > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in > _next > raise self > _Rendezvous: <_Rendezvous of RPC that terminated with > (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)> > Traceback (most recent call last): > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 145, in _execute > response = task() > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 180, in <lambda> > self._execute(lambda: worker.do_instruction(work), work) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 253, in do_instruction > request.instruction_id) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 269, in process_bundle > bundle_processor.process_bundle(instruction_id) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", > line 481, in process_bundle > instruction_id, expected_targets): > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 209, in input_elements > raise_(t, v, tb) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 260, in _read_inputs > for elements in elements_iterator: > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in > next > return self._next() > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in > _next > raise self > _Rendezvous: <_Rendezvous of RPC that terminated with > (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)> > Traceback (most recent call last): > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 145, in _execute > response = task() > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 180, in <lambda> > self._execute(lambda: worker.do_instruction(work), work) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 253, in do_instruction > request.instruction_id) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 269, in process_bundle > bundle_processor.process_bundle(instruction_id) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", > line 481, in process_bundle > instruction_id, expected_targets): > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 209, in input_elements > raise_(t, v, tb) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 260, in _read_inputs > for elements in elements_iterator: > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in > next > return self._next() > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in > _next > raise self > _Rendezvous: <_Rendezvous of RPC that terminated with > (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)> > Traceback (most recent call last): > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 145, in _execute > response = task() > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 180, in <lambda> > self._execute(lambda: worker.do_instruction(work), work) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 253, in do_instruction > request.instruction_id) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 269, in process_bundle > bundle_processor.process_bundle(instruction_id) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", > line 481, in process_bundle > instruction_id, expected_targets): > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 209, in input_elements > raise_(t, v, tb) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 260, in _read_inputs > for elements in elements_iterator: > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in > next > return self._next() > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in > _next > raise self > _Rendezvous: <_Rendezvous of RPC that terminated with > (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)> > Traceback (most recent call last): > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 145, in _execute > response = task() > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 180, in <lambda> > self._execute(lambda: worker.do_instruction(work), work) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 253, in do_instruction > request.instruction_id) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", > line 269, in process_bundle > bundle_processor.process_bundle(instruction_id) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", > line 481, in process_bundle > instruction_id, expected_targets): > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 209, in input_elements > raise_(t, v, tb) > File > "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", > line 260, in _read_inputs > for elements in elements_iterator: > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in > next > return self._next() > File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in > _next > raise self > _Rendezvous: <_Rendezvous of RPC that terminated with > (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)> -- This message was sent by Atlassian JIRA (v7.6.3#76005)