[
https://issues.apache.org/jira/browse/BEAM-6242?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16723099#comment-16723099
]
Maximilian Michels commented on BEAM-6242:
------------------------------------------
Ah, because parallelism is only set to 1 for batch. We might have to decrease
the parallelism also ins streaming or add the
{{--environment_cache_millis=10000}} fix.
> PreCommit Portable WordCount is flaky
> -------------------------------------
>
> Key: BEAM-6242
> URL: https://issues.apache.org/jira/browse/BEAM-6242
> Project: Beam
> Issue Type: Test
> Components: runner-flink
> Reporter: Maximilian Michels
> Priority: Major
>
> [https://builds.apache.org/job/beam_PreCommit_Portable_Python_Phrase/42/consoleText]
>
> {noformat}
> Caused by: java.util.concurrent.ExecutionException:
> java.lang.RuntimeException: Error received from SDK harness for instruction
> 533: Traceback (most recent call last):
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 145, in _execute
> response = task()
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 253, in do_instruction
> request.instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
> line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
> line 489, in process_bundle
> ].process_encoded(data.data)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
> line 126, in process_encoded
> self.output(decoded_value)
> File "apache_beam/runners/worker/operations.py", line 182, in
> apache_beam.runners.worker.operations.Operation.output
> def output(self, windowed_value, output_index=0):
> File "apache_beam/runners/worker/operations.py", line 183, in
> apache_beam.runners.worker.operations.Operation.output
> cython.cast(Receiver,
> self.receivers[output_index]).receive(windowed_value)
> File "apache_beam/runners/worker/operations.py", line 89, in
> apache_beam.runners.worker.operations.ConsumerSet.receive
> cython.cast(Operation, consumer).process(windowed_value)
> File "apache_beam/runners/worker/operations.py", line 497, in
> apache_beam.runners.worker.operations.DoOperation.process
> with self.scoped_process_state:
> File "apache_beam/runners/worker/operations.py", line 498, in
> apache_beam.runners.worker.operations.DoOperation.process
> self.dofn_receiver.receive(o)
> File "apache_beam/runners/common.py", line 680, in
> apache_beam.runners.common.DoFnRunner.receive
> self.process(windowed_value)
> File "apache_beam/runners/common.py", line 686, in
> apache_beam.runners.common.DoFnRunner.process
> self._reraise_augmented(exn)
> File "apache_beam/runners/common.py", line 724, in
> apache_beam.runners.common.DoFnRunner._reraise_augmented
> raise_with_traceback(new_exn)
> File "apache_beam/runners/common.py", line 684, in
> apache_beam.runners.common.DoFnRunner.process
> self.do_fn_invoker.invoke_process(windowed_value)
> File "apache_beam/runners/common.py", line 535, in
> apache_beam.runners.common.PerWindowInvoker.invoke_process
> self._invoke_per_window(
> File "apache_beam/runners/common.py", line 604, in
> apache_beam.runners.common.PerWindowInvoker._invoke_per_window
> output_processor.process_outputs(
> File "apache_beam/runners/common.py", line 755, in
> apache_beam.runners.common._OutputProcessor.process_outputs
> def process_outputs(self, windowed_input_element, results):
> File "apache_beam/runners/common.py", line 770, in
> apache_beam.runners.common._OutputProcessor.process_outputs
> for result in results:
> File "/usr/local/lib/python2.7/site-packages/apache_beam/io/iobase.py",
> line 1077, in <genexpr>
> window.TimestampedValue(v, timestamp.MAX_TIMESTAMP) for v in outputs)
> File
> "/usr/local/lib/python2.7/site-packages/apache_beam/io/filebasedsink.py",
> line 321, in finalize_write
> 'Encountered exceptions in finalize_write: %s' % all_exceptions)
> Exception: Encountered exceptions in finalize_write: [IOError(OSError(2, 'No
> such file or directory'),), IOError(OSError(2, 'No such file or
> directory'),)] [while running 'write/Write/WriteImpl/FinalizeWrite']
> at
> java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)
> at
> java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895)
> at org.apache.beam.sdk.util.MoreFutures.get(MoreFutures.java:57)
> at
> org.apache.beam.runners.fnexecution.control.SdkHarnessClient$ActiveBundle.close(SdkHarnessClient.java:263)
> at
> org.apache.beam.runners.flink.translation.wrappers.streaming.ExecutableStageDoFnOperator$SdkHarnessDoFnRunner.finishBundle(ExecutableStageDoFnOperator.java:540)
> ... 14 more{noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
